MDL-69078 questions: check for UTF-8 encoding of the import file

Signed-off-by: Daniel Ziegenberg <[email protected]>

lang/en/question.php

@@ -202,6 +202,7 @@ $string['importparseerror'] = 'Error(s) found parsing the import file. No questi
 $string['importquestions'] = 'Import questions from file';
 $string['importquestions_help'] = 'This function enables questions in a variety of formats to be imported via text file. Note that the file must use UTF-8 encoding.';
 $string['importquestions_link'] = 'question/import';
+$string['importwrongfileencoding'] = 'The file you selected is not in UFT-8 character encoding. {$a} files must use UTF-8.';
 $string['importwrongfiletype'] = 'The type of the file you selected ({$a->actualtype}) does not match the type expected by this import format ({$a->expectedtype}).';
 $string['invalidarg'] = 'No valid arguments supplied or incorrect server configuration';
 $string['invalidcategoryidforparent'] = 'Invalid category id for parent!';

question/bank/importquestions/classes/form/import_form.php

@@ -158,6 +158,12 @@ class question_import_form extends moodleform {
             $a->actualtype = $file->get_mimetype();
             $a->expectedtype = $qformat->mime_type();
             $errors['newfile'] = get_string('importwrongfiletype', 'question', $a);
+            return $errors;
+        }
+
+        $fileerrors = $qformat->validate_file($file);
+        if ($fileerrors) {
+            $errors['newfile'] = $fileerrors;
         }
 
         return $errors;

question/format.php

@@ -95,6 +95,40 @@ class qformat_default {
         return ($file->get_mimetype() == $this->mime_type());
     }
 
+    /**
+     * Validate the given file.
+     *
+     * For more expensive or detailed integrity checks.
+     *
+     * @param stored_file $file the file to check
+     * @return string the error message that occurred while validating the given file
+     */
+    public function validate_file(stored_file $file): string {
+        return '';
+    }
+
+    /**
+     * Check if the given file has the required utf8 encoding.
+     *
+     * @param stored_file $file the file to check
+     * @return string the error message if the file encoding is not UTF-8
+     */
+    protected function validate_is_utf8_file(stored_file $file): string {
+        if (!mb_check_encoding($file->get_content(), "UTF-8")) {
+            return get_string('importwrongfileencoding', 'question',  $this->get_name());
+        }
+        return '';
+    }
+
+    /**
+     * Return the localized pluginname string for the question format.
+     *
+     * @return string the pluginname string for the question format
+     */
+    protected function get_name(): string {
+        return get_string('pluginname', get_class($this));
+    }
+
     // Accessor methods
 
     /**

question/format/gift/format.php

@@ -73,6 +73,18 @@ class qformat_gift extends qformat_default {
         return '.txt';
     }
 
+    /**
+     * Validate the given file.
+     *
+     * For more expensive or detailed integrity checks.
+     *
+     * @param stored_file $file the file to check
+     * @return string the error message that occurred while validating the given file
+     */
+    public function validate_file(stored_file $file): string {
+        return $this->validate_is_utf8_file($file);
+    }
+
     protected function answerweightparser(&$answer) {
         $answer = substr($answer, 1);                        // Removes initial %.
         $endposition  = strpos($answer, "%");

question/format/gift/tests/behat/import_export.feature

@@ -50,3 +50,12 @@ Feature: Test importing questions from GIFT format.
     And I should see "Match the activity to the description."
     When I press "Continue"
     Then I should see "Moodle activities"
+
+  @javascript @_file_upload
+  Scenario: import some GIFT questions with unsupported encoding
+    When I navigate to "Question bank" in current page administration
+    And I select "Import" from the "Question bank tertiary navigation" singleselect
+    And I set the field "id_format_gift" to "1"
+    And I upload "question/format/gift/tests/fixtures/questions_encoding_windows-1252.gift.txt" file to "Import" filemanager
+    And I press "id_submitbutton"
+    Then I should see "The file you selected is not in UFT-8 character encoding. GIFT format files must use UTF-8."

question/format/gift/tests/fixtures/questions_encoding_windows-1252.gift.txt

@@ -0,0 +1,18 @@
+// question: 0  name: Switch category to $course$/top/Default for LTTEST
+$CATEGORY: $course$/top/Default for LTTEST
+
+
+// question: 19756780  name: asdf
+::asdf::[html]<p>asdf<br></p>{}
+
+
+// question: 19756810  name: test daniel
+::test daniel::[html]<p>asdfasdf</p>{}
+
+
+// question: 19756750  name: asdf
+::asdf::[html]<p>asdf<br></p>{
+	=<p>aödf<br></p> -> asdf
+	=<p>asdf<br></p> -> asdf
+	=<p>asdf<br></p> -> asdf
+}

question/format/missingword/format.php

@@ -55,6 +55,18 @@ class qformat_missingword extends qformat_default {
         return true;
     }
 
+    /**
+     * Validate the given file.
+     *
+     * For more expensive or detailed integrity checks.
+     *
+     * @param stored_file $file the file to check
+     * @return string the error message that occurred while validating the given file
+     */
+    public function validate_file(stored_file $file): string {
+        return $this->validate_is_utf8_file($file);
+    }
+
     public function readquestion($lines) {
         // Given an array of lines known to define a question in
         // this format, this function converts it into a question

question/format/multianswer/format.php

@@ -39,6 +39,18 @@ class qformat_multianswer extends qformat_default {
         return true;
     }
 
+    /**
+     * Validate the given file.
+     *
+     * For more expensive or detailed integrity checks.
+     *
+     * @param stored_file $file the file to check
+     * @return string the error message that occurred while validating the given file
+     */
+    public function validate_file(stored_file $file): string {
+        return $this->validate_is_utf8_file($file);
+    }
+
     public function readquestions($lines) {
         question_bank::get_qtype('multianswer'); // Ensure the multianswer code is loaded.
 

question/format/upgrade.txt

@@ -9,6 +9,8 @@ discussions about Examview in the last 10 years.
 
 * The WebCT question format has been completely removed (WebCT was acquired by Blackboard in 2006).
 
+* The new validate_file() method in question/format.php can be overwritten
+  to implement more expensive or detailed file integrity checks for question imports.
 
 === 3.6 ===
 

question/format/xml/format.php

@@ -58,6 +58,18 @@ class qformat_xml extends qformat_default {
         return 'application/xml';
     }
 
+    /**
+     * Validate the given file.
+     *
+     * For more expensive or detailed integrity checks.
+     *
+     * @param stored_file $file the file to check
+     * @return string the error message that occurred while validating the given file
+     */
+    public function validate_file(stored_file $file): string {
+        return $this->validate_is_utf8_file($file);
+    }
+
     // IMPORT FUNCTIONS START HERE.
 
     /**