MDL-62042 core_search: Unicode non-characters cause indexing problems

Unicode characters such as U+FFEF can be entered into Moodle data and
cause indexing failures. This change strips them out of search fields.

lib/tests/text_test.php

@@ -412,6 +412,27 @@ class core_text_testcase extends advanced_testcase {
         $this->assertSame($str.$bom, core_text::trim_utf8_bom($bom.$str.$bom));
     }
 
+    /**
+     * Tests the static remove_unicode_non_characters method.
+     */
+    public function test_remove_unicode_non_characters() {
+        // Confirm that texts which don't contain these characters are unchanged.
+        $this->assertSame('Frogs!', core_text::remove_unicode_non_characters('Frogs!'));
+
+        // Even if they contain some very scary characters.
+        $example = html_entity_decode('A�𝅘𝅥B');
+        $this->assertSame($example, core_text::remove_unicode_non_characters($example));
+
+        // Non-characters are removed wherever they may be, with other characters left.
+        $example = html_entity_decode('ABCD�E');
+        $expected = html_entity_decode('ABCD�E');
+        $this->assertSame($expected, core_text::remove_unicode_non_characters($example));
+
+        // If you only have a non-character, you get empty string.
+        $example = html_entity_decode('');
+        $this->assertSame('', core_text::remove_unicode_non_characters($example));
+    }
+
     /**
      * Tests the static get_encodings method.
      */