MDL-62042 core_search: Unicode non-characters cause indexing problems

Unicode characters such as U+FFEF can be entered into Moodle data and
cause indexing failures. This change strips them out of search fields.

lib/classes/text.php

@@ -48,6 +48,11 @@ defined('MOODLE_INTERNAL') || die();
  */
 class core_text {
 
+    /**
+     * @var string[] Array of strings representing Unicode non-characters
+     */
+    protected static $noncharacters;
+
     /**
      * Return t3lib helper class, which is used for conversion between charsets
      *
@@ -628,6 +633,39 @@ class core_text {
         return $str;
     }
 
+    /**
+     * There are a number of Unicode non-characters including the byte-order mark (which may appear
+     * multiple times in a string) and also other ranges. These can cause problems for some
+     * processing.
+     *
+     * This function removes the characters using string replace, so that the rest of the string
+     * remains unchanged.
+     *
+     * @param string $value Input string
+     * @return string Cleaned string value
+     * @since Moodle 3.5
+     */
+    public static function remove_unicode_non_characters($value) {
+        // Set up list of all Unicode non-characters for fast replacing.
+        if (!self::$noncharacters) {
+            self::$noncharacters = [];
+            // This list of characters is based on the Unicode standard. It includes the last two
+            // characters of each code planes 0-16 inclusive...
+            for ($plane = 0; $plane <= 16; $plane++) {
+                $base = ($plane === 0 ? '' : dechex($plane));
+                self::$noncharacters[] = html_entity_decode('&#x' . $base . 'fffe;');
+                self::$noncharacters[] = html_entity_decode('&#x' . $base . 'ffff;');
+            }
+            // ...And the character range U+FDD0 to U+FDEF.
+            for ($char = 0xfdd0; $char <= 0xfdef; $char++) {
+                self::$noncharacters[] = html_entity_decode('&#x' . dechex($char) . ';');
+            }
+        }
+
+        // Do character replacement.
+        return str_replace(self::$noncharacters, '', $value);
+    }
+
     /**
      * Returns encoding options for select boxes, utf-8 and platform encoding first
      *