From d534708fd3198c2fe9b0c11bbf9e5fa45acc1a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20S=CC=8Ckoda?= Date: Mon, 24 Jun 2013 12:39:32 +0200 Subject: [PATCH] MDL-40299 textlib conversion to core_text and core_collator --- admin/cli/install.php | 2 +- install.php | 2 +- lib/classes/collator.php | 307 +++++++++++ lib/classes/component.php | 6 +- lib/classes/text.php | 676 +++++++++++++++++++++++++ lib/setup.php | 1 - lib/tests/collator_test.php | 306 +++++++++++ lib/tests/text_test.php | 387 ++++++++++++++ lib/tests/textlib_test.php | 655 ------------------------ lib/textlib.class.php | 924 +--------------------------------- lib/upgrade.txt | 1 + lib/upgradelib.php | 1 - repository/googledocs/lib.php | 1 - 13 files changed, 1685 insertions(+), 1584 deletions(-) create mode 100644 lib/classes/collator.php create mode 100644 lib/classes/text.php create mode 100644 lib/tests/collator_test.php create mode 100644 lib/tests/text_test.php delete mode 100644 lib/tests/textlib_test.php diff --git a/admin/cli/install.php b/admin/cli/install.php index 684acb5d168..d02d1e31230 100644 --- a/admin/cli/install.php +++ b/admin/cli/install.php @@ -163,10 +163,10 @@ $CFG->admin = array_pop($parts); ini_set('include_path', $CFG->libdir.'/pear' . PATH_SEPARATOR . ini_get('include_path')); require_once($CFG->libdir.'/classes/component.php'); +require_once($CFG->libdir.'/classes/text.php'); require_once($CFG->libdir.'/installlib.php'); require_once($CFG->libdir.'/clilib.php'); require_once($CFG->libdir.'/setuplib.php'); -require_once($CFG->libdir.'/textlib.class.php'); require_once($CFG->libdir.'/weblib.php'); require_once($CFG->libdir.'/dmllib.php'); require_once($CFG->libdir.'/moodlelib.php'); diff --git a/install.php b/install.php index ebca5227390..950b39a4f7d 100644 --- a/install.php +++ b/install.php @@ -193,7 +193,7 @@ if (!empty($memlimit) and $memlimit != -1) { } // Continue with lib loading -require_once($CFG->libdir.'/textlib.class.php'); +require_once($CFG->libdir.'/classes/text.php'); require_once($CFG->libdir.'/weblib.php'); require_once($CFG->libdir.'/outputlib.php'); require_once($CFG->libdir.'/dmllib.php'); diff --git a/lib/classes/collator.php b/lib/classes/collator.php new file mode 100644 index 00000000000..b3341b4418e --- /dev/null +++ b/lib/classes/collator.php @@ -0,0 +1,307 @@ +. + +/** + * Defines string apis + * + * @package core + * @copyright 2011 Sam Hemelryk + * 2012 Petr Skoda + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * A collator class with static methods that can be used for sorting. + * + * @package core + * @copyright 2011 Sam Hemelryk + * 2012 Petr Skoda + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_collator { + /** @const compare items using general PHP comparison, equivalent to Collator::SORT_REGULAR, this may bot be locale aware! */ + const SORT_REGULAR = 0; + + /** @const compare items as strings, equivalent to Collator::SORT_STRING */ + const SORT_STRING = 1; + + /** @const compare items as numbers, equivalent to Collator::SORT_NUMERIC */ + const SORT_NUMERIC = 2; + + /** @const compare items like natsort(), equivalent to SORT_NATURAL */ + const SORT_NATURAL = 6; + + /** @const do not ignore case when sorting, use bitwise "|" with SORT_NATURAL or SORT_STRING, equivalent to Collator::UPPER_FIRST */ + const CASE_SENSITIVE = 64; + + /** @var Collator|false|null **/ + protected static $collator = null; + + /** @var string|null The locale that was used in instantiating the current collator **/ + protected static $locale = null; + + /** + * Prevent class instances, all methods are static. + */ + private function __construct() { + } + + /** + * Ensures that a collator is available and created + * + * @return bool Returns true if collation is available and ready + */ + protected static function ensure_collator_available() { + $locale = get_string('locale', 'langconfig'); + if (is_null(self::$collator) || $locale != self::$locale) { + self::$collator = false; + self::$locale = $locale; + if (class_exists('Collator', false)) { + $collator = new Collator($locale); + if (!empty($collator) && $collator instanceof Collator) { + // Check for non fatal error messages. This has to be done immediately + // after instantiation as any further calls to collation will cause + // it to reset to 0 again (or another error code if one occurred) + $errorcode = $collator->getErrorCode(); + $errormessage = $collator->getErrorMessage(); + // Check for an error code, 0 means no error occurred + if ($errorcode !== 0) { + // Get the actual locale being used, e.g. en, he, zh + $localeinuse = $collator->getLocale(Locale::ACTUAL_LOCALE); + // Check for the common fallback warning error codes. If this occurred + // there is normally little to worry about: + // - U_USING_DEFAULT_WARNING (127) - default fallback locale used (pt => UCA) + // - U_USING_FALLBACK_WARNING (128) - fallback locale used (de_CH => de) + // (UCA: Unicode Collation Algorithm http://unicode.org/reports/tr10/) + if ($errorcode === -127 || $errorcode === -128) { + // Check if the locale in use is UCA default one ('root') or + // if it is anything like the locale we asked for + if ($localeinuse !== 'root' && strpos($locale, $localeinuse) !== 0) { + // The locale we asked for is completely different to the locale + // we have received, let the user know via debugging + debugging('Invalid locale: "' . $locale . '", with warning (not fatal) "' . $errormessage . + '", falling back to "' . $collator->getLocale(Locale::VALID_LOCALE) . '"'); + } else { + // Nothing to do here, this is expected! + // The Moodle locale setting isn't what the collator expected but + // it is smart enough to match the first characters of our locale + // to find the correct locale or to use UCA collation + } + } else { + // We've received some other sort of non fatal warning - let the + // user know about it via debugging. + debugging('Problem with locale: "' . $locale . '", with message "' . $errormessage . + '", falling back to "' . $collator->getLocale(Locale::VALID_LOCALE) . '"'); + } + } + // Store the collator object now that we can be sure it is in a workable condition + self::$collator = $collator; + } else { + // Fatal error while trying to instantiate the collator... something went wrong + debugging('Error instantiating collator for locale: "' . $locale . '", with error [' . + intl_get_error_code() . '] ' . intl_get_error_message($collator)); + } + } + } + return (self::$collator instanceof Collator); + } + + /** + * Restore array contents keeping new keys. + * @static + * @param array $arr + * @param array $original + * @return void modifies $arr + */ + protected static function restore_array(array &$arr, array &$original) { + foreach ($arr as $key => $ignored) { + $arr[$key] = $original[$key]; + } + } + + /** + * Normalise numbers in strings for natural sorting comparisons. + * @static + * @param string $string + * @return string string with normalised numbers + */ + protected static function naturalise($string) { + return preg_replace_callback('/[0-9]+/', array('collatorlib', 'callback_naturalise'), $string); + } + + /** + * @internal + * @static + * @param array $matches + * @return string + */ + public static function callback_naturalise($matches) { + return str_pad($matches[0], 20, '0', STR_PAD_LEFT); + } + + /** + * Locale aware sorting, the key associations are kept, values are sorted alphabetically. + * + * @param array $arr array to be sorted (reference) + * @param int $sortflag One of collatorlib::SORT_NUMERIC, collatorlib::SORT_STRING, collatorlib::SORT_NATURAL, collatorlib::SORT_REGULAR + * optionally "|" collatorlib::CASE_SENSITIVE + * @return bool True on success + */ + public static function asort(array &$arr, $sortflag = collatorlib::SORT_STRING) { + if (empty($arr)) { + // nothing to do + return true; + } + + $original = null; + + $casesensitive = (bool)($sortflag & collatorlib::CASE_SENSITIVE); + $sortflag = ($sortflag & ~collatorlib::CASE_SENSITIVE); + if ($sortflag != collatorlib::SORT_NATURAL and $sortflag != collatorlib::SORT_STRING) { + $casesensitive = false; + } + + if (self::ensure_collator_available()) { + if ($sortflag == collatorlib::SORT_NUMERIC) { + $flag = Collator::SORT_NUMERIC; + + } else if ($sortflag == collatorlib::SORT_REGULAR) { + $flag = Collator::SORT_REGULAR; + + } else { + $flag = Collator::SORT_STRING; + } + + if ($sortflag == collatorlib::SORT_NATURAL) { + $original = $arr; + if ($sortflag == collatorlib::SORT_NATURAL) { + foreach ($arr as $key => $value) { + $arr[$key] = self::naturalise((string)$value); + } + } + } + if ($casesensitive) { + self::$collator->setAttribute(Collator::CASE_FIRST, Collator::UPPER_FIRST); + } else { + self::$collator->setAttribute(Collator::CASE_FIRST, Collator::OFF); + } + $result = self::$collator->asort($arr, $flag); + if ($original) { + self::restore_array($arr, $original); + } + return $result; + } + + // try some fallback that works at least for English + + if ($sortflag == collatorlib::SORT_NUMERIC) { + return asort($arr, SORT_NUMERIC); + + } else if ($sortflag == collatorlib::SORT_REGULAR) { + return asort($arr, SORT_REGULAR); + } + + if (!$casesensitive) { + $original = $arr; + foreach ($arr as $key => $value) { + $arr[$key] = textlib::strtolower($value); + } + } + + if ($sortflag == collatorlib::SORT_NATURAL) { + $result = natsort($arr); + + } else { + $result = asort($arr, SORT_LOCALE_STRING); + } + + if ($original) { + self::restore_array($arr, $original); + } + + return $result; + } + + /** + * Locale aware sort of objects by a property in common to all objects + * + * @param array $objects An array of objects to sort (handled by reference) + * @param string $property The property to use for comparison + * @param int $sortflag One of collatorlib::SORT_NUMERIC, collatorlib::SORT_STRING, collatorlib::SORT_NATURAL, collatorlib::SORT_REGULAR + * optionally "|" collatorlib::CASE_SENSITIVE + * @return bool True on success + */ + public static function asort_objects_by_property(array &$objects, $property, $sortflag = collatorlib::SORT_STRING) { + $original = $objects; + foreach ($objects as $key => $object) { + $objects[$key] = $object->$property; + } + $result = self::asort($objects, $sortflag); + self::restore_array($objects, $original); + return $result; + } + + /** + * Locale aware sort of objects by a method in common to all objects + * + * @param array $objects An array of objects to sort (handled by reference) + * @param string $method The method to call to generate a value for comparison + * @param int $sortflag One of collatorlib::SORT_NUMERIC, collatorlib::SORT_STRING, collatorlib::SORT_NATURAL, collatorlib::SORT_REGULAR + * optionally "|" collatorlib::CASE_SENSITIVE + * @return bool True on success + */ + public static function asort_objects_by_method(array &$objects, $method, $sortflag = collatorlib::SORT_STRING) { + $original = $objects; + foreach ($objects as $key => $object) { + $objects[$key] = $object->{$method}(); + } + $result = self::asort($objects, $sortflag); + self::restore_array($objects, $original); + return $result; + } + + /** + * Locale aware sorting, the key associations are kept, keys are sorted alphabetically. + * + * @param array $arr array to be sorted (reference) + * @param int $sortflag One of collatorlib::SORT_NUMERIC, collatorlib::SORT_STRING, collatorlib::SORT_NATURAL, collatorlib::SORT_REGULAR + * optionally "|" collatorlib::CASE_SENSITIVE + * @return bool True on success + */ + public static function ksort(array &$arr, $sortflag = collatorlib::SORT_STRING) { + $keys = array_keys($arr); + if (!self::asort($keys, $sortflag)) { + return false; + } + // This is a bit slow, but we need to keep the references + $original = $arr; + $arr = array(); // Surprisingly this does not break references outside + foreach ($keys as $key) { + $arr[$key] = $original[$key]; + } + + return true; + } +} + +/** + * Legacy collatorlib. + * @deprecated since 2.6, use core_collator:: instead. + */ +class collatorlib extends core_collator { +} diff --git a/lib/classes/component.php b/lib/classes/component.php index 84f46a9b0fd..18952450a66 100644 --- a/lib/classes/component.php +++ b/lib/classes/component.php @@ -420,9 +420,9 @@ $cache = '.var_export($cache, true).'; } } - // Note: Add a few extra legacy classes here if necessary. - //self::$classmap['textlib'] = "$CFG->dirroot/lib/textlib.class.php"; - //self::$classmap['collatorlib'] = "$CFG->dirroot/lib/textlib.class.php"; + // Note: Add extra deprecated legacy classes here as necessary. + self::$classmap['textlib'] = "$CFG->dirroot/lib/classes/text.php"; + self::$classmap['collatorlib'] = "$CFG->dirroot/lib/classes/collator.php"; } /** diff --git a/lib/classes/text.php b/lib/classes/text.php new file mode 100644 index 00000000000..1ca51f8608b --- /dev/null +++ b/lib/classes/text.php @@ -0,0 +1,676 @@ +. + +/** + * Defines string apis + * + * @package core + * @copyright (C) 2001-3001 Eloy Lafuente (stronk7) {@link http://contiento.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * defines string api's for manipulating strings + * + * This class is used to manipulate strings under Moodle 1.6 an later. As + * utf-8 text become mandatory a pool of safe functions under this encoding + * become necessary. The name of the methods is exactly the + * same than their PHP originals. + * + * A big part of this class acts as a wrapper over the Typo3 charset library, + * really a cool group of utilities to handle texts and encoding conversion. + * + * Take a look to its own copyright and license details. + * + * IMPORTANT Note: Typo3 libraries always expect lowercase charsets to use 100% + * its capabilities so, don't forget to make the conversion + * from every wrapper function! + * + * @package core + * @category string + * @copyright 1999 onwards Martin Dougiamas {@link http://moodle.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_text { + + /** + * Return t3lib helper class, which is used for conversion between charsets + * + * @param bool $reset + * @return t3lib_cs + */ + protected static function typo3($reset = false) { + static $typo3cs = null; + + if ($reset) { + $typo3cs = null; + return null; + } + + if (isset($typo3cs)) { + return $typo3cs; + } + + global $CFG; + + // Required files + require_once($CFG->libdir.'/typo3/class.t3lib_cs.php'); + require_once($CFG->libdir.'/typo3/class.t3lib_div.php'); + require_once($CFG->libdir.'/typo3/interface.t3lib_singleton.php'); + require_once($CFG->libdir.'/typo3/class.t3lib_l10n_locales.php'); + + // do not use mbstring or recode because it may return invalid results in some corner cases + $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'iconv'; + $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] = 'iconv'; + + // Tell Typo3 we are curl enabled always (mandatory since 2.0) + $GLOBALS['TYPO3_CONF_VARS']['SYS']['curlUse'] = '1'; + + // And this directory must exist to allow Typo to cache conversion + // tables when using internal functions + make_temp_directory('typo3temp/cs'); + + // Make sure typo is using our dir permissions + $GLOBALS['TYPO3_CONF_VARS']['BE']['folderCreateMask'] = decoct($CFG->directorypermissions); + + // Default mask for Typo + $GLOBALS['TYPO3_CONF_VARS']['BE']['fileCreateMask'] = $CFG->directorypermissions; + + // This full path constants must be defined too, transforming backslashes + // to forward slashed because Typo3 requires it. + if (!defined('PATH_t3lib')) { + define('PATH_t3lib', str_replace('\\','/',$CFG->libdir.'/typo3/')); + define('PATH_typo3', str_replace('\\','/',$CFG->libdir.'/typo3/')); + define('PATH_site', str_replace('\\','/',$CFG->tempdir.'/')); + define('TYPO3_OS', stristr(PHP_OS,'win')&&!stristr(PHP_OS,'darwin')?'WIN':''); + } + + $typo3cs = new t3lib_cs(); + + return $typo3cs; + } + + /** + * Reset internal textlib caches. + * @static + */ + public static function reset_caches() { + self::typo3(true); + } + + /** + * Standardise charset name + * + * Please note it does not mean the returned charset is actually supported. + * + * @static + * @param string $charset raw charset name + * @return string normalised lowercase charset name + */ + public static function parse_charset($charset) { + $charset = strtolower($charset); + + // shortcuts so that we do not have to load typo3 on every page + + if ($charset === 'utf8' or $charset === 'utf-8') { + return 'utf-8'; + } + + if (preg_match('/^(cp|win|windows)-?(12[0-9]{2})$/', $charset, $matches)) { + return 'windows-'.$matches[2]; + } + + if (preg_match('/^iso-8859-[0-9]+$/', $charset, $matches)) { + return $charset; + } + + if ($charset === 'euc-jp') { + return 'euc-jp'; + } + if ($charset === 'iso-2022-jp') { + return 'iso-2022-jp'; + } + if ($charset === 'shift-jis' or $charset === 'shift_jis') { + return 'shift_jis'; + } + if ($charset === 'gb2312') { + return 'gb2312'; + } + if ($charset === 'gb18030') { + return 'gb18030'; + } + + // fallback to typo3 + return self::typo3()->parse_charset($charset); + } + + /** + * Converts the text between different encodings. It uses iconv extension with //TRANSLIT parameter, + * falls back to typo3. If both source and target are utf-8 it tries to fix invalid characters only. + * + * @param string $text + * @param string $fromCS source encoding + * @param string $toCS result encoding + * @return string|bool converted string or false on error + */ + public static function convert($text, $fromCS, $toCS='utf-8') { + $fromCS = self::parse_charset($fromCS); + $toCS = self::parse_charset($toCS); + + $text = (string)$text; // we can work only with strings + + if ($text === '') { + return ''; + } + + if ($toCS === 'utf-8' and $fromCS === 'utf-8') { + return fix_utf8($text); + } + + $result = iconv($fromCS, $toCS.'//TRANSLIT', $text); + + if ($result === false or $result === '') { + // note: iconv is prone to return empty string when invalid char encountered, or false if encoding unsupported + $oldlevel = error_reporting(E_PARSE); + $result = self::typo3()->conv((string)$text, $fromCS, $toCS); + error_reporting($oldlevel); + } + + return $result; + } + + /** + * Multibyte safe substr() function, uses mbstring or iconv for UTF-8, falls back to typo3. + * + * @param string $text string to truncate + * @param int $start negative value means from end + * @param int $len maximum length of characters beginning from start + * @param string $charset encoding of the text + * @return string portion of string specified by the $start and $len + */ + public static function substr($text, $start, $len=null, $charset='utf-8') { + $charset = self::parse_charset($charset); + + if ($charset === 'utf-8') { + if (function_exists('mb_substr')) { + // this is much faster than iconv - see MDL-31142 + if ($len === null) { + $oldcharset = mb_internal_encoding(); + mb_internal_encoding('UTF-8'); + $result = mb_substr($text, $start); + mb_internal_encoding($oldcharset); + return $result; + } else { + return mb_substr($text, $start, $len, 'UTF-8'); + } + + } else { + if ($len === null) { + $len = iconv_strlen($text, 'UTF-8'); + } + return iconv_substr($text, $start, $len, 'UTF-8'); + } + } + + $oldlevel = error_reporting(E_PARSE); + if ($len === null) { + $result = self::typo3()->substr($charset, (string)$text, $start); + } else { + $result = self::typo3()->substr($charset, (string)$text, $start, $len); + } + error_reporting($oldlevel); + + return $result; + } + + /** + * Multibyte safe strlen() function, uses mbstring or iconv for UTF-8, falls back to typo3. + * + * @param string $text input string + * @param string $charset encoding of the text + * @return int number of characters + */ + public static function strlen($text, $charset='utf-8') { + $charset = self::parse_charset($charset); + + if ($charset === 'utf-8') { + if (function_exists('mb_strlen')) { + return mb_strlen($text, 'UTF-8'); + } else { + return iconv_strlen($text, 'UTF-8'); + } + } + + $oldlevel = error_reporting(E_PARSE); + $result = self::typo3()->strlen($charset, (string)$text); + error_reporting($oldlevel); + + return $result; + } + + /** + * Multibyte safe strtolower() function, uses mbstring, falls back to typo3. + * + * @param string $text input string + * @param string $charset encoding of the text (may not work for all encodings) + * @return string lower case text + */ + public static function strtolower($text, $charset='utf-8') { + $charset = self::parse_charset($charset); + + if ($charset === 'utf-8' and function_exists('mb_strtolower')) { + return mb_strtolower($text, 'UTF-8'); + } + + $oldlevel = error_reporting(E_PARSE); + $result = self::typo3()->conv_case($charset, (string)$text, 'toLower'); + error_reporting($oldlevel); + + return $result; + } + + /** + * Multibyte safe strtoupper() function, uses mbstring, falls back to typo3. + * + * @param string $text input string + * @param string $charset encoding of the text (may not work for all encodings) + * @return string upper case text + */ + public static function strtoupper($text, $charset='utf-8') { + $charset = self::parse_charset($charset); + + if ($charset === 'utf-8' and function_exists('mb_strtoupper')) { + return mb_strtoupper($text, 'UTF-8'); + } + + $oldlevel = error_reporting(E_PARSE); + $result = self::typo3()->conv_case($charset, (string)$text, 'toUpper'); + error_reporting($oldlevel); + + return $result; + } + + /** + * Find the position of the first occurrence of a substring in a string. + * UTF-8 ONLY safe strpos(), uses mbstring, falls back to iconv. + * + * @param string $haystack the string to search in + * @param string $needle one or more charachters to search for + * @param int $offset offset from begining of string + * @return int the numeric position of the first occurrence of needle in haystack. + */ + public static function strpos($haystack, $needle, $offset=0) { + if (function_exists('mb_strpos')) { + return mb_strpos($haystack, $needle, $offset, 'UTF-8'); + } else { + return iconv_strpos($haystack, $needle, $offset, 'UTF-8'); + } + } + + /** + * Find the position of the last occurrence of a substring in a string + * UTF-8 ONLY safe strrpos(), uses mbstring, falls back to iconv. + * + * @param string $haystack the string to search in + * @param string $needle one or more charachters to search for + * @return int the numeric position of the last occurrence of needle in haystack + */ + public static function strrpos($haystack, $needle) { + if (function_exists('mb_strpos')) { + return mb_strrpos($haystack, $needle, null, 'UTF-8'); + } else { + return iconv_strrpos($haystack, $needle, 'UTF-8'); + } + } + + /** + * Try to convert upper unicode characters to plain ascii, + * the returned string may contain unconverted unicode characters. + * + * @param string $text input string + * @param string $charset encoding of the text + * @return string converted ascii string + */ + public static function specialtoascii($text, $charset='utf-8') { + $charset = self::parse_charset($charset); + $oldlevel = error_reporting(E_PARSE); + $result = self::typo3()->specCharsToASCII($charset, (string)$text); + error_reporting($oldlevel); + return $result; + } + + /** + * Generate a correct base64 encoded header to be used in MIME mail messages. + * This function seems to be 100% compliant with RFC1342. Credits go to: + * paravoid (http://www.php.net/manual/en/function.mb-encode-mimeheader.php#60283). + * + * @param string $text input string + * @param string $charset encoding of the text + * @return string base64 encoded header + */ + public static function encode_mimeheader($text, $charset='utf-8') { + if (empty($text)) { + return (string)$text; + } + // Normalize charset + $charset = self::parse_charset($charset); + // If the text is pure ASCII, we don't need to encode it + if (self::convert($text, $charset, 'ascii') == $text) { + return $text; + } + // Although RFC says that line feed should be \r\n, it seems that + // some mailers double convert \r, so we are going to use \n alone + $linefeed="\n"; + // Define start and end of every chunk + $start = "=?$charset?B?"; + $end = "?="; + // Accumulate results + $encoded = ''; + // Max line length is 75 (including start and end) + $length = 75 - strlen($start) - strlen($end); + // Multi-byte ratio + $multilength = self::strlen($text, $charset); + // Detect if strlen and friends supported + if ($multilength === false) { + if ($charset == 'GB18030' or $charset == 'gb18030') { + while (strlen($text)) { + // try to encode first 22 chars - we expect most chars are two bytes long + if (preg_match('/^(([\x00-\x7f])|([\x81-\xfe][\x40-\x7e])|([\x81-\xfe][\x80-\xfe])|([\x81-\xfe][\x30-\x39]..)){1,22}/m', $text, $matches)) { + $chunk = $matches[0]; + $encchunk = base64_encode($chunk); + if (strlen($encchunk) > $length) { + // find first 11 chars - each char in 4 bytes - worst case scenario + preg_match('/^(([\x00-\x7f])|([\x81-\xfe][\x40-\x7e])|([\x81-\xfe][\x80-\xfe])|([\x81-\xfe][\x30-\x39]..)){1,11}/m', $text, $matches); + $chunk = $matches[0]; + $encchunk = base64_encode($chunk); + } + $text = substr($text, strlen($chunk)); + $encoded .= ' '.$start.$encchunk.$end.$linefeed; + } else { + break; + } + } + $encoded = trim($encoded); + return $encoded; + } else { + return false; + } + } + $ratio = $multilength / strlen($text); + // Base64 ratio + $magic = $avglength = floor(3 * $length * $ratio / 4); + // basic infinite loop protection + $maxiterations = strlen($text)*2; + $iteration = 0; + // Iterate over the string in magic chunks + for ($i=0; $i <= $multilength; $i+=$magic) { + if ($iteration++ > $maxiterations) { + return false; // probably infinite loop + } + $magic = $avglength; + $offset = 0; + // Ensure the chunk fits in length, reducing magic if necessary + do { + $magic -= $offset; + $chunk = self::substr($text, $i, $magic, $charset); + $chunk = base64_encode($chunk); + $offset++; + } while (strlen($chunk) > $length); + // This chunk doesn't break any multi-byte char. Use it. + if ($chunk) + $encoded .= ' '.$start.$chunk.$end.$linefeed; + } + // Strip the first space and the last linefeed + $encoded = substr($encoded, 1, -strlen($linefeed)); + + return $encoded; + } + + /** + * Returns HTML entity transliteration table. + * @return array with (html entity => utf-8) elements + */ + protected static function get_entities_table() { + static $trans_tbl = null; + + // Generate/create $trans_tbl + if (!isset($trans_tbl)) { + if (version_compare(phpversion(), '5.3.4') < 0) { + $trans_tbl = array(); + foreach (get_html_translation_table(HTML_ENTITIES) as $val=>$key) { + $trans_tbl[$key] = self::convert($val, 'ISO-8859-1', 'utf-8'); + } + + } else if (version_compare(phpversion(), '5.4.0') < 0) { + $trans_tbl = get_html_translation_table(HTML_ENTITIES, ENT_COMPAT, 'UTF-8'); + $trans_tbl = array_flip($trans_tbl); + + } else { + $trans_tbl = get_html_translation_table(HTML_ENTITIES, ENT_COMPAT | ENT_HTML401, 'UTF-8'); + $trans_tbl = array_flip($trans_tbl); + } + } + + return $trans_tbl; + } + + /** + * Converts all the numeric entities &#nnnn; or &#xnnn; to UTF-8 + * Original from laurynas dot butkus at gmail at: + * http://php.net/manual/en/function.html-entity-decode.php#75153 + * with some custom mods to provide more functionality + * + * @param string $str input string + * @param boolean $htmlent convert also html entities (defaults to true) + * @return string encoded UTF-8 string + */ + public static function entities_to_utf8($str, $htmlent=true) { + static $callback1 = null ; + static $callback2 = null ; + + if (!$callback1 or !$callback2) { + $callback1 = create_function('$matches', 'return core_text::code2utf8(hexdec($matches[1]));'); + $callback2 = create_function('$matches', 'return core_text::code2utf8($matches[1]);'); + } + + $result = (string)$str; + $result = preg_replace_callback('/&#x([0-9a-f]+);/i', $callback1, $result); + $result = preg_replace_callback('/&#([0-9]+);/', $callback2, $result); + + // Replace literal entities (if desired) + if ($htmlent) { + $trans_tbl = self::get_entities_table(); + // It should be safe to search for ascii strings and replace them with utf-8 here. + $result = strtr($result, $trans_tbl); + } + // Return utf8-ised string + return $result; + } + + /** + * Converts all Unicode chars > 127 to numeric entities &#nnnn; or &#xnnn;. + * + * @param string $str input string + * @param boolean $dec output decadic only number entities + * @param boolean $nonnum remove all non-numeric entities + * @return string converted string + */ + public static function utf8_to_entities($str, $dec=false, $nonnum=false) { + static $callback = null ; + + if ($nonnum) { + $str = self::entities_to_utf8($str, true); + } + + // Avoid some notices from Typo3 code + $oldlevel = error_reporting(E_PARSE); + $result = self::typo3()->utf8_to_entities((string)$str); + error_reporting($oldlevel); + + if ($dec) { + if (!$callback) { + $callback = create_function('$matches', 'return \'&#\'.(hexdec($matches[1])).\';\';'); + } + $result = preg_replace_callback('/&#x([0-9a-f]+);/i', $callback, $result); + } + + return $result; + } + + /** + * Removes the BOM from unicode string {@link http://unicode.org/faq/utf_bom.html} + * + * @param string $str input string + * @return string + */ + public static function trim_utf8_bom($str) { + $bom = "\xef\xbb\xbf"; + if (strpos($str, $bom) === 0) { + return substr($str, strlen($bom)); + } + return $str; + } + + /** + * Returns encoding options for select boxes, utf-8 and platform encoding first + * + * @return array encodings + */ + public static function get_encodings() { + $encodings = array(); + $encodings['UTF-8'] = 'UTF-8'; + $winenc = strtoupper(get_string('localewincharset', 'langconfig')); + if ($winenc != '') { + $encodings[$winenc] = $winenc; + } + $nixenc = strtoupper(get_string('oldcharset', 'langconfig')); + $encodings[$nixenc] = $nixenc; + + foreach (self::typo3()->synonyms as $enc) { + $enc = strtoupper($enc); + $encodings[$enc] = $enc; + } + return $encodings; + } + + /** + * Returns the utf8 string corresponding to the unicode value + * (from php.net, courtesy - romans@void.lv) + * + * @param int $num one unicode value + * @return string the UTF-8 char corresponding to the unicode value + */ + public static function code2utf8($num) { + if ($num < 128) { + return chr($num); + } + if ($num < 2048) { + return chr(($num >> 6) + 192) . chr(($num & 63) + 128); + } + if ($num < 65536) { + return chr(($num >> 12) + 224) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128); + } + if ($num < 2097152) { + return chr(($num >> 18) + 240) . chr((($num >> 12) & 63) + 128) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128); + } + return ''; + } + + /** + * Returns the code of the given UTF-8 character + * + * @param string $utf8char one UTF-8 character + * @return int the code of the given character + */ + public static function utf8ord($utf8char) { + if ($utf8char == '') { + return 0; + } + $ord0 = ord($utf8char{0}); + if ($ord0 >= 0 && $ord0 <= 127) { + return $ord0; + } + $ord1 = ord($utf8char{1}); + if ($ord0 >= 192 && $ord0 <= 223) { + return ($ord0 - 192) * 64 + ($ord1 - 128); + } + $ord2 = ord($utf8char{2}); + if ($ord0 >= 224 && $ord0 <= 239) { + return ($ord0 - 224) * 4096 + ($ord1 - 128) * 64 + ($ord2 - 128); + } + $ord3 = ord($utf8char{3}); + if ($ord0 >= 240 && $ord0 <= 247) { + return ($ord0 - 240) * 262144 + ($ord1 - 128 )* 4096 + ($ord2 - 128) * 64 + ($ord3 - 128); + } + return false; + } + + /** + * Makes first letter of each word capital - words must be separated by spaces. + * Use with care, this function does not work properly in many locales!!! + * + * @param string $text input string + * @return string + */ + public static function strtotitle($text) { + if (empty($text)) { + return $text; + } + + if (function_exists('mb_convert_case')) { + return mb_convert_case($text, MB_CASE_TITLE, 'UTF-8'); + } + + $text = self::strtolower($text); + $words = explode(' ', $text); + foreach ($words as $i=>$word) { + $length = self::strlen($word); + if (!$length) { + continue; + + } else if ($length == 1) { + $words[$i] = self::strtoupper($word); + + } else { + $letter = self::substr($word, 0, 1); + $letter = self::strtoupper($letter); + $rest = self::substr($word, 1); + $words[$i] = $letter.$rest; + } + } + return implode(' ', $words); + } +} + +/** + * Legacy tectlib. + * @deprecated since 2.6, use core_text:: instead. + */ +class textlib extends core_text { + /** + * Locale aware sorting, the key associations are kept, values are sorted alphabetically. + * + * @param array $arr array to be sorted (reference) + * @param int $sortflag One of Collator::SORT_REGULAR, Collator::SORT_NUMERIC, Collator::SORT_STRING + * @return void modifies parameter + */ + public static function asort(array &$arr, $sortflag = null) { + debugging('textlib::asort has been superseeded by collatorlib::asort please upgrade your code to use that', DEBUG_DEVELOPER); + collatorlib::asort($arr, $sortflag); + } +} \ No newline at end of file diff --git a/lib/setup.php b/lib/setup.php index 7d51e38d5ed..bd590d7bf2d 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -514,7 +514,6 @@ if (defined('COMPONENT_CLASSLOADER')) { } // Load up standard libraries -require_once($CFG->libdir .'/textlib.class.php'); // Functions to handle multibyte strings require_once($CFG->libdir .'/filterlib.php'); // Functions for filtering test as it is output require_once($CFG->libdir .'/ajax/ajaxlib.php'); // Functions for managing our use of JavaScript and YUI require_once($CFG->libdir .'/weblib.php'); // Functions relating to HTTP and content diff --git a/lib/tests/collator_test.php b/lib/tests/collator_test.php new file mode 100644 index 00000000000..9003be53c93 --- /dev/null +++ b/lib/tests/collator_test.php @@ -0,0 +1,306 @@ +. + +/** + * Collator unit tests. + * + * @package core + * @category phpunit + * @copyright 2011 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Unit tests for our utf-8 aware collator. + * + * Used for sorting. + * + * @package core + * @category phpunit + * @copyright 2011 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_collator_testcase extends basic_testcase { + + /** + * @var string The initial lang, stored because we change it during testing + */ + protected $initiallang = null; + + /** + * @var string The last error that has occured + */ + protected $error = null; + + /** + * Prepares things for this test case + * @return void + */ + protected function setUp() { + global $SESSION; + if (isset($SESSION->lang)) { + $this->initiallang = $SESSION->lang; + } + $SESSION->lang = 'en'; // make sure we test en language to get consistent results, hopefully all systems have this locale + if (extension_loaded('intl')) { + $this->error = 'Collation aware sorting not supported'; + } else { + $this->error = 'Collation aware sorting not supported, PHP extension "intl" is not available.'; + } + parent::setUp(); + } + + /** + * Cleans things up after this test case has run + * @return void + */ + protected function tearDown() { + global $SESSION; + parent::tearDown(); + if ($this->initiallang !== null) { + $SESSION->lang = $this->initiallang; + $this->initiallang = null; + } else { + unset($SESSION->lang); + } + } + + /** + * Tests the static asort method + * @return void + */ + public function test_asort() { + $arr = array('b' => 'ab', 1 => 'aa', 0 => 'cc'); + $result = core_collator::asort($arr); + $this->assertSame(array_values($arr), array('aa', 'ab', 'cc')); + $this->assertSame(array_keys($arr), array(1, 'b', 0)); + $this->assertTrue($result); + + $arr = array('b' => 'ab', 1 => 'aa', 0 => 'cc'); + $result = core_collator::asort($arr, core_collator::SORT_STRING); + $this->assertSame(array_values($arr), array('aa', 'ab', 'cc')); + $this->assertSame(array_keys($arr), array(1, 'b', 0)); + $this->assertTrue($result); + + $arr = array('b' => 'aac', 1 => 'Aac', 0 => 'cc'); + $result = core_collator::asort($arr, (core_collator::SORT_STRING | core_collator::CASE_SENSITIVE)); + $this->assertSame(array_values($arr), array('Aac', 'aac', 'cc')); + $this->assertSame(array_keys($arr), array(1, 'b', 0)); + $this->assertTrue($result); + + $arr = array('b' => 'a1', 1 => 'a10', 0 => 'a3b'); + $result = core_collator::asort($arr); + $this->assertSame(array_values($arr), array('a1', 'a10', 'a3b')); + $this->assertSame(array_keys($arr), array('b', 1, 0)); + $this->assertTrue($result); + + $arr = array('b' => 'a1', 1 => 'a10', 0 => 'a3b'); + $result = core_collator::asort($arr, core_collator::SORT_NATURAL); + $this->assertSame(array_values($arr), array('a1', 'a3b', 'a10')); + $this->assertSame(array_keys($arr), array('b', 0, 1)); + $this->assertTrue($result); + + $arr = array('b' => '1.1.1', 1 => '1.2', 0 => '1.20.2'); + $result = core_collator::asort($arr, core_collator::SORT_NATURAL); + $this->assertSame(array_values($arr), array('1.1.1', '1.2', '1.20.2')); + $this->assertSame(array_keys($arr), array('b', 1, 0)); + $this->assertTrue($result); + + $arr = array('b' => '-1', 1 => 1000, 0 => -1.2, 3 => 1, 4 => false); + $result = core_collator::asort($arr, core_collator::SORT_NUMERIC); + $this->assertSame(array_values($arr), array(-1.2, '-1', false, 1, 1000)); + $this->assertSame(array_keys($arr), array(0, 'b', 4, 3, 1)); + $this->assertTrue($result); + + $arr = array('b' => array(1), 1 => array(2, 3), 0 => 1); + $result = core_collator::asort($arr, core_collator::SORT_REGULAR); + $this->assertSame(array_values($arr), array(1, array(1), array(2, 3))); + $this->assertSame(array_keys($arr), array(0, 'b', 1)); + $this->assertTrue($result); + + // test sorting of array of arrays - first element should be used for actual comparison + $arr = array(0=>array('bb', 'z'), 1=>array('ab', 'a'), 2=>array('zz', 'x')); + $result = core_collator::asort($arr, core_collator::SORT_REGULAR); + $this->assertSame(array_keys($arr), array(1, 0, 2)); + $this->assertTrue($result); + + $arr = array('a' => 'áb', 'b' => 'ab', 1 => 'aa', 0=>'cc', 'x' => 'Áb',); + $result = core_collator::asort($arr); + $this->assertSame(array_values($arr), array('aa', 'ab', 'áb', 'Áb', 'cc'), $this->error); + $this->assertSame(array_keys($arr), array(1, 'b', 'a', 'x', 0), $this->error); + $this->assertTrue($result); + + $a = array(2=>'b', 1=>'c'); + $c =& $a; + $b =& $a; + core_collator::asort($b); + $this->assertSame($a, $b); + $this->assertSame($c, $b); + } + + /** + * Tests the static asort_objects_by_method method + * @return void + */ + public function test_asort_objects_by_method() { + $objects = array( + 'b' => new string_test_class('ab'), + 1 => new string_test_class('aa'), + 0 => new string_test_class('cc') + ); + $result = core_collator::asort_objects_by_method($objects, 'get_protected_name'); + $this->assertSame(array_keys($objects), array(1, 'b', 0)); + $this->assertSame($this->get_ordered_names($objects, 'get_protected_name'), array('aa', 'ab', 'cc')); + $this->assertTrue($result); + + $objects = array( + 'b' => new string_test_class('a20'), + 1 => new string_test_class('a1'), + 0 => new string_test_class('a100') + ); + $result = core_collator::asort_objects_by_method($objects, 'get_protected_name', core_collator::SORT_NATURAL); + $this->assertSame(array_keys($objects), array(1, 'b', 0)); + $this->assertSame($this->get_ordered_names($objects, 'get_protected_name'), array('a1', 'a20', 'a100')); + $this->assertTrue($result); + } + + /** + * Tests the static asort_objects_by_method method + * @return void + */ + public function test_asort_objects_by_property() { + $objects = array( + 'b' => new string_test_class('ab'), + 1 => new string_test_class('aa'), + 0 => new string_test_class('cc') + ); + $result = core_collator::asort_objects_by_property($objects, 'publicname'); + $this->assertSame(array_keys($objects), array(1, 'b', 0)); + $this->assertSame($this->get_ordered_names($objects, 'publicname'), array('aa', 'ab', 'cc')); + $this->assertTrue($result); + + $objects = array( + 'b' => new string_test_class('a20'), + 1 => new string_test_class('a1'), + 0 => new string_test_class('a100') + ); + $result = core_collator::asort_objects_by_property($objects, 'publicname', core_collator::SORT_NATURAL); + $this->assertSame(array_keys($objects), array(1, 'b', 0)); + $this->assertSame($this->get_ordered_names($objects, 'publicname'), array('a1', 'a20', 'a100')); + $this->assertTrue($result); + } + + /** + * Returns an array of sorted names + * @param array $objects + * @param string $methodproperty + * @return type + */ + protected function get_ordered_names($objects, $methodproperty = 'get_protected_name') { + $return = array(); + foreach ($objects as $object) { + if ($methodproperty == 'publicname') { + $return[] = $object->publicname; + } else { + $return[] = $object->$methodproperty(); + } + } + return $return; + } + + /** + * Tests the static ksort method + * @return void + */ + public function test_ksort() { + $arr = array('b' => 'ab', 1 => 'aa', 0 => 'cc'); + $result = core_collator::ksort($arr); + $this->assertSame(array_keys($arr), array(0, 1, 'b')); + $this->assertSame(array_values($arr), array('cc', 'aa', 'ab')); + $this->assertTrue($result); + + $obj = new stdClass(); + $arr = array('1.1.1'=>array(), '1.2'=>$obj, '1.20.2'=>null); + $result = core_collator::ksort($arr, core_collator::SORT_NATURAL); + $this->assertSame(array_keys($arr), array('1.1.1', '1.2', '1.20.2')); + $this->assertSame(array_values($arr), array(array(), $obj, null)); + $this->assertTrue($result); + + $a = array(2=>'b', 1=>'c'); + $c =& $a; + $b =& $a; + core_collator::ksort($b); + $this->assertSame($a, $b); + $this->assertSame($c, $b); + } + + public function test_legacy_collatorlib() { + $arr = array('b' => 'ab', 1 => 'aa', 0 => 'cc'); + $result = collatorlib::asort($arr); + $this->assertSame(array_values($arr), array('aa', 'ab', 'cc')); + $this->assertSame(array_keys($arr), array(1, 'b', 0)); + $this->assertTrue($result); + } +} + + +/** + * Simple class used to work with the unit test. + * + * @package core + * @category phpunit + * @copyright 2011 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class string_test_class extends stdClass { + /** + * @var string A public property + */ + public $publicname; + /** + * @var string A protected property + */ + protected $protectedname; + /** + * @var string A private property + */ + private $privatename; + /** + * Constructs the test instance + * @param string $name + */ + public function __construct($name) { + $this->publicname = $name; + $this->protectedname = $name; + $this->privatename = $name; + } + /** + * Returns the protected property + * @return string + */ + public function get_protected_name() { + return $this->protectedname; + } + /** + * Returns the protected property + * @return string + */ + public function get_private_name() { + return $this->publicname; + } +} \ No newline at end of file diff --git a/lib/tests/text_test.php b/lib/tests/text_test.php new file mode 100644 index 00000000000..9903743e7c5 --- /dev/null +++ b/lib/tests/text_test.php @@ -0,0 +1,387 @@ +. + +/** + * core_text unit tests + * + * @package core + * @category phpunit + * @copyright 2012 Petr Skoda {@link http://skodak.org} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + + +/** + * Unit tests for our utf-8 aware text processing + * + * @package core + * @category phpunit + * @copyright 2010 Petr Skoda (http://skodak.org) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_text_testcase extends advanced_testcase { + + /** + * Tests the static parse charset method + * @return void + */ + public function test_parse_charset() { + $this->assertSame(core_text::parse_charset('Cp1250'), 'windows-1250'); + // does typo3 work? some encoding moodle does not use + $this->assertSame(core_text::parse_charset('ms-ansi'), 'windows-1252'); + } + + /** + * Tests the static convert method + * @return void + */ + public function test_convert() { + $utf8 = "Žluťoučký koníček"; + $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); + $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::convert($utf8, 'utf-8', 'iso-8859-2'), $iso2); + $this->assertSame(core_text::convert($iso2, 'iso-8859-2', 'utf-8'), $utf8); + $this->assertSame(core_text::convert($utf8, 'utf-8', 'win-1250'), $win); + $this->assertSame(core_text::convert($win, 'win-1250', 'utf-8'), $utf8); + $this->assertSame(core_text::convert($win, 'win-1250', 'iso-8859-2'), $iso2); + $this->assertSame(core_text::convert($iso2, 'iso-8859-2', 'win-1250'), $win); + $this->assertSame(core_text::convert($iso2, 'iso-8859-2', 'iso-8859-2'), $iso2); + $this->assertSame(core_text::convert($win, 'win-1250', 'cp1250'), $win); + $this->assertSame(core_text::convert($utf8, 'utf-8', 'utf-8'), $utf8); + + + $utf8 = '言語設定'; + $str = pack("H*", "b8c0b8ecc0dfc4ea"); //EUC-JP + $this->assertSame(core_text::convert($utf8, 'utf-8', 'EUC-JP'), $str); + $this->assertSame(core_text::convert($str, 'EUC-JP', 'utf-8'), $utf8); + $this->assertSame(core_text::convert($utf8, 'utf-8', 'utf-8'), $utf8); + + $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP + $this->assertSame(core_text::convert($utf8, 'utf-8', 'ISO-2022-JP'), $str); + $this->assertSame(core_text::convert($str, 'ISO-2022-JP', 'utf-8'), $utf8); + $this->assertSame(core_text::convert($utf8, 'utf-8', 'utf-8'), $utf8); + + $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS + $this->assertSame(core_text::convert($utf8, 'utf-8', 'SHIFT-JIS'), $str); + $this->assertSame(core_text::convert($str, 'SHIFT-JIS', 'utf-8'), $utf8); + $this->assertSame(core_text::convert($utf8, 'utf-8', 'utf-8'), $utf8); + + $utf8 = '简体中文'; + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 + $this->assertSame(core_text::convert($utf8, 'utf-8', 'GB2312'), $str); + $this->assertSame(core_text::convert($str, 'GB2312', 'utf-8'), $utf8); + $this->assertSame(core_text::convert($utf8, 'utf-8', 'utf-8'), $utf8); + + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 + $this->assertSame(core_text::convert($utf8, 'utf-8', 'GB18030'), $str); + $this->assertSame(core_text::convert($str, 'GB18030', 'utf-8'), $utf8); + $this->assertSame(core_text::convert($utf8, 'utf-8', 'utf-8'), $utf8); + } + + /** + * Tests the static sub string method + * @return void + */ + public function test_substr() { + $str = "Žluťoučký koníček"; + $this->assertSame(core_text::substr($str, 0), $str); + $this->assertSame(core_text::substr($str, 1), 'luťoučký koníček'); + $this->assertSame(core_text::substr($str, 1, 3), 'luť'); + $this->assertSame(core_text::substr($str, 0, 100), $str); + $this->assertSame(core_text::substr($str, -3, 2), 'če'); + + $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::substr($iso2, 1, 3, 'iso-8859-2'), core_text::convert('luť', 'utf-8', 'iso-8859-2')); + $this->assertSame(core_text::substr($iso2, 0, 100, 'iso-8859-2'), core_text::convert($str, 'utf-8', 'iso-8859-2')); + $this->assertSame(core_text::substr($iso2, -3, 2, 'iso-8859-2'), core_text::convert('če', 'utf-8', 'iso-8859-2')); + + $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::substr($win, 1, 3, 'cp1250'), core_text::convert('luť', 'utf-8', 'cp1250')); + $this->assertSame(core_text::substr($win, 0, 100, 'cp1250'), core_text::convert($str, 'utf-8', 'cp1250')); + $this->assertSame(core_text::substr($win, -3, 2, 'cp1250'), core_text::convert('če', 'utf-8', 'cp1250')); + + + $str = pack("H*", "b8c0b8ecc0dfc4ea"); //EUC-JP + $s = pack("H*", "b8ec"); //EUC-JP + $this->assertSame(core_text::substr($str, 1, 1, 'EUC-JP'), $s); + + $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP + $s = pack("H*", "1b2442386c1b2842"); //ISO-2022-JP + $this->assertSame(core_text::substr($str, 1, 1, 'ISO-2022-JP'), $s); + + $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS + $s = pack("H*", "8cea"); //SHIFT-JIS + $this->assertSame(core_text::substr($str, 1, 1, 'SHIFT-JIS'), $s); + + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 + $s = pack("H*", "cce5"); //GB2312 + $this->assertSame(core_text::substr($str, 1, 1, 'GB2312'), $s); + + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 + $s = pack("H*", "cce5"); //GB18030 + $this->assertSame(core_text::substr($str, 1, 1, 'GB18030'), $s); + } + + /** + * Tests the static string length method + * @return void + */ + public function test_strlen() { + $str = "Žluťoučký koníček"; + $this->assertSame(core_text::strlen($str), 17); + + $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::strlen($iso2, 'iso-8859-2'), 17); + + $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::strlen($win, 'cp1250'), 17); + + + $str = pack("H*", "b8ec"); //EUC-JP + $this->assertSame(core_text::strlen($str, 'EUC-JP'), 1); + $str = pack("H*", "b8c0b8ecc0dfc4ea"); //EUC-JP + $this->assertSame(core_text::strlen($str, 'EUC-JP'), 4); + + $str = pack("H*", "1b2442386c1b2842"); //ISO-2022-JP + $this->assertSame(core_text::strlen($str, 'ISO-2022-JP'), 1); + $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP + $this->assertSame(core_text::strlen($str, 'ISO-2022-JP'), 4); + + $str = pack("H*", "8cea"); //SHIFT-JIS + $this->assertSame(core_text::strlen($str, 'SHIFT-JIS'), 1); + $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS + $this->assertSame(core_text::strlen($str, 'SHIFT-JIS'), 4); + + $str = pack("H*", "cce5"); //GB2312 + $this->assertSame(core_text::strlen($str, 'GB2312'), 1); + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 + $this->assertSame(core_text::strlen($str, 'GB2312'), 4); + + $str = pack("H*", "cce5"); //GB18030 + $this->assertSame(core_text::strlen($str, 'GB18030'), 1); + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 + $this->assertSame(core_text::strlen($str, 'GB18030'), 4); + } + + /** + * Tests the static strtolower method + * @return void + */ + public function test_strtolower() { + $str = "Žluťoučký koníček"; + $low = 'žluťoučký koníček'; + $this->assertSame(core_text::strtolower($str), $low); + + $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::strtolower($iso2, 'iso-8859-2'), core_text::convert($low, 'utf-8', 'iso-8859-2')); + + $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::strtolower($win, 'cp1250'), core_text::convert($low, 'utf-8', 'cp1250')); + + + $str = '言語設定'; + $this->assertSame(core_text::strtolower($str), $str); + + $str = '简体中文'; + $this->assertSame(core_text::strtolower($str), $str); + + $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP + $this->assertSame(core_text::strtolower($str, 'ISO-2022-JP'), $str); + + $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS + $this->assertSame(core_text::strtolower($str, 'SHIFT-JIS'), $str); + + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 + $this->assertSame(core_text::strtolower($str, 'GB2312'), $str); + + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 + $this->assertSame(core_text::strtolower($str, 'GB18030'), $str); + + // typo3 has problems with integers + $str = 1309528800; + $this->assertSame((string)$str, core_text::strtolower($str)); + } + + /** + * Tests the static strtoupper + * @return void + */ + public function test_strtoupper() { + $str = "Žluťoučký koníček"; + $up = 'ŽLUŤOUČKÝ KONÍČEK'; + $this->assertSame(core_text::strtoupper($str), $up); + + $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::strtoupper($iso2, 'iso-8859-2'), core_text::convert($up, 'utf-8', 'iso-8859-2')); + + $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); + $this->assertSame(core_text::strtoupper($win, 'cp1250'), core_text::convert($up, 'utf-8', 'cp1250')); + + + $str = '言語設定'; + $this->assertSame(core_text::strtoupper($str), $str); + + $str = '简体中文'; + $this->assertSame(core_text::strtoupper($str), $str); + + $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP + $this->assertSame(core_text::strtoupper($str, 'ISO-2022-JP'), $str); + + $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS + $this->assertSame(core_text::strtoupper($str, 'SHIFT-JIS'), $str); + + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 + $this->assertSame(core_text::strtoupper($str, 'GB2312'), $str); + + $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 + $this->assertSame(core_text::strtoupper($str, 'GB18030'), $str); + } + + /** + * Tests the static strpos method + * @return void + */ + public function test_strpos() { + $str = "Žluťoučký koníček"; + $this->assertSame(core_text::strpos($str, 'koníč'), 10); + } + + /** + * Tests the static strrpos + * @return void + */ + public function test_strrpos() { + $str = "Žluťoučký koníček"; + $this->assertSame(core_text::strrpos($str, 'o'), 11); + } + + /** + * Tests the static specialtoascii method + * @return void + */ + public function test_specialtoascii() { + $str = "Žluťoučký koníček"; + $this->assertSame(core_text::specialtoascii($str), 'Zlutoucky konicek'); + } + + /** + * Tests the static encode_mimeheader method + * @return void + */ + public function test_encode_mimeheader() { + $str = "Žluťoučký koníček"; + $this->assertSame(core_text::encode_mimeheader($str), '=?utf-8?B?xb1sdcWlb3XEjWvDvSBrb27DrcSNZWs=?='); + } + + /** + * Tests the static entities_to_utf8 method + * @return void + */ + public function test_entities_to_utf8() { + $str = "Žluťoučký koníček©"&<>§«"; + $this->assertSame("Žluťoučký koníček©\"&<>§«", core_text::entities_to_utf8($str)); + } + + /** + * Tests the static utf8_to_entities method + * @return void + */ + public function test_utf8_to_entities() { + $str = "Žluťoučký koníček©"&<>§«"; + $this->assertSame("Žluťoučký koníček©"&<>§«", core_text::utf8_to_entities($str)); + $this->assertSame("Žluťoučký koníček©"&<>§«", core_text::utf8_to_entities($str, true)); + + $str = "Žluťoučký koníček©"&<>§«"; + $this->assertSame("Žluťoučký koníček©\"&<>§«", core_text::utf8_to_entities($str, false, true)); + $this->assertSame("Žluťoučký koníček©\"&<>§«", core_text::utf8_to_entities($str, true, true)); + } + + /** + * Tests the static trim_utf8_bom method + * @return void + */ + public function test_trim_utf8_bom() { + $bom = "\xef\xbb\xbf"; + $str = "Žluťoučký koníček"; + $this->assertSame(core_text::trim_utf8_bom($bom.$str.$bom), $str.$bom); + } + + /** + * Tests the static get_encodings method + * @return void + */ + public function test_get_encodings() { + $encodings = core_text::get_encodings(); + $this->assertTrue(is_array($encodings)); + $this->assertTrue(count($encodings) > 1); + $this->assertTrue(isset($encodings['UTF-8'])); + } + + /** + * Tests the static code2utf8 method + * @return void + */ + public function test_code2utf8() { + $this->assertSame(core_text::code2utf8(381), 'Ž'); + } + + /** + * Tests the static utf8ord method + * @return void + */ + public function test_utf8ord() { + $this->assertSame(core_text::utf8ord(''), ord('')); + $this->assertSame(core_text::utf8ord('f'), ord('f')); + $this->assertSame(core_text::utf8ord('α'), 0x03B1); + $this->assertSame(core_text::utf8ord('й'), 0x0439); + $this->assertSame(core_text::utf8ord('𯨟'), 0x2FA1F); + $this->assertSame(core_text::utf8ord('Ž'), 381); + } + + /** + * Tests the static strtotitle method + * @return void + */ + public function test_strtotitle() { + $str = "žluťoučký koníček"; + $this->assertSame(core_text::strtotitle($str), "Žluťoučký Koníček"); + } + + public function test_deprecated_textlib() { + $this->assertSame(core_text::strtolower('HUH'), textlib::strtolower('HUH')); + } + + /** + * Tests the deprecated method of textlib that still require an instance. + * @return void + */ + public function test_deprecated_textlib_get_instance() { + $textlib = textlib_get_instance(); + $this->assertDebuggingCalled(); + $this->assertSame($textlib->substr('abc', 1, 1), 'b'); + $this->assertSame($textlib->strlen('abc'), 3); + $this->assertSame($textlib->strtoupper('Abc'), 'ABC'); + $this->assertSame($textlib->strtolower('Abc'), 'abc'); + $this->assertSame($textlib->strpos('abc', 'a'), 0); + $this->assertSame($textlib->strpos('abc', 'd'), false); + $this->assertSame($textlib->strrpos('abcabc', 'a'), 3); + $this->assertSame($textlib->specialtoascii('ábc'), 'abc'); + $this->assertSame($textlib->strtotitle('abc ABC'), 'Abc Abc'); + } +} + diff --git a/lib/tests/textlib_test.php b/lib/tests/textlib_test.php deleted file mode 100644 index 64f04827188..00000000000 --- a/lib/tests/textlib_test.php +++ /dev/null @@ -1,655 +0,0 @@ -. - -/** - * textlib unit tests - * - * @package core - * @category phpunit - * @copyright 2012 Petr Skoda {@link http://skodak.org} - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ - -defined('MOODLE_INTERNAL') || die(); - - -/** - * Unit tests for our utf-8 aware text processing - * - * @package core - * @category phpunit - * @copyright 2010 Petr Skoda (http://skodak.org) - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ -class core_textlib_testcase extends advanced_testcase { - - /** - * Tests the static parse charset method - * @return void - */ - public function test_parse_charset() { - $this->assertSame(textlib::parse_charset('Cp1250'), 'windows-1250'); - // does typo3 work? some encoding moodle does not use - $this->assertSame(textlib::parse_charset('ms-ansi'), 'windows-1252'); - } - - /** - * Tests the static convert method - * @return void - */ - public function test_convert() { - $utf8 = "Žluťoučký koníček"; - $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); - $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::convert($utf8, 'utf-8', 'iso-8859-2'), $iso2); - $this->assertSame(textlib::convert($iso2, 'iso-8859-2', 'utf-8'), $utf8); - $this->assertSame(textlib::convert($utf8, 'utf-8', 'win-1250'), $win); - $this->assertSame(textlib::convert($win, 'win-1250', 'utf-8'), $utf8); - $this->assertSame(textlib::convert($win, 'win-1250', 'iso-8859-2'), $iso2); - $this->assertSame(textlib::convert($iso2, 'iso-8859-2', 'win-1250'), $win); - $this->assertSame(textlib::convert($iso2, 'iso-8859-2', 'iso-8859-2'), $iso2); - $this->assertSame(textlib::convert($win, 'win-1250', 'cp1250'), $win); - $this->assertSame(textlib::convert($utf8, 'utf-8', 'utf-8'), $utf8); - - - $utf8 = '言語設定'; - $str = pack("H*", "b8c0b8ecc0dfc4ea"); //EUC-JP - $this->assertSame(textlib::convert($utf8, 'utf-8', 'EUC-JP'), $str); - $this->assertSame(textlib::convert($str, 'EUC-JP', 'utf-8'), $utf8); - $this->assertSame(textlib::convert($utf8, 'utf-8', 'utf-8'), $utf8); - - $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP - $this->assertSame(textlib::convert($utf8, 'utf-8', 'ISO-2022-JP'), $str); - $this->assertSame(textlib::convert($str, 'ISO-2022-JP', 'utf-8'), $utf8); - $this->assertSame(textlib::convert($utf8, 'utf-8', 'utf-8'), $utf8); - - $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS - $this->assertSame(textlib::convert($utf8, 'utf-8', 'SHIFT-JIS'), $str); - $this->assertSame(textlib::convert($str, 'SHIFT-JIS', 'utf-8'), $utf8); - $this->assertSame(textlib::convert($utf8, 'utf-8', 'utf-8'), $utf8); - - $utf8 = '简体中文'; - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 - $this->assertSame(textlib::convert($utf8, 'utf-8', 'GB2312'), $str); - $this->assertSame(textlib::convert($str, 'GB2312', 'utf-8'), $utf8); - $this->assertSame(textlib::convert($utf8, 'utf-8', 'utf-8'), $utf8); - - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 - $this->assertSame(textlib::convert($utf8, 'utf-8', 'GB18030'), $str); - $this->assertSame(textlib::convert($str, 'GB18030', 'utf-8'), $utf8); - $this->assertSame(textlib::convert($utf8, 'utf-8', 'utf-8'), $utf8); - } - - /** - * Tests the static sub string method - * @return void - */ - public function test_substr() { - $str = "Žluťoučký koníček"; - $this->assertSame(textlib::substr($str, 0), $str); - $this->assertSame(textlib::substr($str, 1), 'luťoučký koníček'); - $this->assertSame(textlib::substr($str, 1, 3), 'luť'); - $this->assertSame(textlib::substr($str, 0, 100), $str); - $this->assertSame(textlib::substr($str, -3, 2), 'če'); - - $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::substr($iso2, 1, 3, 'iso-8859-2'), textlib::convert('luť', 'utf-8', 'iso-8859-2')); - $this->assertSame(textlib::substr($iso2, 0, 100, 'iso-8859-2'), textlib::convert($str, 'utf-8', 'iso-8859-2')); - $this->assertSame(textlib::substr($iso2, -3, 2, 'iso-8859-2'), textlib::convert('če', 'utf-8', 'iso-8859-2')); - - $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::substr($win, 1, 3, 'cp1250'), textlib::convert('luť', 'utf-8', 'cp1250')); - $this->assertSame(textlib::substr($win, 0, 100, 'cp1250'), textlib::convert($str, 'utf-8', 'cp1250')); - $this->assertSame(textlib::substr($win, -3, 2, 'cp1250'), textlib::convert('če', 'utf-8', 'cp1250')); - - - $str = pack("H*", "b8c0b8ecc0dfc4ea"); //EUC-JP - $s = pack("H*", "b8ec"); //EUC-JP - $this->assertSame(textlib::substr($str, 1, 1, 'EUC-JP'), $s); - - $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP - $s = pack("H*", "1b2442386c1b2842"); //ISO-2022-JP - $this->assertSame(textlib::substr($str, 1, 1, 'ISO-2022-JP'), $s); - - $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS - $s = pack("H*", "8cea"); //SHIFT-JIS - $this->assertSame(textlib::substr($str, 1, 1, 'SHIFT-JIS'), $s); - - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 - $s = pack("H*", "cce5"); //GB2312 - $this->assertSame(textlib::substr($str, 1, 1, 'GB2312'), $s); - - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 - $s = pack("H*", "cce5"); //GB18030 - $this->assertSame(textlib::substr($str, 1, 1, 'GB18030'), $s); - } - - /** - * Tests the static string length method - * @return void - */ - public function test_strlen() { - $str = "Žluťoučký koníček"; - $this->assertSame(textlib::strlen($str), 17); - - $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::strlen($iso2, 'iso-8859-2'), 17); - - $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::strlen($win, 'cp1250'), 17); - - - $str = pack("H*", "b8ec"); //EUC-JP - $this->assertSame(textlib::strlen($str, 'EUC-JP'), 1); - $str = pack("H*", "b8c0b8ecc0dfc4ea"); //EUC-JP - $this->assertSame(textlib::strlen($str, 'EUC-JP'), 4); - - $str = pack("H*", "1b2442386c1b2842"); //ISO-2022-JP - $this->assertSame(textlib::strlen($str, 'ISO-2022-JP'), 1); - $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP - $this->assertSame(textlib::strlen($str, 'ISO-2022-JP'), 4); - - $str = pack("H*", "8cea"); //SHIFT-JIS - $this->assertSame(textlib::strlen($str, 'SHIFT-JIS'), 1); - $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS - $this->assertSame(textlib::strlen($str, 'SHIFT-JIS'), 4); - - $str = pack("H*", "cce5"); //GB2312 - $this->assertSame(textlib::strlen($str, 'GB2312'), 1); - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 - $this->assertSame(textlib::strlen($str, 'GB2312'), 4); - - $str = pack("H*", "cce5"); //GB18030 - $this->assertSame(textlib::strlen($str, 'GB18030'), 1); - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 - $this->assertSame(textlib::strlen($str, 'GB18030'), 4); - } - - /** - * Tests the static strtolower method - * @return void - */ - public function test_strtolower() { - $str = "Žluťoučký koníček"; - $low = 'žluťoučký koníček'; - $this->assertSame(textlib::strtolower($str), $low); - - $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::strtolower($iso2, 'iso-8859-2'), textlib::convert($low, 'utf-8', 'iso-8859-2')); - - $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::strtolower($win, 'cp1250'), textlib::convert($low, 'utf-8', 'cp1250')); - - - $str = '言語設定'; - $this->assertSame(textlib::strtolower($str), $str); - - $str = '简体中文'; - $this->assertSame(textlib::strtolower($str), $str); - - $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP - $this->assertSame(textlib::strtolower($str, 'ISO-2022-JP'), $str); - - $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS - $this->assertSame(textlib::strtolower($str, 'SHIFT-JIS'), $str); - - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 - $this->assertSame(textlib::strtolower($str, 'GB2312'), $str); - - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 - $this->assertSame(textlib::strtolower($str, 'GB18030'), $str); - - // typo3 has problems with integers - $str = 1309528800; - $this->assertSame((string)$str, textlib::strtolower($str)); - } - - /** - * Tests the static strtoupper - * @return void - */ - public function test_strtoupper() { - $str = "Žluťoučký koníček"; - $up = 'ŽLUŤOUČKÝ KONÍČEK'; - $this->assertSame(textlib::strtoupper($str), $up); - - $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::strtoupper($iso2, 'iso-8859-2'), textlib::convert($up, 'utf-8', 'iso-8859-2')); - - $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b"); - $this->assertSame(textlib::strtoupper($win, 'cp1250'), textlib::convert($up, 'utf-8', 'cp1250')); - - - $str = '言語設定'; - $this->assertSame(textlib::strtoupper($str), $str); - - $str = '简体中文'; - $this->assertSame(textlib::strtoupper($str), $str); - - $str = pack("H*", "1b24423840386c405f446a1b2842"); //ISO-2022-JP - $this->assertSame(textlib::strtoupper($str, 'ISO-2022-JP'), $str); - - $str = pack("H*", "8cbe8cea90dd92e8"); //SHIFT-JIS - $this->assertSame(textlib::strtoupper($str, 'SHIFT-JIS'), $str); - - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB2312 - $this->assertSame(textlib::strtoupper($str, 'GB2312'), $str); - - $str = pack("H*", "bcf2cce5d6d0cec4"); //GB18030 - $this->assertSame(textlib::strtoupper($str, 'GB18030'), $str); - } - - /** - * Tests the static strpos method - * @return void - */ - public function test_strpos() { - $str = "Žluťoučký koníček"; - $this->assertSame(textlib::strpos($str, 'koníč'), 10); - } - - /** - * Tests the static strrpos - * @return void - */ - public function test_strrpos() { - $str = "Žluťoučký koníček"; - $this->assertSame(textlib::strrpos($str, 'o'), 11); - } - - /** - * Tests the static specialtoascii method - * @return void - */ - public function test_specialtoascii() { - $str = "Žluťoučký koníček"; - $this->assertSame(textlib::specialtoascii($str), 'Zlutoucky konicek'); - } - - /** - * Tests the static encode_mimeheader method - * @return void - */ - public function test_encode_mimeheader() { - $str = "Žluťoučký koníček"; - $this->assertSame(textlib::encode_mimeheader($str), '=?utf-8?B?xb1sdcWlb3XEjWvDvSBrb27DrcSNZWs=?='); - } - - /** - * Tests the static entities_to_utf8 method - * @return void - */ - public function test_entities_to_utf8() { - $str = "Žluťoučký koníček©"&<>§«"; - $this->assertSame("Žluťoučký koníček©\"&<>§«", textlib::entities_to_utf8($str)); - } - - /** - * Tests the static utf8_to_entities method - * @return void - */ - public function test_utf8_to_entities() { - $str = "Žluťoučký koníček©"&<>§«"; - $this->assertSame("Žluťoučký koníček©"&<>§«", textlib::utf8_to_entities($str)); - $this->assertSame("Žluťoučký koníček©"&<>§«", textlib::utf8_to_entities($str, true)); - - $str = "Žluťoučký koníček©"&<>§«"; - $this->assertSame("Žluťoučký koníček©\"&<>§«", textlib::utf8_to_entities($str, false, true)); - $this->assertSame("Žluťoučký koníček©\"&<>§«", textlib::utf8_to_entities($str, true, true)); - } - - /** - * Tests the static trim_utf8_bom method - * @return void - */ - public function test_trim_utf8_bom() { - $bom = "\xef\xbb\xbf"; - $str = "Žluťoučký koníček"; - $this->assertSame(textlib::trim_utf8_bom($bom.$str.$bom), $str.$bom); - } - - /** - * Tests the static get_encodings method - * @return void - */ - public function test_get_encodings() { - $encodings = textlib::get_encodings(); - $this->assertTrue(is_array($encodings)); - $this->assertTrue(count($encodings) > 1); - $this->assertTrue(isset($encodings['UTF-8'])); - } - - /** - * Tests the static code2utf8 method - * @return void - */ - public function test_code2utf8() { - $this->assertSame(textlib::code2utf8(381), 'Ž'); - } - - /** - * Tests the static utf8ord method - * @return void - */ - public function test_utf8ord() { - $this->assertSame(textlib::utf8ord(''), ord('')); - $this->assertSame(textlib::utf8ord('f'), ord('f')); - $this->assertSame(textlib::utf8ord('α'), 0x03B1); - $this->assertSame(textlib::utf8ord('й'), 0x0439); - $this->assertSame(textlib::utf8ord('𯨟'), 0x2FA1F); - $this->assertSame(textlib::utf8ord('Ž'), 381); - } - - /** - * Tests the static strtotitle method - * @return void - */ - public function test_strtotitle() { - $str = "žluťoučký koníček"; - $this->assertSame(textlib::strtotitle($str), "Žluťoučký Koníček"); - } - - /** - * Tests the deprecated method of textlib that still require an instance. - * @return void - */ - public function test_deprecated_textlib_get_instance() { - $textlib = textlib_get_instance(); - $this->assertDebuggingCalled(); - $this->assertSame($textlib->substr('abc', 1, 1), 'b'); - $this->assertSame($textlib->strlen('abc'), 3); - $this->assertSame($textlib->strtoupper('Abc'), 'ABC'); - $this->assertSame($textlib->strtolower('Abc'), 'abc'); - $this->assertSame($textlib->strpos('abc', 'a'), 0); - $this->assertSame($textlib->strpos('abc', 'd'), false); - $this->assertSame($textlib->strrpos('abcabc', 'a'), 3); - $this->assertSame($textlib->specialtoascii('ábc'), 'abc'); - $this->assertSame($textlib->strtotitle('abc ABC'), 'Abc Abc'); - } -} - - -/** - * Unit tests for our utf-8 aware collator. - * - * Used for sorting. - * - * @package core - * @category phpunit - * @copyright 2011 Sam Hemelryk - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ -class collatorlib_testcase extends basic_testcase { - - /** - * @var string The initial lang, stored because we change it during testing - */ - protected $initiallang = null; - - /** - * @var string The last error that has occured - */ - protected $error = null; - - /** - * Prepares things for this test case - * @return void - */ - protected function setUp() { - global $SESSION; - if (isset($SESSION->lang)) { - $this->initiallang = $SESSION->lang; - } - $SESSION->lang = 'en'; // make sure we test en language to get consistent results, hopefully all systems have this locale - if (extension_loaded('intl')) { - $this->error = 'Collation aware sorting not supported'; - } else { - $this->error = 'Collation aware sorting not supported, PHP extension "intl" is not available.'; - } - parent::setUp(); - } - - /** - * Cleans things up after this test case has run - * @return void - */ - protected function tearDown() { - global $SESSION; - parent::tearDown(); - if ($this->initiallang !== null) { - $SESSION->lang = $this->initiallang; - $this->initiallang = null; - } else { - unset($SESSION->lang); - } - } - - /** - * Tests the static asort method - * @return void - */ - public function test_asort() { - $arr = array('b' => 'ab', 1 => 'aa', 0 => 'cc'); - $result = collatorlib::asort($arr); - $this->assertSame(array_values($arr), array('aa', 'ab', 'cc')); - $this->assertSame(array_keys($arr), array(1, 'b', 0)); - $this->assertTrue($result); - - $arr = array('b' => 'ab', 1 => 'aa', 0 => 'cc'); - $result = collatorlib::asort($arr, collatorlib::SORT_STRING); - $this->assertSame(array_values($arr), array('aa', 'ab', 'cc')); - $this->assertSame(array_keys($arr), array(1, 'b', 0)); - $this->assertTrue($result); - - $arr = array('b' => 'aac', 1 => 'Aac', 0 => 'cc'); - $result = collatorlib::asort($arr, (collatorlib::SORT_STRING | collatorlib::CASE_SENSITIVE)); - $this->assertSame(array_values($arr), array('Aac', 'aac', 'cc')); - $this->assertSame(array_keys($arr), array(1, 'b', 0)); - $this->assertTrue($result); - - $arr = array('b' => 'a1', 1 => 'a10', 0 => 'a3b'); - $result = collatorlib::asort($arr); - $this->assertSame(array_values($arr), array('a1', 'a10', 'a3b')); - $this->assertSame(array_keys($arr), array('b', 1, 0)); - $this->assertTrue($result); - - $arr = array('b' => 'a1', 1 => 'a10', 0 => 'a3b'); - $result = collatorlib::asort($arr, collatorlib::SORT_NATURAL); - $this->assertSame(array_values($arr), array('a1', 'a3b', 'a10')); - $this->assertSame(array_keys($arr), array('b', 0, 1)); - $this->assertTrue($result); - - $arr = array('b' => '1.1.1', 1 => '1.2', 0 => '1.20.2'); - $result = collatorlib::asort($arr, collatorlib::SORT_NATURAL); - $this->assertSame(array_values($arr), array('1.1.1', '1.2', '1.20.2')); - $this->assertSame(array_keys($arr), array('b', 1, 0)); - $this->assertTrue($result); - - $arr = array('b' => '-1', 1 => 1000, 0 => -1.2, 3 => 1, 4 => false); - $result = collatorlib::asort($arr, collatorlib::SORT_NUMERIC); - $this->assertSame(array_values($arr), array(-1.2, '-1', false, 1, 1000)); - $this->assertSame(array_keys($arr), array(0, 'b', 4, 3, 1)); - $this->assertTrue($result); - - $arr = array('b' => array(1), 1 => array(2, 3), 0 => 1); - $result = collatorlib::asort($arr, collatorlib::SORT_REGULAR); - $this->assertSame(array_values($arr), array(1, array(1), array(2, 3))); - $this->assertSame(array_keys($arr), array(0, 'b', 1)); - $this->assertTrue($result); - - // test sorting of array of arrays - first element should be used for actual comparison - $arr = array(0=>array('bb', 'z'), 1=>array('ab', 'a'), 2=>array('zz', 'x')); - $result = collatorlib::asort($arr, collatorlib::SORT_REGULAR); - $this->assertSame(array_keys($arr), array(1, 0, 2)); - $this->assertTrue($result); - - $arr = array('a' => 'áb', 'b' => 'ab', 1 => 'aa', 0=>'cc', 'x' => 'Áb',); - $result = collatorlib::asort($arr); - $this->assertSame(array_values($arr), array('aa', 'ab', 'áb', 'Áb', 'cc'), $this->error); - $this->assertSame(array_keys($arr), array(1, 'b', 'a', 'x', 0), $this->error); - $this->assertTrue($result); - - $a = array(2=>'b', 1=>'c'); - $c =& $a; - $b =& $a; - collatorlib::asort($b); - $this->assertSame($a, $b); - $this->assertSame($c, $b); - } - - /** - * Tests the static asort_objects_by_method method - * @return void - */ - public function test_asort_objects_by_method() { - $objects = array( - 'b' => new string_test_class('ab'), - 1 => new string_test_class('aa'), - 0 => new string_test_class('cc') - ); - $result = collatorlib::asort_objects_by_method($objects, 'get_protected_name'); - $this->assertSame(array_keys($objects), array(1, 'b', 0)); - $this->assertSame($this->get_ordered_names($objects, 'get_protected_name'), array('aa', 'ab', 'cc')); - $this->assertTrue($result); - - $objects = array( - 'b' => new string_test_class('a20'), - 1 => new string_test_class('a1'), - 0 => new string_test_class('a100') - ); - $result = collatorlib::asort_objects_by_method($objects, 'get_protected_name', collatorlib::SORT_NATURAL); - $this->assertSame(array_keys($objects), array(1, 'b', 0)); - $this->assertSame($this->get_ordered_names($objects, 'get_protected_name'), array('a1', 'a20', 'a100')); - $this->assertTrue($result); - } - - /** - * Tests the static asort_objects_by_method method - * @return void - */ - public function test_asort_objects_by_property() { - $objects = array( - 'b' => new string_test_class('ab'), - 1 => new string_test_class('aa'), - 0 => new string_test_class('cc') - ); - $result = collatorlib::asort_objects_by_property($objects, 'publicname'); - $this->assertSame(array_keys($objects), array(1, 'b', 0)); - $this->assertSame($this->get_ordered_names($objects, 'publicname'), array('aa', 'ab', 'cc')); - $this->assertTrue($result); - - $objects = array( - 'b' => new string_test_class('a20'), - 1 => new string_test_class('a1'), - 0 => new string_test_class('a100') - ); - $result = collatorlib::asort_objects_by_property($objects, 'publicname', collatorlib::SORT_NATURAL); - $this->assertSame(array_keys($objects), array(1, 'b', 0)); - $this->assertSame($this->get_ordered_names($objects, 'publicname'), array('a1', 'a20', 'a100')); - $this->assertTrue($result); - } - - /** - * Returns an array of sorted names - * @param array $objects - * @param string $methodproperty - * @return type - */ - protected function get_ordered_names($objects, $methodproperty = 'get_protected_name') { - $return = array(); - foreach ($objects as $object) { - if ($methodproperty == 'publicname') { - $return[] = $object->publicname; - } else { - $return[] = $object->$methodproperty(); - } - } - return $return; - } - - /** - * Tests the static ksort method - * @return void - */ - public function test_ksort() { - $arr = array('b' => 'ab', 1 => 'aa', 0 => 'cc'); - $result = collatorlib::ksort($arr); - $this->assertSame(array_keys($arr), array(0, 1, 'b')); - $this->assertSame(array_values($arr), array('cc', 'aa', 'ab')); - $this->assertTrue($result); - - $obj = new stdClass(); - $arr = array('1.1.1'=>array(), '1.2'=>$obj, '1.20.2'=>null); - $result = collatorlib::ksort($arr, collatorlib::SORT_NATURAL); - $this->assertSame(array_keys($arr), array('1.1.1', '1.2', '1.20.2')); - $this->assertSame(array_values($arr), array(array(), $obj, null)); - $this->assertTrue($result); - - $a = array(2=>'b', 1=>'c'); - $c =& $a; - $b =& $a; - collatorlib::ksort($b); - $this->assertSame($a, $b); - $this->assertSame($c, $b); - } -} - - -/** - * Simple class used to work with the unit test. - * - * @package core - * @category phpunit - * @copyright 2011 Sam Hemelryk - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ -class string_test_class extends stdClass { - /** - * @var string A public property - */ - public $publicname; - /** - * @var string A protected property - */ - protected $protectedname; - /** - * @var string A private property - */ - private $privatename; - /** - * Constructs the test instance - * @param string $name - */ - public function __construct($name) { - $this->publicname = $name; - $this->protectedname = $name; - $this->privatename = $name; - } - /** - * Returns the protected property - * @return string - */ - public function get_protected_name() { - return $this->protectedname; - } - /** - * Returns the protected property - * @return string - */ - public function get_private_name() { - return $this->publicname; - } -} \ No newline at end of file diff --git a/lib/textlib.class.php b/lib/textlib.class.php index c91c7e5c1f6..0f52d31c34a 100644 --- a/lib/textlib.class.php +++ b/lib/textlib.class.php @@ -15,7 +15,8 @@ // along with Moodle. If not, see . /** - * Defines string apis + * Defines string apis. + * @deprecated * * @package core * @copyright (C) 2001-3001 Eloy Lafuente (stronk7) {@link http://contiento.com} @@ -24,923 +25,4 @@ defined('MOODLE_INTERNAL') || die(); -/** - * defines string api's for manipulating strings - * - * This class is used to manipulate strings under Moodle 1.6 an later. As - * utf-8 text become mandatory a pool of safe functions under this encoding - * become necessary. The name of the methods is exactly the - * same than their PHP originals. - * - * A big part of this class acts as a wrapper over the Typo3 charset library, - * really a cool group of utilities to handle texts and encoding conversion. - * - * Take a look to its own copyright and license details. - * - * IMPORTANT Note: Typo3 libraries always expect lowercase charsets to use 100% - * its capabilities so, don't forget to make the conversion - * from every wrapper function! - * - * @package core - * @category string - * @copyright 1999 onwards Martin Dougiamas {@link http://moodle.com} - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ -class textlib { - - /** - * Return t3lib helper class, which is used for conversion between charsets - * - * @param bool $reset - * @return t3lib_cs - */ - protected static function typo3($reset = false) { - static $typo3cs = null; - - if ($reset) { - $typo3cs = null; - return null; - } - - if (isset($typo3cs)) { - return $typo3cs; - } - - global $CFG; - - // Required files - require_once($CFG->libdir.'/typo3/class.t3lib_cs.php'); - require_once($CFG->libdir.'/typo3/class.t3lib_div.php'); - require_once($CFG->libdir.'/typo3/interface.t3lib_singleton.php'); - require_once($CFG->libdir.'/typo3/class.t3lib_l10n_locales.php'); - - // do not use mbstring or recode because it may return invalid results in some corner cases - $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'iconv'; - $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] = 'iconv'; - - // Tell Typo3 we are curl enabled always (mandatory since 2.0) - $GLOBALS['TYPO3_CONF_VARS']['SYS']['curlUse'] = '1'; - - // And this directory must exist to allow Typo to cache conversion - // tables when using internal functions - make_temp_directory('typo3temp/cs'); - - // Make sure typo is using our dir permissions - $GLOBALS['TYPO3_CONF_VARS']['BE']['folderCreateMask'] = decoct($CFG->directorypermissions); - - // Default mask for Typo - $GLOBALS['TYPO3_CONF_VARS']['BE']['fileCreateMask'] = $CFG->directorypermissions; - - // This full path constants must be defined too, transforming backslashes - // to forward slashed because Typo3 requires it. - if (!defined('PATH_t3lib')) { - define('PATH_t3lib', str_replace('\\','/',$CFG->libdir.'/typo3/')); - define('PATH_typo3', str_replace('\\','/',$CFG->libdir.'/typo3/')); - define('PATH_site', str_replace('\\','/',$CFG->tempdir.'/')); - define('TYPO3_OS', stristr(PHP_OS,'win')&&!stristr(PHP_OS,'darwin')?'WIN':''); - } - - $typo3cs = new t3lib_cs(); - - return $typo3cs; - } - - /** - * Reset internal textlib caches. - * @static - */ - public static function reset_caches() { - self::typo3(true); - } - - /** - * Standardise charset name - * - * Please note it does not mean the returned charset is actually supported. - * - * @static - * @param string $charset raw charset name - * @return string normalised lowercase charset name - */ - public static function parse_charset($charset) { - $charset = strtolower($charset); - - // shortcuts so that we do not have to load typo3 on every page - - if ($charset === 'utf8' or $charset === 'utf-8') { - return 'utf-8'; - } - - if (preg_match('/^(cp|win|windows)-?(12[0-9]{2})$/', $charset, $matches)) { - return 'windows-'.$matches[2]; - } - - if (preg_match('/^iso-8859-[0-9]+$/', $charset, $matches)) { - return $charset; - } - - if ($charset === 'euc-jp') { - return 'euc-jp'; - } - if ($charset === 'iso-2022-jp') { - return 'iso-2022-jp'; - } - if ($charset === 'shift-jis' or $charset === 'shift_jis') { - return 'shift_jis'; - } - if ($charset === 'gb2312') { - return 'gb2312'; - } - if ($charset === 'gb18030') { - return 'gb18030'; - } - - // fallback to typo3 - return self::typo3()->parse_charset($charset); - } - - /** - * Converts the text between different encodings. It uses iconv extension with //TRANSLIT parameter, - * falls back to typo3. If both source and target are utf-8 it tries to fix invalid characters only. - * - * @param string $text - * @param string $fromCS source encoding - * @param string $toCS result encoding - * @return string|bool converted string or false on error - */ - public static function convert($text, $fromCS, $toCS='utf-8') { - $fromCS = self::parse_charset($fromCS); - $toCS = self::parse_charset($toCS); - - $text = (string)$text; // we can work only with strings - - if ($text === '') { - return ''; - } - - if ($toCS === 'utf-8' and $fromCS === 'utf-8') { - return fix_utf8($text); - } - - $result = iconv($fromCS, $toCS.'//TRANSLIT', $text); - - if ($result === false or $result === '') { - // note: iconv is prone to return empty string when invalid char encountered, or false if encoding unsupported - $oldlevel = error_reporting(E_PARSE); - $result = self::typo3()->conv((string)$text, $fromCS, $toCS); - error_reporting($oldlevel); - } - - return $result; - } - - /** - * Multibyte safe substr() function, uses mbstring or iconv for UTF-8, falls back to typo3. - * - * @param string $text string to truncate - * @param int $start negative value means from end - * @param int $len maximum length of characters beginning from start - * @param string $charset encoding of the text - * @return string portion of string specified by the $start and $len - */ - public static function substr($text, $start, $len=null, $charset='utf-8') { - $charset = self::parse_charset($charset); - - if ($charset === 'utf-8') { - if (function_exists('mb_substr')) { - // this is much faster than iconv - see MDL-31142 - if ($len === null) { - $oldcharset = mb_internal_encoding(); - mb_internal_encoding('UTF-8'); - $result = mb_substr($text, $start); - mb_internal_encoding($oldcharset); - return $result; - } else { - return mb_substr($text, $start, $len, 'UTF-8'); - } - - } else { - if ($len === null) { - $len = iconv_strlen($text, 'UTF-8'); - } - return iconv_substr($text, $start, $len, 'UTF-8'); - } - } - - $oldlevel = error_reporting(E_PARSE); - if ($len === null) { - $result = self::typo3()->substr($charset, (string)$text, $start); - } else { - $result = self::typo3()->substr($charset, (string)$text, $start, $len); - } - error_reporting($oldlevel); - - return $result; - } - - /** - * Multibyte safe strlen() function, uses mbstring or iconv for UTF-8, falls back to typo3. - * - * @param string $text input string - * @param string $charset encoding of the text - * @return int number of characters - */ - public static function strlen($text, $charset='utf-8') { - $charset = self::parse_charset($charset); - - if ($charset === 'utf-8') { - if (function_exists('mb_strlen')) { - return mb_strlen($text, 'UTF-8'); - } else { - return iconv_strlen($text, 'UTF-8'); - } - } - - $oldlevel = error_reporting(E_PARSE); - $result = self::typo3()->strlen($charset, (string)$text); - error_reporting($oldlevel); - - return $result; - } - - /** - * Multibyte safe strtolower() function, uses mbstring, falls back to typo3. - * - * @param string $text input string - * @param string $charset encoding of the text (may not work for all encodings) - * @return string lower case text - */ - public static function strtolower($text, $charset='utf-8') { - $charset = self::parse_charset($charset); - - if ($charset === 'utf-8' and function_exists('mb_strtolower')) { - return mb_strtolower($text, 'UTF-8'); - } - - $oldlevel = error_reporting(E_PARSE); - $result = self::typo3()->conv_case($charset, (string)$text, 'toLower'); - error_reporting($oldlevel); - - return $result; - } - - /** - * Multibyte safe strtoupper() function, uses mbstring, falls back to typo3. - * - * @param string $text input string - * @param string $charset encoding of the text (may not work for all encodings) - * @return string upper case text - */ - public static function strtoupper($text, $charset='utf-8') { - $charset = self::parse_charset($charset); - - if ($charset === 'utf-8' and function_exists('mb_strtoupper')) { - return mb_strtoupper($text, 'UTF-8'); - } - - $oldlevel = error_reporting(E_PARSE); - $result = self::typo3()->conv_case($charset, (string)$text, 'toUpper'); - error_reporting($oldlevel); - - return $result; - } - - /** - * Find the position of the first occurrence of a substring in a string. - * UTF-8 ONLY safe strpos(), uses mbstring, falls back to iconv. - * - * @param string $haystack the string to search in - * @param string $needle one or more charachters to search for - * @param int $offset offset from begining of string - * @return int the numeric position of the first occurrence of needle in haystack. - */ - public static function strpos($haystack, $needle, $offset=0) { - if (function_exists('mb_strpos')) { - return mb_strpos($haystack, $needle, $offset, 'UTF-8'); - } else { - return iconv_strpos($haystack, $needle, $offset, 'UTF-8'); - } - } - - /** - * Find the position of the last occurrence of a substring in a string - * UTF-8 ONLY safe strrpos(), uses mbstring, falls back to iconv. - * - * @param string $haystack the string to search in - * @param string $needle one or more charachters to search for - * @return int the numeric position of the last occurrence of needle in haystack - */ - public static function strrpos($haystack, $needle) { - if (function_exists('mb_strpos')) { - return mb_strrpos($haystack, $needle, null, 'UTF-8'); - } else { - return iconv_strrpos($haystack, $needle, 'UTF-8'); - } - } - - /** - * Try to convert upper unicode characters to plain ascii, - * the returned string may contain unconverted unicode characters. - * - * @param string $text input string - * @param string $charset encoding of the text - * @return string converted ascii string - */ - public static function specialtoascii($text, $charset='utf-8') { - $charset = self::parse_charset($charset); - $oldlevel = error_reporting(E_PARSE); - $result = self::typo3()->specCharsToASCII($charset, (string)$text); - error_reporting($oldlevel); - return $result; - } - - /** - * Generate a correct base64 encoded header to be used in MIME mail messages. - * This function seems to be 100% compliant with RFC1342. Credits go to: - * paravoid (http://www.php.net/manual/en/function.mb-encode-mimeheader.php#60283). - * - * @param string $text input string - * @param string $charset encoding of the text - * @return string base64 encoded header - */ - public static function encode_mimeheader($text, $charset='utf-8') { - if (empty($text)) { - return (string)$text; - } - // Normalize charset - $charset = self::parse_charset($charset); - // If the text is pure ASCII, we don't need to encode it - if (self::convert($text, $charset, 'ascii') == $text) { - return $text; - } - // Although RFC says that line feed should be \r\n, it seems that - // some mailers double convert \r, so we are going to use \n alone - $linefeed="\n"; - // Define start and end of every chunk - $start = "=?$charset?B?"; - $end = "?="; - // Accumulate results - $encoded = ''; - // Max line length is 75 (including start and end) - $length = 75 - strlen($start) - strlen($end); - // Multi-byte ratio - $multilength = self::strlen($text, $charset); - // Detect if strlen and friends supported - if ($multilength === false) { - if ($charset == 'GB18030' or $charset == 'gb18030') { - while (strlen($text)) { - // try to encode first 22 chars - we expect most chars are two bytes long - if (preg_match('/^(([\x00-\x7f])|([\x81-\xfe][\x40-\x7e])|([\x81-\xfe][\x80-\xfe])|([\x81-\xfe][\x30-\x39]..)){1,22}/m', $text, $matches)) { - $chunk = $matches[0]; - $encchunk = base64_encode($chunk); - if (strlen($encchunk) > $length) { - // find first 11 chars - each char in 4 bytes - worst case scenario - preg_match('/^(([\x00-\x7f])|([\x81-\xfe][\x40-\x7e])|([\x81-\xfe][\x80-\xfe])|([\x81-\xfe][\x30-\x39]..)){1,11}/m', $text, $matches); - $chunk = $matches[0]; - $encchunk = base64_encode($chunk); - } - $text = substr($text, strlen($chunk)); - $encoded .= ' '.$start.$encchunk.$end.$linefeed; - } else { - break; - } - } - $encoded = trim($encoded); - return $encoded; - } else { - return false; - } - } - $ratio = $multilength / strlen($text); - // Base64 ratio - $magic = $avglength = floor(3 * $length * $ratio / 4); - // basic infinite loop protection - $maxiterations = strlen($text)*2; - $iteration = 0; - // Iterate over the string in magic chunks - for ($i=0; $i <= $multilength; $i+=$magic) { - if ($iteration++ > $maxiterations) { - return false; // probably infinite loop - } - $magic = $avglength; - $offset = 0; - // Ensure the chunk fits in length, reducing magic if necessary - do { - $magic -= $offset; - $chunk = self::substr($text, $i, $magic, $charset); - $chunk = base64_encode($chunk); - $offset++; - } while (strlen($chunk) > $length); - // This chunk doesn't break any multi-byte char. Use it. - if ($chunk) - $encoded .= ' '.$start.$chunk.$end.$linefeed; - } - // Strip the first space and the last linefeed - $encoded = substr($encoded, 1, -strlen($linefeed)); - - return $encoded; - } - - /** - * Returns HTML entity transliteration table. - * @return array with (html entity => utf-8) elements - */ - protected static function get_entities_table() { - static $trans_tbl = null; - - // Generate/create $trans_tbl - if (!isset($trans_tbl)) { - if (version_compare(phpversion(), '5.3.4') < 0) { - $trans_tbl = array(); - foreach (get_html_translation_table(HTML_ENTITIES) as $val=>$key) { - $trans_tbl[$key] = textlib::convert($val, 'ISO-8859-1', 'utf-8'); - } - - } else if (version_compare(phpversion(), '5.4.0') < 0) { - $trans_tbl = get_html_translation_table(HTML_ENTITIES, ENT_COMPAT, 'UTF-8'); - $trans_tbl = array_flip($trans_tbl); - - } else { - $trans_tbl = get_html_translation_table(HTML_ENTITIES, ENT_COMPAT | ENT_HTML401, 'UTF-8'); - $trans_tbl = array_flip($trans_tbl); - } - } - - return $trans_tbl; - } - - /** - * Converts all the numeric entities &#nnnn; or &#xnnn; to UTF-8 - * Original from laurynas dot butkus at gmail at: - * http://php.net/manual/en/function.html-entity-decode.php#75153 - * with some custom mods to provide more functionality - * - * @param string $str input string - * @param boolean $htmlent convert also html entities (defaults to true) - * @return string encoded UTF-8 string - */ - public static function entities_to_utf8($str, $htmlent=true) { - static $callback1 = null ; - static $callback2 = null ; - - if (!$callback1 or !$callback2) { - $callback1 = create_function('$matches', 'return textlib::code2utf8(hexdec($matches[1]));'); - $callback2 = create_function('$matches', 'return textlib::code2utf8($matches[1]);'); - } - - $result = (string)$str; - $result = preg_replace_callback('/&#x([0-9a-f]+);/i', $callback1, $result); - $result = preg_replace_callback('/&#([0-9]+);/', $callback2, $result); - - // Replace literal entities (if desired) - if ($htmlent) { - $trans_tbl = self::get_entities_table(); - // It should be safe to search for ascii strings and replace them with utf-8 here. - $result = strtr($result, $trans_tbl); - } - // Return utf8-ised string - return $result; - } - - /** - * Converts all Unicode chars > 127 to numeric entities &#nnnn; or &#xnnn;. - * - * @param string $str input string - * @param boolean $dec output decadic only number entities - * @param boolean $nonnum remove all non-numeric entities - * @return string converted string - */ - public static function utf8_to_entities($str, $dec=false, $nonnum=false) { - static $callback = null ; - - if ($nonnum) { - $str = self::entities_to_utf8($str, true); - } - - // Avoid some notices from Typo3 code - $oldlevel = error_reporting(E_PARSE); - $result = self::typo3()->utf8_to_entities((string)$str); - error_reporting($oldlevel); - - if ($dec) { - if (!$callback) { - $callback = create_function('$matches', 'return \'&#\'.(hexdec($matches[1])).\';\';'); - } - $result = preg_replace_callback('/&#x([0-9a-f]+);/i', $callback, $result); - } - - return $result; - } - - /** - * Removes the BOM from unicode string {@link http://unicode.org/faq/utf_bom.html} - * - * @param string $str input string - * @return string - */ - public static function trim_utf8_bom($str) { - $bom = "\xef\xbb\xbf"; - if (strpos($str, $bom) === 0) { - return substr($str, strlen($bom)); - } - return $str; - } - - /** - * Returns encoding options for select boxes, utf-8 and platform encoding first - * - * @return array encodings - */ - public static function get_encodings() { - $encodings = array(); - $encodings['UTF-8'] = 'UTF-8'; - $winenc = strtoupper(get_string('localewincharset', 'langconfig')); - if ($winenc != '') { - $encodings[$winenc] = $winenc; - } - $nixenc = strtoupper(get_string('oldcharset', 'langconfig')); - $encodings[$nixenc] = $nixenc; - - foreach (self::typo3()->synonyms as $enc) { - $enc = strtoupper($enc); - $encodings[$enc] = $enc; - } - return $encodings; - } - - /** - * Returns the utf8 string corresponding to the unicode value - * (from php.net, courtesy - romans@void.lv) - * - * @param int $num one unicode value - * @return string the UTF-8 char corresponding to the unicode value - */ - public static function code2utf8($num) { - if ($num < 128) { - return chr($num); - } - if ($num < 2048) { - return chr(($num >> 6) + 192) . chr(($num & 63) + 128); - } - if ($num < 65536) { - return chr(($num >> 12) + 224) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128); - } - if ($num < 2097152) { - return chr(($num >> 18) + 240) . chr((($num >> 12) & 63) + 128) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128); - } - return ''; - } - - /** - * Returns the code of the given UTF-8 character - * - * @param string $utf8char one UTF-8 character - * @return int the code of the given character - */ - public static function utf8ord($utf8char) { - if ($utf8char == '') { - return 0; - } - $ord0 = ord($utf8char{0}); - if ($ord0 >= 0 && $ord0 <= 127) { - return $ord0; - } - $ord1 = ord($utf8char{1}); - if ($ord0 >= 192 && $ord0 <= 223) { - return ($ord0 - 192) * 64 + ($ord1 - 128); - } - $ord2 = ord($utf8char{2}); - if ($ord0 >= 224 && $ord0 <= 239) { - return ($ord0 - 224) * 4096 + ($ord1 - 128) * 64 + ($ord2 - 128); - } - $ord3 = ord($utf8char{3}); - if ($ord0 >= 240 && $ord0 <= 247) { - return ($ord0 - 240) * 262144 + ($ord1 - 128 )* 4096 + ($ord2 - 128) * 64 + ($ord3 - 128); - } - return false; - } - - /** - * Makes first letter of each word capital - words must be separated by spaces. - * Use with care, this function does not work properly in many locales!!! - * - * @param string $text input string - * @return string - */ - public static function strtotitle($text) { - if (empty($text)) { - return $text; - } - - if (function_exists('mb_convert_case')) { - return mb_convert_case($text, MB_CASE_TITLE, 'UTF-8'); - } - - $text = self::strtolower($text); - $words = explode(' ', $text); - foreach ($words as $i=>$word) { - $length = self::strlen($word); - if (!$length) { - continue; - - } else if ($length == 1) { - $words[$i] = self::strtoupper($word); - - } else { - $letter = self::substr($word, 0, 1); - $letter = self::strtoupper($letter); - $rest = self::substr($word, 1); - $words[$i] = $letter.$rest; - } - } - return implode(' ', $words); - } - - /** - * Locale aware sorting, the key associations are kept, values are sorted alphabetically. - * - * @param array $arr array to be sorted (reference) - * @param int $sortflag One of Collator::SORT_REGULAR, Collator::SORT_NUMERIC, Collator::SORT_STRING - * @return void modifies parameter - */ - public static function asort(array &$arr, $sortflag = null) { - debugging('textlib::asort has been superseeded by collatorlib::asort please upgrade your code to use that', DEBUG_DEVELOPER); - collatorlib::asort($arr, $sortflag); - } -} - - -/** - * A collator class with static methods that can be used for sorting. - * - * @package core - * @copyright 2011 Sam Hemelryk - * 2012 Petr Skoda - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ -class collatorlib { - /** @const compare items using general PHP comparison, equivalent to Collator::SORT_REGULAR, this may bot be locale aware! */ - const SORT_REGULAR = 0; - - /** @const compare items as strings, equivalent to Collator::SORT_STRING */ - const SORT_STRING = 1; - - /** @const compare items as numbers, equivalent to Collator::SORT_NUMERIC */ - const SORT_NUMERIC = 2; - - /** @const compare items like natsort(), equivalent to SORT_NATURAL */ - const SORT_NATURAL = 6; - - /** @const do not ignore case when sorting, use bitwise "|" with SORT_NATURAL or SORT_STRING, equivalent to Collator::UPPER_FIRST */ - const CASE_SENSITIVE = 64; - - /** @var Collator|false|null **/ - protected static $collator = null; - - /** @var string|null The locale that was used in instantiating the current collator **/ - protected static $locale = null; - - /** - * Prevent class instances, all methods are static. - */ - private function __construct() { - } - - /** - * Ensures that a collator is available and created - * - * @return bool Returns true if collation is available and ready - */ - protected static function ensure_collator_available() { - $locale = get_string('locale', 'langconfig'); - if (is_null(self::$collator) || $locale != self::$locale) { - self::$collator = false; - self::$locale = $locale; - if (class_exists('Collator', false)) { - $collator = new Collator($locale); - if (!empty($collator) && $collator instanceof Collator) { - // Check for non fatal error messages. This has to be done immediately - // after instantiation as any further calls to collation will cause - // it to reset to 0 again (or another error code if one occurred) - $errorcode = $collator->getErrorCode(); - $errormessage = $collator->getErrorMessage(); - // Check for an error code, 0 means no error occurred - if ($errorcode !== 0) { - // Get the actual locale being used, e.g. en, he, zh - $localeinuse = $collator->getLocale(Locale::ACTUAL_LOCALE); - // Check for the common fallback warning error codes. If this occurred - // there is normally little to worry about: - // - U_USING_DEFAULT_WARNING (127) - default fallback locale used (pt => UCA) - // - U_USING_FALLBACK_WARNING (128) - fallback locale used (de_CH => de) - // (UCA: Unicode Collation Algorithm http://unicode.org/reports/tr10/) - if ($errorcode === -127 || $errorcode === -128) { - // Check if the locale in use is UCA default one ('root') or - // if it is anything like the locale we asked for - if ($localeinuse !== 'root' && strpos($locale, $localeinuse) !== 0) { - // The locale we asked for is completely different to the locale - // we have received, let the user know via debugging - debugging('Invalid locale: "' . $locale . '", with warning (not fatal) "' . $errormessage . - '", falling back to "' . $collator->getLocale(Locale::VALID_LOCALE) . '"'); - } else { - // Nothing to do here, this is expected! - // The Moodle locale setting isn't what the collator expected but - // it is smart enough to match the first characters of our locale - // to find the correct locale or to use UCA collation - } - } else { - // We've received some other sort of non fatal warning - let the - // user know about it via debugging. - debugging('Problem with locale: "' . $locale . '", with message "' . $errormessage . - '", falling back to "' . $collator->getLocale(Locale::VALID_LOCALE) . '"'); - } - } - // Store the collator object now that we can be sure it is in a workable condition - self::$collator = $collator; - } else { - // Fatal error while trying to instantiate the collator... something went wrong - debugging('Error instantiating collator for locale: "' . $locale . '", with error [' . - intl_get_error_code() . '] ' . intl_get_error_message($collator)); - } - } - } - return (self::$collator instanceof Collator); - } - - /** - * Restore array contents keeping new keys. - * @static - * @param array $arr - * @param array $original - * @return void modifies $arr - */ - protected static function restore_array(array &$arr, array &$original) { - foreach ($arr as $key => $ignored) { - $arr[$key] = $original[$key]; - } - } - - /** - * Normalise numbers in strings for natural sorting comparisons. - * @static - * @param string $string - * @return string string with normalised numbers - */ - protected static function naturalise($string) { - return preg_replace_callback('/[0-9]+/', array('collatorlib', 'callback_naturalise'), $string); - } - - /** - * @internal - * @static - * @param array $matches - * @return string - */ - public static function callback_naturalise($matches) { - return str_pad($matches[0], 20, '0', STR_PAD_LEFT); - } - - /** - * Locale aware sorting, the key associations are kept, values are sorted alphabetically. - * - * @param array $arr array to be sorted (reference) - * @param int $sortflag One of collatorlib::SORT_NUMERIC, collatorlib::SORT_STRING, collatorlib::SORT_NATURAL, collatorlib::SORT_REGULAR - * optionally "|" collatorlib::CASE_SENSITIVE - * @return bool True on success - */ - public static function asort(array &$arr, $sortflag = collatorlib::SORT_STRING) { - if (empty($arr)) { - // nothing to do - return true; - } - - $original = null; - - $casesensitive = (bool)($sortflag & collatorlib::CASE_SENSITIVE); - $sortflag = ($sortflag & ~collatorlib::CASE_SENSITIVE); - if ($sortflag != collatorlib::SORT_NATURAL and $sortflag != collatorlib::SORT_STRING) { - $casesensitive = false; - } - - if (self::ensure_collator_available()) { - if ($sortflag == collatorlib::SORT_NUMERIC) { - $flag = Collator::SORT_NUMERIC; - - } else if ($sortflag == collatorlib::SORT_REGULAR) { - $flag = Collator::SORT_REGULAR; - - } else { - $flag = Collator::SORT_STRING; - } - - if ($sortflag == collatorlib::SORT_NATURAL) { - $original = $arr; - if ($sortflag == collatorlib::SORT_NATURAL) { - foreach ($arr as $key => $value) { - $arr[$key] = self::naturalise((string)$value); - } - } - } - if ($casesensitive) { - self::$collator->setAttribute(Collator::CASE_FIRST, Collator::UPPER_FIRST); - } else { - self::$collator->setAttribute(Collator::CASE_FIRST, Collator::OFF); - } - $result = self::$collator->asort($arr, $flag); - if ($original) { - self::restore_array($arr, $original); - } - return $result; - } - - // try some fallback that works at least for English - - if ($sortflag == collatorlib::SORT_NUMERIC) { - return asort($arr, SORT_NUMERIC); - - } else if ($sortflag == collatorlib::SORT_REGULAR) { - return asort($arr, SORT_REGULAR); - } - - if (!$casesensitive) { - $original = $arr; - foreach ($arr as $key => $value) { - $arr[$key] = textlib::strtolower($value); - } - } - - if ($sortflag == collatorlib::SORT_NATURAL) { - $result = natsort($arr); - - } else { - $result = asort($arr, SORT_LOCALE_STRING); - } - - if ($original) { - self::restore_array($arr, $original); - } - - return $result; - } - - /** - * Locale aware sort of objects by a property in common to all objects - * - * @param array $objects An array of objects to sort (handled by reference) - * @param string $property The property to use for comparison - * @param int $sortflag One of collatorlib::SORT_NUMERIC, collatorlib::SORT_STRING, collatorlib::SORT_NATURAL, collatorlib::SORT_REGULAR - * optionally "|" collatorlib::CASE_SENSITIVE - * @return bool True on success - */ - public static function asort_objects_by_property(array &$objects, $property, $sortflag = collatorlib::SORT_STRING) { - $original = $objects; - foreach ($objects as $key => $object) { - $objects[$key] = $object->$property; - } - $result = self::asort($objects, $sortflag); - self::restore_array($objects, $original); - return $result; - } - - /** - * Locale aware sort of objects by a method in common to all objects - * - * @param array $objects An array of objects to sort (handled by reference) - * @param string $method The method to call to generate a value for comparison - * @param int $sortflag One of collatorlib::SORT_NUMERIC, collatorlib::SORT_STRING, collatorlib::SORT_NATURAL, collatorlib::SORT_REGULAR - * optionally "|" collatorlib::CASE_SENSITIVE - * @return bool True on success - */ - public static function asort_objects_by_method(array &$objects, $method, $sortflag = collatorlib::SORT_STRING) { - $original = $objects; - foreach ($objects as $key => $object) { - $objects[$key] = $object->{$method}(); - } - $result = self::asort($objects, $sortflag); - self::restore_array($objects, $original); - return $result; - } - - /** - * Locale aware sorting, the key associations are kept, keys are sorted alphabetically. - * - * @param array $arr array to be sorted (reference) - * @param int $sortflag One of collatorlib::SORT_NUMERIC, collatorlib::SORT_STRING, collatorlib::SORT_NATURAL, collatorlib::SORT_REGULAR - * optionally "|" collatorlib::CASE_SENSITIVE - * @return bool True on success - */ - public static function ksort(array &$arr, $sortflag = collatorlib::SORT_STRING) { - $keys = array_keys($arr); - if (!self::asort($keys, $sortflag)) { - return false; - } - // This is a bit slow, but we need to keep the references - $original = $arr; - $arr = array(); // Surprisingly this does not break references outside - foreach ($keys as $key) { - $arr[$key] = $original[$key]; - } - - return true; - } -} - +debugging('Do not include textlib.class.php directly, it is now using automatic class loading.'); diff --git a/lib/upgrade.txt b/lib/upgrade.txt index de7abfb271d..b10aa3dfe55 100644 --- a/lib/upgrade.txt +++ b/lib/upgrade.txt @@ -3,6 +3,7 @@ information provided here is intended especially for developers. === 2.6 === * Use new core_component::* plugin listing and component normalisation methods. +* Use core_text::* instead of textlib:: and also core_collator::* instead of collatorlib::*. === 2.5.1 === diff --git a/lib/upgradelib.php b/lib/upgradelib.php index c1ada2eb91a..99d254d820a 100644 --- a/lib/upgradelib.php +++ b/lib/upgradelib.php @@ -1973,7 +1973,6 @@ function upgrade_rename_old_backup_files_using_shortname() { return; } - require_once($CFG->libdir.'/textlib.class.php'); require_once($CFG->dirroot.'/backup/util/includes/backup_includes.php'); $backupword = str_replace(' ', '_', textlib::strtolower(get_string('backupfilename'))); $backupword = trim(clean_filename($backupword), '_'); diff --git a/repository/googledocs/lib.php b/repository/googledocs/lib.php index 62eb57956b8..8ef735d36b2 100644 --- a/repository/googledocs/lib.php +++ b/repository/googledocs/lib.php @@ -26,7 +26,6 @@ defined('MOODLE_INTERNAL') || die(); require_once($CFG->dirroot . '/repository/lib.php'); -require_once($CFG->libdir . '/textlib.class.php'); require_once($CFG->libdir . '/google/Google_Client.php'); require_once($CFG->libdir . '/google/contrib/Google_DriveService.php');