L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MDL-29027 improve textlib api and add missing unittests

Browse Source
This commit is contained in:
Petr Skoda
2011-08-20 17:00:53 +02:00
parent 4f3632441a
commit 5a7e4c1158
2 changed files with 372 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/textlib.class.php
+263 -189
View File
@@ -1,5 +1,4 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
@@ -25,72 +24,17 @@
defined('MOODLE_INTERNAL') || die();
/**
* As we implement the singleton pattern to use this class (only one instance
* is shared globally), we need this helper function
* Original singleton helper function, please use static methods instead,
* ex: textlib::convert()
*
* IMPORTANT Note: Typo3 libraries always expect lowercase charsets to use 100%
* its capabilities so, don't forget to make the conversion
* from every wrapper function!
*
* @return textlib singleton instance of textlib
* @deprecated
* @return textlib instance
*/
function textlib_get_instance() {
global $CFG;
static $instance = null;
if (!$instance) {
/// initialisation is delayed because we do not want this on each page ;-)
/// Required files
require_once($CFG->libdir.'/typo3/class.t3lib_cs.php');
require_once($CFG->libdir.'/typo3/class.t3lib_div.php');
/// If ICONV is available, lets Typo3 library use it for convert
if (extension_loaded('iconv')) {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'iconv';
/// Else if mbstring is available, lets Typo3 library use it
} else if (extension_loaded('mbstring')) {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'mbstring';
/// Else if recode is available, lets Typo3 library use it
} else if (extension_loaded('recode')) {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'recode';
} else {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = '';
}
/// If mbstring is available, lets Typo3 library use it for functions
if (extension_loaded('mbstring')) {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] = 'mbstring';
} else {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] = '';
}
/// Tell Typo3 we are curl enabled always (mandatory since 2.0)
$GLOBALS['TYPO3_CONF_VARS']['SYS']['curlUse'] = '1';
/// And this directory must exist to allow Typo to cache conversion
/// tables when using internal functions
make_upload_directory('temp/typo3temp/cs');
/// Make sure typo is using our dir permissions
$GLOBALS['TYPO3_CONF_VARS']['BE']['folderCreateMask'] = decoct($CFG->directorypermissions);
/// Default mask for Typo
$GLOBALS['TYPO3_CONF_VARS']['BE']['fileCreateMask'] = $CFG->directorypermissions;
/// This full path constants must be defined too, transforming backslashes
/// to forward slashed beacuse Typo3 requires it.
define ('PATH_t3lib', str_replace('\\','/',$CFG->libdir.'/typo3/'));
define ('PATH_typo3', str_replace('\\','/',$CFG->libdir.'/typo3/'));
define ('PATH_site', str_replace('\\','/',$CFG->dataroot.'/temp/'));
define ('TYPO3_OS', stristr(PHP_OS,'win')&&!stristr(PHP_OS,'darwin')?'WIN':'');
$instance = new textlib();
}
return $instance;
return new textlib();
}
/**
* This class is used to manipulate strings under Moodle 1.6 an later. As
* utf-8 text become mandatory a pool of safe functions under this encoding
@@ -102,133 +46,253 @@ function textlib_get_instance() {
*
* Take a look to its own copyright and license details.
*
* @package moodlecore
* IMPORTANT Note: Typo3 libraries always expect lowercase charsets to use 100%
* its capabilities so, don't forget to make the conversion
* from every wrapper function!
*
* @package core
* @subpackage lib
* @copyright 1999 onwards Martin Dougiamas {@link http://moodle.com}
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class textlib {
var $typo3cs;
/**
* Standard constructor of the class. All it does is to instantiate
* a new t3lib_cs object to have all their functions ready.
*
* Instead of istantiating a lot of objects of this class everytime
* some of their functions is going to be used, you can invoke the:
* textlib_get_instance() function, avoiding the creation of them
* (following the singleton pattern)
* Return t3lib helper class
* @return t3lib_cs
*/
function textlib() {
/// Instantiate a conversor object some of the methods in typo3
/// reference to $this and cannot be executed in a static context
$this->typo3cs = new t3lib_cs();
protected static function typo3() {
static $typo3cs = null;
if (isset($typo3cs)) {
return $typo3cs;
}
global $CFG;
// Required files
require_once($CFG->libdir.'/typo3/class.t3lib_cs.php');
require_once($CFG->libdir.'/typo3/class.t3lib_div.php');
// If ICONV is available, lets Typo3 library use it for convert
if (extension_loaded('iconv')) {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'iconv';
// Else if mbstring is available, lets Typo3 library use it
} else if (extension_loaded('mbstring')) {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'mbstring';
// Else if recode is available, lets Typo3 library use it
} else if (extension_loaded('recode')) {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'recode';
} else {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = '';
}
// If mbstring is available, lets Typo3 library use it for functions
if (extension_loaded('mbstring')) {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] = 'mbstring';
} else {
$GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] = '';
}
// Tell Typo3 we are curl enabled always (mandatory since 2.0)
$GLOBALS['TYPO3_CONF_VARS']['SYS']['curlUse'] = '1';
// And this directory must exist to allow Typo to cache conversion
// tables when using internal functions
make_upload_directory('temp/typo3temp/cs');
// Make sure typo is using our dir permissions
$GLOBALS['TYPO3_CONF_VARS']['BE']['folderCreateMask'] = decoct($CFG->directorypermissions);
// Default mask for Typo
$GLOBALS['TYPO3_CONF_VARS']['BE']['fileCreateMask'] = $CFG->directorypermissions;
// This full path constants must be defined too, transforming backslashes
// to forward slashed because Typo3 requires it.
define ('PATH_t3lib', str_replace('\\','/',$CFG->libdir.'/typo3/'));
define ('PATH_typo3', str_replace('\\','/',$CFG->libdir.'/typo3/'));
define ('PATH_site', str_replace('\\','/',$CFG->dataroot.'/temp/'));
define ('TYPO3_OS', stristr(PHP_OS,'win')&&!stristr(PHP_OS,'darwin')?'WIN':'');
$typo3cs = new t3lib_cs();
return $typo3cs;
}
/**
* Converts the text between different encodings. It will use iconv, mbstring
* or internal (typo3) methods to try such conversion. Returns false if fails.
* Standardise charset name
*
* Please note it does not mean the returned charset is actually supported.
*
* @static
* @param string $charset raw charset name
* @return string normalised lowercase charset name
*/
function convert($text, $fromCS, $toCS='utf-8') {
/// Normalize charsets
$fromCS = $this->typo3cs->parse_charset($fromCS);
$toCS = $this->typo3cs->parse_charset($toCS);
/// Avoid some notices from Typo3 code
$oldlevel = error_reporting(E_PARSE);
/// Call Typo3 conv() function. It will do all the work
$result = $this->typo3cs->conv($text, $fromCS, $toCS);
/// Restore original debug level
error_reporting($oldlevel);
return $result;
public static function parse_charset($charset) {
// shortcuts so that we do not have to load typo on every page
$charset = strtolower($charset);
switch ($charset) {
case 'utf8':
case 'utf-8':
return 'utf-8';
case 'cp1250':
case 'win-1250':
case 'windowns-1250':
return 'windows-1250';
case 'cp1252':
case 'win-1252':
case 'windowns-1252':
return 'windows-1252';
case 'l1':
case 'latin1':
case 'iso-8859-1':
case 'iso-8859-1':
return 'iso-8859-1';
case 'l2':
case 'latin2':
case 'iso-8859-2':
case 'iso-8859-2':
return 'iso-8859-2';
//TODO: add more common charsets here - all win and unix encodings from our lang packs
}
// fallback to typo3
return self::typo3()->parse_charset($charset);
}
/**
* Converts the text between different encodings. It uses iconv extension with //TRANSLIT parameter.
* Returns false if fails.
*
* @param string $text
* @param string $fromCS source encoding
* @param string $toCS result encoding
* @return string|bool
*/
public static function convert($text, $fromCS, $toCS='utf-8') {
$fromCS = self::parse_charset($fromCS);
$toCS = self::parse_charset($toCS);
return iconv($fromCS, $toCS.'//TRANSLIT', $text);
}
/**
* Multibyte safe substr() function, uses mbstring if available.
*
* @param string $text
* @param int $start negative value means from end
* @param int $len
* @param string $charset encoding of the text
* @return string
*/
function substr($text, $start, $len=null, $charset='utf-8') {
/// Normalize charset
$charset = $this->typo3cs->parse_charset($charset);
/// Avoid some notices from Typo3 code
$oldlevel = error_reporting(E_PARSE);
/// Call Typo3 substr() function. It will do all the work
$result = $this->typo3cs->substr($charset,$text,$start,$len);
/// Restore original debug level
error_reporting($oldlevel);
return $result;
public static function substr($text, $start, $len=null, $charset='utf-8') {
$charset = self::parse_charset($charset);
return iconv_substr($text, $start, $len, $charset);
}
/**
* Multibyte safe strlen() function, uses mbstring if available.
*
* @param string $text
* @param string $charset encoding of the text
* @return int number of characters
*/
function strlen($text, $charset='utf-8') {
/// Normalize charset
$charset = $this->typo3cs->parse_charset($charset);
/// Avoid some notices from Typo3 code
$oldlevel = error_reporting(E_PARSE);
/// Call Typo3 strlen() function. It will do all the work
$result = $this->typo3cs->strlen($charset,$text);
/// Restore original debug level
error_reporting($oldlevel);
return $result;
public static function strlen($text, $charset='utf-8') {
$charset = self::parse_charset($charset);
return iconv_strlen($text, $charset);
}
/**
* Multibyte safe strtolower() function, uses mbstring if available.
*
* @param string $text
* @param string $charset encoding of the text (may not work for all encodings)
* @return string lower case text
*/
function strtolower($text, $charset='utf-8') {
/// Normalize charset
$charset = $this->typo3cs->parse_charset($charset);
/// Avoid some notices from Typo3 code
public static function strtolower($text, $charset='utf-8') {
$charset = self::parse_charset($charset);
if (function_exists('mb_strtolower')) {
return mb_strtolower($text, $charset);
}
// Avoid some notices from Typo3 code
$oldlevel = error_reporting(E_PARSE);
/// Call Typo3 conv_case() function. It will do all the work
$result = $this->typo3cs->conv_case($charset,$text,'toLower');
/// Restore original debug level
// Call Typo3 conv_case() function. It will do all the work
$result = self::typo3()->conv_case($charset, $text, 'toLower');
// Restore original debug level
error_reporting($oldlevel);
return $result;
}
/**
* Multibyte safe strtoupper() function, uses mbstring if available.
*
* @param string $text
* @param string $charset encoding of the text (may not work for all encodings)
* @return string upper case text
*/
function strtoupper($text, $charset='utf-8') {
/// Normalize charset
$charset = $this->typo3cs->parse_charset($charset);
/// Avoid some notices from Typo3 code
public static function strtoupper($text, $charset='utf-8') {
$charset = self::parse_charset($charset);
if (function_exists('mb_strtoupper')) {
return mb_strtoupper($text, $charset);
}
// Avoid some notices from Typo3 code
$oldlevel = error_reporting(E_PARSE);
/// Call Typo3 conv_case() function. It will do all the work
$result = $this->typo3cs->conv_case($charset,$text,'toUpper');
/// Restore original debug level
// Call Typo3 conv_case() function. It will do all the work
$result = self::typo3()->conv_case($charset, $text, 'toUpper');
// Restore original debug level
error_reporting($oldlevel);
return $result;
}
/**
* UTF-8 ONLY safe strpos() function, uses mbstring if available.
* UTF-8 ONLY safe strpos().
*
* @param string $haystack
* @param string $needle
* @param int $offset
* @return string
*/
function strpos($haystack,$needle,$offset=0) {
/// Call Typo3 utf8_strpos() function. It will do all the work
return $this->typo3cs->utf8_strpos($haystack,$needle,$offset);
public static function strpos($haystack, $needle, $offset=0) {
return iconv_strpos($haystack, $needle, $offset, 'utf-8');
}
/**
* UTF-8 ONLY safe strrpos() function, uses mbstring if available.
* UTF-8 ONLY safe strrpos().
*
* @param string $haystack
* @param string $needle
* @return string
*/
function strrpos($haystack,$needle) {
/// Call Typo3 utf8_strrpos() function. It will do all the work
return $this->typo3cs->utf8_strrpos($haystack,$needle);
public static function strrpos($haystack, $needle) {
return iconv_strrpos($haystack, $needle, 'utf-8');
}
/**
* Try to convert upper unicode characters to plain ascii,
* the returned string may cantain unconverted unicode characters.
* the returned string may contain unconverted unicode characters.
*
* @param string $text
* @param string $charset encoding of the text
* @return string
*/
function specialtoascii($text,$charset='utf-8') {
/// Normalize charset
$charset = $this->typo3cs->parse_charset($charset);
/// Avoid some notices from Typo3 code
public static function specialtoascii($text, $charset='utf-8') {
$charset = self::parse_charset($charset);
// Avoid some notices from Typo3 code
$oldlevel = error_reporting(E_PARSE);
$result = $this->typo3cs->specCharsToASCII($charset,$text);
/// Restore original debug level
$result = self::typo3()->specCharsToASCII($charset, $text);
// Restore original debug level
error_reporting($oldlevel);
return $result;
}
@@ -237,30 +301,34 @@ class textlib {
* Generate a correct base64 encoded header to be used in MIME mail messages.
* This function seems to be 100% compliant with RFC1342. Credits go to:
* paravoid (http://www.php.net/manual/en/function.mb-encode-mimeheader.php#60283).
*
* @param string $text
* @param string $charset encoding of the text
* @return string
*/
function encode_mimeheader($text, $charset='utf-8') {
public static function encode_mimeheader($text, $charset='utf-8') {
if (empty($text)) {
return (string)$text;
}
/// Normalize charset
$charset = $this->typo3cs->parse_charset($charset);
/// If the text is pure ASCII, we don't need to encode it
if ($this->convert($text, $charset, 'ascii') == $text) {
// Normalize charset
$charset = self::parse_charset($charset);
// If the text is pure ASCII, we don't need to encode it
if (self::convert($text, $charset, 'ascii') == $text) {
return $text;
}
/// Although RFC says that line feed should be \r\n, it seems that
/// some mailers double convert \r, so we are going to use \n alone
// Although RFC says that line feed should be \r\n, it seems that
// some mailers double convert \r, so we are going to use \n alone
$linefeed="\n";
/// Define start and end of every chunk
// Define start and end of every chunk
$start = "=?$charset?B?";
$end = "?=";
/// Acumulate results
// Accumulate results
$encoded = '';
/// Max line length is 75 (including start and end)
// Max line length is 75 (including start and end)
$length = 75 - strlen($start) - strlen($end);
/// Multi-byte ratio
$multilength = $this->strlen($text, $charset);
/// Detect if strlen and friends supported
// Multi-byte ratio
$multilength = self::strlen($text, $charset);
// Detect if strlen and friends supported
if ($multilength === false) {
if ($charset == 'GB18030' or $charset == 'gb18030') {
while (strlen($text)) {
@@ -287,30 +355,30 @@ class textlib {
}
}
$ratio = $multilength / strlen($text);
/// Base64 ratio
// Base64 ratio
$magic = $avglength = floor(3 * $length * $ratio / 4);
/// basic infinite loop protection
// basic infinite loop protection
$maxiterations = strlen($text)*2;
$iteration = 0;
/// Iterate over the string in magic chunks
// Iterate over the string in magic chunks
for ($i=0; $i <= $multilength; $i+=$magic) {
if ($iteration++ > $maxiterations) {
return false; // probably infinite loop
}
$magic = $avglength;
$offset = 0;
/// Ensure the chunk fits in length, reduding magic if necessary
// Ensure the chunk fits in length, reducing magic if necessary
do {
$magic -= $offset;
$chunk = $this->substr($text, $i, $magic, $charset);
$chunk = self::substr($text, $i, $magic, $charset);
$chunk = base64_encode($chunk);
$offset++;
} while (strlen($chunk) > $length);
/// This chunk doen't break any multi-byte char. Use it.
// This chunk doesn't break any multi-byte char. Use it.
if ($chunk)
$encoded .= ' '.$start.$chunk.$end.$linefeed;
}
/// Strip the first space and the last linefeed
// Strip the first space and the last linefeed
$encoded = substr($encoded, 1, -strlen($linefeed));
return $encoded;
@@ -324,23 +392,23 @@ class textlib {
*
* @param string $str input string
* @param boolean $htmlent convert also html entities (defaults to true)
* @return string
*
* NOTE: we could have used typo3 entities_to_utf8() here
* but the direct alternative used runs 400% quicker
* and uses 0.5Mb less memory, so, let's use it
* (tested agains 10^6 conversions)
* (tested against 10^6 conversions)
*/
function entities_to_utf8($str, $htmlent=true) {
public static function entities_to_utf8($str, $htmlent=true) {
static $trans_tbl; // Going to use static transliteration table
static $trans_tbl; /// Going to use static translit table
/// Replace numeric entities
// Replace numeric entities
$result = preg_replace('~&#x([0-9a-f]+);~ei', 'textlib::code2utf8(hexdec("\\1"))', $str);
$result = preg_replace('~&#([0-9]+);~e', 'textlib::code2utf8(\\1)', $result);
/// Replace literal entities (if desired)
// Replace literal entities (if desired)
if ($htmlent) {
/// Generate/create $trans_tbl
// Generate/create $trans_tbl
if (!isset($trans_tbl)) {
$trans_tbl = array();
foreach (get_html_translation_table(HTML_ENTITIES) as $val=>$key) {
@@ -349,37 +417,40 @@ class textlib {
}
$result = strtr($result, $trans_tbl);
}
/// Return utf8-ised string
// Return utf8-ised string
return $result;
}
/**
* Converts all Unicode chars > 127 to numeric entities &#nnnn; or &#xnnn;.
*
* @param string input string
* @param boolean output decadic only number entities
* @param boolean remove all nonumeric entities
* @return string converted string
* @param string $str input string
* @param boolean $dec output decadic only number entities
* @param boolean $nonnum remove all non-numeric entities
* @return string converted string
*/
function utf8_to_entities($str, $dec=false, $nonnum=false) {
/// Avoid some notices from Typo3 code
public static function utf8_to_entities($str, $dec=false, $nonnum=false) {
// Avoid some notices from Typo3 code
$oldlevel = error_reporting(E_PARSE);
if ($nonnum) {
$str = $this->typo3cs->entities_to_utf8($str, true);
$str = self::typo3()->entities_to_utf8($str, true);
}
$result = $this->typo3cs->utf8_to_entities($str);
$result = self::typo3()->utf8_to_entities($str);
if ($dec) {
$result = preg_replace('/&#x([0-9a-f]+);/ie', "'&#'.hexdec('$1').';'", $result);
}
/// Restore original debug level
// Restore original debug level
error_reporting($oldlevel);
return $result;
}
/**
* Removes the BOM from unicode string - see http://unicode.org/faq/utf_bom.html
*
* @param string $str
* @return string
*/
function trim_utf8_bom($str) {
public static function trim_utf8_bom($str) {
$bom = "\xef\xbb\xbf";
if (strpos($str, $bom) === 0) {
return substr($str, strlen($bom));
@@ -391,7 +462,7 @@ class textlib {
* Returns encoding options for select boxes, utf-8 and platform encoding first
* @return array encodings
*/
function get_encodings() {
public static function get_encodings() {
$encodings = array();
$encodings['UTF-8'] = 'UTF-8';
$winenc = strtoupper(get_string('localewincharset', 'langconfig'));
@@ -401,7 +472,7 @@ class textlib {
$nixenc = strtoupper(get_string('oldcharset', 'langconfig'));
$encodings[$nixenc] = $nixenc;
foreach ($this->typo3cs->synonyms as $enc) {
foreach (self::typo3()->synonyms as $enc) {
$enc = strtoupper($enc);
$encodings[$enc] = $enc;
}
@@ -415,7 +486,7 @@ class textlib {
* @param int $num one unicode value
* @return string the UTF-8 char corresponding to the unicode value
*/
function code2utf8($num) {
public static function code2utf8($num) {
if ($num < 128) {
return chr($num);
}
@@ -434,32 +505,33 @@ class textlib {
/**
* Makes first letter of each word capital - words must be separated by spaces.
* Use with care, this function does not work properly in many locales!!!
*
* @param string $text
* @return string
*/
function strtotitle($text) {
public static function strtotitle($text) {
if (empty($text)) {
return $text;
}
if (function_exists('mb_convert_case')) {
return mb_convert_case($text, MB_CASE_TITLE,"UTF-8");
return mb_convert_case($text, MB_CASE_TITLE, 'UTF-8');
}
$text = $this->strtolower($text);
$text = self::strtolower($text);
$words = explode(' ', $text);
foreach ($words as $i=>$word) {
$length = $this->strlen($word);
$length = self::strlen($word);
if (!$length) {
continue;
} else if ($length == 1) {
$words[$i] = $this->strtoupper($word);
$words[$i] = self::strtoupper($word);
} else {
$letter = $this->substr($word, 0, 1);
$letter = $this->strtoupper($letter);
$rest = $this->substr($word, 1);
$letter = self::substr($word, 0, 1);
$letter = self::strtoupper($letter);
$rest = self::substr($word, 1);
$words[$i] = $letter.$rest;
}
}
@@ -468,11 +540,13 @@ class textlib {
/**
* Locale aware sorting, the key associations are kept, values are sorted alphabetically.
*
* Note: this function is using current moodle locale.
*
* @param array $arr array to be sorted
* @param string $lang moodle language
* @return void, modifies parameter
*/
function asort(array &$arr) {
public static function asort(array &$arr) {
if (function_exists('collator_asort')) {
if ($coll = collator_create(get_string('locale', 'langconfig'))) {
collator_asort($coll, $arr);