From cfc673c1120c28fd9f476d2e3ba7a811e08293ba Mon Sep 17 00:00:00 2001
From: stronk7 <stronk7>
Date: Wed, 13 Jun 2007 23:21:36 +0000
Subject: [PATCH] Now the tokeniser is using the new
 textlib->entities_to_utf8() own function (PHP4 and PHP compatible)

---
 lib/tokeniserlib.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tokeniserlib.php b/lib/tokeniserlib.php
index 737875b126b..4cf3d4c40e8 100644
--- a/lib/tokeniserlib.php
+++ b/lib/tokeniserlib.php
@@ -312,7 +312,7 @@ function tokenise_simplify($text, $overlap_cjk, $join_numbers) {
     $textlib = textlib_get_instance();
 
     // Decode entities to UTF-8
-    $text = html_entity_decode($text, ENT_QUOTES, 'UTF-8');
+    $text = $textlib->entities_to_utf8($text, true);
 
     // Lowercase
     $text = $textlib->strtolower($text);