From 4ef233d993d6fd951b33dbdeca69ecb3cb3d0e36 Mon Sep 17 00:00:00 2001 From: Andrew Nicols Date: Tue, 20 Aug 2024 11:07:13 +0800 Subject: [PATCH 1/4] MDL-82828 core: Update Html2Text to version 4.3.2 --- lib/classes/component.php | 1 + lib/html2text/README.md | 26 ++++++++++++++++++++++++++ lib/html2text/composer.json | 23 +++++++++++++++++++++++ lib/html2text/lib.php | 1 - lib/html2text/readme_moodle.txt | 6 ++++-- lib/html2text/{ => src}/Html2Text.php | 22 ++++++++++++++++------ lib/thirdpartylibs.xml | 2 +- 7 files changed, 71 insertions(+), 10 deletions(-) create mode 100644 lib/html2text/README.md create mode 100644 lib/html2text/composer.json rename lib/html2text/{ => src}/Html2Text.php (97%) diff --git a/lib/classes/component.php b/lib/classes/component.php index fa4e64daa77..3cc12bfa2d0 100644 --- a/lib/classes/component.php +++ b/lib/classes/component.php @@ -102,6 +102,7 @@ class component { ]; /** @var array> associative array of PRS-4 namespaces and corresponding paths. */ protected static $psr4namespaces = [ + \Html2Text::class => 'lib/html2text/src', \MaxMind::class => 'lib/maxmind/MaxMind', \GeoIp2::class => 'lib/maxmind/GeoIp2', \Sabberworm\CSS::class => 'lib/php-css-parser', diff --git a/lib/html2text/README.md b/lib/html2text/README.md new file mode 100644 index 00000000000..cb619b3fa16 --- /dev/null +++ b/lib/html2text/README.md @@ -0,0 +1,26 @@ +# Html2Text + +A PHP library for converting HTML to formatted plain text. + +[![Build status](https://github.com/mtibben/html2text/actions/workflows/ci.yml/badge.svg)](https://github.com/mtibben/html2text/actions/workflows/ci.yml) + +## Installing + +``` +composer require html2text/html2text +``` + +## Basic Usage +```php +$html = new \Html2Text\Html2Text('Hello, "world"'); + +echo $html->getText(); // Hello, "WORLD" +``` + +## History + +This library started life on the blog of Jon Abernathy http://www.chuggnutt.com/html2text + +A number of projects picked up the library and started using it - among those was RoundCube mail. They made a number of updates to it over time to suit their webmail client. + +Now it has been extracted as a standalone library. Hopefully it can be of use to others. diff --git a/lib/html2text/composer.json b/lib/html2text/composer.json new file mode 100644 index 00000000000..7cfb7fe1066 --- /dev/null +++ b/lib/html2text/composer.json @@ -0,0 +1,23 @@ +{ + "name": "html2text/html2text", + "description": "Converts HTML to formatted plain text", + "type": "library", + "license": "GPL-2.0-or-later", + "autoload": { + "psr-4": { + "Html2Text\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Html2Text\\": "test/" + } + }, + "require-dev": { + "phpunit/phpunit": "~4|^9.0" + }, + "suggest": { + "ext-mbstring": "For best performance", + "symfony/polyfill-mbstring": "If you can't install ext-mbstring" + } +} \ No newline at end of file diff --git a/lib/html2text/lib.php b/lib/html2text/lib.php index 6c4588e22a6..d99b109157d 100644 --- a/lib/html2text/lib.php +++ b/lib/html2text/lib.php @@ -26,7 +26,6 @@ defined('MOODLE_INTERNAL') || die(); -require_once($CFG->libdir . '/html2text/Html2Text.php'); require_once(__DIR__ . '/override.php'); /** diff --git a/lib/html2text/readme_moodle.txt b/lib/html2text/readme_moodle.txt index 37cab254a25..fa41d264f4a 100644 --- a/lib/html2text/readme_moodle.txt +++ b/lib/html2text/readme_moodle.txt @@ -1,4 +1,4 @@ -Description of Html2Text v4.3.1 library import into Moodle +Description of Html2Text library import into Moodle Please note that we override some mb_* functions in Html2Text's namespace at run time. Until Html2Text adds some sort of fallback for the mb_* functions @@ -9,6 +9,8 @@ Instructions ------------ 1. Download the latest release of Html2Text from https://github.com/mtibben/html2text/releases/ 2. Extract the contents of the release archive into a directory. -3. Copy src/Html2Text.php to lib/html2text/ +3. Copy src to lib/html2text/src +4. Update README +5. Update composer.json Imported from: https://github.com/mtibben/html2text/releases/ diff --git a/lib/html2text/Html2Text.php b/lib/html2text/src/Html2Text.php similarity index 97% rename from lib/html2text/Html2Text.php rename to lib/html2text/src/Html2Text.php index 9fd91235927..6e0f9e50e83 100644 --- a/lib/html2text/Html2Text.php +++ b/lib/html2text/src/Html2Text.php @@ -236,16 +236,19 @@ class Html2Text */ public function __construct($html = '', $options = array()) { + $this->htmlFuncFlags = (PHP_VERSION_ID < 50400) + ? ENT_QUOTES + : ENT_QUOTES | ENT_HTML5; + // for backwards compatibility if (!is_array($options)) { - return call_user_func_array(array($this, 'legacyConstruct'), func_get_args()); + // phpcs:ignore (PHPCompatibility.FunctionUse.ArgumentFunctionsReportCurrentValue.NeedsInspection + call_user_func_array(array($this, 'legacyConstruct'), func_get_args()); + return; } $this->html = $html; $this->options = array_merge($this->options, $options); - $this->htmlFuncFlags = (PHP_VERSION_ID < 50400) - ? ENT_COMPAT - : ENT_COMPAT | ENT_HTML5; } /** @@ -351,7 +354,11 @@ class Html2Text { $this->linkList = array(); - $text = trim($this->html); + if ($this->html === null) { + $text = ''; + } else { + $text = trim($this->html); + } $this->converter($text); @@ -389,6 +396,9 @@ class Html2Text $text = preg_replace("/[\n]{3,}/", "\n\n", $text); // remove leading empty lines (can be produced by eg. P tag on the beginning) + if ($text === null) { + $text = ''; + } $text = ltrim($text, "\n"); if ($this->options['width'] > 0) { @@ -417,7 +427,7 @@ class Html2Text } // Ignored link types - if (preg_match('!^(javascript:|mailto:|#)!i', html_entity_decode($link))) { + if (preg_match('!^(javascript:|mailto:|#)!i', html_entity_decode($link, $this->htmlFuncFlags, self::ENCODING))) { return $display; } diff --git a/lib/thirdpartylibs.xml b/lib/thirdpartylibs.xml index 62fa9f82985..f535554e2c4 100644 --- a/lib/thirdpartylibs.xml +++ b/lib/thirdpartylibs.xml @@ -235,7 +235,7 @@ html2text HTML2Text PHP script to convert HTML into an approximate text equivalent. - 4.3.1 + 4.3.2 GPL 2.0+ https://github.com/mtibben/html2text From 082ed827b5df8c43d46f0275c879ab8c870207aa Mon Sep 17 00:00:00 2001 From: Andrew Nicols Date: Tue, 20 Aug 2024 11:14:05 +0800 Subject: [PATCH 2/4] MDL-82828 core: Remove html2text overrides for missing mbstring --- lib/html2text/lib.php | 25 ------ lib/html2text/override.php | 133 -------------------------------- lib/html2text/readme_moodle.txt | 5 -- 3 files changed, 163 deletions(-) delete mode 100644 lib/html2text/override.php diff --git a/lib/html2text/lib.php b/lib/html2text/lib.php index d99b109157d..e96d677136a 100644 --- a/lib/html2text/lib.php +++ b/lib/html2text/lib.php @@ -14,20 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Moodle. If not, see . -/** - * Wrapper for Html2Text - * - * This wrapper allows us to modify the upstream library without hacking it too much. - * - * @package core - * @copyright 2015 Andrew Nicols - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ - -defined('MOODLE_INTERNAL') || die(); - -require_once(__DIR__ . '/override.php'); - /** * Wrapper for Html2Text * @@ -38,7 +24,6 @@ require_once(__DIR__ . '/override.php'); * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class core_html2text extends \Html2Text\Html2Text { - /** * Constructor. * @@ -57,14 +42,4 @@ class core_html2text extends \Html2Text\Html2Text { $this->entSearch[] = '/[ ]+([\n\t])/'; $this->entReplace[] = '\\1'; } - - /** - * Strtoupper multibyte wrapper function with HTML entities handling. - * - * @param string $str Text to convert - * @return string Converted text - */ - protected function strtoupper($str) { - return core_text::strtoupper($str); - } } diff --git a/lib/html2text/override.php b/lib/html2text/override.php deleted file mode 100644 index 46bea72230c..00000000000 --- a/lib/html2text/override.php +++ /dev/null @@ -1,133 +0,0 @@ -. - -/** - * Run time overrides for Html2Text - * - * This allows us to monkey patch the mb_* functions used in Html2Text to use Moodle's core_text functionality. - * - * @package core - * @copyright 2016 Andrew Nicols - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ - -namespace Html2Text; - -/** - * Set the encoding to be used by our monkey patched mb_ functions. - * - * When called with $encoding !== null, we set the static $intenalencoding - * variable, which is used for subsequent calls. - * - * When called with no value for $encoding, we return the previously defined - * $internalencoding. - * - * This is necessary as we need to maintain the state of mb_internal_encoding - * across calls to other mb_* functions. Check how it is used in the other mb_* - * functions defined here - if no encoding is provided we fallback to what was - * set here, otherwise we used the given encoding. - * - * @staticvar string $internalencoding The encoding to be used across mb_* calls. - * @param string $encoding When given, sets $internalencoding - * @return mixed - */ -function mb_internal_encoding($encoding = null) { - static $internalencoding = 'utf-8'; - if ($encoding !== null) { - $internalencoding = $encoding; - return true; - } else { - return $internalencoding; - } -} - -/** - * Performs a multi-byte safe substr() operation based on number of characters. - * Position is counted from the beginning of str. First character's position is - * 0. Second character position is 1, and so on. - * - * @param string $str The string to extract the substring from. - * @param int $start If start is non-negative, the returned string will - * start at the start'th position in string, counting - * from zero. For instance, in the string 'abcdef', - * the character at position 0 is 'a', the character - * at position 2 is 'c', and so forth. - * @param int $length Maximum number of characters to use from str. If - * omitted or NULL is passed, extract all characters - * to the end of the string. - * @param string $encoding The encoding parameter is the character encoding. - * If it is omitted, the internal character encoding - * value will be used. - * - * @return string The portion of str specified by the start and length parameters. - */ -function mb_substr($str, $start, $length = null, $encoding = null) { - if ($encoding === null) { - $encoding = mb_internal_encoding(); - } - return \core_text::substr($str, $start, $length, $encoding); -} - -/** - * Gets the length of a string. - * - * @param string $str The string being checked for length. - * @param string $encoding The encoding parameter is the character encoding. - * If it is omitted, the internal character encoding - * value will be used. - * - * @return int The number of characters in str having character encoding $encoding. - * A multibyte character is counted as 1. - */ -function mb_strlen($str, $encoding = null) { - if ($encoding === null) { - $encoding = mb_internal_encoding(); - } - return \core_text::strlen($str, $encoding); -} - -/** - * Returns $str with all alphabetic chatacters converted to lowercase. - * - * @param string $str The string being lowercased. - * @param string $encoding The encoding parameter is the character encoding. - * If it is omitted, the internal character encoding - * value will be used. - * - * @return string The string with all alphabetic characters converted to lowercase. - */ -function mb_strtolower($str, $encoding = null) { - if ($encoding === null) { - $encoding = mb_internal_encoding(); - } - return \core_text::strtolower($str, $encoding); -} - -/** - * - * @param string The string being uppercased - * @param string $encoding The encoding parameter is the character encoding. - * If it is omitted, the internal character encoding - * value will be used. - * - * @return string The string with all alphabetic characters converted to uppercase. - */ -function mb_strtoupper($str, $encoding = null) { - if ($encoding === null) { - $encoding = mb_internal_encoding(); - } - return \core_text::strtoupper($str, $encoding); -} diff --git a/lib/html2text/readme_moodle.txt b/lib/html2text/readme_moodle.txt index fa41d264f4a..3ec4d309e94 100644 --- a/lib/html2text/readme_moodle.txt +++ b/lib/html2text/readme_moodle.txt @@ -1,10 +1,5 @@ Description of Html2Text library import into Moodle -Please note that we override some mb_* functions in Html2Text's namespace at -run time. Until Html2Text adds some sort of fallback for the mb_* functions -(or we make mbstring a hard requirement) we are forced to do this so that people -running PHP without mbstring don't see nasty undefined function errors. - Instructions ------------ 1. Download the latest release of Html2Text from https://github.com/mtibben/html2text/releases/ From 32fce988e9af673e3736437c83e22c82c626136f Mon Sep 17 00:00:00 2001 From: Andrew Nicols Date: Tue, 20 Aug 2024 14:41:19 +0800 Subject: [PATCH 3/4] MDL-82828 core: Rewrite html2text tests --- lib/tests/html2text_test.php | 252 +++++++++++++++++++---------------- 1 file changed, 140 insertions(+), 112 deletions(-) diff --git a/lib/tests/html2text_test.php b/lib/tests/html2text_test.php index 51d462d5473..15c7a2de9f3 100644 --- a/lib/tests/html2text_test.php +++ b/lib/tests/html2text_test.php @@ -25,49 +25,154 @@ namespace core; * @category test * @copyright 2012 Petr Skoda {@link http://skodak.org} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @covers ::html_to_text */ -class html2text_test extends \basic_testcase { - +final class html2text_test extends \basic_testcase { /** - * ALT as image replacements. + * Data provider for general tests. + * + * @return array */ - public function test_images(): void { - $this->assertSame('[edit]', html_to_text('edit')); + public static function examples_provider(): array { + // Used in the line wrapping tests. + // phpcs:ignore Generic.Files.LineLength.TooLong + $long = "Here is a long string, more than 75 characters long, since by default html_to_text wraps text at 75 chars."; + // phpcs:ignore Generic.Files.LineLength.TooLong + $wrapped = "Here is a long string, more than 75 characters long, since by default\nhtml_to_text wraps text at 75 chars."; - $text = 'xxsome gifxx'; - $result = html_to_text($text, null, false, false); - $this->assertSame($result, 'xx[some gif]xx'); + // These two are used in the PRE parsing tests. + // phpcs:ignore Generic.Files.LineLength.TooLong + $strorig = 'Consider the following function:
void FillMeUp(char* in_string) {'.
+            '
  int i = 0;
  while (in_string[i] != \'\0\') {
    in_string[i] = \'X\';
    i++;
  }
'.
+            '}
What would happen if a non-terminated string were input to this function?

'; + + // Note, the spaces in the 
 section are Unicode NBSPs - they may not be displayed in your editor.
+        $strconv = << [
+                '[edit]',
+                [],
+                'edit',
+            ],
+            'Image alt tag between strings' => [
+                'xx[some gif]xx',
+                [
+                    'dolinks' => false,
+                ],
+                'xxsome gifxx',
+            ],
+            'core_text integration' => [
+                'ŽLUŤOUČKÝ KONÍČEK',
+                ['dolinks' => false],
+                'Žluťoučký koníček',
+            ],
+            'No strip slashes in a tag' => [
+                '[\edit]',
+                [],
+                '\edit',
+            ],
+            'No strip slashes in a string' => [
+                '\\magic\\quotes\\are\\\\horrible',
+                [],
+                '\\magic\\quotes\\are\\\\horrible',
+            ],
+            'Protect "0"' => [
+                '0',
+                ['dolinks' => false],
+                '0',
+            ],
+            'Invalid HTML 1' => [
+                'Gin & Tonic',
+                [],
+                'Gin & Tonic',
+            ],
+            'Invalid HTML 2' => [
+                'Gin > Tonic',
+                [],
+                'Gin > Tonic',
+            ],
+            'Invalid HTML 3' => [
+                'Gin < Tonic',
+                [],
+                'Gin < Tonic',
+            ],
+            'Simple test 1' => [
+                "_Hello_ WORLD!\n",
+                [],
+                '
Hello world!
',
+            ],
+            'Simple test 2' => [
+                "All the WORLD’S a stage.\n\n-- William Shakespeare\n",
+                [],
+                '
All the world’s a stage.
-- William Shakespeare
',
+            ],
+            'Simple test 3' => [
+                "HELLO WORLD!\n\n",
+                [],
+                '
Hello world!
',
+            ],
+            'Simple test 4' => [
+                "Hello\nworld!",
+                [],
+                'Hello
world!',
+            ],
+            'No wrapping when width set to 0' => [
+                $long,
+                ['width' => 0],
+                $long,
+            ],
+            'Wrapping when width set to default' => [
+                $wrapped,
+                [],
+                $long,
+            ],
+            'Trailing whitespace removal' => [
+                'With trailing whitespace and some more text',
+                [],
+                "With trailing whitespace   \nand some   more text",
+            ],
+            'PRE parsing' => [
+                $strconv,
+                [],
+                $strorig,
+            ],
+            'Strip script tags' => [
+                'Interesting text',
+                [],
+                'Interesting  text',
+            ],
+        ];
     }
 
     /**
-     * No magic quotes messing.
+     * Test html2text with various examples.
+     *
+     * @dataProvider examples_provider
+     * @param string $expected
+     * @param array $options
+     * @param string $html
      */
-    public function test_no_strip_slashes(): void {
-        $this->assertSame('[\edit]', html_to_text('\edit'));
-
-        $text = '\\magic\\quotes\\are\\\\horrible';
-        $result = html_to_text($text, null, false, false);
-        $this->assertSame($result, $text);
-    }
-
-    /**
-     * core_text integration.
-     */
-    public function test_core_text(): void {
-        $text = 'Žluťoučký koníček';
-        $result = html_to_text($text, null, false, false);
-        $this->assertSame($result, 'ŽLUŤOUČKÝ KONÍČEK');
-    }
-
-    /**
-     * Protect 0.
-     */
-    public function test_zero(): void {
-        $text = '0';
-        $result = html_to_text($text, null, false, false);
-        $this->assertSame($result, $text);
-
-        $this->assertSame('0', html_to_text('0'));
+    public function test_runner(
+        string $expected,
+        array $options,
+        string $html,
+    ): void {
+        $this->assertSame($expected, html_to_text($html, ...$options));
     }
 
     /**
@@ -108,81 +213,4 @@ have been fixed last weekassertSame(1, preg_match('|^'.preg_quote('[4] https://www.google.fr').'$|m', $result));
         $this->assertSame(false, strpos($result, '[5]'));
     }
-
-    /**
-     * Various invalid HTML typed by users that ignore html strict.
-     **/
-    public function test_invalid_html(): void {
-        $text = 'Gin & Tonic';
-        $result = html_to_text($text, null, false, false);
-        $this->assertSame($result, $text);
-
-        $text = 'Gin > Tonic';
-        $result = html_to_text($text, null, false, false);
-        $this->assertSame($result, $text);
-
-        $text = 'Gin < Tonic';
-        $result = html_to_text($text, null, false, false);
-        $this->assertSame($result, $text);
-    }
-
-    /**
-     * Basic text formatting.
-     */
-    public function test_simple(): void {
-        $this->assertSame("_Hello_ WORLD!\n", html_to_text('
Hello world!
'));
-        $this->assertSame("All the WORLD’S a stage.\n\n-- William Shakespeare\n", html_to_text('
All the world’s a stage.
-- William Shakespeare
'));
-        $this->assertSame("HELLO WORLD!\n\n", html_to_text('
Hello world!
'));
-        $this->assertSame("Hello\nworld!", html_to_text('Hello
world!'));
-    }
-
-    /**
-     * Test line wrapping.
-     */
-    public function test_text_nowrap(): void {
-        $long = "Here is a long string, more than 75 characters long, since by default html_to_text wraps text at 75 chars.";
-        $wrapped = "Here is a long string, more than 75 characters long, since by default\nhtml_to_text wraps text at 75 chars.";
-        $this->assertSame($long, html_to_text($long, 0));
-        $this->assertSame($wrapped, html_to_text($long));
-    }
-
-    /**
-     * Whitespace removal.
-     */
-    public function test_trailing_whitespace(): void {
-        $this->assertSame('With trailing whitespace and some more text', html_to_text("With trailing whitespace   \nand some   more text", 0));
-    }
-
-    /**
-     * PRE parsing.
-     */
-    public function test_html_to_text_pre_parsing_problem(): void {
-        $strorig = 'Consider the following function:
void FillMeUp(char* in_string) {'.
-            '
  int i = 0;
  while (in_string[i] != \'\0\') {
    in_string[i] = \'X\';
    i++;
  }
'.
-            '}
What would happen if a non-terminated string were input to this function?

';
-
-        // Note, the spaces in the 
 section are Unicode NBSPs - they may not be displayed in your editor.
-        $strconv = 'Consider the following function:
-
-void FillMeUp(char* in_string) {
-  int i = 0;
-  while (in_string[i] != \'\0\') {
-    in_string[i] = \'X\';
-    i++;
-  }
-}
-What would happen if a non-terminated string were input to this function?
-
-';
-
-        $this->assertSame($strconv, html_to_text($strorig));
-    }
-
-    /**
-     * Scripts should be stripped.
-     */
-    public function test_strip_scripts(): void {
-        $this->assertSame('Interesting text',
-                html_to_text('Interesting  text', 0));
-    }
 }

From 3d5bca08bf3a8b22d91ae54a9a70fd66de344ea6 Mon Sep 17 00:00:00 2001
From: Andrew Nicols 
Date: Tue, 20 Aug 2024 14:50:06 +0800
Subject: [PATCH 4/4] MDL-82828 core: Add test for html2text trailing space
 between tags

---
 lib/tests/html2text_test.php | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/tests/html2text_test.php b/lib/tests/html2text_test.php
index 15c7a2de9f3..295a0698c5b 100644
--- a/lib/tests/html2text_test.php
+++ b/lib/tests/html2text_test.php
@@ -156,6 +156,16 @@ final class html2text_test extends \basic_testcase {
                 [],
                 'Interesting  text',
             ],
+            'Trailing spaces before newline or tab' => [
+                "Some text with trailing space\n\nAnd some more text\n",
+                [],
+                '
Some text with trailing space 
 
And some more text
',
+            ],
+            'Trailing spaces before newline or tab (list)' => [
+                "\t* Some text with trailing space\n\t* And some more text\n\n",
+                [],
+                '
  • Some text with trailing space 
    •  
  •  And some more text 
    •  
',
+            ],
         ];
     }