From 2e76c14e11c14392263a69c7064ff6adbdfca450 Mon Sep 17 00:00:00 2001 From: Damyon Wiese Date: Wed, 17 Feb 2016 16:16:13 +0800 Subject: [PATCH 1/9] MDL-52954 core: Add a document converter to file_storage This lets us convert between common office formats. E.g. docx -> pdf html -> pdf, html -> ods. This commit also updates assignment editpdf plugin to use this converter on all compatible submission files. --- admin/settings/server.php | 1 + config-dist.php | 6 + lang/en/admin.php | 2 + lib/behat/lib.php | 2 +- lib/filestorage/file_storage.php | 142 ++++++++++++++++++ lib/phpunit/bootstrap.php | 3 +- .../editpdf/classes/document_services.php | 59 +++++++- 7 files changed, 211 insertions(+), 4 deletions(-) diff --git a/admin/settings/server.php b/admin/settings/server.php index 573ed4b196f..07b7c7f895d 100644 --- a/admin/settings/server.php +++ b/admin/settings/server.php @@ -12,6 +12,7 @@ $temp->add(new admin_setting_configexecutable('pathtodu', new lang_string('patht $temp->add(new admin_setting_configexecutable('aspellpath', new lang_string('aspellpath', 'admin'), new lang_string('edhelpaspellpath'), '')); $temp->add(new admin_setting_configexecutable('pathtodot', new lang_string('pathtodot', 'admin'), new lang_string('pathtodot_help', 'admin'), '')); $temp->add(new admin_setting_configexecutable('pathtogs', new lang_string('pathtogs', 'admin'), new lang_string('pathtogs_help', 'admin'), '/usr/bin/gs')); +$temp->add(new admin_setting_configexecutable('pathtopandoc', new lang_string('pathtopandoc', 'admin'), new lang_string('pathtopandoc_help', 'admin'), '/usr/bin/pandoc')); $ADMIN->add('server', $temp); diff --git a/config-dist.php b/config-dist.php index cdf3de959cf..894e1df24c5 100644 --- a/config-dist.php +++ b/config-dist.php @@ -837,6 +837,12 @@ $CFG->admin = 'admin'; // Note that, for now, this only used by the profiling features // (Development->Profiling) built into Moodle. // $CFG->pathtodot = ''; +// +// Path to pandoc. +// Probably something like /usr/bin/pandoc. Used to convert between document formats. +// It is recommended to install the latest stable release of pandoc. +// Download packages for all platforms are available from http://pandoc.org/ +// $CFG->pathtopandoc = ''; //========================================================================= // ALL DONE! To continue installation, visit your main page with a browser diff --git a/lang/en/admin.php b/lang/en/admin.php index fb392c17989..5b64a22be08 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -779,6 +779,8 @@ $string['passwordpolicy'] = 'Password policy'; $string['passwordresettime'] = 'Maximum time to validate password reset request'; $string['passwordreuselimit'] = 'Password rotation limit'; $string['passwordreuselimit_desc'] = 'Number of times a user must change their password before they are allowed to reuse a password. Hashes of previously used passwords are stored in local database table. This feature might not be compatible with some external authentication plugins.'; +$string['pathtopandoc'] = 'Path to pandoc document converter'; +$string['pathtopandoc_help'] = 'Path to pandoc document converter. This is an executable that is capable of converting between document formats. This is optional, but if specified, Moodle will be able to perform automated conversion of documents from a wide range of file formats. This is used by the Assignment module "Annotate PDF" feature.'; $string['pathtodot'] = 'Path to dot'; $string['pathtodot_help'] = 'Path to dot. Probably something like /usr/bin/dot. To be able to generate graphics from DOT files, you must have installed the dot executable and point to it here. Note that, for now, this only used by the profiling features (Development->Profiling) built into Moodle.'; $string['pathtodu'] = 'Path to du'; diff --git a/lib/behat/lib.php b/lib/behat/lib.php index 1ab06c2a957..91b46f3fdee 100644 --- a/lib/behat/lib.php +++ b/lib/behat/lib.php @@ -166,7 +166,7 @@ function behat_clean_init_config() { 'umaskpermissions', 'dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'prefix', 'dboptions', 'proxyhost', 'proxyport', 'proxytype', 'proxyuser', 'proxypassword', 'proxybypass', 'theme', 'pathtogs', 'pathtodu', 'aspellpath', 'pathtodot', 'skiplangupgrade', - 'altcacheconfigpath' + 'altcacheconfigpath', 'pathtopandoc' )); // Add extra allowed settings. diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index 92908142e6a..2503b4a1fbc 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -156,6 +156,128 @@ class file_storage { return $storedfile; } + /** + * Get converted document. + * + * Get an alternate version of the specified document, if it is possible to convert. + * + * @param stored_file $file the file we want to preview + * @param string $format The desired format - e.g. 'pdf'. Formats are specified by file extension. + * @return stored_file|bool false if unable to create the conversion, stored file otherwise + */ + public function get_converted_document(stored_file $file, $format) { + + $context = context_system::instance(); + $path = '/' . $format . '/'; + $conversion = $this->get_file($context->id, 'core', 'documentconversion', 0, $path, $file->get_contenthash()); + + if (!$conversion) { + $conversion = $this->create_converted_document($file, $format); + if (!$conversion) { + return false; + } + } + + return $conversion; + } + + /** + * Verify the format is supported. + * + * @param string $format The desired format - e.g. 'pdf'. Formats are specified by file extension. + * @return bool - True if the format is supported for input. + */ + protected function is_input_format_supported_by_pandoc($format) { + $sanitized = trim(strtolower($format)); + return in_array($sanitized, array('md', 'html', 'tex', 'docx', 'odt', 'epub', 'png', 'jpg', 'gif')); + } + + /** + * Verify the format is supported. + * + * @param string $format The desired format - e.g. 'pdf'. Formats are specified by file extension. + * @return bool - True if the format is supported for output. + */ + protected function is_output_format_supported_by_pandoc($format) { + $sanitized = trim(strtolower($format)); + return in_array($sanitized, array('md', 'pdf', 'html', 'tex', 'docx', 'odt', 'odf', 'epub')); + } + + /** + * Perform a file format conversion on the specified document. + * + * @param stored_file $file the file we want to preview + * @param string $format The desired format - e.g. 'pdf'. Formats are specified by file extension. + * @return stored_file|bool false if unable to create the conversion, stored file otherwise + */ + protected function create_converted_document(stored_file $file, $format) { + global $CFG; + + if (empty($CFG->pathtopandoc) || !is_executable(trim($CFG->pathtopandoc))) { + // No conversions are possible, sorry. + return false; + } + + $fileextension = strtolower(pathinfo($file->get_filename(), PATHINFO_EXTENSION)); + if (!self::is_input_format_supported_by_pandoc($fileextension)) { + return false; + } + + if (!self::is_output_format_supported_by_pandoc($format)) { + return false; + } + + // Copy the file to the local tmp dir. + $tmp = make_request_directory(); + $localfilename = $file->get_filename(); + // Safety. + $localfilename = clean_param($localfilename, PARAM_FILE); + + $filename = $tmp . '/' . $localfilename; + $file->copy_content_to($filename); + + if (in_array($fileextension, array('gif', 'jpg', 'png'))) { + // We wrap images in a tiny html file - pandoc will generate documents from them. + $htmlwrapperfile = $tmp . '/wrapper.html'; + + file_put_contents($htmlwrapperfile, ""); + + $filename = $htmlwrapperfile; + } + + $newtmpfile = pathinfo($filename, PATHINFO_FILENAME) . '.' . $format; + + // Safety. + $newtmpfile = $tmp . '/' . clean_param($newtmpfile, PARAM_FILE); + + $cmd = escapeshellcmd(trim($CFG->pathtopandoc)) . ' ' . + escapeshellarg('-o') . ' ' . + escapeshellarg($newtmpfile) . ' ' . + escapeshellarg($filename); + + $e = file_exists($filename); + $output = null; + $currentdir = getcwd(); + chdir($tmp); + $result = exec($cmd, $output); + chdir($currentdir); + if (!file_exists($newtmpfile)) { + return false; + } + + $context = context_system::instance(); + $record = array( + 'contextid' => $context->id, + 'component' => 'core', + 'filearea' => 'documentconversion', + 'itemid' => 0, + 'filepath' => '/' . $format . '/', + 'filename' => $file->get_contenthash(), + ); + + return $this->create_file_from_pathname($record, $newtmpfile); + } + /** * Returns an image file that represent the given stored file as a preview * @@ -2282,6 +2404,26 @@ class file_storage { $rs->close(); mtrace('done.'); + // remove orphaned converted files (that is files in the core documentconversion filearea without + // the existing original file) + mtrace('Deleting orphaned document conversion files... ', ''); + cron_trace_time_and_memory(); + $sql = "SELECT p.* + FROM {files} p + LEFT JOIN {files} o ON (p.filename = o.contenthash) + WHERE p.contextid = ? AND p.component = 'core' AND p.filearea = 'documentconversion' AND p.itemid = 0 + AND o.id IS NULL"; + $syscontext = context_system::instance(); + $rs = $DB->get_recordset_sql($sql, array($syscontext->id)); + foreach ($rs as $orphan) { + $file = $this->get_file_instance($orphan); + if (!$file->is_directory()) { + $file->delete(); + } + } + $rs->close(); + mtrace('done.'); + // remove trash pool files once a day // if you want to disable purging of trash put $CFG->fileslastcleanup=time(); into config.php if (empty($CFG->fileslastcleanup) or $CFG->fileslastcleanup < time() - 60*60*24) { diff --git a/lib/phpunit/bootstrap.php b/lib/phpunit/bootstrap.php index 16bad007321..39237783a42 100644 --- a/lib/phpunit/bootstrap.php +++ b/lib/phpunit/bootstrap.php @@ -185,7 +185,8 @@ $CFG->dboptions = isset($CFG->phpunit_dboptions) ? $CFG->phpunit_dboptions : $CF $allowed = array('wwwroot', 'dataroot', 'dirroot', 'admin', 'directorypermissions', 'filepermissions', 'dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'prefix', 'dboptions', 'proxyhost', 'proxyport', 'proxytype', 'proxyuser', 'proxypassword', 'proxybypass', // keep proxy settings from config.php - 'altcacheconfigpath', 'pathtogs', 'pathtodu', 'aspellpath', 'pathtodot' + 'altcacheconfigpath', 'pathtogs', 'pathtodu', 'aspellpath', 'pathtodot', + 'pathtopandoc' ); $productioncfg = (array)$CFG; $CFG = new stdClass(); diff --git a/mod/assign/feedback/editpdf/classes/document_services.php b/mod/assign/feedback/editpdf/classes/document_services.php index 48556c166fd..83684e3fad5 100644 --- a/mod/assign/feedback/editpdf/classes/document_services.php +++ b/mod/assign/feedback/editpdf/classes/document_services.php @@ -24,6 +24,8 @@ namespace assignfeedback_editpdf; +use DOMDocument; + /** * Functions for generating the annotated pdf. * @@ -40,6 +42,8 @@ class document_services { const FINAL_PDF_FILEAREA = 'download'; /** File area for combined pdf */ const COMBINED_PDF_FILEAREA = 'combined'; + /** File area for importing html */ + const IMPORT_HTML_FILEAREA = 'importhtml'; /** File area for page images */ const PAGE_IMAGE_FILEAREA = 'pages'; /** File area for readonly page images */ @@ -84,6 +88,32 @@ class document_services { return sha1($assignmentid . '_' . $userid . '_' . $attemptnumber); } + /** + * Use a DOM parser to accurately replace images with their alt text. + * @param string $html + * @return string New html with no image tags. + */ + protected static function strip_images($html) { + $dom = new DOMDocument(); + $dom->loadHTML($html); + $images = $dom->getElementsByTagName('img'); + $i = 0; + + for ($i = ($images->length - 1); $i >= 0; $i--) { + $node = $images->item($i); + + if ($node->hasAttribute('alt')) { + $replacement = ' [ ' . $node->getAttribute('alt') . ' ] '; + } else { + $replacement = ' '; + } + + $text = $dom->createTextNode($replacement); + $node->parentNode->replaceChild($text, $node); + } + return $dom->saveHTML(); + } + /** * This function will search for all files that can be converted * and concatinated into a PDF (1.4) - for any submission plugin @@ -116,13 +146,38 @@ class document_services { if (!$submission) { return $files; } + + $fs = get_file_storage(); // Ask each plugin for it's list of files. foreach ($assignment->get_submission_plugins() as $plugin) { if ($plugin->is_enabled() && $plugin->is_visible()) { $pluginfiles = $plugin->get_files($submission, $user); foreach ($pluginfiles as $filename => $file) { - if (($file instanceof \stored_file) && ($file->get_mimetype() === 'application/pdf')) { - $files[$filename] = $file; + if ($file instanceof \stored_file) { + if ($file->get_mimetype() === 'application/pdf') { + $files[$filename] = $file; + } else if ($convertedfile = $fs->get_converted_document($file, 'pdf')) { + $files[$filename] = $convertedfile; + } + } else { + // Create a tmp stored_file from this html string. + $file = reset($file); + // Strip image tags, because they will not be resolvable. + $file = self::strip_images($file); + $record = new \stdClass(); + $record->contextid = $assignment->get_context()->id; + $record->component = 'assignfeedback_editpdf'; + $record->filearea = self::IMPORT_HTML_FILEAREA; + $record->itemid = $submission->id; + $record->filepath = '/'; + $record->filename = $plugin->get_type() . '-' . $filename; + + $htmlfile = $fs->create_file_from_string($record, $file); + $convertedfile = $fs->get_converted_document($htmlfile, 'pdf'); + $htmlfile->delete(); + if ($convertedfile) { + $files[$filename] = $convertedfile; + } } } } From 128d8736d3e76a27052ccc821a9ae5becb28de5c Mon Sep 17 00:00:00 2001 From: Damyon Wiese Date: Mon, 22 Feb 2016 10:14:04 +0800 Subject: [PATCH 2/9] MDL-52954 core: Unit tests for pandoc document converter. --- lib/tests/fixtures/pandoc-source.docx | Bin 0 -> 11065 bytes lib/tests/fixtures/pandoc-source.html | 12 +++ lib/tests/pandoc_test.php | 104 ++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 lib/tests/fixtures/pandoc-source.docx create mode 100644 lib/tests/fixtures/pandoc-source.html create mode 100644 lib/tests/pandoc_test.php diff --git a/lib/tests/fixtures/pandoc-source.docx b/lib/tests/fixtures/pandoc-source.docx new file mode 100644 index 0000000000000000000000000000000000000000..286c58a89265855790206b771ad41b8960e96e62 GIT binary patch literal 11065 zcmeHtWmjFxvi8Cq7VZwgf&|y#?(S~E-6cS9_h1Pw!QI^nPH=)laCg5eIrog6owLXL z{(yVthdI_9Yd$rrtE;-IpDqPy2uKV7GyoO=0FVNfACoxDzyJUeC;$K*01K`yY-j6i zYU`}8;$d&ZM&I(t?K*yBJFBn*&)Tnumty< z^%D8S28T!*SJKz#_YbzL)1Z$nZwd3tZjv;ZlA>(Zi9&1K}i#yiBN5eHA)shEF0370sGmk1(KzzzUI_fLD+b3%X{9&1Z!+%m8her~$^RJU$EEukg- z+@M{`Jdw}Z%C(quwMz&5#pLV#0{8DRwZAiXPH{!QrI7RDN+;eQrBmS2L?y`yc6AvL zhY_hhMC&<;ZRxjD@=J7E8?AX?4}&x!`8C`(EGz|&GyO#UTx&OhOpVE>BjsVWvn8K35REJ1QEm+hgSM-hvX#A6*u@5v&3>k_e+La1D3^ic{NKJx<3}xfn1F#NLG2&rz36$K zvI7U35(Uc39suEO&)L-!wdwV~5AecDA<#D)5e8!Kek9bUAGz7}QbF%$sK!##z}DuM za4%82GrGEslZ4il)jD8dB1%+_+S##Jyki{@o_75#B2$%@WqNHz^ToPqE|D{a{dk(; zGA$P|>azK90_m%2+S?(fQNKqrD~FXOH*%SpEzUyWmuD^vyg;paG(1weepyl$?(Q^I0Sj>x-drh@%v!JcN3HpzA97(IhR4MRORSlb=rVI@hdHEQHnVF6rI~K3 zEezHtkz0mf7|wvQIrmJwCm5_?1co=8pSn_b9bOe=Au<|dcloUdGd)OK03pLh;o{Za z;n>1q8t9`1XTXeT!lX?kVt;2cK?9fQXAJXC%*%i$OL7(>2;EALeB5kArKL=k?pF>e zJ$XGug9oO407JGbrv#p+RjuA696$^_ACAeyh|v$*qxvyiI#X$xd7o)k_I!R(dd1t# zm2)S3`y^RjWT`keW~jJt=-*dC?8q8JmD^Zc9JN5S+>SqRv1>C>inOfMU#Z^9USYWI zBBo4H+OiE58KG@P?45KcQ^k9B?B&nQDYzawKmOxDGUKI_JC){RAyx4MVBqZCWii7l zQfGQ`F_(^-D(mX)wmqq%($x}z$YDA;DL%=VI=s7yf<`xG*P>L)x5W-vZ)s&!jHVqe z$rmY{%dy0hyjFCL;KO1by>`=s)wGA^y#xJfHrsRQidE5ui~l+1=U*B&3F^hC5KlXeR_)Q(6N+T& zr51A6#}+K-S%L1Phzyp+)vHRBz?1otv_k5kra;UF#5RI~nOA{%V!4)pgJ7JUMJuic zlz&PP$#rEL$`wlk)3#e;DP zpRY=%m&Nb34I#1!OP*KA5tx$n!C&0;1M6uv48feY2{F|6$C}T7U|rz0CY-3K6jzGx z)6k1&pARp4OM!N``S3zS{H0{>cmRnXCT^uJ>Snbid`?xwD8RsnY|ddRB2jAYV{#r!kA~h48c74&5z$D@)^K|o;4SQK zzqLh^nznBE8(H4OHJrf}yqGWY6AB5J?2a3J&=q0vo}K1KS(51HL0!-=E>W?{3#Qt?ap1)>{l*tu- zvo@{P$bmK?#&^1{TeLKGTIj-$?c>-vjT$p+XxU#w8r)FdIenrln+cBK$O@g@GC(dP zTs$$KMi@+~Ig+}?>zuR1^$rerN1Y>GgL5H5A6|#Ex|xFlKr+#og#2P5)juCXeZ1jL zK|PCj%H5TZ%d<7yL^_HfsXw7o*M_2ui2Ek@X8-P{;xoVK0Lo`V{@cCV^`1=_el4M~ zVw;S>ZB@pOr^E39;#~gEz8B|{*fZ{${V(@DR2`3x9&ErV^H*aLjG2$uheHt+Z=ar8 z#4%50{l!ghWBWyN(zjrbE^shd)Ep?O$Q{}-dtkqvhmTSua(1OjLp^;CC%7D?F-vtL z;NtGBTO$P9&V_gPaU4f=2Zn{^U}m`G%!3^}j&bX#Xc>xM(58BDN_JcRFjDG2GWOQ! zYl%P~=xCFKcVR)17C*BUA$W6xBZq-3DTFcQ^yn80e{`(0KkxGKK0x}d+Af1Ap$wfY za~?TWnps4>1)^`N+8*Mwt96YRj0Tv`cDM?7a z5G6%YgIy}OraJPpr_K^M;%$Bsnsq;t0Y1`lt6d6Jw9Df%R* z;!p4d-?mP^zR6XHu|d?);R*)7-8dM)jJvAG+K`=3C}uW%5V_M0a}VW)R&trR9w?uWMgI*4SEW&-yh zG7%!!HkN@b>0l&9?mn{L`~d-q-70o%pS(CnFi`nj<-zAyXkR0psTIFc5*&`$Q`$s@ zywY7THdH3YvF3}_I6Ch{uGn3F)KQ3i(<1v0Ig=|FQ+BO%M6KLJMDS?NP;E}=Q>ylI zsF~EKo-zeoZ_WxlM6B6BXh^{&a5cT4izYK981xZ|tRvf-mGAWZ2-tgMxa)oK!R zABt=<5(mPY;QZ#loW{M~`eamx){y+_0s8ycMjmrZKe%ht%}TGd2jqBb-o>>DcXeJy z#%n=wn~E77ThGA(izWP?R4k%Rj8+Dh@eoZ|Y~gk^_CtADvxIFHyL!jI?-WHnY(aGo z8QpS(5ROIW)HKKvtcg%0erDB_le)XdhEhW>fHeBZE@+?)vN6A*Y!| zV2xvmGV8br0*8mrpuvjNhQDo6HU9QkMY!Z0f>J&~{|k%ir7t)K6rKOIAIe-KIQgdv zn-~O4haw>hk*9XQ{;p}{P^`v?APZR9Mj7@OD6Z)jn=wtiS7qy+h1V9LkU>upq5-jd z)Y7l+f8~qfI?IEpkX?~*J4zj9R|vE$VmEZo zeY_2CEJLiTJ!Dh@VQp8Qq~j4%|{>+COGHB`zN!TooF{ha;YQ?&10HgdhzcIg{eGnpu5nYuVA{j;wKQh5Eg8VE+G zqB=ln>OXfn@k;PI+@NZ87Us8V)ydS^*}~S`>1PtlP+f9NX9D`1$h`QkH${U%)LUdk zGCD9O#%BX$M8zZ+e?*^szM6S(`#2zDY}_g1MbhQ+%IsP1Oa>d%OacXm zCUL^Z3OdLjb>3Fee<^0wvnW*`m%L`Z(xn-Q=A^I$-xoWHEp!wdwIhY$72Jd?CK!v1 zSuiNj*-WOy!*<m0mZ3P(I5<3L^b>{_)uS(M#TW`7Nze9H5f9qjXDQlJSO`!+0Rlo zA1Wh=g>ZJTAP-paMsxdnstM6$;yu2L-C)t5p%LtD~3D1jqjr@a3`tl1WwXSzfhgW-gUh^0bDp14L5x(lSb-`cH zHIzR~_d1Hkx|yfrNQId;dFlX99DVe+jSouJ>~nWM`WIFA~25}P+ z4bg)3WxzAxBquq-64^DmgrlgE1LDBd5GjpPqb>p&pUZmMu&QSRS(+f(S!2I)+K4Hv zGAEF~o7!1A!VW5bBXoEiIpgj{Ko-q8h$7gn?gANJ#)NGGS>T9}`4KV!^DcEm&io7pPY9T(KA^yied&HqJ)8oyD`1z!q(yN^5<6NjE}uX<7U_yJ;Sp6O zt8uJ@8$w#pq~J%W3@@u^Rthd2bf^@VOa46q>$dO(l3J*LBWxVq{zM9!-yC3W3 zCLQg{u`i7JK8#DPNf1q|hBk?x?`+{aePcjbim*SD`Yi@9xVuW>}I6_y>7KD19-1aTi&N4Pml3Bt&ef zE5ahy2}{7%1XxUe@r$!v858fjCv&pt{@_g;vgPd$B3M~Pc=oyYY+&*^ug*0%h^Wro zI53=SYQuE>QEucAq1kD%5zvs%$)JXZabun_R3q*Gy_Mdqfo&cTo zKV30NNhne0GEQ89>4;k(R+&)PtXyBRS$|yi#K^ooJ3-O6uz9P<%8YbcZIdks&nw!D zWbeVxcap7m!Frbj2XiYH)N@%l!zXgK^x&z{afgr96L{%s_+SxlX8rs|cbW0E^{9k?xWVs9_ zS$WWew~?O_7q4h?L&C;l2eOkQdrQ*UN24`FNhDAY0UvHSoe#nr;}SBDT6y_lY}-AR zy^TT_7eu)-RZacXxbtmVsPzw85{9coB{e7`YZ|E=aE9@Hy&)O9JP;*~Ux#-0i!S7E zRp3Tc_vrQFCef?nVSC2G1}8TaOrojF*?QK7eNvl&>Zu4+%jP~KH{@c{dtdMpPfJzP zeye832lqq4_#{eVFteI~MVaBMv;Xxp1Dx^!(H(BSIDj#yZ*ZqG7_g$N5I8>Ok3{;K z2Q^S;i=?Bp$RS{T)+;w(VvGL$G;`x_nhNhrqr%ohX92yySgE3IfSk4-Sh|q=L~^1I z03TclO9Vgg#PwVZ(uxL=!NPk579Dk-6ij-~%do+F8?h{F&|Y{kA}msZ<*@lYm-uXE z&NiuSY4S-<4$*z?i?$c3=t@D{I9cz+_8Zf)e1$@(stpjnk7|~q zg<3$#{sa(1c$WjE8enWox8J@;~V3O8oikkQ!)g7 z_ieVru^)jCb8*(rR;Bnˇf5dz;xHOL_*nqznGZ#8{USV{{H=7i|u)qO40zs z-BkZEWl}-X%3TX?BpczkK0aeIs|~ap(e>MX0ZlyK1~>Y$o>*(61yLol2}hK9pJjy8 z4yZV$uN{e|XfBLhUKIGx!?`=g;8QX6Eop`WlAuN9K-O~Y0ZAT#33Zb-jEvvl3_17Y3`d9^#)d&&XWQ?|XX@b0bgXaK14- z4-O&djV6028eb}RePe7+`jN~RyW9kD^TV(f15QWm=3(9YnCqO(c`6?<{zC2_r&BOQ$?hnMh&p>?t%J)% zf5)Egs}Ou$bl}z#|%#bt0F%z_u3H$(z~X zRVC#u(YTv$!!MY4R}`!Fqogp|+7)Ti`Z;LXrf$XZDezW{R0KBhlL9%~6^HCVKb()f zdr&2GwrTM;tYfjR%Rb&JwRI%;-P3D6F6w zwm=K&^4W|Oa8r}MnaS{O(xDeQ?20Zh+=rnrd{!Irws^wyuqB_)W@wj^X?(M=*lp{* zJ}A1|!dz{;T{A|#`Bah3Hq#O*wxd;Wwb-IhG3)G4-+*eZ!m@Y8l6|D_^=7PVtdF%A zE`NxVCL}klxs>Bsv{|otq%gP=AJ^iod0rEwKg03o9EhZx{e8iqP z*@3md`H1;>s<_^iwg8UZZ`>{fx5H6Ng@-z&9*O?lav@?}ePS^|pAylIu5eMS&{ewe zm?{??=pjO7oB6kL3!l$=ZJ-CI%qkWdGeX!{0Q8}?A;z{?rkOWo89ZW1YML z(5>1lsSHv%^q0x&s7}mcj|bEJ`hGz7eEFs7UKarKr(J$PyNqFtMa0JxUR#Cxd5)3r z&j>&hOjCrxr$F5bkC{SBy;|_*&Z9ISB88|b;4LEk9JKyHlxLo=lm^J0$Z^pIjB)N* zqxlp7G~`^#rf7U}we^_-Px*=I2^Z2izGVs(|4G%iT1OMYKX%H+ z6pds$+Xq?@2pb_#_=*fWiC=0TVmzmu7X5Kez5$oX7iJ+no&Vbbq0V*m_&9SEx|Ypp z56KH*`90=D-7`#;8Mun{Y9cJ0t`<*(8w97Wt`owg2Fe{hlyf^U?(^I*)iyRT-xftJL zn3W0mH*6s!?L9GNjyz{hA>ZYeTSq9|X|M3ZTcSAT!g<(Q)xM8j9%ded61ZTo-Z*k{ zw{rJy#~ATZ7y_Hu_Xe&i@BM*f{4G;mSQy^ocM^hKA;v(iV8l!|mn{LbAc$%nDLw?Y z7NhZxaIIOh!uhk7kT^J=byCcv{hlr5nM<6=h{9JZR8r0YX9PrvzJ1zl1oVjvyELWu zuu1}##Y2)#!6=z>X6+1Yx9406R`_{&8dYgx-_5f}yPur)9`qrx%+SjBiLbVG=w+Fv zPu5pTOf0cpNj8?g=NGaNDg#T|&SAM%f>Pvv5^T;Gf~^hWQ9AvY=zsDA*%qMXk#16i zqIHmmFBAfZO({axKUslR$BzySuFJ=8#urxSCY2Cb=!+$B5vuhJZ^c6jE==X*$Q_(X zjc_qiiekw(8Axs1pX=?PQV9H_p+pJ&1-f0I@M@f4x|2N~Oy7Xa>A|Xf6xOwobcnB0 zO3kE_W=K`T-t&@6GAMJ;hPegzq@DE17HIn(F;P1eVVlb~%s$|&vy~Qb>@It3 zaQ;USE%~o<2n?Jav={&99>ZTH?r*n$=r~l6{=0y`cR~Gz1!RIMu-|&3{sjKLHRN|- z59mDb|J5k+r=&j{1^$xt0xI?Xqix_%_@76Rf5Ep9{}=w}f#jbe{v?0@5>bZ!?;`Uz z%HvP?pOn^La4Zn5@@oKpBe?#=|4Fm_h404sU;H1W%bybd&RzV42LLK?0f0Z5j6czT z-#z}0wj%fq{qMcxpYT76^1lQqgDAjX!~d&9SCEDRxyR3rI|M)%h-opR{`vGj$C+`Q literal 0 HcmV?d00001 diff --git a/lib/tests/fixtures/pandoc-source.html b/lib/tests/fixtures/pandoc-source.html new file mode 100644 index 00000000000..45ad289a658 --- /dev/null +++ b/lib/tests/fixtures/pandoc-source.html @@ -0,0 +1,12 @@ + + + + +

My First Heading

+ +

My first paragraph.

+ + + + + diff --git a/lib/tests/pandoc_test.php b/lib/tests/pandoc_test.php new file mode 100644 index 00000000000..68fcdcc1202 --- /dev/null +++ b/lib/tests/pandoc_test.php @@ -0,0 +1,104 @@ +. + +/** + * Test pandoc functionality. + * + * @package core + * @category phpunit + * @copyright 2016 Damyon Wiese + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + + +/** + * A set of tests for some of the pandoc functionality within Moodle. + * + * @package core + * @category phpunit + * @copyright 2016 Damyon Wiese + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_pandoc_testcase extends advanced_testcase { + + private $testfile1 = null; + private $testfile2 = null; + + public function setUp() { + $this->fixturepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR; + + $fs = get_file_storage(); + $filerecord = array( + 'contextid' => context_system::instance()->id, + 'component' => 'test', + 'filearea' => 'unittest', + 'itemid' => 0, + 'filepath' => '/', + 'filename' => 'test.html' + ); + $teststring = file_get_contents($this->fixturepath . DIRECTORY_SEPARATOR . 'pandoc-source.html'); + $this->testfile1 = $fs->create_file_from_string($filerecord, $teststring); + + $filerecord = array( + 'contextid' => context_system::instance()->id, + 'component' => 'test', + 'filearea' => 'unittest', + 'itemid' => 0, + 'filepath' => '/', + 'filename' => 'test.docx' + ); + $teststring = file_get_contents($this->fixturepath . DIRECTORY_SEPARATOR . 'pandoc-source.docx'); + $this->testfile2 = $fs->create_file_from_string($filerecord, $teststring); + + $this->resetAfterTest(); + } + + public function test_generate_pdf() { + global $CFG; + + if (empty($CFG->pathtopandoc) || !is_executable(trim($CFG->pathtopandoc))) { + // No conversions are possible, sorry. + return $this->markTestSkipped(); + } + $fs = get_file_storage(); + + $result = $fs->get_converted_document($this->testfile1, 'pdf'); + $this->assertSame($result->get_mimetype(), 'application/pdf'); + $this->assertGreaterThan(0, $result->get_filesize()); + $result = $fs->get_converted_document($this->testfile2, 'pdf'); + $this->assertSame($result->get_mimetype(), 'application/pdf'); + $this->assertGreaterThan(0, $result->get_filesize()); + } + + public function test_generate_markdown() { + global $CFG; + + if (empty($CFG->pathtopandoc) || !is_executable(trim($CFG->pathtopandoc))) { + // No conversions are possible, sorry. + return $this->markTestSkipped(); + } + $fs = get_file_storage(); + + $result = $fs->get_converted_document($this->testfile1, 'md'); + $this->assertSame($result->get_mimetype(), 'text/plain'); + $this->assertGreaterThan(0, $result->get_filesize()); + $result = $fs->get_converted_document($this->testfile2, 'md'); + $this->assertSame($result->get_mimetype(), 'text/plain'); + $this->assertGreaterThan(0, $result->get_filesize()); + } +} From 1356d8515163e7d8ad4b1063179f12f9951d70c4 Mon Sep 17 00:00:00 2001 From: Damyon Wiese Date: Fri, 26 Feb 2016 11:52:40 +0800 Subject: [PATCH 3/9] MDL-52954 core: Change from pandoc to unoconv - it gives better results Most importantly it retains formatting better, and supports different charsets far better than pandoc. --- admin/settings/server.php | 2 +- config-dist.php | 10 ++-- lang/en/admin.php | 4 +- lib/behat/lib.php | 2 +- lib/filestorage/file_storage.php | 52 +++++++++--------- lib/phpunit/bootstrap.php | 2 +- ...pandoc-source.docx => unoconv-source.docx} | Bin ...pandoc-source.html => unoconv-source.html} | 0 .../{pandoc_test.php => unoconv_test.php} | 22 +++++--- .../editpdf/classes/document_services.php | 30 ++++++++-- 10 files changed, 75 insertions(+), 49 deletions(-) rename lib/tests/fixtures/{pandoc-source.docx => unoconv-source.docx} (100%) rename lib/tests/fixtures/{pandoc-source.html => unoconv-source.html} (100%) rename lib/tests/{pandoc_test.php => unoconv_test.php} (85%) diff --git a/admin/settings/server.php b/admin/settings/server.php index 07b7c7f895d..548031bb771 100644 --- a/admin/settings/server.php +++ b/admin/settings/server.php @@ -12,7 +12,7 @@ $temp->add(new admin_setting_configexecutable('pathtodu', new lang_string('patht $temp->add(new admin_setting_configexecutable('aspellpath', new lang_string('aspellpath', 'admin'), new lang_string('edhelpaspellpath'), '')); $temp->add(new admin_setting_configexecutable('pathtodot', new lang_string('pathtodot', 'admin'), new lang_string('pathtodot_help', 'admin'), '')); $temp->add(new admin_setting_configexecutable('pathtogs', new lang_string('pathtogs', 'admin'), new lang_string('pathtogs_help', 'admin'), '/usr/bin/gs')); -$temp->add(new admin_setting_configexecutable('pathtopandoc', new lang_string('pathtopandoc', 'admin'), new lang_string('pathtopandoc_help', 'admin'), '/usr/bin/pandoc')); +$temp->add(new admin_setting_configexecutable('pathtounoconv', new lang_string('pathtounoconv', 'admin'), new lang_string('pathtounoconv_help', 'admin'), '/usr/bin/unoconv')); $ADMIN->add('server', $temp); diff --git a/config-dist.php b/config-dist.php index 894e1df24c5..ae794095569 100644 --- a/config-dist.php +++ b/config-dist.php @@ -838,11 +838,11 @@ $CFG->admin = 'admin'; // (Development->Profiling) built into Moodle. // $CFG->pathtodot = ''; // -// Path to pandoc. -// Probably something like /usr/bin/pandoc. Used to convert between document formats. -// It is recommended to install the latest stable release of pandoc. -// Download packages for all platforms are available from http://pandoc.org/ -// $CFG->pathtopandoc = ''; +// Path to unoconv. +// Probably something like /usr/bin/unoconv. Used as a fallback to convert between document formats. +// Unoconv is used convert between file formats supported by LibreOffice. +// Use a recent version of unoconv ( >= 0.7 ), older versions have trouble running from a webserver. +// $CFG->pathtounoconv = ''; //========================================================================= // ALL DONE! To continue installation, visit your main page with a browser diff --git a/lang/en/admin.php b/lang/en/admin.php index 5b64a22be08..9455d539064 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -779,8 +779,6 @@ $string['passwordpolicy'] = 'Password policy'; $string['passwordresettime'] = 'Maximum time to validate password reset request'; $string['passwordreuselimit'] = 'Password rotation limit'; $string['passwordreuselimit_desc'] = 'Number of times a user must change their password before they are allowed to reuse a password. Hashes of previously used passwords are stored in local database table. This feature might not be compatible with some external authentication plugins.'; -$string['pathtopandoc'] = 'Path to pandoc document converter'; -$string['pathtopandoc_help'] = 'Path to pandoc document converter. This is an executable that is capable of converting between document formats. This is optional, but if specified, Moodle will be able to perform automated conversion of documents from a wide range of file formats. This is used by the Assignment module "Annotate PDF" feature.'; $string['pathtodot'] = 'Path to dot'; $string['pathtodot_help'] = 'Path to dot. Probably something like /usr/bin/dot. To be able to generate graphics from DOT files, you must have installed the dot executable and point to it here. Note that, for now, this only used by the profiling features (Development->Profiling) built into Moodle.'; $string['pathtodu'] = 'Path to du'; @@ -792,6 +790,8 @@ $string['pathtopgdumpinvalid'] = 'Invalid path to pg_dump - either wrong path or $string['pathtopsql'] = 'Path to psql'; $string['pathtopsqldesc'] = 'This is only necessary to enter if you have more than one psql on your system (for example if you have more than one version of postgresql installed)'; $string['pathtopsqlinvalid'] = 'Invalid path to psql - either wrong path or not executable'; +$string['pathtounoconv'] = 'Path to unoconv document converter'; +$string['pathtounoconv_help'] = 'Path to unoconv document converter. This is an executable that is capable of converting between document formats supported by LibreOffice. This is optional, but if specified, Moodle will use it to automatically convert between document formats. This is used to support a wider range of input files for the assignment annotate PDF feature.'; $string['pcreunicodewarning'] = 'It is strongly recommended to use PCRE PHP extension that is compatible with Unicode characters.'; $string['perfdebug'] = 'Performance info'; $string['performance'] = 'Performance'; diff --git a/lib/behat/lib.php b/lib/behat/lib.php index 91b46f3fdee..201831fcda5 100644 --- a/lib/behat/lib.php +++ b/lib/behat/lib.php @@ -166,7 +166,7 @@ function behat_clean_init_config() { 'umaskpermissions', 'dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'prefix', 'dboptions', 'proxyhost', 'proxyport', 'proxytype', 'proxyuser', 'proxypassword', 'proxybypass', 'theme', 'pathtogs', 'pathtodu', 'aspellpath', 'pathtodot', 'skiplangupgrade', - 'altcacheconfigpath', 'pathtopandoc' + 'altcacheconfigpath', 'pathtounoconv' )); // Add extra allowed settings. diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index 2503b4a1fbc..6034798e99a 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -187,21 +187,29 @@ class file_storage { * @param string $format The desired format - e.g. 'pdf'. Formats are specified by file extension. * @return bool - True if the format is supported for input. */ - protected function is_input_format_supported_by_pandoc($format) { + protected function is_format_supported_by_unoconv($format) { + global $CFG; + + if (!isset($this->unoconvformats)) { + // Ask unoconv for it's list of supported document formats. + $cmd = escapeshellcmd(trim($CFG->pathtounoconv)) . ' --show'; + $pipes = array(); + $pipesspec = array(2 => array('pipe', 'w')); + $proc = proc_open($cmd, $pipesspec, $pipes); + $programoutput = stream_get_contents($pipes[2]); + fclose($pipes[2]); + proc_close($proc); + $matches = array(); + preg_match_all('/\[\.(.*)\]/', $programoutput, $matches); + + $this->unoconvformats = $matches[1]; + $this->unoconvformats = array_unique($this->unoconvformats); + } + $sanitized = trim(strtolower($format)); - return in_array($sanitized, array('md', 'html', 'tex', 'docx', 'odt', 'epub', 'png', 'jpg', 'gif')); + return in_array($sanitized, $this->unoconvformats); } - /** - * Verify the format is supported. - * - * @param string $format The desired format - e.g. 'pdf'. Formats are specified by file extension. - * @return bool - True if the format is supported for output. - */ - protected function is_output_format_supported_by_pandoc($format) { - $sanitized = trim(strtolower($format)); - return in_array($sanitized, array('md', 'pdf', 'html', 'tex', 'docx', 'odt', 'odf', 'epub')); - } /** * Perform a file format conversion on the specified document. @@ -213,17 +221,17 @@ class file_storage { protected function create_converted_document(stored_file $file, $format) { global $CFG; - if (empty($CFG->pathtopandoc) || !is_executable(trim($CFG->pathtopandoc))) { + if (empty($CFG->pathtounoconv) || !is_executable(trim($CFG->pathtounoconv))) { // No conversions are possible, sorry. return false; } $fileextension = strtolower(pathinfo($file->get_filename(), PATHINFO_EXTENSION)); - if (!self::is_input_format_supported_by_pandoc($fileextension)) { + if (!self::is_format_supported_by_unoconv($fileextension)) { return false; } - if (!self::is_output_format_supported_by_pandoc($format)) { + if (!self::is_format_supported_by_unoconv($format)) { return false; } @@ -236,21 +244,14 @@ class file_storage { $filename = $tmp . '/' . $localfilename; $file->copy_content_to($filename); - if (in_array($fileextension, array('gif', 'jpg', 'png'))) { - // We wrap images in a tiny html file - pandoc will generate documents from them. - $htmlwrapperfile = $tmp . '/wrapper.html'; - - file_put_contents($htmlwrapperfile, ""); - - $filename = $htmlwrapperfile; - } - $newtmpfile = pathinfo($filename, PATHINFO_FILENAME) . '.' . $format; // Safety. $newtmpfile = $tmp . '/' . clean_param($newtmpfile, PARAM_FILE); - $cmd = escapeshellcmd(trim($CFG->pathtopandoc)) . ' ' . + $cmd = escapeshellcmd(trim($CFG->pathtounoconv)) . ' ' . + escapeshellarg('-f') . ' ' . + escapeshellarg($format) . ' ' . escapeshellarg('-o') . ' ' . escapeshellarg($newtmpfile) . ' ' . escapeshellarg($filename); @@ -259,6 +260,7 @@ class file_storage { $output = null; $currentdir = getcwd(); chdir($tmp); + $result = exec('env 1>&2', $output); $result = exec($cmd, $output); chdir($currentdir); if (!file_exists($newtmpfile)) { diff --git a/lib/phpunit/bootstrap.php b/lib/phpunit/bootstrap.php index 39237783a42..aff8506868e 100644 --- a/lib/phpunit/bootstrap.php +++ b/lib/phpunit/bootstrap.php @@ -186,7 +186,7 @@ $allowed = array('wwwroot', 'dataroot', 'dirroot', 'admin', 'directorypermission 'dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'prefix', 'dboptions', 'proxyhost', 'proxyport', 'proxytype', 'proxyuser', 'proxypassword', 'proxybypass', // keep proxy settings from config.php 'altcacheconfigpath', 'pathtogs', 'pathtodu', 'aspellpath', 'pathtodot', - 'pathtopandoc' + 'pathtounoconv' ); $productioncfg = (array)$CFG; $CFG = new stdClass(); diff --git a/lib/tests/fixtures/pandoc-source.docx b/lib/tests/fixtures/unoconv-source.docx similarity index 100% rename from lib/tests/fixtures/pandoc-source.docx rename to lib/tests/fixtures/unoconv-source.docx diff --git a/lib/tests/fixtures/pandoc-source.html b/lib/tests/fixtures/unoconv-source.html similarity index 100% rename from lib/tests/fixtures/pandoc-source.html rename to lib/tests/fixtures/unoconv-source.html diff --git a/lib/tests/pandoc_test.php b/lib/tests/unoconv_test.php similarity index 85% rename from lib/tests/pandoc_test.php rename to lib/tests/unoconv_test.php index 68fcdcc1202..6c266f7bdff 100644 --- a/lib/tests/pandoc_test.php +++ b/lib/tests/unoconv_test.php @@ -15,7 +15,7 @@ // along with Moodle. If not, see . /** - * Test pandoc functionality. + * Test unoconv functionality. * * @package core * @category phpunit @@ -27,14 +27,14 @@ defined('MOODLE_INTERNAL') || die(); /** - * A set of tests for some of the pandoc functionality within Moodle. + * A set of tests for some of the unoconv functionality within Moodle. * * @package core * @category phpunit * @copyright 2016 Damyon Wiese * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ -class core_pandoc_testcase extends advanced_testcase { +class core_unoconv_testcase extends advanced_testcase { private $testfile1 = null; private $testfile2 = null; @@ -51,7 +51,7 @@ class core_pandoc_testcase extends advanced_testcase { 'filepath' => '/', 'filename' => 'test.html' ); - $teststring = file_get_contents($this->fixturepath . DIRECTORY_SEPARATOR . 'pandoc-source.html'); + $teststring = file_get_contents($this->fixturepath . DIRECTORY_SEPARATOR . 'unoconv-source.html'); $this->testfile1 = $fs->create_file_from_string($filerecord, $teststring); $filerecord = array( @@ -62,7 +62,7 @@ class core_pandoc_testcase extends advanced_testcase { 'filepath' => '/', 'filename' => 'test.docx' ); - $teststring = file_get_contents($this->fixturepath . DIRECTORY_SEPARATOR . 'pandoc-source.docx'); + $teststring = file_get_contents($this->fixturepath . DIRECTORY_SEPARATOR . 'unoconv-source.docx'); $this->testfile2 = $fs->create_file_from_string($filerecord, $teststring); $this->resetAfterTest(); @@ -71,16 +71,18 @@ class core_pandoc_testcase extends advanced_testcase { public function test_generate_pdf() { global $CFG; - if (empty($CFG->pathtopandoc) || !is_executable(trim($CFG->pathtopandoc))) { + if (empty($CFG->pathtounoconv) || !is_executable(trim($CFG->pathtounoconv))) { // No conversions are possible, sorry. return $this->markTestSkipped(); } $fs = get_file_storage(); $result = $fs->get_converted_document($this->testfile1, 'pdf'); + $this->assertNotFalse($result); $this->assertSame($result->get_mimetype(), 'application/pdf'); $this->assertGreaterThan(0, $result->get_filesize()); $result = $fs->get_converted_document($this->testfile2, 'pdf'); + $this->assertNotFalse($result); $this->assertSame($result->get_mimetype(), 'application/pdf'); $this->assertGreaterThan(0, $result->get_filesize()); } @@ -88,16 +90,18 @@ class core_pandoc_testcase extends advanced_testcase { public function test_generate_markdown() { global $CFG; - if (empty($CFG->pathtopandoc) || !is_executable(trim($CFG->pathtopandoc))) { + if (empty($CFG->pathtounoconv) || !is_executable(trim($CFG->pathtounoconv))) { // No conversions are possible, sorry. return $this->markTestSkipped(); } $fs = get_file_storage(); - $result = $fs->get_converted_document($this->testfile1, 'md'); + $result = $fs->get_converted_document($this->testfile1, 'txt'); + $this->assertNotFalse($result); $this->assertSame($result->get_mimetype(), 'text/plain'); $this->assertGreaterThan(0, $result->get_filesize()); - $result = $fs->get_converted_document($this->testfile2, 'md'); + $result = $fs->get_converted_document($this->testfile2, 'txt'); + $this->assertNotFalse($result); $this->assertSame($result->get_mimetype(), 'text/plain'); $this->assertGreaterThan(0, $result->get_filesize()); } diff --git a/mod/assign/feedback/editpdf/classes/document_services.php b/mod/assign/feedback/editpdf/classes/document_services.php index 83684e3fad5..4ada0a79829 100644 --- a/mod/assign/feedback/editpdf/classes/document_services.php +++ b/mod/assign/feedback/editpdf/classes/document_services.php @@ -53,6 +53,30 @@ class document_services { /** Filename for combined pdf */ const COMBINED_PDF_FILENAME = 'combined.pdf'; + /** Base64 encoded blank pdf. This is the most reliable/fastest way to generate a blank pdf. */ + const BLANK_PDF_BASE64 = <<Output(self::COMBINED_PDF_FILENAME, 'S'); - $file = $fs->create_file_from_string($record, $content); - $blankpdf->Close(); // No real need to close this pdf, because it has been outputted, but for clarity. + $file = $fs->create_file_from_string($record, base64_decode(self::BLANK_PDF_BASE64)); } else { // This was a combined pdf. $file = $fs->create_file_from_pathname($record, $tmpfile); From b803df8170f0fca8eb6d5b79ee9443725f73aa28 Mon Sep 17 00:00:00 2001 From: Damyon Wiese Date: Fri, 19 Feb 2016 15:56:51 +0800 Subject: [PATCH 4/9] MDL-52954 atto: Stop autosave timer, when editor no-longer exists. --- .../moodle-editor_atto-editor-debug.js | 16 +++++++++++++++- .../moodle-editor_atto-editor-min.js | 6 +++--- .../moodle-editor_atto-editor.js | 16 +++++++++++++++- lib/editor/atto/yui/src/editor/js/autosave.js | 16 +++++++++++++++- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/lib/editor/atto/yui/build/moodle-editor_atto-editor/moodle-editor_atto-editor-debug.js b/lib/editor/atto/yui/build/moodle-editor_atto-editor/moodle-editor_atto-editor-debug.js index d5ed2a6f9d9..0332fd002f2 100644 --- a/lib/editor/atto/yui/build/moodle-editor_atto-editor/moodle-editor_atto-editor-debug.js +++ b/lib/editor/atto/yui/build/moodle-editor_atto-editor/moodle-editor_atto-editor-debug.js @@ -820,6 +820,14 @@ EditorAutosave.prototype = { */ autosaveInstance: null, + /** + * Autosave Timer. + * + * @property autosaveTimer + * @type object + */ + autosaveTimer: null, + /** * Initialize the autosave process * @@ -904,7 +912,7 @@ EditorAutosave.prototype = { // Now setup the timer for periodic saves. var delay = parseInt(this.get('autosaveFrequency'), 10) * 1000; - Y.later(delay, this, this.saveDraft, false, true); + this.autosaveTimer = Y.later(delay, this, this.saveDraft, false, true); // Now setup the listener for form submission. form = this.textarea.ancestor('form'); @@ -968,6 +976,12 @@ EditorAutosave.prototype = { */ saveDraft: function() { var url, params; + + if (!this.editor.getDOMNode()) { + // Stop autosaving if the editor was removed from the page. + this.autosaveTimer.cancel(); + return; + } // Only copy the text from the div to the textarea if the textarea is not currently visible. if (!this.editor.get('hidden')) { this.updateOriginal(); diff --git a/lib/editor/atto/yui/build/moodle-editor_atto-editor/moodle-editor_atto-editor-min.js b/lib/editor/atto/yui/build/moodle-editor_atto-editor/moodle-editor_atto-editor-min.js index 6d7777988fd..10d60c3641f 100644 --- a/lib/editor/atto/yui/build/moodle-editor_atto-editor/moodle-editor_atto-editor-min.js +++ b/lib/editor/atto/yui/build/moodle-editor_atto-editor/moodle-editor_atto-editor-min.js @@ -1,4 +1,4 @@ YUI.add("moodle-editor_atto-editor",function(e,t){function s(){s.superclass.constructor.apply(this,arguments)}function f(){}function l(){}function d(){}function v(){}function m(){}function g(){}function y(){}function b(){}function w(){}function E(){}var n="moodle-editor_atto-editor",r={CONTENT:"editor_atto_content",CONTENTWRAPPER:"editor_atto_content_wrap",TOOLBAR:"editor_atto_toolbar",WRAPPER:"editor_atto",HIGHLIGHT:"highlight"},i=window.rangy;e.extend(s,e.Base,{BLOCK_TAGS:["address","article","aside","audio","blockquote","canvas","dd","div","dl","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","hr","noscript","ol","output","p","pre","section","table","tfoot","ul","video"],PLACEHOLDER_CLASS:"atto-tmp-class",ALL_NODES_SELECTOR:"[style],font[face]",FONT_FAMILY:"fontFamily",_wrapper:null,editor:null,textarea:null,textareaLabel:null,plugins:null,_eventHandles:null,initializer:function(){var t;this.textarea=e.one(document.getElementById(this.get("elementid")));if(!this.textarea)return;this._eventHandles=[],this._wrapper=e.Node.create('
'),t=e.Handlebars.compile('
'),this.editor=e.Node.create(t({elementid:this.get("elementid"),CSS:r})),this.textareaLabel=e.one('[for="'+this.get("elementid")+'"]'),this.textareaLabel&&(this.textareaLabel.generateID(),this.editor.setAttribute("aria-labelledby",this.textareaLabel.get("id"))),this.setupToolbar();var n=e.Node.create('
');n.appendChild(this.editor),this._wrapper.appendChild(n),this.editor.setStyle("minHeight",20*this.textarea.getAttribute("rows")+8+"px"),e.UA.ie===0&&this.editor.setStyle("height",20*this.textarea.getAttribute("rows")+8+"px"),this.disableCssStyling(),document.queryCommandSupported("DefaultParagraphSeparator")&&document.execCommand("DefaultParagraphSeparator",!1,"p"),this.textarea.get("parentNode").insert(this._wrapper,this.textarea).setAttribute("class","editor_atto_wrap"),this.textarea.hide(),this.updateFromTextArea(),this.publishEvents(),this.setupSelectionWatchers(),this.setupAutomaticPolling(),this.setupPlugins(),this.setupAutosave(),this.setupNotifications()},focus:function(){return this.editor.focus(),this},publishEvents:function(){return this.publish("change",{broadcast:!0,preventable:!0}),this.publish("pluginsloaded",{fireOnce:!0}),this.publish("atto:selectionchanged",{prefix:"atto"}),this},setupAutomaticPolling:function(){return this._registerEventHandle(this.editor.on(["keyup","cut"],this.updateOriginal,this)),this._registerEventHandle(this.editor.on("paste",this.pasteCleanup,this)),this._registerEventHandle(this.editor.on("drop",this.updateOriginalDelayed,this)),this},updateOriginalDelayed:function(){return e.soon(e.bind(this.updateOriginal,this)),this},setupPlugins:function(){this.plugins={};var t=this.get("plugins"),n,r,i,s,o;for(n in t){r=t[n];if(!r.plugins)continue;for(i in r.plugins){s=r.plugins[i],o=e.mix({name:s.name,group:r.group,editor:this.editor,toolbar:this.toolbar,host:this},s);if(typeof e.M["atto_"+s.name]=="undefined")continue;this.plugins[s.name]=new e.M["atto_"+s.name].Button(o)}}return this.fire("pluginsloaded"),this},enablePlugins:function(e){this._setPluginState(!0,e)},disablePlugins:function(e){this._setPluginState(!1,e)},_setPluginState:function(t,n){var r="disableButtons";t&&(r="enableButtons"),n?this.plugins[n][r]():e.Object.each(this.plugins,function(e){e[r]()},this)},_registerEventHandle:function(e){this._eventHandles.push(e)}},{NS:"editor_atto",ATTRS:{elementid:{value:null,writeOnce:!0},contextid:{value:null,writeOnce:!0},plugins:{value:{},writeOnce:!0}}}),e.augment(s,e.EventTarget),e.namespace("M.editor_atto").Editor=s,e.namespace("M.editor_atto.Editor").init=function(t){return new e.M.editor_atto.Editor(t)};var o="moodle-editor_atto-editor-notify",u="info",a="warning";f.ATTRS={},f.prototype={messageOverlay:null,hideTimer:null,setupNotifications:function(){var e=new Image,t=new Image;return e.src=M.util.image_url("i/warning","moodle"),t.src=M.util.image_url("i/info","moodle"),this},showMessage:function(t,n,r){var i="",s,o;return this.messageOverlay===null&&(this.messageOverlay=e.Node.create('
'),this.messageOverlay.hide(!0),this.textarea.get("parentNode").append(this.messageOverlay),this.messageOverlay.on("click",function(){this.messageOverlay.hide(!0)},this)),this.hideTimer!==null&&this.hideTimer.cancel(),n===a?i=''+M.util.get_string(':n===u&&(i=''+M.util.get_string('),s=parseInt(r,10),s<=0&&(s=6e4),n="atto_"+n,o=e.Node.create('"),this.messageOverlay.empty(),this.messageOverlay.append(o),this.messageOverlay.show(!0),this.hideTimer=e.later(s,this,function(){this.hideTimer=null,this.messageOverlay.hide(!0)}),this}},e.Base.mix(e.M.editor_atto.Editor,[f]),l.ATTRS={},l.prototype={_getEmptyContent:function(){return e.UA.ie&&e.UA.ie<10?"

":"


"},updateFromTextArea:function(){return this.editor.setHTML(""),this.editor.append(this._cleanHTML(this.textarea.get("value"))),this.editor.getHTML()===""&&this.editor.setHTML(this._getEmptyContent()),this},updateOriginal:function(){var e=this.textarea.get("value"),t=this.getCleanHTML();return t===""&&this.isActive()&&(t=this._getEmptyContent()),e!==t&&(this.textarea.set("value",t),this.textarea.simulate("change"),this.fire("change")),this}},e.Base.mix(e.M.editor_atto.Editor,[l]);var c=5e3,h=6e4,p="moodle-editor_atto-editor-autosave";d.ATTRS={autosaveEnabled:{value:!0,writeOnce:!0},autosaveFrequency:{value:60,writeOnce:!0},pageHash:{value:"",writeOnce:!0},autosaveAjaxScript:{value:"/lib/editor/atto/autosave-ajax.php",readOnly:!0}} -,d.prototype={lastText:"",autosaveInstance:null,setupAutosave:function(){var t=-1,n,r=null,i=this.get("filepickeroptions"),s,o;if(!this.get("autosaveEnabled"))return;this.autosaveInstance=e.stamp(this);for(r in i)typeof i[r].itemid!="undefined"&&(t=i[r].itemid);o=M.cfg.wwwroot+this.get("autosaveAjaxScript"),s={sesskey:M.cfg.sesskey,contextid:this.get("contextid"),action:"resume",drafttext:"",draftid:t,elementid:this.get("elementid"),pageinstance:this.autosaveInstance,pagehash:this.get("pageHash")},e.io(o,{method:"POST",data:s,context:this,on:{success:function(e,t){var n;if(typeof t.responseText!="undefined"&&t.responseText!==""){n=JSON.parse(t.responseText);if(n.result==="

"||n.result==="


"||n.result==="
")n.result="";if(n.result==="

 

"||n.result==="


 

")n.result="";n.error||typeof n.result=="undefined"?this.showMessage(M.util.get_string("errortextrecovery","editor_atto"),a,h):n.result!==this.textarea.get("value")&&n.result!==""&&this.recoverText(n.result),this._fireSelectionChanged()}},failure:function(){this.showMessage(M.util.get_string("errortextrecovery","editor_atto"),a,h)}}});var u=parseInt(this.get("autosaveFrequency"),10)*1e3;return e.later(u,this,this.saveDraft,!1,!0),n=this.textarea.ancestor("form"),n&&n.on("submit",this.resetAutosave,this),this},resetAutosave:function(){var t=M.cfg.wwwroot+this.get("autosaveAjaxScript"),n={sesskey:M.cfg.sesskey,contextid:this.get("contextid"),action:"reset",elementid:this.get("elementid"),pageinstance:this.autosaveInstance,pagehash:this.get("pageHash")};return e.io(t,{method:"POST",data:n,sync:!0}),this},recoverText:function(e){return this.editor.setHTML(e),this.saveSelection(),this.updateOriginal(),this.lastText=e,this.showMessage(M.util.get_string("textrecovered","editor_atto"),u,h),this},saveDraft:function(){var t,n;this.editor.get("hidden")||this.updateOriginal();var r=this.textarea.get("value");if(r!==this.lastText){t=M.cfg.wwwroot+this.get("autosaveAjaxScript"),n={sesskey:M.cfg.sesskey,contextid:this.get("contextid"),action:"save",drafttext:r,elementid:this.get("elementid"),pagehash:this.get("pageHash"),pageinstance:this.autosaveInstance};var i=function(e,t){var n=parseInt(this.get("autosaveFrequency"),10)*1e3;this.showMessage(M.util.get_string("autosavefailed","editor_atto"),a,n)};e.io(t,{method:"POST",data:n,context:this,on:{error:i,failure:i,success:function(t,n){n.responseText!==""?e.soon(e.bind(i,this,[t,n])):(this.lastText=r,this.showMessage(M.util.get_string("autosavesucceeded","editor_atto"),u,c))}}})}return this}},e.Base.mix(e.M.editor_atto.Editor,[d]),v.ATTRS={},v.prototype={getCleanHTML:function(){var t=this.editor.cloneNode(!0),n;return e.each(t.all('[id^="yui"]'),function(e){e.removeAttribute("id")}),t.all(".atto_control").remove(!0),n=t.get("innerHTML"),n==="

"||n==="


"?"":this._cleanHTML(n)},cleanEditorHTML:function(){var e=this.editor.get("innerHTML");return this.editor.set("innerHTML",this._cleanHTML(e)),this},_cleanHTML:function(e){var t=[{regex:/]*>[\s\S]*?<\/style>/gi,replace:""},{regex:/)/gi,replace:""},{regex:/<\/?(?:title|meta|style|st\d|head|font|html|body|link)[^>]*?>/gi,replace:""}];return this._filterContentWithRules(e,t)},_filterContentWithRules:function(e,t){var n=0;for(n=0;n-1);if(r){var i;try{i=t.clipboardData.getData("text/html")}catch(s){return this.fallbackPasteCleanupDelayed(),!0}e.preventDefault(),i=this._cleanPasteHTML(i);var o=window.rangy.saveSelection();return this.insertContentAtFocusPoint(i),window.rangy.restoreSelection(o),window.rangy.getSelection().collapseToEnd(),this.updateOriginal(),!1}return this.fallbackPasteCleanupDelayed(),!0}return this.fallbackPasteCleanupDelayed(),!0}return this.updateOriginalDelayed(),!0},fallbackPasteCleanup:function(){var e=window.rangy.saveSelection(),t=this.editor.get("innerHTML");return this.editor.set("innerHTML",this._cleanPasteHTML(t)),this.updateOriginal(),window.rangy.restoreSelection(e),this},fallbackPasteCleanupDelayed:function(){return e.soon(e.bind(this.fallbackPasteCleanup,this)),this},_cleanPasteHTML:function(e){if(!e||e.length===0)return"";var t=[{regex:/<\s*\/html\s*>([\s\S]+)$/gi,replace:""},{regex://gi,replace:""},{regex://gi,replace:""},{regex:/]*>[\s\S]*?<\/xml>/gi,replace:""},{regex:/<\?xml[^>]*>[\s\S]*?<\\\?xml>/gi,replace:""},{regex:/<\/?\w+:[^>]*>/gi,replace:""}];e=this._filterContentWithRules(e,t),e=this._cleanHTML(e);if(e.length===0||!e.match(/\S/))return e;var n=document.createElement("div");return n.innerHTML=e,e=n.innerHTML,n.innerHTML="",t=[{regex:/(<[^>]*?style\s*?=\s*?")([^>"]*)(")/gi,replace:function(e,t,n,r){return n=n.replace(/(?:^|;)[\s]*MSO[-:](?:&[\w]*;|[^;"])*/gi,""),t+n+r}},{regex:/(<[^>]*?class\s*?=\s*?")([^>"]*)(")/gi,replace:function(e,t,n,r){return n=n.replace(/(?:^|[\s])[\s]*MSO[_a-zA-Z0-9\-]*/gi,""),n=n.replace(/(?:^|[\s])[\s]*Apple-[_a-zA-Z0-9\-]*/gi,""),t+n+r}},{regex:/]*?name\s*?=\s*?"OLE_LINK\d*?"[^>]*?>\s*?<\/a>/gi,replace:""}],e=this._filterContentWithRules(e,t),e=this._cleanHTML(e),e=this._cleanSpans(e),e},_cleanSpans:function(e){if(!e||e.length===0)return"";if(e.length===0||!e.match(/\S/))return e;var t=[{regex:/(<[^>]*?)(?:[\s]*(?:class|style|id)\s*?=\s*?"\s*?")+/gi,replace:"$1"}];e=this._filterContentWithRules(e,t);var n=document.createElement("div");n.innerHTML=e;var r=n.getElementsByTagName("span"),i=Array.prototype.slice.call(r,0);return i.forEach(function(e){if(!e.hasAttributes()){while(e.firstChild)e.parentNode.insertBefore(e.firstChild,e);e.parentNode.removeChild(e)}}),n.innerHTML}},e.Base.mix(e.M.editor_atto.Editor,[v]),m -.ATTRS={},m.prototype={applyFormat:function(t,n,r,i){function s(t,n,r,i,s,o){e.soon(e.bind(function(e,t,n,r,i,s){var o=window.rangy.getSelection(),u=o.getRangeAt(0);u.setStart(i,s),o.setSingleRange(u),t.apply(n,[e,r]),o.collapseToEnd(),this.saveSelection(),this.updateOriginal()},this,t,n,r,i,s,o))}r=r||this;var o=window.rangy.getSelection();if(o.isCollapsed){var u=this.editor.once("input",s,this,n,r,i,o.anchorNode,o.anchorOffset);this.editor.onceAfter(["click","selectstart"],u.detach,u);return}n.apply(r,[t,i]),this.saveSelection(),this.updateOriginal()},replaceTags:function(t,n){t.setAttribute("data-iterate",!0);var r=this.editor.one('[data-iterate="true"]');while(r){var i=e.Node.create("<"+n+" />").setAttrs(r.getAttrs()).removeAttribute("data-iterate");r.getAttribute("style")&&i.setAttribute("style",r.getAttribute("style")),r.getAttribute("class")&&i.setAttribute("class",r.getAttribute("class"));var s=r.getDOMNode().childNodes,o;o=s[0];while(typeof o!="undefined")i.append(o),o=s[0];r.replace(i),r=this.editor.one('[data-iterate="true"]')}},changeToCSS:function(e,t){var n=window.rangy.saveSelection();this.editor.all(".rangySelectionBoundary").setStyle("display",null),this.editor.all(e).addClass(t),this.replaceTags(this.editor.all("."+t),"span"),window.rangy.restoreSelection(n)},changeToTags:function(e,t){var n=window.rangy.saveSelection();this.editor.all(".rangySelectionBoundary").setStyle("display",null),this.replaceTags(this.editor.all('span[class="'+e+'"]'),t),this.editor.all(t+'[class="'+e+'"]').removeAttribute("class"),this.editor.all("."+e).each(function(n){n.wrap("<"+t+"/>"),n.removeClass(e)}),this.editor.all('[class="'+e+'"]').removeAttribute("class"),this.editor.all(t).removeClass(e),window.rangy.restoreSelection(n)}},e.Base.mix(e.M.editor_atto.Editor,[m]),g.ATTRS={},g.prototype={toolbar:null,openMenus:null,setupToolbar:function(){return this.toolbar=e.Node.create('