From 99302c5e96991b817f7e556ec52cb0d368cf8613 Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Sat, 28 Apr 2012 18:00:06 +0200 Subject: [PATCH 1/5] MDL-26028 cleanup all IE6 gzip hacks --- calendar/export_execute.php | 5 ----- lib/configonlylib.php | 17 +++++------------ lib/filelib.php | 6 ------ lib/setup.php | 9 +++++++++ lib/setuplib.php | 3 +++ mod/scorm/api.php | 10 +++------- mod/scorm/loadSCO.php | 3 +++ mod/scorm/loaddatamodel.php | 8 +++----- mod/scorm/player.php | 6 ------ 9 files changed, 26 insertions(+), 41 deletions(-) diff --git a/calendar/export_execute.php b/calendar/export_execute.php index ee0bed88d7a..229868237da 100644 --- a/calendar/export_execute.php +++ b/calendar/export_execute.php @@ -174,11 +174,6 @@ if(empty($serialized)) { die('bad serialization'); } -//IE compatibility HACK! -if (ini_get_bool('zlib.output_compression')) { - ini_set('zlib.output_compression', 'Off'); -} - $filename = 'icalexport.ics'; header('Last-Modified: '. gmdate('D, d M Y H:i:s', time()) .' GMT'); diff --git a/lib/configonlylib.php b/lib/configonlylib.php index 62391626d2d..185415d1f37 100644 --- a/lib/configonlylib.php +++ b/lib/configonlylib.php @@ -89,19 +89,12 @@ function min_enable_zlib_compression() { } // zlib.output_compression is preferred over ob_gzhandler() - if (!empty($_SERVER['HTTP_USER_AGENT'])) { - $agent = $_SERVER['HTTP_USER_AGENT']; - // try to detect IE6 and prevent gzip because it is extremely buggy browser - $parts = explode(';', $agent); - if (isset($parts[1])) { - $parts = explode(' ', trim($parts[1])); - if (count($parts) > 1) { - if ($parts[0] === 'MSIE' and (float)$parts[1] < 7) { - @ini_set('zlib.output_compression', '0'); - return false; - } - } + if (!empty($_SERVER['HTTP_USER_AGENT']) and strpos($_SERVER['HTTP_USER_AGENT'], 'MSIE 6') !== false) { + @ini_set('zlib.output_compression', 'Off'); + if (function_exists('apache_setenv')) { + @apache_setenv('no-gzip', 1); } + return false; } @ini_set('output_handler', ''); diff --git a/lib/filelib.php b/lib/filelib.php index cb30f23a572..e11b4a0ab69 100644 --- a/lib/filelib.php +++ b/lib/filelib.php @@ -1624,12 +1624,6 @@ function prepare_file_content_sending() { $olddebug = error_reporting(0); - // IE compatibility HACK - it does not like zlib compression much - // there is also a problem with the length header in older PHP versions - if (ini_get_bool('zlib.output_compression')) { - ini_set('zlib.output_compression', 'Off'); - } - // flush and close all buffers if possible while(ob_get_level()) { if (!ob_end_flush()) { diff --git a/lib/setup.php b/lib/setup.php index cd652f669b0..dfaea6aad6e 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -887,6 +887,15 @@ if (PHPUNIT_TEST) { } +// // try to detect IE6 and prevent gzip because it is extremely buggy browser +if (!empty($_SERVER['HTTP_USER_AGENT']) and strpos($_SERVER['HTTP_USER_AGENT'], 'MSIE 6') !== false) { + @ini_set('zlib.output_compression', 'Off'); + if (function_exists('apache_setenv')) { + @apache_setenv('no-gzip', 1); + } +} + + // note: we can not block non utf-8 installations here, because empty mysql database // might be converted to utf-8 in admin/index.php during installation diff --git a/lib/setuplib.php b/lib/setuplib.php index 5bc617b0b0b..181afe4675c 100644 --- a/lib/setuplib.php +++ b/lib/setuplib.php @@ -1079,6 +1079,9 @@ function disable_output_buffering() { } } + // disable any other output handlers + ini_set('output_handler', ''); + error_reporting($olddebug); } diff --git a/mod/scorm/api.php b/mod/scorm/api.php index a23cfb136bf..9d906634429 100644 --- a/mod/scorm/api.php +++ b/mod/scorm/api.php @@ -23,13 +23,6 @@ $scoid = required_param('scoid', PARAM_INT); // sco ID $mode = optional_param('mode', '', PARAM_ALPHA); // navigation mode $attempt = required_param('attempt', PARAM_INT); // new attempt -//IE 6 Bug workaround -if (strpos($_SERVER['HTTP_USER_AGENT'], 'MSIE 6') !== false) { - @ini_set('zlib.output_compression', 'Off'); - @apache_setenv('no-gzip', 1); - header( 'Content-Type: application/javascript' ); -} - if (!empty($id)) { if (! $cm = get_coursemodule_from_id('scorm', $id)) { print_error('invalidcoursemodule'); @@ -96,6 +89,9 @@ if (!$sco = scorm_get_sco($scoid)) { if (scorm_version_check($scorm->version, SCORM_13)) { $userdata->{'cmi.scaled_passing_score'} = $DB->get_field('scorm_seq_objective', 'minnormalizedmeasure', array('scoid'=>$scoid)); } + +header('Content-Type: text/javascript; charset=UTF-8'); + $scorm->version = strtolower(clean_param($scorm->version, PARAM_SAFEDIR)); // Just to be safe if (file_exists($CFG->dirroot.'/mod/scorm/datamodels/'.$scorm->version.'.js.php')) { include_once($CFG->dirroot.'/mod/scorm/datamodels/'.$scorm->version.'.js.php'); diff --git a/mod/scorm/loadSCO.php b/mod/scorm/loadSCO.php index 90fd45bfed1..29f49722f17 100644 --- a/mod/scorm/loadSCO.php +++ b/mod/scorm/loadSCO.php @@ -145,6 +145,9 @@ add_to_log($course->id, 'scorm', 'launch', 'view.php?id='.$cm->id, $result, $cm- // which API are we looking for $LMS_api = (scorm_version_check($scorm->version, SCORM_12) || empty($scorm->version)) ? 'API' : 'API_1484_11'; + +header('Content-Type: text/html; charset=UTF-8'); + ?> diff --git a/mod/scorm/loaddatamodel.php b/mod/scorm/loaddatamodel.php index fb6e2eb329a..ce8005ca47b 100644 --- a/mod/scorm/loaddatamodel.php +++ b/mod/scorm/loaddatamodel.php @@ -23,11 +23,6 @@ $scoid = required_param('scoid', PARAM_INT); // sco ID $mode = optional_param('mode', '', PARAM_ALPHA); // navigation mode $attempt = required_param('attempt', PARAM_INT); // new attempt -//IE 6 Bug workaround -if (strpos($_SERVER['HTTP_USER_AGENT'], 'MSIE 6') !== false && ini_get('zlib.output_compression') == 'On') { - ini_set('zlib.output_compression', 'Off'); -} - if (!empty($id)) { if (! $cm = get_coursemodule_from_id('scorm', $id)) { print_error('invalidcoursemodule'); @@ -94,6 +89,9 @@ if ($scodatas = scorm_get_sco($scoid, SCO_DATA)) { if (!$sco = scorm_get_sco($scoid)) { print_error('cannotfindsco', 'scorm'); } + +header('Content-Type: text/javascript; charset=UTF-8'); + $scorm->version = strtolower(clean_param($scorm->version, PARAM_SAFEDIR)); // Just to be safe if (file_exists($CFG->dirroot.'/mod/scorm/datamodels/'.$scorm->version.'.js.php')) { include($CFG->dirroot.'/mod/scorm/datamodels/'.$scorm->version.'.js.php'); diff --git a/mod/scorm/player.php b/mod/scorm/player.php index e39a61902b2..53051dbc6c8 100644 --- a/mod/scorm/player.php +++ b/mod/scorm/player.php @@ -28,12 +28,6 @@ $currentorg = optional_param('currentorg', '', PARAM_RAW); // selected organizat $newattempt = optional_param('newattempt', 'off', PARAM_ALPHA); // the user request to start a new attempt $displaymode = optional_param('display','',PARAM_ALPHA); -//IE 6 Bug workaround -if (strpos($_SERVER['HTTP_USER_AGENT'], 'MSIE 6') !== false) { - @ini_set('zlib.output_compression', 'Off'); - @apache_setenv('no-gzip', 1); -} - // IE 9 workaround for Flash bug: MDL-29213 // Note that it's not clear if appending the meta tag via $CFG->additionalhtmlhead // is correct at all, both because of the mechanism itself and because MS says From d5dd0540c21c3443f37b47f131c2b997540025d5 Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Sun, 29 Apr 2012 01:16:53 +0200 Subject: [PATCH 2/5] MDL-26028 send_file performance improvements Includes X-Sendfile support, ETags for stored_files and general cleanup. --- config-dist.php | 6 + lib/filelib.php | 462 +++++++++++++------------------ lib/filestorage/file_storage.php | 16 ++ lib/xsendfilelib.php | 68 +++++ 4 files changed, 279 insertions(+), 273 deletions(-) create mode 100644 lib/xsendfilelib.php diff --git a/config-dist.php b/config-dist.php index 495319c8203..9684ced01d9 100644 --- a/config-dist.php +++ b/config-dist.php @@ -220,6 +220,12 @@ $CFG->admin = 'admin'; // about students being served outdated versions of uploaded files. // $CFG->filelifetime = 86400; // +// Some web servers can offload the file serving from PHP process, +// comment out one the following options to enable it in Moodle: +// $CFG->xsendfile = 'X-Sendfile'; // Apache {@see https://tn123.org/mod_xsendfile/} +// $CFG->xsendfile = 'X-LIGHTTPD-send-file'; // Lighttpd {@see http://redmine.lighttpd.net/projects/lighttpd/wiki/X-LIGHTTPD-send-file} +// $CFG->xsendfile = 'X-Accel-Redirect'; // Nginx {@see http://wiki.nginx.org/XSendfile} +// // This setting will prevent the 'My Courses' page being displayed when a student // logs in. The site front page will always show the same (logged-out) view. // $CFG->disablemycourses = true; diff --git a/lib/filelib.php b/lib/filelib.php index e11b4a0ab69..335c8892f63 100644 --- a/lib/filelib.php +++ b/lib/filelib.php @@ -1611,32 +1611,159 @@ function send_header_404() { } /** - * Check output buffering settings before sending file. - * Please note you should not send any other headers after calling this function. - * - * To be called only from lib/filelib.php ! + * Enhanced readfile() with optional acceleration. + * @param string|stored_file $file + * @param string $mimetype + * @param bool $accelerate + * @return void */ -function prepare_file_content_sending() { - // We needed to be able to send headers up until now - if (headers_sent()) { - throw new file_serving_exception('Headers already sent, can not serve file.'); +function readfile_accel($file, $mimetype, $accelerate) { + global $CFG; + + if ($mimetype === 'text/plain') { + // there is no encoding specified in text files, we need something consistent + header('Content-Type: text/plain; charset=utf-8'); + } else { + header('Content-Type: '.$mimetype); } - $olddebug = error_reporting(0); + $lastmodified = is_object($file) ? $file->get_timemodified() : filemtime($file); + header('Last-Modified: '. gmdate('D, d M Y H:i:s', $lastmodified) .' GMT'); - // flush and close all buffers if possible - while(ob_get_level()) { - if (!ob_end_flush()) { - // prevent infinite loop when buffer can not be closed - break; + if (is_object($file)) { + header('ETag: ' . $file->get_contenthash()); + if (isset($_SERVER['HTTP_IF_NONE_MATCH']) and $_SERVER['HTTP_IF_NONE_MATCH'] === $file->get_contenthash()) { + header('HTTP/1.1 304 Not Modified'); + return; } } - error_reporting($olddebug); + // if etag present for stored file rely on it exclusively + if (!empty($_SERVER['HTTP_IF_MODIFIED_SINCE']) and (empty($_SERVER['HTTP_IF_NONE_MATCH']) or !is_object($file))) { + // get unixtime of request header; clip extra junk off first + $since = strtotime(preg_replace('/;.*$/', '', $_SERVER["HTTP_IF_MODIFIED_SINCE"])); + if ($since && $since >= $lastmodified) { + header('HTTP/1.1 304 Not Modified'); + return; + } + } - //NOTE: we can not reliable test headers_sent() here because - // the headers might be sent which trying to close the buffers, - // this happens especially if browser does not support gzip or deflate + if ($accelerate and !empty($CFG->xsendfile)) { + if (empty($CFG->disablebyteserving) and $mimetype !== 'text/plain') { + header('Accept-Ranges: bytes'); + } else { + header('Accept-Ranges: none'); + } + + if (is_object($file)) { + $fs = get_file_storage(); + if ($fs->xsendfile($file->get_contenthash())) { + return; + } + + } else { + require_once("$CFG->libdir/xsendfilelib.php"); + if (xsendfile($file)) { + return; + } + } + } + + $filesize = is_object($file) ? $file->get_filesize() : filesize($file); + + header('Last-Modified: '. gmdate('D, d M Y H:i:s', $lastmodified) .' GMT'); + + if ($accelerate and empty($CFG->disablebyteserving) and $mimetype !== 'text/plain') { + header('Accept-Ranges: bytes'); + + if (!empty($_SERVER['HTTP_RANGE']) and strpos($_SERVER['HTTP_RANGE'],'bytes=') !== FALSE) { + // byteserving stuff - for acrobat reader and download accelerators + // see: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 + // inspired by: http://www.coneural.org/florian/papers/04_byteserving.php + $ranges = false; + if (preg_match_all('/(\d*)-(\d*)/', $_SERVER['HTTP_RANGE'], $ranges, PREG_SET_ORDER)) { + foreach ($ranges as $key=>$value) { + if ($ranges[$key][1] == '') { + //suffix case + $ranges[$key][1] = $filesize - $ranges[$key][2]; + $ranges[$key][2] = $filesize - 1; + } else if ($ranges[$key][2] == '' || $ranges[$key][2] > $filesize - 1) { + //fix range length + $ranges[$key][2] = $filesize - 1; + } + if ($ranges[$key][2] != '' && $ranges[$key][2] < $ranges[$key][1]) { + //invalid byte-range ==> ignore header + $ranges = false; + break; + } + //prepare multipart header + $ranges[$key][0] = "\r\n--".BYTESERVING_BOUNDARY."\r\nContent-Type: $mimetype\r\n"; + $ranges[$key][0] .= "Content-Range: bytes {$ranges[$key][1]}-{$ranges[$key][2]}/$filesize\r\n\r\n"; + } + } else { + $ranges = false; + } + if ($ranges) { + if (is_object($file)) { + $handle = $file->get_content_file_handle(); + } else { + $handle = fopen($file, 'rb'); + } + byteserving_send_file($handle, $mimetype, $ranges, $filesize); + } + } + } else { + // Do not byteserve + header('Accept-Ranges: none'); + } + + header('Content-Length: '.$filesize); + + if ($filesize > 10000000) { + // for large files try to flush and close all buffers to conserve memory + while(@ob_get_level()) { + if (!@ob_end_flush()) { + break; + } + } + } + + // send the whole file content + if (is_object($file)) { + $file->readfile(); + } else { + readfile($file); + } +} + +/** + * Similar to readfile_accel() but designed for strings. + * @param string $string + * @param string $mimetype + * @param bool $accelerate + * @return void + */ +function readstring_accel($string, $mimetype, $accelerate) { + global $CFG; + + if ($mimetype === 'text/plain') { + // there is no encoding specified in text files, we need something consistent + header('Content-Type: text/plain; charset=utf-8'); + } else { + header('Content-Type: '.$mimetype); + } + header('Last-Modified: '. gmdate('D, d M Y H:i:s', time()) .' GMT'); + header('Accept-Ranges: none'); + + if ($accelerate and !empty($CFG->xsendfile)) { + $fs = get_file_storage(); + if ($fs->xsendfile(sha1($string))) { + return; + } + } + + header('Content-Length: '.strlen($string)); + echo $string; } /** @@ -1650,8 +1777,15 @@ function prepare_file_content_sending() { function send_temp_file($path, $filename, $pathisstring=false) { global $CFG; + if (check_browser_version('Firefox', '1.5')) { + // only FF is known to correctly save to disk before opening... + $mimetype = mimeinfo('type', $filename); + } else { + $mimetype = 'application/x-forcedownload'; + } + // close session - not needed anymore - @session_get_instance()->write_close(); + session_get_instance()->write_close(); if (!$pathisstring) { if (!file_exists($path)) { @@ -1667,10 +1801,7 @@ function send_temp_file($path, $filename, $pathisstring=false) { $filename = urlencode($filename); } - $filesize = $pathisstring ? strlen($path) : filesize($path); - header('Content-Disposition: attachment; filename='.$filename); - header('Content-Length: '.$filesize); if (strpos($CFG->wwwroot, 'https://') === 0) { //https sites - watch out for IE! KB812935 and KB316431 header('Cache-Control: max-age=10'); header('Expires: '. gmdate('D, d M Y H:i:s', 0) .' GMT'); @@ -1680,17 +1811,13 @@ function send_temp_file($path, $filename, $pathisstring=false) { header('Expires: '. gmdate('D, d M Y H:i:s', 0) .' GMT'); header('Pragma: no-cache'); } - header('Accept-Ranges: none'); // Do not allow byteserving - //flush the buffers - save memory and disable sid rewrite - // this also disables zlib compression - prepare_file_content_sending(); - - // send the contents + // send the contents - we can not accelerate this because the file will be deleted asap if ($pathisstring) { - echo $path; + readstring_accel($path, $mimetype, false); } else { - @readfile($path); + readfile_accel($path, $mimetype, false); + @unlink($path); } die; //no more chars to output @@ -1712,9 +1839,6 @@ function send_temp_file_finished($path) { * byteranges etc. * * @category files - * @global stdClass $CFG - * @global stdClass $COURSE - * @global moodle_session $SESSION * @param string $path Path of file on disk (including real filename), or actual content of file as string * @param string $filename Filename to send * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours) @@ -1729,7 +1853,7 @@ function send_temp_file_finished($path) { * @return null script execution stopped unless $dontdie is true */ function send_file($path, $filename, $lifetime = 'default' , $filter=0, $pathisstring=false, $forcedownload=false, $mimetype='', $dontdie=false) { - global $CFG, $COURSE, $SESSION; + global $CFG, $COURSE; if ($dontdie) { ignore_user_abort(true); @@ -1753,46 +1877,6 @@ function send_file($path, $filename, $lifetime = 'default' , $filter=0, $pathiss $mimetype = ($forcedownload and !$isFF) ? 'application/x-forcedownload' : ($mimetype ? $mimetype : mimeinfo('type', $filename)); - $lastmodified = $pathisstring ? time() : filemtime($path); - $filesize = $pathisstring ? strlen($path) : filesize($path); - -/* - MDL-13949 - //Adobe Acrobat Reader XSS prevention - if ($mimetype=='application/pdf' or mimeinfo('type', $filename)=='application/pdf') { - //please note that it prevents opening of pdfs in browser when http referer disabled - //or file linked from another site; browser caching of pdfs is now disabled too - if (!empty($_SERVER['HTTP_RANGE'])) { - //already byteserving - $lifetime = 1; // >0 needed for byteserving - } else if (empty($_SERVER['HTTP_REFERER']) or strpos($_SERVER['HTTP_REFERER'], $CFG->wwwroot)!==0) { - $mimetype = 'application/x-forcedownload'; - $forcedownload = true; - $lifetime = 0; - } else { - $lifetime = 1; // >0 needed for byteserving - } - } -*/ - - if ($lifetime > 0 && !empty($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { - // get unixtime of request header; clip extra junk off first - $since = strtotime(preg_replace('/;.*$/','',$_SERVER["HTTP_IF_MODIFIED_SINCE"])); - if ($since && $since >= $lastmodified) { - header('HTTP/1.1 304 Not Modified'); - header('Expires: '. gmdate('D, d M Y H:i:s', time() + $lifetime) .' GMT'); - header('Cache-Control: max-age='.$lifetime); - header('Content-Type: '.$mimetype); - if ($dontdie) { - return; - } - die; - } - } - - //do not put '@' before the next header to detect incorrect moodle configurations, - //error should be better than "weird" empty lines for admins/users - header('Last-Modified: '. gmdate('D, d M Y H:i:s', $lastmodified) .' GMT'); - // if user is using IE, urlencode the filename so that multibyte file name will show up correctly on popup if (check_browser_version('MSIE')) { $filename = rawurlencode($filename); @@ -1805,51 +1889,13 @@ function send_file($path, $filename, $lifetime = 'default' , $filter=0, $pathiss } if ($lifetime > 0) { + $nobyteserving = false; header('Cache-Control: max-age='.$lifetime); header('Expires: '. gmdate('D, d M Y H:i:s', time() + $lifetime) .' GMT'); header('Pragma: '); - if (empty($CFG->disablebyteserving) && !$pathisstring && $mimetype != 'text/plain' && $mimetype != 'text/html') { - - header('Accept-Ranges: bytes'); - - if (!empty($_SERVER['HTTP_RANGE']) && strpos($_SERVER['HTTP_RANGE'],'bytes=') !== FALSE) { - // byteserving stuff - for acrobat reader and download accelerators - // see: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 - // inspired by: http://www.coneural.org/florian/papers/04_byteserving.php - $ranges = false; - if (preg_match_all('/(\d*)-(\d*)/', $_SERVER['HTTP_RANGE'], $ranges, PREG_SET_ORDER)) { - foreach ($ranges as $key=>$value) { - if ($ranges[$key][1] == '') { - //suffix case - $ranges[$key][1] = $filesize - $ranges[$key][2]; - $ranges[$key][2] = $filesize - 1; - } else if ($ranges[$key][2] == '' || $ranges[$key][2] > $filesize - 1) { - //fix range length - $ranges[$key][2] = $filesize - 1; - } - if ($ranges[$key][2] != '' && $ranges[$key][2] < $ranges[$key][1]) { - //invalid byte-range ==> ignore header - $ranges = false; - break; - } - //prepare multipart header - $ranges[$key][0] = "\r\n--".BYTESERVING_BOUNDARY."\r\nContent-Type: $mimetype\r\n"; - $ranges[$key][0] .= "Content-Range: bytes {$ranges[$key][1]}-{$ranges[$key][2]}/$filesize\r\n\r\n"; - } - } else { - $ranges = false; - } - if ($ranges) { - $handle = fopen($path, 'rb'); - byteserving_send_file($handle, $mimetype, $ranges, $filesize); - } - } - } else { - /// Do not byteserve (disabled, strings, text and html files). - header('Accept-Ranges: none'); - } } else { // Do not cache files in proxies and browsers + $nobyteserving = true; if (strpos($CFG->wwwroot, 'https://') === 0) { //https sites - watch out for IE! KB812935 and KB316431 header('Cache-Control: max-age=10'); header('Expires: '. gmdate('D, d M Y H:i:s', 0) .' GMT'); @@ -1859,29 +1905,18 @@ function send_file($path, $filename, $lifetime = 'default' , $filter=0, $pathiss header('Expires: '. gmdate('D, d M Y H:i:s', 0) .' GMT'); header('Pragma: no-cache'); } - header('Accept-Ranges: none'); // Do not allow byteserving when caching disabled } if (empty($filter)) { - if ($mimetype == 'text/plain') { - header('Content-Type: Text/plain; charset=utf-8'); //add encoding - } else { - header('Content-Type: '.$mimetype); - } - header('Content-Length: '.$filesize); - - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); - // send the contents if ($pathisstring) { - echo $path; + readstring_accel($path, $mimetype, !$dontdie); } else { - @readfile($path); + readfile_accel($path, $mimetype, !$dontdie); } - } else { // Try to put the file through filters + } else { + // Try to put the file through filters if ($mimetype == 'text/html') { $options = new stdClass(); $options->noclean = true; @@ -1891,46 +1926,24 @@ function send_file($path, $filename, $lifetime = 'default' , $filter=0, $pathiss $text = file_modify_html_header($text); $output = format_text($text, FORMAT_HTML, $options, $COURSE->id); - header('Content-Length: '.strlen($output)); - header('Content-Type: text/html'); + readstring_accel($output, $mimetype, false); - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); - - // send the contents - echo $output; - // only filter text if filter all files is selected } else if (($mimetype == 'text/plain') and ($filter == 1)) { + // only filter text if filter all files is selected $options = new stdClass(); $options->newlines = false; $options->noclean = true; $text = htmlentities($pathisstring ? $path : implode('', file($path))); $output = '
'. format_text($text, FORMAT_MOODLE, $options, $COURSE->id) .'
'; - header('Content-Length: '.strlen($output)); - header('Content-Type: text/html; charset=utf-8'); //add encoding - - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); - - // send the contents - echo $output; - - } else { // Just send it out raw - header('Content-Length: '.$filesize); - header('Content-Type: '.$mimetype); - - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); + readstring_accel($output, $mimetype, false); + } else { // send the contents if ($pathisstring) { - echo $path; - }else { - @readfile($path); + readstring_accel($path, $mimetype, !$dontdie); + } else { + readfile_accel($path, $mimetype, !$dontdie); } } } @@ -1953,9 +1966,6 @@ function send_file($path, $filename, $lifetime = 'default' , $filter=0, $pathiss * and should not be reopened. * * @category files - * @global stdClass $CFG - * @global stdClass $COURSE - * @global moodle_session $SESSION * @param stored_file $stored_file local file object * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours) * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only @@ -1964,7 +1974,7 @@ function send_file($path, $filename, $lifetime = 'default' , $filter=0, $pathiss * @return null script execution stopped unless $options['dontdie'] is true */ function send_stored_file($stored_file, $lifetime=86400 , $filter=0, $forcedownload=false, array $options=array()) { - global $CFG, $COURSE, $SESSION; + global $CFG, $COURSE; if (empty($options['filename'])) { $filename = null; @@ -2017,28 +2027,6 @@ function send_stored_file($stored_file, $lifetime=86400 , $filter=0, $forcedownl $mimetype = ($forcedownload and !$isFF) ? 'application/x-forcedownload' : ($stored_file->get_mimetype() ? $stored_file->get_mimetype() : mimeinfo('type', $filename)); - $lastmodified = $stored_file->get_timemodified(); - $filesize = $stored_file->get_filesize(); - - if ($lifetime > 0 && !empty($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { - // get unixtime of request header; clip extra junk off first - $since = strtotime(preg_replace('/;.*$/','',$_SERVER["HTTP_IF_MODIFIED_SINCE"])); - if ($since && $since >= $lastmodified) { - header('HTTP/1.1 304 Not Modified'); - header('Expires: '. gmdate('D, d M Y H:i:s', time() + $lifetime) .' GMT'); - header('Cache-Control: max-age='.$lifetime); - header('Content-Type: '.$mimetype); - if ($dontdie) { - return; - } - die; - } - } - - //do not put '@' before the next header to detect incorrect moodle configurations, - //error should be better than "weird" empty lines for admins/users - header('Last-Modified: '. gmdate('D, d M Y H:i:s', $lastmodified) .' GMT'); - // if user is using IE, urlencode the filename so that multibyte file name will show up correctly on popup if (check_browser_version('MSIE')) { $filename = rawurlencode($filename); @@ -2055,45 +2043,6 @@ function send_stored_file($stored_file, $lifetime=86400 , $filter=0, $forcedownl header('Expires: '. gmdate('D, d M Y H:i:s', time() + $lifetime) .' GMT'); header('Pragma: '); - if (empty($CFG->disablebyteserving) && $mimetype != 'text/plain' && $mimetype != 'text/html') { - - header('Accept-Ranges: bytes'); - - if (!empty($_SERVER['HTTP_RANGE']) && strpos($_SERVER['HTTP_RANGE'],'bytes=') !== FALSE) { - // byteserving stuff - for acrobat reader and download accelerators - // see: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 - // inspired by: http://www.coneural.org/florian/papers/04_byteserving.php - $ranges = false; - if (preg_match_all('/(\d*)-(\d*)/', $_SERVER['HTTP_RANGE'], $ranges, PREG_SET_ORDER)) { - foreach ($ranges as $key=>$value) { - if ($ranges[$key][1] == '') { - //suffix case - $ranges[$key][1] = $filesize - $ranges[$key][2]; - $ranges[$key][2] = $filesize - 1; - } else if ($ranges[$key][2] == '' || $ranges[$key][2] > $filesize - 1) { - //fix range length - $ranges[$key][2] = $filesize - 1; - } - if ($ranges[$key][2] != '' && $ranges[$key][2] < $ranges[$key][1]) { - //invalid byte-range ==> ignore header - $ranges = false; - break; - } - //prepare multipart header - $ranges[$key][0] = "\r\n--".BYTESERVING_BOUNDARY."\r\nContent-Type: $mimetype\r\n"; - $ranges[$key][0] .= "Content-Range: bytes {$ranges[$key][1]}-{$ranges[$key][2]}/$filesize\r\n\r\n"; - } - } else { - $ranges = false; - } - if ($ranges) { - byteserving_send_file($stored_file->get_content_file_handle(), $mimetype, $ranges, $filesize); - } - } - } else { - /// Do not byteserve (disabled, strings, text and html files). - header('Accept-Ranges: none'); - } } else { // Do not cache files in proxies and browsers if (strpos($CFG->wwwroot, 'https://') === 0) { //https sites - watch out for IE! KB812935 and KB316431 header('Cache-Control: max-age=10'); @@ -2104,23 +2053,11 @@ function send_stored_file($stored_file, $lifetime=86400 , $filter=0, $forcedownl header('Expires: '. gmdate('D, d M Y H:i:s', 0) .' GMT'); header('Pragma: no-cache'); } - header('Accept-Ranges: none'); // Do not allow byteserving when caching disabled } if (empty($filter)) { - if ($mimetype == 'text/plain') { - header('Content-Type: Text/plain; charset=utf-8'); //add encoding - } else { - header('Content-Type: '.$mimetype); - } - header('Content-Length: '.$filesize); - - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); - // send the contents - $stored_file->readfile(); + readfile_accel($stored_file, $mimetype, !$dontdie); } else { // Try to put the file through filters if ($mimetype == 'text/html') { @@ -2131,15 +2068,7 @@ function send_stored_file($stored_file, $lifetime=86400 , $filter=0, $forcedownl $text = file_modify_html_header($text); $output = format_text($text, FORMAT_HTML, $options, $COURSE->id); - header('Content-Length: '.strlen($output)); - header('Content-Type: text/html'); - - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); - - // send the contents - echo $output; + readstring_accel($output, $mimetype, false); } else if (($mimetype == 'text/plain') and ($filter == 1)) { // only filter text if filter all files is selected @@ -2149,26 +2078,10 @@ function send_stored_file($stored_file, $lifetime=86400 , $filter=0, $forcedownl $text = $stored_file->get_content(); $output = '
'. format_text($text, FORMAT_MOODLE, $options, $COURSE->id) .'
'; - header('Content-Length: '.strlen($output)); - header('Content-Type: text/html; charset=utf-8'); //add encoding - - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); - - // send the contents - echo $output; + readstring_accel($output, $mimetype, false); } else { // Just send it out raw - header('Content-Length: '.$filesize); - header('Content-Type: '.$mimetype); - - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); - - // send the contents - $stored_file->readfile(); + readfile_accel($stored_file, $mimetype, !$dontdie); } } if ($dontdie) { @@ -2347,9 +2260,11 @@ function fulldelete($location) { * @param string $mimetype The mimetype for the output * @param array $ranges An array of ranges to send * @param string $filesize The size of the content if only one range is used - * @todo MDL-31088 check if "multipart/x-byteranges" is more compatible with current readers/browsers/servers */ function byteserving_send_file($handle, $mimetype, $ranges, $filesize) { + // better turn off any kind of compression and buffering + @ini_set('zlib.output_compression', 'Off'); + $chunksize = 1*(1024*1024); // 1MB chunks - must be less than 2MB! if ($handle === false) { die; @@ -2361,11 +2276,12 @@ function byteserving_send_file($handle, $mimetype, $ranges, $filesize) { header('Content-Range: bytes '.$ranges[0][1].'-'.$ranges[0][2].'/'.$filesize); header('Content-Type: '.$mimetype); - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); + while(@ob_get_level()) { + if (!@ob_end_flush()) { + break; + } + } - $buffer = ''; fseek($handle, $ranges[0][1]); while (!feof($handle) && $length > 0) { @set_time_limit(60*60); //reset time limit to 60 min - should be enough for 1 MB chunk @@ -2385,16 +2301,16 @@ function byteserving_send_file($handle, $mimetype, $ranges, $filesize) { header('HTTP/1.1 206 Partial content'); header('Content-Length: '.$totallength); header('Content-Type: multipart/byteranges; boundary='.BYTESERVING_BOUNDARY); - //TODO: check if "multipart/x-byteranges" is more compatible with current readers/browsers/servers - //flush the buffers - save memory and disable sid rewrite - //this also disables zlib compression - prepare_file_content_sending(); + while(@ob_get_level()) { + if (!@ob_end_flush()) { + break; + } + } foreach($ranges as $range) { $length = $range[2] - $range[1] + 1; echo $range[0]; - $buffer = ''; fseek($handle, $range[1]); while (!feof($handle) && $length > 0) { @set_time_limit(60*60); //reset time limit to 60 min - should be enough for 1 MB chunk diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index 463f9d4f763..9855a6d0973 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -1294,6 +1294,22 @@ class file_storage { return array($contenthash, $filesize, $newfile); } + /** + * Serve file content using X-Sendfile header. + * Please make sure that all headers are already sent + * and the all access control checks passed. + * + * @param string $contenthash sah1 hash of the file content to be served + * @return bool success + */ + public function xsendfile($contenthash) { + global $CFG; + require_once("$CFG->libdir/xsendfilelib.php"); + + $hashpath = $this->path_from_hash($contenthash); + return xsendfile("$hashpath/$contenthash"); + } + /** * Return path to file with given hash. * diff --git a/lib/xsendfilelib.php b/lib/xsendfilelib.php new file mode 100644 index 00000000000..784facb8ca7 --- /dev/null +++ b/lib/xsendfilelib.php @@ -0,0 +1,68 @@ +. + +/** + * X-Sendfile support + * + * @package core_files + * @copyright 2012 Petr Skoda {@link http://skodak.org} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +//NOTE: do not verify MOODLE_INTERNAL here, this is used from themes too + +/** + * Serve file using X-Sendfile header, this needs special server module + * or configuration. Please make sure that all headers are already sent + * and the all access control checks passed. + * + * @param string $filepath + * @return bool success + */ +function xsendfile($filepath) { + global $CFG; + + if (empty($CFG->xsendfile)) { + return false; + } + + if (!file_exists($filepath)) { + return false; + } + + if (headers_sent()) { + return false; + } + + if ($CFG->xsendfile === 'X-LIGHTTPD-send-file') { + // http://redmine.lighttpd.net/projects/lighttpd/wiki/X-LIGHTTPD-send-file says 1.4 it does not support byteserving + header('Accept-Ranges: none'); + + } else if ($CFG->xsendfile === 'X-Accel-Redirect') { + // http://wiki.nginx.org/XSendfile + // Nginx is using relative path to protected folder, please note you can not use cache dir, tempdir and file pool outside of dataroot! + $filepath = realpath($filepath); + $dataroot = realpath($CFG->dataroot); + if (strpos($filepath, $dataroot) !== 0) { + return false; + } + $filepath = substr($filepath, strlen($dataroot)); + } + + header("$CFG->xsendfile: $filepath"); + + return true; +} From 7e9f1b63e509263d2143b311862ebcd261226b93 Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Sun, 29 Apr 2012 18:59:08 +0200 Subject: [PATCH 3/5] MDL-26028 use X-Sendfile for theme images and javascript --- theme/image.php | 7 +++++++ theme/javascript.php | 8 ++++++++ theme/yui_image.php | 7 +++++++ 3 files changed, 22 insertions(+) diff --git a/theme/image.php b/theme/image.php index 63acfaf0053..c8dd7acc643 100644 --- a/theme/image.php +++ b/theme/image.php @@ -138,6 +138,9 @@ if ($rev > -1) { // parameters to get the best performance. function send_cached_image($imagepath, $rev) { + global $CFG; + require("$CFG->dirroot/lib/xsendfilelib.php"); + $lifetime = 60*60*24*30; // 30 days $pathinfo = pathinfo($imagepath); $imagename = $pathinfo['filename'].'.'.$pathinfo['extension']; @@ -154,6 +157,10 @@ function send_cached_image($imagepath, $rev) { header('Content-Type: '.$mimetype); header('Content-Length: '.filesize($imagepath)); + if (xsendfile($imagepath)) { + die; + } + // no need to gzip already compressed images ;-) readfile($imagepath); diff --git a/theme/javascript.php b/theme/javascript.php index 6f7ebadc8be..4151172a711 100644 --- a/theme/javascript.php +++ b/theme/javascript.php @@ -99,6 +99,9 @@ if ($rev > -1) { // parameters to get the best performance. function send_cached_js($jspath) { + global $CFG; + require("$CFG->dirroot/lib/xsendfilelib.php"); + $lifetime = 60*60*24*30; // 30 days header('Content-Disposition: inline; filename="javascript.php"'); @@ -108,6 +111,11 @@ function send_cached_js($jspath) { header('Cache-Control: max-age='.$lifetime); header('Accept-Ranges: none'); header('Content-Type: application/javascript; charset=utf-8'); + + if (xsendfile($jspath)) { + die; + } + if (!min_enable_zlib_compression()) { header('Content-Length: '.filesize($jspath)); } diff --git a/theme/yui_image.php b/theme/yui_image.php index 23ac987ff29..37dd6ba5075 100644 --- a/theme/yui_image.php +++ b/theme/yui_image.php @@ -66,6 +66,9 @@ yui_image_cached($imagepath); function yui_image_cached($imagepath) { + global $CFG; + require("$CFG->dirroot/lib/xsendfilelib.php"); + $lifetime = 60*60*24*300; // 300 days === forever $pathinfo = pathinfo($imagepath); $imagename = $pathinfo['filename'].'.'.$pathinfo['extension']; @@ -88,6 +91,10 @@ function yui_image_cached($imagepath) { header('Content-Type: '.$mimetype); header('Content-Length: '.filesize($imagepath)); + if (xsendfile($imagepath)) { + die; + } + // no need to gzip already compressed images ;-) readfile($imagepath); From f7d26a054534ea680e69a13e0eb199ccaf547be1 Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Mon, 30 Apr 2012 15:52:42 +0200 Subject: [PATCH 4/5] MDL-26028 implement Nginx X-Sendfile directory aliases setting --- config-dist.php | 10 ++++++++++ lib/xsendfilelib.php | 21 ++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/config-dist.php b/config-dist.php index 9684ced01d9..4b3534fa32b 100644 --- a/config-dist.php +++ b/config-dist.php @@ -225,6 +225,16 @@ $CFG->admin = 'admin'; // $CFG->xsendfile = 'X-Sendfile'; // Apache {@see https://tn123.org/mod_xsendfile/} // $CFG->xsendfile = 'X-LIGHTTPD-send-file'; // Lighttpd {@see http://redmine.lighttpd.net/projects/lighttpd/wiki/X-LIGHTTPD-send-file} // $CFG->xsendfile = 'X-Accel-Redirect'; // Nginx {@see http://wiki.nginx.org/XSendfile} +// If your X-Sendfile implementation (usually Nginx) uses directory aliases specify them +// in the following array setting: +// $CFG->xsendfilealiases = array( +// '/dataroot/' => $CFG->dataroot, +// '/cachedir/' => '/var/www/moodle/cache', // for custom $CFG->cachedir locations +// '/tempdir/' => '/var/www/moodle/temp', // for custom $CFG->tempdir locations +// '/filedir' => '/var/www/moodle/filedir', // for custom $CFG->filedir locations +// ); +// +// // // This setting will prevent the 'My Courses' page being displayed when a student // logs in. The site front page will always show the same (logged-out) view. diff --git a/lib/xsendfilelib.php b/lib/xsendfilelib.php index 784facb8ca7..39106f7324d 100644 --- a/lib/xsendfilelib.php +++ b/lib/xsendfilelib.php @@ -47,19 +47,30 @@ function xsendfile($filepath) { return false; } + $filepath = realpath($filepath); + + $aliased = false; + if (!empty($CFG->xsendfilealiases) and is_array($CFG->xsendfilealiases)) { + foreach ($CFG->xsendfilealiases as $alias=>$dir) { + $dir = realpath($dir).PATH_SEPARATOR; + if (strpos($filepath, $dir) === 0) { + $filepath = $alias.substr($filepath, strlen($dir)); + $aliased = true; + break; + } + } + } + if ($CFG->xsendfile === 'X-LIGHTTPD-send-file') { // http://redmine.lighttpd.net/projects/lighttpd/wiki/X-LIGHTTPD-send-file says 1.4 it does not support byteserving header('Accept-Ranges: none'); } else if ($CFG->xsendfile === 'X-Accel-Redirect') { // http://wiki.nginx.org/XSendfile - // Nginx is using relative path to protected folder, please note you can not use cache dir, tempdir and file pool outside of dataroot! - $filepath = realpath($filepath); - $dataroot = realpath($CFG->dataroot); - if (strpos($filepath, $dataroot) !== 0) { + // Nginx requires paths relative to aliases, you need to specify them in config.php + if (!$aliased) { return false; } - $filepath = substr($filepath, strlen($dataroot)); } header("$CFG->xsendfile: $filepath"); From e61f888d36a4012e4744044a9e58402c7a981a14 Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Mon, 30 Apr 2012 17:54:24 +0200 Subject: [PATCH 5/5] MDL-26028 fix invalid separator constant and support root dirs in aliases Credit goes to Loic Jeannin, thanks. --- lib/xsendfilelib.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/xsendfilelib.php b/lib/xsendfilelib.php index 39106f7324d..3a563fd892a 100644 --- a/lib/xsendfilelib.php +++ b/lib/xsendfilelib.php @@ -52,7 +52,14 @@ function xsendfile($filepath) { $aliased = false; if (!empty($CFG->xsendfilealiases) and is_array($CFG->xsendfilealiases)) { foreach ($CFG->xsendfilealiases as $alias=>$dir) { - $dir = realpath($dir).PATH_SEPARATOR; + $dir = realpath($dir); + if ($dir === false) { + continue; + } + if (substr($dir, -1) !== DIRECTORY_SEPARATOR) { + // add trailing dir separator + $dir .= DIRECTORY_SEPARATOR; + } if (strpos($filepath, $dir) === 0) { $filepath = $alias.substr($filepath, strlen($dir)); $aliased = true;