diff --git a/lib/file/file_storage.php b/lib/file/file_storage.php index 32b898f0da7..7e175255102 100644 --- a/lib/file/file_storage.php +++ b/lib/file/file_storage.php @@ -45,6 +45,8 @@ class file_storage { private $filedir; /** @var string Contents of deleted files not needed any more */ private $trashdir; + /** @var string tempdir */ + private $tempdir; /** @var int Permissions for new directories */ private $dirpermissions; /** @var int Permissions for new files */ @@ -55,12 +57,14 @@ class file_storage { * * @param string $filedir full path to pool directory * @param string $trashdir temporary storage of deleted area + * @param string $tempdir temporary storage of various files * @param int $dirpermissions new directory permissions * @param int $filepermissions new file permissions */ - public function __construct($filedir, $trashdir, $dirpermissions, $filepermissions) { + public function __construct($filedir, $trashdir, $tempdir, $dirpermissions, $filepermissions) { $this->filedir = $filedir; $this->trashdir = $trashdir; + $this->tempdir = $tempdir; $this->dirpermissions = $dirpermissions; $this->filepermissions = $filepermissions; @@ -575,9 +579,10 @@ class file_storage { * @param mixed $file_record object or array describing file * @param string $path path to file or content of file * @param array $options @see download_file_content() options + * @param bool $usetempfile use temporary file for download, may prevent out of memory problems * @return stored_file instance */ - public function create_file_from_url($file_record, $url, array $options = NULL) { + public function create_file_from_url($file_record, $url, array $options = NULL, $usetempfile = false) { $file_record = (array)$file_record; //do not modify the submitted record, this cast unlinks objects $file_record = (object)$file_record; // we support arrays too @@ -589,11 +594,6 @@ class file_storage { $connecttimeout = isset($options['connecttimeout']) ? $options['connecttimeout'] : 20; $skipcertverify = isset($options['skipcertverify']) ? $options['skipcertverify'] : false; - // TODO: it might be better to add a new option to download file content to temp file, - // the problem here is that the size of file is limited by available memory - - $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify); - if (!isset($file_record->filename)) { $parts = explode('/', $url); $filename = array_pop($parts); @@ -602,7 +602,29 @@ class file_storage { $source = !empty($file_record->source) ? $file_record->source : $url; $file_record->source = clean_param($source, PARAM_URL); - return $this->create_file_from_string($file_record, $content); + if ($usetempfile) { + check_dir_exists($this->tempdir, true, true); + $tmpfile = tempnam($this->tempdir, 'newfromurl'); + $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, $tmpfile); + if ($content === false) { + throw new file_exception('storedfileproblem', 'Can not fetch file form URL'); + } + try { + $newfile = $this->create_file_from_pathname($file_record, $tmpfile); + @unlink($tmpfile); + return $newfile; + } catch (Exception $e) { + @unlink($tmpfile); + throw $e; + } + + } else { + $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify); + if ($content === false) { + throw new file_exception('storedfileproblem', 'Can not fetch file form URL'); + } + return $this->create_file_from_string($file_record, $content); + } } /** diff --git a/lib/filelib.php b/lib/filelib.php index be696ff518d..3eac5a23ab6 100644 --- a/lib/filelib.php +++ b/lib/filelib.php @@ -901,9 +901,10 @@ function format_postdata_for_curlcall($postdata) { * usually happens if the remote server is completely down (default 20 seconds); * may not work when using proxy * @param bool $skipcertverify If true, the peer's SSL certificate will not be checked. Only use this when already in a trusted location. - * @return mixed false if request failed or content of the file as string if ok. + * @param string $tofile store the downloaded content to file instead of returning it + * @return mixed false if request failed or content of the file as string if ok. true if file downloaded into $tofile successfully. */ -function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false) { +function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false, $tofile=NULL) { global $CFG; // some extra security @@ -958,7 +959,7 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons } curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_HEADER, true); + curl_setopt($ch, CURLOPT_HEADER, false); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $connecttimeout); curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); if (!ini_get('open_basedir') and !ini_get('safe_mode')) { @@ -1006,12 +1007,26 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons } } - $data = curl_exec($ch); + // set up header and ocntent handlers + $received = new object(); + $received->headers = array(); // received headers array + $received->tofile = $tofile; + $received->fh = null; + curl_setopt($ch, CURLOPT_HEADERFUNCTION, partial('download_file_content_header_handler', $received)); + if ($tofile) { + curl_setopt($ch, CURLOPT_WRITEFUNCTION, partial('download_file_content_write_handler', $received)); + } + + $result = curl_exec($ch); // try to detect encoding problems if ((curl_errno($ch) == 23 or curl_errno($ch) == 61) and defined('CURLOPT_ENCODING')) { curl_setopt($ch, CURLOPT_ENCODING, 'none'); - $data = curl_exec($ch); + $result = curl_exec($ch); + } + + if ($received->fh) { + fclose($received->fh); } if (curl_errno($ch)) { @@ -1028,7 +1043,7 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons } $response->headers = array(); $response->response_code = $error; - $response->results = ''; + $response->results = false; $response->error = $error; return $response; } else { @@ -1046,21 +1061,15 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons $response->status = '0'; $response->headers = array(); $response->response_code = 'Unknown cURL error'; - $response->results = ''; // do NOT change this! + $response->results = false; // do NOT change this, we really want to ignore the result! $response->error = 'Unknown cURL error'; } else { - // strip redirect headers and get headers array and content - $data = explode("\r\n\r\n", $data, $info['redirect_count'] + 2); - $results = array_pop($data); - $headers = array_pop($data); - $headers = explode("\r\n", trim($headers)); - $response = new object();; $response->status = (string)$info['http_code']; - $response->headers = $headers; - $response->response_code = $headers[0]; - $response->results = $results; + $response->headers = $received->headers; + $response->response_code = $received->headers[0]; + $response->results = $result; $response->error = ''; } @@ -1075,6 +1084,32 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons } } +/** + * internal implementation + */ +function download_file_content_header_handler($received, $ch, $header) { + $received->headers[] = $header; + return strlen($header); +} + +/** + * internal implementation + */ +function download_file_content_write_handler($received, $ch, $data) { + if (!$received->fh) { + $received->fh = fopen($received->tofile, 'w'); + if ($received->fh === false) { + // bad luck, file creation or overriding failed + return 0; + } + } + if (fwrite($received->fh, $data) === false) { + // bad luck, write failed, let's abort completely + return 0; + } + return strlen($data); +} + /** * @return array List of information about file types based on extensions. * Associative array of extension (lower-case) to associative array diff --git a/lib/moodlelib.php b/lib/moodlelib.php index 5443aacad05..de84f3a17e0 100644 --- a/lib/moodlelib.php +++ b/lib/moodlelib.php @@ -5156,7 +5156,7 @@ function get_file_storage() { $trashdirdir = $CFG->dataroot.'/trashdir'; } - $fs = new file_storage($filedir, $trashdirdir, $CFG->directorypermissions, $CFG->filepermissions); + $fs = new file_storage($filedir, $trashdirdir, "$CFG->dataroot/temp/filestorage", $CFG->directorypermissions, $CFG->filepermissions); return $fs; }