L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MDL-22532 support for fetching of large files via temp files

Browse Source
This commit is contained in:
Petr Skoda
2010-05-20 17:13:07 +00:00
parent 645ba122dc
commit 3a1055a548
3 changed files with 82 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/file/file_storage.php
+30 -8
View File
@@ -45,6 +45,8 @@ class file_storage {
private $filedir;
/** @var string Contents of deleted files not needed any more */
private $trashdir;
/** @var string tempdir */
private $tempdir;
/** @var int Permissions for new directories */
private $dirpermissions;
/** @var int Permissions for new files */
@@ -55,12 +57,14 @@ class file_storage {
*
* @param string $filedir full path to pool directory
* @param string $trashdir temporary storage of deleted area
* @param string $tempdir temporary storage of various files
* @param int $dirpermissions new directory permissions
* @param int $filepermissions new file permissions
*/
public function __construct($filedir, $trashdir, $dirpermissions, $filepermissions) {
public function __construct($filedir, $trashdir, $tempdir, $dirpermissions, $filepermissions) {
$this->filedir = $filedir;
$this->trashdir = $trashdir;
$this->tempdir = $tempdir;
$this->dirpermissions = $dirpermissions;
$this->filepermissions = $filepermissions;
@@ -575,9 +579,10 @@ class file_storage {
* @param mixed $file_record object or array describing file
* @param string $path path to file or content of file
* @param array $options @see download_file_content() options
* @param bool $usetempfile use temporary file for download, may prevent out of memory problems
* @return stored_file instance
*/
public function create_file_from_url($file_record, $url, array $options = NULL) {
public function create_file_from_url($file_record, $url, array $options = NULL, $usetempfile = false) {
$file_record = (array)$file_record; //do not modify the submitted record, this cast unlinks objects
$file_record = (object)$file_record; // we support arrays too
@@ -589,11 +594,6 @@ class file_storage {
$connecttimeout = isset($options['connecttimeout']) ? $options['connecttimeout'] : 20;
$skipcertverify = isset($options['skipcertverify']) ? $options['skipcertverify'] : false;
// TODO: it might be better to add a new option to download file content to temp file,
// the problem here is that the size of file is limited by available memory
$content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify);
if (!isset($file_record->filename)) {
$parts = explode('/', $url);
$filename = array_pop($parts);
@@ -602,7 +602,29 @@ class file_storage {
$source = !empty($file_record->source) ? $file_record->source : $url;
$file_record->source = clean_param($source, PARAM_URL);
return $this->create_file_from_string($file_record, $content);
if ($usetempfile) {
check_dir_exists($this->tempdir, true, true);
$tmpfile = tempnam($this->tempdir, 'newfromurl');
$content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, $tmpfile);
if ($content === false) {
throw new file_exception('storedfileproblem', 'Can not fetch file form URL');
}
try {
$newfile = $this->create_file_from_pathname($file_record, $tmpfile);
@unlink($tmpfile);
return $newfile;
} catch (Exception $e) {
@unlink($tmpfile);
throw $e;
}
} else {
$content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify);
if ($content === false) {
throw new file_exception('storedfileproblem', 'Can not fetch file form URL');
}
return $this->create_file_from_string($file_record, $content);
}
}
/**
lib/filelib.php
+51 -16
View File
@@ -901,9 +901,10 @@ function format_postdata_for_curlcall($postdata) {
* usually happens if the remote server is completely down (default 20 seconds);
* may not work when using proxy
* @param bool $skipcertverify If true, the peer's SSL certificate will not be checked. Only use this when already in a trusted location.
* @return mixed false if request failed or content of the file as string if ok.
* @param string $tofile store the downloaded content to file instead of returning it
* @return mixed false if request failed or content of the file as string if ok. true if file downloaded into $tofile successfully.
*/
function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false) {
function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false, $tofile=NULL) {
global $CFG;
// some extra security
@@ -958,7 +959,7 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
}
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $connecttimeout);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
if (!ini_get('open_basedir') and !ini_get('safe_mode')) {
@@ -1006,12 +1007,26 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
}
}
$data = curl_exec($ch);
// set up header and ocntent handlers
$received = new object();
$received->headers = array(); // received headers array
$received->tofile = $tofile;
$received->fh = null;
curl_setopt($ch, CURLOPT_HEADERFUNCTION, partial('download_file_content_header_handler', $received));
if ($tofile) {
curl_setopt($ch, CURLOPT_WRITEFUNCTION, partial('download_file_content_write_handler', $received));
}
$result = curl_exec($ch);
// try to detect encoding problems
if ((curl_errno($ch) == 23 or curl_errno($ch) == 61) and defined('CURLOPT_ENCODING')) {
curl_setopt($ch, CURLOPT_ENCODING, 'none');
$data = curl_exec($ch);
$result = curl_exec($ch);
}
if ($received->fh) {
fclose($received->fh);
}
if (curl_errno($ch)) {
@@ -1028,7 +1043,7 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
}
$response->headers = array();
$response->response_code = $error;
$response->results = '';
$response->results = false;
$response->error = $error;
return $response;
} else {
@@ -1046,21 +1061,15 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
$response->status = '0';
$response->headers = array();
$response->response_code = 'Unknown cURL error';
$response->results = ''; // do NOT change this!
$response->results = false; // do NOT change this, we really want to ignore the result!
$response->error = 'Unknown cURL error';
} else {
// strip redirect headers and get headers array and content
$data = explode("\r\n\r\n", $data, $info['redirect_count'] + 2);
$results = array_pop($data);
$headers = array_pop($data);
$headers = explode("\r\n", trim($headers));
$response = new object();;
$response->status = (string)$info['http_code'];
$response->headers = $headers;
$response->response_code = $headers[0];
$response->results = $results;
$response->headers = $received->headers;
$response->response_code = $received->headers[0];
$response->results = $result;
$response->error = '';
}
@@ -1075,6 +1084,32 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
}
}
/**
* internal implementation
*/
function download_file_content_header_handler($received, $ch, $header) {
$received->headers[] = $header;
return strlen($header);
}
/**
* internal implementation
*/
function download_file_content_write_handler($received, $ch, $data) {
if (!$received->fh) {
$received->fh = fopen($received->tofile, 'w');
if ($received->fh === false) {
// bad luck, file creation or overriding failed
return 0;
}
}
if (fwrite($received->fh, $data) === false) {
// bad luck, write failed, let's abort completely
return 0;
}
return strlen($data);
}
/**
* @return array List of information about file types based on extensions.
* Associative array of extension (lower-case) to associative array
lib/moodlelib.php
+1 -1
View File
@@ -5156,7 +5156,7 @@ function get_file_storage() {
$trashdirdir = $CFG->dataroot.'/trashdir';
}
$fs = new file_storage($filedir, $trashdirdir, $CFG->directorypermissions, $CFG->filepermissions);
$fs = new file_storage($filedir, $trashdirdir, "$CFG->dataroot/temp/filestorage", $CFG->directorypermissions, $CFG->filepermissions);
return $fs;
}