L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'wip-MDL-42016-master' of git://github.com/marinaglancy/moodle

Browse Source 
Conflicts:
	lib/db/install.xml
	lib/db/upgrade.php
	version.php
This commit is contained in:
Sam Hemelryk
2013-10-09 15:58:45 +13:00
parent e9091a28f5 5e5923ea28
commit f18f96f767
20 changed files with 465 additions and 344 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/db/install.xml
+1 -2
View File
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<XMLDB PATH="lib/db" VERSION="20131001" COMMENT="XMLDB file for core Moodle tables"
<XMLDB PATH="lib/db" VERSION="20131009" COMMENT="XMLDB file for core Moodle tables"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="../../lib/xmldb/xmldb.xsd"
>
@@ -2455,7 +2455,6 @@
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
<FIELD NAME="repositoryid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
<FIELD NAME="lastsync" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="Last time the proxy file was synced with repository"/>
<FIELD NAME="lifetime" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="How often do we have to sync proxy file with repository"/>
<FIELD NAME="reference" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="Identification of the external file. Repository plugins are interpreting it to locate the external file."/>
<FIELD NAME="referencehash" TYPE="char" LENGTH="40" NOTNULL="true" SEQUENCE="false" COMMENT="Internal implementation detail, contains SHA1 hash of the reference field. Can be indexed and used for comparison. Not meant to be used by a non-core code."/>
</FIELDS>
lib/db/upgrade.php
+15 -1
View File
@@ -2655,10 +2655,24 @@ function xmldb_main_upgrade($oldversion) {
if ($dbman->field_exists($table, $field)) {
$dbman->drop_field($table, $field);
}
// Main savepoint reached.
upgrade_main_savepoint(true, 2013100800.02);
}
if ($oldversion < 2013100900.00) {
// Define field lifetime to be dropped from files_reference.
$table = new xmldb_table('files_reference');
$field = new xmldb_field('lifetime');
// Conditionally launch drop field lifetime.
if ($dbman->field_exists($table, $field)) {
$dbman->drop_field($table, $field);
}
// Main savepoint reached.
upgrade_main_savepoint(true, 2013100900.00);
}
return true;
}
lib/filelib.php
-2
View File
@@ -928,8 +928,6 @@ function file_save_draft_area_files($draftitemid, $contextid, $component, $filea
$oldfile->get_referencefileid() != $newfile->get_referencefileid() ||
$oldfile->get_userid() != $newfile->get_userid())) {
$oldfile->replace_file_with($newfile);
// push changes to all local files that are referencing this file
$fs->update_references_to_storedfile($oldfile);
}
// unchanged file or directory - we keep it as is
lib/filestorage/file_storage.php
+11 -18
View File
@@ -1031,7 +1031,7 @@ class file_storage {
}
}
if ($key == 'referencefileid' or $key == 'referencelastsync' or $key == 'referencelifetime') {
if ($key == 'referencefileid' or $key == 'referencelastsync') {
$value = clean_param($value, PARAM_INT);
}
@@ -2102,10 +2102,7 @@ class file_storage {
$now = time();
foreach ($rs as $record) {
require_once($CFG->dirroot.'/repository/lib.php');
$repo = repository::get_instance($record->repositoryid);
$lifetime = $repo->get_reference_file_lifetime($reference);
$this->update_references($record->id, $now, $lifetime,
$this->update_references($record->id, $now, null,
$storedfile->get_contenthash(), $storedfile->get_filesize(), 0);
}
$rs->close();
@@ -2238,8 +2235,7 @@ class file_storage {
$referencefields = array('repositoryid' => 'repositoryid',
'reference' => 'reference',
'lastsync' => 'referencelastsync',
'lifetime' => 'referencelifetime');
'lastsync' => 'referencelastsync');
// id is specifically named to prevent overlaping between the two tables.
$fields = array();
@@ -2259,10 +2255,12 @@ class file_storage {
* Returns the id of the record in {files_reference} that matches the passed repositoryid and reference
*
* If the record already exists, its id is returned. If there is no such record yet,
* new one is created (using the lastsync and lifetime provided, too) and its id is returned.
* new one is created (using the lastsync provided, too) and its id is returned.
*
* @param int $repositoryid
* @param string $reference
* @param int $lastsync
* @param int $lifetime argument not used any more
* @return int
*/
private function get_or_create_referencefileid($repositoryid, $reference, $lastsync = null, $lifetime = null) {
@@ -2281,8 +2279,7 @@ class file_storage {
'repositoryid' => $repositoryid,
'reference' => $reference,
'referencehash' => sha1($reference),
'lastsync' => $lastsync,
'lifetime' => $lifetime));
'lastsync' => $lastsync));
} catch (dml_exception $e) {
// if inserting the new record failed, chances are that the race condition has just
// occured and the unique index did not allow to create the second record with the same
@@ -2316,12 +2313,11 @@ class file_storage {
*
* This function is called after synchronisation of an external file and updates the
* contenthash, filesize and status of all files that reference this external file
* as well as time last synchronised and sync lifetime (how long we don't need to call
* synchronisation for this reference).
* as well as time last synchronised.
*
* @param int $referencefileid
* @param int $lastsync
* @param int $lifetime
* @param int $lifetime argument not used any more, liefetime is returned by repository
* @param string $contenthash
* @param int $filesize
* @param int $status 0 if ok or 666 if source is missing
@@ -2330,20 +2326,17 @@ class file_storage {
global $DB;
$referencefileid = clean_param($referencefileid, PARAM_INT);
$lastsync = clean_param($lastsync, PARAM_INT);
$lifetime = clean_param($lifetime, PARAM_INT);
validate_param($contenthash, PARAM_TEXT, NULL_NOT_ALLOWED);
$filesize = clean_param($filesize, PARAM_INT);
$status = clean_param($status, PARAM_INT);
$params = array('contenthash' => $contenthash,
'filesize' => $filesize,
'status' => $status,
'referencefileid' => $referencefileid,
'lastsync' => $lastsync,
'lifetime' => $lifetime);
'referencefileid' => $referencefileid);
$DB->execute('UPDATE {files} SET contenthash = :contenthash, filesize = :filesize,
status = :status
WHERE referencefileid = :referencefileid', $params);
$data = array('id' => $referencefileid, 'lastsync' => $lastsync, 'lifetime' => $lifetime);
$data = array('id' => $referencefileid, 'lastsync' => $lastsync);
$DB->update_record('files_reference', (object)$data);
}
}
lib/filestorage/stored_file.php
+52 -42
View File
@@ -73,7 +73,7 @@ class stored_file {
$this->repository = null;
}
// make sure all reference fields exist in file_record even when it is not a reference
foreach (array('referencelastsync', 'referencelifetime', 'referencefileid', 'reference', 'repositoryid') as $key) {
foreach (array('referencelastsync', 'referencefileid', 'reference', 'repositoryid') as $key) {
if (empty($this->file_record->$key)) {
$this->file_record->$key = null;
}
@@ -97,6 +97,7 @@ class stored_file {
*/
protected function update($dataobject) {
global $DB;
$updatereferencesneeded = false;
$keys = array_keys((array)$this->file_record);
foreach ($dataobject as $field => $value) {
if (in_array($field, $keys)) {
@@ -156,6 +157,10 @@ class stored_file {
}
}
if (($field == 'contenthash' || $field == 'filesize') && $this->file_record->$field != $value) {
$updatereferencesneeded = true;
}
// adding the field
$this->file_record->$field = $value;
} else {
@@ -175,6 +180,10 @@ class stored_file {
$this->file_record->mimetype = $mimetype;
$DB->update_record('files', $this->file_record);
if ($updatereferencesneeded) {
// Either filesize or contenthash of this file have changed. Update all files that reference to it.
$this->fs->update_references_to_storedfile($this);
}
}
/**
@@ -198,12 +207,21 @@ class stored_file {
/**
* Replace the content by providing another stored_file instance
*
* @deprecated since 2.6
* @see stored_file::replace_file_with()
* @param stored_file $storedfile
*/
public function replace_content_with(stored_file $storedfile) {
debugging('Function stored_file::replace_content_with() is deprecated. Please use stored_file::replace_file_with()', DEBUG_DEVELOPER);
$filerecord = new stdClass;
$contenthash = $storedfile->get_contenthash();
$this->set_contenthash($contenthash);
$this->set_filesize($storedfile->get_filesize());
if ($this->fs->content_exists($contenthash)) {
$filerecord->contenthash = $contenthash;
} else {
throw new file_exception('storedfileproblem', 'Invalid contenthash, content must be already in filepool', $contenthash);
}
$filerecord->filesize = $storedfile->get_filesize();
$this->update($filerecord);
}
/**
@@ -278,7 +296,6 @@ class stored_file {
$this->file_record->reference = null;
$this->file_record->referencefileid = null;
$this->file_record->referencelastsync = null;
$this->file_record->referencelifetime = null;
}
/**
@@ -602,10 +619,8 @@ class stored_file {
* Updates contenthash and filesize
*/
public function sync_external_file() {
global $CFG;
if (!empty($this->file_record->referencefileid)) {
require_once($CFG->dirroot.'/repository/lib.php');
repository::sync_external_file($this);
if (!empty($this->repository)) {
$this->repository->sync_reference($this);
}
}
@@ -684,12 +699,13 @@ class stored_file {
return $this->file_record->filesize;
}
/**
/**
* Returns the size of file in bytes.
*
* @param int $filesize bytes
*/
public function set_filesize($filesize) {
debugging('Function stored_file::set_filesize() is deprecated. Please use stored_file::replace_file_with()', DEBUG_DEVELOPER);
$filerecord = new stdClass;
$filerecord->filesize = $filesize;
$this->update($filerecord);
@@ -762,22 +778,6 @@ class stored_file {
return $this->file_record->contenthash;
}
/**
* Set contenthash
*
* @param string $contenthash
*/
protected function set_contenthash($contenthash) {
// make sure the content exists in moodle file pool
if ($this->fs->content_exists($contenthash)) {
$filerecord = new stdClass;
$filerecord->contenthash = $contenthash;
$this->update($filerecord);
} else {
throw new file_exception('storedfileproblem', 'Invalid contenthash, content must be already in filepool', $contenthash);
}
}
/**
* Returns sha1 hash of all file path components sha1("contextid/component/filearea/itemid/dir/dir/filename.ext").
*
@@ -899,11 +899,26 @@ class stored_file {
}
/**
* Get reference last sync time
* Get reference life time (in seconds) after which sync is required
*
* This data is no longer stored in DB or returned by repository. Each
* repository should decide by itself when to synchronise the references.
*
* @deprecated since 2.6
* @see repository::sync_reference()
* @return int
*/
public function get_referencelifetime() {
return $this->file_record->referencelifetime;
debugging('Function stored_file::get_referencelifetime() is deprecated.', DEBUG_DEVELOPER);
if ($this->repository) {
if (method_exists($this->repository, 'get_reference_file_lifetime')) {
return $this->repository->get_reference_file_lifetime($this->get_reference());
} else {
return 24 * 60 * 60;
}
} else {
return 0;
}
}
/**
* Returns file reference
@@ -929,31 +944,28 @@ class stored_file {
* We update contenthash, filesize and status in files table if changed
* and we always update lastsync in files_reference table
*
* @param string $contenthash
* @param int $filesize
* @param int $status
* @param int $lifetime the life time of this synchronisation results
* @param null|string $contenthash if set to null contenthash is not changed
* @param int $filesize new size of the file
* @param int $status new status of the file (0 means OK, 666 - source missing)
*/
public function set_synchronized($contenthash, $filesize, $status = 0, $lifetime = null) {
global $DB;
public function set_synchronized($contenthash, $filesize, $status = 0) {
if (!$this->is_external_file()) {
return;
}
$now = time();
if ($contenthash === null) {
$contenthash = $this->file_record->contenthash;
}
if ($contenthash != $this->file_record->contenthash) {
$oldcontenthash = $this->file_record->contenthash;
}
if ($lifetime === null) {
$lifetime = $this->file_record->referencelifetime;
}
// this will update all entries in {files} that have the same filereference id
$this->fs->update_references($this->file_record->referencefileid, $now, $lifetime, $contenthash, $filesize, $status);
$this->fs->update_references($this->file_record->referencefileid, $now, null, $contenthash, $filesize, $status);
// we don't need to call update() for this object, just set the values of changed fields
$this->file_record->contenthash = $contenthash;
$this->file_record->filesize = $filesize;
$this->file_record->status = $status;
$this->file_record->referencelastsync = $now;
$this->file_record->referencelifetime = $lifetime;
if (isset($oldcontenthash)) {
$this->fs->deleted_file_cleanup($oldcontenthash);
}
@@ -961,11 +973,9 @@ class stored_file {
/**
* Sets the error status for a file that could not be synchronised
*
* @param int $lifetime the life time of this synchronisation results
*/
public function set_missingsource($lifetime = null) {
$this->set_synchronized($this->get_contenthash(), $this->get_filesize(), 666, $lifetime);
public function set_missingsource() {
$this->set_synchronized($this->file_record->contenthash, $this->file_record->filesize, 666);
}
/**
lib/filestorage/tests/file_storage_test.php
+108 -1
View File
@@ -413,7 +413,7 @@ class core_files_file_storage_testcase extends advanced_testcase {
$file2 = clone($file1);
$file2->filename = '2.txt';
$userfile2 = $fs->create_file_from_string($file2, 'file2 content');
$userfile2 = $fs->create_file_from_string($file2, 'file2 content longer');
$this->assertInstanceOf('stored_file', $userfile2);
$file3 = clone($file1);
@@ -1515,6 +1515,113 @@ class core_files_file_storage_testcase extends advanced_testcase {
$this->assertTrue($symlink2->is_external_file());
}
/**
* Make sure that when internal file is updated all references to it are
* updated immediately. When it is deleted, the references are converted
* to true copies.
*/
public function test_update_reference_internal() {
purge_all_caches();
$this->resetAfterTest(true);
$user = $this->setup_three_private_files();
$fs = get_file_storage();
$repos = repository::get_instances(array('type' => 'user'));
$repo = reset($repos);
// Create two aliases linking the same original.
$areafiles = array_values($fs->get_area_files($user->ctxid, 'user', 'private', false, 'filename', false));
$originalfile = $areafiles[0];
$this->assertInstanceOf('stored_file', $originalfile);
$contenthash = $originalfile->get_contenthash();
$filesize = $originalfile->get_filesize();
$substitutefile = $areafiles[1];
$this->assertInstanceOf('stored_file', $substitutefile);
$newcontenthash = $substitutefile->get_contenthash();
$newfilesize = $substitutefile->get_filesize();
$originalrecord = array(
'contextid' => $originalfile->get_contextid(),
'component' => $originalfile->get_component(),
'filearea' => $originalfile->get_filearea(),
'itemid' => $originalfile->get_itemid(),
'filepath' => $originalfile->get_filepath(),
'filename' => $originalfile->get_filename(),
);
$aliasrecord = $this->generate_file_record();
$aliasrecord->filepath = '/A/';
$aliasrecord->filename = 'symlink.txt';
$ref = $fs->pack_reference($originalrecord);
$symlink1 = $fs->create_file_from_reference($aliasrecord, $repo->id, $ref);
// Make sure created alias is a reference and has the same size and contenthash as source.
$this->assertEquals($contenthash, $symlink1->get_contenthash());
$this->assertEquals($filesize, $symlink1->get_filesize());
$this->assertEquals($repo->id, $symlink1->get_repository_id());
$this->assertNotEmpty($symlink1->get_referencefileid());
$referenceid = $symlink1->get_referencefileid();
$aliasrecord->filepath = '/B/';
$aliasrecord->filename = 'symlink.txt';
$ref = $fs->pack_reference($originalrecord);
$symlink2 = $fs->create_file_from_reference($aliasrecord, $repo->id, $ref);
// Make sure created alias is a reference and has the same size and contenthash as source.
$this->assertEquals($contenthash, $symlink2->get_contenthash());
$this->assertEquals($filesize, $symlink2->get_filesize());
$this->assertEquals($repo->id, $symlink2->get_repository_id());
// Make sure both aliases have the same reference id.
$this->assertEquals($referenceid, $symlink2->get_referencefileid());
// Overwrite ofiginal file.
$originalfile->replace_file_with($substitutefile);
$this->assertEquals($newcontenthash, $originalfile->get_contenthash());
$this->assertEquals($newfilesize, $originalfile->get_filesize());
// References to the internal files must be synchronised immediately.
// Refetch A/symlink.txt file.
$symlink1 = $fs->get_file($aliasrecord->contextid, $aliasrecord->component,
$aliasrecord->filearea, $aliasrecord->itemid, '/A/', 'symlink.txt');
$this->assertTrue($symlink1->is_external_file());
$this->assertEquals($newcontenthash, $symlink1->get_contenthash());
$this->assertEquals($newfilesize, $symlink1->get_filesize());
$this->assertEquals($repo->id, $symlink1->get_repository_id());
$this->assertEquals($referenceid, $symlink1->get_referencefileid());
// Refetch B/symlink.txt file.
$symlink2 = $fs->get_file($aliasrecord->contextid, $aliasrecord->component,
$aliasrecord->filearea, $aliasrecord->itemid, '/B/', 'symlink.txt');
$this->assertTrue($symlink2->is_external_file());
$this->assertEquals($newcontenthash, $symlink2->get_contenthash());
$this->assertEquals($newfilesize, $symlink2->get_filesize());
$this->assertEquals($repo->id, $symlink2->get_repository_id());
$this->assertEquals($referenceid, $symlink2->get_referencefileid());
// Remove original file.
$originalfile->delete();
// References must be converted to independend files.
// Refetch A/symlink.txt file.
$symlink1 = $fs->get_file($aliasrecord->contextid, $aliasrecord->component,
$aliasrecord->filearea, $aliasrecord->itemid, '/A/', 'symlink.txt');
$this->assertFalse($symlink1->is_external_file());
$this->assertEquals($newcontenthash, $symlink1->get_contenthash());
$this->assertEquals($newfilesize, $symlink1->get_filesize());
$this->assertNull($symlink1->get_repository_id());
$this->assertNull($symlink1->get_referencefileid());
// Refetch B/symlink.txt file.
$symlink2 = $fs->get_file($aliasrecord->contextid, $aliasrecord->component,
$aliasrecord->filearea, $aliasrecord->itemid, '/B/', 'symlink.txt');
$this->assertFalse($symlink2->is_external_file());
$this->assertEquals($newcontenthash, $symlink2->get_contenthash());
$this->assertEquals($newfilesize, $symlink2->get_filesize());
$this->assertNull($symlink2->get_repository_id());
$this->assertNull($symlink2->get_referencefileid());
}
public function test_get_unused_filename() {
global $USER;
$this->resetAfterTest(true);
lib/phpunit/classes/util.php
-3
View File
@@ -201,9 +201,6 @@ class phpunit_util extends testing_util {
events_get_handlers('reset');
core_text::reset_caches();
get_message_processors(false, true);
if (class_exists('repository')) {
repository::reset_caches();
}
filter_manager::reset_caches();
//TODO MDL-25290: add more resets here and probably refactor them to new core function
lib/setuplib.php
+1 -1
View File
@@ -1161,7 +1161,7 @@ function disable_output_buffering() {
*/
function redirect_if_major_upgrade_required() {
global $CFG;
$lastmajordbchanges = 2013091000.03;
$lastmajordbchanges = 2013100400.02;
if (empty($CFG->version) or (float)$CFG->version < $lastmajordbchanges or
during_initial_install() or !empty($CFG->adminsetuppending)) {
try {
lib/upgrade.txt
+11
View File
@@ -101,6 +101,17 @@ Navigation:
* settings_navigation::
get_course_modules() -> (no replacement)
Files and repositories:
* stored_file::replace_content_with() -> stored_file::replace_file_with()
* stored_file::set_filesize() -> stored_file::replace_file_with()
* stored_file::get_referencelifetime() -> (no replacement)
* repository::sync_external_file() -> see repository::sync_reference()
* repository::get_file_by_reference() -> repository::sync_reference()
* repository::
get_reference_file_lifetime() -> (no replacement)
* repository::sync_individual_file() -> (no replacement)
* repository::reset_caches() -> (no replacement)
Calendar:
* add_event() -> calendar_event::create()
* update_event() -> calendar_event->update()
mod/assign/feedback/file/importziplib.php
+1 -1
View File
@@ -253,7 +253,7 @@ class assignfeedback_file_zip_importer {
'/',
$filename)) {
// Update existing feedback file.
$oldfile->replace_content_with($unzippedfile);
$oldfile->replace_file_with($unzippedfile);
$feedbackfilesupdated++;
} else {
// Create a new feedback file.
repository/boxnet/lib.php
+28 -34
View File
@@ -267,34 +267,34 @@ class repository_boxnet extends repository {
return $source;
}
/**
* Returns information about file in this repository by reference
* {@link repository::get_file_reference()}
* {@link repository::get_file()}
*
* Returns null if file not found or is not readable
*
* @param stdClass $reference file reference db record
* @return null|stdClass with attribute 'filepath'
*/
public function get_file_by_reference($reference) {
$array = explode('/', $reference->reference);
$fileid = array_pop($array);
$fileinfo = $this->boxclient->get_file_info($fileid, self::SYNCFILE_TIMEOUT);
if ($fileinfo) {
$size = (int)$fileinfo->size;
if (file_extension_in_typegroup($fileinfo->file_name, 'web_image')) {
// this is an image - download it to moodle
$path = $this->prepare_file('');
$c = new curl;
$result = $c->download_one($reference->reference, null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT));
if ($result === true) {
return (object)array('filepath' => $path);
}
}
return (object)array('filesize' => $size);
public function sync_reference(stored_file $file) {
if ($file->get_referencelastsync() + DAYSECS > time()) {
// Synchronise not more often than once a day.
return false;
}
return null;
$c = new curl;
if (file_extension_in_typegroup($file->get_filename(), 'web_image')) {
$path = $this->prepare_file('');
$result = $c->download_one($file->get_reference(), null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT));
$info = $c->get_info();
if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path);
$file->set_synchronized($contenthash, $filesize);
return true;
}
}
$c->get($file->get_reference(), null, array('timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true, 'nobody' => true));
$info = $c->get_info();
if (isset($info['http_code']) && $info['http_code'] == 200 &&
array_key_exists('download_content_length', $info) &&
$info['download_content_length'] >= 0) {
$filesize = (int)$info['download_content_length'];
$file->set_synchronized(null, $filesize);
return true;
}
$file->set_missingsource();
return true;
}
/**
@@ -307,14 +307,8 @@ class repository_boxnet extends repository {
*/
public function get_reference_details($reference, $filestatus = 0) {
// Indicate it's from box.net repository + secure URL
$array = explode('/', $reference);
$fileid = array_pop($array);
$fileinfo = $this->boxclient->get_file_info($fileid, self::SYNCFILE_TIMEOUT);
if (!empty($fileinfo)) {
$reference = (string)$fileinfo->file_name;
}
$details = $this->get_name() . ': ' . $reference;
if (!empty($fileinfo)) {
if (!$filestatus) {
return $details;
} else {
return get_string('lostsource', 'repository', $details);
repository/coursefiles/lib.php
-11
View File
@@ -211,17 +211,6 @@ class repository_coursefiles extends repository {
return true;
}
/**
* Return reference file life time
*
* @param string $ref
* @return int
*/
public function get_reference_file_lifetime($ref) {
// this should be realtime
return 0;
}
/**
* Is this repository accessing private data?
*
repository/dropbox/lib.php
+18 -20
View File
@@ -546,25 +546,18 @@ class repository_dropbox extends repository {
return serialize($reference);
}
/**
* Returns information about file in this repository by reference
* {@link repository::get_file_reference()}
* {@link repository::get_file()}
*
* Returns null if file not found or is not readable
*
* @param stdClass $reference file reference db record
* @return null|stdClass that has 'filepath' property
*/
public function get_file_by_reference($reference) {
global $USER;
$ref = unserialize($reference->reference);
public function sync_reference(stored_file $file) {
if ($file->get_referencelastsync() + DAYSECS > time()) {
// Synchronise not more often than once a day.
return false;
}
$ref = unserialize($file->get_reference());
if (!isset($ref->url)) {
// this is an old-style reference in DB. We need to fix it
$ref = unserialize($this->fix_old_style_reference($reference->reference));
$ref = unserialize($this->fix_old_style_reference($file->get_reference()));
}
if (!isset($ref->url)) {
return null;
return false;
}
$c = new curl;
$url = $this->get_file_download_link($ref->url);
@@ -574,7 +567,10 @@ class repository_dropbox extends repository {
$result = $c->download_one($url, array(), array('filepath' => $saveas, 'timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true));
$info = $c->get_info();
if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
return (object)array('filepath' => $saveas);
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($saveas);
$file->set_synchronized($contenthash, $filesize);
return true;
}
} catch (Exception $e) {}
}
@@ -583,9 +579,12 @@ class repository_dropbox extends repository {
if (isset($info['http_code']) && $info['http_code'] == 200 &&
array_key_exists('download_content_length', $info) &&
$info['download_content_length'] >= 0) {
return (object)array('filesize' => (int)$info['download_content_length']);
$filesize = (int)$info['download_content_length'];
$file->set_synchronized(null, $filesize);
return true;
}
return null;
$file->set_missingsource();
return true;
}
/**
@@ -648,8 +647,7 @@ class repository_dropbox extends repository {
/**
* Returns the maximum size of the Dropbox files to cache in moodle
*
* Note that {@link repository_dropbox::get_file_by_reference()} called by
* {@link repository::sync_external_file()} will try to cache images even
* Note that {@link repository_dropbox::sync_reference()} will try to cache images even
* when they are bigger in order to generate thumbnails. However there is
* a small timeout for downloading images for synchronisation and it will
* probably fail if the image is too big.
repository/equella/lib.php
+19 -33
View File
@@ -134,7 +134,7 @@ class repository_equella extends repository {
* If we received the connection timeout more than 3 times in a row, we don't attemt to
* connect to the server any more during this request.
*
* This function is used by {@link repository_equella::get_file_by_reference()} that
* This function is used by {@link repository_equella::sync_reference()} that
* synchronises the file size of referenced files.
*
* @param int $errno omit if we just want to know the return value, the last curl_errno otherwise
@@ -159,18 +159,6 @@ class repository_equella extends repository {
return ($countfailures[$sess] < 3);
}
/**
* Decide whether or not the file should be synced
*
* @param stored_file $storedfile
* @return bool
*/
public function sync_individual_file(stored_file $storedfile) {
// if we had several unsuccessfull attempts to connect to server - do not try any more
return $this->connection_result();
}
/**
* Download a file, this function can be overridden by subclass. {@link curl}
*
@@ -202,34 +190,30 @@ class repository_equella extends repository {
return array('path'=>$path, 'url'=>$url);
}
/**
* Returns information about file in this repository by reference
*
* If the file is an image we download the contents and save it in our filesystem
* so we can generate thumbnails. Otherwise we just request the file size.
* Returns null if file not found or can not be accessed
*
* @param stdClass $reference file reference db record
* @return stdClass|null contains one of the following:
* - 'filesize' (for non-image files or files we failed to retrieve fully because of timeout)
* - 'filepath' (for image files that we retrieived and saved)
*/
public function get_file_by_reference($reference) {
public function sync_reference(stored_file $file) {
global $USER;
$ref = @unserialize(base64_decode($reference->reference));
if ($file->get_referencelastsync() + DAYSECS > time() || !$this->connection_result()) {
// Synchronise not more often than once a day.
// if we had several unsuccessfull attempts to connect to server - do not try any more.
return false;
}
$ref = @unserialize(base64_decode($file->get_reference()));
if (!isset($ref->url) || !($url = $this->appendtoken($ref->url))) {
// Occurs when the user isn't known..
return null;
$file->set_missingsource();
return true;
}
$return = null;
$cookiepathname = $this->prepare_file($USER->id. '_'. uniqid('', true). '.cookie');
$c = new curl(array('cookie' => $cookiepathname));
if (file_extension_in_typegroup($ref->filename, 'web_image')) {
$path = $this->prepare_file('');
$result = $c->download_one($url, null, array('filepath' => $path, 'followlocation' => true, 'timeout' => self::SYNCIMAGE_TIMEOUT));
if ($result === true) {
$return = (object)array('filepath' => $path);
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path);
$file->set_synchronized($contenthash, $filesize);
return true;
}
} else {
$result = $c->head($url, array('followlocation' => true, 'timeout' => self::SYNCFILE_TIMEOUT));
@@ -241,13 +225,15 @@ class repository_equella extends repository {
$this->connection_result($c->get_errno());
$curlinfo = $c->get_info();
if ($return === null && isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200
if (isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200
&& array_key_exists('download_content_length', $curlinfo)
&& $curlinfo['download_content_length'] >= 0) {
// we received a correct header and at least can tell the file size
$return = (object)array('filesize' => $curlinfo['download_content_length']);
$file->set_synchronized(null, $curlinfo['download_content_length']);
return true;
}
return $return;
$file->set_missingsource();
return true;
}
/**
repository/filesystem/lib.php
+27 -35
View File
@@ -265,17 +265,6 @@ class repository_filesystem extends repository {
return FILE_INTERNAL | FILE_REFERENCE;
}
/**
* Return reference file life time
*
* @param string $ref
* @return int
*/
public function get_reference_file_lifetime($ref) {
// Does not cost us much to synchronise within our own filesystem, set to 1 minute
return 60;
}
/**
* Return human readable reference information
*
@@ -292,37 +281,40 @@ class repository_filesystem extends repository {
}
}
/**
* Returns information about file in this repository by reference
*
* Returns null if file not found or is not readable
*
* @param stdClass $reference file reference db record
* @return stdClass|null contains one of the following:
* - 'filesize' if file should not be copied to moodle filepool
* - 'filepath' if file should be copied to moodle filepool
*/
public function get_file_by_reference($reference) {
$ref = $reference->reference;
if ($ref{0} == '/') {
$filepath = $this->root_path.substr($ref, 1, strlen($ref)-1);
} else {
$filepath = $this->root_path.$ref;
public function sync_reference(stored_file $file) {
if ($file->get_referencelastsync() + 60 > time()) {
// Does not cost us much to synchronise within our own filesystem, check every 1 minute.
return false;
}
static $issyncing = false;
if ($issyncing) {
// Avoid infinite recursion when calling $file->get_filesize() and get_contenthash().
return;
}
$filepath = $this->root_path.ltrim($file->get_reference(), '/');
if (file_exists($filepath) && is_readable($filepath)) {
$fs = get_file_storage();
$issyncing = true;
if (file_extension_in_typegroup($filepath, 'web_image')) {
// return path to image files so it will be copied into moodle filepool
// we need the file in filepool to generate an image thumbnail
return (object)array('filepath' => $filepath);
$contenthash = sha1_file($filepath);
if ($file->get_contenthash() == $contenthash) {
// File did not change since the last synchronisation.
$filesize = filesize($filepath);
} else {
// Copy file into moodle filepool (used to generate an image thumbnail).
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($filepath);
}
} else {
// return just the file size so file will NOT be copied into moodle filepool
return (object)array(
'filesize' => filesize($filepath)
);
// Update only file size so file will NOT be copied into moodle filepool.
$contenthash = null;
$filesize = filesize($filepath);
}
$issyncing = false;
$file->set_synchronized($contenthash, $filesize);
} else {
return null;
$file->set_missingsource();
}
return true;
}
/**
repository/lib.php
+160 -115
View File
@@ -559,6 +559,35 @@ abstract class repository implements cacheable_object {
$this->super_called = true;
}
/**
* Magic method for non-existing (usually deprecated) class methods.
*
* @param string $name
* @param array $arguments
* @return mixed
* @throws coding_exception
*/
public function __call($name, $arguments) {
if ($name === 'sync_individual_file') {
// Method repository::sync_individual_file() was deprecated in Moodle 2.6.
// See repository::sync_reference().
debugging('Function repository::sync_individual_file() is deprecated.', DEBUG_DEVELOPER);
return true;
} else if ($name === 'get_file_by_reference') {
// Method repository::get_file_by_reference() was deprecated in Moodle 2.6.
// See repository::sync_reference().
debugging('Function repository::get_file_by_reference() is deprecated.', DEBUG_DEVELOPER);
return null;
} else if ($name === 'get_reference_file_lifetime') {
// Method repository::get_file_by_reference() was deprecated in Moodle 2.6.
// See repository::sync_reference().
debugging('Function repository::get_reference_file_lifetime() is deprecated.', DEBUG_DEVELOPER);
return 24 * 60 * 60;
} else {
throw new coding_exception('Tried to call unknown method '.get_class($this).'::'.$name);
}
}
/**
* Get repository instance using repository id
*
@@ -1276,27 +1305,6 @@ abstract class repository implements cacheable_object {
}
}
/**
* Return reference file life time
*
* @param string $ref
* @return int
*/
public function get_reference_file_lifetime($ref) {
// One day
return 60 * 60 * 24;
}
/**
* Decide whether or not the file should be synced
*
* @param stored_file $storedfile
* @return bool
*/
public function sync_individual_file(stored_file $storedfile) {
return true;
}
/**
* Return human readable reference information
*
@@ -1345,50 +1353,6 @@ abstract class repository implements cacheable_object {
public function cache_file_by_reference($reference, $storedfile) {
}
/**
* Returns information about file in this repository by reference
*
* This function must be implemented for repositories supporting FILE_REFERENCE, it is called
* for existing aliases when the lifetime of the previous syncronisation has expired.
*
* Returns null if file not found or is not readable or timeout occured during request.
* Note that this function may be run for EACH file that needs to be synchronised at the
* moment. If anything is being downloaded or requested from external sources there
* should be a small timeout. The synchronisation is performed to update the size of
* the file and/or to update image and re-generated image preview. There is nothing
* fatal if syncronisation fails but it is fatal if syncronisation takes too long
* and hangs the script generating a page.
*
* If get_file_by_reference() returns filesize just the record in {files} table is being updated.
* If filepath, handle or content are returned - the file is also stored in moodle filepool
* (recommended for images to generate the thumbnails). For non-image files it is not
* recommended to download them to moodle during syncronisation since it may take
* unnecessary long time.
*
* @param stdClass $reference record from DB table {files_reference}
* @return stdClass|null contains one of the following:
* - 'filesize' and optionally 'contenthash'
* - 'filepath'
* - 'handle'
* - 'content'
*/
public function get_file_by_reference($reference) {
if ($this->has_moodle_files() && isset($reference->reference)) {
$fs = get_file_storage();
$params = file_storage::unpack_reference($reference->reference, true);
if (!is_array($params) || !($storedfile = $fs->get_file($params['contextid'],
$params['component'], $params['filearea'], $params['itemid'], $params['filepath'],
$params['filename']))) {
return null;
}
return (object)array(
'contenthash' => $storedfile->get_contenthash(),
'filesize' => $storedfile->get_filesize()
);
}
return null;
}
/**
* Return the source information
*
@@ -1771,7 +1735,7 @@ abstract class repository implements cacheable_object {
/**
* Downloads the file from external repository and saves it in moodle filepool.
* This function is different from {@link repository::sync_external_file()} because it has
* This function is different from {@link repository::sync_reference()} because it has
* bigger request timeout and always downloads the content.
*
* This function is invoked when we try to unlink the file from the source and convert
@@ -1809,10 +1773,7 @@ abstract class repository implements cacheable_object {
// content for the file that was not actually downloaded
$contentexists = false;
}
$now = time();
if ($file->get_referencelastsync() + $file->get_referencelifetime() >= $now &&
!$file->get_status() &&
$contentexists) {
if (!$file->get_status() && $contentexists) {
// we already have the content in moodle filepool and it was synchronised recently.
// Repositories may overwrite it if they want to force synchronisation anyway!
return;
@@ -1823,8 +1784,7 @@ abstract class repository implements cacheable_object {
if (isset($fileinfo['path'])) {
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($fileinfo['path']);
// set this file and other similar aliases synchronised
$lifetime = $this->get_reference_file_lifetime($file->get_reference());
$file->set_synchronized($contenthash, $filesize, 0, $lifetime);
$file->set_synchronized($contenthash, $filesize);
} else {
throw new moodle_exception('errorwhiledownload', 'repository', '', '');
}
@@ -2724,67 +2684,150 @@ abstract class repository implements cacheable_object {
}
/**
* Called from phpunit between tests, resets whatever was cached
* Method deprecated, cache is handled by MUC now.
* @deprecated since 2.6
*/
public static function reset_caches() {
self::sync_external_file(null, true);
debugging('Function repository::reset_caches() is deprecated.', DEBUG_DEVELOPER);
}
/**
* Performs synchronisation of reference to an external file if the previous one has expired.
*
* @param stored_file $file
* @param bool $resetsynchistory whether to reset all history of sync (used by phpunit)
* @return bool success
* Method deprecated
* @deprecated since 2.6
* @see repository::sync_reference()
*/
public static function sync_external_file($file, $resetsynchistory = false) {
global $DB;
// TODO MDL-25290 static should be replaced with MUC code.
static $synchronized = array();
if ($resetsynchistory) {
$synchronized = array();
}
$fs = get_file_storage();
if (!$file || !$file->get_referencefileid()) {
debugging('Function repository::sync_external_file() is deprecated.',
DEBUG_DEVELOPER);
if ($resetsynchistory || !$file || !$file->get_repository_id() ||
!($repository = self::get_repository_by_id($file->get_repository_id(), SYSCONTEXTID))) {
return false;
}
if (array_key_exists($file->get_id(), $synchronized)) {
return $synchronized[$file->get_id()];
}
return $repository->sync_reference($file);
}
// remember that we already cached in current request to prevent from querying again
$synchronized[$file->get_id()] = false;
if (!$reference = $DB->get_record('files_reference', array('id'=>$file->get_referencefileid()))) {
/**
* Performs synchronisation of an external file if the previous one has expired.
*
* This function must be implemented for external repositories supporting
* FILE_REFERENCE, it is called for existing aliases when their filesize,
* contenthash or timemodified are requested. It is not called for internal
* repositories (see {@link repository::has_moodle_files()}), references to
* internal files are updated immediately when source is modified.
*
* Referenced files may optionally keep their content in Moodle filepool (for
* thumbnail generation or to be able to serve cached copy). In this
* case both contenthash and filesize need to be synchronized. Otherwise repositories
* should use contenthash of empty file and correct filesize in bytes.
*
* Note that this function may be run for EACH file that needs to be synchronised at the
* moment. If anything is being downloaded or requested from external sources there
* should be a small timeout. The synchronisation is performed to update the size of
* the file and/or to update image and re-generated image preview. There is nothing
* fatal if syncronisation fails but it is fatal if syncronisation takes too long
* and hangs the script generating a page.
*
* Note: If you wish to call $file->get_filesize(), $file->get_contenthash() or
* $file->get_timemodified() make sure that recursion does not happen.
*
* Called from {@link stored_file::sync_external_file()}
*
* @uses stored_file::set_missingsource()
* @uses stored_file::set_synchronized()
* @param stored_file $file
* @return bool false when file does not need synchronisation, true if it was synchronised
*/
public function sync_reference(stored_file $file) {
if ($file->get_repository_id() != $this->id) {
// This should not really happen because the function can be called from stored_file only.
return false;
}
if (!empty($reference->lastsync) and ($reference->lastsync + $reference->lifetime > time())) {
$synchronized[$file->get_id()] = true;
if ($this->has_moodle_files()) {
// References to local files need to be synchronised only once.
// Later they will be synchronised automatically when the source is changed.
if ($file->get_referencelastsync()) {
return false;
}
$fs = get_file_storage();
$params = file_storage::unpack_reference($file->get_reference(), true);
if (!is_array($params) || !($storedfile = $fs->get_file($params['contextid'],
$params['component'], $params['filearea'], $params['itemid'], $params['filepath'],
$params['filename']))) {
$file->set_missingsource();
} else {
$file->set_synchronized($storedfile->get_contenthash(), $storedfile->get_filesize());
}
return true;
}
if (!$repository = self::get_repository_by_id($reference->repositoryid, SYSCONTEXTID)) {
// Backward compatibility (Moodle 2.3-2.5) implementation that calls
// methods repository::get_reference_file_lifetime(), repository::sync_individual_file()
// and repository::get_file_by_reference(). These methods are removed from the
// base repository class but may still be implemented by the child classes.
// THIS IS NOT A GOOD EXAMPLE of implementation. For good examples see the overwriting methods.
if (!method_exists($this, 'get_file_by_reference')) {
// Function get_file_by_reference() is not implemented. No synchronisation.
return false;
}
if (!$repository->sync_individual_file($file)) {
// Check if the previous sync result is still valid.
if (method_exists($this, 'get_reference_file_lifetime')) {
$lifetime = $this->get_reference_file_lifetime($file->get_reference());
} else {
// Default value that was hardcoded in Moodle 2.3 - 2.5.
$lifetime = 60 * 60 * 24;
}
if (($lastsynced = $file->get_referencelastsync()) && $lastsynced + $lifetime >= time()) {
return false;
}
$lifetime = $repository->get_reference_file_lifetime($reference);
$fileinfo = $repository->get_file_by_reference($reference);
if ($fileinfo === null) {
// does not exist any more - set status to missing
$file->set_missingsource($lifetime);
$synchronized[$file->get_id()] = true;
return true;
$cache = cache::make('core', 'repositories');
if (($lastsyncresult = $cache->get('sync:'.$file->get_referencefileid())) !== false) {
if ($lastsyncresult === true) {
// We are in the process of synchronizing this reference.
// Avoid recursion when calling $file->get_filesize() and $file->get_contenthash().
return false;
} else {
// We have synchronised the same reference inside this request already.
// It looks like the object $file was created before the synchronisation and contains old data.
if (!empty($lastsyncresult['missing'])) {
$file->set_missingsource();
} else {
$cache->set('sync:'.$file->get_referencefileid(), true);
if ($file->get_contenthash() != $lastsyncresult['contenthash'] ||
$file->get_filesize() != $lastsyncresult['filesize']) {
$file->set_synchronized($lastsyncresult['contenthash'], $lastsyncresult['filesize']);
}
$cache->set('sync:'.$file->get_referencefileid(), $lastsyncresult);
}
return true;
}
}
// Weird function sync_individual_file() that was present in API in 2.3 - 2.5, default value was true.
if (method_exists($this, 'sync_individual_file') && !$this->sync_individual_file($file)) {
return false;
}
// Set 'true' into the cache to indicate that file is in the process of synchronisation.
$cache->set('sync:'.$file->get_referencefileid(), true);
// Create object with the structure that repository::get_file_by_reference() expects.
$reference = new stdClass();
$reference->id = $file->get_referencefileid();
$reference->reference = $file->get_reference();
$reference->referencehash = sha1($file->get_reference());
$reference->lastsync = $file->get_referencelastsync();
$reference->lifetime = $lifetime;
$fileinfo = $this->get_file_by_reference($reference);
$contenthash = null;
$filesize = null;
$fs = get_file_storage();
if (!empty($fileinfo->filesize)) {
// filesize returned
if (!empty($fileinfo->contenthash) && $fs->content_exists($fileinfo->contenthash)) {
@@ -2796,8 +2839,7 @@ abstract class repository implements cacheable_object {
$contenthash = $file->get_contenthash();
} else {
// we can't save empty contenthash so generate contenthash from empty string
$fs->add_string_to_pool('');
$contenthash = sha1('');
list($contenthash, $unused1, $unused2) = $fs->add_string_to_pool('');
}
$filesize = $fileinfo->filesize;
} else if (!empty($fileinfo->filepath)) {
@@ -2807,22 +2849,25 @@ abstract class repository implements cacheable_object {
// File handle returned
$contents = '';
while (!feof($fileinfo->handle)) {
$contents .= fread($handle, 8192);
$contents .= fread($fileinfo->handle, 8192);
}
fclose($fileinfo->handle);
list($contenthash, $filesize, $newfile) = $fs->add_string_to_pool($content);
list($contenthash, $filesize, $newfile) = $fs->add_string_to_pool($contents);
} else if (isset($fileinfo->content)) {
// File content returned
list($contenthash, $filesize, $newfile) = $fs->add_string_to_pool($fileinfo->content);
}
if (!isset($contenthash) or !isset($filesize)) {
return false;
$file->set_missingsource(null);
$cache->set('sync:'.$file->get_referencefileid(), array('missing' => true));
} else {
// update files table
$file->set_synchronized($contenthash, $filesize);
$cache->set('sync:'.$file->get_referencefileid(),
array('contenthash' => $contenthash, 'filesize' => $filesize));
}
// update files table
$file->set_synchronized($contenthash, $filesize, 0, $lifetime);
$synchronized[$file->get_id()] = true;
return true;
}
repository/local/lib.php
-11
View File
@@ -126,17 +126,6 @@ class repository_local extends repository {
return true;
}
/**
* Return reference file life time
*
* @param string $ref
* @return int
*/
public function get_reference_file_lifetime($ref) {
// this should be realtime
return 0;
}
/**
* Returns all children elements that have one of the specified extensions
*
repository/upgrade.txt
+12 -2
View File
@@ -7,11 +7,21 @@ http://docs.moodle.org/dev/Repository_API
* get_option() now always return null when the first parameter ($config) is not empty, and
no value was found for this $config. Previously this could sometimes return an empty array().
* The function repository_attach_id() was removed, it was never used and was not useful.
* New functions send_relative_file() and supports_relative_file() to allow sending relative linked
files - see filesystem repository for example.
* DB fields files.referencelifetime and files.referencelastsync are deleted.
Their values are stored only in files_reference.lastsync and files_reference.lifetime.
* DB fields files.referencelifetime, files.referencelastsync and files_reference.lifetime
are deleted. The last synchronization time is stored only in files_reference.lastsync
and lifetime is not stored in DB any more, each repository must decide for itself
when to synchronize the file in function repository::sync_reference().
* The following methods in class repository are deprecated: sync_external_file(),
get_file_by_reference(), get_reference_file_lifetime(), sync_individual_file() and
reset_caches(). Instead there is one method repository::sync_reference() - this simplifies
the repositories API and reduces the number of DB queries.
=== 2.5 ===
repository/user/lib.php
-11
View File
@@ -159,17 +159,6 @@ class repository_user extends repository {
return FILE_INTERNAL | FILE_REFERENCE;
}
/**
* Return reference file life time
*
* @param string $ref
* @return int
*/
public function get_reference_file_lifetime($ref) {
// this should be realtime
return 0;
}
/**
* Is this repository accessing private data?
*
version.php
+1 -1
View File
@@ -29,7 +29,7 @@
defined('MOODLE_INTERNAL') || die();
$version = 2013100800.02; // YYYYMMDD = weekly release date of this DEV branch.
$version = 2013100900.00; // YYYYMMDD = weekly release date of this DEV branch.
// RR = release increments - 00 in DEV branches.
// .XX = incremental changes.