diff --git a/lang/en/error.php b/lang/en/error.php index 52f787ed0c1..a9fde2459cf 100644 --- a/lang/en/error.php +++ b/lang/en/error.php @@ -398,6 +398,7 @@ $string['loginasnoenrol'] = 'You cannot use enrol or unenrol when in course "Log $string['loginasonecourse'] = 'You cannot enter this course.
You have to terminate the "Login as" session before entering any other course.'; $string['maxbytesfile'] = 'The file {$a->file} is too large. The maximum size you can upload is {$a->size}.'; $string['maxareabytes'] = 'The file is larger than the space remaining in this area.'; +$string['maxdraftitemids'] = 'Due to uploading a high volume of files, your file uploads are temporarily limited. Please try again after a few seconds.'; $string['messageundeliveredbynotificationsettings'] = 'The message could not be sent because personal messages between users (in Notification settings) has been disabled by a site administrator.'; $string['messagingdisable'] = 'Messaging is disabled on this site'; $string['mimetexisnotexist'] = 'Your system is not configured to run mimeTeX. You need to obtain the C source from https://www.forkosh.com/mimetex.zip, compile it and put the executable into your moodle/filter/tex/ directory.'; diff --git a/lib/filelib.php b/lib/filelib.php index b19e630c9ab..5e6849a9c93 100644 --- a/lib/filelib.php +++ b/lib/filelib.php @@ -40,6 +40,16 @@ define('IGNORE_FILE_MERGE', -1); */ define('FILE_AREA_MAX_BYTES_UNLIMITED', -1); +/** + * Capacity of the draft area bucket when using the leaking bucket technique to limit the draft upload rate. + */ +define('DRAFT_AREA_BUCKET_CAPACITY', 50); + +/** + * Leaking rate of the draft area bucket when using the leaking bucket technique to limit the draft upload rate. + */ +define('DRAFT_AREA_BUCKET_LEAK', 0.2); + require_once("$CFG->libdir/filestorage/file_exceptions.php"); require_once("$CFG->libdir/filestorage/file_storage.php"); require_once("$CFG->libdir/filestorage/zip_packer.php"); @@ -390,7 +400,7 @@ function file_get_unused_draft_itemid() { * @return string|null returns string if $text was passed in, the rewritten $text is returned. Otherwise NULL. */ function file_prepare_draft_area(&$draftitemid, $contextid, $component, $filearea, $itemid, array $options=null, $text=null) { - global $CFG, $USER, $CFG; + global $CFG, $USER; $options = (array)$options; if (!isset($options['subdirs'])) { @@ -606,6 +616,55 @@ function file_is_draft_area_limit_reached($draftitemid, $areamaxbytes, $newfiles return false; } +/** + * Returns whether a user has reached their draft area upload rate. + * + * @param int $userid The user id + * @return bool + */ +function file_is_draft_areas_limit_reached(int $userid): bool { + global $CFG; + + $capacity = $CFG->draft_area_bucket_capacity ?? DRAFT_AREA_BUCKET_CAPACITY; + $leak = $CFG->draft_area_bucket_leak ?? DRAFT_AREA_BUCKET_LEAK; + + $since = time() - floor($capacity / $leak); // The items that were in the bucket before this time are already leaked by now. + // We are going to be a bit generous to the user when using the leaky bucket + // algorithm below. We are going to assume that the bucket is empty at $since. + // We have to do an assumption here unless we really want to get ALL user's draft + // items without any limit and put all of them in the leaking bucket. + // I decided to favour performance over accuracy here. + + $fs = get_file_storage(); + $items = $fs->get_user_draft_items($userid, $since); + $items = array_reverse($items); // So that the items are sorted based on time in the ascending direction. + + // We only need to store the time that each element in the bucket is going to leak. So $bucket is array of leaking times. + $bucket = []; + foreach ($items as $item) { + $now = $item->timemodified; + // First let's see if items can be dropped from the bucket as a result of leakage. + while (!empty($bucket) && ($now >= $bucket[0])) { + array_shift($bucket); + } + + // Calculate the time that the new item we put into the bucket will be leaked from it, and store it into the bucket. + if ($bucket) { + $bucket[] = max($bucket[count($bucket) - 1], $now) + (1 / $leak); + } else { + $bucket[] = $now + (1 / $leak); + } + } + + // Recalculate the bucket's content based on the leakage until now. + $now = time(); + while (!empty($bucket) && ($now >= $bucket[0])) { + array_shift($bucket); + } + + return count($bucket) >= $capacity; +} + /** * Get used space of files * @global moodle_database $DB diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index 9e599542e2e..65945ef7200 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -663,6 +663,41 @@ class file_storage { return $result; } + /** + * Returns the file area item ids and their updatetime for a user's draft uploads, sorted by updatetime DESC. + * + * @param int $userid user id + * @param int $updatedsince only return draft areas updated since this time + * @param int $lastnum only return the last specified numbers + * @return array + */ + public function get_user_draft_items(int $userid, int $updatedsince = 0, int $lastnum = 0): array { + global $DB; + + $params = [ + 'component' => 'user', + 'filearea' => 'draft', + 'contextid' => context_user::instance($userid)->id, + ]; + + $updatedsincesql = ''; + if ($updatedsince) { + $updatedsincesql = 'AND f.timemodified > :time'; + $params['time'] = $updatedsince; + } + $sql = "SELECT itemid, + MAX(f.timemodified) AS timemodified + FROM {files} f + WHERE component = :component + AND filearea = :filearea + AND contextid = :contextid + $updatedsincesql + GROUP BY itemid + ORDER BY MAX(f.timemodified) DESC"; + + return $DB->get_records_sql($sql, $params, 0, $lastnum); + } + /** * Returns array based tree structure of area files * diff --git a/lib/tests/filelib_test.php b/lib/tests/filelib_test.php index 113273b8848..fd7a7b98b9b 100644 --- a/lib/tests/filelib_test.php +++ b/lib/tests/filelib_test.php @@ -1667,6 +1667,65 @@ EOF; $draftfiles = $fs->get_area_files($usercontext->id, 'user', 'draft', $file2->get_itemid(), 'itemid', 0); $this->assertCount(1, $draftfiles); } + + /** + * Test file_is_draft_areas_limit_reached + */ + public function test_file_is_draft_areas_limit_reached() { + global $CFG; + $this->resetAfterTest(true); + + $capacity = $CFG->draft_area_bucket_capacity = 5; + $leak = $CFG->draft_area_bucket_leak = 0.2; // Leaks every 5 seconds. + + $generator = $this->getDataGenerator(); + $user = $generator->create_user(); + + $this->setUser($user); + + $itemids = []; + for ($i = 0; $i < $capacity; $i++) { + $itemids[$i] = file_get_unused_draft_itemid(); + } + + // This test highly depends on time. We try to make sure that the test starts at the early moments on the second. + // This was not needed if MDL-37327 was implemented. + $after = time(); + while (time() === $after) { + usleep(100000); + } + + // Burst up to the capacity and make sure that the bucket allows it. + for ($i = 0; $i < $capacity; $i++) { + if ($i) { + sleep(1); // A little delay so we have different timemodified value for files. + } + $this->assertFalse(file_is_draft_areas_limit_reached($user->id)); + self::create_draft_file([ + 'filename' => 'file1.png', + 'itemid' => $itemids[$i], + ]); + } + + // The bucket should be full after bursting. + $this->assertTrue(file_is_draft_areas_limit_reached($user->id)); + + // The bucket leaks so it shouldn't be full after a certain time. + // Reiterating that this test could have been faster if MDL-37327 was implemented. + sleep(ceil(1 / $leak) - ($capacity - 1)); + $this->assertFalse(file_is_draft_areas_limit_reached($user->id)); + + // Only one item was leaked from the bucket. So the bucket should become full again if we add a single item to it. + self::create_draft_file([ + 'filename' => 'file2.png', + 'itemid' => $itemids[0], + ]); + $this->assertTrue(file_is_draft_areas_limit_reached($user->id)); + + // The bucket leaks at a constant rate. It doesn't matter if it is filled as the result of bursting or not. + sleep(ceil(1 / $leak)); + $this->assertFalse(file_is_draft_areas_limit_reached($user->id)); + } } /** diff --git a/repository/filepicker.php b/repository/filepicker.php index 112e653f4b8..9ea912e2b80 100644 --- a/repository/filepicker.php +++ b/repository/filepicker.php @@ -334,6 +334,11 @@ case 'download': unlink($thefile['path']); print_error('maxareabytes'); } + // Ensure the user does not upload too many draft files in a short period. + if (file_is_draft_areas_limit_reached($USER->id)) { + unlink($thefile['path']); + print_error('maxdraftitemids'); + } try { $info = repository::move_to_filepool($thefile['path'], $record); redirect($home_url, get_string('downloadsucc', 'repository')); diff --git a/repository/repository_ajax.php b/repository/repository_ajax.php index 285bc4f6fb9..9f8ccea8079 100644 --- a/repository/repository_ajax.php +++ b/repository/repository_ajax.php @@ -309,6 +309,10 @@ switch ($action) { if (file_is_draft_area_limit_reached($itemid, $areamaxbytes, filesize($downloadedfile['path']))) { throw new file_exception('maxareabytes'); } + // Ensure the user does not upload too many draft files in a short period. + if (file_is_draft_areas_limit_reached($USER->id)) { + throw new file_exception('maxdraftitemids'); + } $info = repository::move_to_filepool($downloadedfile['path'], $record); if (empty($info)) { diff --git a/repository/upload/lib.php b/repository/upload/lib.php index 713ffbda9d1..b3dbe525a6b 100644 --- a/repository/upload/lib.php +++ b/repository/upload/lib.php @@ -199,6 +199,10 @@ class repository_upload extends repository { if (file_is_draft_area_limit_reached($record->itemid, $areamaxbytes, filesize($_FILES[$elname]['tmp_name']))) { throw new file_exception('maxareabytes'); } + // Ensure the user does not upload too many draft files in a short period. + if (file_is_draft_areas_limit_reached($USER->id)) { + throw new file_exception('maxdraftitemids'); + } $record->contextid = $context->id; $record->userid = $USER->id;