From 09373a5a620fc31f9d1d0c9711e144ae4a7fb0fc Mon Sep 17 00:00:00 2001 From: David Monllao Date: Tue, 17 Sep 2013 17:32:45 +0800 Subject: [PATCH 1/3] MDL-41827 tool_generator: New unit test checking the generated files size --- .../generator/tests/maketestcourse_test.php | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/admin/tool/generator/tests/maketestcourse_test.php b/admin/tool/generator/tests/maketestcourse_test.php index b8637f4eb34..b5afea0bc04 100644 --- a/admin/tool/generator/tests/maketestcourse_test.php +++ b/admin/tool/generator/tests/maketestcourse_test.php @@ -35,7 +35,7 @@ class tool_generator_maketestcourse_testcase extends advanced_testcase { $this->setAdminUser(); // Create the XS course. - $backend = new tool_generator_course_backend('TOOL_MAKELARGECOURSE_XS', 0, false, false); + $backend = new tool_generator_course_backend('TOOL_MAKELARGECOURSE_XS', 0, false, false, false); $courseid = $backend->make(); // Get course details. @@ -118,7 +118,7 @@ class tool_generator_maketestcourse_testcase extends advanced_testcase { $this->setAdminUser(); // Create the S course (more sections and activities than XS). - $backend = new tool_generator_course_backend('TOOL_S_COURSE_1', 1, true, false); + $backend = new tool_generator_course_backend('TOOL_S_COURSE_1', 1, true, false, false); $courseid = $backend->make(); // Get course details. @@ -151,4 +151,57 @@ class tool_generator_maketestcourse_testcase extends advanced_testcase { } } + + /** + * Creates a small test course specifying a maximum size and checks the generated files size is limited. + */ + public function test_filesize_limit() { + + $this->resetAfterTest(); + $this->setAdminUser(); + + // Limit. + $filesizelimit = 100; + + // Create a limited XS course. + $backend = new tool_generator_course_backend('TOOL_XS_LIMITED', 0, false, $filesizelimit, false); + $courseid = $backend->make(); + + $course = get_course($courseid); + $modinfo = get_fast_modinfo($course); + + // Check there are small files. + $fs = get_file_storage(); + $resources = $modinfo->get_instances_of('resource'); + foreach ($resources as $resource) { + $resourcecontext = context_module::instance($resource->id); + $files = $fs->get_area_files($resourcecontext->id, 'mod_resource', 'content', false, 'filename', false); + foreach ($files as $file) { + if ($file->get_mimetype() == 'application/octet-stream') { + $this->assertLessThanOrEqual($filesizelimit, $file->get_filesize()); + } + } + } + + // Create a non-limited XS course. + $backend = new tool_generator_course_backend('TOOL_XS_NOLIMITS', 0, false, false, false); + $courseid = $backend->make(); + + $course = get_course($courseid); + $modinfo = get_fast_modinfo($course); + + // Check there are small files. + $fs = get_file_storage(); + $resources = $modinfo->get_instances_of('resource'); + foreach ($resources as $resource) { + $resourcecontext = context_module::instance($resource->id); + $files = $fs->get_area_files($resourcecontext->id, 'mod_resource', 'content', false, 'filename', false); + foreach ($files as $file) { + if ($file->get_mimetype() == 'application/octet-stream') { + $this->assertGreaterThan($filesizelimit, (int)$file->get_filesize()); + } + } + } + + } } From 53e6d79c39a63b73d6a331749316e447a84362df Mon Sep 17 00:00:00 2001 From: David Monllao Date: Tue, 17 Sep 2013 17:33:07 +0800 Subject: [PATCH 2/3] MDL-41827 tool_generator: Adding a file size limit --- admin/tool/generator/classes/backend.php | 9 +++++- .../tool/generator/classes/course_backend.php | 30 +++++++++++++++---- admin/tool/generator/classes/site_backend.php | 5 ++-- admin/tool/generator/cli/maketestcourse.php | 15 ++++++---- admin/tool/generator/cli/maketestsite.php | 13 ++++---- 5 files changed, 52 insertions(+), 20 deletions(-) diff --git a/admin/tool/generator/classes/backend.php b/admin/tool/generator/classes/backend.php index 20b3370bd1f..da931db0f90 100644 --- a/admin/tool/generator/classes/backend.php +++ b/admin/tool/generator/classes/backend.php @@ -51,6 +51,11 @@ abstract class tool_generator_backend { */ protected $fixeddataset; + /** + * @var int|bool Maximum number of bytes for file. + */ + protected $filesizelimit; + /** * @var bool True if displaying progress */ @@ -81,10 +86,11 @@ abstract class tool_generator_backend { * * @param int $size Size as numeric index * @param bool $fixeddataset To use fixed or random data + * @param int|bool $filesizelimit The max number of bytes for a generated file * @param bool $progress True if progress information should be displayed * @throws coding_exception If parameters are invalid */ - public function __construct($size, $fixeddataset = false, $progress = true) { + public function __construct($size, $fixeddataset = false, $filesizelimit = false, $progress = true) { // Check parameter. if ($size < self::MIN_SIZE || $size > self::MAX_SIZE) { @@ -94,6 +100,7 @@ abstract class tool_generator_backend { // Set parameters. $this->size = $size; $this->fixeddataset = $fixeddataset; + $this->filesizelimit = $filesizelimit; $this->progress = $progress; } diff --git a/admin/tool/generator/classes/course_backend.php b/admin/tool/generator/classes/course_backend.php index 07e1e7b1c85..1034b93bea0 100644 --- a/admin/tool/generator/classes/course_backend.php +++ b/admin/tool/generator/classes/course_backend.php @@ -100,15 +100,15 @@ class tool_generator_course_backend extends tool_generator_backend { * @param string $shortname Course shortname * @param int $size Size as numeric index * @param bool $fixeddataset To use fixed or random data + * @param int|bool $filesizelimit The max number of bytes for a generated file * @param bool $progress True if progress information should be displayed - * @return int Course id */ - public function __construct($shortname, $size, $fixeddataset = false, $progress = true) { + public function __construct($shortname, $size, $fixeddataset = false, $filesizelimit = false, $progress = true) { // Set parameters. $this->shortname = $shortname; - parent::__construct($size, $fixeddataset, $progress); + parent::__construct($size, $fixeddataset, $filesizelimit, $progress); } /** @@ -345,7 +345,7 @@ class tool_generator_course_backend extends tool_generator_backend { // Generate random binary data (different for each file so it // doesn't compress unrealistically). - $data = self::get_random_binary(self::$paramsmallfilesize[$this->size]); + $data = self::get_random_binary($this->limit_filesize(self::$paramsmallfilesize[$this->size])); $fs->create_file_from_string($filerecord, $data); $this->dot($i, $count); @@ -362,6 +362,7 @@ class tool_generator_course_backend extends tool_generator_backend { * @return Random data */ private static function get_random_binary($length) { + $data = microtime(true); if (strlen($data) > $length) { // Use last digits of data. @@ -382,8 +383,9 @@ class tool_generator_course_backend extends tool_generator_backend { // Work out how many files and how many blocks to use (up to 64KB). $count = self::$parambigfilecount[$this->size]; - $blocks = ceil(self::$parambigfilesize[$this->size] / 65536); - $blocksize = floor(self::$parambigfilesize[$this->size] / $blocks); + $filesize = $this->limit_filesize(self::$parambigfilesize[$this->size]); + $blocks = ceil($filesize / 65536); + $blocksize = floor($filesize / $blocks); $this->log('createbigfiles', $count, true); @@ -504,4 +506,20 @@ class tool_generator_course_backend extends tool_generator_backend { return $userid; } + /** + * Restricts the binary file size if necessary + * + * @param int $length The total length + * @return int The limited length if a limit was specified. + */ + private function limit_filesize($length) { + + // Limit to $this->filesizelimit. + if (is_numeric($this->filesizelimit) && $length > $this->filesizelimit) { + $length = floor($this->filesizelimit); + } + + return $length; + } + } diff --git a/admin/tool/generator/classes/site_backend.php b/admin/tool/generator/classes/site_backend.php index 01bde976ec5..b634c54d14c 100644 --- a/admin/tool/generator/classes/site_backend.php +++ b/admin/tool/generator/classes/site_backend.php @@ -61,15 +61,16 @@ class tool_generator_site_backend extends tool_generator_backend { * @param int $size Size as numeric index * @param bool $bypasscheck If debugging level checking was skipped. * @param bool $fixeddataset To use fixed or random data + * @param int|bool $filesizelimit The max number of bytes for a generated file * @param bool $progress True if progress information should be displayed * @return int Course id */ - public function __construct($size, $bypasscheck, $fixeddataset = false, $progress = true) { + public function __construct($size, $bypasscheck, $fixeddataset = false, $filesizelimit = false, $progress = true) { // Set parameters. $this->bypasscheck = $bypasscheck; - parent::__construct($size, $fixeddataset, $progress); + parent::__construct($size, $fixeddataset, $filesizelimit, $progress); } /** diff --git a/admin/tool/generator/cli/maketestcourse.php b/admin/tool/generator/cli/maketestcourse.php index 407a58f12db..ce242e35d3b 100644 --- a/admin/tool/generator/cli/maketestcourse.php +++ b/admin/tool/generator/cli/maketestcourse.php @@ -35,6 +35,7 @@ list($options, $unrecognized) = cli_get_params( 'shortname' => false, 'size' => false, 'fixeddataset' => false, + 'filesizelimit' => false, 'bypasscheck' => false, 'quiet' => false ), @@ -52,11 +53,12 @@ Not for use on live sites; only normally works if debugging is set to DEVELOPER level. Options: ---shortname Shortname of course to create (required) ---size Size of course to create XS, S, M, L, XL, or XXL (required) ---fixeddataset Use a fixed data set instead of randomly generated data ---bypasscheck Bypasses the developer-mode check (be careful!) ---quiet Do not show any output +--shortname Shortname of course to create (required) +--size Size of course to create XS, S, M, L, XL, or XXL (required) +--fixeddataset Use a fixed data set instead of randomly generated data +--filesizelimit Limits the size of the generated files to the specified bytes +--bypasscheck Bypasses the developer-mode check (be careful!) +--quiet Do not show any output -h, --help Print out this help @@ -76,6 +78,7 @@ if (empty($options['bypasscheck']) && !debugging('', DEBUG_DEVELOPER)) { $shortname = $options['shortname']; $sizename = $options['size']; $fixeddataset = $options['fixeddataset']; +$filesizelimit = $options['filesizelimit']; // Check size. try { @@ -93,5 +96,5 @@ if ($error = tool_generator_course_backend::check_shortname_available($shortname session_set_user(get_admin()); // Do backend code to generate course. -$backend = new tool_generator_course_backend($shortname, $size, $fixeddataset, empty($options['quiet'])); +$backend = new tool_generator_course_backend($shortname, $size, $fixeddataset, $filesizelimit, empty($options['quiet'])); $id = $backend->make(); diff --git a/admin/tool/generator/cli/maketestsite.php b/admin/tool/generator/cli/maketestsite.php index bc91d7d8a9f..7bff781d06f 100644 --- a/admin/tool/generator/cli/maketestsite.php +++ b/admin/tool/generator/cli/maketestsite.php @@ -34,6 +34,7 @@ list($options, $unrecognized) = cli_get_params( 'help' => false, 'size' => false, 'fixeddataset' => false, + 'filesizelimit' => false, 'bypasscheck' => false, 'quiet' => false ), @@ -57,10 +58,11 @@ Consider that, depending on the size you select, this CLI tool can really genera $sitesizes Options: ---size Size of the generated site, this value affects the number of courses and their size. Accepted values: XS, S, M, L, XL, or XXL (required) ---fixeddataset Use a fixed data set instead of randomly generated data ---bypasscheck Bypasses the developer-mode check (be careful!) ---quiet Do not show any output +--size Size of the generated site, this value affects the number of courses and their size. Accepted values: XS, S, M, L, XL, or XXL (required) +--fixeddataset Use a fixed data set instead of randomly generated data +--filesizelimit Limits the size of the generated files to the specified bytes +--bypasscheck Bypasses the developer-mode check (be careful!) +--quiet Do not show any output -h, --help Print out this help @@ -79,6 +81,7 @@ if (empty($options['bypasscheck']) && !$CFG->debugdeveloper) { // Get options. $sizename = $options['size']; $fixeddataset = $options['fixeddataset']; +$filesizelimit = $options['filesizelimit']; // Check size. try { @@ -91,5 +94,5 @@ try { session_set_user(get_admin()); // Do backend code to generate site. -$backend = new tool_generator_site_backend($size, $options['bypasscheck'], $fixeddataset, empty($options['quiet'])); +$backend = new tool_generator_site_backend($size, $options['bypasscheck'], $fixeddataset, $filesizelimit, empty($options['quiet'])); $backend->make(); From c43ef6c624953e4c94f5c0a7c364af7e3c6f1036 Mon Sep 17 00:00:00 2001 From: David Monllao Date: Thu, 19 Sep 2013 16:47:04 +0800 Subject: [PATCH 3/3] MDL-41889 tool_generator: Fixing problem detecting the last used course shortname --- admin/tool/generator/classes/site_backend.php | 23 ++-- .../generator/tests/maketestsite_test.php | 106 ++++++++++++++++++ 2 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 admin/tool/generator/tests/maketestsite_test.php diff --git a/admin/tool/generator/classes/site_backend.php b/admin/tool/generator/classes/site_backend.php index b634c54d14c..41511a8a8ba 100644 --- a/admin/tool/generator/classes/site_backend.php +++ b/admin/tool/generator/classes/site_backend.php @@ -105,7 +105,7 @@ class tool_generator_site_backend extends tool_generator_backend { // Create courses. $prevchdir = getcwd(); chdir($CFG->dirroot); - $ncourse = $this->get_last_testcourse_id(); + $ncourse = self::get_last_testcourse_id(); foreach (self::$sitecourses as $coursesize => $ncourses) { for ($i = 1; $i <= $ncourses[$this->size]; $i++) { // Non language-dependant shortname. @@ -149,6 +149,10 @@ class tool_generator_site_backend extends tool_generator_backend { $options[] = '--quiet'; } + if ($this->filesizelimit) { + $options[] = '--filesizelimit="' . $this->filesizelimit . '"'; + } + // Extend options. $optionstoextend = array( 'fixeddataset' => 'fixeddataset', @@ -178,26 +182,29 @@ class tool_generator_site_backend extends tool_generator_backend { * * @return int The last generated numeric value. */ - protected function get_last_testcourse_id() { + protected static function get_last_testcourse_id() { global $DB; $params = array(); $params['shortnameprefix'] = $DB->sql_like_escape(self::SHORTNAMEPREFIX) . '%'; $like = $DB->sql_like('shortname', ':shortnameprefix'); - if (!$testcourses = $DB->get_records_select('course', $like, $params, 'shortname DESC')) { + if (!$testcourses = $DB->get_records_select('course', $like, $params, '', 'shortname')) { return 0; } + // SQL order by is not appropiate here as is ordering strings. + $shortnames = array_keys($testcourses); + rsort($shortnames, SORT_NATURAL); // They come ordered by shortname DESC, so non-numeric values will be the first ones. - foreach ($testcourses as $testcourse) { - $sufix = substr($testcourse->shortname, strlen(self::SHORTNAMEPREFIX)); - if (is_numeric($sufix)) { + $prefixnchars = strlen(self::SHORTNAMEPREFIX); + foreach ($shortnames as $shortname) { + $sufix = substr($shortname, $prefixnchars); + if (preg_match('/^[\d]+$/', $sufix)) { return $sufix; } } - - // If all sufixes are not numeric this is the fist make test site run. + // If all sufixes are not numeric this is the first make test site run. return 0; } diff --git a/admin/tool/generator/tests/maketestsite_test.php b/admin/tool/generator/tests/maketestsite_test.php new file mode 100644 index 00000000000..18f6aed73ab --- /dev/null +++ b/admin/tool/generator/tests/maketestsite_test.php @@ -0,0 +1,106 @@ +. + +/** + * Unit test for the site generator + * + * @package tool_generator + * @copyright 2013 David Monllaó + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Silly class to access site_backend internal methods. + * + * @package tool_generator + * @copyright 2013 David Monllaó + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class testable_tool_generator_site_backend extends tool_generator_site_backend { + + /** + * Public accessor. + * + * @return int + */ + public static function get_last_testcourse_id() { + return parent::get_last_testcourse_id(); + } +} + +/** + * Unit test for the site generator + * + * @package tool_generator + * @copyright 2013 David Monllaó + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tool_generator_maketestsite_testcase extends advanced_testcase { + + /** + * Checks that site courses shortnames are properly generated. + */ + public function test_shortnames_generation() { + + $this->resetAfterTest(); + $this->setAdminUser(); + + $generator = $this->getDataGenerator(); + + // Shortname common prefix. + $prefix = tool_generator_site_backend::SHORTNAMEPREFIX; + + $record = array(); + + // Without courses will be 0. + $lastshortname = testable_tool_generator_site_backend::get_last_testcourse_id(); + $this->assertEquals(0, $lastshortname); + + // Without {$prefix} + {no integer} courses will be 0. + $record['shortname'] = $prefix . 'AA'; + $generator->create_course($record); + $record['shortname'] = $prefix . '__'; + $generator->create_course($record); + $record['shortname'] = $prefix . '12.2'; + $generator->create_course($record); + + $lastshortname = testable_tool_generator_site_backend::get_last_testcourse_id(); + $this->assertEquals(0, $lastshortname); + + // With {$prefix} + {integer} courses will be the higher one. + $record['shortname'] = $prefix . '2'; + $generator->create_course($record); + $record['shortname'] = $prefix . '20'; + $generator->create_course($record); + $record['shortname'] = $prefix . '8'; + $generator->create_course($record); + + $lastshortname = testable_tool_generator_site_backend::get_last_testcourse_id(); + $this->assertEquals(20, $lastshortname); + + // Numeric order. + for ($i = 9; $i < 14; $i++) { + $record['shortname'] = $prefix . $i; + $generator->create_course($record); + } + + $lastshortname = testable_tool_generator_site_backend::get_last_testcourse_id(); + $this->assertEquals(20, $lastshortname); + } + +}