From ad3532dea1bc0fc14c92c9c00f0409f799b1d0fb Mon Sep 17 00:00:00 2001 From: Andrew Nicols Date: Wed, 25 Mar 2015 10:55:03 +0800 Subject: [PATCH 1/2] MDL-44874 core: Move uuid generation to setuplib --- lib/classes/lock/db_record_lock_factory.php | 34 +-------------- lib/setuplib.php | 46 +++++++++++++++++++++ 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/lib/classes/lock/db_record_lock_factory.php b/lib/classes/lock/db_record_lock_factory.php index 70cbdb2e6b0..a7fbcfb07e9 100644 --- a/lib/classes/lock/db_record_lock_factory.php +++ b/lib/classes/lock/db_record_lock_factory.php @@ -103,41 +103,9 @@ class db_record_lock_factory implements lock_factory { * to duplicates in a clustered environment (especially on VMs due to poor time precision). */ protected function generate_unique_token() { - $uuid = ''; - - if (function_exists("uuid_create")) { - $context = null; - uuid_create($context); - - uuid_make($context, UUID_MAKE_V4); - uuid_export($context, UUID_FMT_STR, $uuid); - } else { - // Fallback uuid generation based on: - // "http://www.php.net/manual/en/function.uniqid.php#94959". - $uuid = sprintf('%04x%04x-%04x-%04x-%04x-%04x%04x%04x', - - // 32 bits for "time_low". - mt_rand(0, 0xffff), mt_rand(0, 0xffff), - - // 16 bits for "time_mid". - mt_rand(0, 0xffff), - - // 16 bits for "time_hi_and_version", - // four most significant bits holds version number 4. - mt_rand(0, 0x0fff) | 0x4000, - - // 16 bits, 8 bits for "clk_seq_hi_res", - // 8 bits for "clk_seq_low", - // two most significant bits holds zero and one for variant DCE1.1. - mt_rand(0, 0x3fff) | 0x8000, - - // 48 bits for "node". - mt_rand(0, 0xffff), mt_rand(0, 0xffff), mt_rand(0, 0xffff)); - } - return trim($uuid); + return generate_uuid(); } - /** * Create and get a lock * @param string $resource - The identifier for the lock. Should use frankenstyle prefix. diff --git a/lib/setuplib.php b/lib/setuplib.php index 848df4a4779..6fc8b079d95 100644 --- a/lib/setuplib.php +++ b/lib/setuplib.php @@ -575,6 +575,52 @@ function get_exception_info($ex) { return $info; } +/** + * Generate a uuid. + * + * Unique is hard. Very hard. Attempt to use the PECL UUID functions if available, and if not then revert to + * constructing the uuid using mt_rand. + * + * It is important that this token is not solely based on time as this could lead + * to duplicates in a clustered environment (especially on VMs due to poor time precision). + * + * @return string The uuid. + */ +function generate_uuid() { + $uuid = ''; + + if (function_exists("uuid_create")) { + $context = null; + uuid_create($context); + + uuid_make($context, UUID_MAKE_V4); + uuid_export($context, UUID_FMT_STR, $uuid); + } else { + // Fallback uuid generation based on: + // "http://www.php.net/manual/en/function.uniqid.php#94959". + $uuid = sprintf('%04x%04x-%04x-%04x-%04x-%04x%04x%04x', + + // 32 bits for "time_low". + mt_rand(0, 0xffff), mt_rand(0, 0xffff), + + // 16 bits for "time_mid". + mt_rand(0, 0xffff), + + // 16 bits for "time_hi_and_version", + // four most significant bits holds version number 4. + mt_rand(0, 0x0fff) | 0x4000, + + // 16 bits, 8 bits for "clk_seq_hi_res", + // 8 bits for "clk_seq_low", + // two most significant bits holds zero and one for variant DCE1.1. + mt_rand(0, 0x3fff) | 0x8000, + + // 48 bits for "node". + mt_rand(0, 0xffff), mt_rand(0, 0xffff), mt_rand(0, 0xffff)); + } + return trim($uuid); +} + /** * Returns the Moodle Docs URL in the users language for a given 'More help' link. * From 70be2642fc311df22a32a36f4672b034964ab0d5 Mon Sep 17 00:00:00 2001 From: Andrew Nicols Date: Thu, 19 Mar 2015 14:57:56 +0800 Subject: [PATCH 2/2] MDL-44874 core: Add per-request directory functions This adds functionality to create individual request directories which can be only be used for the current request. They are removed by a shutdown handler. --- config-dist.php | 2 +- lib/setuplib.php | 99 +++++++++++++++++++++++++++++++++- lib/tests/setuplib_test.php | 104 ++++++++++++++++++++++++++++++++++++ lib/upgrade.txt | 1 + 4 files changed, 204 insertions(+), 2 deletions(-) diff --git a/config-dist.php b/config-dist.php index 9ba0d91ff6d..6196a57a847 100644 --- a/config-dist.php +++ b/config-dist.php @@ -384,7 +384,7 @@ $CFG->admin = 'admin'; // Localcachedir is intended for server clusters, it does not have to be shared by cluster nodes. // The directories must not be accessible via web. // -// $CFG->tempdir = '/var/www/moodle/temp'; // Files used during one HTTP request only. +// $CFG->tempdir = '/var/www/moodle/temp'; // Directory MUST BE SHARED by all clsuter nodes. // $CFG->cachedir = '/var/www/moodle/cache'; // Directory MUST BE SHARED by all cluster nodes, locking required. // $CFG->localcachedir = '/var/local/cache'; // Intended for local node caching. // diff --git a/lib/setuplib.php b/lib/setuplib.php index 6fc8b079d95..a0814bee8eb 100644 --- a/lib/setuplib.php +++ b/lib/setuplib.php @@ -1389,6 +1389,50 @@ function check_dir_exists($dir, $create = true, $recursive = true) { return mkdir($dir, $CFG->directorypermissions, $recursive); } +/** + * Create a new unique directory within the specified directory. + * + * @param string $basedir The directory to create your new unique directory within. + * @param bool $exceptiononerror throw exception if error encountered + * @return string The created directory + * @throws invalid_dataroot_permissions + */ +function make_unique_writable_directory($basedir, $exceptiononerror = true) { + if (!is_dir($basedir) || !is_writable($basedir)) { + // The basedir is not writable. We will not be able to create the child directory. + if ($exceptiononerror) { + throw new invalid_dataroot_permissions($basedir . ' is not writable. Unable to create a unique directory within it.'); + } else { + return false; + } + } + + do { + // Generate a new (hopefully unique) directory name. + $uniquedir = $basedir . DIRECTORY_SEPARATOR . generate_uuid(); + } while ( + // Ensure that basedir is still writable - if we do not check, we could get stuck in a loop here. + is_writable($basedir) && + + // Make the new unique directory. If the directory already exists, it will return false. + !make_writable_directory($uniquedir, $exceptiononerror) && + + // Ensure that the directory now exists + file_exists($uniquedir) && is_dir($uniquedir) + ); + + // Check that the directory was correctly created. + if (!file_exists($uniquedir) || !is_dir($uniquedir) || !is_writable($uniquedir)) { + if ($exceptiononerror) { + throw new invalid_dataroot_permissions('Unique directory creation failed.'); + } else { + return false; + } + } + + return $uniquedir; +} + /** * Create a directory and make sure it is writable. * @@ -1480,9 +1524,62 @@ function make_upload_directory($directory, $exceptiononerror = true) { return make_writable_directory("$CFG->dataroot/$directory", $exceptiononerror); } +/** + * Get a per-request storage directory in the tempdir. + * + * The directory is automatically cleaned up during the shutdown handler. + * + * @param bool $exceptiononerror throw exception if error encountered + * @return string|false Returns full path to directory if successful, false if not; may throw exception + */ +function get_request_storage_directory($exceptiononerror = true) { + global $CFG; + + static $requestdir = null; + + if (!$requestdir || !file_exists($requestdir) || !is_dir($requestdir) || !is_writable($requestdir)) { + if ($CFG->localcachedir !== "$CFG->dataroot/localcache") { + check_dir_exists($CFG->localcachedir, true, true); + protect_directory($CFG->localcachedir); + } else { + protect_directory($CFG->dataroot); + } + + if ($requestdir = make_unique_writable_directory($CFG->localcachedir, $exceptiononerror)) { + // Register a shutdown handler to remove the directory. + \core_shutdown_manager::register_function('remove_dir', array($requestdir)); + } + } + + return $requestdir; +} + +/** + * Create a per-request directory and make sure it is writable. + * This can only be used during the current request and will be tidied away + * automatically afterwards. + * + * A new, unique directory is always created within the current request directory. + * + * @param bool $exceptiononerror throw exception if error encountered + * @return string full path to directory if successful, false if not; may throw exception + */ +function make_request_directory($exceptiononerror = true) { + $basedir = get_request_storage_directory($exceptiononerror); + return make_unique_writable_directory($basedir, $exceptiononerror); +} + /** * Create a directory under tempdir and make sure it is writable. - * Temporary files should be used during the current request only! + * + * Where possible, please use make_request_directory() and limit the scope + * of your data to the current HTTP request. + * + * Do not use for storing cache files - see make_cache_directory(), and + * make_localcache_directory() instead for this purpose. + * + * Temporary files must be on a shared storage, and heavy usage is + * discouraged due to the performance impact upon clustered environments. * * @param string $directory the full path of the directory to be created under $CFG->tempdir * @param bool $exceptiononerror throw exception if error encountered diff --git a/lib/tests/setuplib_test.php b/lib/tests/setuplib_test.php index f1f999d5c38..96fffada93b 100644 --- a/lib/tests/setuplib_test.php +++ b/lib/tests/setuplib_test.php @@ -217,6 +217,110 @@ class core_setuplib_testcase extends advanced_testcase { $this->assertTimeCurrent(filemtime($timestampfile)); } + public function test_make_unique_directory_basedir_is_file() { + global $CFG; + + // Start with a file instead of a directory. + $base = $CFG->tempdir . DIRECTORY_SEPARATOR . md5(microtime() + rand()); + touch($base); + + // First the false test. + $this->assertFalse(make_unique_writable_directory($base, false)); + + // Now check for exception. + $this->setExpectedException('invalid_dataroot_permissions', + $base . ' is not writable. Unable to create a unique directory within it.' + ); + make_unique_writable_directory($base); + + unlink($base); + } + + public function test_make_unique_directory() { + global $CFG; + + // Create directories should be both directories, and writable. + $firstdir = make_unique_writable_directory($CFG->tempdir); + $this->assertTrue(is_dir($firstdir)); + $this->assertTrue(is_writable($firstdir)); + + $seconddir = make_unique_writable_directory($CFG->tempdir); + $this->assertTrue(is_dir($seconddir)); + $this->assertTrue(is_writable($seconddir)); + + // Directories should be different each iteration. + $this->assertNotEquals($firstdir, $seconddir); + } + + public function test_get_request_storage_directory() { + // Making a call to get_request_storage_directory should always give the same result. + $firstdir = get_request_storage_directory(); + $seconddir = get_request_storage_directory(); + $this->assertTrue(is_dir($firstdir)); + $this->assertEquals($firstdir, $seconddir); + + // Removing the directory and calling get_request_storage_directory() again should cause a new directory to be created. + remove_dir($firstdir); + $this->assertFalse(file_exists($firstdir)); + $this->assertFalse(is_dir($firstdir)); + + $thirddir = get_request_storage_directory(); + $this->assertTrue(is_dir($thirddir)); + $this->assertNotEquals($firstdir, $thirddir); + + // Removing it and replacing it with a file should cause it to be regenerated again. + remove_dir($thirddir); + $this->assertFalse(file_exists($thirddir)); + $this->assertFalse(is_dir($thirddir)); + touch($thirddir); + $this->assertTrue(file_exists($thirddir)); + $this->assertFalse(is_dir($thirddir)); + + $fourthdir = get_request_storage_directory(); + $this->assertTrue(is_dir($fourthdir)); + $this->assertNotEquals($thirddir, $fourthdir); + } + + + public function test_make_request_directory() { + // Every request directory should be unique. + $firstdir = make_request_directory(); + $seconddir = make_request_directory(); + $thirddir = make_request_directory(); + $fourthdir = make_request_directory(); + + $this->assertNotEquals($firstdir, $seconddir); + $this->assertNotEquals($firstdir, $thirddir); + $this->assertNotEquals($firstdir, $fourthdir); + $this->assertNotEquals($seconddir, $thirddir); + $this->assertNotEquals($seconddir, $fourthdir); + $this->assertNotEquals($thirddir, $fourthdir); + + // They should also all be within the request storage directory. + $requestdir = get_request_storage_directory(); + $this->assertEquals(0, strpos($firstdir, $requestdir)); + $this->assertEquals(0, strpos($seconddir, $requestdir)); + $this->assertEquals(0, strpos($thirddir, $requestdir)); + $this->assertEquals(0, strpos($fourthdir, $requestdir)); + + // Removing the requestdir should mean that new request directories are still created successfully. + remove_dir($requestdir); + $this->assertFalse(file_exists($requestdir)); + $this->assertFalse(is_dir($requestdir)); + + $fifthdir = make_request_directory(); + $this->assertNotEquals($firstdir, $fifthdir); + $this->assertNotEquals($seconddir, $fifthdir); + $this->assertNotEquals($thirddir, $fifthdir); + $this->assertNotEquals($fourthdir, $fifthdir); + $this->assertTrue(is_dir($fifthdir)); + $this->assertFalse(strpos($fifthdir, $requestdir)); + + // And it should be within the new request directory. + $newrequestdir = get_request_storage_directory(); + $this->assertEquals(0, strpos($fifthdir, $newrequestdir)); + } + public function test_merge_query_params() { $original = array( 'id' => '1', diff --git a/lib/upgrade.txt b/lib/upgrade.txt index 5b176107455..2acf52cc3bb 100644 --- a/lib/upgrade.txt +++ b/lib/upgrade.txt @@ -27,6 +27,7 @@ information provided here is intended especially for developers. set $CFG->usezipbackups to store them in zip format. This does not affect the restore process, which continues accepting both. * Added support for custom string manager implementations via $CFG->customstringmanager directive in the config.php. See MDL-49361 for details. +* Add new make_request_directory() for creation of per-request files. === 2.8 ===