From 62704f33d4e99264e97b6ade92154d30aa9cf2d3 Mon Sep 17 00:00:00 2001 From: Sam Hemelryk Date: Mon, 10 Sep 2012 15:28:57 +1200 Subject: [PATCH] MDL-25290 cache_file: Added default file cache store --- cache/stores/file/addinstanceform.php | 59 ++ cache/stores/file/lang/en/cache_file.php | 37 ++ cache/stores/file/lib.php | 653 +++++++++++++++++++++++ cache/stores/file/version.php | 32 ++ 4 files changed, 781 insertions(+) create mode 100644 cache/stores/file/addinstanceform.php create mode 100644 cache/stores/file/lang/en/cache_file.php create mode 100644 cache/stores/file/lib.php create mode 100644 cache/stores/file/version.php diff --git a/cache/stores/file/addinstanceform.php b/cache/stores/file/addinstanceform.php new file mode 100644 index 00000000000..532c3c553a6 --- /dev/null +++ b/cache/stores/file/addinstanceform.php @@ -0,0 +1,59 @@ +. + +/** + * The library file for the file cache store. + * + * This file is part of the file cache store, it contains the API for interacting with an instance of the store. + * This is used as a default cache store within the Cache API. It should never be deleted. + * + * @package cache_file + * @category cache + * @copyright 2012 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +require_once($CFG->dirroot.'/cache/forms.php'); +require_once($CFG->dirroot.'/cache/stores/file/lib.php'); + +/** + * Form for adding a file instance. + * + * @copyright 2012 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cache_store_file_addinstance_form extends cache_store_addinstance_form { + + /** + * Adds the desired form elements. + */ + protected function configuration_definition() { + $form = $this->_form; + + $form->addElement('text', 'path', get_string('path', 'cache_file')); + $form->setType('path', PARAM_SAFEPATH); + $form->addHelpButton('path', 'path', 'cache_file'); + + $form->addElement('checkbox', 'autocreate', get_string('autocreate', 'cache_file')); + $form->setType('autocreate', PARAM_BOOL); + $form->addHelpButton('autocreate', 'autocreate', 'cache_file'); + $form->disabledIf('autocreate', 'path', 'eq', ''); + + $form->addElement('checkbox', 'prescan', get_string('prescan', 'cache_file')); + $form->setType('prescan', PARAM_BOOL); + $form->addHelpButton('prescan', 'prescan', 'cache_file'); + } +} \ No newline at end of file diff --git a/cache/stores/file/lang/en/cache_file.php b/cache/stores/file/lang/en/cache_file.php new file mode 100644 index 00000000000..1a3204041f2 --- /dev/null +++ b/cache/stores/file/lang/en/cache_file.php @@ -0,0 +1,37 @@ +. + +/** + * The library file for the file cache store. + * + * This file is part of the file cache store, it contains the API for interacting with an instance of the store. + * This is used as a default cache store within the Cache API. It should never be deleted. + * + * @package cache_file + * @category cache + * @copyright 2012 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +$string['autocreate'] = 'Auto create directory'; +$string['autocreate_help'] = 'If enabled the directory specified in path will be automatically created if it does not already exist.'; +$string['path'] = 'Cache path'; +$string['path_help'] = 'The directory that should be used to store files for this cache store. If left blank (default) a directory will be automatically created in the moodledata directory. This can be used to point a file store towards a directory on a better performing drive (such as one in memory).'; +$string['pluginname'] = 'File cache'; +$string['prescan'] = 'Prescan directory'; +$string['prescan_help'] = 'If enabled the directory is scanned when the cache is first used and requests for files are first checked against the scan data. This can help if you have a slow file system and are finding that file operations are causing you a bottle neck.'; diff --git a/cache/stores/file/lib.php b/cache/stores/file/lib.php new file mode 100644 index 00000000000..dc6d298b7a4 --- /dev/null +++ b/cache/stores/file/lib.php @@ -0,0 +1,653 @@ +. + +/** + * The library file for the file cache store. + * + * This file is part of the file cache store, it contains the API for interacting with an instance of the store. + * This is used as a default cache store within the Cache API. It should never be deleted. + * + * @package cache_file + * @category cache + * @copyright 2012 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * The file store class. + * + * Configuration options + * path: string: path to the cache directory, if left empty one will be created in the cache directory + * autocreate: true, false + * prescan: true, false + * + * @copyright 2012 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cache_store_file implements cache_store, cache_is_lockable, cache_is_key_aware { + + /** + * The name of the store. + * @var string + */ + protected $name; + + /** + * The path to use for the file storage. + * @var string + */ + protected $path = null; + + /** + * Set to true when a prescan has been performed. + * @var bool + */ + protected $prescan = false; + + /** + * Set to true when the path should be automatically created if it does not yet exist. + * @var bool + */ + protected $autocreate = false; + + /** + * Set to true if a custom path is being used. + * @var bool + */ + protected $custompath = false; + + /** + * An array of keys we are sure about presently. + * @var array + */ + protected $keys = array(); + + /** + * True when the store is ready to be initialised. + * @var bool + */ + protected $isready = false; + + /** + * An array containing the locks this store instance owns presently. + * @var array + */ + protected $locks = array(); + + /** + * The cache definition this instance has been initialised with. + * @var cache_definition + */ + protected $definition; + + /** + * Constructs the store instance. + * + * Noting that this function is not an initialisation. It is used to prepare the store for use. + * The store will be initialised when required and will be provided with a cache_definition at that time. + * + * @param string $name + * @param array $configuration + */ + public function __construct($name, array $configuration = array()) { + $this->name = $name; + if (array_key_exists('path', $configuration) && $configuration['path'] !== '') { + $this->custompath = true; + $this->autocreate = !empty($configuration['autocreate']); + $path = (string)$configuration['path']; + if (!is_dir($path)) { + if ($this->autocreate) { + if (!make_writable_directory($path, false)) { + $path = false; + debugging('Error trying to autocreate file store path. '.$path, DEBUG_DEVELOPER); + } + } else { + $path = false; + debugging('The given file cache store path does not exist. '.$path, DEBUG_DEVELOPER); + } + } + if ($path !== false && !is_writable($path)) { + $path = false; + debugging('The given file cache store path is not writable. '.$path, DEBUG_DEVELOPER); + } + } else { + $path = make_cache_directory('cache_store_file/'.preg_replace('#[^a-zA-Z0-9\.\-_]+#', '', $name)); + } + $this->isready = $path !== false; + $this->path = $path; + $this->prescan = array_key_exists('prescan', $configuration) ? (bool)$configuration['prescan'] : false; + } + + /** + * Returns true if this store instance is ready to be used. + * @return bool + */ + public function is_ready() { + return ($this->path !== null); + } + + /** + * Returns true once this instance has been initialised. + * + * @return bool + */ + public function is_initialised() { + return true; + } + + /** + * Returns the supported features as a combined int. + * + * @param array $configuration + * @return int + */ + public static function get_supported_features(array $configuration = array()) { + $supported = self::SUPPORTS_DATA_GUARANTEE + + self::SUPPORTS_NATIVE_TTL; + return $supported; + } + + /** + * Returns the supported modes as a combined int. + * + * @param array $configuration + * @return int + */ + public static function get_supported_modes(array $configuration = array()) { + return self::MODE_APPLICATION + self::MODE_SESSION; + } + + /** + * Returns true if the store requirements are met. + * + * @return bool + */ + public static function are_requirements_met() { + return true; + } + + /** + * Returns true if the given mode is supported by this store. + * + * @param int $mode One of cache_store::MODE_* + * @return bool + */ + public static function is_supported_mode($mode) { + return ($mode === self::MODE_APPLICATION || $mode === self::MODE_SESSION); + } + + /** + * Returns true if the store instance supports multiple identifiers. + * + * @return bool + */ + public function supports_multiple_indentifiers() { + return false; + } + + /** + * Returns true if the store instance guarantees data. + * + * @return bool + */ + public function supports_data_guarantee() { + return true; + } + + /** + * Returns true if the store instance supports native ttl. + * + * @return bool + */ + public function supports_native_ttl() { + return true; + } + + /** + * Initialises the cache. + * + * Once this has been done the cache is all set to be used. + * + * @param cache_definition $definition + */ + public function initialise(cache_definition $definition) { + $this->definition = $definition; + $hash = preg_replace('#[^a-zA-Z0-9]+#', '_', $this->definition->get_id()); + $this->path .= '/'.$hash; + make_writable_directory($this->path); + if ($this->prescan && $definition->get_mode() !== self::MODE_REQUEST) { + $this->prescan = false; + } + if ($this->prescan) { + $pattern = $this->path.'/*.cache'; + foreach (glob($pattern, GLOB_MARK | GLOB_NOSORT) as $filename) { + $this->keys[basename($filename)] = filemtime($filename); + } + } + } + + /** + * Retrieves an item from the cache store given its key. + * + * @param string $key The key to retrieve + * @return mixed The data that was associated with the key, or false if the key did not exist. + */ + public function get($key) { + $filename = $key.'.cache'; + $file = $this->path.'/'.$filename; + $ttl = $this->definition->get_ttl(); + if ($ttl) { + $maxtime = cache::now() - $ttl; + } + $readfile = false; + if ($this->prescan && array_key_exists($key, $this->keys)) { + if (!$ttl || $this->keys[$filename] >= $maxtime && file_exists($file)) { + $readfile = true; + } else { + $this->delete($key); + } + } else if (file_exists($file) && (!$ttl || filemtime($file) >= $maxtime)) { + $readfile = true; + } + if (!$readfile) { + return false; + } + // Check the filesize first, likely not needed but important none the less + $filesize = filesize($file); + if (!$filesize) { + return false; + } + // Open ensuring the file for writing, truncating it and setting the pointer to the start. + if (!$handle = fopen($file, 'rb')) { + return false; + } + // Lock it up! + // We don't care if this succeeds or not, on some systems it will, on some it won't, meah either way + flock($handle, LOCK_SH); + // HACK ALERT + // There is a problem when reading from the file during PHPUNIT tests. For one reason or another the filesize is not correct + // Doesn't happen during normal operation, just during unit tests. + // Read it + $data = fread($handle, $filesize+128); + // Unlock it + flock($handle, LOCK_UN); + // Return it unserialised. + return $this->prep_data_after_read($data); + } + + /** + * Retrieves several items from the cache store in a single transaction. + * + * If not all of the items are available in the cache then the data value for those that are missing will be set to false. + * + * @param array $keys The array of keys to retrieve + * @return array An array of items from the cache. There will be an item for each key, those that were not in the store will + * be set to false. + */ + public function get_many($keys) { + $result = array(); + foreach ($keys as $key) { + $result[$key] = $this->get($key); + } + return $result; + } + + /** + * Deletes an item from the cache store. + * + * @param string $key The key to delete. + * @return bool Returns true if the operation was a success, false otherwise. + */ + public function delete($key) { + $filename = $key.'.cache'; + $file = $this->path.'/'.$filename; + $result = @unlink($file); + unset($this->keys[$filename]); + return $result; + } + + /** + * Deletes several keys from the cache in a single action. + * + * @param array $keys The keys to delete + * @return int The number of items successfully deleted. + */ + public function delete_many(array $keys) { + $count = 0; + foreach ($keys as $key) { + if ($this->delete($key)) { + $count++; + } + } + return $count; + } + + /** + * Sets an item in the cache given its key and data value. + * + * @param string $key The key to use. + * @param mixed $data The data to set. + * @return bool True if the operation was a success false otherwise. + */ + public function set($key, $data) { + $this->ensure_path_exists(); + $filename = $key.'.cache'; + $file = $this->path.'/'.$filename; + $result = $this->write_file($file, $this->prep_data_before_save($data)); + if (!$result) { + // Couldn't write the file. + return false; + } + // Record the key if required + if ($this->prescan) { + $this->keys[$filename] = cache::now() + 1; + } + // Return true.. it all worked **miracles** + return true; + } + + /** + * Prepares data to be stored in a file. + * + * @param mixed $data + * @return string + */ + protected function prep_data_before_save($data) { + return serialize($data); + } + + /** + * Prepares the data it has been read from the cache. Undoing what was done in prep_data_before_save. + * + * @param string $data + * @return mixed + * @throws coding_exception + */ + protected function prep_data_after_read($data) { + $result = @unserialize($data); + if ($result === false) { + throw new coding_exception('Failed to unserialise data from file. Either failed to read, or failed to write.'); + } + return $result; + } + + /** + * Sets many items in the cache in a single transaction. + * + * @param array $keyvaluearray An array of key value pairs. Each item in the array will be an associative array with two + * keys, 'key' and 'value'. + * @return int The number of items successfully set. It is up to the developer to check this matches the number of items + * sent ... if they care that is. + */ + public function set_many(array $keyvaluearray) { + $count = 0; + foreach ($keyvaluearray as $pair) { + if ($this->set($pair['key'], $pair['value'])) { + $count++; + } + } + return $count; + } + + /** + * Checks if the store has a record for the given key and returns true if so. + * + * @param string $key + * @return bool + */ + public function has($key) { + $filename = $key.'.cache'; + $file = $this->path.'/'.$key.'.cache'; + $maxtime = cache::now() - $this->definition->get_ttl(); + if ($this->prescan) { + return array_key_exists($filename, $this->keys) && $this->keys[$filename] >= $maxtime; + } + return (file_exists($file) && ($this->definition->get_ttl() == 0 || filemtime($file) >= $maxtime)); + } + + /** + * Returns true if the store contains records for all of the given keys. + * + * @param array $keys + * @return bool + */ + public function has_all(array $keys) { + foreach ($keys as $key) { + if (!$this->has($key)) { + return false; + } + } + return true; + } + + /** + * Returns true if the store contains records for any of the given keys. + * + * @param array $keys + * @return bool + */ + public function has_any(array $keys) { + foreach ($keys as $key) { + if ($this->has($key)) { + return true; + } + } + return false; + } + + /** + * Acquires a lock for the key with the given identifier. + * + * @param string $key The key to acquire a lock for. + * @param string $identifier The identifier who will own the lock. + * @return bool True if the lock could be acquired, false otherwise. + */ + public function acquire_lock($key, $identifier) { + if (array_key_exists($key, $this->locks) && $this->locks[$key] == $identifier) { + // We already have the lock, return true. + return true; + } + $result = cache_lock::lock($key, false); + if ($result) { + $this->locks[$key] = $identifier; + } + return $result; + } + + /** + * Releases the lock provided it belongs to the identifier. + * + * @param string $key The key to the lock is for. + * @param string $identifier The identifier of the caller. + * @return bool True if the lock has been released, false if there was a problem releasing the lock. + */ + public function release_lock($key, $identifier) { + if (array_key_exists($key, $this->locks) && $this->locks[$key] == $identifier) { + $outcome = cache_lock::unlock($key); + return $outcome; + } + return false; + } + + /** + * Returns true if the given key has a lock and it belongs to the identifier. + * + * @param string $key The key to the lock is for. + * @param string $identifier The identifier of the caller. + * @return bool True if this code has the lock, false if there is a lock but this code doesn't have it, null if there is no lock. + */ + public function has_lock($key, $identifier) { + return (array_key_exists($key, $this->locks) && $this->locks[$key] == $identifier); + } + + /** + * Returns the path to the lock file. + * + * @param string $key + * @return string The absolute path to use for a lock file for this key. + */ + protected function get_lock_file($key) { + return $this->path.'/lock-'.$key.'.lock'; + } + + /** + * Cleans up any left over lock files. + * + * There shouldn't be any left over lock files but clean them up just in case. + */ + public function __destruct() { + $errors = false; + foreach ($this->locks as $file) { + try { + @unlink($file); + } catch (Exception $e) { + // We just want to ensure we unlink everything possible. + $errors = true; + } + } + if ($errors) { + error_log('ERROR ERROR ERROR!!! Unable to release all file cache store locks!'); + } + } + + /** + * Purges the cache deleting all items within it. + * + * @return boolean True on success. False otherwise. + */ + public function purge() { + $pattern = $this->path.'/*.cache'; + foreach (glob($pattern, GLOB_MARK | GLOB_NOSORT) as $filename) { + @unlink($filename); + } + $this->keys = array(); + return true; + } + + /** + * Checks to make sure that the path for the file cache exists. + * + * @return bool + * @throws coding_exception + */ + protected function ensure_path_exists() { + if (!is_writable($this->path)) { + if ($this->custompath && !$this->autocreate) { + throw new coding_exception('File store path does not exist. You must create it and make it writable to the web server.'); + } + if (!make_writable_directory($this->path, false)) { + throw new coding_exception('File store path does not exist and can not be created.'); + } + } + return true; + } + + /** + * Returns true if the user can add an instance of the store plugin. + * + * @return bool + */ + public static function can_add_instance() { + return true; + } + + /** + * Performs any necessary clean up when the store instance is being deleted. + * + * 1. Purges the cache directory. + * 2. Deletes the directory we created for this cache instances data. + */ + public function cleanup() { + $this->purge(); + @rmdir($this->path); + } + + /** + * Generates an instance of the cache store that can be used for testing. + * + * Returns an instance of the cache store, or false if one cannot be created. + * + * @param cache_definition $definition + * @return cache_store_file + */ + public static function initialise_test_instance(cache_definition $definition) { + $name = 'File test'; + $path = make_cache_directory('cache_store_file_test'); + $cache = new cache_store_file($name, array('path' => $path)); + $cache->initialise($definition); + return $cache; + } + + /** + * Writes your madness to a file. + * + * There are several things going on in this function to try to ensure what we don't end up with partial writes etc. + * 1. Files for writing are opened with the mode xb, the file must be created and can not already exist. + * 2. We use cache_mutex to ensure we acquire a lock. + * 3. Renaming, data is written to a temporary file, where it can be verified using md5 and is then renamed. + * + * @param string $file Absolute file path + * @param string $content The content to write. + * @return bool + */ + protected function write_file($file, $content) { + // Generate a temp file that is going to be unique. We'll rename it at the end to the desired file name. + // in this way we avoid partial writes. + $path = dirname($file); + while (true) { + $tempfile = $path.'/'.uniqid(sesskey().'.', true) . '.temp'; + if (!file_exists($tempfile)) { + break; + } + } + + // Lock the temp file before we write. + if (!cache_lock::lock($tempfile, false)) { + return false; + } + + // Open the file with mode=x. This acts to create and open the file for writing only. + // If the file already exists this will return false. + // We also force binary. + $handle = @fopen($tempfile, 'xb+'); + if ($handle === false) { + // File already exists... lock already exists, return false. + return false; + } + // We have the lock. Write our content. + fwrite($handle, $content); + fflush($handle); + // Close the handle, we're done. + fclose($handle); + + // Unlock the temp file. + cache_lock::unlock($tempfile); + + if (md5_file($tempfile) !== md5($content)) { + // The md5 of the content of the file must match the md5 of the content given to be written. + @unlink($tempfile); + return false; + } + + // Finally rename the temp file to the desired file, returning the true|false result. + $result = rename($tempfile, $file); + if (!$result) { + // Failed to rename, don't leave files lying around. + @unlink($tempfile); + } + return $result; + } +} \ No newline at end of file diff --git a/cache/stores/file/version.php b/cache/stores/file/version.php new file mode 100644 index 00000000000..b92d50eccb0 --- /dev/null +++ b/cache/stores/file/version.php @@ -0,0 +1,32 @@ +. + +/** + * Cache file store version information. + * + * This is used as a default cache store within the Cache API. It should never be deleted. + * + * @package cache_file + * @category cache + * @copyright 2012 Sam Hemelryk + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die; + +$plugin->version = 2012091000; // The current module version (Date: YYYYMMDDXX) +$plugin->requires = 2012090700; // Requires this Moodle version +$plugin->component = 'cache_file'; // Full name of the plugin (used for diagnostics) \ No newline at end of file