diff --git a/cache/classes/interfaces.php b/cache/classes/interfaces.php index 28bcf159ce9..543041f3989 100644 --- a/cache/classes/interfaces.php +++ b/cache/classes/interfaces.php @@ -48,6 +48,34 @@ interface cache_loader { */ public function get($key, $strictness = IGNORE_MISSING); + /** + * Retrieves the value and actual version for the given key, with at least the required version. + * + * If there is no value for the key, or there is a value but it doesn't have the required + * version, then this function will return false (or throw an exception if you set strictness + * to MUST_EXIST). + * + * This function can be used to make it easier to support localisable caches (where the cache + * could be stored on a local server as well as a shared cache). Specifying the version means + * that it will automatically retrieve the correct version if available, either from the local + * server or [if that has an older version] from the shared server. + * + * If the cached version is newer than specified version, it will be returned regardless. For + * example, if you request version 4, but the locally cached version is 5, it will be returned. + * If you request version 6, and the locally cached version is 5, then the system will look in + * higher-level caches (if any); if there still isn't a version 6 or greater, it will return + * null. + * + * You must use this function if you use set_versioned. + * + * @param string|int $key The key for the data being requested. + * @param int $requiredversion Minimum required version of the data + * @param int $strictness One of IGNORE_MISSING or MUST_EXIST. + * @param mixed $actualversion If specified, will be set to the actual version number retrieved + * @return mixed Data from the cache, or false if the key did not exist or was too old + */ + public function get_versioned($key, int $requiredversion, int $strictness = IGNORE_MISSING, &$actualversion = null); + /** * Retrieves an array of values for an array of keys. * @@ -82,6 +110,29 @@ interface cache_loader { */ public function set($key, $data); + /** + * Sets the value for the given key with the given version. + * + * The cache does not store multiple versions - any existing version will be overwritten with + * this one. This function should only be used if there is a known 'current version' (e.g. + * stored in a database table). It only ensures that the cache does not return outdated data. + * + * This function can be used to help implement localisable caches (where the cache could be + * stored on a local server as well as a shared cache). The version will be recorded alongside + * the item and get_versioned will always return the correct version. + * + * The version number must be an integer that always increases. This could be based on the + * current time, or a stored value that increases by 1 each time it changes, etc. + * + * If you use this function you must use get_versioned to retrieve the data. + * + * @param string|int $key The key for the data being set. + * @param int $version Integer for the version of the data + * @param mixed $data The data to set against the key. + * @return bool True on success, false otherwise. + */ + public function set_versioned($key, int $version, $data): bool; + /** * Sends several key => value pairs to the cache. * @@ -436,6 +487,32 @@ interface cache_data_source { public function load_many_for_cache(array $keys); } +/** + * Versionable cache data source. + * + * This interface extends the main cache data source interface to add an extra required method if + * the data source is to be used for a versioned cache. + * + * @package core_cache + */ +interface cache_data_source_versionable extends cache_data_source { + /** + * Loads the data for the key provided ready formatted for caching. + * + * If there is no data for that key, or if the data for the required key has an older version + * than the specified $requiredversion, then this returns null. + * + * If there is data then $actualversion should be set to the actual version number retrieved + * (may be the same as $requiredversion or newer). + * + * @param string|int $key The key to load. + * @param int $requiredversion Minimum required version + * @param mixed $actualversion Should be set to the actual version number retrieved + * @return mixed What ever data should be returned, or false if it can't be loaded. + */ + public function load_for_cache_versioned($key, int $requiredversion, &$actualversion); +} + /** * Cacheable object. * @@ -521,4 +598,4 @@ interface cache_lock_interface { * Things such as unfortunate timeouts etc could cause this situation. */ public function __destruct(); -} \ No newline at end of file +} diff --git a/cache/classes/loaders.php b/cache/classes/loaders.php index 39e1d50a8f5..cfbc65e59ea 100644 --- a/cache/classes/loaders.php +++ b/cache/classes/loaders.php @@ -42,6 +42,11 @@ defined('MOODLE_INTERNAL') || die(); */ class cache implements cache_loader { + /** + * @var int Constant for cache entries that do not have a version number + */ + const VERSION_NONE = -1; + /** * We need a timestamp to use within the cache API. * This stamp needs to be used for all ttl and time based operations to ensure that we don't end up with @@ -397,13 +402,101 @@ class cache implements cache_loader { * @throws coding_exception */ public function get($key, $strictness = IGNORE_MISSING) { + return $this->get_implementation($key, self::VERSION_NONE, $strictness); + } + + /** + * Retrieves the value and actual version for the given key, with at least the required version. + * + * If there is no value for the key, or there is a value but it doesn't have the required + * version, then this function will return null (or throw an exception if you set strictness + * to MUST_EXIST). + * + * This function can be used to make it easier to support localisable caches (where the cache + * could be stored on a local server as well as a shared cache). Specifying the version means + * that it will automatically retrieve the correct version if available, either from the local + * server or [if that has an older version] from the shared server. + * + * If the cached version is newer than specified version, it will be returned regardless. For + * example, if you request version 4, but the locally cached version is 5, it will be returned. + * If you request version 6, and the locally cached version is 5, then the system will look in + * higher-level caches (if any); if there still isn't a version 6 or greater, it will return + * null. + * + * You must use this function if you use set_versioned. + * + * @param string|int $key The key for the data being requested. + * @param int $requiredversion Minimum required version of the data + * @param int $strictness One of IGNORE_MISSING or MUST_EXIST. + * @param mixed $actualversion If specified, will be set to the actual version number retrieved + * @return mixed Data from the cache, or false if the key did not exist or was too old + * @throws \coding_exception If you call get_versioned on a non-versioned cache key + */ + public function get_versioned($key, int $requiredversion, int $strictness = IGNORE_MISSING, &$actualversion = null) { + return $this->get_implementation($key, $requiredversion, $strictness, $actualversion); + } + + /** + * Checks returned data to see if it matches the specified version number. + * + * For versioned data, this returns the version_wrapper object (or false). For other + * data, it returns the actual data (or false). + * + * @param mixed $result Result data + * @param int $requiredversion Required version number or VERSION_NONE if there must be no version + * @return bool True if version is current, false if not (or if result is false) + * @throws \coding_exception If unexpected type of data (versioned vs non-versioned) is found + */ + protected static function check_version($result, int $requiredversion): bool { + if ($requiredversion === self::VERSION_NONE) { + if ($result instanceof \core_cache\version_wrapper) { + throw new \coding_exception('Unexpectedly found versioned cache entry'); + } else { + // No version checks, so version is always correct. + return true; + } + } else { + // If there's no result, obviously it doesn't meet the required version. + if (!$result) { + return false; + } + if (!($result instanceof \core_cache\version_wrapper)) { + throw new \coding_exception('Unexpectedly found non-versioned cache entry'); + } + // If the result doesn't match the required version tag, return false. + if ($result->version < $requiredversion) { + return false; + } + // The version meets the requirement. + return true; + } + } + + /** + * Retrieves the value for the given key from the cache. + * + * @param string|int $key The key for the data being requested. + * It can be any structure although using a scalar string or int is recommended in the interests of performance. + * In advanced cases an array may be useful such as in situations requiring the multi-key functionality. + * @param int $requiredversion Minimum required version of the data or cache::VERSION_NONE + * @param int $strictness One of IGNORE_MISSING | MUST_EXIST + * @param mixed $actualversion If specified, will be set to the actual version number retrieved + * @return mixed|false The data from the cache or false if the key did not exist within the cache. + * @throws coding_exception + */ + protected function get_implementation($key, int $requiredversion, int $strictness, &$actualversion = null) { // 1. Get it from the static acceleration array if we can (only when it is enabled and it has already been requested/set). $usesstaticacceleration = $this->use_static_acceleration(); if ($usesstaticacceleration) { $result = $this->static_acceleration_get($key); - if ($result !== false) { - return $result; + if ($result && self::check_version($result, $requiredversion)) { + if ($requiredversion === self::VERSION_NONE) { + return $result; + } else { + $actualversion = $result->version; + return $result->data; + } } } @@ -412,18 +505,58 @@ class cache implements cache_loader { // 3. Get it from the store. Obviously wasn't in the static acceleration array. $result = $this->store->get($parsedkey); + if ($result) { + // Check the result has at least the required version. + try { + $validversion = self::check_version($result, $requiredversion); + } catch (\coding_exception $e) { + // If we get an exception because there is incorrect data in the cache (not + // versioned when it ought to be), delete it so this exception goes away next time. + // The exception should only happen if there is a code bug (which is why we still + // throw it) but there are unusual scenarios where it might happen and that would + // be annoying if it doesn't fix itself. + $this->store->delete($parsedkey); + throw $e; + } + + if (!$validversion) { + // If the result was too old, don't use it. + $result = false; + + // Also delete it immediately. This improves performance in the + // case when the cache item is large and there may be multiple clients simultaneously + // requesting it - they won't all have to do a megabyte of IO just in order to find + // that it's out of date. + $this->store->delete($parsedkey); + } + } if ($result !== false) { - if ($result instanceof cache_ttl_wrapper) { - if ($result->has_expired()) { + // Look to see if there's a TTL wrapper. It might be inside a version wrapper. + if ($requiredversion !== self::VERSION_NONE) { + $ttlconsider = $result->data; + } else { + $ttlconsider = $result; + } + if ($ttlconsider instanceof cache_ttl_wrapper) { + if ($ttlconsider->has_expired()) { $this->store->delete($parsedkey); $result = false; + } else if ($requiredversion === self::VERSION_NONE) { + // Use the data inside the TTL wrapper as the result. + $result = $ttlconsider->data; } else { - $result = $result->data; + // Put the data from the TTL wrapper directly inside the version wrapper. + $result->data = $ttlconsider->data; } } if ($usesstaticacceleration) { $this->static_acceleration_set($key, $result); } + // Remove version wrapper if necessary. + if ($requiredversion !== self::VERSION_NONE) { + $actualversion = $result->version; + $result = $result->data; + } if ($result instanceof cache_cached_object) { $result = $result->restore_object(); } @@ -439,9 +572,23 @@ class cache implements cache_loader { // We must pass the original (unparsed) key to the next loader in the chain. // The next loader will parse the key as it sees fit. It may be parsed differently // depending upon the capabilities of the store associated with the loader. - $result = $this->loader->get($key); + if ($requiredversion === self::VERSION_NONE) { + $result = $this->loader->get($key); + } else { + $result = $this->loader->get_versioned($key, $requiredversion, IGNORE_MISSING, $actualversion); + } } else if ($this->datasource !== false) { - $result = $this->datasource->load_for_cache($key); + if ($requiredversion === self::VERSION_NONE) { + $result = $this->datasource->load_for_cache($key); + } else { + if (!$this->datasource instanceof cache_data_source_versionable) { + throw new \coding_exception('Data source is not versionable'); + } + $result = $this->datasource->load_for_cache_versioned($key, $requiredversion, $actualversion); + if ($result && $actualversion < $requiredversion) { + throw new \coding_exception('Data source returned outdated version'); + } + } } $setaftervalidation = ($result !== false); } else if ($this->perfdebug) { @@ -452,9 +599,14 @@ class cache implements cache_loader { if ($strictness === MUST_EXIST && $result === false) { throw new coding_exception('Requested key did not exist in any cache stores and could not be loaded.'); } - // 6. Set it to the store if we got it from the loader/datasource. + // 6. Set it to the store if we got it from the loader/datasource. Only set to this direct + // store; parent method will have set it to all stores if needed. if ($setaftervalidation) { - $this->set($key, $result); + if ($requiredversion === self::VERSION_NONE) { + $this->set_implementation($key, self::VERSION_NONE, $result, false); + } else { + $this->set_implementation($key, $actualversion, $result, false); + } } // 7. Make sure we don't pass back anything that could be a reference. // We don't want people modifying the data in the cache. @@ -630,10 +782,52 @@ class cache implements cache_loader { * @return bool True on success, false otherwise. */ public function set($key, $data) { - if ($this->loader !== false) { + return $this->set_implementation($key, self::VERSION_NONE, $data); + } + + /** + * Sets the value for the given key with the given version. + * + * The cache does not store multiple versions - any existing version will be overwritten with + * this one. This function should only be used if there is a known 'current version' (e.g. + * stored in a database table). It only ensures that the cache does not return outdated data. + * + * This function can be used to help implement localisable caches (where the cache could be + * stored on a local server as well as a shared cache). The version will be recorded alongside + * the item and get_versioned will always return the correct version. + * + * The version number must be an integer that always increases. This could be based on the + * current time, or a stored value that increases by 1 each time it changes, etc. + * + * If you use this function you must use get_versioned to retrieve the data. + * + * @param string|int $key The key for the data being set. + * @param int $version Integer for the version of the data + * @param mixed $data The data to set against the key. + * @return bool True on success, false otherwise. + */ + public function set_versioned($key, int $version, $data): bool { + return $this->set_implementation($key, $version, $data); + } + + /** + * Sets the value for the given key, optionally with a version tag. + * + * @param string|int $key The key for the data being set. + * @param int $version Version number for the data or cache::VERSION_NONE if none + * @param mixed $data The data to set against the key. + * @param bool $setparents If true, sets all parent loaders, otherwise only this one + * @return bool True on success, false otherwise. + */ + protected function set_implementation($key, int $version, $data, bool $setparents = true): bool { + if ($this->loader !== false && $setparents) { // We have a loader available set it there as well. // We have to let the loader do its own parsing of data as it may be unique. - $this->loader->set($key, $data); + if ($version === self::VERSION_NONE) { + $this->loader->set($key, $data); + } else { + $this->loader->set_versioned($key, $version, $data); + } } $usestaticacceleration = $this->use_static_acceleration(); @@ -648,7 +842,12 @@ class cache implements cache_loader { } if ($usestaticacceleration) { - $this->static_acceleration_set($key, $data); + // Static acceleration cache should include the cache version wrapper, but not TTL. + if ($version === self::VERSION_NONE) { + $this->static_acceleration_set($key, $data); + } else { + $this->static_acceleration_set($key, new \core_cache\version_wrapper($data, $version)); + } } if ($this->has_a_ttl() && !$this->store_supports_native_ttl()) { @@ -656,6 +855,10 @@ class cache implements cache_loader { } $parsedkey = $this->parse_key($key); + if ($version !== self::VERSION_NONE) { + $data = new \core_cache\version_wrapper($data, $version); + } + $success = $this->store->set($parsedkey, $data); if ($this->perfdebug) { cache_helper::record_cache_set($this->store, $this->definition, 1, @@ -1505,14 +1708,16 @@ class cache_application extends cache implements cache_loader_with_locking { * * * @param string|int $key The key for the data being requested. + * @param int $version Version number * @param mixed $data The data to set against the key. + * @param bool $setparents If true, sets all parent loaders, otherwise only this one * @return bool True on success, false otherwise. */ - public function set($key, $data) { + protected function set_implementation($key, int $version, $data, bool $setparents = true): bool { if ($this->requirelockingwrite && !$this->acquire_lock($key)) { return false; } - $result = parent::set($key, $data); + $result = parent::set_implementation($key, $version, $data, $setparents); if ($this->requirelockingwrite && !$this->release_lock($key)) { debugging('Failed to release cache lock on set operation... this should not happen.', DEBUG_DEVELOPER); } @@ -1569,15 +1774,17 @@ class cache_application extends cache implements cache_loader_with_locking { * Retrieves the value for the given key from the cache. * * @param string|int $key The key for the data being requested. + * @param int $requiredversion Minimum required version of the data or cache::VERSION_NONE * @param int $strictness One of IGNORE_MISSING | MUST_EXIST + * @param mixed &$actualversion If specified, will be set to the actual version number retrieved * @return mixed|false The data from the cache or false if the key did not exist within the cache. */ - public function get($key, $strictness = IGNORE_MISSING) { + protected function get_implementation($key, int $requiredversion, int $strictness, &$actualversion = null) { if ($this->requirelockingread && $this->check_lock_state($key) === false) { // Read locking required and someone else has the read lock. return false; } - return parent::get($key, $strictness); + return parent::get_implementation($key, $requiredversion, $strictness, $actualversion); } /** @@ -1830,16 +2037,18 @@ class cache_session extends cache { * @param string|int $key The key for the data being requested. * It can be any structure although using a scalar string or int is recommended in the interests of performance. * In advanced cases an array may be useful such as in situations requiring the multi-key functionality. + * @param int $requiredversion Minimum required version of the data or cache::VERSION_NONE * @param int $strictness One of IGNORE_MISSING | MUST_EXIST + * @param mixed &$actualversion If specified, will be set to the actual version number retrieved * @return mixed|false The data from the cache or false if the key did not exist within the cache. * @throws coding_exception */ - public function get($key, $strictness = IGNORE_MISSING) { + protected function get_implementation($key, int $requiredversion, int $strictness, &$actualversion = null) { // Check the tracked user. $this->check_tracked_user(); // Use parent code. - return parent::get($key, $strictness); + return parent::get_implementation($key, $requiredversion, $strictness, $actualversion); } /** diff --git a/cache/classes/version_wrapper.php b/cache/classes/version_wrapper.php new file mode 100644 index 00000000000..875ee6156b0 --- /dev/null +++ b/cache/classes/version_wrapper.php @@ -0,0 +1,50 @@ +. + +namespace core_cache; + +/** + * Class wrapping information in the cache that is tagged with a version number. + * + * @package core_cache + * @copyright 2021 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class version_wrapper { + + /** + * The data being stored. + * @var mixed + */ + public $data; + + /** + * Version number for the data + * @var int + */ + public $version; + + /** + * Constructs a version tag wrapper. + * + * @param mixed $data + * @param int $version Version number + */ + public function __construct($data, int $version) { + $this->data = $data; + $this->version = $version; + } +} diff --git a/cache/disabledlib.php b/cache/disabledlib.php index a86c704cd8d..e27318dead6 100644 --- a/cache/disabledlib.php +++ b/cache/disabledlib.php @@ -61,14 +61,27 @@ class cache_disabled extends cache { * Gets a key from the cache. * * @param int|string $key + * @param int $requiredversion Minimum required version of the data or cache::VERSION_NONE * @param int $strictness Unused. + * @param mixed &$actualversion If specified, will be set to the actual version number retrieved * @return bool */ - public function get($key, $strictness = IGNORE_MISSING) { - if ($this->get_datasource() !== false) { - return $this->get_datasource()->load_for_cache($key); + protected function get_implementation($key, int $requiredversion, int $strictness, &$actualversion = null) { + $datasource = $this->get_datasource(); + if ($datasource !== false) { + if ($requiredversion === cache::VERSION_NONE) { + return $datasource->load_for_cache($key); + } else { + if (!$datasource instanceof cache_data_source_versionable) { + throw new \coding_exception('Data source is not versionable'); + } + $result = $datasource->load_for_cache_versioned($key, $requiredversion, $actualversion); + if ($result && $actualversion < $requiredversion) { + throw new \coding_exception('Data source returned outdated version'); + } + return $result; + } } - return false; } @@ -91,10 +104,12 @@ class cache_disabled extends cache { * Sets a key value pair in the cache. * * @param int|string $key Unused. + * @param int $version Unused. * @param mixed $data Unused. + * @param bool $setparents Unused. * @return bool */ - public function set($key, $data) { + protected function set_implementation($key, int $version, $data, bool $setparents = true): bool { return false; } diff --git a/cache/lib.php b/cache/lib.php index 1ee4100e7cf..e6941de9397 100644 --- a/cache/lib.php +++ b/cache/lib.php @@ -225,4 +225,4 @@ class cacheable_object_array extends ArrayObject implements cacheable_object { $class = __CLASS__; return new $class($result); } -} \ No newline at end of file +} diff --git a/cache/tests/cache_test.php b/cache/tests/cache_test.php index 8934e771f35..c587bc82588 100644 --- a/cache/tests/cache_test.php +++ b/cache/tests/cache_test.php @@ -40,6 +40,7 @@ use stdClass; * @package core_cache * @copyright 2012 Sam Hemelryk * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @coversDefaultClass \cache */ class cache_test extends advanced_testcase { @@ -51,6 +52,7 @@ class cache_test extends advanced_testcase { require_once($CFG->dirroot . '/cache/locallib.php'); require_once($CFG->dirroot . '/cache/tests/fixtures/lib.php'); + require_once($CFG->dirroot . '/cache/tests/fixtures/cache_phpunit_dummy_datasource_versionable.php'); } /** @@ -561,6 +563,61 @@ class cache_test extends advanced_testcase { $this->assertEquals('c has no value really.', $result['c']); } + /** + * Tests a definition using a data loader with versioned keys. + * + * @covers ::get_versioned + * @covers ::set_versioned + */ + public function test_definition_data_loader_versioned() { + // Create two definitions, one using a non-versionable data source and the other using + // a versionable one. + $instance = cache_config_testing::instance(true); + $instance->phpunit_add_definition('phpunit/datasourcetest1', array( + 'mode' => cache_store::MODE_APPLICATION, + 'component' => 'phpunit', + 'area' => 'datasourcetest1', + 'datasource' => 'cache_phpunit_dummy_datasource', + 'datasourcefile' => 'cache/tests/fixtures/lib.php' + )); + $instance->phpunit_add_definition('phpunit/datasourcetest2', array( + 'mode' => cache_store::MODE_APPLICATION, + 'component' => 'phpunit', + 'area' => 'datasourcetest2', + 'datasource' => 'cache_phpunit_dummy_datasource_versionable', + 'datasourcefile' => 'cache/tests/fixtures/lib.php' + )); + + // The first data source works for normal 'get'. + $cache1 = cache::make('phpunit', 'datasourcetest1'); + $this->assertEquals('Frog has no value really.', $cache1->get('Frog')); + + // But it doesn't work for get_versioned. + try { + $cache1->get_versioned('zombie', 1); + $this->fail(); + } catch (\coding_exception $e) { + $this->assertStringContainsString('Data source is not versionable', $e->getMessage()); + } + + // The second data source works for get_versioned. Set up the datasource first. + $cache2 = cache::make('phpunit', 'datasourcetest2'); + + $datasource = \cache_phpunit_dummy_datasource_versionable::get_last_instance(); + $datasource->has_value('frog', 3, 'Kermit'); + + // Check data with no value. + $this->assertFalse($cache2->get_versioned('zombie', 1)); + + // Check data with value in datastore of required version. + $result = $cache2->get_versioned('frog', 3, IGNORE_MISSING, $actualversion); + $this->assertEquals('Kermit', $result); + $this->assertEquals(3, $actualversion); + + // Check when the datastore doesn't have required version. + $this->assertFalse($cache2->get_versioned('frog', 4)); + } + /** * Tests a definition using an overridden loader */ @@ -1355,7 +1412,9 @@ class cache_test extends advanced_testcase { $this->assertInstanceOf('cache_disabled', $cache); $this->assertFalse($cache->get('test')); + $this->assertFalse($cache->get_versioned('v', 1)); $this->assertFalse($cache->set('test', 'test')); + $this->assertFalse($cache->set_versioned('v', 1, 'data')); $this->assertFalse($cache->delete('test')); $this->assertTrue($cache->purge()); @@ -1364,7 +1423,9 @@ class cache_test extends advanced_testcase { $this->assertInstanceOf('cache_disabled', $cache); $this->assertFalse($cache->get('test')); + $this->assertFalse($cache->get_versioned('v', 1)); $this->assertFalse($cache->set('test', 'test')); + $this->assertFalse($cache->set_versioned('v', 1, 'data')); $this->assertFalse($cache->delete('test')); $this->assertTrue($cache->purge()); @@ -1373,7 +1434,9 @@ class cache_test extends advanced_testcase { $this->assertInstanceOf('cache_disabled', $cache); $this->assertFalse($cache->get('test')); + $this->assertFalse($cache->get_versioned('v', 1)); $this->assertFalse($cache->set('test', 'test')); + $this->assertFalse($cache->set_versioned('v', 1, 'data')); $this->assertFalse($cache->delete('test')); $this->assertTrue($cache->purge()); @@ -1452,6 +1515,369 @@ class cache_test extends advanced_testcase { $this->assertSame(array('two' => 'two', 'three' => 'three'), $cache->get_many(array('two', 'three'))); } + /** + * Data provider to try using a TTL or non-TTL cache. + * + * @return array + */ + public function ttl_or_not(): array { + return [[false], [true]]; + } + + /** + * Data provider to try using a TTL or non-TTL cache, and static acceleration or not. + * + * @return array + */ + public function ttl_and_static_acceleration_or_not(): array { + return [[false, false], [false, true], [true, false], [true, true]]; + } + + /** + * Data provider to try using a TTL or non-TTL cache, and simple data on or off. + * + * @return array + */ + public function ttl_and_simple_data_or_not(): array { + // Same values as for ttl and static acceleration (two booleans). + return $this->ttl_and_static_acceleration_or_not(); + } + + /** + * Shared code to set up a two or three-layer versioned cache for testing. + * + * @param bool $ttl If true, sets TTL in the definition + * @param bool $threelayer If true, uses a 3-layer instead of 2-layer cache + * @param bool $staticacceleration If true, enables static acceleration + * @param bool $simpledata If true, enables simple data + * @return \cache_application Cache + */ + protected function create_versioned_cache(bool $ttl, bool $threelayer = false, + bool $staticacceleration = false, bool $simpledata = false): \cache_application { + $instance = cache_config_testing::instance(true); + $instance->phpunit_add_file_store('a', false); + $instance->phpunit_add_file_store('b', false); + if ($threelayer) { + $instance->phpunit_add_file_store('c', false); + } + $defarray = [ + 'mode' => cache_store::MODE_APPLICATION, + 'component' => 'phpunit', + 'area' => 'multi_loader' + ]; + if ($ttl) { + $defarray['ttl'] = '600'; + } + if ($staticacceleration) { + $defarray['staticacceleration'] = true; + $defarray['staticaccelerationsize'] = 10; + } + if ($simpledata) { + $defarray['simpledata'] = true; + } + $instance->phpunit_add_definition('phpunit/multi_loader', $defarray, false); + $instance->phpunit_add_definition_mapping('phpunit/multi_loader', 'a', 1); + $instance->phpunit_add_definition_mapping('phpunit/multi_loader', 'b', 2); + if ($threelayer) { + $instance->phpunit_add_definition_mapping('phpunit/multi_loader', 'c', 3); + } + + $multicache = cache::make('phpunit', 'multi_loader'); + return $multicache; + } + + /** + * Tests basic use of versioned cache. + * + * @dataProvider ttl_and_simple_data_or_not + * @param bool $ttl If true, uses a TTL cache. + * @param bool $simpledata If true, turns on simple data flag + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_basic(bool $ttl, bool $simpledata): void { + $multicache = $this->create_versioned_cache($ttl, false, false, $simpledata); + + $this->assertTrue($multicache->set_versioned('game', 1, 'Pooh-sticks')); + + $result = $multicache->get_versioned('game', 1, IGNORE_MISSING, $actualversion); + $this->assertEquals('Pooh-sticks', $result); + $this->assertEquals(1, $actualversion); + } + + /** + * Tests versioned cache with objects. + * + * @dataProvider ttl_and_static_acceleration_or_not + * @param bool $ttl If true, uses a TTL cache. + * @param bool $staticacceleration If true, enables static acceleration + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_objects(bool $ttl, bool $staticacceleration): void { + $multicache = $this->create_versioned_cache($ttl, false, $staticacceleration); + + // Set an object value. + $data = (object)['game' => 'Pooh-sticks']; + $this->assertTrue($multicache->set_versioned('game', 1, $data)); + + // Get it. + $result = $multicache->get_versioned('game', 1); + $this->assertEquals('Pooh-sticks', $result->game); + + // Mess about with the value in the returned object. + $result->game = 'Tag'; + + // Get it again and confirm the cached object has not been affected. + $result = $multicache->get_versioned('game', 1); + $this->assertEquals('Pooh-sticks', $result->game); + } + + /** + * Tests requesting a version that doesn't exist. + * + * @dataProvider ttl_or_not + * @param bool $ttl If true, uses a TTL cache. + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_not_exist(bool $ttl): void { + $multicache = $this->create_versioned_cache($ttl); + + $multicache->set_versioned('game', 1, 'Pooh-sticks'); + + // Exists but with wrong version. + $this->assertFalse($multicache->get_versioned('game', 2)); + + // Doesn't exist at all. + $this->assertFalse($multicache->get_versioned('frog', 0)); + } + + /** + * Tests attempts to use get after set_version or get_version after set. + * + * @dataProvider ttl_or_not + * @param bool $ttl If true, uses a TTL cache. + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_incompatible_versioning(bool $ttl): void { + $multicache = $this->create_versioned_cache($ttl); + + // What if you use get on a get_version cache? + $multicache->set_versioned('game', 1, 'Pooh-sticks'); + try { + $multicache->get('game'); + $this->fail(); + } catch (\coding_exception $e) { + $this->assertStringContainsString('Unexpectedly found versioned cache entry', $e->getMessage()); + } + + // Or get_version on a get cache? + $multicache->set('toy', 'Train set'); + try { + $multicache->get_versioned('toy', 1); + $this->fail(); + } catch (\coding_exception $e) { + $this->assertStringContainsString('Unexpectedly found non-versioned cache entry', $e->getMessage()); + } + } + + /** + * Versions are only stored once, so if you set a newer version you will always get it even + * if you ask for the lower version number. + * + * @dataProvider ttl_or_not + * @param bool $ttl If true, uses a TTL cache. + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_single_copy(bool $ttl): void { + $multicache = $this->create_versioned_cache($ttl); + + $multicache->set_versioned('game', 1, 'Pooh-sticks'); + $multicache->set_versioned('game', 2, 'Tag'); + $this->assertEquals('Tag', $multicache->get_versioned('game', 1, IGNORE_MISSING, $actualversion)); + + // The reported version number matches the one returned, not requested. + $this->assertEquals(2, $actualversion); + } + + /** + * If the first (local) store has an outdated copy but the second (shared) store has a newer + * one, then it should automatically be retrieved. + * + * @dataProvider ttl_or_not + * @param bool $ttl If true, uses a TTL cache. + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_outdated_local(bool $ttl): void { + $multicache = $this->create_versioned_cache($ttl); + + // Set initial value to version 2, 'Tag', in both stores. + $multicache->set_versioned('game', 2, 'Tag'); + + // Get the two separate cache stores for the multi-level cache. + $factory = cache_factory::instance(); + $definition = $factory->create_definition('phpunit', 'multi_loader'); + [0 => $storea, 1 => $storeb] = $factory->get_store_instances_in_use($definition); + + // Simulate what happens if the shared cache is updated with a new version but the + // local one still has an old version. + $hashgame = cache_helper::hash_key('game', $definition); + $data = 'British Bulldog'; + if ($ttl) { + $data = new \cache_ttl_wrapper($data, 600); + } + $storeb->set($hashgame, new \core_cache\version_wrapper($data, 3)); + + // If we ask for the old one we'll get it straight off from local cache. + $this->assertEquals('Tag', $multicache->get_versioned('game', 2)); + + // But if we ask for the new one it will still get it via the shared cache. + $this->assertEquals('British Bulldog', $multicache->get_versioned('game', 3)); + + // Also, now it will have been updated in the local cache as well. + $localvalue = $storea->get($hashgame); + if ($ttl) { + // In case the time has changed slightly since the first set, we can't do an exact + // compare, so check it ignoring the time field. + $this->assertEquals(3, $localvalue->version); + $ttldata = $localvalue->data; + $this->assertInstanceOf('cache_ttl_wrapper', $ttldata); + $this->assertEquals('British Bulldog', $ttldata->data); + } else { + $this->assertEquals(new \core_cache\version_wrapper('British Bulldog', 3), $localvalue); + } + } + + /** + * When we request a newer version, older ones are automatically deleted in every level of the + * cache (to save I/O if there are multiple requests, as if there is another request it will + * not have to retrieve the values to find out that they're old). + * + * @dataProvider ttl_or_not + * @param bool $ttl If true, uses a TTL cache. + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_deleting_outdated(bool $ttl): void { + $multicache = $this->create_versioned_cache($ttl); + + // Set initial value to version 2, 'Tag', in both stores. + $multicache->set_versioned('game', 2, 'Tag'); + + // Get the two separate cache stores for the multi-level cache. + $factory = cache_factory::instance(); + $definition = $factory->create_definition('phpunit', 'multi_loader'); + [0 => $storea, 1 => $storeb] = $factory->get_store_instances_in_use($definition); + + // If we request a newer version, then any older version should be deleted in each + // cache level. + $this->assertFalse($multicache->get_versioned('game', 4)); + $hashgame = cache_helper::hash_key('game', $definition); + $this->assertFalse($storea->get($hashgame)); + $this->assertFalse($storeb->get($hashgame)); + } + + /** + * Tests a versioned cache when using static cache. + * + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_static(): void { + $staticcache = $this->create_versioned_cache(false, false, true); + + // Set a value in the cache, version 1. This will store it in static acceleration. + $staticcache->set_versioned('game', 1, 'Pooh-sticks'); + + // Get the first cache store (we don't need the second one for this test). + $factory = cache_factory::instance(); + $definition = $factory->create_definition('phpunit', 'multi_loader'); + [0 => $storea] = $factory->get_store_instances_in_use($definition); + + // Hack a newer version into cache store without directly calling set (now the static + // has v1, store has v2). This simulates another client updating the cache. + $hashgame = cache_helper::hash_key('game', $definition); + $storea->set($hashgame, new \core_cache\version_wrapper('Tag', 2)); + + // Get the key from the cache, v1. This will use static acceleration. + $this->assertEquals('Pooh-sticks', $staticcache->get_versioned('game', 1)); + + // Now if we ask for a newer version, it should not use the static cached one. + $this->assertEquals('Tag', $staticcache->get_versioned('game', 2)); + + // This get should have updated static acceleration, so it will be used next time without + // a store request. + $storea->set($hashgame, new \core_cache\version_wrapper('British Bulldog', 3)); + $this->assertEquals('Tag', $staticcache->get_versioned('game', 2)); + + // Requesting the higher version will get rid of static acceleration again. + $this->assertEquals('British Bulldog', $staticcache->get_versioned('game', 3)); + + // Finally ask for a version that doesn't exist anywhere, just to confirm it returns null. + $this->assertFalse($staticcache->get_versioned('game', 4)); + } + + /** + * Tests basic use of 3-layer versioned caches. + * + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_3_layers_basic(): void { + $multicache = $this->create_versioned_cache(false, true); + + // Basic use of set_versioned and get_versioned. + $multicache->set_versioned('game', 1, 'Pooh-sticks'); + $this->assertEquals('Pooh-sticks', $multicache->get_versioned('game', 1)); + + // What if you ask for a version that doesn't exist? + $this->assertFalse($multicache->get_versioned('game', 2)); + + // Setting a new version wipes out the old version; if you request it, you get the new one. + $multicache->set_versioned('game', 2, 'Tag'); + $this->assertEquals('Tag', $multicache->get_versioned('game', 1)); + } + + /** + * Tests use of 3-layer versioned caches where the 3 layers currently have different versions. + * + * @covers ::set_versioned + * @covers ::get_versioned + */ + public function test_versioned_cache_3_layers_different_data(): void { + // Set version 2 using normal method. + $multicache = $this->create_versioned_cache(false, true); + $multicache->set_versioned('game', 2, 'Tag'); + + // Get the three separate cache stores for the multi-level cache. + $factory = cache_factory::instance(); + $definition = $factory->create_definition('phpunit', 'multi_loader'); + [0 => $storea, 1 => $storeb, 2 => $storec] = $factory->get_store_instances_in_use($definition); + + // Set up two other versions so every level has a different version. + $hashgame = cache_helper::hash_key('game', $definition); + $storeb->set($hashgame, new \core_cache\version_wrapper('British Bulldog', 3)); + $storec->set($hashgame, new \core_cache\version_wrapper('Hopscotch', 4)); + + // First request can be satisfied from A; second request requires B... + $this->assertEquals('Tag', $multicache->get_versioned('game', 2)); + $this->assertEquals('British Bulldog', $multicache->get_versioned('game', 3)); + + // And should update the data in A. + $this->assertEquals(new \core_cache\version_wrapper('British Bulldog', 3), $storea->get($hashgame)); + $this->assertEquals('British Bulldog', $multicache->get_versioned('game', 1)); + + // But newer data should still be in C. + $this->assertEquals('Hopscotch', $multicache->get_versioned('game', 4)); + // Now it's stored in A and B too. + $this->assertEquals(new \core_cache\version_wrapper('Hopscotch', 4), $storea->get($hashgame)); + $this->assertEquals(new \core_cache\version_wrapper('Hopscotch', 4), $storeb->get($hashgame)); + } + /** * Test that multiple application loaders work ok. */ diff --git a/cache/tests/fixtures/cache_phpunit_dummy_datasource_versionable.php b/cache/tests/fixtures/cache_phpunit_dummy_datasource_versionable.php new file mode 100644 index 00000000000..ef0743e3f89 --- /dev/null +++ b/cache/tests/fixtures/cache_phpunit_dummy_datasource_versionable.php @@ -0,0 +1,84 @@ +. + +/** + * A dummy datasource which supports versioning. + * + * @package core_cache + * @copyright 2021 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cache_phpunit_dummy_datasource_versionable extends cache_phpunit_dummy_datasource + implements cache_data_source_versionable { + /** @var array Data in cache */ + protected $data = []; + + /** @var cache_phpunit_dummy_datasource_versionable Last created instance */ + protected static $lastinstance; + + /** + * Returns an instance of this object for use with the cache. + * + * @param cache_definition $definition + * @return cache_phpunit_dummy_datasource New object + */ + public static function get_instance_for_cache(cache_definition $definition): + cache_phpunit_dummy_datasource_versionable { + self::$lastinstance = new cache_phpunit_dummy_datasource_versionable(); + return self::$lastinstance; + } + + /** + * Gets the last instance that was created. + * + * @return cache_phpunit_dummy_datasource_versionable + */ + public static function get_last_instance(): cache_phpunit_dummy_datasource_versionable { + return self::$lastinstance; + } + + /** + * Sets up the datasource so that it has a value for a particular key. + * + * @param string $key Key + * @param int $version Version for key + * @param mixed $data + */ + public function has_value(string $key, int $version, $data): void { + $this->data[$key] = new \core_cache\version_wrapper($data, $version); + } + + /** + * Loads versioned data. + * + * @param int|string $key Key + * @param int $requiredversion Minimum version number + * @param mixed $actualversion Should be set to the actual version number retrieved + * @return mixed Data retrieved from cache or false if none + */ + public function load_for_cache_versioned($key, int $requiredversion, &$actualversion) { + if (!array_key_exists($key, $this->data)) { + return false; + } + $value = $this->data[$key]; + if ($value->version < $requiredversion) { + return false; + } + $actualversion = $value->version; + return $value->data; + } +} + diff --git a/cache/tests/fixtures/cachestore_file_with_ttl_wrappers.php b/cache/tests/fixtures/cachestore_file_with_ttl_wrappers.php new file mode 100644 index 00000000000..c5752a7904f --- /dev/null +++ b/cache/tests/fixtures/cachestore_file_with_ttl_wrappers.php @@ -0,0 +1,37 @@ +. + +/** + * A subclass of cachestore_file but which doesn't report that it has TTL support. + * + * This is so we can easily test behaviour involving the TTL wrapper objects. + * + * @package core_cache + * @copyright 2021 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cachestore_file_with_ttl_wrappers extends cachestore_file { + /** + * Reports the same supported features as the parent, but without SUPPORTS_NATIVE_TTL. + * + * @param array $configuration Configuration + * @return int Supported features + */ + public static function get_supported_features(array $configuration = array()) { + return parent::get_supported_features($configuration) - self::SUPPORTS_NATIVE_TTL; + } +} + diff --git a/cache/tests/fixtures/lib.php b/cache/tests/fixtures/lib.php index bee66090d1a..37ae3731b99 100644 --- a/cache/tests/fixtures/lib.php +++ b/cache/tests/fixtures/lib.php @@ -225,9 +225,15 @@ class cache_config_testing extends cache_config_writer { /** * Forcefully adds a file store. * + * You can turn off native TTL support if you want a way to test TTL wrapper objects. + * * @param string $name + * @param bool $nativettl If false, uses fixture that turns off native TTL support */ - public function phpunit_add_file_store($name) { + public function phpunit_add_file_store(string $name, bool $nativettl = true): void { + if (!$nativettl) { + require_once(__DIR__ . '/cachestore_file_with_ttl_wrappers.php'); + } $this->configstores[$name] = array( 'name' => $name, 'plugin' => 'file', @@ -237,7 +243,7 @@ class cache_config_testing extends cache_config_writer { 'features' => 6, 'modes' => 3, 'mappingsonly' => false, - 'class' => 'cachestore_file', + 'class' => $nativettl ? 'cachestore_file' : 'cachestore_file_with_ttl_wrappers', 'default' => false, 'lock' => 'cachelock_file_default' ); diff --git a/cache/upgrade.txt b/cache/upgrade.txt index 16eaa8f9af0..b42024bff76 100644 --- a/cache/upgrade.txt +++ b/cache/upgrade.txt @@ -7,6 +7,10 @@ Information provided here is intended especially for developers. * The cache_store class now has functions cache_size_details(), store_total_size(), and estimate_stored_size(), related to size used by the cache. These can be overridden by a cache store to provide better information for the new cache usage admin page. +* New functions cache::set_versioned() and cache::get_versioned() can be used to ensure correct + behaviour when using a multi-level cache with early cache levels stored locally. (Used when + rebuilding modinfo.) There is also a new interface cache_data_source_versionable which can + be implemented if you want to make a data source that supports versioning. === 3.10 === * The function supports_recursion() from the lock_factory interface has been deprecated including the related implementations.