From e2d4a543b0c30e85fde4c7369bcb25cff47d9655 Mon Sep 17 00:00:00 2001 From: Safat Date: Wed, 30 Nov 2022 01:29:18 +1100 Subject: [PATCH] MDL-76135 core: Implement Guzzle cache middleware --- lib/classes/http_client.php | 30 +- lib/classes/local/guzzle/cache_handler.php | 266 ++++++++++++++++++ lib/classes/local/guzzle/cache_item.php | 199 +++++++++++++ lib/classes/local/guzzle/cache_storage.php | 109 +++++++ .../guzzlecache/src/CacheMiddleware.php | 4 +- lib/tests/http_client_test.php | 56 ++++ 6 files changed, 659 insertions(+), 5 deletions(-) create mode 100644 lib/classes/local/guzzle/cache_handler.php create mode 100644 lib/classes/local/guzzle/cache_item.php create mode 100644 lib/classes/local/guzzle/cache_storage.php diff --git a/lib/classes/http_client.php b/lib/classes/http_client.php index 228923054bd..08ccf05ad40 100644 --- a/lib/classes/http_client.php +++ b/lib/classes/http_client.php @@ -16,6 +16,8 @@ namespace core; +use core\local\guzzle\cache_handler; +use core\local\guzzle\cache_storage; use core\local\guzzle\check_request; use core\local\guzzle\redirect_middleware; use GuzzleHttp\Client; @@ -76,10 +78,6 @@ class http_client extends Client { } } - // Cache. - // TODO Look at whether to implement our own cache, or something like this: - // @see {https://github.com/Kevinrob/guzzle-cache-middleware}. - return $settings; } @@ -107,6 +105,30 @@ class http_client extends Client { $stack->after('allow_redirects', redirect_middleware::setup($settings), 'moodle_allow_redirect'); $stack->remove('allow_redirects'); + // Use cache middleware if cache is enabled. + if (!empty($settings['cache'])) { + $module = 'misc'; + if (!empty($settings['module_cache'])) { + $module = $settings['module_cache']; + } + + // Set TTL for the cache. + if ($module === 'repository') { + if (empty($CFG->repositorycacheexpire)) { + $CFG->repositorycacheexpire = 120; + } + $ttl = $CFG->repositorycacheexpire; + } else { + if (empty($CFG->curlcache)) { + $CFG->curlcache = 120; + } + $ttl = $CFG->curlcache; + } + + $stack->push(new CacheMiddleware (new PrivateCacheStrategy (new cache_storage (new cache_handler($module), $ttl))), + 'cache'); + } + return $stack; } diff --git a/lib/classes/local/guzzle/cache_handler.php b/lib/classes/local/guzzle/cache_handler.php new file mode 100644 index 00000000000..7a878a771bb --- /dev/null +++ b/lib/classes/local/guzzle/cache_handler.php @@ -0,0 +1,266 @@ +. + +namespace core\local\guzzle; + +/** + * Class to handle and generates CacheItemPoolInterface objects. + * + * This class will handle save, delete, cleanup etc. for the cache item. + * For individual cache objects, this class will rely on {@cache_item} class. + * + * @package core + * @copyright 2022 Safat Shahin + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cache_handler { + + /** + * This array will have the kay and value for that key. + * Mainly individual data will be handled by {@cache_item} class for each array element. + * + * @var array $items cached items or the items currently in use by the cache pool. + */ + private array $items; + + /** + * This array will have the cache items which might need to persisted later. + * It will not save the items in the cache pool using cache_item class until the commit is done for these elements. + * + * @var array $deferreditems cache items to be persisted later. + */ + private array $deferreditems; + + /** + * Constructor for class cache_handler. + * This class will accept the module which will determine the location of cached files. + * + * @param string $module module string for cache directory. + */ + public function __construct(string $module = 'repository') { + global $CFG; + $this->module = $module; + + // Set the directory for cache. + $this->dir = $CFG->cachedir . '/' . $module . '/'; + if (!file_exists($this->dir) && !mkdir($concurrentdirectory = $this->dir, $CFG->directorypermissions, true) && + !is_dir($concurrentdirectory)) { + throw new \moodle_exception(sprintf('Directory "%s" was not created', $concurrentdirectory)); + } + + } + + /** + * Returns a Cache Item representing the specified key. + * + * This method must always return a CacheItemInterface object, even in case of + * a cache miss. It MUST NOT return null. + * + * @param string $key The key for which to return the corresponding Cache Item.. + * @param int|null $ttl Number of seconds for the cache item to live. + * @return cache_item The corresponding Cache Item. + */ + public function get_item(string$key, ?int $ttl = null): cache_item { + return new cache_item($key, $this->module, $ttl); + } + + /** + * Returns a traversable set of cache items. + * + * @param string[] $keys An indexed array of keys of items to retrieve. + * @return iterable + * An iterable collection of Cache Items keyed by the cache keys of + * each item. A Cache item will be returned for each key, even if that + * key is not found. However, if no keys are specified then an empty + * traversable MUST be returned instead. + */ + public function get_items(array $keys = []): iterable { + $items = []; + + foreach ($keys as $key) { + $items[$key] = $this->has_item($key) ? clone $this->items[$key] : $this->get_item($key); + } + + return $items; + } + + /** + * Confirms if the cache contains specified cache item. + * + * Note: This method MAY avoid retrieving the cached value for performance reasons. + * This could result in a race condition with CacheItemInterface::get(). To avoid + * such situation use CacheItemInterface::isHit() instead. + * + * @param string $key The key for which to check existence. + * @return bool True if item exists in the cache, false otherwise. + */ + public function has_item($key): bool { + $this->assert_key_is_valid($key); + + return isset($this->items[$key]) && $this->items[$key]->isHit(); + } + + /** + * Deletes all items in the pool. + * + * @return bool True if the pool was successfully cleared. False if there was an error. + */ + public function clear(): bool { + global $USER; + + if (isset($this->items)) { + foreach ($this->items as $key => $item) { + // Delete cache file. + if ($dir = opendir($this->dir)) { + $filename = 'u' . $USER->id . '_' . md5(serialize($key)); + $filename = $dir . $filename; + if (file_exists($filename) && $this->items[$key]->isHit()) { + @unlink($filename); + } + closedir($dir); + } + } + } + + $this->items = []; + $this->deferreditems = []; + + return true; + } + + /** + * Refreshes all items in the pool. + * + * @param int $ttl Seconds to live. + * @return void + */ + public function refresh(int $ttl): void { + if ($dir = opendir($this->dir)) { + while (false !== ($file = readdir($dir))) { + if (!is_dir($file) && $file !== '.' && $file !== '..') { + $lasttime = @filemtime($this->dir . $file); + if (time() - $lasttime > $ttl) { + mtrace($this->dir . $file); + @unlink($this->dir . $file); + } + } + } + closedir($dir); + } + } + + /** + * Removes the item from the pool. + * + * @param string $key The key to delete. + * @return bool True if the item was successfully removed. False if there was an error. + */ + public function delete_item(string $key): bool { + return $this->delete_items([$key]); + } + + /** + * Removes multiple items from the pool. + * + * @param string[] $keys An array of keys that should be removed from the pool. + * @return bool True if the items were successfully removed. False if there was an error. + */ + public function delete_items(array $keys): bool { + global $USER; + array_walk($keys, [$this, 'assert_key_is_valid']); + + foreach ($keys as $key) { + // Delete cache file. + if ($dir = opendir($this->dir)) { + $filename = 'u' . $USER->id . '_' . md5(serialize($key)); + $filename = $dir . $filename; + if (file_exists($filename)) { + @unlink($filename); + } + } + + unset($this->items[$key]); + } + + return true; + } + + /** + * Persists a cache item immediately. + * + * @param cache_item $item The cache item to save. + * @return bool True if the item was successfully persisted. False if there was an error. + */ + public function save(cache_item $item): bool { + global $CFG, $USER; + $key = $item->get_key(); + + // File and directory setup. + $filename = 'u' . $USER->id . '_' . md5(serialize($key)); + $fp = fopen($this->dir . $filename, 'wb'); + + // Store the item. + fwrite($fp, serialize($item->get())); + fclose($fp); + @chmod($this->dir . $filename, $CFG->filepermissions); + + $this->items[$key] = $item; + + return true; + } + + /** + * Sets a cache item to be persisted later. + * + * @param cache_item $item The cache item to save. + * @return bool False if the item could not be queued or if a commit was attempted and failed. True otherwise. + */ + public function save_deferred(cache_item $item): bool { + $this->deferreditems[$item->get_key()] = $item; + + return true; + } + + /** + * Persists any deferred cache items. + * + * @return bool True if all not-yet-saved items were successfully saved or there were none. False otherwise. + */ + public function commit(): bool { + foreach ($this->deferreditems as $item) { + $this->save($item); + } + + $this->deferreditems = []; + + return true; + } + + /** + * Asserts that the given key is valid. + * Some simple validation to make sure the passed key is a valid one. + * + * @param string $key The key to validate. + */ + private function assert_key_is_valid(string $key): void { + $invalidcharacters = '{}()/\\\\@:'; + + if (!is_string($key) || preg_match("#[$invalidcharacters]#", $key)) { + throw new \moodle_exception('Invalid cache key'); + } + } + +} diff --git a/lib/classes/local/guzzle/cache_item.php b/lib/classes/local/guzzle/cache_item.php new file mode 100644 index 00000000000..e6ef8e00657 --- /dev/null +++ b/lib/classes/local/guzzle/cache_item.php @@ -0,0 +1,199 @@ +. + +namespace core\local\guzzle; + +/** + * Class to define an interface for interacting with objects inside a cache. + * + * @package core + * @copyright 2022 safatshahin + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cache_item { + + /** + * Key to be used to store and identify the cache. + * + * @var string $key cache item key. + */ + private string $key; + + /** + * Actual data of the cache. + * + * @var mixed $value cache data. + */ + private $value; + + /** + * The expiry of the cache item according to TTL. + * + * @var \DateTime|null $expiration TTL time for the cache item. + */ + private ?\DateTime $expiration; + + /** + * Confirms if the cache item lookup resulted in a cache hit. + * + * @var bool $ishit determine the cache lookup. + */ + private bool $ishit = false; + + /** + * Constructor for the cache_item to get the key and module to retrieve or set the cache item. + * + * @param string $key The key for the current cache item. + * @param string $module determines the location of the cache item. + * @param int|null $ttl Time to live for the cache item. + */ + public function __construct(string $key, string $module, ?int $ttl = null) { + global $CFG, $USER; + $this->key = $key; + // Set the directory for cache. + $this->dir = $CFG->cachedir . '/' . $module . '/'; + if (file_exists($this->dir)) { + $filename = 'u' . $USER->id . '_' . md5(serialize($key)); + // If the cache fine exists, set the value from the cache file. + if (file_exists($this->dir . $filename)) { + $this->ishit = true; + $this->expires_after($ttl); + $fp = fopen($this->dir . $filename, 'rb'); + $size = filesize($this->dir . $filename); + $content = fread($fp, $size); + $this->value = unserialize($content); + } + } + } + + /** + * Returns the key for the current cache item. + * + * The key is loaded by the Implementing Library, but should be available to + * the higher level callers when needed. + * + * @return string The key string for this cache item. + */ + public function get_key(): string { + return $this->key; + } + + /** + * Retrieves the value of the item from the cache associated with this object's key. + * + * The value returned must be identical to the value originally stored by set(). + * + * If isHit() returns false, this method MUST return null. Note that null + * is a legitimate cache value, so the isHit() method SHOULD be used to + * differentiate between "null value was found" and "no value was found." + * + * @return mixed The value corresponding to this cache item's key, or null if not found. + */ + public function get() { + return $this->is_hit() ? $this->value : null; + } + + /** + * Confirms if the cache item lookup resulted in a cache hit. + * + * Note: This method MUST NOT have a race condition between calling isHit() + * and calling get(). + * + * @return bool True if the request resulted in a cache hit. False otherwise. + */ + public function is_hit():bool { + if (!$this->ishit) { + return false; + } + + if ($this->expiration === null) { + return true; + } + + return $this->current_time()->getTimestamp() < $this->expiration->getTimestamp(); + } + + /** + * Sets the value represented by this cache item. + * + * The $value argument may be any item that can be serialized by PHP, + * although the method of serialization is left up to the Implementing + * Library. + * + * @param mixed $value The serializable value to be stored. + * @return static The invoked object. + */ + public function set($value): cache_item { + $this->ishit = true; + $this->value = $value; + + return $this; + } + + /** + * Sets the absolute expiration time for this cache item. + * + * @param \DateTimeInterface|null $expiration + * The point in time after which the item MUST be considered expired. + * If null is passed explicitly, a default value MAY be used. If none is set, + * the value should be stored permanently or for as long as the + * implementation allows. + * + * @return static The called object. + */ + public function expires_at($expiration): cache_item { + if (null === $expiration || $expiration instanceof \DateTimeInterface) { + $this->expiration = $expiration; + + return $this; + } + + throw new \coding_exception('Invalid argument passed'); + } + + /** + * Sets the relative expiration time for this cache item. + * + * @param int|\DateInterval|null $time + * The period of time from the present after which the item MUST be considered + * expired. An integer parameter is understood to be the time in seconds until + * expiration. If null is passed explicitly, a default value MAY be used. + * If none is set, the value should be stored permanently or for as long as the + * implementation allows. + * + * @return static The called object. + */ + public function expires_after($time): cache_item { + if (is_int($time) && $time >= 0) { + $this->expiration = $this->current_time()->add(new \DateInterval("PT{$time}S")); + } else if ($time instanceof \DateInterval) { + $this->expiration = $this->current_time()->add($time); + } else { + $this->expiration = $time; + } + + return $this; + } + + /** + * Gets the current time in the user timezone. + * + * @return \DateTime + */ + private function current_time(): \DateTime { + return new \DateTime('now', new \DateTimeZone(get_user_timezone())); + } +} diff --git a/lib/classes/local/guzzle/cache_storage.php b/lib/classes/local/guzzle/cache_storage.php new file mode 100644 index 00000000000..58e0d386e08 --- /dev/null +++ b/lib/classes/local/guzzle/cache_storage.php @@ -0,0 +1,109 @@ +. + +namespace core\local\guzzle; + +use Kevinrob\GuzzleCache\CacheEntry; +use Kevinrob\GuzzleCache\Storage\CacheStorageInterface; + +/** + * Cache storage handler to handle cache objects, TTL etc. + * + * @package core + * @copyright 2022 safatshahin + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cache_storage implements CacheStorageInterface { + + /** + * The cache pool. + * + * @var cache_handler + */ + protected $cachepool; + + /** + * The last item retrieved from the cache. + * + * This item is transiently stored so that save() can reuse the cache item + * usually retrieved by fetch() beforehand, instead of requesting it a second time. + * + * @var cache_item|null + */ + protected $lastitem; + + /** + * TTL for the cache. + * + * @var int|null time to live. + */ + private int $ttl; + + public function __construct(cache_handler $cachepool, ?int $ttl = null) { + $this->cachepool = $cachepool; + $this->ttl = $ttl; + } + + public function fetch($key): ?CacheEntry { + // Refresh the cache files. + if ($this->ttl) { + $this->cachepool->refresh($this->ttl); + } + $item = $this->cachepool->get_item($key, $this->ttl); + $this->lastitem = $item; + + $cache = $item->get(); + + if ($cache instanceof CacheEntry) { + return $cache; + } + + return null; + } + + public function save($key, CacheEntry $data): bool { + if ($this->lastitem && $this->lastitem->get_key() === $key) { + $item = $this->lastitem; + } else { + $item = $this->cachepool->get_item($key); + } + + $this->lastitem = null; + + $item->set($data); + + // Check if the TTL is set, otherwise use from data. + $ttl = $this->ttl ?? $data->getTTL(); + + if ($ttl === 0) { + // No expiration. + $item->expires_after(null); + } else { + $item->expires_after($ttl); + } + + return $this->cachepool->save($item); + } + + public function delete($key): bool { + if (null !== $this->lastitem && $this->lastitem->get_key() === $key) { + $this->lastitem = null; + } + + return $this->cachepool->delete_item($key); + } + +} diff --git a/lib/guzzlehttp/kevinrob/guzzlecache/src/CacheMiddleware.php b/lib/guzzlehttp/kevinrob/guzzlecache/src/CacheMiddleware.php index bb55f0cd5be..f5fdc5e8f98 100644 --- a/lib/guzzlehttp/kevinrob/guzzlecache/src/CacheMiddleware.php +++ b/lib/guzzlehttp/kevinrob/guzzlecache/src/CacheMiddleware.php @@ -7,6 +7,7 @@ use GuzzleHttp\Exception\TransferException; use GuzzleHttp\Promise\FulfilledPromise; use GuzzleHttp\Promise\Promise; use GuzzleHttp\Promise\RejectedPromise; +use GuzzleHttp\Promise\Utils; use GuzzleHttp\Psr7\Response; use Kevinrob\GuzzleCache\Strategy\CacheStrategyInterface; use Kevinrob\GuzzleCache\Strategy\PrivateCacheStrategy; @@ -110,7 +111,8 @@ class CacheMiddleware */ public function purgeReValidation() { - \GuzzleHttp\Promise\inspect_all($this->waitingRevalidate); + // Call to \GuzzleHttp\Promise\inspect_all throws error, replacing with the latest one. + Utils::inspectAll($this->waitingRevalidate); } /** diff --git a/lib/tests/http_client_test.php b/lib/tests/http_client_test.php index 8da9f8d7710..e7e7e8e9984 100644 --- a/lib/tests/http_client_test.php +++ b/lib/tests/http_client_test.php @@ -30,6 +30,11 @@ use GuzzleHttp\Psr7\Uri; * @copyright 2022 Safat Shahin * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later * @coversDefaultClass \core\http_client + * @coversDefaultClass \core\local\guzzle\redirect_middleware + * @coversDefaultClass \core\local\guzzle\check_request + * @coversDefaultClass \core\local\guzzle\cache_item + * @coversDefaultClass \core\local\guzzle\cache_handler + * @coversDefaultClass \core\local\guzzle\cache_storage */ class http_client_test extends \advanced_testcase { @@ -353,4 +358,55 @@ class http_client_test extends \advanced_testcase { $this->assertSame($testurl, (string)$mock->getLastRequest()->getUri()); $this->assertSame('done', $response->getBody()->getContents()); } + + /** + * Test guzzle cache middleware. + * + * @covers \core\local\guzzle\cache_item + * @covers \core\local\guzzle\cache_handler + * @covers \core\local\guzzle\cache_storage + */ + public function test_http_client_cache_item() { + global $CFG, $USER; + $module = 'core_guzzle'; + $cachedir = "$CFG->cachedir/$module/"; + + $testhtml = $this->getExternalTestFileUrl('/test.html'); + + // Test item is cached in the specified module. + $client = new \core\http_client([ + 'cache' => true, + 'module_cache' => $module + ]); + $response = $client->get($testhtml); + + $cachecontent = ''; + if ($dir = opendir($cachedir)) { + while (false !== ($file = readdir($dir))) { + if (!is_dir($file) && $file !== '.' && $file !== '..') { + if (strpos($file, 'u' . $USER->id . '_') !== false) { + $cachecontent = file_get_contents($cachedir . $file); + } + } + } + } + + $this->assertNotEmpty($cachecontent); + @unlink($cachedir . $file); + + // Test cache item objects returns correct values. + $key = 'sample_key'; + $cachefilename = 'u' . $USER->id . '_' . md5(serialize($key)); + $cachefile = $cachedir.$cachefilename; + + $content = $response->getBody()->getContents(); + file_put_contents($cachefile, serialize($content)); + + $cacheitemobject = new \core\local\guzzle\cache_item($key, $module, null); + + // Test the cache item matches with the cached response. + $this->assertSame($content, $cacheitemobject->get()); + + @unlink($cachefile); + } }