MDL-63127 cachestore_redis: Implemented compression

cache/stores/redis/addinstanceform.php

@@ -58,5 +58,11 @@ class cachestore_redis_addinstance_form extends cachestore_addinstance_form {
         $form->addHelpButton('serializer', 'useserializer', 'cachestore_redis');
         $form->setDefault('serializer', Redis::SERIALIZER_PHP);
         $form->setType('serializer', PARAM_INT);
+
+        $compressoroptions = cachestore_redis::config_get_compressor_options();
+        $form->addElement('select', 'compressor', get_string('usecompressor', 'cachestore_redis'), $compressoroptions);
+        $form->addHelpButton('compressor', 'usecompressor', 'cachestore_redis');
+        $form->setDefault('compressor', cachestore_redis::COMPRESSOR_NONE);
+        $form->setType('compressor', PARAM_INT);
     }
 }

cache/stores/redis/lang/en/cachestore_redis.php

@@ -24,6 +24,8 @@
 
 defined('MOODLE_INTERNAL') || die();
 
+$string['compressor_none'] = 'No compression.';
+$string['compressor_php_gzip'] = 'Use gzip compression.';
 $string['pluginname'] = 'Redis';
 $string['prefix'] = 'Key prefix';
 $string['prefix_help'] = 'This prefix is used for all key names on the Redis server.
@@ -48,3 +50,5 @@ $string['useserializer'] = 'Use serializer';
 $string['useserializer_help'] = 'Specifies the serializer to use for serializing.
 The valid serializers are Redis::SERIALIZER_PHP or Redis::SERIALIZER_IGBINARY.
 The latter is supported only when phpredis is configured with --enable-redis-igbinary option and the igbinary extension is loaded.';
+$string['usecompressor'] = 'Use compressor';
+$string['usecompressor_help'] = 'Specifies the compressor to use after serializing. It is done at Moodle Cache API level, not at php-redis level.';

cache/stores/redis/lib.php

@@ -38,6 +38,16 @@ defined('MOODLE_INTERNAL') || die();
  */
 class cachestore_redis extends cache_store implements cache_is_key_aware, cache_is_lockable,
         cache_is_configurable, cache_is_searchable {
+    /**
+     * Compressor: none.
+     */
+    const COMPRESSOR_NONE = 0;
+
+    /**
+     * Compressor: PHP GZip.
+     */
+    const COMPRESSOR_PHP_GZIP = 1;
+
     /**
      * Name of this store.
      *
@@ -80,6 +90,13 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
      */
     protected $serializer = Redis::SERIALIZER_PHP;
 
+    /**
+     * Compressor for this store.
+     *
+     * @var int
+     */
+    protected $compressor = self::COMPRESSOR_NONE;
+
     /**
      * Determines if the requirements for this type of store are met.
      *
@@ -134,6 +151,9 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
         if (array_key_exists('serializer', $configuration)) {
             $this->serializer = (int)$configuration['serializer'];
         }
+        if (array_key_exists('compressor', $configuration)) {
+            $this->compressor = (int)$configuration['compressor'];
+        }
         $password = !empty($configuration['password']) ? $configuration['password'] : '';
         $prefix = !empty($configuration['prefix']) ? $configuration['prefix'] : '';
         $this->redis = $this->new_redis($configuration['server'], $prefix, $password);
@@ -161,7 +181,10 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
             if (!empty($password)) {
                 $redis->auth($password);
             }
-            $redis->setOption(Redis::OPT_SERIALIZER, $this->serializer);
+            // If using compressor, serialisation will be done at cachestore level, not php-redis.
+            if ($this->compressor == self::COMPRESSOR_NONE) {
+                $redis->setOption(Redis::OPT_SERIALIZER, $this->serializer);
+            }
             if (!empty($prefix)) {
                 $redis->setOption(Redis::OPT_PREFIX, $prefix);
             }
@@ -236,7 +259,13 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
      * @return mixed The value of the key, or false if there is no value associated with the key.
      */
     public function get($key) {
-        return $this->redis->hGet($this->hash, $key);
+        $value = $this->redis->hGet($this->hash, $key);
+
+        if ($this->compressor == self::COMPRESSOR_NONE) {
+            return $value;
+        }
+
+        return $this->uncompress($value);
     }
 
     /**
@@ -246,7 +275,17 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
      * @return array An array of the values of the given keys.
      */
     public function get_many($keys) {
-        return $this->redis->hMGet($this->hash, $keys);
+        $values = $this->redis->hMGet($this->hash, $keys);
+
+        if ($this->compressor == self::COMPRESSOR_NONE) {
+            return $values;
+        }
+
+        foreach ($values as &$value) {
+            $value = $this->uncompress($value);
+        }
+
+        return $values;
     }
 
     /**
@@ -257,6 +296,8 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
      * @return bool True if the operation succeeded, false otherwise.
      */
     public function set($key, $value) {
+        $value = $this->compress($value);
+
         return ($this->redis->hSet($this->hash, $key, $value) !== false);
     }
 
@@ -270,7 +311,8 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
     public function set_many(array $keyvaluearray) {
         $pairs = [];
         foreach ($keyvaluearray as $pair) {
-            $pairs[$pair['key']] = $pair['value'];
+            $key = $pair['key'];
+            $pairs[$key] = $this->compress($pairs[$key]);
         }
         if ($this->redis->hMSet($this->hash, $pairs)) {
             return count($pairs);
@@ -446,7 +488,8 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
             'server' => $data->server,
             'prefix' => $data->prefix,
             'password' => $data->password,
-            'serializer' => $data->serializer
+            'serializer' => $data->serializer,
+            'compressor' => $data->compressor,
         );
     }
 
@@ -465,6 +508,9 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
         if (!empty($config['serializer'])) {
             $data['serializer'] = $config['serializer'];
         }
+        if (!empty($config['compressor'])) {
+            $data['compressor'] = $config['compressor'];
+        }
         $editform->set_data($data);
     }
 
@@ -538,4 +584,104 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
         }
         return $options;
     }
+
+    /**
+     * Gets an array of options to use as the compressor.
+     *
+     * @return array
+     */
+    public static function config_get_compressor_options() {
+        return [
+            self::COMPRESSOR_NONE     => get_string('compressor_none', 'cachestore_redis'),
+            self::COMPRESSOR_PHP_GZIP => get_string('compressor_php_gzip', 'cachestore_redis'),
+        ];
+    }
+
+    /**
+     * Compress the given value, serializing it first.
+     *
+     * @param mixed $value
+     * @return string
+     */
+    private function compress($value) {
+        $value = $this->serialize($value);
+
+        switch ($this->compressor) {
+            case self::COMPRESSOR_NONE:
+                return $value;
+
+            case self::COMPRESSOR_PHP_GZIP:
+                return gzencode($value);
+
+            default:
+                debugging("Invalid compressor: {$this->compressor}");
+                return $value;
+        }
+    }
+
+    /**
+     * Uncompresses (deflates) the data, unserialising it afterwards.
+     *
+     * @param string $value
+     * @return mixed
+     */
+    private function uncompress($value) {
+        if ($value === false) {
+            return false;
+        }
+
+        if ($this->compressor == self::COMPRESSOR_NONE) {
+            return $value;
+        }
+
+        switch ($this->compressor) {
+            case self::COMPRESSOR_PHP_GZIP:
+                $value = gzdecode($value);
+                break;
+            default:
+                debugging("Invalid compressor: {$this->compressor}");
+        }
+
+        return $this->unserialize($value);
+    }
+
+    /**
+     * Serializes the data according to the configured serializer.
+     *
+     * @param mixed $value
+     * @return string
+     */
+    private function serialize($value) {
+        switch ($this->serializer) {
+            case Redis::SERIALIZER_NONE:
+                return $value;
+            case Redis::SERIALIZER_PHP:
+                return serialize($value);
+            case Redis::SERIALIZER_IGBINARY:
+                return igbinary_serialize($value);
+            default:
+                debugging("Invalid serializer: {$this->serializer}");
+                return $value;
+        }
+    }
+
+    /**
+     * Unserializes the data according to the configured serializer
+     *
+     * @param string $value
+     * @return mixed
+     */
+    private function unserialize($value) {
+        switch ($this->serializer) {
+            case Redis::SERIALIZER_NONE:
+                return $value;
+            case Redis::SERIALIZER_PHP:
+                return unserialize($value);
+            case Redis::SERIALIZER_IGBINARY:
+                return igbinary_unserialize($value);
+            default:
+                debugging("Invalid serializer: {$this->serializer}");
+                return $value;
+        }
+    }
 }

cache/stores/redis/tests/compressor_test.php

@@ -0,0 +1,404 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Redis cache test.
+ *
+ * If you wish to use these unit tests all you need to do is add the following definition to
+ * your config.php file.
+ *
+ * define('TEST_CACHESTORE_REDIS_TESTSERVERS', '127.0.0.1');
+ *
+ * @package   cachestore_redis
+ * @copyright 2018 Catalyst IT Australia {@link http://www.catalyst-au.net}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once(__DIR__.'/../../../tests/fixtures/stores.php');
+require_once(__DIR__.'/../lib.php');
+
+/**
+ * Redis cache test - compressor settings.
+ *
+ * @package   cachestore_redis
+ * @author    Daniel Thee Roperto <[email protected]>
+ * @copyright 2018 Catalyst IT Australia {@link http://www.catalyst-au.net}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class cachestore_redis_compressor_test extends advanced_testcase {
+    /**
+     * @var cachestore_redis
+     */
+    protected $store = null;
+
+    /**
+     * Create a cachestore.
+     *
+     * @param int      $compressor
+     * @param int|null $serializer
+     * @return cachestore_redis|null
+     */
+    public function create_store($compressor, $serializer = null) {
+        if (!cachestore_redis::are_requirements_met() || !defined('TEST_CACHESTORE_REDIS_TESTSERVERS')) {
+            $this->markTestSkipped('Could not test cachestore_redis. Requirements are not met.');
+            return null;
+        }
+
+        if (is_null($serializer)) {
+            $serializer = Redis::SERIALIZER_PHP;
+        }
+
+        /** @var cache_definition $definition */
+        $definition = cache_definition::load_adhoc(cache_store::MODE_APPLICATION, 'cachestore_redis', 'phpunit_test');
+        $config = cachestore_redis::unit_test_configuration();
+        $config['compressor'] = $compressor;
+        $config['serializer'] = $serializer;
+        $store = new cachestore_redis('Test', $config);
+        $store->initialise($definition);
+
+        return $store;
+    }
+
+    /**
+     * Create a cache store.
+     */
+    public function setUp() {
+        parent::setUp();
+
+        $this->store = $this->create_store(cachestore_redis::COMPRESSOR_PHP_GZIP);
+    }
+
+    /**
+     * Destroy cache store.
+     */
+    protected function tearDown() {
+        parent::tearDown();
+
+        if ($this->store instanceof cachestore_redis) {
+            $this->store->purge();
+        }
+    }
+
+    /**
+     * Set a value.
+     */
+    public function test_it_can_set() {
+        if (is_null($this->store)) {
+            return;
+        }
+
+        $this->store->set('the key', 'the value');
+        $expected = gzencode(serialize('the value'));
+
+        // Disable compressor to check stored value.
+        $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE);
+        $actual = $rawstore->get('the key'); // Compressor was disabled.
+
+        self::assertSame($expected, $actual);
+    }
+
+    /**
+     * Set many values.
+     */
+    public function test_it_can_set_many() {
+        if (is_null($this->store)) {
+            return;
+        }
+
+        // Create values.
+        $values = [];
+        for ($i = 0; $i < 10; $i++) {
+            $values[] = [
+                'key'   => "key_{$i}",
+                'value' => "value #{$i}",
+            ];
+        }
+
+        // Store it.
+        $this->store->set_many($values);
+
+        // Disable compressor to check stored value.
+        $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE);
+
+        foreach ($values as $value) {
+            $expected = gzencode(serialize($value['value']));
+            $actual = $rawstore->get($value['key']); // Compressor was disabled.
+            self::assertSame($expected, $actual, "Invalid value for key={$value['key']}");
+        }
+    }
+
+    /**
+     * Gets a value.
+     */
+    public function test_it_can_get() {
+        if (is_null($this->store)) {
+            return;
+        }
+
+        $this->store->set('the key', 'the value');
+        $actual = $this->store->get('the key');
+        self::assertSame('the value', $actual);
+    }
+
+    /**
+     * Gets many values.
+     */
+    public function test_it_can_get_many() {
+        if (is_null($this->store)) {
+            return;
+        }
+
+        // Create values.
+        $values = [];
+        $keys = [];
+        $expected = [];
+        for ($i = 0; $i < 10; $i++) {
+            $key = "getkey_{$i}";
+            $value = "getvalue #{$i}";
+            $keys[] = $key;
+            $values[] = [
+                'key'   => $key,
+                'value' => $value,
+            ];
+            $expected[$key] = $value;
+        }
+
+        $this->store->set_many($values);
+        $actual = $this->store->get_many($keys);
+        self::assertSame($expected, $actual);
+    }
+
+    /**
+     * It misses a value.
+     */
+    public function test_it_can_miss_one() {
+        if (is_null($this->store)) {
+            return;
+        }
+        $actual = $this->store->get('missme');
+        self::assertFalse($actual);
+    }
+
+    /**
+     * It misses many values.
+     */
+    public function test_it_can_miss_many() {
+        if (is_null($this->store)) {
+            return;
+        }
+        $expected = ['missme' => false, 'missmetoo' => false];
+        $actual = $this->store->get_many(array_keys($expected));
+        self::assertSame($expected, $actual);
+    }
+
+    /**
+     * It misses some values.
+     */
+    public function test_it_can_miss_some() {
+        if (is_null($this->store)) {
+            return;
+        }
+
+        $this->store->set('iamhere', 'youfoundme');
+
+        $expected = ['missme' => false, 'missmetoo' => false, 'iamhere' => 'youfoundme'];
+        $actual = $this->store->get_many(array_keys($expected));
+        self::assertSame($expected, $actual);
+    }
+
+    /**
+     * A provider for test_works_with_different_types
+     *
+     * @return array
+     */
+    public function provider_for_test_it_works_with_different_types() {
+        $object = new stdClass();
+        $object->field = 'value';
+
+        return [
+            ['string', 'Abc Def'],
+            ['string_empty', ''],
+            ['string_binary', gzencode('some binary data')],
+            ['int', 123],
+            ['int_zero', 0],
+            ['int_negative', -100],
+            ['int_huge', PHP_INT_MAX],
+            ['float', 3.14],
+            ['boolean_true', true],
+            // Boolean 'false' is not tested as it is not allowed in Moodle.
+            ['array', [1, 'b', 3.4]],
+            ['array_map', ['a' => 'b', 'c' => 'd']],
+            ['object_stdClass', $object],
+            ['null', null],
+        ];
+    }
+
+    /**
+     * It works with different types.
+     *
+     * @dataProvider provider_for_test_it_works_with_different_types
+     * @param string $key
+     * @param mixed $value
+     */
+    public function test_it_works_with_different_types($key, $value) {
+        if (is_null($this->store)) {
+            return;
+        }
+
+        $this->store->set($key, $value);
+        $actual = $this->store->get($key);
+        self::assertEquals($value, $actual, "Failed set/get for: {$key}");
+    }
+
+    /**
+     * Test it works with different types for many.
+     */
+    public function test_it_works_with_different_types_for_many() {
+        if (is_null($this->store)) {
+            return;
+        }
+
+        $provider = $this->provider_for_test_it_works_with_different_types();
+        $keys = [];
+        $values = [];
+        $expected = [];
+        foreach ($provider as $item) {
+            $keys[] = $item[0];
+            $values[] = ['key' => $item[0], 'value' => $item[1]];
+            $expected[$item[0]] = $item[1];
+        }
+
+        $this->store->set_many($values);
+        $actual = $this->store->get_many($keys);
+        self::assertEquals($expected, $actual);
+    }
+
+    /**
+     * Test it does not use PHP Redis serialization if compression is on.
+     */
+    public function test_it_does_not_use_phpredis_serialisation() {
+        if (is_null($this->store)) {
+            return; // Redis not enabled.
+        }
+
+        $this->store->set('my key', 'my value');
+
+        // Create a connection without serialisation or compressor to fetch raw data.
+        $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE);
+
+        $rawdata = $rawstore->get('my key');
+        $expected = gzencode(serialize('my value')); // It should not have an extra serialisation.
+        self::assertSame($expected, $rawdata);
+    }
+
+    /**
+     * Provider for serializer tests.
+     *
+     * @return array
+     */
+    public function provider_for_test_it_can_use_serializers() {
+        if (!class_exists('Redis')) {
+            return [];
+        }
+
+        $data = [
+            ['none', Redis::SERIALIZER_NONE, gzencode('value1'), gzencode('value2')],
+            ['php', Redis::SERIALIZER_PHP, gzencode(serialize('value1')), gzencode(serialize('value2'))],
+        ];
+
+        if (defined('Redis::SERIALIZER_IGBINARY')) {
+            $data[] = [
+                'igbinary',
+                Redis::SERIALIZER_IGBINARY,
+                gzencode(igbinary_serialize('value1')),
+                gzencode(igbinary_serialize('value2')),
+            ];
+        }
+
+        return $data;
+    }
+
+    /**
+     * Test it can use  serializers with get and set.
+     *
+     * @dataProvider provider_for_test_it_can_use_serializers
+     * @param string $name
+     * @param int $serializer
+     * @param string $rawexpected1
+     * @param string $rawexpected2
+     */
+    public function test_it_can_use_serializers_getset($name, $serializer, $rawexpected1, $rawexpected2) {
+        if (is_null($this->store)) {
+            return; // Redis not enabled.
+        }
+
+        // Create a connection with the desired serialisation.
+        $store = $this->create_store(cachestore_redis::COMPRESSOR_PHP_GZIP, $serializer);
+
+        // Create a connection without serialisation or compressor to fetch raw data.
+        $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE);
+
+        $store->set('key', 'value1');
+        $data = $store->get('key');
+        $rawdata = $rawstore->get('key');
+        self::assertSame('value1', $data, "Invalid serialisation/unserialisation for: {$name}");
+        self::assertSame($rawexpected1, $rawdata, "Invalid rawdata for: {$name}");
+    }
+
+    /**
+     * Test it can use  serializers with get and set many.
+     *
+     * @dataProvider provider_for_test_it_can_use_serializers
+     * @param string $name
+     * @param int $serializer
+     * @param string $rawexpected1
+     * @param string $rawexpected2
+     */
+    public function test_it_can_use_serializers_getsetmany($name, $serializer, $rawexpected1, $rawexpected2) {
+        if (is_null($this->store)) {
+            return; // Redis not enabled.
+        }
+
+        $many = [
+            ['key' => 'key1', 'value' => 'value1'],
+            ['key' => 'key2', 'value' => 'value2'],
+        ];
+        $keys = ['key1', 'key2'];
+        $expectations = ['key1' => 'value1', 'key2' => 'value2'];
+        $rawexpectations = ['key1' => $rawexpected1, 'key2' => $rawexpected2];
+
+        // Create a connection with the desired serialisation.
+        $store = $this->create_store(cachestore_redis::COMPRESSOR_PHP_GZIP, $serializer);
+        $store->set_many($many);
+
+        // Create a connection without serialisation or compressor to fetch raw data.
+        $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE);
+
+        $data = $store->get_many($keys);
+        $rawdata = $rawstore->get_many($keys);
+
+        foreach ($keys as $key) {
+            self::assertSame($expectations[$key],
+                             $data[$key],
+                             "Invalid serialisation/unserialisation for {$key} with serializer {$name}");
+            self::assertSame($rawexpectations[$key],
+                             $rawdata[$key],
+                             "Invalid rawdata for {$key} with serializer {$name}");
+        }
+    }
+}