From 7f8a816a7d17d028c26f1749fa1daa52c2e35b01 Mon Sep 17 00:00:00 2001 From: Russell Smith Date: Sun, 19 Jun 2016 16:24:11 +1000 Subject: [PATCH] MDL-54606 session: Add redis handler with locking. --- config-dist.php | 9 + lib/classes/session/redis.php | 382 +++++++++++++++++++++++++++++++ lib/tests/session_redis_test.php | 272 ++++++++++++++++++++++ 3 files changed, 663 insertions(+) create mode 100644 lib/classes/session/redis.php create mode 100644 lib/tests/session_redis_test.php diff --git a/config-dist.php b/config-dist.php index faa77deda23..ac9297e799f 100644 --- a/config-dist.php +++ b/config-dist.php @@ -249,6 +249,15 @@ $CFG->admin = 'admin'; // $CFG->session_memcached_acquire_lock_timeout = 120; // $CFG->session_memcached_lock_expire = 7200; // Ignored if PECL memcached is below version 2.2.0 // +// Redis session handler (requires redis server and redis extension): +// $CFG->session_handler_class = '\core\session\redis'; +// $CFG->session_redis_host = '127.0.0.1'; +// $CFG->session_redis_port = 6379; // Optional. +// $CFG->session_redis_database = 0; // Optional, default is db 0. +// $CFG->session_redis_prefix = ''; // Optional, default is don't set one. +// $CFG->session_redis_acquire_lock_timeout = 120; +// $CFG->session_redis_lock_expire = 7200; +// // Memcache session handler (requires memcached server and memcache extension): // $CFG->session_handler_class = '\core\session\memcache'; // $CFG->session_memcache_save_path = '127.0.0.1:11211'; diff --git a/lib/classes/session/redis.php b/lib/classes/session/redis.php new file mode 100644 index 00000000000..d8b06107a5d --- /dev/null +++ b/lib/classes/session/redis.php @@ -0,0 +1,382 @@ +. + +/** + * Redis based session handler. + * + * @package core + * @copyright 2015 Russell Smith + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\session; + +use RedisException; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Redis based session handler. + * + * The default Redis session handler does not handle locking in 2.2.7, so we have written a php session handler + * that uses locking. The places where locking is used was modeled from the memcached code that is used in Moodle + * https://github.com/php-memcached-dev/php-memcached/blob/master/php_memcached_session.c + * + * @package core + * @copyright 2016 Russell Smith + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class redis extends handler { + /** @var string $host save_path string */ + protected $host = ''; + /** @var int $port The port to connect to */ + protected $port = 6379; + /** @var int $database the Redis database to store sesions in */ + protected $database = 0; + /** @var array $servers list of servers parsed from save_path */ + protected $prefix = ''; + /** @var int $acquiretimeout how long to wait for session lock in seconds */ + protected $acquiretimeout = 120; + /** + * @var int $lockexpire how long to wait in seconds before expiring the lock automatically + * so that other requests may continue execution, ignored if PECL redis is below version 2.2.0. + */ + protected $lockexpire = 7200; + + /** @var Redis Connection */ + protected $connection = null; + + /** @var array $locks List of currently held locks by this page. */ + protected $locks = array(); + + /** + * Create new instance of handler. + */ + public function __construct() { + global $CFG; + + if (isset($CFG->session_redis_host)) { + $this->host = $CFG->session_redis_host; + } + + if (isset($CFG->session_redis_port)) { + $this->port = (int)$CFG->session_redis_port; + } + + if (isset($CFG->session_redis_database)) { + $this->database = (int)$CFG->session_redis_database; + } + + if (isset($CFG->session_redis_prefix)) { + $this->prefix = $CFG->session_redis_prefix; + } + + if (isset($CFG->session_redis_acquire_lock_timeout)) { + $this->acquiretimeout = (int)$CFG->session_redis_acquire_lock_timeout; + } + + if (isset($CFG->session_redis_lock_expire)) { + $this->lockexpire = (int)$CFG->session_redis_lock_expire; + } + } + + /** + * Start the session. + * + * @return bool success + */ + public function start() { + $result = parent::start(); + + return $result; + } + + /** + * Init session handler. + */ + public function init() { + if (!extension_loaded('redis')) { + throw new exception('sessionhandlerproblem', 'error', '', null, 'redis extension is not loaded'); + } + + if (empty($this->host)) { + throw new exception('sessionhandlerproblem', 'error', '', null, + '$CFG->session_redis_host must be specified in config.php'); + } + + // The session handler requires a version of Redis with the SETEX command (at least 2.0). + $version = phpversion('Redis'); + if (!$version or version_compare($version, '2.0') <= 0) { + throw new exception('sessionhandlerproblem', 'error', '', null, 'redis extension version must be at least 2.0'); + } + + $this->connection = new \Redis(); + + $result = session_set_save_handler(array($this, 'handler_open'), + array($this, 'handler_close'), + array($this, 'handler_read'), + array($this, 'handler_write'), + array($this, 'handler_destroy'), + array($this, 'handler_gc')); + if (!$result) { + throw new exception('redissessionhandlerproblem', 'error'); + } + + try { + // One second timeout was chosen as it is long for connection, but short enough for a user to be patient. + if (!$this->connection->connect($this->host, $this->port, 1)) { + throw new RedisException('Unable to connect to host.'); + } + if (!$this->connection->setOption(\Redis::OPT_SERIALIZER, \Redis::SERIALIZER_PHP)) { + throw new RedisException('Unable to set Redis PHP Serializer option.'); + } + + if ($this->prefix !== '') { + // Use custom prefix on sessions. + if (!$this->connection->setOption(\Redis::OPT_PREFIX, $this->prefix)) { + throw new RedisException('Unable to set Redis Prefix option.'); + } + } + if ($this->database !== 0) { + if (!$this->connection->select($this->database)) { + throw new RedisException('Unable to select Redis database '.$this->database.'.'); + } + } + $this->connection->ping(); + return true; + } catch (RedisException $e) { + error_log('Failed to connect to redis at '.$this->host.':'.$this->port.', error returned was: '.$e->getMessage()); + return false; + } + } + + /** + * Update our session search path to include session name when opened. + * + * @param string $savepath unused session save path. (ignored) + * @param string $sessionname Session name for this session. (ignored) + * @return bool true always as we will succeed. + */ + public function handler_open($savepath, $sessionname) { + return true; + } + + /** + * Close the session completely. We also remove all locks we may have obtained that aren't expired. + * + * @return bool true on success. false on unable to unlock sessions. + */ + public function handler_close() { + try { + foreach ($this->locks as $id => $expirytime) { + if ($expirytime > $this->time()) { + $this->unlock_session($id); + } + unset($this->locks[$id]); + } + } catch (RedisException $e) { + error_log('Failed talking to redis: '.$e->getMessage()); + return false; + } + + return true; + } + /** + * Read the session data from storage + * + * @param string $id The session id to read from storage. + * @return string The session data for PHP to process. + * + * @throws RedisException when we are unable to talk to the Redis server. + */ + public function handler_read($id) { + try { + $this->lock_session($id); + $sessiondata = $this->connection->get($id); + if ($sessiondata === false) { + $this->unlock_session($id); + return ''; + } + $this->connection->expire($id, $this->lockexpire); + } catch (RedisException $e) { + error_log('Failed talking to redis: '.$e->getMessage()); + throw $e; + } + return $sessiondata; + } + + /** + * Write the serialized session data to our session store. + * + * @param string $id session id to write. + * @param string $data session data + * @return bool true on write success, false on failure + */ + public function handler_write($id, $data) { + if (is_null($this->connection)) { + // The session has already been closed, don't attempt another write. + error_log('Tried to write session: '.$id.' before open or after close.'); + return false; + } + + // We do not do locking here because memcached doesn't. Also + // PHP does open, read, destroy, write, close. When a session doesn't exist. + // There can be race conditions on new sessions racing each other but we can + // address that in the future. + try { + $this->connection->setex($id, $this->lockexpire, $data); + } catch (RedisException $e) { + error_log('Failed talking to redis: '.$e->getMessage()); + return false; + } + return true; + } + + /** + * Handle destroying a session. + * + * @param string $id the session id to destroy. + * @return bool true if the session was deleted, false otherwise. + */ + public function handler_destroy($id) { + try { + $this->connection->del($id); + $this->unlock_session($id); + } catch (RedisException $e) { + error_log('Failed talking to redis: '.$e->getMessage()); + return false; + } + + return true; + } + + /** + * Garbage collect sessions. We don't we any as Redis does it for us. + * + * @param integer $maxlifetime All sessions older than this should be removed. + * @return bool true, as Redis handles expiry for us. + */ + public function handler_gc($maxlifetime) { + return true; + } + + /** + * Unlock a session. + * + * @param string $id Session id to be unlocked. + */ + protected function unlock_session($id) { + if (isset($this->locks[$id])) { + $this->connection->del($id.".lock"); + unset($this->locks[$id]); + } + } + + /** + * Obtain a session lock so we are the only one using it at the moent. + * + * @param string $id The session id to lock. + * @return bool true when session was locked, exception otherwise. + * @throws exception When we are unable to obtain a session lock. + */ + protected function lock_session($id) { + $lockkey = $id.".lock"; + + $haslock = isset($this->locks[$id]) && $this->time() < $this->locks[$id]; + $startlocktime = $this->time(); + + /* To be able to ensure sessions don't write out of order we must obtain an exclusive lock + * on the session for the entire time it is open. If another AJAX call, or page is using + * the session then we just wait until it finishes before we can open the session. + */ + while (!$haslock) { + $haslock = $this->connection->setnx($lockkey, '1'); + if (!$haslock) { + usleep(rand(100000, 1000000)); + if ($this->time() > $startlocktime + $this->acquiretimeout) { + // This is a fatal error, better inform users. + // It should not happen very often - all pages that need long time to execute + // should close session immediately after access control checks. + error_log('Cannot obtain session lock for sid: '.$id.' within '.$this->acquiretimeout. + '. It is likely another page has a long session lock, or the session lock was never released.'); + throw new exception("Unable to obtain session lock"); + } + } else { + $this->locks[$id] = $this->time() + $this->lockexpire; + $this->connection->expire($lockkey, $this->lockexpire); + return true; + } + } + } + + /** + * Return the current time. + * + * @return int the current time as a unixtimestamp. + */ + protected function time() { + return time(); + } + + /** + * Check the backend contains data for this session id. + * + * Note: this is intended to be called from manager::session_exists() only. + * + * @param string $sid + * @return bool true if session found. + */ + public function session_exists($sid) { + if (!$this->connection) { + return false; + } + + try { + return $this->connection->exists($sid); + } catch (RedisException $e) { + return false; + } + } + + /** + * Kill all active sessions, the core sessions table is purged afterwards. + */ + public function kill_all_sessions() { + global $DB; + if (!$this->connection) { + return; + } + + $rs = $DB->get_recordset('sessions', array(), 'id DESC', 'id, sid'); + foreach ($rs as $record) { + $this->handler_destroy($record->sid); + } + $rs->close(); + } + + /** + * Kill one session, the session record is removed afterwards. + * + * @param string $sid + */ + public function kill_session($sid) { + if (!$this->connection) { + return; + } + + $this->handler_destroy($sid); + } +} \ No newline at end of file diff --git a/lib/tests/session_redis_test.php b/lib/tests/session_redis_test.php new file mode 100644 index 00000000000..8bc12029e0c --- /dev/null +++ b/lib/tests/session_redis_test.php @@ -0,0 +1,272 @@ +. + +/** + * Redis session tests. + * + * NOTE: in order to execute this test you need to set up + * Redis server and add configuration a constant + * to config.php or phpunit.xml configuration file: + * + * define('TEST_SESSION_REDIS_HOST', '127.0.0.1'); + * + * @package core + * @author Russell Smith + * @copyright 2016 Russell Smith + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Unit tests for classes/session/redis.php. + * + * @package core + * @author Russell Smith + * @copyright 2016 Russell Smith + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_session_redis_testcase extends advanced_testcase { + + /** @var $keyprefix This key prefix used when testing Redis */ + protected $keyprefix = null; + /** @var $redis The current testing redis connection */ + protected $redis = null; + + public function setUp() { + global $CFG; + + if (!extension_loaded('redis')) { + $this->markTestSkipped('Redis extension not loaded.'); + } + if (!defined('TEST_SESSION_REDIS_HOST')) { + $this->markTestSkipped('Session test server not set. define: TEST_SESSION_REDIS_HOST'); + } + + $this->resetAfterTest(); + + $this->keyprefix = 'phpunit'.rand(1, 100000); + + $CFG->session_redis_host = TEST_SESSION_REDIS_HOST; + $CFG->session_redis_prefix = $this->keyprefix; + + // Set a very short lock timeout to ensure tests run quickly. We are running single threaded, + // so unless we lock and expect it to be there, we will always see a lock. + $CFG->session_redis_acquire_lock_timeout = 1; + $CFG->session_redis_lock_expire = 70; + + $this->redis = new Redis(); + $this->redis->connect(TEST_SESSION_REDIS_HOST); + } + + public function tearDown() { + if (!extension_loaded('redis') || !defined('TEST_SESSION_REDIS_HOST')) { + return; + } + + $list = $this->redis->keys($this->keyprefix.'*'); + foreach ($list as $keyname) { + $this->redis->del($keyname); + } + $this->redis->close(); + } + + public function test_normal_session_start_stop_works() { + $sess = new \core\session\redis(); + $sess->init(); + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertSame('', $sess->handler_read('sess1')); + $this->assertTrue($sess->handler_write('sess1', 'DATA')); + $this->assertTrue($sess->handler_close()); + + // Read the session again to ensure locking did what it should. + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertSame('DATA', $sess->handler_read('sess1')); + $this->assertTrue($sess->handler_write('sess1', 'DATA-new')); + $this->assertTrue($sess->handler_close()); + $this->assertSessionNoLocks(); + } + + public function test_session_blocks_with_existing_session() { + $sess = new \core\session\redis(); + $sess->init(); + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertSame('', $sess->handler_read('sess1')); + $this->assertTrue($sess->handler_write('sess1', 'DATA')); + $this->assertTrue($sess->handler_close()); + + // Sessions are not locked until they have been saved once. + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertSame('DATA', $sess->handler_read('sess1')); + + $sessblocked = new \core\session\redis(); + $sessblocked->init(); + $this->assertTrue($sessblocked->handler_open('Not used', 'Not used')); + + // Trap the error log and send it to stdOut so we can expect output at the right times. + $errorlog = tempnam(sys_get_temp_dir(), "rediserrorlog"); + $this->iniSet('error_log', $errorlog); + try { + $sessblocked->handler_read('sess1'); + $this->fail('Session lock must fail to be obtained.'); + } catch (\core\session\exception $e) { + $this->assertContains("Unable to obtain session lock", $e->getMessage()); + $this->assertContains('Cannot obtain session lock for sid: sess1', file_get_contents($errorlog)); + } + + $this->assertTrue($sessblocked->handler_close()); + $this->assertTrue($sess->handler_write('sess1', 'DATA-new')); + $this->assertTrue($sess->handler_close()); + $this->assertSessionNoLocks(); + } + + public function test_session_is_destroyed_when_it_does_not_exist() { + $sess = new \core\session\redis(); + $sess->init(); + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertTrue($sess->handler_destroy('sess-destroy')); + $this->assertSessionNoLocks(); + } + + public function test_session_is_destroyed_when_we_have_it_open() { + $sess = new \core\session\redis(); + $sess->init(); + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertSame('', $sess->handler_read('sess-destroy')); + $this->assertTrue($sess->handler_destroy('sess-destroy')); + $this->assertTrue($sess->handler_close()); + $this->assertSessionNoLocks(); + } + + public function test_multiple_sessions_do_not_interfere_with_each_other() { + $sess1 = new \core\session\redis(); + $sess1->init(); + $sess2 = new \core\session\redis(); + $sess2->init(); + + // Initialize session 1. + $this->assertTrue($sess1->handler_open('Not used', 'Not used')); + $this->assertSame('', $sess1->handler_read('sess1')); + $this->assertTrue($sess1->handler_write('sess1', 'DATA')); + $this->assertTrue($sess1->handler_close()); + + // Initialize session 2. + $this->assertTrue($sess2->handler_open('Not used', 'Not used')); + $this->assertSame('', $sess2->handler_read('sess2')); + $this->assertTrue($sess2->handler_write('sess2', 'DATA2')); + $this->assertTrue($sess2->handler_close()); + + // Open and read session 1 and 2. + $this->assertTrue($sess1->handler_open('Not used', 'Not used')); + $this->assertSame('DATA', $sess1->handler_read('sess1')); + $this->assertTrue($sess2->handler_open('Not used', 'Not used')); + $this->assertSame('DATA2', $sess2->handler_read('sess2')); + + // Write both sessions. + $this->assertTrue($sess1->handler_write('sess1', 'DATAX')); + $this->assertTrue($sess2->handler_write('sess2', 'DATA2X')); + + // Read both sessions. + $this->assertTrue($sess1->handler_open('Not used', 'Not used')); + $this->assertTrue($sess2->handler_open('Not used', 'Not used')); + $this->assertEquals('DATAX', $sess1->handler_read('sess1')); + $this->assertEquals('DATA2X', $sess2->handler_read('sess2')); + + // Close both sessions + $this->assertTrue($sess1->handler_close()); + $this->assertTrue($sess2->handler_close()); + + // Read the session again to ensure locking did what it should. + $this->assertSessionNoLocks(); + } + + public function test_multiple_sessions_work_with_a_single_instance() { + $sess = new \core\session\redis(); + $sess->init(); + + // Initialize session 1. + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertSame('', $sess->handler_read('sess1')); + $this->assertTrue($sess->handler_write('sess1', 'DATA')); + $this->assertSame('', $sess->handler_read('sess2')); + $this->assertTrue($sess->handler_write('sess2', 'DATA2')); + $this->assertSame('DATA', $sess->handler_read('sess1')); + $this->assertSame('DATA2', $sess->handler_read('sess2')); + $this->assertTrue($sess->handler_destroy('sess2')); + + $this->assertTrue($sess->handler_close()); + $this->assertSessionNoLocks(); + + $this->assertTrue($sess->handler_close()); + } + + public function test_session_exists_returns_valid_values() { + $sess = new \core\session\redis(); + $sess->init(); + + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertSame('', $sess->handler_read('sess1')); + + $this->assertFalse($sess->session_exists('sess1'), 'Session must not exist yet, it has not been saved'); + $this->assertTrue($sess->handler_write('sess1', 'DATA')); + $this->assertTrue($sess->session_exists('sess1'), 'Session must exist now.'); + $this->assertTrue($sess->handler_destroy('sess1')); + $this->assertFalse($sess->session_exists('sess1'), 'Session should be destroyed.'); + } + + public function test_kill_sessions_removes_the_session_from_redis() { + global $DB; + + $sess = new \core\session\redis(); + $sess->init(); + + $this->assertTrue($sess->handler_open('Not used', 'Not used')); + $this->assertTrue($sess->handler_write('sess1', 'DATA')); + $this->assertTrue($sess->handler_write('sess2', 'DATA')); + $this->assertTrue($sess->handler_write('sess3', 'DATA')); + + $sessiondata = new \stdClass(); + $sessiondata->userid = 2; + $sessiondata->timecreated = time(); + $sessiondata->timemodified = time(); + + $sessiondata->sid = 'sess1'; + $DB->insert_record('sessions', $sessiondata); + $sessiondata->sid = 'sess2'; + $DB->insert_record('sessions', $sessiondata); + $sessiondata->sid = 'sess3'; + $DB->insert_record('sessions', $sessiondata); + + $this->assertNotEquals('', $sess->handler_read('sess1')); + $sess->kill_session('sess1'); + $this->assertEquals('', $sess->handler_read('sess1')); + + $this->assertEmpty($this->redis->keys($this->keyprefix.'sess1.lock')); + + $sess->kill_all_sessions(); + + $this->assertEquals(3, $DB->count_records('sessions'), 'Moodle handles session database, plugin must not change it.'); + $this->assertSessionNoLocks(); + $this->assertEmpty($this->redis->keys($this->keyprefix.'*'), 'There should be no session data left.'); + } + + /** + * Assert that we don't have any session locks in Redis. + */ + protected function assertSessionNoLocks() { + $this->assertEmpty($this->redis->keys($this->keyprefix.'*.lock')); + } +}