L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'MDL-73713-readonly-txn-latency' of https://github.com/catalyst/moodle

Browse Source
This commit is contained in:
Jun Pataleta
2022-03-24 10:17:30 +08:00
parent 7a65b2a3c7 d8baf29573
commit efe227d1bf
13 changed files with 227 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config-dist.php
+3 -2
View File
@@ -105,8 +105,9 @@ $CFG->dboptions = array(
'latency' => 0.5, // Set read-only slave sync latency in seconds.
// When 'latency' seconds have lapsed after an update to a table
// it is deemed safe to use readonly slave for reading from the table.
// It is optional. If omitted once written to a table it will always
// use master handle for reading.
// It is optional, defaults to 1 second. If you want once written to a table
// to always use master handle for reading set it to something ridiculosly big,
// eg 10.
// Lower values increase the performance, but setting it too low means
// missing the master-slave sync.
'exclude_tables' => [ // Tables to exclude from read-only slave feature.
lib/dml/moodle_read_slave_trait.php
+27 -33
View File
@@ -43,8 +43,8 @@ defined('MOODLE_INTERNAL') || die();
* - It supports multiple 'instance' entries, in case one is not accessible,
* but only one (first connectable) instance is used.
* - 'latency' option: master -> slave sync latency in seconds (will probably
* be a fraction of a second). If specified, a table being written to is
* deemed fully synced and suitable for slave read.
* be a fraction of a second). A table being written to is deemed fully synced
* after that period and suitable for slave read. Defaults to 1 sec.
* - 'exclude_tables' option: a list of tables that never go to the slave for
* querying. The feature is meant to be used in emergency only, so the
* readonly feature can still be used in case there is a rogue query that
@@ -54,8 +54,8 @@ defined('MOODLE_INTERNAL') || die();
*
* Choice of the database handle is based on following:
* - SQL_QUERY_INSERT, UPDATE and STRUCTURE record table from the query
* in the $written array and microtime() the event if the 'latency' option
* is set. For those queries master write handle is used.
* in the $written array and microtime() the event. For those queries master
* write handle is used.
* - SQL_QUERY_AUX queries will always use the master write handle because they
* are used for transactionstart/end, locking etc. In that respect, query_start() and
* query_end() *must not* be used during the connection phase.
@@ -63,11 +63,9 @@ defined('MOODLE_INTERNAL') || die();
* -- any of the tables involved is a temp table
* -- any of the tables involved is listed in the 'exclude_tables' option
* -- any of the tables involved is in the $written array:
* * If the 'latency' option is set then the microtime() is compared to
* the write microrime, and if more then latency time has passed the slave
* handle is used.
* * Otherwise (not enough time passed or 'latency' option not set)
* we choose the master write handle
* * current microtime() is compared to the write microrime, and if more than
* latency time has passed the slave handle is used
* * otherwise (not enough time passed) we choose the master write handle
* If none of the above conditions are met the slave instance is used.
*
* A 'latency' example:
@@ -92,7 +90,7 @@ trait moodle_read_slave_trait {
private $wantreadslave = false;
private $readsslave = 0;
private $slavelatency = 0;
private $slavelatency = 1;
private $written = []; // Track tables being written to.
private $readexclude = []; // Tables to exclude from using dbhreadonly.
@@ -324,23 +322,27 @@ trait moodle_read_slave_trait {
}
if (isset($this->written[$tablename])) {
if ($this->slavelatency) {
$now = $now ?: microtime(true);
if ($now - $this->written[$tablename] < $this->slavelatency) {
return false;
}
unset($this->written[$tablename]);
} else {
$now = $now ?: microtime(true);
// Paranoid check.
if ($this->written[$tablename] === true) {
debugging(
"$tablename last written set to true outside transaction - should not happen!",
DEBUG_DEVELOPER
);
$this->written[$tablename] = $now;
return false;
}
if ($now - $this->written[$tablename] < $this->slavelatency) {
return false;
}
unset($this->written[$tablename]);
}
}
return true;
case SQL_QUERY_INSERT:
case SQL_QUERY_UPDATE:
// If we are in transaction we cannot set the written time yet.
$now = $this->slavelatency && !$this->transactions ? microtime(true) : true;
$now = $this->transactions ? true : microtime(true);
foreach ($this->table_names($sql) as $tablename) {
$this->written[$tablename] = $now;
}
@@ -364,23 +366,15 @@ trait moodle_read_slave_trait {
* @throws dml_transaction_exception Creates and throws transaction related exceptions.
*/
public function commit_delegated_transaction(moodle_transaction $transaction) {
parent::commit_delegated_transaction($transaction);
if ($this->transactions) {
return;
}
if (!$this->slavelatency) {
return;
}
$now = null;
foreach ($this->written as $tablename => $when) {
if ($when === true) {
$now = $now ?: microtime(true);
if ($this->written) {
// Adjust the written time.
$now = microtime(true);
foreach ($this->written as $tablename => $when) {
$this->written[$tablename] = $now;
}
}
parent::commit_delegated_transaction($transaction);
}
/**
lib/dml/tests/dml_mysqli_read_slave_test.php
+5 -2
View File
@@ -21,8 +21,11 @@
* @category dml
* @copyright 2018 Srdjan Janković, Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
* @coversDefaultClass \mysqli_native_moodle_database
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/fixtures/read_slave_moodle_database_mock_mysqli.php');
@@ -35,7 +38,7 @@ require_once(__DIR__.'/fixtures/read_slave_moodle_database_mock_mysqli.php');
* @copyright 2018 Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class core_dml_mysqli_read_slave_testcase extends base_testcase {
class dml_mysqli_read_slave_test extends \base_testcase {
/**
* Test readonly handle is not used for reading from special pg_*() call queries,
* pg_try_advisory_lock and pg_advisory_unlock.
@@ -80,7 +83,7 @@ class core_dml_mysqli_read_slave_testcase extends base_testcase {
'connecttimeout' => 1
];
$db2 = moodle_database::get_driver_instance($cfg->dbtype, $cfg->dblibrary);
$db2 = \moodle_database::get_driver_instance($cfg->dbtype, $cfg->dblibrary);
$db2->connect($cfg->dbhost, $cfg->dbuser, $cfg->dbpass, $cfg->dbname, $cfg->prefix, $cfg->dboptions);
$this->assertTrue(count($db2->get_records('user')) > 0);
}
lib/dml/tests/dml_pgsql_read_slave_test.php
+8 -5
View File
@@ -21,8 +21,11 @@
* @category dml
* @copyright 2018 Srdjan Janković, Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
* @coversDefaultClass \pgsql_native_moodle_database
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/fixtures/read_slave_moodle_database_mock_pgsql.php');
@@ -35,7 +38,7 @@ require_once(__DIR__.'/fixtures/read_slave_moodle_database_mock_pgsql.php');
* @copyright 2018 Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class core_dml_pgsql_read_slave_testcase extends base_testcase {
class dml_pgsql_read_slave_test extends \base_testcase {
/**
* Test correct database handles are used for cursors
*
@@ -135,12 +138,12 @@ class core_dml_pgsql_read_slave_testcase extends base_testcase {
}
// Get a separate disposable db connection handle with guaranteed 'readonly' config.
$db2 = moodle_database::get_driver_instance($cfg->dbtype, $cfg->dblibrary);
$db2 = \moodle_database::get_driver_instance($cfg->dbtype, $cfg->dblibrary);
$db2->connect($cfg->dbhost, $cfg->dbuser, $cfg->dbpass, $cfg->dbname, $cfg->prefix, $cfg->dboptions);
$dbman = $db2->get_manager();
$table = new xmldb_table('silly_test_table');
$table = new \xmldb_table('silly_test_table');
$table->add_field('id', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, XMLDB_SEQUENCE);
$table->add_field('msg', XMLDB_TYPE_CHAR, 255);
$table->add_key('primary', XMLDB_KEY_PRIMARY, ['id']);
@@ -155,7 +158,7 @@ class core_dml_pgsql_read_slave_testcase extends base_testcase {
$db2->get_records('silly_test_table');
$this->assertEquals($reads, $db2->perf_get_reads_slave());
$table2 = new xmldb_table('silly_test_table2');
$table2 = new \xmldb_table('silly_test_table2');
$table2->add_field('id', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, XMLDB_SEQUENCE);
$table2->add_field('msg', XMLDB_TYPE_CHAR, 255);
$table2->add_key('primary', XMLDB_KEY_PRIMARY, ['id']);
@@ -193,7 +196,7 @@ class core_dml_pgsql_read_slave_testcase extends base_testcase {
'connecttimeout' => 1
];
$db2 = moodle_database::get_driver_instance($cfg->dbtype, $cfg->dblibrary);
$db2 = \moodle_database::get_driver_instance($cfg->dbtype, $cfg->dblibrary);
$db2->connect($cfg->dbhost, $cfg->dbuser, $cfg->dbpass, $cfg->dbname, $cfg->prefix, $cfg->dboptions);
$this->assertTrue(count($db2->get_records('user')) > 0);
}
lib/dml/tests/dml_read_slave_test.php
+105 -6
View File
@@ -21,12 +21,16 @@
* @category dml
* @copyright 2018 Srdjan Janković, Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
* @coversDefaultClass \moodle_temptables
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/fixtures/read_slave_moodle_database_table_names.php');
require_once(__DIR__.'/fixtures/read_slave_moodle_database_special.php');
require_once(__DIR__.'/../../tests/fixtures/event_fixtures.php');
/**
* DML read/read-write database handle use tests
@@ -36,10 +40,12 @@ require_once(__DIR__.'/fixtures/read_slave_moodle_database_special.php');
* @copyright 2018 Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class core_dml_read_slave_testcase extends base_testcase {
class dml_read_slave_test extends \base_testcase {
/** @var float */
static private $dbreadonlylatency = 0.8;
/** @var float */
static private $defaultlatency = 1;
/**
* Instantiates a test database interface object.
@@ -223,18 +229,27 @@ class core_dml_read_slave_testcase extends base_testcase {
$this->assertEquals('test_rw::test:test', $handle);
$this->assertEquals(0, $DB->perf_get_reads_slave());
sleep(1);
$handle = $DB->get_records('table');
$this->assertEquals('test_rw::test:test', $handle);
$this->assertEquals(0, $DB->perf_get_reads_slave());
$handle = $DB->get_records('table2');
$handle = $DB->get_records_sql("SELECT * FROM {table2} JOIN {table}");
$this->assertEquals('test_rw::test:test', $handle);
$this->assertEquals(0, $DB->perf_get_reads_slave());
sleep(1);
$handle = $DB->get_records('table');
$this->assert_readonly_handle($handle);
$this->assertEquals(1, $DB->perf_get_reads_slave());
$handle = $DB->get_records('table2');
$this->assert_readonly_handle($handle);
$this->assertEquals(2, $DB->perf_get_reads_slave());
$handle = $DB->get_records_sql("SELECT * FROM {table2} JOIN {table}");
$this->assertEquals('test_rw::test:test', $handle);
$this->assertEquals(1, $DB->perf_get_reads_slave());
$this->assert_readonly_handle($handle);
$this->assertEquals(3, $DB->perf_get_reads_slave());
}
/**
@@ -278,12 +293,15 @@ class core_dml_read_slave_testcase extends base_testcase {
* so the latency parameter is applied properly.
*
* @return void
* @covers ::can_use_readonly
* @covers ::commit_delegated_transaction
*/
public function test_transaction() : void {
public function test_transaction(): void {
$DB = $this->new_db(true);
$this->assertNull($DB->get_dbhwrite());
$skip = false;
$transaction = $DB->start_delegated_transaction();
$now = microtime(true);
$handle = $DB->get_records_sql("SELECT * FROM {table}");
@@ -304,11 +322,92 @@ class core_dml_read_slave_testcase extends base_testcase {
// Make sure enough time passes.
sleep(1);
} else {
$skip = true;
}
// Exceeded latency time, use ro handle.
$handle = $DB->get_records_sql("SELECT * FROM {table}");
$this->assert_readonly_handle($handle);
if ($skip) {
$this->markTestSkipped("Delay too long to test write handle immediately after transaction");
}
}
/**
* Test readonly handle is not used with events
* when the latency parameter is applied properly.
*
* @return void
* @covers ::can_use_readonly
* @covers ::commit_delegated_transaction
*/
public function test_transaction_with_events(): void {
$this->with_global_db(function () {
global $DB;
$DB = $this->new_db(true, ['test_ro'], read_slave_moodle_database_special::class);
$DB->set_tables([
'config_plugins' => [
'columns' => [
'plugin' => (object)['meta_type' => ''],
]
]
]);
$this->assertNull($DB->get_dbhwrite());
$this->_called = false;
$transaction = $DB->start_delegated_transaction();
$now = microtime(true);
$observers = [
[
'eventname' => '\core_tests\event\unittest_executed',
'callback' => function (\core_tests\event\unittest_executed $event) use ($DB, $now) {
$this->_called = true;
$this->assertFalse($DB->is_transaction_started());
// This condition should always evaluate true, however we need to
// safeguard from an unaccounted delay that can break this test.
if (microtime(true) - $now < 1 + self::$dbreadonlylatency) {
// Not enough time passed, use rw handle.
$handle = $DB->get_records_sql_p("SELECT * FROM {table}");
$this->assertEquals('test_rw::test:test', $handle);
// Make sure enough time passes.
sleep(1);
} else {
$this->markTestSkipped("Delay too long to test write handle immediately after transaction");
}
// Exceeded latency time, use ro handle.
$handle = $DB->get_records_sql_p("SELECT * FROM {table}");
$this->assertEquals('test_ro::test:test', $handle);
},
'internal' => 0,
],
];
\core\event\manager::phpunit_replace_observers($observers);
$handle = $DB->get_records_sql_p("SELECT * FROM {table}");
// Use rw handle during transaction.
$this->assertEquals('test_rw::test:test', $handle);
$handle = $DB->insert_record_raw('table', array('name' => 'blah'));
// Introduce delay so we can check that table write timestamps
// are adjusted properly.
sleep(1);
$event = \core_tests\event\unittest_executed::create([
'context' => \context_system::instance(),
'other' => ['sample' => 1]
]);
$event->trigger();
$transaction->allow_commit();
$this->assertTrue($this->_called);
});
}
/**
lib/dml/tests/fixtures/read_slave_moodle_database.php
Vendored
+4 -2
View File
@@ -23,6 +23,8 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/test_moodle_database.php');
@@ -37,7 +39,7 @@ require_once(__DIR__.'/../../moodle_read_slave_trait.php');
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class read_slave_moodle_database extends test_moodle_database {
use moodle_read_slave_trait;
use \moodle_read_slave_trait;
/** @var string */
protected $handle;
@@ -58,7 +60,7 @@ class read_slave_moodle_database extends test_moodle_database {
$this->prefix = $prefix;
if ($dbhost == 'test_ro_fail') {
throw new dml_connection_exception($dbhost);
throw new \dml_connection_exception($dbhost);
}
return true;
lib/dml/tests/fixtures/read_slave_moodle_database_mock_mysqli.php
Vendored
+3 -1
View File
@@ -23,6 +23,8 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/../../mysqli_native_moodle_database.php');
@@ -36,7 +38,7 @@ require_once(__DIR__.'/test_moodle_read_slave_trait.php');
* @copyright 2018 Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class read_slave_moodle_database_mock_mysqli extends mysqli_native_moodle_database {
class read_slave_moodle_database_mock_mysqli extends \mysqli_native_moodle_database {
use test_moodle_read_slave_trait;
/**
lib/dml/tests/fixtures/read_slave_moodle_database_mock_pgsql.php
Vendored
+3 -1
View File
@@ -23,6 +23,8 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/../../pgsql_native_moodle_database.php');
@@ -36,6 +38,6 @@ require_once(__DIR__.'/test_moodle_read_slave_trait.php');
* @copyright 2018 Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class read_slave_moodle_database_mock_pgsql extends pgsql_native_moodle_database {
class read_slave_moodle_database_mock_pgsql extends \pgsql_native_moodle_database {
use test_moodle_read_slave_trait;
}
lib/dml/tests/fixtures/read_slave_moodle_database_special.php
Vendored
+58 -1
View File
@@ -23,10 +23,11 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/read_slave_moodle_database.php');
require_once(__DIR__.'/read_slave_moodle_recordset_special.php');
/**
* Database driver mock test class that uses read_slave_moodle_recordset_special
@@ -50,6 +51,19 @@ class read_slave_moodle_database_special extends read_slave_moodle_database {
return [];
}
/**
* Returns read_slave_moodle_database::get_records_sql()
* For the tests where we need both fake result and dbhandle info.
* @param string $sql the SQL select query to execute.
* @param array $params array of sql parameters
* @param int $limitfrom return a subset of records, starting at this point (optional).
* @param int $limitnum return a subset comprising this many records (optional, required if $limitfrom is set).
* @return string $handle handle property
*/
public function get_records_sql_p($sql, array $params = null, $limitfrom = 0, $limitnum = 0) {
return parent::get_records_sql($sql, $params);
}
/**
* Returns fake recordset
* @param string $sql
@@ -74,3 +88,46 @@ class read_slave_moodle_database_special extends read_slave_moodle_database {
return 1;
}
}
/**
* Database recordset mock test class
*
* @package core
* @category dml
* @copyright 2018 Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class read_slave_moodle_recordset_special extends \moodle_recordset {
/**
* Iterator interface
* @return void
*/
public function close() {
}
/**
* Iterator interface
* @return stdClass
*/
public function current() {
return new stdClass();
}
/**
* Iterator interface
* @return void
*/
public function next() {
}
/**
* Iterator interface
* @return mixed
*/
public function key() {
}
/**
* Iterator interface
* @return bool
*/
public function valid() {
return false;
}
}
lib/dml/tests/fixtures/read_slave_moodle_database_table_names.php
Vendored
+2
View File
@@ -23,6 +23,8 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/read_slave_moodle_database.php');
lib/dml/tests/fixtures/test_moodle_database.php
Vendored
+4 -2
View File
@@ -23,6 +23,8 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/../../moodle_database.php');
@@ -38,7 +40,7 @@ require_once(__DIR__.'/test_sql_generator.php');
* @copyright 2018 Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
abstract class test_moodle_database extends moodle_database {
abstract class test_moodle_database extends \moodle_database {
/** @var string */
private $error;
@@ -53,7 +55,7 @@ abstract class test_moodle_database extends moodle_database {
public function __construct($external = false) {
parent::__construct($external);
$this->temptables = new moodle_temptables($this);
$this->temptables = new \moodle_temptables($this);
}
/**
lib/dml/tests/fixtures/test_moodle_read_slave_trait.php
Vendored
+2 -4
View File
@@ -23,9 +23,7 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/../../pgsql_native_moodle_database.php');
namespace core;
/**
* Read slave helper that exposes selected moodle_read_slave_trait metods
@@ -55,7 +53,7 @@ trait test_moodle_read_slave_trait {
$this->dbhreadonly = $ro;
$this->set_db_handle($this->dbhwrite);
$this->temptables = new moodle_temptables($this);
$this->temptables = new \moodle_temptables($this);
}
/**
lib/dml/tests/fixtures/test_sql_generator.php
Vendored
+3 -1
View File
@@ -23,6 +23,8 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core;
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__.'/../../../ddl/sql_generator.php');
@@ -36,7 +38,7 @@ require_once(__DIR__.'/../../../ddl/sql_generator.php');
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*
*/
class test_sql_generator extends sql_generator {
class test_sql_generator extends \sql_generator {
// phpcs:disable moodle.NamingConventions.ValidFunctionName.LowercaseMethod
/**