From b04ca8c8a1c82634a6a85b22df76497c22bb2e83 Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Thu, 22 Sep 2016 12:23:40 +0800 Subject: [PATCH] MDL-55986 behat: Added option to optimize parallel runs by tags run.php should only be used for running behat before this patch if tags were passed, then behat.yml was being re-written with features havings tags. As this is not ideal, this option is now removed and features with tags will be optimally split by init.php when --optimize-run option is passed --- admin/tool/behat/cli/init.php | 28 ++-- admin/tool/behat/cli/run.php | 31 ----- admin/tool/behat/cli/util.php | 10 +- admin/tool/behat/cli/util_single_run.php | 4 +- admin/tool/behat/tests/manager_util_test.php | 139 ++++--------------- lib/behat/classes/behat_config_util.php | 116 +++++++++++----- lib/behat/classes/util.php | 5 +- 7 files changed, 135 insertions(+), 198 deletions(-) diff --git a/admin/tool/behat/cli/init.php b/admin/tool/behat/cli/init.php index f58a1bc5dc5..37e1dc24240 100644 --- a/admin/tool/behat/cli/init.php +++ b/admin/tool/behat/cli/init.php @@ -48,6 +48,7 @@ list($options, $unrecognized) = cli_get_params( 'fromrun' => 1, 'torun' => 0, 'run-with-theme' => false, + 'optimize-runs' => '', ), array( 'j' => 'parallel', @@ -68,6 +69,7 @@ Options: -m, --maxruns Max parallel processes to be executed at one time. --fromrun Execute run starting from (Used for parallel runs on different vms) --torun Execute run till (Used for parallel runs on different vms) +--optimize-runs Split features with specified tags in all parallel runs. --run-with-theme Run all core features with specified theme. -h, --help Print out this help @@ -85,21 +87,17 @@ if (!empty($options['help'])) { // Check which util file to call. $utilfile = 'util_single_run.php'; -$paralleloption = ""; +$commandoptions = ""; // If parallel run then use utilparallel. if ($options['parallel'] && $options['parallel'] > 1) { $utilfile = 'util.php'; - $paralleloption = ""; - foreach ($options as $option => $value) { - if ($value) { - $paralleloption .= " --$option=\"$value\""; - } - } } -$themesuitewithallfeatures = ''; -if ($options['run-with-theme']) { - $themesuitewithallfeatures = '--run-with-theme="true"'; +// Sanitize input options, so they can be passed to util. +foreach ($options as $option => $value) { + if ($value) { + $commandoptions .= " --$option='$value'"; + } } // Changing the cwd to admin/tool/behat/cli. @@ -111,7 +109,7 @@ testing_update_composer_dependencies(); // Check whether the behat test environment needs to be updated. chdir(__DIR__); -exec("php $utilfile --diag $paralleloption $themesuitewithallfeatures", $output, $code); +exec("php $utilfile --diag $commandoptions", $output, $code); if ($code == 0) { echo "Behat test environment already installed\n"; @@ -119,7 +117,7 @@ if ($code == 0) { } else if ($code == BEHAT_EXITCODE_INSTALL) { // Behat and dependencies are installed and we need to install the test site. chdir(__DIR__); - passthru("php $utilfile --install $paralleloption $themesuitewithallfeatures", $code); + passthru("php $utilfile --install $commandoptions", $code); if ($code != 0) { chdir($cwd); exit($code); @@ -128,14 +126,14 @@ if ($code == 0) { } else if ($code == BEHAT_EXITCODE_REINSTALL) { // Test site data is outdated. chdir(__DIR__); - passthru("php $utilfile --drop $paralleloption $themesuitewithallfeatures", $code); + passthru("php $utilfile --drop $commandoptions", $code); if ($code != 0) { chdir($cwd); exit($code); } chdir(__DIR__); - passthru("php $utilfile --install $paralleloption $themesuitewithallfeatures", $code); + passthru("php $utilfile --install $commandoptions", $code); if ($code != 0) { chdir($cwd); exit($code); @@ -150,7 +148,7 @@ if ($code == 0) { // Enable editing mode according to config.php vars. chdir(__DIR__); -passthru("php $utilfile --enable $paralleloption $themesuitewithallfeatures", $code); +passthru("php $utilfile --enable $commandoptions", $code); if ($code != 0) { echo "Error enabling site" . PHP_EOL; chdir($cwd); diff --git a/admin/tool/behat/cli/run.php b/admin/tool/behat/cli/run.php index 853a17ff35c..2775a5f1237 100644 --- a/admin/tool/behat/cli/run.php +++ b/admin/tool/behat/cli/run.php @@ -55,7 +55,6 @@ list($options, $unrecognised) = cli_get_params( 'fromrun' => 1, 'torun' => 0, 'single-run' => false, - 'run-with-theme' => false, ), array( 'h' => 'help', @@ -79,7 +78,6 @@ Options: --replace Replace args string with run process number, useful for output. --fromrun Execute run starting from (Used for parallel runs on different vms) --torun Execute run till (Used for parallel runs on different vms) ---run-with-theme Run all core features with specified theme. -h, --help Print out this help @@ -120,8 +118,6 @@ array_walk($unrecognised, function (&$v) { }); $extraopts = $unrecognised; -$tags = ''; - if ($options['profile']) { $profile = $options['profile']; @@ -174,33 +170,6 @@ if (empty($parallelrun)) { exit($code); } -// Update config file if tags defined. -if ($tags) { - define('ABORT_AFTER_CONFIG_CANCEL', true); - require("$CFG->dirroot/lib/setup.php"); - // Hack to set proper dataroot and wwwroot. - $behatdataroot = $CFG->behat_dataroot; - $behatwwwroot = $CFG->behat_wwwroot; - for ($i = 1; $i <= $parallelrun; $i++) { - $CFG->behatrunprocess = $i; - - if (!empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) { - $CFG->behat_wwwroot = $CFG->behat_parallel_run[$i - 1]['behat_wwwroot']; - } else { - $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i; - } - if (!empty($CFG->behat_parallel_run[$i - 1]['behat_dataroot'])) { - $CFG->behat_dataroot = $CFG->behat_parallel_run[$i - 1]['behat_dataroot']; - } else { - $CFG->behat_dataroot = $behatdataroot . $i; - } - behat_config_manager::update_config_file('', true, $tags, $options['run-with-theme'], $parallelrun); - } - $CFG->behat_dataroot = $behatdataroot; - $CFG->behat_wwwroot = $behatwwwroot; - unset($CFG->behatrunprocess); -} - $cmds = array(); echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL; diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index a30e30a85b0..e82e7fb3e22 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -57,6 +57,7 @@ list($options, $unrecognized) = cli_get_params( 'fromrun' => 1, 'torun' => 0, 'run-with-theme' => false, + 'optimize-runs' => '', ), array( 'h' => 'help', @@ -80,8 +81,9 @@ Options: --diag Get behat test environment status code --updatesteps Update feature step file. --j, --parallel Number of parallel behat run operation --m, --maxruns Max parallel processes to be executed at one time. +-j, --parallel Number of parallel behat run operation +-m, --maxruns Max parallel processes to be executed at one time. +--optimize-runs Split features with specified tags in all parallel runs. --run-with-theme Run all core features with specified theme. -h, --help Print out this help @@ -185,7 +187,7 @@ if ($options['diag'] || $options['enable'] || $options['disable']) { $CFG->behatrunprocess = $i; // Update config file for each run. - behat_config_manager::update_config_file('', true, '', $options['run-with-theme'], + behat_config_manager::update_config_file('', true, $options['optimize-runs'], $options['run-with-theme'], $options['parallel'], $i); } unset($CFG->behatrunprocess); @@ -267,7 +269,7 @@ function commands_to_execute($options) { if ($options[$option]) { $extra .= " --$option"; if ($value) { - $extra .= "=$value"; + $extra .= "='$value'"; } } } diff --git a/admin/tool/behat/cli/util_single_run.php b/admin/tool/behat/cli/util_single_run.php index 70aca4407af..85987a0f1fa 100644 --- a/admin/tool/behat/cli/util_single_run.php +++ b/admin/tool/behat/cli/util_single_run.php @@ -48,6 +48,7 @@ list($options, $unrecognized) = cli_get_params( 'tags' => '', 'updatesteps' => false, 'run-with-theme' => false, + 'optimize-runs' => '', ), array( 'h' => 'help', @@ -72,6 +73,7 @@ Options: --disable Disables test environment --diag Get behat test environment status code --updatesteps Update feature step file. +--optimize-runs Split features with specified tags in all parallel runs. --run-with-theme Run all core features with specified theme. -h, --help Print out this help @@ -178,7 +180,7 @@ if ($options['install']) { } // Enable test mode. - behat_util::start_test_mode($options['run-with-theme'], $parallel, $run); + behat_util::start_test_mode($options['run-with-theme'], $options['optimize-runs'], $parallel, $run); // This is only displayed once for parallel install. if (empty($run)) { diff --git a/admin/tool/behat/tests/manager_util_test.php b/admin/tool/behat/tests/manager_util_test.php index 2f2a931ce1b..28182c3140d 100644 --- a/admin/tool/behat/tests/manager_util_test.php +++ b/admin/tool/behat/tests/manager_util_test.php @@ -41,77 +41,27 @@ require_once($CFG->libdir . '/behat/classes/behat_config_manager.php'); */ class tool_behat_manager_util_testcase extends advanced_testcase { - /** - * @var array core features. - */ - private $corefeatures = array( - 'feedback_editpdf_behat_test1' => '/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature', - 'feedback_file_behat_test2' => "C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature", - 'moodle_login_behat_test3' => "C:\\test\\moodle/login/tests/behat/behat_test3.feature", - ); - - - /** - * @var array theme features. - */ - private $themefeatures = array( - 'behat_themetest1_core_behat_tests_testtheme_theme' => '/test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature', - 'behat_themetest2_mod_assign_behat_tests_testtheme_theme' => "C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature", - 'behat_themetest3_behat_tests_testtheme_theme_moodle' => "C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature", - ); - - /** - * @var array core contexts. - */ - private $corecontexts = array( - 'behat_context1' => '/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_context1.php', - 'behat_context2' => "C:\\test\\moodle\\blocks\\comments\\tests\\behat\\behat_context2.php", - 'behat_context3' => "C:\\test\\moodle/lib/editor/atto/tests/behat/behat_context3.php", - ); - - /** - * @var array Theme contexts for test. - */ - private $themecontexts = array( - 'behat_theme_testtheme_behat_context1' => - '/test/moodle/theme/testtheme/tests/behat/mod_assign/behat_theme_testtheme_behat_context1.php', - 'behat_theme_testtheme_behat_context2' => - "C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\block_comments\\behat_theme_testtheme_behat_context2.php", - 'behat_theme_testtheme_behat_context3' => - "C:\\test\\moodle/theme/testtheme/tests/behat/editor_atto/behat_theme_testtheme_behat_context3.php" - ); - - /** - * Keep instance of behat_config_util mock object. - * - * @var null - */ - private $behatconfigutil = null; - - /** - * Test setup. - */ - public function setUp() { - $this->resetAfterTest(true); - $mockbuilder = $this->getMockBuilder('behat_config_util'); - $mockbuilder->setMethods(array('get_behat_features_for_theme', 'get_behat_contexts_for_theme', - 'get_list_of_themes', 'get_overridden_theme_contexts')); - - $this->behatconfigutil = $mockbuilder->getMock(); - + private function get_config($behatconfigutil) { // List of themes is const for test. - $this->behatconfigutil->expects($this->any()) + $behatconfigutil->expects($this->any()) ->method('get_list_of_themes') ->will($this->returnValue(array('testtheme'))); - $this->behatconfigutil->expects($this->any()) - ->method('get_behat_contexts_for_theme') + // Theme directory for testing. + $behatconfigutil->expects($this->any()) + ->method('get_test_directories_overridden_for_theme') ->with($this->equalTo('testtheme')) ->will($this->returnValue(array( - 'contexts' => $this->themecontexts, - 'suitecontexts' => $this->themecontexts + __DIR__ . '/fixtures/testtheme/tests/behat' ))); + // Core components list for testing. + $behatconfigutil->expects($this->any()) + ->method('get_components_with_tests') + ->will($this->returnValue(array('testtheme' => __DIR__.'/fixtures/core'))); + + return $behatconfigutil->get_config_file_contents(); + } /** @@ -121,59 +71,28 @@ class tool_behat_manager_util_testcase extends advanced_testcase { public function test_get_config_file_contents_with_single_run() { global $CFG; + $this->resetAfterTest(); $CFG->behat_wwwroot = 'http://example.com/behat'; - $behatconfigutil = $this->behatconfigutil; + $mockbuilder = $this->getMockBuilder('behat_config_util'); + $mockbuilder->setMethods(array('get_test_directories_overridden_for_theme', 'get_list_of_themes', + 'get_components_with_tests')); - // No theme feature exists. - $behatconfigutil->expects($this->once()) - ->method('get_behat_features_for_theme') - ->with($this->anything()) - ->will($this->returnValue(array( - 'blacklistfeatures' => array(), - 'features' => array())) - ); + $behatconfigutil = $mockbuilder->getMock(); - $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts); + $config = $this->get_config($behatconfigutil); - $expectedconfigwithfeatures = "default: - formatters: - moodle_progress: - output_styles: - comment: - - magenta - suites: - default: - paths: - - /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature - - 'C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature' - - 'C:\\test\\moodle/login/tests/behat/behat_test3.feature' - contexts: - - behat_context1 - - behat_context2 - - behat_context3 - testtheme: - paths: { } - contexts: - - behat_theme_testtheme_behat_context1 - - behat_theme_testtheme_behat_context2 - - behat_theme_testtheme_behat_context3 - extensions: - Behat\\MinkExtension: - base_url: 'http://example.com/behat' - goutte: null - selenium2: - wd_host: 'http://localhost:4444/wd/hub' -"; +var_dump($config); + // Contains core features and contexts. + $suites = $config['default']['suites']; + $this->assertContains('test_1.feature', $suites['default']['paths'][0]); + $this->assertContains('test_2.feature', $suites['default']['paths'][1]); + $this->assertContains('behat_test_context_1', $suites['default']['contexts'][0]); + $this->assertContains('behat_test_context_2', $suites['default']['contexts'][1]); - $this->assertContains($expectedconfigwithfeatures, $config); - - $expectedstepdefinitions = "steps_definitions: - behat_context1: /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_context1.php - behat_context2: 'C:\\test\\moodle\\blocks\\comments\\tests\\behat\\behat_context2.php' - behat_context3: 'C:\\test\\moodle/lib/editor/atto/tests/behat/behat_context3.php' -"; - $this->assertContains($expectedstepdefinitions, $config); + // Contains theme specific features and contexts. + $this->assertContains('theme_test_1.feature', $suites['testtheme']['paths'][0]); + $this->assertContains('theme_test_2.feature', $suites['testtheme']['paths'][1]); } /** diff --git a/lib/behat/classes/behat_config_util.php b/lib/behat/classes/behat_config_util.php index e9bb702ab2d..acf1f57f3b8 100644 --- a/lib/behat/classes/behat_config_util.php +++ b/lib/behat/classes/behat_config_util.php @@ -210,7 +210,15 @@ class behat_config_util { $this->features = $features; - return $this->filtered_features_with_tags($features, $tags); + // If tags are passed then filter features which has sepecified tags. + if (!empty($tags)) { + $features = $this->filtered_features_with_tags($features, $tags); + } + + // Return sorted list. + ksort($features); + + return $features; } /** @@ -301,6 +309,9 @@ class behat_config_util { } } + // Sort contexts with there name. + ksort($this->contexts); + return $this->get_component_contexts($component); } @@ -351,6 +362,11 @@ class behat_config_util { $config = $this->merge_behat_profiles($config); + // Return config array for phpunit, so it can be tested. + if (defined('PHPUNIT_TEST')) { + return $config; + } + return Symfony\Component\Yaml\Yaml::dump($config, 10, 2); } @@ -390,7 +406,7 @@ class behat_config_util { $andtags[] = preg_replace('/,.*/', '', $tag); } - foreach ($features as $featurefile) { + foreach ($features as $key => $featurefile) { $contents = file_get_contents($featurefile); $includefeature = true; foreach ($andtags as $tag) { @@ -418,7 +434,7 @@ class behat_config_util { } if ($includefeature) { - $newfeaturelist[] = $featurefile; + $newfeaturelist[$key] = $featurefile; } } return $newfeaturelist; @@ -520,20 +536,31 @@ class behat_config_util { // If parallel run, then only divide features. if (!empty($currentrun) && !empty($parallelruns)) { - // Attempt to split into weighted buckets using timing information, if available. - if ($alloc = $this->profile_guided_allocate($features, max(1, $parallelruns), $currentrun)) { - $allocatedfeatures = $alloc; - } else { - // Divide the list of feature files amongst the parallel runners. - // Pull out the features for just this worker. - if (count($features)) { - $features = array_chunk($features, ceil(count($features) / max(1, $parallelruns))); - // Check if there is any feature file for this process. - if (!empty($features[$currentrun - 1])) { - $allocatedfeatures = $features[$currentrun - 1]; - } else { - $allocatedfeatures = array(); + $featurestodivide['withtags'] = $features; + $allocatedfeatures = array(); + + // If tags are set then split features with tags first. + if (!empty($this->tags)) { + $featurestodivide['withtags'] = $this->filtered_features_with_tags($features); + $featurestodivide['withouttags'] = $this->remove_blacklisted_features_from_list($features, + $featurestodivide['withtags']); + } + + // Attempt to split into weighted buckets using timing information, if available. + foreach ($featurestodivide as $tagfeatures) { + if ($alloc = $this->profile_guided_allocate($tagfeatures, max(1, $parallelruns), $currentrun)) { + $allocatedfeatures = array_merge($allocatedfeatures, $alloc); + } else { + // Divide the list of feature files amongst the parallel runners. + // Pull out the features for just this worker. + if (count($tagfeatures)) { + $splitfeatures = array_chunk($tagfeatures, ceil(count($tagfeatures) / max(1, $parallelruns))); + + // Check if there is any feature file for this process. + if (!empty($splitfeatures[$currentrun - 1])) { + $allocatedfeatures = array_merge($allocatedfeatures, $splitfeatures[$currentrun - 1]); + } } } } @@ -863,7 +890,7 @@ class behat_config_util { * * @return array */ - private function get_components_with_tests() { + protected function get_components_with_tests() { if (empty($this->componentswithtests)) { $this->componentswithtests = tests_finder::get_components_with_tests('behat'); } @@ -902,9 +929,7 @@ class behat_config_util { if (isset($features[$key])) { $features[$key] = null; unset($features[$key]); - } else if (empty($this->tags)) { - // If tags not set, then ensure we have a blacklisted feature in core. Else, let user know that - // blacklisted feature is invalid. + } else { $featurestocheck = $this->get_components_features(); if (!isset($featurestocheck[$key]) && !defined('PHPUNIT_TEST')) { behat_error(BEHAT_EXITCODE_REQUIREMENT, 'Blacklisted feature "' . $blacklistpath . '" not found.'); @@ -972,7 +997,6 @@ class behat_config_util { } } - // Return sub-set of features if parallel run. $featuresforrun = $this->get_features_for_the_run($features, $parallelruns, $currentrun); // Default suite. @@ -1057,6 +1081,20 @@ class behat_config_util { return $selectablethemes; } + /** + * Return theme directory. + * + * @param string $themename + * @return string theme directory + */ + protected function get_theme_directory($themename) { + global $CFG; + + $themetestdir = "/theme/" . $themename; + + return $CFG->dirroot . $themetestdir . self::get_behat_tests_path(); + } + /** * Returns all the directories having overridden tests. * @@ -1071,8 +1109,7 @@ class behat_config_util { 'contexts' => '|behat_.*\.php$|', 'features' => '|.*\.feature$|', ); - $themetestdir = "/theme/" . $theme . '/tests/behat'; - $themetestdirfullpath = $CFG->dirroot . $themetestdir; + $themetestdirfullpath = $this->get_theme_directory($theme) . '/tests/behat'; // If test directory doesn't exist then return. if (!is_dir($themetestdirfullpath)) { @@ -1082,7 +1119,7 @@ class behat_config_util { $directoriestosearch = glob($themetestdirfullpath . DIRECTORY_SEPARATOR . '*' , GLOB_ONLYDIR); // Include theme directory to find tests. - $dirs[realpath($themetestdirfullpath)] = trim(str_replace('/', '_', $themetestdir), '_'); + $dirs[realpath($themetestdirfullpath)] = trim(str_replace('/', '_', $themetestdirfullpath), '_'); // Search for tests in valid directories. foreach ($directoriestosearch as $dir) { @@ -1109,10 +1146,8 @@ class behat_config_util { * @return array list of blacklisted contexts or features */ protected function get_blacklisted_tests_for_theme($theme, $testtype) { - global $CFG; - $themetestpath = $CFG->dirroot . DIRECTORY_SEPARATOR . "theme" . DIRECTORY_SEPARATOR . $theme . - self::get_behat_tests_path(); + $themetestpath = $this->get_theme_test_directory($theme); if (file_exists($themetestpath . DIRECTORY_SEPARATOR . 'blacklist.json')) { // Blacklist file exist. Leave it for last to clear the feature and contexts. @@ -1177,16 +1212,24 @@ class behat_config_util { $themefeatures = $this->get_tests_for_theme($theme, 'features'); $themeblacklistfeatures = $this->get_blacklisted_tests_for_theme($theme, 'features'); - // If tags are specified then we just want features with specified tags. - if (!empty($this->tags)) { - if (!empty($themefeatures)) { - $themefeatures = $this->filtered_features_with_tags($themefeatures); - } + // Clean feature key and path. + $features = array(); + $blacklistfeatures = array(); + + foreach ($themefeatures as $themefeature) { + list($featurekey, $featurepath) = $this->get_clean_feature_key_and_path($themefeature); + $features[$featurekey] = $featurepath; + } + foreach ($themeblacklistfeatures as $themeblacklistfeature) { + list($blacklistfeaturekey, $blacklistfeaturepath) = $this->get_clean_feature_key_and_path($themeblacklistfeature); + $blacklistfeatures[$blacklistfeaturekey] = $blacklistfeaturepath; } + ksort($features); + $retval = array( - 'blacklistfeatures' => $themeblacklistfeatures, - 'features' => $themefeatures + 'blacklistfeatures' => $blacklistfeatures, + 'features' => $features ); return $retval; @@ -1215,7 +1258,10 @@ class behat_config_util { // If we already have this list then just return. This will not change by run. if (!empty($this->themecontexts[$theme]) && !empty($this->themesuitecontexts)) { - return array(array_keys($this->themecontexts[$theme]), $this->themesuitecontexts[$theme]); + return array( + 'contexts' => $this->themecontexts[$theme], + 'suitecontexts' => $this->themesuitecontexts[$theme], + ); } if (empty($this->overriddenthemescontexts)) { diff --git a/lib/behat/classes/util.php b/lib/behat/classes/util.php index c0c03633c50..11aa8664b6e 100644 --- a/lib/behat/classes/util.php +++ b/lib/behat/classes/util.php @@ -216,12 +216,13 @@ class behat_util extends testing_util { * Stores a file in dataroot/behat to allow Moodle to switch * to the test environment when using cli-server. * @param bool $themesuitewithallfeatures if only theme specific features need to be included in the suite. + * @param string $tags comma separated tag, which will be given preference while distributing features in parallel run. * @param int $parallelruns number of parallel runs. * @param int $run current run. * @throws coding_exception * @return void */ - public static function start_test_mode($themesuitewithallfeatures = false, $parallelruns = 0, $run = 0) { + public static function start_test_mode($themesuitewithallfeatures = false, $tags = '', $parallelruns = 0, $run = 0) { global $CFG; if (!defined('BEHAT_UTIL')) { @@ -237,7 +238,7 @@ class behat_util extends testing_util { self::test_environment_problem(); // Updates all the Moodle features and steps definitions. - behat_config_manager::update_config_file('', true, '', $themesuitewithallfeatures, $parallelruns, $run); + behat_config_manager::update_config_file('', true, $tags, $themesuitewithallfeatures, $parallelruns, $run); if (self::is_test_mode_enabled()) { return;