diff --git a/admin/tool/behat/cli/init.php b/admin/tool/behat/cli/init.php index d4300d989bb..c80abb9b3e3 100644 --- a/admin/tool/behat/cli/init.php +++ b/admin/tool/behat/cli/init.php @@ -59,6 +59,9 @@ list($options, $unrecognized) = cli_get_params( $help = " Behat utilities to initialise behat tests +Usage: + php init.php [--parallel=value [--maxruns=value] [--fromrun=value --torun=value]] [--help] + Options: -j, --parallel Number of parallel behat run to initialise -m, --maxruns Max parallel processes to be executed at one time. @@ -82,7 +85,7 @@ if (!empty($options['help'])) { $utilfile = 'util_single_run.php'; $paralleloption = ""; // If parallel run then use utilparallel. -if ($options['parallel']) { +if ($options['parallel'] && $options['parallel'] > 1) { $utilfile = 'util.php'; $paralleloption = ""; foreach ($options as $option => $value) { diff --git a/admin/tool/behat/cli/run.php b/admin/tool/behat/cli/run.php index 42407471611..0a5e0aab193 100644 --- a/admin/tool/behat/cli/run.php +++ b/admin/tool/behat/cli/run.php @@ -50,24 +50,29 @@ list($options, $unrecognised) = cli_get_params( 'help' => false, 'tags' => '', 'profile' => '', + 'feature' => '', 'fromrun' => 1, 'torun' => 0, + 'single-run' => false, ), array( 'h' => 'help', 't' => 'tags', 'p' => 'profile', + 's' => 'single-run', ) ); // Checking run.php CLI script usage. $help = " Behat utilities to run behat tests in parallel + +Usage: + php run.php [--BEHAT_OPTION=\"value\"] [--feature=\"value\"] [--replace] [--fromrun=value --torun=value] [--help] + Options: --t, --tags Tags to execute. --p, --profile Profile to execute. ---stop-on-failure Stop on failure in any parallel run. ---verbose Verbose output +--BEHAT_OPTION Any combination of behat option specified in http://behat.readthedocs.org/en/v2.5/guides/6.cli.html +--feature Only execute specified feature file (Absolute path of feature file). --replace Replace args string with run process number, useful for output. --fromrun Execute run starting from (Used for parallel runs on different vms) --torun Execute run till (Used for parallel runs on different vms) @@ -75,7 +80,7 @@ Options: -h, --help Print out this help Example from Moodle root directory: -\$ php admin/tool/behat/cli/run.php --parallel=2 +\$ php admin/tool/behat/cli/run.php --tags=\"@javascript\" More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests "; @@ -101,25 +106,6 @@ if (extension_loaded('pcntl')) { } } -// If empty parallelrun then just check with user if it's a run single behat test. -if (empty($parallelrun)) { - if (cli_input("This is not a parallel site, do you want to run single behat run? (Y/N)", 'n', array('y', 'n')) == 'y') { - $runtestscommand = behat_command::get_behat_command(); - $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); - exec("php $runtestscommand", $output, $code); - echo implode(PHP_EOL, $output) . PHP_EOL; - exit($code); - } else { - exit(1); - } -} - -// Create site symlink if necessary. -if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) { - echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL; - exit(1); -} - $time = microtime(true); array_walk($unrecognised, function (&$v) { if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1='\$2'", $v)) { @@ -151,6 +137,30 @@ if ($options['tags']) { $extraopts[] = '--tags="' . $tags . '"'; } +// Feature should be added to last, for behat command. +if ($options['feature']) { + $extraopts[] = $options['feature']; + // Only run 1 process as process. + // Feature file is picked from absolute path provided, so no need to check for behat.yml. + $options['torun'] = $options['fromrun']; +} + +// Set of options to pass to behat. +$extraopts = implode(' ', $extraopts); + +// If empty parallelrun then just check with user if it's a run single behat test. +if (empty($parallelrun)) { + $cwd = getcwd(); + chdir(__DIR__); + $runtestscommand = '../../../../' . behat_command::get_behat_command(); + $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); + $runtestscommand .= ' ' . $extraopts; + echo "Running single behat site:" . PHP_EOL; + passthru("php $runtestscommand", $code); + chdir($cwd); + exit($code); +} + // Update config file if tags defined. if ($tags) { // Hack to set proper dataroot and wwwroot. @@ -158,12 +168,17 @@ if ($tags) { $behatwwwroot = $CFG->behat_wwwroot; for ($i = 1; $i <= $parallelrun; $i++) { $CFG->behatrunprocess = $i; - $CFG->behat_dataroot = $behatdataroot . $i; - if (!empty($CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'])) { - $CFG->behat_wwwroot = $CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot']; + + if (!empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) { + $CFG->behat_wwwroot = $CFG->behat_parallel_run[$i - 1]['behat_wwwroot']; } else { $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i; } + if (!empty($CFG->behat_parallel_run[$i - 1]['behat_dataroot'])) { + $CFG->behat_dataroot = $CFG->behat_parallel_run[$i - 1]['behat_dataroot']; + } else { + $CFG->behat_dataroot = $behatdataroot . $i; + } behat_config_manager::update_config_file('', true, $tags); } $CFG->behat_dataroot = $behatdataroot; @@ -172,7 +187,6 @@ if ($tags) { } $cmds = array(); -$extraopts = implode(' ', $extraopts); echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL; for ($i = $options['fromrun']; $i <= $options['torun']; $i++) { @@ -185,7 +199,7 @@ for ($i = $options['fromrun']; $i <= $options['torun']; $i++) { $behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i); // Command to execute behat run. - $cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts; + $cmds[BEHAT_PARALLEL_SITE_NAME . $i] = '../../../../' . $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts; echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL; } @@ -194,8 +208,14 @@ if (empty($cmds)) { exit(1); } +// Create site symlink if necessary. +if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) { + echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL; + exit(1); +} + // Execute all commands. -$processes = cli_execute_parallel($cmds); +$processes = cli_execute_parallel($cmds, __DIR__); $stoponfail = empty($options['stop-on-failure']) ? false : true; // Print header. @@ -213,21 +233,31 @@ foreach ($exitcodes as $exitcode) { $status = (bool)$status || (bool)$exitcode; } +// Run finished. Show exit code and output from individual process. +$verbose = empty($options['verbose']) ? false : true; +$verbose = $verbose || $status; + // Show exit code from each process, if any process failed. -if ($status) { - echo "Exit codes: " . implode(" ", $exitcodes) . PHP_EOL; - echo "To re-run failed processes, you can use following commands:" . PHP_EOL; - foreach ($cmds as $name => $cmd) { - if (!empty($exitcodes[$name])) { - echo "[" . $name . "] " . $cmd . PHP_EOL; +if ($verbose) { + // Echo exit codes. + echo "Exit codes for each behat run: " . PHP_EOL; + ksort($exitcodes); + foreach ($exitcodes as $run => $exitcode) { + echo $run . ": " . $exitcode . PHP_EOL; + } + + // Show failed re-run commands. + if ($status) { + echo "To re-run failed processes, you can use following commands:" . PHP_EOL; + foreach ($cmds as $name => $cmd) { + if (!empty($exitcodes[$name])) { + echo "[" . $name . "] " . $cmd . PHP_EOL; + } } } echo PHP_EOL; } -// Run finished. Show exit code and output from individual process. -$verbose = empty($options['verbose']) ? false : true; -$verbose = $verbose || $status; print_each_process_info($processes, $verbose); // Remove site symlink if necessary. diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index 1bdb72dd833..23a47fd6b8f 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -68,15 +68,18 @@ list($options, $unrecognized) = cli_get_params( $help = " Behat utilities to manage the test environment +Usage: + php util.php [--install|--drop|--enable|--disable|--diag|--updatesteps|--help] [--parallel=value [--maxruns=value]] + Options: --install Installs the test environment for acceptance tests --drop Drops the database tables and the dataroot contents --enable Enables test environment and updates tests list --disable Disables test environment --diag Get behat test environment status code +--updatesteps Update feature step file. -j, --parallel Number of parallel behat run operation -m, --maxruns Max parallel processes to be executed at one time. ---updatesteps Update feature step file. -h, --help Print out this help @@ -205,8 +208,12 @@ if ($options['install']) { } else if ($options['disable']) { echo "Acceptance tests environment disabled for " . $options['parallel'] . " parallel sites" . PHP_EOL; +} else if ($options['diag']) { + // Valid option, so nothing to do. } else { echo $help; + chdir($cwd); + exit(1); } chdir($cwd); diff --git a/admin/tool/behat/cli/util_single_run.php b/admin/tool/behat/cli/util_single_run.php index fa90359f174..b089539958e 100644 --- a/admin/tool/behat/cli/util_single_run.php +++ b/admin/tool/behat/cli/util_single_run.php @@ -61,6 +61,9 @@ if ($options['install'] or $options['drop']) { $help = " Behat utilities to manage the test environment +Usage: + php util_single_run.php [--install|--drop|--enable|--disable|--diag|--updatesteps|--help] + Options: --install Installs the test environment for acceptance tests --drop Drops the database tables and the dataroot contents diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index f3b8680378d..eaa94303b62 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -130,21 +130,44 @@ class behat_config_manager { return $features; } $newfeaturelist = array(); - $tagstosearch = explode('&&', $tags); + // Split tags in and and or. + $tags = explode('&&', $tags); + $andtags = array(); + $ortags = array(); + foreach ($tags as $tag) { + // Explode all tags seperated by , and add it to ortags. + $ortags = array_merge($ortags, explode(',', $tag)); + // And tags will be the first one before comma(,). + $andtags[] = preg_replace('/,.*/', '', $tag); + } + foreach ($features as $featurefile) { $contents = file_get_contents($featurefile); $includefeature = true; - foreach ($tagstosearch as $tag) { + foreach ($andtags as $tag) { // If negitive tag, then ensure it don't exist. if (strpos($tag, '~') !== false) { $tag = substr($tag, 1); if ($contents && strpos($contents, $tag) !== false) { $includefeature = false; + break; } } else if ($contents && strpos($contents, $tag) === false) { $includefeature = false; + break; } } + + // If feature not included then check or tags. + if (!$includefeature && !empty($ortags)) { + foreach ($ortags as $tag) { + if ($contents && (strpos($tag, '~') === false) && (strpos($contents, $tag) !== false)) { + $includefeature = true; + break; + } + } + } + if ($includefeature) { $newfeaturelist[] = $featurefile; } @@ -354,7 +377,12 @@ class behat_config_manager { // Pull out the features for just this worker. if (count($features)) { $features = array_chunk($features, ceil(count($features) / max(1, $parallelruns))); - $features = $features[$CFG->behatrunprocess - 1]; + // Check if there is any feature file for this process. + if (!empty($features[$CFG->behatrunprocess - 1])) { + $features = $features[$CFG->behatrunprocess - 1]; + } else { + $features = null; + } } } // Set proper selenium2 wd_host if defined.