diff --git a/admin/tool/behat/cli/init.php b/admin/tool/behat/cli/init.php index 5547b56eae3..97d613556d0 100644 --- a/admin/tool/behat/cli/init.php +++ b/admin/tool/behat/cli/init.php @@ -40,10 +40,41 @@ define('CACHE_DISABLE_ALL', true); require_once(__DIR__ . '/../../../../lib/clilib.php'); require_once(__DIR__ . '/../../../../lib/behat/lib.php'); + +list($options, $unrecognized) = cli_get_params( + array( + 'parallel' => 0, + 'suffix' => '', + ) +); + + +$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']); +$suffixarg = $options['suffix'] ? "--suffix={$options['suffix']} --parallel=$nproc" : ''; + + +if ($nproc && !$suffixarg) { + foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) { + if (file_exists($dir) && is_link($dir) && preg_match('#/behat\d+$#', $dir)) { + unlink($dir); + } + } + $cmds = array(); + for ($i = 1; $i <= $nproc; $i++) { + $cmds[] = "php ".__FILE__." --suffix=$i --parallel=$nproc 2>&1"; + } + // This is intensive compared to behat itself so halve the parallelism. + foreach (array_chunk($cmds, max(1, floor($nproc/2)), true) as $chunk) { + ns_parallel_popen($chunk, true); + } + exit(0); +} + + // Changing the cwd to admin/tool/behat/cli. chdir(__DIR__); $output = null; -exec("php util.php --diag", $output, $code); +exec("php util.php --diag $suffixarg", $output, $code); if ($code == 0) { echo "Behat test environment already installed\n"; @@ -53,7 +84,7 @@ if ($code == 0) { // Behat and dependencies are installed and we need to install the test site. chdir(__DIR__); - passthru("php util.php --install", $code); + passthru("php util.php --install $suffixarg", $code); if ($code != 0) { exit($code); } @@ -64,12 +95,12 @@ if ($code == 0) { // Test site data is outdated. chdir(__DIR__); - passthru("php util.php --drop", $code); + passthru("php util.php --drop $suffixarg", $code); if ($code != 0) { exit($code); } - passthru("php util.php --install", $code); + passthru("php util.php --install $suffixarg", $code); if ($code != 0) { exit($code); } @@ -81,7 +112,7 @@ if ($code == 0) { // Returning to admin/tool/behat/cli. chdir(__DIR__); - passthru("php util.php --install", $code); + passthru("php util.php --install $suffixarg", $code); if ($code != 0) { exit($code); } @@ -93,7 +124,7 @@ if ($code == 0) { } // Enable editing mode according to config.php vars. -passthru("php util.php --enable", $code); +passthru("php util.php --enable $suffixarg", $code); if ($code != 0) { exit($code); } diff --git a/admin/tool/behat/cli/ns_parallel.php b/admin/tool/behat/cli/ns_parallel.php new file mode 100644 index 00000000000..2f1ae7e7cfc --- /dev/null +++ b/admin/tool/behat/cli/ns_parallel.php @@ -0,0 +1,212 @@ +. + +/** + * Wrapper to run previously set-up behat tests in parallel. + * + * @package tool_behat + * @copyright 2014 NetSpot Pty Ltd + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + + +if (isset($_SERVER['REMOTE_ADDR'])) { + die(); // No access from web! +} + + +define('BEHAT_UTIL', true); +define('CLI_SCRIPT', true); +define('ABORT_AFTER_CONFIG', true); +define('NO_OUTPUT_BUFFERING', true); + +error_reporting(E_ALL | E_STRICT); +ini_set('display_errors', '1'); +ini_set('log_errors', '1'); + + +require_once __DIR__ .'/../../../../config.php'; +require_once __DIR__.'/../../../../lib/clilib.php'; +require_once __DIR__.'/../../../../lib/behat/lib.php'; + + +list($options, $unrecognised) = cli_get_params( + array( + 'stop-on-failure' => 0, + 'parallel' => 0, + 'verbose' => false, + 'replace' => false, + ) +); + + +if (empty($options['parallel']) && $dirs = glob("{$CFG->dirroot}/behat*")) { + sort($dirs); + if ($max = preg_filter('#.*behat(\d+)#', '$1', end($dirs))) { + $options['parallel'] = $max; + } +} + + +$suffix = ''; +$time = microtime(true); +$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']); +array_walk($unrecognised, function (&$v) { + if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1='\$2'", $v)) { + $v = $x; + } else if (!preg_match("#^-#", $v)) { + $v = escapeshellarg($v); + } +}); +$extraopts = implode(' ', $unrecognised); + + +if (empty($nproc)) { + fwrite(STDERR, "Invalid or missing --parallel parameter, must be >= 1.\n"); + exit(1); +} + + +$checkfail = array(); +$outputs = array(); +$handles = array(); +$pipe2i = array(); +$exits = array(); +$unused = null; +$linelencnt = 0; +$procs = array(); + + +for ($i = 1; $i <= $nproc; $i++) { + $myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts; + $dirroot = dirname($CFG->behat_dataroot)."/behat$i"; + $cmd = "exec {$CFG->dirroot}/vendor/bin/behat --config $dirroot/behat/behat.yml $myopts"; + list($handle, $pipes) = ns_proc_open($cmd, true); + @fclose($pipes[0]); + unset($pipes[0]); + $exits[$i] = 1; + $handles[$i] = array($handle, $pipes[1], $pipes[2]); + $procs[$i] = $handle; + $checkfail[$i] = false; + $outputs[$i] = array(''); + $pipe2i[(int) $pipes[1]] = $i; + $pipe2i[(int) $pipes[2]] = $i; + stream_set_blocking($pipes[1], 0); + stream_set_blocking($pipes[2], 0); +} + + +while (!empty($procs)) { + usleep(10000); + + foreach ($handles as $i => $p) { + if (!($status = @proc_get_status($p[0])) || !$status['running']) { + if ($exits[$i] !== 0) { + $exits[$i] = !empty($status) ? $status['exitcode'] : 1; + } + unset($procs[$i]); + unset($handles[$i][0]); + $last = array_pop($outputs[$i]); + for ($l=2; $l>=1; $l--) + while ($part = @fread($handles[$i][$l], 8192)) + $last .= $part; + $outputs[$i] = array_merge($outputs[$i], explode("\n", $last)); + } + } + + $ready = array(); + foreach ($handles as $i => $set) { + $ready[] = $set[1]; + $ready[] = $set[2]; + } + + // Poll for any process with output or ended. + if (!$result = @stream_select($ready, $unused, $unused, 1)) { + // Nothing; try again. + continue; + } + if (!$fh = reset($ready)) { + continue; + } + + $i = $pipe2i[(int) $fh]; + $last = array_pop($outputs[$i]); + $read = fread($fh, 4096); + $newlines = explode("\n", $last.$read); + $outputs[$i] = array_merge($outputs[$i], $newlines); + + if (!$checkfail[$i]) { + foreach ($newlines as $l => $line) { + unset($newlines[$l]); + if (preg_match('#^Started at [\d\-]+#', $line) || (strlen($line) > 3 && preg_match('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', $line))) { + $checkfail[$i] = true; + break; + } + } + } + if ($progress = preg_filter('#^\s*([FUS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $newlines)) { + if ($checkfail[$i] && preg_filter('#^\s*[S\.\-]*[FU][S\.\-]*(?:\s+\d+)?\s*$#', '$1', $progress)) { + $exits[$i] = 1; + if ($options['stop-on-failure']) { + foreach ($handles as $l => $p) { + $exits[$l] = $l != $i ? 0 : $exits[$i]; + @proc_terminate($p[0], SIGINT); + } + } + } + } + // Process has gone, assume this is the last output for it. + if (empty($procs[$i])) { + unset($handles[$i]); + } + if (empty($checkfail[$i]) || !($update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $read))) { + continue; + } + while ($update) { + $part = substr($update, 0, 70 - $linelencnt); + $update = substr($update, strlen($part)); + $linelencnt += strlen($part); + echo $part; + if ($linelencnt >= 70) { + echo "\n"; + $linelencnt = 0; + } + } +} +echo "\n\n"; + + +$exits = array_filter($exits, function ($v) {return $v !== 0;}); + + +if ($exits || $options['verbose']) { + echo "Exit codes: ".implode(" ", $exits)."\n\n"; + foreach ($outputs as $i => $output) { + unset($outputs[$i]); + if (!end($output)) array_pop($output); + $prefix = "[behat$i] "; + array_walk($output, function (&$l) use ($prefix) { + $l = $prefix.$l; + }); + echo implode("\n", $output)."\n\n"; + } + $failed = true; +} + + +$time = round(microtime(true) - $time, 1); +echo "Finished in {$time}s\n"; +exit(!empty($failed) ? 1 : 0); diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index d843bf2fb6b..b68992e3701 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -40,6 +40,8 @@ list($options, $unrecognized) = cli_get_params( array( 'help' => false, 'install' => false, + 'parallel' => 0, + 'suffix' => '', 'drop' => false, 'enable' => false, 'disable' => false, @@ -63,6 +65,7 @@ Options: --drop Drops the database tables and the dataroot contents --enable Enables test environment and updates tests list --disable Disables test environment +--parallel Run operation for all parallel behat environments. --diag Get behat test environment status code -h, --help Print out this help @@ -78,11 +81,32 @@ if (!empty($options['help'])) { exit(0); } -// Describe this script. + +if (!empty($options['parallel']) && empty($options['suffix'])) { + foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) { + if (file_exists($dir) && is_dir($dir)) { + unlink($dir); + } + } + $cmds = array(); + $extra = preg_filter('#(.*)\s*--parallel=\d+\s*(.*?)#', '$1 $2', implode(' ', array_slice($argv, 1))); + for ($i = 1; $i <= $options['parallel']; $i++) { + $cmds[] = "php ".__FILE__." $extra --suffix=$i 2>&1"; + } + // This is intensive compared to behat itself so halve the parallelism. + foreach (array_chunk($cmds, min(1, floor($options['parallel']/2)), true) as $chunk) { + ns_parallel_popen($chunk, true); + } + exit(0); +} + + +// Checking $CFG->behat_* vars and values. define('BEHAT_UTIL', true); define('CLI_SCRIPT', true); define('NO_OUTPUT_BUFFERING', true); define('IGNORE_COMPONENT_CACHE', true); +define('BEHAT_SUFFIX', $options['suffix']); // Only load CFG from config.php, stop ASAP in lib/setup.php. define('ABORT_AFTER_CONFIG', true); diff --git a/behat_pga_default.json b/behat_pga_default.json new file mode 100644 index 00000000000..dbce41d24e3 --- /dev/null +++ b/behat_pga_default.json @@ -0,0 +1,109 @@ +{ + "course\/tests\/behat\/course_controls.feature": 117.2, + "backup\/util\/ui\/tests\/behat\/restore_moodle2_courses.feature": 108.2, + "mod\/forum\/tests\/behat\/discussion_subscriptions.feature": 92.8, + "course\/tests\/behat\/category_resort.feature": 75.4, + "course\/tests\/behat\/course_resort.feature": 58.9, + "admin\/tool\/behat\/tests\/behat\/data_generators.feature": 53.1, + "course\/tests\/behat\/category_management.feature": 50.1, + "blocks\/recent_activity\/tests\/behat\/structural_changes.feature": 48.3, + "grade\/tests\/behat\/grade_override_letter.feature": 47.3, + "blocks\/activity_modules\/tests\/behat\/block_activity_modules.feature": 42.7, + "calendar\/tests\/behat\/calendar.feature": 42, + "grade\/grading\/form\/rubric\/tests\/behat\/edit_rubric.feature": 41.2, + "mod\/forum\/tests\/behat\/forum_subscriptions.feature": 32.6, + "course\/tests\/behat\/move_activities.feature": 30.8, + "mod\/workshep\/tests\/behat\/workshep_assessment.feature": 29.3, + "blocks\/glossary_random\/tests\/behat\/glossary_random.feature": 29, + "mod\/workshop\/tests\/behat\/workshop_assessment.feature": 28.8, + "blocks\/tests\/behat\/manage_blocks.feature": 28.6, + "badges\/tests\/behat\/navrequirecap.feature": 28, + "mod\/quiz\/tests\/behat\/completion_condition_attempts_used.feature": 26.9, + "mod\/quiz\/tests\/behat\/completion_condition_passing_grade.feature": 26, + "blocks\/html\/tests\/behat\/multiple_instances.feature": 26, + "message\/tests\/behat\/display_history.feature": 24.7, + "course\/tests\/behat\/category_change_visibility.feature": 24.2, + "local\/uneditableblocks\/tests\/behat\/enable_uneditableblocks.feature": 23.1, + "mod\/feedback\/tests\/behat\/defaultshortanswerlength.feature": 22.1, + "backup\/util\/ui\/tests\/behat\/backup_courses.feature": 22, + "course\/tests\/behat\/create_delete_course.feature": 21.9, + "blocks\/course_summary\/tests\/behat\/block_course_summary_course.feature": 21.2, + "course\/tests\/behat\/move_sections.feature": 21.1, + "course\/tests\/behat\/course_category_management_listing.feature": 21, + "local\/userpolicy\/tests\/behat\/fieldvisibility.feature": 20.9, + "admin\/tool\/behat\/tests\/behat\/get_and_set_fields.feature": 20.5, + "mod\/wiki\/tests\/behat\/edit_tags.feature": 19.6, + "mod\/attendance\/tests\/behat\/attendance_mod.feature": 19.5, + "blocks\/participants\/tests\/behat\/block_participants_course.feature": 19.5, + "blocks\/course_summary\/tests\/behat\/block_course_summary_frontpage.feature": 17.7, + "blocks\/tests\/behat\/configure_block_throughout_site.feature": 17.7, + "grade\/report\/singleview\/tests\/behat\/singleview.feature": 17.6, + "grade\/grading\/form\/rubric\/tests\/behat\/reuse_own_rubrics.feature": 16.3, + "mod\/wiki\/tests\/behat\/group_enhancements.feature": 16.2, + "admin\/tests\/behat\/forcelogin_makefrontpagepublic.feature": 16, + "message\/tests\/behat\/disablenotifications.feature": 16, + "completion\/tests\/behat\/teacher_manual_completion.feature": 16, + "mod\/glossary\/tests\/behat\/entries_require_approval.feature": 16, + "backup\/util\/ui\/tests\/behat\/import_course.feature": 15.9, + "admin\/tests\/behat\/set_admin_settings_value.feature": 15.8, + "mod\/mediagallery\/tests\/behat\/separategroups.feature": 15.6, + "blocks\/tests\/behat\/return_block_original_state.feature": 15.6, + "admin\/tests\/behat\/custom_maxbytes.feature": 15.4, + "mod\/oublog\/tests\/behat\/separate_individuals.feature": 15.4, + "mod\/forum\/tests\/behat\/default_displaywordcount.feature": 15.4, + "backup\/util\/ui\/tests\/behat\/restore_moodle2_course_numsections.feature": 14.9, + "question\/tests\/behat\/question_defaultpenalty.feature": 14.8, + "availability\/tests\/behat\/edit_availability.feature": 14.7, + "mod\/data\/tests\/behat\/view_entries.feature": 14.6, + "local\/catdelete\/tests\/behat\/catdelete.feature": 14.3, + "group\/tests\/behat\/showusernameingroup.feature": 14.3, + "mod\/forum\/tests\/behat\/edit_post_student.feature": 14.2, + "admin\/tool\/behat\/tests\/behat\/nasty_strings.feature": 14.1, + "mod\/hsuforum\/tests\/behat\/edit_post_student.feature": 13.8, + "mod\/quiz\/tests\/behat\/add_quiz.feature": 13.5, + "grade\/grading\/form\/rubric\/tests\/behat\/publish_rubric_templates.feature": 13.5, + "blocks\/autocreate_user\/tests\/behat\/add_introtext.feature": 13.5, + "question\/type\/truefalse\/tests\/behat\/custompenalty.feature": 13.3, + "blocks\/participants\/tests\/behat\/block_participants_frontpage.feature": 13.1, + "blocks\/html\/tests\/behat\/course_block.feature": 12.9, + "mod\/quiz\/tests\/behat\/configsubnethide.feature": 12.8, + "mod\/forum\/tests\/behat\/separate_group_single_group_discussions.feature": 12.6, + "blocks\/html\/tests\/behat\/configuring_html_block.feature": 12.3, + "course\/tests\/behat\/course_change_visibility.feature": 12.2, + "course\/tests\/behat\/modchooser_hidden.feature": 11.9, + "blocks\/login\/tests\/behat\/login_block.feature": 11.7, + "course\/tests\/behat\/paged_course_navigation.feature": 11.1, + "course\/tests\/behat\/limitrolerenaming.feature": 11, + "mod\/book\/tests\/behat\/create_chapters.feature": 11, + "mod\/forum\/tests\/behat\/my_forum_posts.feature": 10.7, + "auth\/tests\/behat\/login.feature": 10.5, + "blocks\/glossary_random\/tests\/behat\/glossary_random_frontpage.feature": 10.5, + "mod\/data\/tests\/behat\/add_entries.feature": 10.1, + "mod\/wiki\/tests\/behat\/preview_page.feature": 9.7, + "mod\/oublog\/tests\/behat\/personalblog.feature": 9.7, + "admin\/tests\/behat\/display_short_names.feature": 9.6, + "mod\/forum\/tests\/behat\/separate_group_discussions.feature": 9.5, + "course\/tests\/behat\/add_activities.feature": 9.4, + "mod\/forum\/tests\/behat\/forum_subscriptions_management.feature": 8.8, + "blocks\/tests\/behat\/restrict_available_blocks.feature": 8.7, + "blocks\/comments\/tests\/behat\/add_comment.feature": 8.7, + "mod\/survey\/tests\/behat\/survey_types.feature": 8.5, + "admin\/tool\/langimport\/tests\/behat\/manage_langpacks.feature": 7.3, + "local\/rolerenaming\/tests\/behat\/course_role_renaming.feature": 7.2, + "course\/tests\/behat\/edit_settings.feature": 7.1, + "group\/tests\/behat\/create_groups.feature": 6.8, + "blocks\/tests\/behat\/add_blocks.feature": 6.5, + "message\/tests\/behat\/manage_contacts.feature": 6.1, + "course\/tests\/behat\/course_search.feature": 6, + "course\/tests\/behat\/course_creation.feature": 5.8, + "my\/tests\/behat\/reset_page.feature": 5.4, + "my\/tests\/behat\/restrict_available_blocks.feature": 5.3, + "mod\/oublog\/tests\/behat\/basic.feature": 5.2, + "my\/tests\/behat\/add_blocks.feature": 4.6, + "user\/tests\/behat\/reset_page.feature": 4, + "message\/tests\/behat\/search_history.feature": 3.9, + "blocks\/navigation\/tests\/behat\/expand_my_courses_setting.feature": 3.3, + "user\/tests\/behat\/add_blocks.feature": 2.8, + "report\/usersessions\/tests\/behat\/usersessions_report.feature": 2.7, + "admin\/tool\/behat\/tests\/behat\/test_environment.feature": 1 +} \ No newline at end of file diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index d6c852017de..b6f97154999 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -79,7 +79,10 @@ class behat_config_manager { $featurespaths[$uniquekey] = $path; } } - $features = array_values($featurespaths); + foreach ($featurespaths as $path) { + $additional = glob("$path/*.feature"); + $features = array_merge($features, $additional); + } } // Optionally include features from additional directories. @@ -198,6 +201,29 @@ class behat_config_manager { // We require here when we are sure behat dependencies are available. require_once($CFG->dirroot . '/vendor/autoload.php'); + $instance = 1; + $parallel = 0; + foreach ($_SERVER['argv'] as $arg) { + if (strpos($arg, '--suffix=') === 0) { + $instance = intval(substr($arg, strlen('--suffix='))); + } + if (empty($parallel)) { + $parallel = preg_filter('#--parallel=(\d+)#', '$1', $arg); + } + } + + // Attempt to split into weighted buckets using timing information, if available. + if ($alloc = self::profile_guided_allocate($features, max(1, $parallel), $instance)) { + $features = $alloc; + } else { + // Divide the list of feature files amongst the parallel runners. + srand(crc32(floor(time() / 3600 / 24).var_export($features,true))); + shuffle($features); + // Pull out the features for just this worker. + $features = array_chunk($features, ceil(count($features) / max(1, $parallel))); + $features = $features[$instance-1]; + } + // It is possible that it has no value as we don't require a full behat setup to list the step definitions. if (empty($CFG->behat_wwwroot)) { $CFG->behat_wwwroot = 'http://itwillnotbeused.com'; @@ -242,6 +268,71 @@ class behat_config_manager { return Symfony\Component\Yaml\Yaml::dump($config, 10, 2); } + /** + * Attempt to split feature list into fairish buckets using timing information, if available. + * Simply add each one to lightest buckets until all files allocated. + * PGA = Profile Guided Allocation. I made it up just now. + * CAUTION: workers must agree on allocation, do not be random anywhere! + * + * @param array $features Behat feature files array + * @param int $nbuckets Number of buckets to divide into + * @param int $instance Index number of this instance + * @return array Feature files array, sorted into allocations + */ + protected static function profile_guided_allocate($features, $nbuckets, $instance) { + $pga = __DIR__.'/../../../behat_pga_default.json'; + $pga = defined('BEHAT_PGA_DATA') && file_exists(BEHAT_PGA_DATA) ? BEHAT_PGA_DATA : $pga; + + if (defined('BEHAT_PGA_DISABLE') || !file_exists($pga) || !$pga = @json_decode(file_get_contents($pga), true)) { + // No data available, fall back to relying on shuffle. + return false; + } + + arsort($pga); // Ensure most expensive is first. + + $realroot = realpath(__DIR__.'/../../../').'/'; + $defaultweight = array_sum($pga) / count($pga); // TODO: median is more ideal. + $weights = array_fill(0, $nbuckets, 0); + $buckets = array_fill(0, $nbuckets, array()); + $totalweight = 0; + + // Re-key the features list to match pga data. + foreach ($features as $k => $file) { + $key = str_replace($realroot, '', $file); + $features[$key] = $file; + unset($features[$k]); + if (!isset($pga[$key])) { + $pga[$key] = $defaultweight; + } + } + + // Sort features by known weights; largest ones should be allocated first. + $pgaorder = array(); + foreach ($features as $key => $file) { + $pgaorder[$key] = $pga[$key]; + } + arsort($pgaorder); + + // Finally, add each feature one by one to the lightest bucket. + foreach ($pgaorder as $key => $weight) { + $file = $features[$key]; + $light_bucket = array_search(min($weights), $weights); + $weights[$light_bucket] += $weight; + $buckets[$light_bucket][] = $file; + $totalweight += $weight; + } + + if (!defined('BEHAT_PGA_DISABLE_HISTOGRAM') && $instance == $nbuckets) { + echo "Bucket weightings:\n"; + foreach ($weights as $k => $weight) { + echo "$k: ".str_repeat('*', 70 * $nbuckets * $weight / $totalweight)."\n"; + } + } + + // Return the features for this worker. + return $buckets[$instance-1]; + } + /** * Overrides default config with local config values * diff --git a/lib/behat/classes/util.php b/lib/behat/classes/util.php index 95f42fea62d..4a42c578653 100644 --- a/lib/behat/classes/util.php +++ b/lib/behat/classes/util.php @@ -213,6 +213,22 @@ class behat_util extends testing_util { // Updates all the Moodle features and steps definitions. behat_config_manager::update_config_file(); + // Create suffix symlink if necessary. + global $CFG; + if ($CFG->behat_suffix) { + $extra = preg_filter('#.*/(.+)$#', '$1', $CFG->behat_wwwroot); + $link = $CFG->dirroot.'/'.$extra; + if (file_exists($link)) { + if (!is_link($link)) { + throw new coding_exception("File exists at link location ($link) but is not a link!"); + } + @unlink($link); + } + if (!symlink($CFG->dirroot, $link)) { + throw new coding_exception("Unable to create behat suffix symlink ($link)"); + } + } + if (self::is_test_mode_enabled()) { return; } diff --git a/lib/behat/lib.php b/lib/behat/lib.php index 86718a92a8a..40612f4cd8e 100644 --- a/lib/behat/lib.php +++ b/lib/behat/lib.php @@ -272,6 +272,34 @@ function behat_is_test_site() { return false; } +/** + * Add behat suffix to $CFG vars for parallel testing. + **/ +function behat_add_suffix_to_vars($suffix = '') { + global $CFG; + + if (empty($suffix)) { + if (!empty($CFG->behat_suffix)) { + $suffix = $CFG->behat_suffix; + + } else if (defined('BEHAT_SUFFIX') && BEHAT_SUFFIX) { + $suffix = BEHAT_SUFFIX; + } + } + + $CFG->behat_suffix = $suffix; + + if ($suffix) { + if (isset($CFG->behat_wwwroot) && !preg_match("#/behat$suffix\$#", $CFG->behat_wwwroot)) { + $CFG->behat_wwwroot .= "/behat{$suffix}"; + } + if (!preg_match("#/behat{$suffix}\$#", $CFG->behat_dataroot)) { + $CFG->behat_dataroot = dirname($CFG->behat_dataroot)."/behat{$suffix}"; + } + $CFG->behat_prefix = "behat{$suffix}_"; + } +} + /** * Checks if the URL requested by the user matches the provided argument * diff --git a/lib/clilib.php b/lib/clilib.php index 32272defc04..e4992528144 100644 --- a/lib/clilib.php +++ b/lib/clilib.php @@ -175,3 +175,34 @@ function cli_error($text, $errorcode=1) { fwrite(STDERR, "\n"); die($errorcode); } + + +function ns_proc_open($cmd, $die = false) { + $desc = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('pipe', 'w'), + ); + if (!($handle = proc_open($cmd, $desc, $pipes)) && $die) { + throw new Exception('Error starting worker'); + } + return array($handle, $pipes); +} + + +function ns_parallel_popen($cmds, $doexit = false) { + $procs = array(); + foreach ($cmds as $k => $cmd) { + $procs[] = popen($cmd, 'r'); + } + $status = false; + foreach ($procs as $p) { + if (!$p) continue; + while ($out = fgets($p)) echo $out; + $status |= (bool) pclose($p); + } + if ($doexit && $status) { + exit($status); + } + return $status; +} diff --git a/lib/setup.php b/lib/setup.php index d0a77601fdf..a0d870795d1 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -77,9 +77,34 @@ if (defined('BEHAT_SITE_RUNNING')) { // We already switched to behat test site previously. } else if (!empty($CFG->behat_wwwroot) or !empty($CFG->behat_dataroot) or !empty($CFG->behat_prefix)) { + global $argv; + $suffix = ''; + + if (defined('BEHAT_SUFFIX') && BEHAT_SUFFIX) { + $suffix = BEHAT_SUFFIX; + + } else if (!empty($_SERVER['REMOTE_ADDR'])) { + if (preg_match('#/behat(.+?)/#', $_SERVER['REQUEST_URI'])) { + $afterpath = str_replace(realpath($CFG->dirroot).'/', '', realpath($_SERVER['SCRIPT_FILENAME'])); + if (!$suffix = preg_filter("#.*/behat(.+?)/$afterpath#", '$1', $_SERVER['SCRIPT_FILENAME'])) { + throw new coding_exception("Unable to determine behat suffix [afterpath=$afterpath, scriptfilename={$_SERVER['SCRIPT_FILENAME']}]!"); + } + } + + } else if (defined('BEHAT_TEST') || defined('BEHAT_UTIL')) { + if ($match = preg_filter('#--suffix=(.+)#', '$1', $argv)) { + $suffix = reset($match); + } + if ($k = array_search('--config', $argv)) { + $behatconfig = $argv[$k+1]; + $suffix = preg_filter("#^{$CFG->behat_dataroot}(.+?)/behat/behat\.yml#", '$1', $behatconfig); + } + } + // The behat is configured on this server, we need to find out if this is the behat test // site based on the URL used for access. require_once(__DIR__ . '/../lib/behat/lib.php'); + behat_add_suffix_to_vars($suffix); if (behat_is_test_site()) { // Checking the integrity of the provided $CFG->behat_* vars and the // selected wwwroot to prevent conflicts with production and phpunit environments. @@ -89,7 +114,7 @@ if (defined('BEHAT_SITE_RUNNING')) { if (!file_exists("$CFG->behat_dataroot/behattestdir.txt")) { if ($dh = opendir($CFG->behat_dataroot)) { while (($file = readdir($dh)) !== false) { - if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store') { + if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store' or is_numeric($file)) { continue; } behat_error(BEHAT_EXITCODE_CONFIG, '$CFG->behat_dataroot directory is not empty, ensure this is the directory where you want to install behat test dataroot'); diff --git a/lib/testing/lib.php b/lib/testing/lib.php index c089c6a76cf..bbb93d3e293 100644 --- a/lib/testing/lib.php +++ b/lib/testing/lib.php @@ -156,6 +156,9 @@ function testing_error($errorcode, $text = '') { // do not write to error stream because we need the error message in PHP exec result from web ui echo($text."\n"); + if (isset($_SERVER['REMOTE_ADDR'])) { + header('HTTP/1.1 500 Internal Server Error'); + } exit($errorcode); } diff --git a/lib/tests/behat/behat_hooks.php b/lib/tests/behat/behat_hooks.php index 6e2653f9ed6..060998294f8 100644 --- a/lib/tests/behat/behat_hooks.php +++ b/lib/tests/behat/behat_hooks.php @@ -153,6 +153,43 @@ class behat_hooks extends behat_base { } } + protected static $timings = array(); + + /** @BeforeFeature */ + public static function before_feature($obj) { + $file = $obj->getFeature()->getFile(); + self::$timings[$file] = microtime(true); + } + + /** @AfterFeature */ + public static function teardownFeature($obj) { + $file = $obj->getFeature()->getFile(); + self::$timings[$file] = microtime(true) - self::$timings[$file]; + // Probably didn't actually run this, don't output it. + if (self::$timings[$file] < 1) { + unset(self::$timings[$file]); + } + } + + /** @AfterSuite */ + public static function tearDown($obj) { + global $CFG; + if (!defined('BEHAT_FEATURE_TIMING')) { + return; + } + $realroot = realpath(__DIR__.'/../../../').'/'; + foreach (self::$timings as $k => $v) { + $new = str_replace($realroot, '', $k); + self::$timings[$new] = round($v, 1); + unset(self::$timings[$k]); + } + if ($existing = @json_decode(file_get_contents(BEHAT_FEATURE_TIMING), true)) { + self::$timings = array_merge($existing, self::$timings); + } + arsort(self::$timings); + @file_put_contents(BEHAT_FEATURE_TIMING, json_encode(self::$timings, JSON_PRETTY_PRINT)); + } + /** * Resets the test environment. *