From b9f78ef45c6c6f19cd2b754470c2ae7960f62e7f Mon Sep 17 00:00:00 2001 From: Ferran Recio Date: Wed, 15 Nov 2023 15:23:35 +0100 Subject: [PATCH] MDL-80096 tool_generator: execute behat generators in current instance This is a new feature in tool_generator to execute a feature file containing only generator steps. This new tool can be used to prepared testing scenarios for complex issues that requires manual testing. --- .../generator/classes/form/featureimport.php | 60 +++++ .../local/testscenario/parsedfeature.php | 123 +++++++++ .../classes/local/testscenario/runner.php | 197 +++++++++++++++ .../classes/local/testscenario/steprunner.php | 217 ++++++++++++++++ .../classes/output/parsingresult.php | 88 +++++++ admin/tool/generator/cli/runtestscenario.php | 178 +++++++++++++ .../tool/generator/lang/en/tool_generator.php | 12 + admin/tool/generator/runtestscenario.php | 95 +++++++ admin/tool/generator/settings.php | 10 +- .../templates/parsingresult.mustache | 106 ++++++++ .../tests/behat/testscenario.feature | 64 +++++ .../fixtures/testscenario/scenario.feature | 27 ++ .../testscenario/scenario_wrongempty.feature | 4 + .../testscenario/scenario_wrongformat.feature | 5 + .../scenario_wrongonlybackground.feature | 12 + .../scenario_wrongoutline.feature | 14 ++ .../testscenario/scenario_wrongstep.feature | 13 + .../local/testscenario/parsedfeature_test.php | 216 ++++++++++++++++ .../tests/local/testscenario/runner_test.php | 107 ++++++++ .../local/testscenario/steprunner_test.php | 238 ++++++++++++++++++ 20 files changed, 1785 insertions(+), 1 deletion(-) create mode 100644 admin/tool/generator/classes/form/featureimport.php create mode 100644 admin/tool/generator/classes/local/testscenario/parsedfeature.php create mode 100644 admin/tool/generator/classes/local/testscenario/runner.php create mode 100644 admin/tool/generator/classes/local/testscenario/steprunner.php create mode 100644 admin/tool/generator/classes/output/parsingresult.php create mode 100644 admin/tool/generator/cli/runtestscenario.php create mode 100644 admin/tool/generator/runtestscenario.php create mode 100644 admin/tool/generator/templates/parsingresult.mustache create mode 100644 admin/tool/generator/tests/behat/testscenario.feature create mode 100644 admin/tool/generator/tests/fixtures/testscenario/scenario.feature create mode 100644 admin/tool/generator/tests/fixtures/testscenario/scenario_wrongempty.feature create mode 100644 admin/tool/generator/tests/fixtures/testscenario/scenario_wrongformat.feature create mode 100644 admin/tool/generator/tests/fixtures/testscenario/scenario_wrongonlybackground.feature create mode 100644 admin/tool/generator/tests/fixtures/testscenario/scenario_wrongoutline.feature create mode 100644 admin/tool/generator/tests/fixtures/testscenario/scenario_wrongstep.feature create mode 100644 admin/tool/generator/tests/local/testscenario/parsedfeature_test.php create mode 100644 admin/tool/generator/tests/local/testscenario/runner_test.php create mode 100644 admin/tool/generator/tests/local/testscenario/steprunner_test.php diff --git a/admin/tool/generator/classes/form/featureimport.php b/admin/tool/generator/classes/form/featureimport.php new file mode 100644 index 00000000000..e5c121d58f8 --- /dev/null +++ b/admin/tool/generator/classes/form/featureimport.php @@ -0,0 +1,60 @@ +. + +namespace tool_generator\form; + +defined('MOODLE_INTERNAL') || die(); + +use moodleform; + +global $CFG; +require_once($CFG->dirroot . '/lib/formslib.php'); + +/** + * Form for importting a testing scenario feature file. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class featureimport extends moodleform { + public function definition(): void { + $mform = &$this->_form; + + // File upload. + $mform->addElement( + 'filepicker', + 'featurefile', + get_string('testscenario_file', 'tool_generator'), + null, + ['accepted_types' => ['.feature']] + ); + $mform->addRule('featurefile', null, 'required'); + $this->add_action_buttons(false, get_string('import')); + } + + /** + * Get the feature file contents. + * @return string|null the feature file contents or null if not found. + */ + public function get_feature_contents(): ?string { + $result = $this->get_file_content('featurefile'); + if (!$result) { + return null; + } + return $result; + } +} diff --git a/admin/tool/generator/classes/local/testscenario/parsedfeature.php b/admin/tool/generator/classes/local/testscenario/parsedfeature.php new file mode 100644 index 00000000000..83d174252a9 --- /dev/null +++ b/admin/tool/generator/classes/local/testscenario/parsedfeature.php @@ -0,0 +1,123 @@ +. + +namespace tool_generator\local\testscenario; + +use stdClass; + +/** + * Class with a scenario feature parsed. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class parsedfeature { + /** @var int the number of steps. */ + private int $stepcount = 0; + + /** @var bool if the parser is ok or fail. */ + private bool $isvalid = true; + + /** @var stdClass[] the list of scenarios with all the steps. + * + * scenarionum => {type: string, title: string, steps: steprunner[]}. + */ + private array $scenarios = []; + + /** + * Get the general error, if any. + * @return string + */ + public function get_general_error(): string { + if (!$this->isvalid) { + return get_string('testscenario_invalidfile', 'tool_generator'); + } + if ($this->stepcount == 0) { + return get_string('testscenario_nosteps', 'tool_generator'); + } + return ''; + } + + /** + * Check if the parsed feature is valid. + * @return bool + */ + public function is_valid(): bool { + return $this->isvalid && $this->stepcount > 0; + } + + /** + * Add a line to the current scenario. + * @param steprunner $step the step to add. + */ + public function add_step(steprunner $step) { + if (empty($this->scenarios)) { + $this->add_scenario('scenario', null); + } + $currentscenario = count($this->scenarios) - 1; + $this->scenarios[$currentscenario]->steps[] = $step; + $this->stepcount++; + if (!$step->is_valid()) { + $this->isvalid = false; + } + } + + /** + * Insert a new scenario. + * @param string $type the type of the scenario. + * @param string|null $name the name of the scenario. + */ + public function add_scenario(string $type, ?string $name) { + $this->scenarios[] = (object) [ + 'type' => $type, + 'name' => $name ?? '', + 'steps' => [], + 'error' => '', + ]; + } + + /** + * Add an error to the current scenario. + * @param string $error + */ + public function add_error(string $error) { + $currentscenario = count($this->scenarios) - 1; + $this->scenarios[$currentscenario]->error = $error; + } + + /** + * Get the list of scenarios. + * @return stdClass[] array of scenarionum => {type: string, title: string, steps: steprunner[]} + */ + public function get_scenarios(): array { + return $this->scenarios; + } + + /** + * Get all the steps form all scenarios. + * @return steprunner[] + */ + public function get_all_steps(): array { + $result = []; + foreach ($this->scenarios as $scenario) { + foreach ($scenario->steps as $step) { + $result[] = $step; + } + } + return $result; + } +} diff --git a/admin/tool/generator/classes/local/testscenario/runner.php b/admin/tool/generator/classes/local/testscenario/runner.php new file mode 100644 index 00000000000..ea9d0456259 --- /dev/null +++ b/admin/tool/generator/classes/local/testscenario/runner.php @@ -0,0 +1,197 @@ +. + +namespace tool_generator\local\testscenario; + +use behat_data_generators; +use Behat\Gherkin\Parser; +use Behat\Gherkin\Lexer; +use Behat\Gherkin\Keywords\ArrayKeywords; +use ReflectionClass; +use ReflectionMethod; + +/** + * Class to process a scenario generator file. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class runner { + + /** @var behat_data_generators the behat data generator instance. */ + private behat_data_generators $generator; + + /** @var array of valid steps indexed by given expression tag. */ + private array $validsteps; + + /** + * Initi all composer, behat libraries and load the valid steps. + */ + public function init() { + $this->include_composer_libraries(); + $this->include_behat_libraries(); + $this->load_generator(); + } + + /** + * Include composer autload. + */ + public function include_composer_libraries() { + global $CFG; + if (!file_exists($CFG->dirroot . '/vendor/autoload.php')) { + throw new \moodle_exception('Missing composer.'); + } + require_once($CFG->dirroot . '/vendor/autoload.php'); + return true; + } + + /** + * Include all necessary behat libraries. + */ + public function include_behat_libraries() { + global $CFG; + if (!class_exists('Behat\Gherkin\Lexer')) { + throw new \moodle_exception('Missing behat classes.'); + } + // Behat utilities. + require_once($CFG->libdir . '/behat/classes/util.php'); + require_once($CFG->libdir . '/behat/classes/behat_command.php'); + require_once($CFG->libdir . '/behat/behat_base.php'); + require_once("{$CFG->libdir}/tests/behat/behat_data_generators.php"); + return true; + } + + /** + * Load all generators. + */ + private function load_generator() { + $this->generator = new behat_data_generators(); + $this->validsteps = $this->scan_generator($this->generator); + } + + /** + * Scan a generator to get all valid steps. + * @param behat_data_generators $generator the generator to scan. + * @return array the valid steps. + */ + private function scan_generator(behat_data_generators $generator): array { + $result = []; + $class = new ReflectionClass($generator); + $methods = $class->getMethods(ReflectionMethod::IS_PUBLIC); + foreach ($methods as $method) { + $given = $this->get_method_given($method); + if ($given) { + $result[$given] = $method->getName(); + } + } + return $result; + } + + /** + * Get the given expression tag of a method. + * + * @param ReflectionMethod $method the method to get the given expression tag. + * @return string|null the given expression tag or null if not found. + */ + private function get_method_given(ReflectionMethod $method): ?string { + $doccomment = $method->getDocComment(); + $doccomment = str_replace("\r\n", "\n", $doccomment); + $doccomment = str_replace("\r", "\n", $doccomment); + $doccomment = explode("\n", $doccomment); + foreach ($doccomment as $line) { + $matches = []; + if (preg_match('/.*\@(given|when|then)\s+(.+)$/i', $line, $matches)) { + return $matches[2]; + } + } + return null; + } + + /** + * Parse a feature file. + * @param string $content the feature file content. + * @return parsedfeature + */ + public function parse_feature(string $content): parsedfeature { + $result = new parsedfeature(); + + $parser = $this->get_parser(); + $feature = $parser->parse($content); + + // No need for background in testing scenarios because scenarios can only contain generators. + // In the future the background can be used to define clean up steps (when clean up methods + // are implemented). + if ($feature->hasScenarios()) { + $scenarios = $feature->getScenarios(); + foreach ($scenarios as $scenario) { + if ($scenario->getNodeType() == 'Outline') { + $result->add_scenario($scenario->getNodeType(), $scenario->getTitle()); + $result->add_error(get_string('testscenario_outline', 'tool_generator')); + continue; + } + $result->add_scenario($scenario->getNodeType(), $scenario->getTitle()); + $steps = $scenario->getSteps(); + foreach ($steps as $step) { + $result->add_step(new steprunner($this->generator, $this->validsteps, $step)); + } + } + } + return $result; + } + + /** + * Get the parser. + * @return Parser + */ + private function get_parser(): Parser { + $keywords = new ArrayKeywords([ + 'en' => [ + 'feature' => 'Feature', + // If in the future we have clean up steps, background will be renamed to "Clean up". + 'background' => 'Background', + 'scenario' => 'Scenario', + 'scenario_outline' => 'Scenario Outline|Scenario Template', + 'examples' => 'Examples|Scenarios', + 'given' => 'Given', + 'when' => 'When', + 'then' => 'Then', + 'and' => 'And', + 'but' => 'But', + ], + ]); + $lexer = new Lexer($keywords); + $parser = new Parser($lexer); + return $parser; + } + + /** + * Execute a parsed feature. + * @param parsedfeature $parsedfeature the parsed feature to execute. + * @return bool true if all steps were executed successfully. + */ + public function execute(parsedfeature $parsedfeature): bool { + if (!$parsedfeature->is_valid()) { + return false; + } + $result = true; + $steps = $parsedfeature->get_all_steps(); + foreach ($steps as $step) { + $result = $step->execute() && $result; + } + return $result; + } +} diff --git a/admin/tool/generator/classes/local/testscenario/steprunner.php b/admin/tool/generator/classes/local/testscenario/steprunner.php new file mode 100644 index 00000000000..e315640aad5 --- /dev/null +++ b/admin/tool/generator/classes/local/testscenario/steprunner.php @@ -0,0 +1,217 @@ +. + +namespace tool_generator\local\testscenario; + +use behat_data_generators; +use Behat\Gherkin\Node\StepNode; + +/** + * Class to validate and process a scenario step. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class steprunner { + /** @var behat_data_generators the behat data generator instance. */ + private behat_data_generators $generator; + + /** @var array the valid steps indexed by given expression tag. */ + private array $validsteps; + + /** @var StepNode the step node to process. */ + private StepNode $stepnode; + + /** @var string|null the generator method to call. */ + private ?string $method = null; + + /** @var array the parameters to pass to the generator method. */ + private array $params = []; + + /** @var bool if the step is valid. */ + private bool $isvalid = false; + + /** @var bool if the step has been executed. */ + private bool $executed = false; + + /** @var string the error message if any. */ + private string $error = ''; + + /** + * Constructor. + * @param behat_data_generators $generator the behat data generator instance. + * @param array $validsteps the valid steps indexed by given expression tag. + * @param StepNode $stepnode the step node to process. + */ + public function __construct(behat_data_generators $generator, array $validsteps, StepNode $stepnode) { + $this->generator = $generator; + $this->validsteps = $validsteps; + $this->stepnode = $stepnode; + $this->init(); + } + + /** + * Init the step runner. + * + * This method will check if the step is valid and all the needed information + * in case it is executed. + */ + private function init() { + $matches = []; + $linetext = $this->stepnode->getText(); + foreach ($this->validsteps as $pattern => $method) { + if (!$this->match_given($pattern, $linetext, $matches)) { + continue; + } + $this->method = $method; + $this->params = $this->build_method_params($method, $matches); + $this->isvalid = true; + return; + } + $this->error = get_string('testscenario_invalidstep', 'tool_generator'); + } + + /** + * Build the method parameters. + * @param string $methodname the method name. + * @param array $matches the matches. + * @return array the method parameters. + */ + private function build_method_params($methodname, $matches) { + $method = new \ReflectionMethod($this->generator, $methodname); + $params = []; + foreach ($method->getParameters() as $param) { + $paramname = $param->getName(); + if (isset($matches[$paramname])) { + $params[] = $matches[$paramname]; + unset($matches[$paramname]); + } else if (count($matches) > 0) { + // If the param is not present means the regular expressions does not use + // proper names. So we will try to find the param by position. + $params[] = array_pop($matches); + } else { + // No more params to match. + break; + } + } + return array_merge($params, $this->stepnode->getArguments()); + } + + /** + * Return if the step is valid. + * @return bool + */ + public function is_valid(): bool { + return $this->isvalid; + } + + /** + * Return if the step has been executed. + * @return bool + */ + public function is_executed(): bool { + return $this->executed; + } + + /** + * Return the step text. + * @return string + */ + public function get_text(): string { + return $this->stepnode->getText(); + } + + /** + * Return the step error message. + * @return string + */ + public function get_error(): string { + return $this->error; + } + + /** + * Return the step arguments as string. + * @return string + */ + public function get_arguments_string(): string { + $result = ''; + foreach ($this->stepnode->getArguments() as $argument) { + $result .= $argument->getTableAsString(); + } + return $result; + } + + /** + * Match a given expression with a text. + * @param string $pattern the given expression. + * @param string $text the text to match. + * @param array $matches the matches. + * @return bool if the step matched the generator given expression. + */ + private function match_given(string $pattern, $text, array &$matches) { + $internalmatcher = []; + if (substr($pattern, 0, 1) === '/') { + // Pattern is a regular expression. + $result = preg_match($pattern, $text, $matches); + foreach ($matches as $key => $value) { + if (is_int($key)) { + unset($matches[$key]); + } + } + return $result; + } + + // Patter is a string with parameters. + $elementmatches = []; + preg_match_all('/:([^ ]+)/', $pattern, $elementmatches, PREG_SET_ORDER, 0); + + $pattern = preg_replace('/:([^ ]+)/', '(?P<$1>"[^"]+"|[^" ]+)', $pattern); + $pattern = '/^' . $pattern . '$/'; + $result = preg_match($pattern, $text, $internalmatcher); + if (!$result) { + return false; + } + foreach ($elementmatches as $elementmatch) { + // Remove any possible " at the beggining and end of $internalmatcher[$elementmatch[1]]. + $paramvalue = preg_replace('/^"(.*)"$/', '$1', $internalmatcher[$elementmatch[1]]); + $matches[$elementmatch[1]] = $paramvalue; + } + return true; + } + + /** + * Execute the step. + * @return bool if the step is executed or not. + */ + public function execute(): bool { + if (!$this->isvalid) { + return false; + } + $this->executed = true; + try { + call_user_func_array( + [$this->generator, $this->method], + $this->params + ); + } catch (\moodle_exception $exception) { + $this->error = $exception->getMessage(); + $this->isvalid = false; + return false; + } + return true; + } +} diff --git a/admin/tool/generator/classes/output/parsingresult.php b/admin/tool/generator/classes/output/parsingresult.php new file mode 100644 index 00000000000..cdf99520db4 --- /dev/null +++ b/admin/tool/generator/classes/output/parsingresult.php @@ -0,0 +1,88 @@ +. + +namespace tool_generator\output; + +use renderable; +use renderer_base; +use templatable; +use tool_generator\local\testscenario\parsedfeature; + +/** + * A report to show the feature file parsing process. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class parsingresult implements renderable, templatable { + + /** @var parsedfeature the processed feature object. */ + protected $parsedfeature; + + /** + * Constructor. + * + * @param parsedfeature $parsedfeature + */ + public function __construct(parsedfeature $parsedfeature) { + $this->parsedfeature = $parsedfeature; + } + + /** + * Export for template. + * + * @param renderer_base $output The renderer. + * @return array + */ + public function export_for_template(renderer_base $output): array { + $data = [ + 'scenarios' => [], + 'isvalid' => $this->parsedfeature->is_valid(), + 'generalerror' => $this->parsedfeature->get_general_error(), + ]; + $haslines = false; + foreach ($this->parsedfeature->get_scenarios() as $scenario) { + $scenariodata = [ + 'type' => $scenario->type, + 'name' => $scenario->name, + 'steps' => [], + ]; + if (!empty($scenario->error)) { + $scenariodata['scenarioerror'] = $scenario->error; + } + foreach ($scenario->steps as $step) { + $scenariodata['steps'][] = [ + 'text' => $step->get_text(), + 'arguments' => $step->get_arguments_string(), + 'hasarguments' => !empty($step->get_arguments_string()), + 'isvalid' => $step->is_valid(), + 'error' => $step->get_error(), + 'isexecuted' => $step->is_executed(), + ]; + $haslines = true; + } + if (!empty($scenariodata['steps'])) { + $scenariodata['hassteps'] = true; + } + $data['scenarios'][] = $scenariodata; + } + if ($haslines) { + $data['haslines'] = $haslines; + } + return $data; + } +} diff --git a/admin/tool/generator/cli/runtestscenario.php b/admin/tool/generator/cli/runtestscenario.php new file mode 100644 index 00000000000..efe47811c4f --- /dev/null +++ b/admin/tool/generator/cli/runtestscenario.php @@ -0,0 +1,178 @@ +. + +/** + * Run a test scenario generator feature file. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +if (isset($_SERVER['REMOTE_ADDR'])) { + die(); // No access from web! +} + +define('CLI_SCRIPT', true); + +require_once(__DIR__ . '/../../../../config.php'); +require_once(__DIR__ . '/../../../../lib/clilib.php'); +require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php'); +require_once(__DIR__ . '/../../../../lib/testing/lib.php'); + +ini_set('display_errors', '1'); +ini_set('log_errors', '1'); + +list($options, $unrecognised) = cli_get_params( + [ + 'help' => false, + 'feature' => '', + 'disable-composer' => false, + 'composer-upgrade' => true, + 'composer-self-update' => true, + ], + [ + 'h' => 'help', + 'f' => 'feature', + ] +); + +// Checking run.php CLI script usage. +$help = " +Run a feature file into the current Moodle instance. The feature file can only +contains scenarios with core_data_generator steps. It is not yet compatible +with scenario outlines. All scenarios will be executed at once, event background +steps. + +Usage: + php runtestscenario.php [--feature=\"value\"] [--help] + [--no-composer-self-update] [--no-composer-upgrade] + [--disable-composer] + +Options: +-f, --feature Execute specified feature file (Absolute path of feature file). + +--no-composer-self-update +Prevent upgrade of the composer utility using its self-update command + +--no-composer-upgrade +Prevent update development dependencies using composer + +--disable-composer +A shortcut to disable composer self-update and dependency update +Note: Installation of composer and/or dependencies will still happen as required + +-h, --help Print out this help + +Example from Moodle root directory: +\$ php admin/tool/generator/cli/runtestscenario.php --feature=/path/to/some/testing/scenario.feature +"; + +if (!empty($options['help'])) { + echo $help; + exit(0); +} + +// The command will install composer if not present. Usually composer libraries are +// installed when behat or phpunit are installed, but we do not want to force users +// to create all phpunit or behat databases tables just to run a test scenario locally. +if (!file_exists($CFG->dirroot . '/vendor/autoload.php')) { + // Force OPcache reset if used, we do not want any stale caches + // when preparing test environment. + if (function_exists('opcache_reset')) { + opcache_reset(); + } + + if ($options['disable-composer']) { + // Disable self-update and upgrade easily. + // Note: Installation will still occur regardless of this setting. + $options['composer-self-update'] = false; + $options['composer-upgrade'] = false; + } + + // Install and update composer and dependencies as required. + testing_update_composer_dependencies($options['composer-self-update'], $options['composer-upgrade']); +} + +if (empty($options['feature'])) { + echo "Missing feature file path.\n"; + exit(0); +} + +$featurefile = $options['feature']; +if (!file_exists($featurefile)) { + echo "Feature file not found.\n"; + exit(0); +} + +$runner = new tool_generator\local\testscenario\runner(); + +try { + $runner->init(); +} catch (Exception $e) { + echo "Something is wrong with the behat setup.\n"; + echo " Please,try running \"php admin/tool/behat/cli/init.php\" from your Moodle root directory.\n"; + exit(0); +} + +$content = file_get_contents($featurefile); + +if (empty($content)) { + echo "The feature file is empty.\n"; + exit(0); +} + +try { + $parsedfeature = $runner->parse_feature($content); +} catch (\Exception $error) { + echo "Error parsing feature file: {$error->getMessage()}\n"; + echo "Use the web version of the tool to see the parsing details:\n"; + echo " Site administration -> development -> Create testing scenarios\n"; + exit(0); +} + +if (!$parsedfeature->is_valid()) { + echo "The file is not valid: {$parsedfeature->get_general_error()}\n"; + echo "Use the web version of the tool to see the details:\n"; + echo " Site administration -> development -> Create testing scenarios\n"; + exit(0); +} + +$total = 0; +$success = 0; + +foreach ($parsedfeature->get_all_steps() as $step) { + if ($step->execute()) { + echo "\nOK: {$step->get_text()}\n"; + echo "{$step->get_arguments_string()}\n"; + $success++; + } else { + echo "\nFAIL: {$step->get_text()}\n"; + echo "{$step->get_arguments_string()}\n"; + echo "{$step->get_error()}\n"; + } + $total++; +} + +echo "\n{$success}/{$total} steps executed successfully.\n"; + +if ($success < $total) { + echo "\nSome steps failed.\n"; + exit(1); +} else { + echo "\nAll steps executed successfully.\n"; + exit(0); +} diff --git a/admin/tool/generator/lang/en/tool_generator.php b/admin/tool/generator/lang/en/tool_generator.php index d9365a6d062..8bd2347f9d5 100644 --- a/admin/tool/generator/lang/en/tool_generator.php +++ b/admin/tool/generator/lang/en/tool_generator.php @@ -95,6 +95,18 @@ $string['sitesize_5'] = 'XXL (~20GB; 4177 courses, created in ~10 hours)'; $string['size'] = 'Size of course'; $string['smallfiles'] = 'Small files'; $string['targetcourse'] = 'Test target course'; +$string['testscenario'] = 'Create testing scenarios'; +$string['testscenario_description'] = 'Creating testing scenarios uses a limited feature files syntax to create all necessary elements to run a manual test.'; +$string['testscenario_filedesc'] = 'The upload feature files can only contain scenarios with core_data_generator steps. It is not yet compatible with scenario outlines. All scenarios will be executed at once but background will be ignored.'; +$string['testscenario_errorparsing'] = 'Error parsing feature file: {$a}'; +$string['testscenario_file'] = 'Feature file'; +$string['testscenario_invalidfile'] = 'The file format is not valid or contains invalid steps.'; +$string['testscenario_invalidstep'] = 'Unkown step. Create testing scenarios only accepts generator steps.'; +$string['testscenario_outline'] = 'Scenario outlines are not supported.'; +$string['testscenario_nosteps'] = 'There are no steps to execute in the file.'; +$string['testscenario_notready'] = 'Composer and behat libraries are not installed.

Please, execute this command to enable this tool: php admin/tool/generator/cli/runtestscenario.php'; +$string['testscenario_scenarionosteps'] = 'This scenario does not have any steps.'; +$string['testscenario_steps'] = 'Testing scenario steps:'; $string['testplanexplanation'] = 'This tool creates a JMeter test plan file along with the user credentials file. This test plan is designed to work along with {$a}, which makes easier to run the test plan in a specific Moodle environment, gathers information about the runs and compares the results, so you will need to download it and use it\'s test_runner.sh script or follow the installation and usage instructions. diff --git a/admin/tool/generator/runtestscenario.php b/admin/tool/generator/runtestscenario.php new file mode 100644 index 00000000000..c1253151629 --- /dev/null +++ b/admin/tool/generator/runtestscenario.php @@ -0,0 +1,95 @@ +. + +/** + * Web interface to list and filter steps + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +use tool_generator\local\testscenario\runner; +use tool_generator\form\featureimport; +use tool_generator\output\parsingresult; + +require(__DIR__ . '/../../../config.php'); +require_once($CFG->libdir . '/adminlib.php'); +require_once($CFG->libdir . '/behat/classes/behat_config_manager.php'); + +// Executing behat generator can take some time. +core_php_time_limit::raise(300); + +admin_externalpage_setup('toolgenerator_runtestscenario'); + +$currenturl = new moodle_url('/admin/tool/generator/runtestscenario.php'); +$runner = new runner(); + +/** @var core_renderer $output*/ +$output = $PAGE->get_renderer('core'); + +echo $output->header(); +echo $output->heading(get_string('testscenario', 'tool_generator')); + +echo $output->paragraph(get_string('testscenario_description', 'tool_generator')); + +try { + $runner->init(); +} catch (Exception $e) { + echo $output->notification(get_string('testscenario_notready', 'tool_generator'), null, false); + echo $output->footer(); + die; +} + +echo $output->paragraph(get_string('testscenario_filedesc', 'tool_generator')); + +$mform = new featureimport(); + +$data = null; +if (!$mform->is_cancelled()) { + $data = $mform->get_data(); +} + +if (empty($data)) { + $mform->display(); + echo $OUTPUT->footer(); + die; +} + +$content = $mform->get_feature_contents(); + +if (empty($content)) { + echo $output->notification(get_string('testscenario_invalidfile', 'tool_generator')); + echo $output->continue_button($currenturl); + echo $output->footer(); + die; +} + +try { + $parsedfeature = $runner->parse_feature($content); +} catch (\Throwable $th) { + echo $output->notification(get_string('testscenario_errorparsing', 'tool_generator', $th->getMessage())); + echo $output->continue_button($currenturl); + + echo $output->footer(); + die; +} + +$runner->execute($parsedfeature); + +echo $output->render(new parsingresult($parsedfeature)); +echo $output->continue_button($currenturl); +echo $output->footer(); diff --git a/admin/tool/generator/settings.php b/admin/tool/generator/settings.php index f1f2350fe3a..e7322f71aed 100644 --- a/admin/tool/generator/settings.php +++ b/admin/tool/generator/settings.php @@ -32,5 +32,13 @@ if ($hassiteconfig) { $ADMIN->add('development', new admin_externalpage('toolgeneratortestplan', get_string('maketestplan', 'tool_generator'), $CFG->wwwroot . '/' . $CFG->admin . '/tool/generator/maketestplan.php')); -} + $ADMIN->add( + 'development', + new admin_externalpage( + 'toolgenerator_runtestscenario', + get_string('testscenario', 'tool_generator'), + $CFG->wwwroot . '/' . $CFG->admin . '/tool/generator/runtestscenario.php' + ) + ); +} diff --git a/admin/tool/generator/templates/parsingresult.mustache b/admin/tool/generator/templates/parsingresult.mustache new file mode 100644 index 00000000000..244996b3af3 --- /dev/null +++ b/admin/tool/generator/templates/parsingresult.mustache @@ -0,0 +1,106 @@ +{{! + This file is part of Moodle - https://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template tool_generator/parsingresult + + Display the result of parsing or executing a behat testing scenario. + + Example context (json): + { + "isvalid": true, + "generalerror": "Error message", + "haslines": true, + "scenarios": [ + { + "type": "Scenario", + "title": "Scenario title", + "hassteps": true, + "steps": [ + { + "text": "Step text", + "isvalid": true, + "isexecuted": true, + "hasarguments": true, + "arguments": "Argument text", + "error": "Error message" + } + ] + } + ] + } +}} +{{^isvalid}} + +{{/isvalid}} +{{#haslines}} +

{{#str}} testscenario_steps, tool_generator {{/str}}

+{{/haslines}} +{{#scenarios}} +

{{type}}: {{name}}

+
+ {{#scenarioerror}} + + {{/scenarioerror}} + {{^hassteps}} + + {{/hassteps}} + {{#hassteps}} +
    + {{#steps}} +
  • +
    + {{text}} + {{^isvalid}} + + + {{#str}} error {{/str}} + + {{/isvalid}} + {{#isvalid}} + {{#isexecuted}} + + + {{#str}} success {{/str}} + + {{/isexecuted}} + {{/isvalid}} +
    + {{#hasarguments}} + + {{/hasarguments}} + {{^isvalid}} + + {{/isvalid}} +
    • + {{/steps}} +
+ {{/hassteps}} +
+{{/scenarios}} diff --git a/admin/tool/generator/tests/behat/testscenario.feature b/admin/tool/generator/tests/behat/testscenario.feature new file mode 100644 index 00000000000..588ac3413c4 --- /dev/null +++ b/admin/tool/generator/tests/behat/testscenario.feature @@ -0,0 +1,64 @@ +@tool @tool_generator @_file_upload +Feature: Create testing scenarios using generators + In order to execute manual tests + As a developer + I need to use a feature file to execute generators into the current instance + + @javascript + Scenario: Create a testing scenario with a course enrolled users and activities + Given I log in as "admin" + And I navigate to "Development > Create testing scenarios" in site administration + When I upload "admin/tool/generator/tests/fixtures/testscenario/scenario.feature" file to "Feature file" filemanager + And I press "Import" + And I should see "Scenario: Create course content" + Then I am on the "C1" "Course" page + And I should see "Activity sample 1" in the "Topic 1" "section" + And I should see "Activity sample 2" in the "Topic 1" "section" + And I navigate to course participants + And I should see "Teacher Test1" + And I should see "Student Test1" + And I should see "Student Test2" + And I should see "Student Test3" + And I should see "Student Test4" + And I should see "Student Test5" + + @javascript + Scenario: Prevent creating a testing scenario with no steps to execute + Given I log in as "admin" + And I navigate to "Development > Create testing scenarios" in site administration + When I upload "admin/tool/generator/tests/fixtures/testscenario/scenario_wrongempty.feature" file to "Feature file" filemanager + And I press "Import" + Then I should see "There are no steps to execute in the file." + + @javascript + Scenario: Prevent creating a testing scenario with only background steps to execute + Given I log in as "admin" + And I navigate to "Development > Create testing scenarios" in site administration + When I upload "admin/tool/generator/tests/fixtures/testscenario/scenario_wrongonlybackground.feature" file to "Feature file" filemanager + And I press "Import" + Then I should see "There are no steps to execute in the file." + + @javascript + Scenario: Prevent creating a testing scenario with a wrong file format + Given I log in as "admin" + And I navigate to "Development > Create testing scenarios" in site administration + When I upload "admin/tool/generator/tests/fixtures/testscenario/scenario_wrongformat.feature" file to "Feature file" filemanager + And I press "Import" + Then I should see "Error parsing feature file" + + @javascript + Scenario: Prevent creating a testing scenario with non generator steps + Given I log in as "admin" + And I navigate to "Development > Create testing scenarios" in site administration + When I upload "admin/tool/generator/tests/fixtures/testscenario/scenario_wrongstep.feature" file to "Feature file" filemanager + And I press "Import" + Then I should see "The file format is not valid or contains invalid steps" + + @javascript + Scenario: Prevent creating a testing scenario from a scenario outline + Given I log in as "admin" + And I navigate to "Development > Create testing scenarios" in site administration + When I upload "admin/tool/generator/tests/fixtures/testscenario/scenario_wrongoutline.feature" file to "Feature file" filemanager + And I press "Import" + Then I should see "Scenario outlines are not supported" + Then I should see "There are no steps to execute in the file" diff --git a/admin/tool/generator/tests/fixtures/testscenario/scenario.feature b/admin/tool/generator/tests/fixtures/testscenario/scenario.feature new file mode 100644 index 00000000000..155806ea63d --- /dev/null +++ b/admin/tool/generator/tests/fixtures/testscenario/scenario.feature @@ -0,0 +1,27 @@ +Feature: Prepare scenario for testing + Scenario: Create course content + Given the following "course" exists: + | fullname | Course test | + | shortname | C1 | + | category | 0 | + | numsections | 3 | + And the following "activities" exist: + | activity | name | intro | course | idnumber | section | visible | + | assign | Activity sample 1 | Test assignment description | C1 | sample1 | 1 | 1 | + | assign | Activity sample 2 | Test assignment description | C1 | sample2 | 1 | 0 | + Scenario: Create users + Given the following "users" exist: + | username | firstname | lastname | email | + | teacher1 | Teacher | Test1 | sample@example.com | + And the following "course enrolments" exist: + | user | course | role | + | teacher1 | C1 | editingteacher | + And "5" "users" exist with the following data: + | username | student[count] | + | firstname | Student | + | lastname | Test[count] | + | email | student[count]@example.com | + And "5" "course enrolments" exist with the following data: + | user | student[count] | + | course | C1 | + | role | student | diff --git a/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongempty.feature b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongempty.feature new file mode 100644 index 00000000000..17c07e478f1 --- /dev/null +++ b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongempty.feature @@ -0,0 +1,4 @@ +Feature: Prepare scenario for testing + Scenario: only empty scenario 1. + Scenario: only empty scenario 2. + diff --git a/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongformat.feature b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongformat.feature new file mode 100644 index 00000000000..c5b2d51bcbb --- /dev/null +++ b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongformat.feature @@ -0,0 +1,5 @@ +This is not a valid feature file. + + However: it has lines + indented and stuff + like if it was. diff --git a/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongonlybackground.feature b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongonlybackground.feature new file mode 100644 index 00000000000..51bdf1022ae --- /dev/null +++ b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongonlybackground.feature @@ -0,0 +1,12 @@ +Feature: Invalid texting because background is not compatible + Background: + Given the following "course" exists: + | fullname | Course 1 | + | shortname | C1 | + | category | 0 | + | numsections | 3 | + And the following "activities" exist: + | activity | name | intro | course | idnumber | section | visible | + | assign | Activity sample 1 | Test assignment description | C1 | sample1 | 1 | 1 | + | assign | Activity sample 2 | Test assignment description | C1 | sample2 | 1 | 0 | + diff --git a/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongoutline.feature b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongoutline.feature new file mode 100644 index 00000000000..904c0a3e209 --- /dev/null +++ b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongoutline.feature @@ -0,0 +1,14 @@ +Feature: Prepare scenario for testing + + Scenario Outline: test outline scenarios are not supported yet + Given the following "course" exists: + | fullname | | + | shortname | | + | category | 0 | + | numsections | 3 | + + Examples: + | name | shortname | + | Course 1 | C1 | + | Course 2 | C2 | + | Course 3 | C3 | diff --git a/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongstep.feature b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongstep.feature new file mode 100644 index 00000000000..70c05450ea8 --- /dev/null +++ b/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongstep.feature @@ -0,0 +1,13 @@ +Feature: Contains wrong steps + Scenario: Scenario with non generator steps + Given the following "course" exists: + | fullname | Course test | + | shortname | C1 | + | category | 0 | + | numsections | 3 | + And the following "activities" exist: + | activity | name | intro | course | idnumber | section | visible | + | assign | Activity sample 1 | Test assignment description | C1 | sample1 | 1 | 1 | + | assign | Activity sample 2 | Test assignment description | C1 | sample2 | 1 | 0 | + And I click on "Tokens filter" "link" + diff --git a/admin/tool/generator/tests/local/testscenario/parsedfeature_test.php b/admin/tool/generator/tests/local/testscenario/parsedfeature_test.php new file mode 100644 index 00000000000..6af109d10ea --- /dev/null +++ b/admin/tool/generator/tests/local/testscenario/parsedfeature_test.php @@ -0,0 +1,216 @@ +. + +namespace tool_generator\local\testscenario; + +/** + * Tests for parsedfeature class. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @coversDefaultClass \tool_generator\local\testscenario\parsedfeature + */ +class parsedfeature_test extends \advanced_testcase { + /** + * Get a parsed feature from a content. + * @param string $content the feature content. + * @return parsedfeature the parsed feature. + */ + private function get_feature_from_content(string $content): parsedfeature { + $runner = new runner(); + $runner->init(); + return $runner->parse_feature($content); + } + + /** + * Test for parse_feature. + * @covers ::get_general_error + * @covers ::add_scenario + * @covers ::add_step + */ + public function test_general_error(): void { + $nosteps = get_string('testscenario_nosteps', 'tool_generator'); + $invalidfile = get_string('testscenario_invalidfile', 'tool_generator'); + + $parsedfeature = new parsedfeature(); + $this->assertEquals($nosteps, $parsedfeature->get_general_error()); + + $parsedfeature->add_scenario('Scenario', 'Test scenario'); + $this->assertEquals($nosteps, $parsedfeature->get_general_error()); + + // Add some valid step. + $extrafeature = $this->get_feature_from_content('Feature: Test feature + Scenario: Create users + Given the following "users" exist: + | username | firstname | lastname | email | + | teacher1 | Teacher | Test1 | sample@example.com | + '); + $step = $extrafeature->get_all_steps()[0]; + $parsedfeature->add_step($step); + $this->assertEquals('', $parsedfeature->get_general_error()); + + // Only generator methods are allowed. + $extrafeature = $this->get_feature_from_content('Feature: Test feature + Scenario: Not generator + Given I am in a course + '); + $step = $extrafeature->get_all_steps()[0]; + $parsedfeature->add_step($step); + $this->assertEquals($invalidfile, $parsedfeature->get_general_error()); + } + + /** + * Test for parse_feature. + * @covers ::is_valid + * @covers ::add_scenario + * @covers ::add_step + */ + public function test_is_valid(): void { + $parsedfeature = new parsedfeature(); + $this->assertFalse($parsedfeature->is_valid()); + + $parsedfeature->add_scenario('Scenario', 'Test scenario'); + $this->assertFalse($parsedfeature->is_valid()); + + // Add some valid step. + $extrafeature = $this->get_feature_from_content('Feature: Test feature + Scenario: Create users + Given the following "users" exist: + | username | firstname | lastname | email | + | teacher1 | Teacher | Test1 | sample@example.com | + '); + $step = $extrafeature->get_all_steps()[0]; + $parsedfeature->add_step($step); + $this->assertTrue($parsedfeature->is_valid()); + + // Only generator methods are allowed. + $extrafeature = $this->get_feature_from_content('Feature: Test feature + Scenario: Not generator + Given I am in a course + '); + $step = $extrafeature->get_all_steps()[0]; + $parsedfeature->add_step($step); + $this->assertFalse($parsedfeature->is_valid()); + } + + /** + * Test for ading steps into scenarios. + * @covers ::add_step + * @covers ::add_scenario + * @covers ::get_all_steps + * @covers ::get_scenarios + */ + public function test_add_step(): void { + $parsedfeature = new parsedfeature(); + $this->assertEquals(0, count($parsedfeature->get_all_steps())); + + $extrafeature = $this->get_feature_from_content('Feature: Test feature + Scenario: Create users + Given the following "users" exist: + | username | firstname | lastname | email | + | teacher1 | Teacher | Test1 | sample@example.com | + '); + $step = $extrafeature->get_all_steps()[0]; + $parsedfeature->add_step($step); + $this->assertEquals(1, count($parsedfeature->get_all_steps())); + + // Should create a default scenario. + $scenarios = $parsedfeature->get_scenarios(); + $this->assertEquals(1, count($scenarios)); + $this->assertEquals('scenario', $scenarios[0]->type); + $this->assertEquals('', $scenarios[0]->name); + $this->assertEquals(1, count($scenarios[0]->steps)); + $this->assertEquals($step, $scenarios[0]->steps[0]); + $this->assertEquals('', $scenarios[0]->error); + + // Add a second step. + $extrafeature = $this->get_feature_from_content('Feature: Test feature + Scenario: Not generator + Given I am in a course + '); + $step2 = $extrafeature->get_all_steps()[0]; + $parsedfeature->add_step($step2); + $scenarios = $parsedfeature->get_scenarios(); + $this->assertEquals(1, count($scenarios)); + $this->assertEquals('scenario', $scenarios[0]->type); + $this->assertEquals('', $scenarios[0]->name); + $this->assertEquals(2, count($scenarios[0]->steps)); + $this->assertEquals($step, $scenarios[0]->steps[0]); + $this->assertEquals($step2, $scenarios[0]->steps[1]); + $this->assertEquals('', $scenarios[0]->error); + + // Create a new scenario. + $parsedfeature->add_scenario('scenario', 'Test scenario 2'); + $parsedfeature->add_step($step2); + $scenarios = $parsedfeature->get_scenarios(); + $this->assertEquals(2, count($scenarios)); + // Scenario 1. + $this->assertEquals('scenario', $scenarios[0]->type); + $this->assertEquals('', $scenarios[0]->name); + $this->assertEquals(2, count($scenarios[0]->steps)); + $this->assertEquals($step, $scenarios[0]->steps[0]); + $this->assertEquals($step2, $scenarios[0]->steps[1]); + $this->assertEquals('', $scenarios[0]->error); + // Scenario 2. + $this->assertEquals('scenario', $scenarios[1]->type); + $this->assertEquals('Test scenario 2', $scenarios[1]->name); + $this->assertEquals(1, count($scenarios[1]->steps)); + $this->assertEquals($step2, $scenarios[1]->steps[0]); + $this->assertEquals('', $scenarios[1]->error); + } + + /** + * Test for ading errors into scenarios. + * @covers ::add_error + * @covers ::add_scenario + * @covers ::add_step + * @covers ::get_scenarios + */ + public function test_add_error(): void { + $parsedfeature = new parsedfeature(); + + // Add some valid step. + $extrafeature = $this->get_feature_from_content('Feature: Test feature + Scenario: Create users + Given the following "users" exist: + | username | firstname | lastname | email | + | teacher1 | Teacher | Test1 | sample@example.com | + '); + $step = $extrafeature->get_all_steps()[0]; + + $parsedfeature->add_scenario('scenario', 'Test scenario 1'); + $parsedfeature->add_step($step); + $parsedfeature->add_error('Error message'); + $parsedfeature->add_scenario('scenario', 'Test scenario 2'); + $parsedfeature->add_step($step); + + $scenarios = $parsedfeature->get_scenarios(); + $this->assertEquals(2, count($scenarios)); + // Scenario 1. + $this->assertEquals('scenario', $scenarios[0]->type); + $this->assertEquals('Test scenario 1', $scenarios[0]->name); + $this->assertEquals(1, count($scenarios[0]->steps)); + $this->assertEquals($step, $scenarios[0]->steps[0]); + $this->assertEquals('Error message', $scenarios[0]->error); + // Scenario 2. + $this->assertEquals('scenario', $scenarios[1]->type); + $this->assertEquals('Test scenario 2', $scenarios[1]->name); + $this->assertEquals(1, count($scenarios[1]->steps)); + $this->assertEquals($step, $scenarios[1]->steps[0]); + $this->assertEquals('', $scenarios[1]->error); + } +} diff --git a/admin/tool/generator/tests/local/testscenario/runner_test.php b/admin/tool/generator/tests/local/testscenario/runner_test.php new file mode 100644 index 00000000000..835abeb61b1 --- /dev/null +++ b/admin/tool/generator/tests/local/testscenario/runner_test.php @@ -0,0 +1,107 @@ +. + +namespace tool_generator\local\testscenario; + +/** + * Tests for runner class. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @coversDefaultClass \tool_generator\local\testscenario\runner + */ +class runner_test extends \advanced_testcase { + + /** + * Test for parse_feature. + * @covers ::parse_feature + * @covers ::execute + */ + public function test_parse_and_execute_feature(): void { + global $CFG, $DB; + + $this->resetAfterTest(); + $this->setAdminUser(); + + // Call the init method to include all behat libraries and attributes. + $runner = new runner(); + $runner->init(); + + $featurefile = $CFG->dirroot . '/admin/tool/generator/tests/fixtures/testscenario/scenario.feature'; + $contents = file_get_contents($featurefile); + $feature = $runner->parse_feature($contents); + + $this->assertEquals(2, count($feature->get_scenarios())); + $this->assertEquals(6, count($feature->get_all_steps())); + $this->assertTrue($feature->is_valid()); + + $result = $runner->execute($feature); + $this->assertTrue($result); + + // Validate everything is created. + $this->assertEquals( + 1, + $DB->count_records('course', ['shortname' => 'C1']) + ); + $course = $DB->get_record('course', ['shortname' => 'C1']); + $this->assertEquals( + 2, + $DB->count_records('course_modules', ['course' => $course->id]) + ); + $this->assertEquals( + 1, + $DB->count_records('user', ['firstname' => 'Teacher']) + ); + $this->assertEquals( + 5, + $DB->count_records('user', ['firstname' => 'Student']) + ); + $context = \context_course::instance($course->id); + $this->assertEquals( + 6, + $DB->count_records('role_assignments', ['contextid' => $context->id]) + ); + } + + /** + * Test for parse_feature. + * @covers ::parse_feature + * @covers ::execute + */ + public function test_parse_and_execute_wrong_feature(): void { + global $CFG, $DB; + + $this->resetAfterTest(); + $this->setAdminUser(); + + // Call the init method to include all behat libraries and attributes. + $runner = new runner(); + $runner->init(); + + $featurefile = $CFG->dirroot . '/admin/tool/generator/tests/fixtures/testscenario/scenario_wrongstep.feature'; + $contents = file_get_contents($featurefile); + $feature = $runner->parse_feature($contents); + + $this->assertEquals(1, count($feature->get_scenarios())); + $this->assertEquals(3, count($feature->get_all_steps())); + $this->assertFalse($feature->is_valid()); + + $result = $runner->execute($feature); + $this->assertFalse($result); + $this->assertEquals(0, $DB->count_records('course', ['shortname' => 'C1'])); + } +} diff --git a/admin/tool/generator/tests/local/testscenario/steprunner_test.php b/admin/tool/generator/tests/local/testscenario/steprunner_test.php new file mode 100644 index 00000000000..08d20c567a1 --- /dev/null +++ b/admin/tool/generator/tests/local/testscenario/steprunner_test.php @@ -0,0 +1,238 @@ +. + +namespace tool_generator\local\testscenario; + +use behat_data_generators; +use Behat\Gherkin\Parser; +use Behat\Gherkin\Lexer; +use Behat\Gherkin\Keywords\ArrayKeywords; +use Behat\Gherkin\Node\StepNode; + +/** + * Tests for steprunner class. + * + * @package tool_generator + * @copyright 2023 Ferran Recio + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @coversDefaultClass \tool_generator\local\testscenario\steprunner + */ +class steprunner_test extends \advanced_testcase { + public static function setUpBeforeClass(): void { + // Call the init method to include all behat libraries and attributes. + $runner = new runner(); + $runner->init(); + } + + /** + * Get a step node from a string. + * @param string $step the step string. + * @return StepNode the step node. + */ + private function get_step(string $step): StepNode { + $content = 'Feature: Test feature + Scenario: Test scenario + ' . $step . ' + '; + + $method = new \ReflectionMethod(runner::class, 'get_parser'); + $parser = $method->invoke(new runner()); + + $feature = $parser->parse($content); + $scenario = $feature->getScenarios()[0]; + $steps = $scenario->getSteps(); + return $steps[0]; + } + + /** + * Test for parse_feature. + * @covers ::is_valid + * @param string $step the step to validate. + * @param bool $expected if the step is expected to be valid. + * @dataProvider execute_steps_provider + */ + public function test_is_valid(string $step, bool $expected): void { + $generator = new behat_data_generators(); + $validsteps = [ + '/^the following "(?P(?:[^"]|\\")*)" exist:$/' => 'the_following_entities_exist', + ':count :entitytype exist with the following data:' => 'the_following_repeated_entities_exist', + 'the following :entitytype exists:' => 'the_following_entity_exists', + ]; + + $step = $this->get_step($step); + $steprunner = new steprunner($generator, $validsteps, $step); + $this->assertEquals($expected, $steprunner->is_valid()); + } + + /** + * Test for execute step. + * + * @covers ::is_executed + * @covers ::execute + * @param string $step the step to execute. + * @param bool $expected if the step is expected to be executed. + * @dataProvider execute_steps_provider + */ + public function test_execute(string $step, bool $expected): void { + global $DB; + + $this->resetAfterTest(); + + $generator = new behat_data_generators(); + $validsteps = [ + '/^the following "(?P(?:[^"]|\\")*)" exist:$/' => 'the_following_entities_exist', + ':count :entitytype exist with the following data:' => 'the_following_repeated_entities_exist', + 'the following :entitytype exists:' => 'the_following_entity_exists', + ]; + + $step = $this->get_step($step); + $steprunner = new steprunner($generator, $validsteps, $step); + + $this->assertFalse($steprunner->is_executed()); + + $result = $steprunner->execute(); + + $this->assertEquals($expected, $result); + $this->assertEquals($expected, $steprunner->is_executed()); + + if ($expected) { + // Validate everything is created. + $this->assertEquals( + 1, + $DB->count_records('course', ['shortname' => 'C1']) + ); + } + } + + /** + * Data provider for test_execute. + * @return array the data. + */ + public static function execute_steps_provider(): array { + return [ + 'Following exists' => [ + 'step' => 'Given the following "course" exists: + | fullname | Course test | + | shortname | C1 | + | category | 0 |', + 'expected' => true, + ], + 'Following exist' => [ + 'step' => 'Given the following "course" exist: + | fullname | shortname | category | + | Course test | C1 | 0 |', + 'expected' => true, + ], + 'Repeated entities' => [ + 'step' => 'Given "1" "courses" exist with the following data: + | fullname | Course test | + | shortname | C[count] | + | category | 0 | + | numsections | 3 |', + 'expected' => true, + ], + 'Invalid step' => [ + 'step' => 'Given I click on "Tokens filter" "link"', + 'expected' => false, + ], + ]; + } + + /** + * Test for execute step. + * @covers ::is_executed + * @covers ::execute + * @covers ::get_error + */ + public function test_execute_duplicated(): void { + global $DB; + + $this->resetAfterTest(); + + $generator = new behat_data_generators(); + $validsteps = [ + '/^the following "(?P(?:[^"]|\\")*)" exist:$/' => 'the_following_entities_exist', + ':count :entitytype exist with the following data:' => 'the_following_repeated_entities_exist', + 'the following :entitytype exists:' => 'the_following_entity_exists', + ]; + + $step = $this->get_step('Given the following "course" exists: + | fullname | Course test | + | shortname | C1 | + | category | 0 |'); + $steprunner = new steprunner($generator, $validsteps, $step); + + $this->assertFalse($steprunner->is_executed()); + + $result = $steprunner->execute(); + + $this->assertTrue($result); + $this->assertTrue($steprunner->is_executed()); + $this->assertEquals('', $steprunner->get_error()); + + // Validate everything is created. + $this->assertEquals( + 1, + $DB->count_records('course', ['shortname' => 'C1']) + ); + + // Execute the same course creation. + $steprunner = new steprunner($generator, $validsteps, $step); + $this->assertFalse($steprunner->is_executed()); + $result = $steprunner->execute(); + $this->assertFalse($result); + $this->assertTrue($steprunner->is_executed()); + $this->assertEquals(get_string('shortnametaken', 'error', 'C1'), $steprunner->get_error()); + } + + /** + * Test for parse_feature. + * @covers ::get_text + * @covers ::get_arguments_string + */ + public function test_get_step_content(): void { + $generator = new behat_data_generators(); + $validsteps = [ + '/^the following "(?P(?:[^"]|\\")*)" exist:$/' => 'the_following_entities_exist', + ':count :entitytype exist with the following data:' => 'the_following_repeated_entities_exist', + 'the following :entitytype exists:' => 'the_following_entity_exists', + ]; + + $step = $this->get_step('Given the following "course" exists: + | fullname | Course test | + | shortname | C1 | + | category | 0 | + | numsections | 3 |'); + $steprunner = new steprunner($generator, $validsteps, $step); + + $this->assertEquals( + 'the following "course" exists:', + $steprunner->get_text() + ); + + $data = [ + '| fullname | Course test |', + '| shortname | C1 |', + '| category | 0 |', + '| numsections | 3 |', + ]; + $arguments = explode("\n", $steprunner->get_arguments_string()); + $this->assertEquals( + $data, + $arguments + ); + } +}