From 4234159ecdcf8a2bba8b6aba25f5db12eb8b1d64 Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Wed, 18 Dec 2019 11:58:43 +1100 Subject: [PATCH] MDL-67433 tool_task: Add limits to adhoc CLI runner. --- admin/tool/task/cli/adhoc_task.php | 50 ++++---------------- lib/cronlib.php | 75 +++++++++++++++++++++++------- lib/upgrade.txt | 5 ++ 3 files changed, 71 insertions(+), 59 deletions(-) diff --git a/admin/tool/task/cli/adhoc_task.php b/admin/tool/task/cli/adhoc_task.php index 1bae3806e6d..755026b90fb 100644 --- a/admin/tool/task/cli/adhoc_task.php +++ b/admin/tool/task/cli/adhoc_task.php @@ -35,10 +35,12 @@ list($options, $unrecognized) = cli_get_params( 'keep-alive' => 0, 'showsql' => false, 'showdebugging' => false, + 'ignorelimits' => false, ], [ 'h' => 'help', 'e' => 'execute', 'k' => 'keep-alive', + 'i' => 'ignorelimits', ] ); @@ -57,6 +59,7 @@ Options: --showdebugging Show developer level debugging information -e, --execute Run all queued adhoc tasks -k, --keep-alive=N Keep this script alive for N seconds and poll for new adhoc tasks + -i --ignorelimits Ignore task_adhoc_concurrency_limit and task_adhoc_max_runtime limits Example: \$sudo -u www-data /usr/bin/php admin/tool/task/cli/adhoc_task.php --execute @@ -99,6 +102,8 @@ if (!empty($CFG->showcrondebugging)) { set_debugging(DEBUG_DEVELOPER, true); } +$checklimits = empty($options['ignorelimits']); + core_php_time_limit::raise(); // Increase memory limit. @@ -107,45 +112,8 @@ raise_memory_limit(MEMORY_EXTRA); // Emulate normal session - we use admin account by default. cron_setup_user(); -// Start output log. -$timestart = time(); -$timenow = $timestart; -$finishtime = $timenow + (int)$options['keep-alive']; -$humantimenow = date('r', $timenow); +$humantimenow = date('r', time()); +$keepalive = (int)$options['keep-alive']; + mtrace("Server Time: {$humantimenow}\n"); - -// Run all adhoc tasks. -$taskcount = 0; -$waiting = false; -while (!\core\task\manager::static_caches_cleared_since($timestart)) { - - $task = \core\task\manager::get_next_adhoc_task($timenow); - - if ($task) { - if ($waiting) { - cli_writeln(''); - } - $waiting = false; - cron_run_inner_adhoc_task($task); - $taskcount++; - unset($task); - } else { - if (time() > $finishtime) { - break; - } - if (!$waiting) { - cli_write('Waiting for more adhoc tasks to be queued '); - } else { - cli_write('.'); - } - $waiting = true; - sleep(1); - $timenow = time(); - } -} -if ($waiting) { - cli_writeln(''); -} - -mtrace("Ran {$taskcount} adhoc tasks found at {$humantimenow}"); - +cron_run_adhoc_tasks(time(), $keepalive, $checklimits); diff --git a/lib/cronlib.php b/lib/cronlib.php index c735ac2501b..db0193fe857 100644 --- a/lib/cronlib.php +++ b/lib/cronlib.php @@ -128,41 +128,80 @@ function cron_run_scheduled_tasks(int $timenow) { * Execute all queued adhoc tasks, applying necessary concurrency limits and time limits. * * @param int $timenow The time this process started. + * @param int $keepalive Keep this function alive for N seconds and poll for new adhoc tasks. + * @param bool $checklimits Should we check limits? */ -function cron_run_adhoc_tasks(int $timenow) { +function cron_run_adhoc_tasks(int $timenow, $keepalive = 0, $checklimits = true) { // Allow a restriction on the number of adhoc task runners at once. $cronlockfactory = \core\lock\lock_config::get_lock_factory('cron'); $maxruns = get_config('core', 'task_adhoc_concurrency_limit'); $maxruntime = get_config('core', 'task_adhoc_max_runtime'); - $adhoclock = null; - for ($run = 0; $run < $maxruns; $run++) { - if ($adhoclock = $cronlockfactory->get_lock("adhoc_task_runner_{$run}", 1)) { - break; + if ($checklimits) { + $adhoclock = null; + for ($run = 0; $run < $maxruns; $run++) { + if ($adhoclock = $cronlockfactory->get_lock("adhoc_task_runner_{$run}", 1)) { + break; + } + } + + if (!$adhoclock) { + mtrace("Skipping processing of adhoc tasks. Concurrency limit reached."); + return; } } - if (!$adhoclock) { - mtrace("Skipping processing of adhoc tasks. Concurrency limit reached."); - return; - } - - $starttime = time(); + $humantimenow = date('r', $timenow); + $finishtime = $timenow + $keepalive; + $waiting = false; + $taskcount = 0; // Run all adhoc tasks. - while (!\core\task\manager::static_caches_cleared_since($timenow) && - $task = \core\task\manager::get_next_adhoc_task(time())) { - cron_run_inner_adhoc_task($task); - unset($task); + while (!\core\task\manager::static_caches_cleared_since($timenow)) { - if ((time() - $starttime) > $maxruntime) { + if ($checklimits && (time() - $timenow) >= $maxruntime) { + if ($waiting) { + $waiting = false; + mtrace(''); + } mtrace("Stopping processing of adhoc tasks as time limit has been reached."); break; } + + $task = \core\task\manager::get_next_adhoc_task(time()); + + if ($task) { + if ($waiting) { + mtrace(''); + } + $waiting = false; + cron_run_inner_adhoc_task($task); + $taskcount++; + unset($task); + } else { + if (time() >= $finishtime) { + break; + } + if (!$waiting) { + mtrace('Waiting for more adhoc tasks to be queued ', ''); + } else { + mtrace('.', ''); + } + $waiting = true; + sleep(1); + } } - // Release the adhoc task runner lock. - $adhoclock->release(); + if ($waiting) { + mtrace(''); + } + + mtrace("Ran {$taskcount} adhoc tasks found at {$humantimenow}"); + + if ($adhoclock) { + // Release the adhoc task runner lock. + $adhoclock->release(); + } } /** diff --git a/lib/upgrade.txt b/lib/upgrade.txt index 4a8eef053d5..737f0fd0635 100644 --- a/lib/upgrade.txt +++ b/lib/upgrade.txt @@ -1,6 +1,11 @@ This files describes API changes in core libraries and APIs, information provided here is intended especially for developers. +=== 3.9 === + +* admin/tool/task/cli/adhoc_task.php now observers the concurrency limits. + If you want to get the previous (unlimited) behavior, use the --ignorelimits switch). + === 3.8 === * Add CLI option to notify all cron tasks to stop: admin/cli/cron.php --stop * The rotate_image function has been added to the stored_file class (MDL-63349)