diff --git a/admin/index.php b/admin/index.php index c391089ea4b..ee3b2e3b85e 100644 --- a/admin/index.php +++ b/admin/index.php @@ -843,8 +843,9 @@ $errorsdisplayed = defined('WARN_DISPLAY_ERRORS_ENABLED'); $lastcron = get_config('tool_task', 'lastcronstart'); $cronoverdue = ($lastcron < time() - 3600 * 24); $lastcroninterval = get_config('tool_task', 'lastcroninterval'); -$expectedfrequency = $CFG->expectedcronfrequency ?? 200; -$croninfrequent = !$cronoverdue && ($lastcroninterval > $expectedfrequency || $lastcron < time() - $expectedfrequency); + +$expectedfrequency = $CFG->expectedcronfrequency ?? MINSECS; +$croninfrequent = !$cronoverdue && ($lastcroninterval > ($expectedfrequency + MINSECS) || $lastcron < time() - $expectedfrequency); $dbproblems = $DB->diagnose(); $maintenancemode = !empty($CFG->maintenance_enabled); diff --git a/admin/renderer.php b/admin/renderer.php index 4ab7754a13b..f9043524f91 100644 --- a/admin/renderer.php +++ b/admin/renderer.php @@ -601,19 +601,9 @@ class core_admin_renderer extends plugin_renderer_base { return ''; } - if (empty($CFG->cronclionly)) { - $url = new moodle_url('/admin/cron.php'); - if (!empty($CFG->cronremotepassword)) { - $url = new moodle_url('/admin/cron.php', array('password' => $CFG->cronremotepassword)); - } - - return $this->warning(get_string('cronwarning', 'admin', $url->out()) . ' ' . - $this->help_icon('cron', 'admin')); - } - - // $CFG->cronclionly is not empty: cron can run only from CLI. - return $this->warning(get_string('cronwarningcli', 'admin') . ' ' . - $this->help_icon('cron', 'admin')); + $check = new \tool_task\check\cronrunning(); + $result = $check->get_result(); + return $this->warning($result->get_summary() . ' ' . $this->help_icon('cron', 'admin')); } /** @@ -629,9 +619,9 @@ class core_admin_renderer extends plugin_renderer_base { return ''; } - $expectedfrequency = $CFG->expectedcronfrequency ?? 200; - return $this->warning(get_string('croninfrequent', 'admin', $expectedfrequency) . ' ' . - $this->help_icon('cron', 'admin')); + $check = new \tool_task\check\cronrunning(); + $result = $check->get_result(); + return $this->warning($result->get_summary() . ' ' . $this->help_icon('cron', 'admin')); } /** diff --git a/admin/tool/task/classes/check/adhocqueue.php b/admin/tool/task/classes/check/adhocqueue.php new file mode 100644 index 00000000000..85f98b8f502 --- /dev/null +++ b/admin/tool/task/classes/check/adhocqueue.php @@ -0,0 +1,93 @@ +. + +/** + * Ad hoc queue checks + * + * @package tool_task + * @copyright 2020 Brendan Heywood (brendan@catalyst-au.net) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace tool_task\check; + +defined('MOODLE_INTERNAL') || die(); + +use core\check\check; +use core\check\result; + +/** + * Ad hoc queue checks + * + * @package tool_task + * @copyright 2020 Brendan Heywood (brendan@catalyst-au.net) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class adhocqueue extends check { + + /** + * Constructor + */ + public function __construct() { + global $CFG; + $this->id = 'adhocqueue'; + $this->name = get_string('checkadhocqueue', 'tool_task'); + } + + /** + * Return result + * @return result + */ + public function get_result() : result { + global $DB, $CFG; + + $stats = $DB->get_record_sql(' + SELECT count(*) cnt, + MAX(? - nextruntime) age + FROM {task_adhoc}', [time()]); + + $status = result::OK; + $summary = get_string('adhocempty', 'tool_task'); + $details = ''; + + if ($stats->cnt > 0) { + // A large queue size by itself is not an issue, only when tasks + // are not being processed in a timely fashion is it an issue. + $status = result::INFO; + $summary = get_string('adhocqueuesize', 'tool_task', $stats->cnt); + } + + $max = $CFG->adhoctaskagewarn ?? 10 * MINSECS; + if ($stats->age > $max) { + $status = result::WARNING; + $summary = get_string('adhocqueueold', 'tool_task', [ + 'age' => format_time($stats->age), + 'max' => format_time($max), + ]); + } + + $max = $CFG->adhoctaskageerror ?? 4 * HOURSECS; + if ($stats->age > $max) { + $status = result::ERROR; + $summary = get_string('adhocqueueold', 'tool_task', [ + 'age' => format_time($stats->age), + 'max' => format_time($max), + ]); + } + + return new result($status, $summary, $details); + } +} diff --git a/admin/tool/task/classes/check/cronrunning.php b/admin/tool/task/classes/check/cronrunning.php new file mode 100644 index 00000000000..46aa6908231 --- /dev/null +++ b/admin/tool/task/classes/check/cronrunning.php @@ -0,0 +1,118 @@ +. + +/** + * Cron running check + * + * @package tool_task + * @copyright 2020 Brendan Heywood (brendan@catalyst-au.net) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace tool_task\check; + +defined('MOODLE_INTERNAL') || die(); + +use core\check\check; +use core\check\result; +/** + * Cron running check + * + * @package tool_task + * @copyright 2020 Brendan Heywood (brendan@catalyst-au.net) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cronrunning extends check { + + /** + * Constructor + */ + public function __construct() { + global $CFG; + $this->id = 'cronrunning'; + $this->name = get_string('checkcronrunning', 'tool_task'); + if (empty($CFG->cronclionly)) { + $this->actionlink = new \action_link( + new \moodle_url('/admin/cron.php'), + get_string('cron', 'admin')); + } + } + + /** + * Return result + * @return result + */ + public function get_result() : result { + global $CFG; + + // Eventually this should replace cron_overdue_warning and + // cron_infrequent_warning. + $lastcron = get_config('tool_task', 'lastcronstart'); + $expectedfrequency = $CFG->expectedcronfrequency ?? MINSECS; + + $delta = time() - $lastcron; + + $lastcroninterval = get_config('tool_task', 'lastcroninterval'); + + $formatdelta = format_time($delta); + $formatexpected = format_time($expectedfrequency); + $formatinterval = format_time($lastcroninterval); + + $details = format_time($delta); + + if ($delta > $expectedfrequency + MINSECS) { + $status = result::WARNING; + + if ($delta > DAYSECS) { + $status = result::CRITICAL; + } + + if (empty($lastcron)) { + $summary = get_string('cronwarningnever', 'admin', [ + 'expected' => $formatexpected, + ]); + } else if (empty($CFG->cronclionly)) { + $url = new \moodle_url('/admin/cron.php'); + $summary = get_string('cronwarning', 'admin', [ + 'url' => $url->out(), + 'actual' => $formatdelta, + 'expected' => $formatexpected, + ]); + } else { + $summary = get_string('cronwarningcli', 'admin', [ + 'actual' => $formatdelta, + 'expected' => $formatexpected, + ]); + } + return new result($status, $summary, $details); + } + + if ($lastcroninterval > $expectedfrequency) { + $status = result::WARNING; + $summary = get_string('croninfrequent', 'admin', [ + 'actual' => $formatinterval, + 'expected' => $formatexpected, + ]); + return new result($status, $summary, $details); + } + + $status = result::OK; + $summary = get_string('cronok', 'tool_task'); + + return new result($status, $summary, $details); + } +} + diff --git a/admin/tool/task/classes/check/maxfaildelay.php b/admin/tool/task/classes/check/maxfaildelay.php new file mode 100644 index 00000000000..019aa75edf1 --- /dev/null +++ b/admin/tool/task/classes/check/maxfaildelay.php @@ -0,0 +1,96 @@ +. + +/** + * Task fail delay check + * + * @package tool_task + * @copyright 2020 Brendan Heywood (brendan@catalyst-au.net) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace tool_task\check; + +defined('MOODLE_INTERNAL') || die(); + +use core\check\check; +use core\check\result; + +/** + * Task fail delay check + * + * @package tool_task + * @copyright 2020 Brendan Heywood (brendan@catalyst-au.net) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class maxfaildelay extends check { + + /** + * Constructor + */ + public function __construct() { + global $CFG; + $this->id = 'cronfaildelay'; + $this->name = get_string('checkmaxfaildelay', 'tool_task'); + $this->actionlink = new \action_link( + new \moodle_url('/admin/tool/task/scheduledtasks.php'), + get_string('scheduledtasks', 'tool_task')); + } + + /** + * Return result + * @return result + */ + public function get_result() : result { + global $CFG; + + $status = result::OK; + $summary = get_string('tasknofailures', 'tool_task'); + $details = ''; + $failures = 0; + $maxdelay = 0; + + $tasks = \core\task\manager::get_all_scheduled_tasks(); + foreach ($tasks as $task) { + if ($task->get_disabled()) { + continue; + } + $faildelay = $task->get_fail_delay(); + if ($faildelay > $maxdelay) { + $maxdelay = $faildelay; + } + if ($faildelay > 0) { + $failures++; + $details .= get_string('faildelay', 'tool_task') . ': ' . format_time($faildelay); + $details .= ' - ' . $task->get_name() . ' (' .get_class($task) . ")
"; + } + } + + if ($failures > 0) { + // Intermittent failures are not yet a warning. + $status = result::INFO; + $summary = get_string('taskfailures', 'tool_task', $failures); + } + if ($maxdelay > 5 * MINSECS) { + $status = result::WARNING; + } + if ($maxdelay > 4 * HOURSECS) { + $status = result::ERROR; + } + + return new result($status, $summary, $details); + } +} diff --git a/admin/tool/task/lang/en/tool_task.php b/admin/tool/task/lang/en/tool_task.php index e48348a0e58..43c9540d404 100644 --- a/admin/tool/task/lang/en/tool_task.php +++ b/admin/tool/task/lang/en/tool_task.php @@ -23,12 +23,19 @@ */ $string['asap'] = 'ASAP'; +$string['adhocempty'] = 'Adhoc task queue is empty'; +$string['adhocqueuesize'] = 'Adhoc task queue has {$a} tasks'; +$string['adhocqueueold'] = 'Oldest task is {$a->age} which is more than {$a->max}'; $string['backtoscheduledtasks'] = 'Back to scheduled tasks'; $string['blocking'] = 'Blocking'; $string['cannotfindthepathtothecli'] = 'Cannot find the path to the PHP CLI executable so task execution aborted. Set the \'Path to PHP CLI\' setting in Site administration / Server / System paths.'; +$string['checkadhocqueue'] = 'Adhoc task queue'; +$string['checkcronrunning'] = 'Cron running'; +$string['checkmaxfaildelay'] = 'Tasks max fail delay'; $string['clearfaildelay_confirm'] = 'Are you sure you want to clear the fail delay for task \'{$a}\'? After clearing the delay, the task will run according to its normal schedule.'; $string['component'] = 'Component'; $string['corecomponent'] = 'Core'; +$string['cronok'] = 'Cron is running frequently'; $string['default'] = 'Default'; $string['defaultx'] = 'Default: {$a}'; $string['disabled'] = 'Disabled'; @@ -50,7 +57,9 @@ $string['runpattern'] = 'Run pattern'; $string['scheduledtasks'] = 'Scheduled tasks'; $string['scheduledtaskchangesdisabled'] = 'Modifications to the list of scheduled tasks have been prevented in Moodle configuration'; $string['taskdisabled'] = 'Task disabled'; +$string['taskfailures'] = 'There are {$a} task(s) failing'; $string['tasklogs'] = 'Task logs'; +$string['tasknofailures'] = 'There are no tasks failing'; $string['taskscheduleday'] = 'Day'; $string['taskscheduleday_help'] = 'Day of month field for task schedule. The field uses the same format as unix cron. Some examples are: diff --git a/admin/tool/task/lib.php b/admin/tool/task/lib.php new file mode 100644 index 00000000000..ae23257b904 --- /dev/null +++ b/admin/tool/task/lib.php @@ -0,0 +1,39 @@ +. + +/** + * Task API status checks + * + * @package tool_task + * @copyright 2020 Brendan Heywood (brendan@catalyst-au.net) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Add cron related service status checks + * + * @return array of check objects + */ +function tool_task_status_checks() : array { + return [ + new \tool_task\check\cronrunning(), + new \tool_task\check\maxfaildelay(), + new \tool_task\check\adhocqueue(), + ]; +} + diff --git a/config-dist.php b/config-dist.php index 5da7105ef30..51f7e55e370 100644 --- a/config-dist.php +++ b/config-dist.php @@ -616,6 +616,12 @@ $CFG->admin = 'admin'; // // $CFG->expectedcronfrequency = 200; // +// Moodle 3.9+ checks how old tasks are in the ad hoc queue and warns at 10 minutes +// and errors at 4 hours. Set these to override these limits: +// +// $CFG->adhoctaskagewarn = 10 * 60; +// $CFG->adhoctaskageerror = 4 * 60 * 60; +// // Session lock warning threshold. Long running pages should release the session using \core\session\manager::write_close(). // Set this threshold to any value greater than 0 to add developer warnings when a page locks the session for too long. // The session should rarely be locked for more than 1 second. The input should be in seconds and may be a float. diff --git a/lang/en/admin.php b/lang/en/admin.php index fefbb20fcd3..5cc4192bdff 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -429,10 +429,11 @@ $string['cron_link'] = 'admin/cron'; $string['cronclionly'] = 'Cron execution via command line only'; $string['cronerrorclionly'] = 'Sorry, internet access to this page has been disabled by the administrator.'; $string['cronerrorpassword'] = 'Sorry, you have not provided a valid password to access this page'; -$string['croninfrequent'] = 'The time between the last two runs of the cron maintenance script was over {$a} seconds. We recommend configuring it to run more frequently.'; +$string['croninfrequent'] = 'There was {$a->actual} between the last two runs of the cron maintenance script and it should run every {$a->expected}. We recommend configuring it to run more frequently.'; $string['cronremotepassword'] = 'Cron password for remote access'; -$string['cronwarning'] = 'The cron.php maintenance script has not been run for at least 24 hours.'; -$string['cronwarningcli'] = 'The cli/cron.php maintenance script has not been run for at least 24 hours.'; +$string['cronwarning'] = 'The admin/cron.php script has not been run for {$a->actual} and should run every {$a->expected}.'; +$string['cronwarningcli'] = 'The admin/cli/cron.php script has not been run for {$a->actual} and should run every {$a->expected}.'; +$string['cronwarningnever'] = 'The admin/cli/cron.php script has never been run and should run every {$a->expected}.'; $string['ctyperequired'] = 'The ctype PHP extension is now required by Moodle, in order to improve site performance and to offer multilingual compatibility.'; $string['curlsecurityallowedport'] = 'cURL allowed ports list'; $string['curlsecurityallowedportsyntax'] = 'List of port numbers that cURL can connect to. Valid entries are integer numbers only. Put each entry on a new line. If left empty, then all ports are allowed. If set, in almost all cases, both 443 and 80 should be specified for cURL to connect to standard HTTPS and HTTP ports.';