From af7ca95b3f45cb76af3cecea293d30f0e8bcd35f Mon Sep 17 00:00:00 2001 From: David Woloszyn Date: Thu, 19 Dec 2024 10:51:43 +1100 Subject: [PATCH] MDL-83395 core_ai: AI usage report --- admin/settings/ai.php | 8 + .../local/entities/ai_action_register.php | 256 ++++++++++++++++++ .../local/systemreports/usage.php | 117 ++++++++ ai/tests/behat/ai_usage_report.feature | 29 ++ .../generator/behat_core_ai_generator.php | 49 ++++ ai/tests/generator/lib.php | 90 ++++++ ai/usage_report.php | 45 +++ lang/en/ai.php | 6 + lang/en/role.php | 1 + lib/db/access.php | 9 + version.php | 2 +- 11 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 ai/classes/reportbuilder/local/entities/ai_action_register.php create mode 100644 ai/classes/reportbuilder/local/systemreports/usage.php create mode 100644 ai/tests/behat/ai_usage_report.feature create mode 100644 ai/tests/generator/behat_core_ai_generator.php create mode 100644 ai/tests/generator/lib.php create mode 100644 ai/usage_report.php diff --git a/admin/settings/ai.php b/admin/settings/ai.php index 838ac7abcef..42705f5955e 100644 --- a/admin/settings/ai.php +++ b/admin/settings/ai.php @@ -78,3 +78,11 @@ $aipolicyacceptance = new admin_externalpage( 'moodle/ai:viewaipolicyacceptancereport' ); $ADMIN->add('aireports', $aipolicyacceptance); +// Add AI usage report. +$aiusage = new admin_externalpage( + 'aiusagereport', + get_string('aiusage', 'core_ai'), + new moodle_url('/ai/usage_report.php'), + 'moodle/ai:viewaiusagereport', +); +$ADMIN->add('aireports', $aiusage); diff --git a/ai/classes/reportbuilder/local/entities/ai_action_register.php b/ai/classes/reportbuilder/local/entities/ai_action_register.php new file mode 100644 index 00000000000..354ad423299 --- /dev/null +++ b/ai/classes/reportbuilder/local/entities/ai_action_register.php @@ -0,0 +1,256 @@ +. + +namespace core_ai\reportbuilder\local\entities; + +use core_reportbuilder\local\entities\base; +use core_reportbuilder\local\filters\boolean_select; +use core_reportbuilder\local\filters\date; +use core_reportbuilder\local\filters\number; +use core_reportbuilder\local\filters\text; +use core_reportbuilder\local\helpers\format; +use core_reportbuilder\local\report\column; +use core_reportbuilder\local\report\filter; +use lang_string; + +/** + * AI action register entity. + * + * Defines all the columns and filters that can be added to reports that use this entity. + * + * @package core_ai + * @copyright 2024 David Woloszyn + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class ai_action_register extends base { + + #[\Override] + protected function get_default_tables(): array { + return [ + 'ai_action_register', + ]; + } + + #[\Override] + protected function get_default_entity_title(): lang_string { + return new lang_string('aiactionregister', 'core_ai'); + } + + #[\Override] + public function initialise(): base { + $columns = $this->get_all_columns(); + foreach ($columns as $column) { + $this->add_column($column); + } + + // All the filters defined by the entity can also be used as conditions. + $filters = $this->get_all_filters(); + foreach ($filters as $filter) { + $this + ->add_filter($filter) + ->add_condition($filter); + } + + return $this; + } + + /** + * Returns list of all available columns. + * + * @return column[] + */ + protected function get_all_columns(): array { + $mainalias = $this->get_table_alias('ai_action_register'); + $generatetextalias = 'aagt'; + $summarisetextalias = 'aast'; + + // Action name column. + $columns[] = (new column( + 'actionname', + new lang_string('action', 'core_ai'), + $this->get_entity_name(), + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_field("{$mainalias}.actionname") + ->set_is_sortable(true) + ->add_callback(static function(string $actionname): string { + return get_string("action_{$actionname}", 'core_ai'); + }); + + // Provider column. + $columns[] = (new column( + 'provider', + new lang_string('provider', 'core_ai'), + $this->get_entity_name(), + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_field("{$mainalias}.provider") + ->set_is_sortable(true); + + // Success column. + $columns[] = (new column( + 'success', + new lang_string('success', 'moodle'), + $this->get_entity_name(), + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_BOOLEAN) + ->add_field("{$mainalias}.success") + ->set_is_sortable(true) + ->set_callback([format::class, 'boolean_as_text']); + + // Time created column. + $columns[] = (new column( + 'timecreated', + new lang_string('timegenerated', 'core_ai'), + $this->get_entity_name(), + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TIMESTAMP) + ->add_field("{$mainalias}.timecreated") + ->set_is_sortable(true) + ->add_callback([format::class, 'userdate']); + + // Prompt tokens column. + // Only available for summarise_text and generate_text actions. + $columns[] = (new column( + 'prompttokens', + new lang_string('prompttokens', 'core_ai'), + $this->get_entity_name(), + )) + ->add_joins($this->get_joins()) + ->add_join(" + LEFT JOIN {ai_action_generate_text} {$generatetextalias} + ON {$mainalias}.actionid = {$generatetextalias}.id + AND {$mainalias}.actionname = 'generate_text'") + ->add_join(" + LEFT JOIN {ai_action_summarise_text} {$summarisetextalias} + ON {$mainalias}.actionid = {$summarisetextalias}.id + AND {$mainalias}.actionname = 'summarise_text'") + ->set_type(column::TYPE_INTEGER) + ->add_field("COALESCE({$generatetextalias}.prompttokens, {$summarisetextalias}.prompttokens)", 'prompttokens') + ->set_is_sortable(true) + ->add_callback(static function(?int $value): string { + return $value ?? get_string('unknownvalue', 'core_ai'); + }); + + // Completion tokens column. + // Only available for summarise_text and generate_text actions. + $columns[] = (new column( + 'completiontokens', + new lang_string('completiontokens', 'core_ai'), + $this->get_entity_name(), + )) + ->add_joins($this->get_joins()) + ->add_join(" + LEFT JOIN {ai_action_generate_text} {$generatetextalias} + ON {$mainalias}.actionid = {$generatetextalias}.id + AND {$mainalias}.actionname = 'generate_text'") + ->add_join(" + LEFT JOIN {ai_action_summarise_text} {$summarisetextalias} + ON {$mainalias}.actionid = {$summarisetextalias}.id + AND {$mainalias}.actionname = 'summarise_text'") + ->set_type(column::TYPE_INTEGER) + ->add_field("COALESCE({$generatetextalias}.completiontoken, {$summarisetextalias}.completiontoken)", 'completiontokens') + ->set_is_sortable(true) + ->add_callback(static function(?int $value): string { + return $value ?? get_string('unknownvalue', 'core_ai'); + }); + + return $columns; + } + + /** + * Return list of all available filters. + * + * @return filter[] + */ + protected function get_all_filters(): array { + $mainalias = $this->get_table_alias('ai_action_register'); + $generatetextalias = 'aagt'; + $summarisetextalias = 'aast'; + + // Action name filter. + $filters[] = (new filter( + text::class, + 'actionname', + new lang_string('action', 'core_ai'), + $this->get_entity_name(), + "{$mainalias}.actionname", + )) + ->add_joins($this->get_joins()); + + // Provider filter. + $filters[] = (new filter( + text::class, + 'provider', + new lang_string('provider', 'core_ai'), + $this->get_entity_name(), + "{$mainalias}.provider", + )) + ->add_joins($this->get_joins()); + + // Time created filter. + $filters[] = (new filter( + date::class, + 'timecreated', + new lang_string('timegenerated', 'core_ai'), + $this->get_entity_name(), + "{$mainalias}.timecreated", + )) + ->add_joins($this->get_joins()) + ->set_limited_operators([ + date::DATE_ANY, + date::DATE_RANGE, + date::DATE_PREVIOUS, + date::DATE_CURRENT, + ]); + + // Prompt tokens filter. + $filters[] = (new filter( + number::class, + 'prompttokens', + new lang_string('prompttokens', 'core_ai'), + $this->get_entity_name(), + "COALESCE({$generatetextalias}.prompttokens, {$summarisetextalias}.prompttokens)", + )) + ->add_joins($this->get_joins()); + + // Completion tokens filter. + $filters[] = (new filter( + number::class, + 'completiontokens', + new lang_string('completiontokens', 'core_ai'), + $this->get_entity_name(), + "COALESCE({$generatetextalias}.completiontoken, {$summarisetextalias}.completiontoken)", + )) + ->add_joins($this->get_joins()); + + // Success filter. + $filters[] = (new filter( + boolean_select::class, + 'success', + new lang_string('success', 'moodle'), + $this->get_entity_name(), + "{$mainalias}.success", + )) + ->add_joins($this->get_joins()); + + return $filters; + } +} diff --git a/ai/classes/reportbuilder/local/systemreports/usage.php b/ai/classes/reportbuilder/local/systemreports/usage.php new file mode 100644 index 00000000000..d88f1eaec4e --- /dev/null +++ b/ai/classes/reportbuilder/local/systemreports/usage.php @@ -0,0 +1,117 @@ +. + +namespace core_ai\reportbuilder\local\systemreports; + +use core_reportbuilder\system_report; +use core_ai\reportbuilder\local\entities\ai_action_register; +use core\reportbuilder\local\entities\context; +use core_reportbuilder\local\entities\user; + +/** + * AI usage system report. + * + * @package core_ai + * @copyright 2024 David Woloszyn + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class usage extends system_report { + + #[\Override] + protected function initialise(): void { + $entitymain = new ai_action_register(); + $entitymainalias = $entitymain->get_table_alias('ai_action_register'); + + $this->set_main_table('ai_action_register', $entitymainalias); + $this->add_entity($entitymain); + + // Join the 'user' entity to our main entity. + $entityuser = new user(); + $entituseralias = $entityuser->get_table_alias('user'); + $this->add_entity($entityuser->add_join( + "LEFT JOIN {user} {$entituseralias} ON {$entituseralias}.id = {$entitymainalias}.userid" + )); + + // Join the 'context' entity to our main entity. + $entitycontext = new context(); + $entitycontextalias = $entitycontext->get_table_alias('context'); + $this->add_entity($entitycontext->add_join( + "LEFT JOIN {context} {$entitycontextalias} ON {$entitycontextalias}.id = {$entitymainalias}.contextid" + )); + + // Now we can call our helper methods to add the content we want to include in the report. + $this->add_columns(); + $this->add_filters(); + + // Set if report can be downloaded. + $this->set_downloadable(true, get_string('aiusage', 'core_ai')); + } + + #[\Override] + protected function can_view(): bool { + return has_capability('moodle/ai:viewaiusagereport', $this->get_context()); + } + + #[\Override] + public static function get_name(): string { + return get_string('aiusage', 'core_ai'); + } + + /** + * Adds the columns we want to display in the report. + * + * They are all provided by the entities we previously added in the {@see initialise} method, referencing each by their + * unique identifier. + */ + public function add_columns(): void { + $columns = [ + 'ai_action_register:provider', + 'ai_action_register:actionname', + 'ai_action_register:timecreated', + 'ai_action_register:prompttokens', + 'ai_action_register:completiontokens', + 'ai_action_register:success', + 'context:name', + 'user:fullnamewithlink', + ]; + + $this->add_columns_from_entities($columns); + + // It's possible to set a default initial sort direction for one column. + $this->set_initial_sort_column('ai_action_register:timecreated', SORT_DESC); + } + + /** + * Adds the filters we want to display in the report. + * + * They are all provided by the entities we previously added in the {@see initialise} method, referencing each by their + * unique identifier. + */ + protected function add_filters(): void { + $filters = [ + 'ai_action_register:actionname', + 'ai_action_register:provider', + 'ai_action_register:timecreated', + 'ai_action_register:prompttokens', + 'ai_action_register:completiontokens', + 'ai_action_register:success', + 'context:level', + 'user:fullname', + ]; + + $this->add_filters_from_entities($filters); + } +} diff --git a/ai/tests/behat/ai_usage_report.feature b/ai/tests/behat/ai_usage_report.feature new file mode 100644 index 00000000000..bf5fd3529ae --- /dev/null +++ b/ai/tests/behat/ai_usage_report.feature @@ -0,0 +1,29 @@ +@report @core_ai +Feature: AI usage report displays recorded AI data + In order to view AI usage data + As a manager or admin + I can access the AI usage report + + Background: + Given the following "users" exist: + | username | firstname | lastname | email | + | manager1 | Manager | One | manager1@example.com | + | student1 | Student | One | student1@example.com | + | student2 | Student | Two | student2@example.com | + And the following "role assigns" exist: + | user | role | contextlevel | reference | + | manager1 | manager | System | | + And the following "core_ai > ai actions" exist: + | actionname | user | success | provider | contextid | + | generate_text | student1 | 1 | OpenAI | 1 | + | summarise_text | student1 | 0 | OpenAI | 1 | + | generate_image | student2 | 1 | Azure | 1 | + + Scenario: Managers can view the AI usage report + Given I am logged in as "manager1" + When I navigate to "Reports > AI reports > AI usage" in site administration + Then the following should exist in the "reportbuilder-table" table: + | Action | First name | Provider | Success | + | Generate text | Student One | OpenAI | Yes | + | Summarise text | Student One | OpenAI | No | + | Generate image | Student Two | Azure | Yes | diff --git a/ai/tests/generator/behat_core_ai_generator.php b/ai/tests/generator/behat_core_ai_generator.php new file mode 100644 index 00000000000..af7252cd9bf --- /dev/null +++ b/ai/tests/generator/behat_core_ai_generator.php @@ -0,0 +1,49 @@ +. + +/** + * Behat generator for AI. + * + * @package core_ai + * @copyright 2024 David Woloszyn + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class behat_core_ai_generator extends behat_generator_base { + + /** + * Get the list of creatable entities for core_ai. + * + * @return array + */ + protected function get_creatable_entities(): array { + return [ + 'ai actions' => [ + 'singular' => 'ai action', + 'datagenerator' => 'ai_actions', + 'required' => [ + 'actionname', + 'success', + 'user', + 'contextid', + 'provider', + ], + 'switchids' => [ + 'user' => 'userid', + ], + ], + ]; + } +} diff --git a/ai/tests/generator/lib.php b/ai/tests/generator/lib.php new file mode 100644 index 00000000000..ca178e3d1ac --- /dev/null +++ b/ai/tests/generator/lib.php @@ -0,0 +1,90 @@ +. + +/** + * AI data generator for tests. + * + * @package core_ai + * @copyright 2024 David Woloszyn + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_ai_generator extends component_generator_base { + + /** + * Creates AI action registry records. + * + * @param array $data + */ + public function create_ai_actions(array $data): void { + global $DB; + + if (!isset($data['actionname'])) { + throw new Exception('\'ai actions\' requires the field \'actionname\' to be specified'); + } + if (!isset($data['success'])) { + throw new Exception('\'ai actions\' requires the field \'success\' to be specified'); + } + if (!isset($data['userid'])) { + throw new Exception('\'ai actions\' requires the field \'user\' to be specified'); + } + if (!isset($data['contextid'])) { + throw new Exception('\'ai actions\' requires the field \'contextid\' to be specified'); + } + if (!isset($data['provider'])) { + throw new Exception('\'ai actions\' requires the field \'provider\' to be specified'); + } + + $action = new stdClass(); + foreach ($data as $key => $value) { + // Add data to parent action record. + $action->$key = $value; + + // Create the child action record. + $child = new stdClass(); + $child->prompt = 'Prompt text'; + + if ($key === 'actionname') { + // Generate image actions need to be structured differently. + if ($value === 'generate_image') { + $child->numberimages = 1; + $child->quality = 'hd'; + $child->aspectratio = 'landscape'; + $child->style = 'vivid'; + $child->sourceurl = 'http://localhost/yourimage'; + $child->revisedprompt = 'Revised prompt'; + } else { + // Generate text (and variants). + $child->generatedcontent = 'Your generated content'; + $child->prompttokens = 33; + $child->completiontoken = 44; + } + // Simulate an error. + if ($key === 'success' && $value == 0) { + $action->errorcode = 403; + $action->errormessage = 'Forbidden'; + } + + $childid = $DB->insert_record("ai_action_{$value}", $child); + } + } + + // Finalise some fields before inserting. + $action->actionid = $childid; + $action->timecreated = time(); + $action->timecompleted = time() + 1; + $DB->insert_record('ai_action_register', $action); + } +} diff --git a/ai/usage_report.php b/ai/usage_report.php new file mode 100644 index 00000000000..405528a88c5 --- /dev/null +++ b/ai/usage_report.php @@ -0,0 +1,45 @@ +. + +/** + * Display AI usage report. + * + * @package core_ai + * @copyright 2024 David Woloszyn + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +use core_reportbuilder\system_report_factory; +use core_ai\reportbuilder\local\systemreports\usage; + +require(__DIR__ . '/../config.php'); +require_once($CFG->libdir . '/adminlib.php'); + +admin_externalpage_setup('aiusagereport'); + +// Set up the page. +$systemcontext = context_system::instance(); +$pageurl = new moodle_url($CFG->wwwroot . '/ai/usage_report.php'); +$PAGE->set_url($pageurl); +$PAGE->set_context($systemcontext); +$PAGE->set_pagelayout('report'); +$PAGE->set_primary_active_tab('siteadminnode'); +echo $OUTPUT->header(); + +$report = system_report_factory::create(usage::class, $systemcontext); +echo $OUTPUT->heading($report::get_name()); +echo $report->output(); +echo $OUTPUT->footer(); diff --git a/lang/en/ai.php b/lang/en/ai.php index 40a8def3170..a7f52efb25c 100644 --- a/lang/en/ai.php +++ b/lang/en/ai.php @@ -48,17 +48,20 @@ $string['action_translate_text_desc'] = 'Translate provided text from one langua $string['actionsettingprovider'] = '{$a} action settings'; $string['actionsettingprovider_desc'] = 'These settings control how the {$a->providername} performs the action {$a->actionname}.'; $string['ai'] = 'AI'; +$string['aiactionregister'] = 'AI action register'; $string['aiplacements'] = 'AI placements'; $string['aipolicyacceptance'] = 'AI policy acceptance'; $string['aipolicyregister'] = 'AI policy register'; $string['aiproviders'] = 'AI providers'; $string['aireports'] = 'AI reports'; +$string['aiusage'] = 'AI usage'; $string['aiusagepolicy'] = 'AI usage policy'; $string['availableplacements'] = 'Choose where AI actions are available'; $string['availableplacements_desc'] = 'Placements define how and where AI actions can be used in your site. You can choose which actions are available in each placement through the settings.'; $string['availableproviders'] = 'Manage the AI providers connected to your LMS'; $string['availableproviders_desc'] = 'AI providers add AI functionality to your site through \'actions\' like text summarisation or image generation.
You can manage the actions for each provider in their settings.'; +$string['completiontokens'] = 'Completion tokens'; $string['contentwatermark'] = 'Generated by AI'; $string['dateaccepted'] = 'Date accepted'; $string['declineaipolicy'] = 'Decline'; @@ -105,11 +108,14 @@ $string['privacy:metadata:ai_policy_register'] = 'A table storing the status of $string['privacy:metadata:ai_policy_register:contextid'] = 'The ID of the context whose data was saved.'; $string['privacy:metadata:ai_policy_register:timeaccepted'] = 'The time the user accepted the AI policy.'; $string['privacy:metadata:ai_policy_register:userid'] = 'The ID of the user whose data was saved.'; +$string['prompttokens'] = 'Prompt tokens'; $string['provider'] = 'Provider'; $string['provideractionsettings'] = 'Actions'; $string['provideractionsettings_desc'] = 'Choose and configure the actions that the {$a} can perform on your site.'; $string['providers'] = 'Providers'; $string['providersettings'] = 'Settings'; +$string['timegenerated'] = 'Time generated'; +$string['unknownvalue'] = '—'; $string['userpolicy'] = '

Welcome to the new AI feature!

This Artificial Intelligence (AI) feature is based solely on external Large Language Models (LLM) to improve your learning and teaching experience. Before you start using these AI services, please read this usage policy.

Accuracy of AI-generated content

diff --git a/lang/en/role.php b/lang/en/role.php index 8c84a205dbd..e56b7c5202a 100644 --- a/lang/en/role.php +++ b/lang/en/role.php @@ -26,6 +26,7 @@ $string['ai:acceptpolicy'] = 'Accept AI policy'; $string['ai:fetchanyuserpolicystatus'] = 'Get a users AI policy acceptance'; $string['ai:fetchpolicy'] = 'Get a users AI policy acceptance'; $string['ai:viewaipolicyacceptancereport'] = 'View AI policy acceptance report'; +$string['ai:viewaiusagereport'] = 'View AI usage report'; $string['addinganewrole'] = 'Adding a new role'; $string['addrole'] = 'Add a new role'; $string['advancedoverride'] = 'Advanced role override'; diff --git a/lib/db/access.php b/lib/db/access.php index 8bb36c16e65..9a9f680c051 100644 --- a/lib/db/access.php +++ b/lib/db/access.php @@ -2819,4 +2819,13 @@ $capabilities = array( 'manager' => CAP_ALLOW, ], ], + // Allow managers to view the AI usage report. + 'moodle/ai:viewaiusagereport' => [ + 'riskbitmask' => RISK_PERSONAL, + 'captype' => 'read', + 'contextlevel' => CONTEXT_SYSTEM, + 'archetypes' => [ + 'manager' => CAP_ALLOW, + ], + ], ); diff --git a/version.php b/version.php index c00b3a8a5bd..518f41473bf 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2024121300.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2024121300.01; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes. $release = '5.0dev (Build: 20241213)'; // Human-friendly version name