From 5d385d4bcb071536cc6954d2d1385e46766bcaff Mon Sep 17 00:00:00 2001 From: Paul Holden Date: Fri, 12 Aug 2022 08:39:02 +0100 Subject: [PATCH] MDL-75535 files: implement files datasource for custom reporting. Create file entity, joining to existing user entity to provide data for the reportbuilder editor. --- .../reportbuilder/datasource/files.php | 110 ++++++ .../reportbuilder/local/entities/file.php | 374 ++++++++++++++++++ .../reportbuilder/datasource/files_test.php | 323 +++++++++++++++ lang/en/moodle.php | 2 + 4 files changed, 809 insertions(+) create mode 100644 files/classes/reportbuilder/datasource/files.php create mode 100644 files/classes/reportbuilder/local/entities/file.php create mode 100644 files/tests/reportbuilder/datasource/files_test.php diff --git a/files/classes/reportbuilder/datasource/files.php b/files/classes/reportbuilder/datasource/files.php new file mode 100644 index 00000000000..ddfa4370785 --- /dev/null +++ b/files/classes/reportbuilder/datasource/files.php @@ -0,0 +1,110 @@ +. + +declare(strict_types=1); + +namespace core_files\reportbuilder\datasource; + +use core_files\reportbuilder\local\entities\file; +use core_reportbuilder\datasource; +use core_reportbuilder\local\entities\user; +use core_reportbuilder\local\filters\boolean_select; + +/** + * Files datasource + * + * @package core_files + * @copyright 2022 Paul Holden + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class files extends datasource { + + /** + * Return user friendly name of the report source + * + * @return string + */ + public static function get_name(): string { + return get_string('files'); + } + + /** + * Initialise report + */ + protected function initialise(): void { + $fileentity = new file(); + $filesalias = $fileentity->get_table_alias('files'); + + $this->set_main_table('files', $filesalias); + $this->add_entity($fileentity); + + // Join the user entity. + $userentity = new user(); + $useralias = $userentity->get_table_alias('user'); + $this->add_entity($userentity + ->add_join("LEFT JOIN {user} {$useralias} ON {$useralias}.id = {$filesalias}.userid") + ); + + // Add report elements from each of the entities we added to the report. + $this->add_all_from_entities(); + } + + /** + * Return the columns that will be added to the report upon creation + * + * @return string[] + */ + public function get_default_columns(): array { + return [ + 'file:context', + 'user:fullname', + 'file:name', + 'file:type', + 'file:size', + 'file:timecreated', + ]; + } + + /** + * Return the filters that will be added to the report upon creation + * + * @return string[] + */ + public function get_default_filters(): array { + return [ + 'file:size', + 'file:timecreated', + ]; + } + + /** + * Return the conditions that will be added to the report upon creation + * + * @return string[] + */ + public function get_default_conditions(): array { + return ['file:directory']; + } + + /** + * Return the condition values that will be set for the report upon creation + * + * @return array + */ + public function get_default_condition_values(): array { + return ['file:directory_operator' => boolean_select::NOT_CHECKED]; + } +} diff --git a/files/classes/reportbuilder/local/entities/file.php b/files/classes/reportbuilder/local/entities/file.php new file mode 100644 index 00000000000..53687f98550 --- /dev/null +++ b/files/classes/reportbuilder/local/entities/file.php @@ -0,0 +1,374 @@ +. + +declare(strict_types=1); + +namespace core_files\reportbuilder\local\entities; + +use context; +use context_helper; +use lang_string; +use license_manager; +use html_writer; +use stdClass; +use core_reportbuilder\local\entities\base; +use core_reportbuilder\local\helpers\format; +use core_reportbuilder\local\filters\{boolean_select, date, number, select, text}; +use core_reportbuilder\local\report\{column, filter}; + +/** + * File entity + * + * @package core_files + * @copyright 2022 Paul Holden + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class file extends base { + + /** + * Database tables that this entity uses and their default aliases + * + * @return array + */ + protected function get_default_table_aliases(): array { + return [ + 'files' => 'f', + 'context' => 'fctx', + ]; + } + + /** + * The default title for this entity + * + * @return lang_string + */ + protected function get_default_entity_title(): lang_string { + return new lang_string('file'); + } + + /** + * Initialise the entity + * + * @return base + */ + public function initialise(): base { + $columns = $this->get_all_columns(); + foreach ($columns as $column) { + $this->add_column($column); + } + + // All the filters defined by the entity can also be used as conditions. + $filters = $this->get_all_filters(); + foreach ($filters as $filter) { + $this + ->add_filter($filter) + ->add_condition($filter); + } + + return $this; + } + + /** + * Returns list of all available columns + * + * @return column[] + */ + protected function get_all_columns(): array { + $filesalias = $this->get_table_alias('files'); + $contextalias = $this->get_table_alias('context'); + + // Name. + $columns[] = (new column( + 'name', + new lang_string('filename', 'core_repository'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_field("{$filesalias}.filename") + ->set_is_sortable(true); + + // Size. + $columns[] = (new column( + 'size', + new lang_string('size'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_INTEGER) + ->add_fields("{$filesalias}.filesize, {$filesalias}.filename") + ->set_is_sortable(true) + ->add_callback(static function($filesize, stdClass $fileinfo): string { + // Absent file size and/or directory should not return output. + if ($fileinfo->filesize === null || $fileinfo->filename === '.') { + return ''; + } + return display_size($fileinfo->filesize); + }); + + // Path. + $columns[] = (new column( + 'path', + new lang_string('path'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_field("{$filesalias}.filepath") + ->set_is_sortable(true); + + // Type. + $columns[] = (new column( + 'type', + new lang_string('type', 'core_repository'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_fields("{$filesalias}.mimetype, {$filesalias}.filename") + ->set_is_sortable(true) + ->add_callback(static function($mimetype, stdClass $fileinfo): string { + global $CFG; + require_once("{$CFG->libdir}/filelib.php"); + + // Absent mime type and/or directory has pre-determined output. + if ($fileinfo->filename === '.') { + return get_string('directory'); + } else if ($fileinfo->mimetype === null) { + return ''; + } + + return get_mimetype_description($fileinfo, true); + }); + + // Author. + $columns[] = (new column( + 'author', + new lang_string('author', 'core_repository'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_field("{$filesalias}.author") + ->set_is_sortable(true); + + // License. + $columns[] = (new column( + 'license', + new lang_string('license', 'core_repository'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_field("{$filesalias}.license") + ->set_is_sortable(true) + ->add_callback(static function(?string $license): string { + global $CFG; + require_once("{$CFG->libdir}/licenselib.php"); + + $licenses = license_manager::get_licenses(); + if ($license === null || !array_key_exists($license, $licenses)) { + return ''; + } + return $licenses[$license]->fullname; + }); + + // Context. + $columns[] = (new column( + 'context', + new lang_string('context'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_join("LEFT JOIN {context} {$contextalias} ON {$contextalias}.id = {$filesalias}.contextid") + ->add_fields("{$filesalias}.contextid, " . context_helper::get_preload_record_columns_sql($contextalias)) + // Sorting may not order alphabetically, but will at least group contexts together. + ->set_is_sortable(true) + ->add_callback(static function($contextid, stdClass $context): string { + if ($contextid === null) { + return ''; + } + + context_helper::preload_from_record($context); + return context::instance_by_id($contextid)->get_context_name(); + }); + + // Context link. + $columns[] = (new column( + 'contexturl', + new lang_string('contexturl'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_join("LEFT JOIN {context} {$contextalias} ON {$contextalias}.id = {$filesalias}.contextid") + ->add_fields("{$filesalias}.contextid, " . context_helper::get_preload_record_columns_sql($contextalias)) + // Sorting may not order alphabetically, but will at least group contexts together. + ->set_is_sortable(true) + ->add_callback(static function($contextid, stdClass $context): string { + if ($contextid === null) { + return ''; + } + + context_helper::preload_from_record($context); + $context = context::instance_by_id($contextid); + + return html_writer::link($context->get_url(), $context->get_context_name()); + }); + + // Component. + $columns[] = (new column( + 'component', + new lang_string('plugin'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_fields("{$filesalias}.component") + ->set_is_sortable(true); + + // Area. + $columns[] = (new column( + 'area', + new lang_string('pluginarea'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TEXT) + ->add_fields("{$filesalias}.filearea") + ->set_is_sortable(true); + + // Item ID. + $columns[] = (new column( + 'itemid', + new lang_string('pluginitemid'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_INTEGER) + ->add_fields("{$filesalias}.itemid") + ->set_is_sortable(true) + ->set_disabled_aggregation_all(); + + // Time created. + $columns[] = (new column( + 'timecreated', + new lang_string('timecreated', 'core_reportbuilder'), + $this->get_entity_name() + )) + ->add_joins($this->get_joins()) + ->set_type(column::TYPE_TIMESTAMP) + ->add_field("{$filesalias}.timecreated") + ->add_callback([format::class, 'userdate']) + ->set_is_sortable(true); + + return $columns; + } + + /** + * Return list of all available filters + * + * @return filter[] + */ + protected function get_all_filters(): array { + $filesalias = $this->get_table_alias('files'); + + // Directory. + $filters[] = (new filter( + boolean_select::class, + 'directory', + new lang_string('directory'), + $this->get_entity_name(), + "CASE WHEN {$filesalias}.filename = '.' THEN 1 ELSE 0 END" + )) + ->add_joins($this->get_joins()); + + // Draft. + $filters[] = (new filter( + boolean_select::class, + 'draft', + new lang_string('areauserdraft', 'core_repository'), + $this->get_entity_name(), + "CASE WHEN {$filesalias}.component = 'user' AND {$filesalias}.filearea = 'draft' THEN 1 ELSE 0 END" + )) + ->add_joins($this->get_joins()); + + // Name. + $filters[] = (new filter( + text::class, + 'name', + new lang_string('filename', 'core_repository'), + $this->get_entity_name(), + "{$filesalias}.filename" + )) + ->add_joins($this->get_joins()); + + // Size. + $filters[] = (new filter( + number::class, + 'size', + new lang_string('size'), + $this->get_entity_name(), + "{$filesalias}.filesize" + )) + ->add_joins($this->get_joins()) + ->set_limited_operators([ + number::ANY_VALUE, + number::LESS_THAN, + number::GREATER_THAN, + number::RANGE, + ]); + + // License (consider null = 'unknown/license not specified' for filtering purposes). + $filters[] = (new filter( + select::class, + 'license', + new lang_string('license', 'core_repository'), + $this->get_entity_name(), + "COALESCE({$filesalias}.license, 'unknown')" + )) + ->add_joins($this->get_joins()) + ->set_options_callback(static function(): array { + global $CFG; + require_once("{$CFG->libdir}/licenselib.php"); + + $licenses = license_manager::get_licenses(); + + return array_map(static function(stdClass $license): string { + return $license->fullname; + }, $licenses); + }); + + // Time created. + $filters[] = (new filter( + date::class, + 'timecreated', + new lang_string('timecreated', 'core_reportbuilder'), + $this->get_entity_name(), + "{$filesalias}.timecreated" + )) + ->add_joins($this->get_joins()) + ->set_limited_operators([ + date::DATE_ANY, + date::DATE_RANGE, + date::DATE_LAST, + date::DATE_CURRENT, + ]); + + return $filters; + } +} diff --git a/files/tests/reportbuilder/datasource/files_test.php b/files/tests/reportbuilder/datasource/files_test.php new file mode 100644 index 00000000000..512ad59f970 --- /dev/null +++ b/files/tests/reportbuilder/datasource/files_test.php @@ -0,0 +1,323 @@ +. + +declare(strict_types=1); + +namespace core_files\reportbuilder\datasource; + +use context_course; +use context_user; +use core_collator; +use core_reportbuilder_generator; +use core_reportbuilder_testcase; +use core_reportbuilder\local\filters\{boolean_select, date, number, select, text}; + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once("{$CFG->dirroot}/reportbuilder/tests/helpers.php"); + +/** + * Unit tests for files datasource + * + * @package core_files + * @covers \core_files\reportbuilder\datasource\files + * @copyright 2022 Paul Holden + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class files_test extends core_reportbuilder_testcase { + + /** + * Test default datasource + */ + public function test_datasource_default(): void { + $this->resetAfterTest(); + + $user = $this->getDataGenerator()->create_user(); + $usercontext = context_user::instance($user->id); + + $this->setUser($user); + + $course = $this->getDataGenerator()->create_course(); + $coursecontext = context_course::instance($course->id); + + $this->generate_test_files($coursecontext); + + /** @var core_reportbuilder_generator $generator */ + $generator = $this->getDataGenerator()->get_plugin_generator('core_reportbuilder'); + $report = $generator->create_report(['name' => 'Files', 'source' => files::class, 'default' => 1]); + + $content = $this->get_custom_report_content($report->get('id')); + $content = $this->filter_custom_report_content($content, static function(array $row): bool { + return $row['c0_contextid'] !== 'System'; + }, 'c0_contextid'); + + $this->assertCount(2, $content); + + // First row (course summary file). + [$contextname, $userfullname, $filename, $mimetype, $filesize, $timecreated] = array_values($content[0]); + + $this->assertEquals($coursecontext->get_context_name(), $contextname); + $this->assertEquals(fullname($user), $userfullname); + $this->assertEquals('Hello.txt', $filename); + $this->assertEquals('Text file', $mimetype); + $this->assertEquals("5\xc2\xa0bytes", $filesize); + $this->assertNotEmpty($timecreated); + + // Second row (user draft file). + [$contextname, $userfullname, $filename, $mimetype, $filesize, $timecreated] = array_values($content[1]); + + $this->assertEquals($usercontext->get_context_name(), $contextname); + $this->assertEquals(fullname($user), $userfullname); + $this->assertEquals('Hello.txt', $filename); + $this->assertEquals('Text file', $mimetype); + $this->assertEquals("5\xc2\xa0bytes", $filesize); + $this->assertNotEmpty($timecreated); + } + + /** + * Test datasource columns that aren't added by default + */ + public function test_datasource_non_default_columns(): void { + $this->resetAfterTest(); + $this->setAdminUser(); + + $user = $this->getDataGenerator()->create_user(); + $usercontext = context_user::instance($user->id); + + $this->setUser($user); + + $course = $this->getDataGenerator()->create_course(); + $coursecontext = context_course::instance($course->id); + + $draftitemid = $this->generate_test_files($coursecontext); + + /** @var core_reportbuilder_generator $generator */ + $generator = $this->getDataGenerator()->get_plugin_generator('core_reportbuilder'); + $report = $generator->create_report(['name' => 'Files', 'source' => files::class, 'default' => 0]); + + $generator->create_column(['reportid' => $report->get('id'), 'uniqueidentifier' => 'file:contexturl']); + $generator->create_column(['reportid' => $report->get('id'), 'uniqueidentifier' => 'file:path']); + $generator->create_column(['reportid' => $report->get('id'), 'uniqueidentifier' => 'file:author']); + $generator->create_column(['reportid' => $report->get('id'), 'uniqueidentifier' => 'file:license']); + $generator->create_column(['reportid' => $report->get('id'), 'uniqueidentifier' => 'file:component']); + $generator->create_column(['reportid' => $report->get('id'), 'uniqueidentifier' => 'file:area']); + $generator->create_column(['reportid' => $report->get('id'), 'uniqueidentifier' => 'file:itemid']); + + $content = $this->get_custom_report_content($report->get('id')); + $content = $this->filter_custom_report_content($content, static function(array $row): bool { + return stripos($row['c0_contextid'], 'System') === false; + }, 'c0_contextid'); + + // There should be two entries (directory & file) for each context. + $this->assertEquals([ + [ + "get_url()}\">{$coursecontext->get_context_name()}", + '/', + null, + '', + 'course', + 'summary', + 0, + ], + [ + "get_url()}\">{$coursecontext->get_context_name()}", + '/', + null, + '', + 'course', + 'summary', + 0, + ], + [ + "get_url()}\">{$usercontext->get_context_name()}", + '/', + null, + '', + 'user', + 'draft', + $draftitemid, + ], + [ + "get_url()}\">{$usercontext->get_context_name()}", + '/', + null, + '', + 'user', + 'draft', + $draftitemid, + ], + ], array_map('array_values', $content)); + } + + /** + * Data provider for {@see test_datasource_filters} + * + * @return array[] + */ + public function datasource_filters_provider(): array { + return [ + // File. + 'Filter directory' => ['file:directory', [ + 'file:directory_operator' => boolean_select::CHECKED, + ], 2], + 'Filter draft' => ['file:draft', [ + 'file:draft_operator' => boolean_select::CHECKED, + ], 2], + 'Filter name' => ['file:name', [ + 'file:name_operator' => text::IS_EQUAL_TO, + 'file:name_value' => 'Hello.txt', + ], 2], + 'Filter size' => ['file:size', [ + 'file:size_operator' => number::GREATER_THAN, + 'file:size_value1' => 2, + ], 2], + 'Filter license' => ['file:license', [ + 'file:license_operator' => select::EQUAL_TO, + 'file:license_value' => 'unknown', + ], 4], + 'Filter license (non match)' => ['file:license', [ + 'file:license_operator' => select::EQUAL_TO, + 'file:license_value' => 'public', + ], 0], + 'Filter time created' => ['file:timecreated', [ + 'file:timecreated_operator' => date::DATE_RANGE, + 'file:timecreated_from' => 1622502000, + ], 4], + 'Filter time created (non match)' => ['file:timecreated', [ + 'file:timecreated_operator' => date::DATE_RANGE, + 'file:timecreated_to' => 1622502000, + ], 0], + + // User (just to check the join). + 'Filter user' => ['user:username', [ + 'user:username_operator' => text::IS_EQUAL_TO, + 'user:username_value' => 'alfie', + ], 4], + 'Filter user (no match)' => ['user:username', [ + 'user:username_operator' => text::IS_EQUAL_TO, + 'user:username_value' => 'lionel', + ], 0], + ]; + } + + /** + * Test datasource filters + * + * @param string $filtername + * @param array $filtervalues + * @param int $expectmatchcount + * + * @dataProvider datasource_filters_provider + */ + public function test_datasource_filters( + string $filtername, + array $filtervalues, + int $expectmatchcount + ): void { + $this->resetAfterTest(); + $this->setAdminUser(); + + $user = $this->getDataGenerator()->create_user(['username' => 'alfie']); + $this->setUser($user); + + $course = $this->getDataGenerator()->create_course(); + $coursecontext = context_course::instance($course->id); + + $this->generate_test_files($coursecontext); + + /** @var core_reportbuilder_generator $generator */ + $generator = $this->getDataGenerator()->get_plugin_generator('core_reportbuilder'); + + // Create report containing single column, and given filter. + $report = $generator->create_report(['name' => 'Files', 'source' => files::class, 'default' => 0]); + $generator->create_column(['reportid' => $report->get('id'), 'uniqueidentifier' => 'file:context']); + + // Add filter, set it's values. + $generator->create_filter(['reportid' => $report->get('id'), 'uniqueidentifier' => $filtername]); + $content = $this->get_custom_report_content($report->get('id'), 0, $filtervalues); + $content = $this->filter_custom_report_content($content, static function(array $row): bool { + return stripos($row['c0_contextid'], 'System') === false; + }, 'c0_contextid'); + + $this->assertCount($expectmatchcount, $content); + } + + /** + * Stress test datasource + * + * In order to execute this test PHPUNIT_LONGTEST should be defined as true in phpunit.xml or directly in config.php + */ + public function test_stress_datasource(): void { + if (!PHPUNIT_LONGTEST) { + $this->markTestSkipped('PHPUNIT_LONGTEST is not defined'); + } + + $this->resetAfterTest(); + $this->setAdminUser(); + + $course = $this->getDataGenerator()->create_course(); + $coursecontext = context_course::instance($course->id); + + $this->generate_test_files($coursecontext); + + $this->datasource_stress_test_columns(files::class); + $this->datasource_stress_test_columns_aggregation(files::class); + $this->datasource_stress_test_conditions(files::class, 'file:path'); + } + + /** + * Ensuring report content only includes files we have explicitly created within the test, and ordering them + * + * @param array $content + * @param callable $callback + * @param string $sortfield + * @return array + */ + protected function filter_custom_report_content(array $content, callable $callback, string $sortfield): array { + $content = array_filter($content, $callback); + core_collator::asort_array_of_arrays_by_key($content, $sortfield); + return array_values($content); + } + + /** + * Helper method to generate some test files for reporting on + * + * @param context_course $context + * @return int Draft item ID + */ + protected function generate_test_files(context_course $context): int { + global $USER; + + $draftitemid = file_get_unused_draft_itemid(); + + // Populate user draft. + get_file_storage()->create_file_from_string([ + 'contextid' => context_user::instance($USER->id)->id, + 'userid' => $USER->id, + 'component' => 'user', + 'filearea' => 'draft', + 'itemid' => $draftitemid, + 'filepath' => '/', + 'filename' => 'Hello.txt', + ], 'Hello'); + + // Save draft to course summary file area. + file_save_draft_area_files($draftitemid, $context->id, 'course', 'summary', 0); + + return $draftitemid; + } +} diff --git a/lang/en/moodle.php b/lang/en/moodle.php index c767cc1a21b..c36f447ff46 100644 --- a/lang/en/moodle.php +++ b/lang/en/moodle.php @@ -1656,8 +1656,10 @@ $string['pleaseclose'] = 'Please close this window now.'; $string['pleasesearchmore'] = 'Please search some more'; $string['pleaseusesearch'] = 'Please use the search'; $string['plugin'] = 'Plugin'; +$string['pluginarea'] = 'Area'; $string['plugindeletefiles'] = 'All data associated with the plugin \'{$a->name}\' has been deleted from the database. To prevent the plugin re-installing itself, you should now delete this directory from your server: {$a->directory}'; $string['plugincheck'] = 'Plugins check'; +$string['pluginitemid'] = 'Item ID'; $string['pluginsetup'] = 'Setting up plugin tables'; $string['policyaccept'] = 'I understand and agree'; $string['policyagree'] = 'You must agree to this policy to continue using this site. Do you agree?';