L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MDL-40084 mod_data: Add file import support

Browse Source 
Co-authored-by: Stefan Hanauska <[email protected]>
This commit is contained in:
Philipp Memmel
2023-06-29 19:18:15 +02:00
co-authored by Stefan Hanauska
parent e11b3867ae
commit cf2c91dae2
11 changed files with 284 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mod/data/classes/local/importer.php
+77 -7
View File
@@ -19,11 +19,10 @@ namespace mod_data\local;
use coding_exception;
use core_php_time_limit;
use file_packer;
use file_serving_exception;
use zip_archive;
use moodle_exception;
/**
* Importer class for importing data.
* Importer class for importing data and - if needed - files as well from a zip archive.
*
* @package mod_data
* @copyright 2023 ISB Bayern
@@ -41,10 +40,20 @@ abstract class importer {
/** @var string $importfiletype The file type of the import file. */
protected string $importfiletype;
/** @var file_packer Zip file packer to extract files from a zip archive. */
private file_packer $packer;
/** @var bool Tracks state if zip archive has been extracted already. */
private bool $zipfileextracted;
/** @var string Temporary directory where zip archive is being extracted to. */
private string $extracteddir;
/**
* Creates an importer object.
*
* This object can be used to import data from data files (like csv).
* This object can be used to import data from data files (like csv) and zip archives both including a data file and files to be
* stored in the course module context.
*
* @param string $importfilepath the complete path of the import file including filename
* @param string $importfilename the import file name as uploaded by the user
@@ -54,8 +63,10 @@ abstract class importer {
$this->importfilepath = $importfilepath;
$this->importfilename = $importfilename;
$this->importfiletype = pathinfo($importfilename, PATHINFO_EXTENSION);
if ($this->importfiletype !== $this->get_import_data_file_extension()) {
throw new coding_exception('Only ' . $this->get_import_data_file_extension() . '" files are allowed.');
$this->zipfileextracted = false;
if ($this->importfiletype !== $this->get_import_data_file_extension() && $this->importfiletype !== 'zip') {
throw new coding_exception('Only "zip" or "' . $this->get_import_data_file_extension() . '" files are '
. 'allowed.');
}
}
@@ -69,9 +80,68 @@ abstract class importer {
/**
* Returns the file content of the data file.
*
* Returns the content of the file directly if the importer's file is a data file itself. If the importer's file is a zip
* archive, the content of the first found data file in the zip archive's root will be returned.
*
* @return false|string the data file content as string; false, if file cannot be found/read
* @throws moodle_exception
*/
public function get_data_file_content(): false|string {
return file_get_contents($this->importfilepath);
if ($this->importfiletype !== 'zip') {
// We have no zip archive, so the file itself must be the data file.
return file_get_contents($this->importfilepath);
}
// So we have a zip archive and need to find the right data file in the root of the zip archive.
$this->extract_zip();
$datafilenames = array_filter($this->packer->list_files($this->importfilepath),
fn($file) => pathinfo($file->pathname, PATHINFO_EXTENSION) === $this->get_import_data_file_extension()
&& !str_contains($file->pathname, '/'));
if (empty($datafilenames) || count($datafilenames) > 1) {
return false;
}
return file_get_contents($this->extracteddir . reset($datafilenames)->pathname);
}
/**
* Returns the file content from a file which has been stored in the zip archive.
*
* @param string $filename
* @param string $zipsubdir
* @return false|string the file content as string, false if the file could not be found/read
* @throws moodle_exception
*/
public function get_file_content_from_zip(string $filename, string $zipsubdir = 'files/'): false|string {
if (empty($filename)) {
// Nothing to return.
return false;
}
// Just to be sure extract if not extracted yet.
$this->extract_zip();
if (!str_ends_with($zipsubdir, '/')) {
$zipsubdir .= '/';
}
$filepathinextractedzip = $this->extracteddir . $zipsubdir . $filename;
return file_exists($filepathinextractedzip) ? file_get_contents($filepathinextractedzip) : false;
}
/**
* Extracts (if not already done and if we have a zip file to deal with) the zip file to a temporary directory.
*
* @return void
* @throws moodle_exception
*/
private function extract_zip(): void {
if ($this->zipfileextracted || $this->importfiletype !== 'zip') {
return;
}
$this->packer = get_file_packer();
core_php_time_limit::raise(180);
$this->extracteddir = make_request_directory();
if (!str_ends_with($this->extracteddir, '/')) {
$this->extracteddir .= '/';
}
$this->packer->extract_to_pathname($this->importfilepath, $this->extracteddir);
$this->zipfileextracted = true;
}
}
mod/data/classes/local/mod_data_csv_importer.php
+12 -2
View File
@@ -27,7 +27,7 @@ use moodle_exception;
use stdClass;
/**
* CSV importer class for importing data.
* CSV importer class for importing data and - if needed - files as well from a zip archive.
*
* @package mod_data
* @copyright 2023 ISB Bayern
@@ -143,7 +143,17 @@ class mod_data_csv_importer extends csv_importer {
$content->fieldid = $field->field->id;
$content->content = $value;
$content->recordid = $recordid;
$DB->insert_record('data_content', $content);
if ($field->file_import_supported() && $this->importfiletype === 'zip') {
$filecontent = $this->get_file_content_from_zip($content->content);
if (!$filecontent) {
// No corresponding file in zip archive, so no record for this field being added at all.
continue;
}
$contentid = $DB->insert_record('data_content', $content);
$field->import_file_value($contentid, $filecontent, $content->content);
} else {
$DB->insert_record('data_content', $content);
}
}
}
mod/data/field/file/field.class.php
+32
View File
@@ -246,6 +246,38 @@ class data_field_file extends data_field_base {
return true;
}
/**
* Specifies that this field type supports the import of files.
*
* @return bool true which means that file import is being supported by this field type
*/
public function file_import_supported(): bool {
return true;
}
/**
* Provides the necessary code for importing a file when importing the content of a mod_data instance.
*
* @param int $contentid the id of the mod_data content record
* @param string $filecontent the content of the file to import as string
* @param string $filename the filename the imported file should get
* @return void
* @throws file_exception
* @throws stored_file_creation_exception
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
$filerecord = [
'contextid' => $this->context->id,
'component' => 'mod_data',
'filearea' => 'content',
'itemid' => $contentid,
'filepath' => '/',
'filename' => $filename,
];
$fs = get_file_storage();
$fs->create_file_from_string($filerecord, $filecontent);
}
/**
* Exports the file content for file export.
*
mod/data/field/picture/field.class.php
+33
View File
@@ -391,6 +391,39 @@ class data_field_picture extends data_field_base {
return $file ? $file->get_content() : null;
}
/**
* Specifies that this field type supports the import of files.
*
* @return bool true which means that file import is being supported by this field type
*/
public function file_import_supported(): bool {
return true;
}
/**
* Provides the necessary code for importing a file when importing the content of a mod_data instance.
*
* @param int $contentid the id of the mod_data content record
* @param string $filecontent the content of the file to import as string
* @param string $filename the filename the imported file should get
* @return void
* @throws file_exception
* @throws stored_file_creation_exception
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
$filerecord = [
'contextid' => $this->context->id,
'component' => 'mod_data',
'filearea' => 'content',
'itemid' => $contentid,
'filepath' => '/',
'filename' => $filename,
];
$fs = get_file_storage();
$file = $fs->create_file_from_string($filerecord, $filecontent);
$this->update_thumbnail(null, $file);
}
function file_ok($path) {
return true;
}
mod/data/import_form.php
+2 -1
View File
@@ -14,7 +14,8 @@ class mod_data_import_form extends moodleform {
$dataid = $this->_customdata['dataid'];
$backtourl = $this->_customdata['backtourl'];
$mform->addElement('filepicker', 'recordsfile', get_string('csvfile', 'data'));
$mform->addElement('filepicker', 'recordsfile', get_string('csvorzipfile', 'data'),
null, ['accepted_types' => ['application/zip', 'text/csv']]);
$delimiters = csv_import_reader::get_delimiter_list();
$mform->addElement('select', 'fielddelimiter', get_string('fielddelimiter', 'data'), $delimiters);
mod/data/lang/en/data.php
+5 -9
View File
@@ -83,7 +83,7 @@ $string['createfields'] = 'Create fields to collect different types of data.';
$string['createtemplates'] = 'Templates define the interface of your activity. Once you create fields, templates will be created automatically. Alternatively, you can use a preset, which includes fields and templates.';
$string['csstemplate'] = 'Custom CSS';
$string['csvfailed'] = 'Unable to read the raw data from the CSV file';
$string['csvfile'] = 'CSV file';
$string['csvorzipfile'] = 'CSV or ZIP containing a CSV file';
$string['csvimport'] = 'CSV file import';
$string['csvimport_help'] = 'Entries may be imported via a plain text file with a list of field names as the first line, then the data, with one record per line.';
$string['csvwithselecteddelimiter'] = '<abbr title="Comma Separated Values">CSV</abbr>';
@@ -152,7 +152,7 @@ $string['entrieslefttoadd'] = 'You must add {$a->entriesleft} more entry/entries
$string['entrieslefttoaddtoview'] = 'You must add {$a->entrieslefttoview} more entry/entries before you can view other participants\' entries.';
$string['entry'] = 'Entry';
$string['entrysaved'] = 'Your entry has been saved';
$string['errordatafilenotfound'] = 'The file could not be imported. Please upload a CSV file.';
$string['errordatafilenotfound'] = 'The file could not be imported. Accepted file types are CSV or a ZIP containing a CSV file in the format used for exporting entries.';
$string['errormustbeteacher'] = 'You need to be a teacher to use this page!';
$string['errorpresetexists'] = 'A preset with this name already exists.';
$string['errorpresetexistsbutnotoverwrite'] = 'A preset with this name already exists. Choose a different name.';
@@ -455,14 +455,10 @@ $string['unsupportedfields'] = 'Unsupported fields';
$string['unsupportedfieldslist'] = 'The following fields cannot be exported:';
$string['updatefield'] = 'Update an existing field';
$string['uploadfile'] = 'Upload file';
$string['uploadrecords'] = 'Upload entries from a file';
$string['uploadrecords_help'] = 'Entries may be uploaded via text file. The format of the file should be as follows:
$string['uploadrecords'] = 'Import entries';
$string['uploadrecords_help'] = 'Import entries that you have exported from another database, either via CSV or a ZIP containing a CSV file (if files are included in the export).
* Each line of the file contains one record
* Each record is a series of data separated by the selected separator
* The first record contains a list of fieldnames defining the format of the rest of the file
The field enclosure is a character that surrounds each field in each record. It can normally be left unset.';
Alternatively, to create a CSV file for importing, add one entry to the database and then export it. Edit the CSV file and add more entries.';
$string['uploadrecords_link'] = 'mod/data/import';
$string['url'] = 'URL';
$string['usedate'] = 'Include in search.';
mod/data/lib.php
+23
View File
@@ -661,6 +661,29 @@ class data_field_base { // Base class for Database Field Types (see field/*/
return null;
}
/**
* Per default, a field does not support the import of files.
*
* A field type can overwrite this function and return true. In this case it also has to implement the function
* import_file_value().
*
* @return false means file imports are not supported
*/
public function file_import_supported(): bool {
return false;
}
/**
* Returns a stored_file object for exporting a file of a given record.
*
* @param int $contentid content id
* @param string $filecontent the content of the file as string
* @param string $filename the filename the file should have
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
return;
}
/**
* Per default, return the record's text value only from the "content" field.
* Override this in fields class if necessary.
mod/data/tests/fixtures/test_data_import_with_files.zip
Vendored
BIN
View File
Binary file not shown.
mod/data/tests/fixtures/test_data_import_with_files_missing_file.zip
Vendored
BIN
View File
Binary file not shown.
mod/data/tests/import_test.php
+96 -4
View File
@@ -20,6 +20,7 @@ use coding_exception;
use dml_exception;
use mod_data\local\mod_data_csv_importer;
use moodle_exception;
use zip_archive;
/**
* Unit tests for import.php.
@@ -73,6 +74,13 @@ class import_test extends \advanced_testcase {
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield';
$fieldrecord->type = 'file';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'picturefield';
$fieldrecord->type = 'picture';
$generator->create_field($fieldrecord, $data);
return [
'teacher' => $teacher,
@@ -116,8 +124,6 @@ class import_test extends \advanced_testcase {
*
* At least one entry has an identifiable user, which is assigned as author.
*
* @throws coding_exception
* @throws moodle_exception
* @throws dml_exception
*/
public function test_import_with_userdata(): void {
@@ -151,7 +157,6 @@ class import_test extends \advanced_testcase {
* as the current lang string for username. In that case, the first Username entry is used for the field.
* The second one is used to identify the author.
*
* @throws moodle_exception
* @throws coding_exception
* @throws dml_exception
*/
@@ -214,8 +219,8 @@ class import_test extends \advanced_testcase {
* as the current lang string for username. In that case, the only Username entry is used for the field.
* The author should not be set.
*
* @throws coding_exception
* @throws dml_exception
* @throws moodle_exception
*/
public function test_import_with_field_username_without_userdata(): void {
[
@@ -265,11 +270,98 @@ class import_test extends \advanced_testcase {
}
}
/**
* Tests the import including files from a zip archive.
*
* @covers \mod_data\local\importer
* @covers \mod_data\local\csv_importer
* @return void
* @throws coding_exception
* @throws moodle_exception
* @throws dml_exception
*/
public function test_import_with_files(): void {
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$importer = new mod_data_csv_importer(__DIR__ . '/fixtures/test_data_import_with_files.zip',
'test_data_import_with_files.zip');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$records = $this->get_data_records($data->id);
$ziparchive = new zip_archive();
$ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files.zip');
$importedcontent = array_values($records)[0]->items;
$this->assertEquals(17, $importedcontent['ID']->content);
$this->assertEquals('samplefile.png', $importedcontent['filefield']->content);
$this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
// We now check if content of imported file from zip content is identical to the content of the file
// stored in the mod_data record in the field 'filefield'.
$fileindex = array_values(array_map(fn($file) => $file->index,
array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplefile.png')))[0];
$filestream = $ziparchive->get_stream($fileindex);
$filefield = data_get_field_from_name('filefield', $data);
$filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
$this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
$filefieldfilecontent);
fclose($filestream);
// We now check if content of imported picture from zip content is identical to the content of the picture file
// stored in the mod_data record in the field 'picturefield'.
$fileindex = array_values(array_map(fn($file) => $file->index,
array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplepicture.png')))[0];
$filestream = $ziparchive->get_stream($fileindex);
$filefield = data_get_field_from_name('picturefield', $data);
$filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
$this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
$filefieldfilecontent);
fclose($filestream);
$ziparchive->close();
}
/**
* Tests the import including files from a zip archive.
*
* @covers \mod_data\local\importer
* @covers \mod_data\local\csv_importer
* @return void
* @throws coding_exception
* @throws moodle_exception
* @throws dml_exception
*/
public function test_import_with_files_missing_file(): void {
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$importer = new mod_data_csv_importer(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip',
'test_data_import_with_files_missing_file.zip');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$records = $this->get_data_records($data->id);
$ziparchive = new zip_archive();
$ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip');
$importedcontent = array_values($records)[0]->items;
$this->assertEquals(17, $importedcontent['ID']->content);
$this->assertFalse(isset($importedcontent['filefield']));
$this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
$ziparchive->close();
}
/**
* Returns the records of the data instance.
*
* Each records has an item entry, which contains all fields associated with this item.
* Each fields has the parameters name, type and content.
*
* @param int $dataid Id of the data instance.
* @return array The records of the data instance.
* @throws dml_exception
mod/data/upgrade.txt
+4
View File
@@ -16,6 +16,10 @@ information provided here is intended especially for developers.
type will have to export the name of the file by overwriting text_export_supported() to return true and make the
method export_text_value() return the name of the file.
* The field types file and picture now are able to export the file/picture.
* Field base class now has two new methods file_import_supported() and import_file_value(). The method
file_import_supported() can be overwritten to declare that a field type is able to import a file. In this case this
field type will have to implement the method import_file_value() doing the actual import of the file being passed.
* The field types file and picture now are able to import the file/picture.
== 4.2 ==
* The field base class now has a method validate(). Overwrite it in the field type to provide validation of field type's