From d0e82b746ea167e1d99e9ccca670f694377af73e Mon Sep 17 00:00:00 2001 From: Trisha Milan Date: Fri, 10 May 2024 12:10:32 +1000 Subject: [PATCH] MDL-81807 core: Add scheduled task for updating the GeoIP database This commit includes the following changes: - Implemented a scheduled task to automate the download and update of the GeoIP database. - Integrated checksum verification for the downloaded database file. - Added new configuration settings to enable administrators to enter MaxMind account credentials, allowing authenticated access to both free and paid GeoIP database versions from Maxmind. --- admin/settings/location.php | 21 ++- lang/en/admin.php | 17 +- lang/en/error.php | 2 +- lib/classes/task/update_geoip2file_task.php | 183 ++++++++++++++++++++ lib/db/tasks.php | 10 ++ version.php | 2 +- 6 files changed, 227 insertions(+), 8 deletions(-) create mode 100644 lib/classes/task/update_geoip2file_task.php diff --git a/admin/settings/location.php b/admin/settings/location.php index 5a4cc9010c8..3d6809ad655 100644 --- a/admin/settings/location.php +++ b/admin/settings/location.php @@ -44,7 +44,26 @@ if ($hassiteconfig) { $temp->add(new admin_setting_configfile('geoip2file', new lang_string('geoipfile', 'core_admin'), new lang_string('configgeoipfile', 'core_admin', $CFG->dataroot . '/geoip/'), - $CFG->dataroot . '/geoip/GeoLite2-City.mmdb')); + $CFG->dataroot . '/geoip/GeoIP-City.mmdb')); + + $temp->add(new admin_setting_configselect('geoipdbedition', + new lang_string('geoipdbedition', 'core_admin'), + new lang_string('geoipdbedition_desc', 'core_admin'), 'GeoLite2-City', + ['GeoLite2-City' => 'GeoLite2-City', 'GeoIP2-City' => 'GeoIP2-City'])); + + $temp->add(new admin_setting_configtext('geoipmaxmindaccid', + new lang_string('geoipmaxmindaccid', 'core_admin'), + new lang_string('geoipmaxmindaccid_desc', 'core_admin'), + '', + PARAM_TEXT, + )); + + $temp->add(new admin_setting_configtext('geoipmaxmindlicensekey', + new lang_string('geoipmaxmindlicensekey', 'core_admin'), + new lang_string('geoipmaxmindlicensekey_desc', 'core_admin'), + '', + PARAM_TEXT, + )); $temp->add(new admin_setting_configtext('googlemapkey3', new lang_string('googlemapkey3', 'core_admin'), new lang_string('googlemapkey3_help', 'core_admin'), '', PARAM_RAW, 60)); diff --git a/lang/en/admin.php b/lang/en/admin.php index 664b9790dc7..be392c74838 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -286,7 +286,7 @@ $string['configfrontpageloggedin'] = 'The items selected above will be displayed $string['configfullnamedisplay'] = 'This defines how names are shown when they are displayed in full. The default value, "language", leaves it to the string "fullnamedisplay" in the current language pack to decide. Some languages have different name display conventions. For most mono-lingual sites the most efficient setting is "firstname lastname", but you may choose to hide last names altogether. Placeholders that can be used are: firstname, lastname, firstnamephonetic, lastnamephonetic, middlename, and alternatename.'; -$string['configgeoipfile'] = 'Location of GeoLite2 City binary data file. This file is not part of Moodle distribution and must be obtained separately from MaxMind. You can either buy a commercial version or use the free version. You\'ll need to register to download the City database file, which you can do at https://dev.maxmind.com/geoip/geoip2/geolite2/. Once you\'ve registered and downloaded the file, extract it into "{$a}" directory on your server.'; +$string['configgeoipfile'] = 'Location of GeoIP City binary data file. This file is not part of Moodle distribution and must be obtained separately from MaxMind. You can either buy a commercial version or use the free version. You\'ll need to register to download the City database file, which you can do at https://dev.maxmind.com/geoip/geoip2/geolite2/. Once you\'ve registered and downloaded the file, extract it into "{$a}" directory on your server.'; $string['configgetremoteaddrconf'] = 'If your server is behind a reverse proxy, you can use this setting to specify which HTTP headers can be trusted to contain the remote IP address. The headers are read in order, using the first one that is available.'; $string['configgradebookroles'] = 'This setting allows you to control who appears on the gradebook. Users need to have at least one of these roles in a course to be shown in the gradebook for that course.'; $string['configgradeexport'] = 'Choose which gradebook export formats are your primary methods for exporting grades. Chosen plugins will then set and use a "last exported" field for every grade. For example, this might result in exported records being identified as being "new" or "updated". If you are not sure about this then leave everything unchecked.'; @@ -690,7 +690,13 @@ $string['fullnamedisplay'] = 'Full name format'; $string['fullnamedisplayprivate'] = 'Full name format - private'; $string['gdrequired'] = 'The GD extension is now required by Moodle for image conversion.'; $string['generalsettings'] = 'General settings'; -$string['geoipfile'] = 'GeoLite2 City MaxMind DB'; +$string['geoipdbedition'] = 'GeoIP Database Edition'; +$string['geoipdbedition_desc'] = 'Select the edition of the GeoIP database to download. Please be aware that using the GeoIP2-City database requires a valid paid license.'; +$string['geoipfile'] = 'GeoIP City DB'; +$string['geoipmaxmindaccid'] = 'MaxMind account ID'; +$string['geoipmaxmindaccid_desc'] = 'The account ID of the account created on the MaxMind API service.'; +$string['geoipmaxmindlicensekey'] = 'MaxMind license key'; +$string['geoipmaxmindlicensekey_desc'] = 'The license key of the account created on the MaxMind API service.'; $string['getremoteaddrconf'] = 'Logged IP address source'; $string['globalsearch'] = 'Global search'; $string['globalsearchmanage'] = 'Manage global search'; @@ -775,10 +781,10 @@ $string['ipblocked'] = 'This site is not available currently.'; $string['ipblocker'] = 'IP blocker'; $string['ipblockersyntax'] = 'Put every entry on one line. Valid entries are either full IP address (such as 192.168.10.1) which matches a single host; or partial address (such as 192.168) which matches any address starting with those numbers; or CIDR notation (such as 231.54.211.0/20); or a range of IP addresses (such as 231.3.56.10-20) where the range applies to the last part of the address. Text domain names (like \'example.com\') are not supported. Blank lines, and text following a "#" character are ignored.'; $string['iplookup'] = 'IP address lookup'; -$string['iplookupgeoplugin'] = 'The geoPlugin service is currently being used to look up geographical information. For more accurate results we recommend installing a local copy of the MaxMind GeoLite database.'; -$string['iplookupinfo'] = 'The free online NetGeo server is used to look up IP address locations. The database is not maintained any more though, and may return incorrect data. Installing a local copy of the free GeoLite2 City database from MaxMind is recommended.
+$string['iplookupgeoplugin'] = 'The geoPlugin service is currently being used to look up geographical information. For more accurate results we recommend installing a local copy of the MaxMind GeoIP City database.'; +$string['iplookupinfo'] = 'The free online NetGeo server is used to look up IP address locations. The database is not maintained any more though, and may return incorrect data. Installing a local copy of a GeoIP database from MaxMind is recommended.
If a Google Maps API V3 key is provided, IP address location is displayed using Google Maps. You need a Google account and to apply for a Google Maps API key. If no Google Maps API V3 key is provided, IP address location is displayed using OpenStreetMap.'; -$string['iplookupmaxmindnote'] = 'This product includes GeoLite2 data created by MaxMind, available from https://www.maxmind.com.'; +$string['iplookupmaxmindnote'] = 'This product includes GeoIP City data created by MaxMind, available from https://www.maxmind.com.'; $string['iscustomadminwarnings'] = 'It has been detected that your site is using a custom admin directory. This feature is no longer supported and will be removed after Moodle 4.2.'; $string['ishttpswarning'] = 'It has been detected that your site is not secured using HTTPS. It is strongly recommended to migrate your site to HTTPS for increased security and improved integration with other systems.'; $string['keeptagnamecase'] = 'Keep tag name casing'; @@ -1436,6 +1442,7 @@ $string['tasksessioncleanup'] = 'Cleanup old sessions'; $string['taskstatscron'] = 'Background processing for statistics'; $string['tasktagcron'] = 'Background processing for tags'; $string['tasktempfilecleanup'] = 'Delete stale temp files'; +$string['taskupdategeoip2file'] = 'Update GeoIP file'; $string['tempdatafoldercleanup'] = 'Clean up temporary data files older than'; $string['templates'] = 'Templates'; $string['testoutgoingmailconf'] = 'Test outgoing mail configuration'; diff --git a/lang/en/error.php b/lang/en/error.php index 09764f33a61..9de16f6fe16 100644 --- a/lang/en/error.php +++ b/lang/en/error.php @@ -96,7 +96,7 @@ $string['cannotfindinfo'] = 'Cannot find info for: "{$a}"'; $string['cannotfindlang'] = 'Cannot find "{$a}" language pack!'; $string['cannotfindteacher'] = 'Cannot find teacher'; $string['cannotfinduser'] = 'Cannot find user named "{$a}"'; -$string['cannotgeoplugin'] = 'Cannot connect to the geoPlugin server at https://www.geoplugin.com. Please check your proxy settings or install the MaxMind GeoLite City data file.'; +$string['cannotgeoplugin'] = 'Cannot connect to the geoPlugin server at https://www.geoplugin.com. Please check your proxy settings or install the MaxMind GeoIP City data file.'; $string['cannotgetblock'] = 'Could not retrieve blocks from the database'; $string['cannotgetcats'] = 'Cannot get category record'; $string['cannotgetdata'] = 'Cannot get data'; diff --git a/lib/classes/task/update_geoip2file_task.php b/lib/classes/task/update_geoip2file_task.php new file mode 100644 index 00000000000..2e7fb7798c4 --- /dev/null +++ b/lib/classes/task/update_geoip2file_task.php @@ -0,0 +1,183 @@ +. + +namespace core\task; + +use core\http_client; +use moodle_exception; +use PharData; + +/** + * Simple task to update the GeoIP database file. + * + * @package core + * @author Trisha Milan + * @copyright Monash University 2024 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class update_geoip2file_task extends scheduled_task { + + /** + * Get a descriptive name for this task (shown to admins). + * + * @return string + */ + public function get_name(): string { + return get_string('taskupdategeoip2file', 'admin'); + } + + /** + * Execute the task to update the GeoIP2 database file. + * + * @throws \GuzzleHttp\Exception\GuzzleException + * @throws moodle_exception + */ + public function execute(): void { + global $CFG; + + if (!$CFG->geoipmaxmindaccid || !$CFG->geoipmaxmindlicensekey) { + mtrace("MaxMind account information is incomplete. Please configure the account ID and license key."); + return; + } + + // Setup base directory path and permissions. + $geoip2file = $CFG->geoip2file; + $geoipdirectory = dirname($geoip2file); + if (!check_dir_exists($geoipdirectory) && !mkdir($geoipdirectory, $CFG->directorypermissions, true)) { + throw new moodle_exception("Cannot create output directory $geoipdirectory"); + } + + $geoippermalink = 'https://download.maxmind.com/geoip/databases/' . $CFG->geoipdbedition . '/download'; + + $client = new http_client(['auth' => [$CFG->geoipmaxmindaccid, $CFG->geoipmaxmindlicensekey]]); + $response = $client->head($geoippermalink, ['query' => ['suffix' => 'tar.gz']]); + $headers = $response->getHeaders(); + $lastmodified = strtotime($headers['Last-Modified'][0]); + if (!$this->is_update_needed($geoip2file, $lastmodified)) { + mtrace("No update needed. The GeoIP database is up to date."); + return; + } + + // Define path for downloading the GeoIP2 archive. + $archivefilename = 'GeoIP-City.tar.gz'; + $tempdirectory = make_request_directory(true); + $geoipdownloadpath = $tempdirectory . '/' . $archivefilename; + + mtrace("Downloading $CFG->geoipdbedition database from MaxMind......"); + $response = $client->request('GET', $geoippermalink, [ + 'query' => ['suffix' => 'tar.gz'], + 'sink' => $geoipdownloadpath, + ]); + if ($response->getStatusCode() != 200) { + throw new moodle_exception("Error downloading file."); + } + + mtrace("File downloaded successfully to $geoipdownloadpath"); + mtrace("Verifying checksum......"); + + // Get the latest checksum from MaxMind. + $checksumcontent = $client->get($geoippermalink, ['query' => ['suffix' => 'tar.gz.sha256']])->getBody()->getContents(); + list($checksum) = explode(' ', $checksumcontent); + if (!$this->verify_checksum($checksum, $geoipdownloadpath)) { + throw new moodle_exception("Checksum verification failed."); + } + + mtrace("Checksum verified successfully."); + if ($this->update_geoip2file($geoipdownloadpath, $tempdirectory, $geoip2file)) { + // Store the last seen timestamp. + set_config('geoip_last_seen_timestamp', $lastmodified); + mtrace("GeoIP database update successful!"); + } else { + throw new moodle_exception("GeoIP database update failed."); + } + } + + /** + * Determines if an update is needed for the GeoIP2 file based on the last modified date. + * + * @param string $geoip2file The path to the GeoIP2 file that needs to be checked for updates. + * @param string $lastmodified The last modified date to be compared against the stored last seen timestamp. + * @return bool + */ + private function is_update_needed(string $geoip2file, string $lastmodified): bool { + return !file_exists($geoip2file) || $lastmodified !== get_config('core', 'geoip_last_seen_timestamp'); + } + + /** + * Verify the checksum of the downloaded file against an expected checksum. + * + * @param string $expectedchecksum The checksum expected for the file. + * @param string $geoipdownloadpath The path where the downloaded geoip archive is located. + * @return bool Returns true if the checksums match, returns false otherwise. + */ + private function verify_checksum(string $expectedchecksum, string $geoipdownloadpath): bool { + $actualchecksum = hash_file('sha256', $geoipdownloadpath); + return $expectedchecksum === $actualchecksum; + } + + /** + * Extract the archive and update the GeoIP2 database file. + * + * @param string $archivepath The path to the archive file that needs to be extracted. + * @param string $targetdirectory Directory where the archive contents will be extracted. + * @param string $geoip2file The path to move the extracted GeoIP2 file. + * @return bool Returns true if the file was successfully extracted and moved to the specified location, + * false if any part of the process fails. + */ + private function update_geoip2file(string $archivepath, string $targetdirectory, string $geoip2file): bool { + $archive = new PharData($archivepath); + $archivename = $archive->getFilename(); + + mtrace("Extracting file......"); + $archive->extractTo($targetdirectory); + $sourcefolder = $targetdirectory . '/' . $archivename; + + // Find the mmdb file. + $mmdbfiles = glob($sourcefolder . '/*.mmdb'); + if (count($mmdbfiles) > 1) { + throw new moodle_exception("Multiple .mmdb files found in the extracted folder."); + } else if (count($mmdbfiles) === 0) { + throw new moodle_exception("GeoIP file does not exist."); + } + + // Backup existing GeoIP file before attempting to update. + $geoip2filename = basename($geoip2file); + $backuppath = $targetdirectory . '/' . 'backup_' . $geoip2filename; + if (file_exists($geoip2file)) { + if (!rename($geoip2file, $backuppath)) { + mtrace("Failed to create a backup of the existing GeoIP database."); + } + mtrace("Temporary backup of existing GeoIP file has been created."); + } + + mtrace("Moving {$mmdbfiles[0]} into $geoip2file"); + if (!copy($mmdbfiles[0], $geoip2file)) { + mtrace("Failed to update $geoip2filename."); + // Attempt to restore the original file from the backup. + if (file_exists($backuppath)) { + mtrace("Attempting to restore from backup."); + if (!copy($backuppath, $geoip2file)) { + throw new moodle_exception("Failed to restore the GeoIP database from backup."); + } else { + mtrace("The GeoIP database has been restored from the backup successfully."); + } + } + return false; + } + mtrace("$geoip2filename updated successfully."); + return true; + } +} diff --git a/lib/db/tasks.php b/lib/db/tasks.php index 905c73f06e3..0a9548e4b45 100644 --- a/lib/db/tasks.php +++ b/lib/db/tasks.php @@ -466,4 +466,14 @@ $tasks = array( 'month' => '*', 'dayofweek' => '*', ], + [ + 'classname' => 'core\task\update_geoip2file_task', + 'blocking' => 0, + 'minute' => '0', + 'hour' => 'R', + 'day' => '*', + 'month' => '*', + 'dayofweek' => 'R', + 'disabled' => true, + ], ); diff --git a/version.php b/version.php index 732cea4ccfb..a6c30fe3288 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2024062000.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2024062000.001; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes. $release = '4.5dev (Build: 20240620)'; // Human-friendly version name