L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MDL-63401 tool_dataprivacy: Rewrite expired deletion handling

Browse Source 
This change rewrites the way in which expiry is calculated and addresses
a number of closely related issues:

Users can customise, and add blocks with data to, their dashboard.  When
a user had done so, the user could be flagged for deletion, but the
blocks in their Dashboard were subject to the default block retention
policy. In addition there is no way to override the retention policy for
user blocks.

This change modifies the structure of the expiry mechanism:
- to consider any subcontext of the user context to be a part of the user
  context during calculation. User child contexts are not the property
  of the system, and should not be treated separately.
- the way in which the context expiry mechanism worked was to select
  use a multiple different managers based solely on the context level.
  Because of the use of user blocks, this proved to be unreliable as
  blocks has been attributed purely to courses.
  This has been changed to a single manager which is aware of hierarchy
  and deletions as a whole.
- to prepare for upcoming work relating to more detailed expiry
  mechanisms, a new expiry_info class is introduced and used to
  merge the expiry of child contexts into a working in-memory view.

This changeset includes extensive unit tests.
This commit is contained in:
Andrew Nicols
2018-10-08 20:52:27 +08:00
parent 6b0512cc52
commit 3e90c7df33
12 changed files with 2092 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/tool/dataprivacy/classes/api.php
+1 -29
View File
@@ -929,7 +929,7 @@ class api {
* @param int $forcedvalue Use this purposeid value as if this was this context instance purpose.
* @return purpose|false
*/
public static function get_effective_context_purpose(\context $context, $forcedvalue=false) {
public static function get_effective_context_purpose(\context $context, $forcedvalue = false) {
if (!data_registry::defaults_set()) {
return false;
}
@@ -967,34 +967,6 @@ class api {
return data_registry::get_effective_contextlevel_value($contextlevel, 'purpose', $forcedvalue);
}
/**
* Creates an expired context record for the provided context id.
*
* @param int $contextid
* @return \tool_dataprivacy\expired_context
*/
public static function create_expired_context($contextid) {
$record = (object)[
'contextid' => $contextid,
'status' => expired_context::STATUS_EXPIRED,
];
$expiredctx = new expired_context(0, $record);
$expiredctx->save();
return $expiredctx;
}
/**
* Deletes an expired context record.
*
* @param int $id The tool_dataprivacy_ctxexpire id.
* @return bool True on success.
*/
public static function delete_expired_context($id) {
$expiredcontext = new expired_context($id);
return $expiredcontext->delete();
}
/**
* Updates the status of an expired context.
*
admin/tool/dataprivacy/classes/data_registry.php
+24 -15
View File
@@ -191,14 +191,14 @@ class data_registry {
* @param int|false $forcedvalue Use this value as if this was this context instance value.
* @return persistent|false It return a 'purpose' instance or a 'category' instance, depending on $element
*/
public static function get_effective_context_value(\context $context, $element, $forcedvalue=false) {
public static function get_effective_context_value(\context $context, $element, $forcedvalue = false) {
if ($element !== 'purpose' && $element !== 'category') {
throw new coding_exception('Only \'purpose\' and \'category\' are supported.');
}
$fieldname = $element . 'id';
if ($forcedvalue === false) {
if (empty($forcedvalue)) {
$instance = context_instance::get_record_by_contextid($context->id, false);
if (!$instance) {
@@ -217,20 +217,29 @@ class data_registry {
// The effective value varies depending on the context level.
if ($context->contextlevel == CONTEXT_USER) {
// Use the context level value as we don't allow people to set specific instances values.
return self::get_effective_contextlevel_value($context->contextlevel, $element);
} else {
// Check if we need to pass the plugin name of an activity.
$forplugin = '';
if ($context->contextlevel == CONTEXT_MODULE) {
list($course, $cm) = get_course_and_cm_from_cmid($context->instanceid);
$forplugin = $cm->modname;
}
// Use the default context level value.
list($purposeid, $categoryid) = self::get_effective_default_contextlevel_purpose_and_category(
$context->contextlevel, false, false, $forplugin
);
return self::get_element_instance($element, $$fieldname);
return self::get_effective_contextlevel_value(CONTEXT_USER, $element);
}
$parents = $context->get_parent_contexts(true);
foreach ($parents as $parent) {
if ($parent->contextlevel == CONTEXT_USER) {
// Use the context level value as we don't allow people to set specific instances values.
return self::get_effective_contextlevel_value(CONTEXT_USER, $element);
}
}
// Check if we need to pass the plugin name of an activity.
$forplugin = '';
if ($context->contextlevel == CONTEXT_MODULE) {
list($course, $cm) = get_course_and_cm_from_cmid($context->instanceid);
$forplugin = $cm->modname;
}
// Use the default context level value.
list($purposeid, $categoryid) = self::get_effective_default_contextlevel_purpose_and_category(
$context->contextlevel, false, false, $forplugin
);
return self::get_element_instance($element, $$fieldname);
}
// Specific value for this context instance.
admin/tool/dataprivacy/classes/expired_context.php
+47
View File
@@ -159,4 +159,51 @@ class expired_context extends \core\persistent {
return $DB->count_records_sql($sql, $params);
}
/**
* Create a new expired_context based on the context, and expiry_info object.
*
* @param \context $context
* @param expiry_info $info
* @return expired_context
*/
public static function create_from_expiry_info(\context $context, expiry_info $info) : expired_context {
$record = (object) [
'contextid' => $context->id,
'status' => self::STATUS_EXPIRED,
];
$expiredcontext = new static(0, $record);
$expiredcontext->save();
return $expiredcontext;
}
/**
* Update the expired_context from an expiry_info object which relates to this context.
*
* @param expiry_info $info
* @return $this
*/
public function update_from_expiry_info(expiry_info $info) : expired_context {
return $this;
}
/**
* Check whether this expired_context record is in a state ready for deletion to actually take place.
*
* @return bool
*/
public function can_process_deletion() : bool {
return ($this->get('status') == self::STATUS_APPROVED);
}
/**
* Check whether this expired_context record has already been cleaned.
*
* @return bool
*/
public function is_complete() : bool {
return ($this->get('status') == self::STATUS_CLEANED);
}
}
admin/tool/dataprivacy/classes/expired_contexts_manager.php
+567 -55
View File
@@ -34,121 +34,633 @@ defined('MOODLE_INTERNAL') || die();
* @copyright 2018 David Monllao
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
abstract class expired_contexts_manager {
class expired_contexts_manager {
/**
* Number of deleted contexts for each scheduled task run.
*/
const DELETE_LIMIT = 200;
/**
* Returns the list of expired context instances.
*
* @return \stdClass[]
*/
abstract protected function get_expired_contexts();
/** @var progress_trace The log progress tracer */
protected $progresstracer = null;
/**
* Specify with context levels this expired contexts manager is deleting.
*
* @return int[]
*/
abstract protected function get_context_levels();
/** @var manager The privacy manager */
protected $manager = null;
/**
* Flag expired contexts as expired.
*
* @return int The number of contexts flagged as expired.
* @return int[] The number of contexts flagged as expired for courses, and users.
*/
public function flag_expired() {
public function flag_expired_contexts() : array {
if (!$this->check_requirements()) {
return 0;
return [0, 0];
}
$contexts = $this->get_expired_contexts();
foreach ($contexts as $context) {
api::create_expired_context($context->id);
// Clear old and stale records first.
static::clear_old_records();
$data = static::get_nested_expiry_info_for_courses();
$coursecount = 0;
foreach ($data as $expiryrecord) {
if ($this->update_from_expiry_info($expiryrecord)) {
$coursecount++;
}
}
return count($contexts);
$data = static::get_nested_expiry_info_for_user();
$usercount = 0;
foreach ($data as $expiryrecord) {
if ($this->update_from_expiry_info($expiryrecord)) {
$usercount++;
}
}
return [$coursecount, $usercount];
}
/**
* Clear old and stale records.
*/
protected static function clear_old_records() {
global $DB;
$sql = "SELECT dpctx.*
FROM {tool_dataprivacy_ctxexpired} dpctx
LEFT JOIN {context} ctx ON ctx.id = dpctx.contextid
WHERE ctx.id IS NULL";
$orphaned = $DB->get_recordset_sql($sql);
foreach ($orphaned as $orphan) {
$expiredcontext = new expired_context(0, $orphan);
$expiredcontext->delete();
}
// Delete any child of a user context.
$parentpath = $DB->sql_concat('ctxuser.path', "'/%'");
$params = [
'contextuser' => CONTEXT_USER,
];
$sql = "SELECT dpctx.*
FROM {tool_dataprivacy_ctxexpired} dpctx
WHERE dpctx.contextid IN (
SELECT ctx.id
FROM {context} ctxuser
JOIN {context} ctx ON ctx.path LIKE {$parentpath}
WHERE ctxuser.contextlevel = :contextuser
)";
$userchildren = $DB->get_recordset_sql($sql, $params);
foreach ($userchildren as $child) {
$expiredcontext = new expired_context(0, $child);
$expiredcontext->delete();
}
}
/**
* Get the full nested set of expiry data relating to all contexts.
*
* @param string $contextpath A contexpath to restrict results to
* @return \stdClass[]
*/
protected static function get_nested_expiry_info($contextpath = '') : array {
$coursepaths = self::get_nested_expiry_info_for_courses($contextpath);
$userpaths = self::get_nested_expiry_info_for_user($contextpath);
return array_merge($coursepaths, $userpaths);
}
/**
* Get the full nested set of expiry data relating to course-related contexts.
*
* @param string $contextpath A contexpath to restrict results to
* @return \stdClass[]
*/
protected static function get_nested_expiry_info_for_courses($contextpath = '') : array {
global $DB;
$contextfields = \context_helper::get_preload_record_columns_sql('ctx');
$expiredfields = expired_context::get_sql_fields('expiredctx', 'expiredctx');
$purposefields = 'dpctx.purposeid';
$coursefields = 'ctxcourse.expirydate AS expirydate';
$fields = implode(', ', ['ctx.id', $contextfields, $expiredfields, $coursefields, $purposefields]);
// We want all contexts at course-dependant levels.
$parentpath = $DB->sql_concat('ctxcourse.path', "'/%'");
// This SQL query returns all course-dependant contexts (including the course context)
// which course end date already passed.
// This is ordered by the context path in reverse order, which will give the child nodes before any parent node.
$params = [
'contextlevel' => CONTEXT_COURSE,
];
$where = '';
if (!empty($contextpath)) {
$where = "AND (ctx.path = :pathmatchexact OR ctx.path LIKE :pathmatchchildren)";
$params['pathmatchexact'] = $contextpath;
$params['pathmatchchildren'] = "{$contextpath}/%";
}
$sql = "SELECT $fields
FROM {context} ctx
JOIN (
SELECT c.enddate AS expirydate, subctx.path
FROM {context} subctx
JOIN {course} c
ON subctx.contextlevel = :contextlevel
AND subctx.instanceid = c.id
AND c.format != 'site'
) ctxcourse
ON ctx.path LIKE {$parentpath} OR ctx.path = ctxcourse.path
LEFT JOIN {tool_dataprivacy_ctxinstance} dpctx
ON dpctx.contextid = ctx.id
LEFT JOIN {tool_dataprivacy_ctxexpired} expiredctx
ON ctx.id = expiredctx.contextid
WHERE 1 = 1 {$where}
ORDER BY ctx.path DESC";
return self::get_nested_expiry_info_from_sql($sql, $params);
}
/**
* Get the full nested set of expiry data.
*
* @param string $contextpath A contexpath to restrict results to
* @return \stdClass[]
*/
protected static function get_nested_expiry_info_for_user($contextpath = '') : array {
global $DB;
$contextfields = \context_helper::get_preload_record_columns_sql('ctx');
$expiredfields = expired_context::get_sql_fields('expiredctx', 'expiredctx');
$purposefields = 'dpctx.purposeid';
$userfields = 'u.lastaccess AS expirydate';
$fields = implode(', ', ['ctx.id', $contextfields, $expiredfields, $userfields, $purposefields]);
// We want all contexts at user-dependant levels.
$parentpath = $DB->sql_concat('ctxuser.path', "'/%'");
// This SQL query returns all user-dependant contexts (including the user context)
// This is ordered by the context path in reverse order, which will give the child nodes before any parent node.
$params = [
'contextlevel' => CONTEXT_USER,
];
$where = '';
if (!empty($contextpath)) {
$where = "AND ctx.path = :pathmatchexact";
$params['pathmatchexact'] = $contextpath;
}
$sql = "SELECT $fields, u.deleted AS userdeleted
FROM {context} ctx
JOIN {user} u ON ctx.instanceid = u.id
LEFT JOIN {tool_dataprivacy_ctxinstance} dpctx
ON dpctx.contextid = ctx.id
LEFT JOIN {tool_dataprivacy_ctxexpired} expiredctx
ON ctx.id = expiredctx.contextid
WHERE ctx.contextlevel = :contextlevel {$where}
ORDER BY ctx.path DESC";
return self::get_nested_expiry_info_from_sql($sql, $params);
}
/**
* Get the full nested set of expiry data given appropriate SQL.
*
* @param string $sql The SQL used to select the nested information.
* @param array $params The params required by the SQL.
* @return \stdClass[]
*/
protected static function get_nested_expiry_info_from_sql(string $sql, array $params) : array {
global $DB;
$fulllist = $DB->get_recordset_sql($sql, $params);
$datalist = [];
$expiredcontents = [];
$pathstoskip = [];
foreach ($fulllist as $record) {
\context_helper::preload_from_record($record);
$context = \context::instance_by_id($record->id, false);
if (!self::is_eligible_for_deletion($pathstoskip, $context)) {
// We should skip this context, and therefore all of it's children.
$datalist = array_filter($datalist, function($data, $path) use ($context) {
// Remove any child of this context.
// Technically this should never be fulfilled because the query is ordered in path DESC, but is kept
// in to be certain.
return (false === strpos($path, "{$context->path}/"));
}, ARRAY_FILTER_USE_BOTH);
if ($record->expiredctxid) {
// There was previously an expired context record.
// Delete it to be on the safe side.
$expiredcontext = new expired_context(null, expired_context::extract_record($record, 'expiredctx'));
$expiredcontext->delete();
}
continue;
}
$purposevalue = $record->purposeid !== null ? $record->purposeid : context_instance::NOTSET;
$purpose = api::get_effective_context_purpose($context, $purposevalue);
if ($context instanceof \context_user && !empty($record->userdeleted)) {
$expiryinfo = static::get_expiry_info($purpose, $record->userdeleted);
} else {
$expiryinfo = static::get_expiry_info($purpose, $record->expirydate);
}
foreach ($datalist as $path => $data) {
// Merge with already-processed children.
if (strpos($path, $context->path) !== 0) {
continue;
}
$expiryinfo->merge_with_child($data->info);
}
$datalist[$context->path] = (object) [
'context' => $context,
'record' => $record,
'purpose' => $purpose,
'info' => $expiryinfo,
];
}
$fulllist->close();
return $datalist;
}
/**
* Check whether the supplied context would be elible for deletion.
*
* @param array $pathstoskip A set of paths which should be skipped
* @param \context $context
* @return bool
*/
protected static function is_eligible_for_deletion(array &$pathstoskip, \context $context) : bool {
$shouldskip = false;
// Check whether any of the child contexts are ineligble.
$shouldskip = !empty(array_filter($pathstoskip, function($path) use ($context) {
// If any child context has already been skipped then it will appear in this list.
// Since paths include parents, test if the context under test appears as the haystack in the skipped
// context's needle.
return false !== (strpos($context->path, $path));
}));
if (!$shouldskip && $context instanceof \context_user) {
// The context instanceid is the user's ID.
if (isguestuser($context->instanceid) || is_siteadmin($context->instanceid)) {
// This is an admin, or the guest and cannot be deleted.
$shouldskip = true;
}
if (!$shouldskip) {
$courses = enrol_get_users_courses($context->instanceid, false, ['enddate']);
foreach ($courses as $course) {
if (empty($course->enddate) || $course->enddate >= time()) {
// This user still has an active enrolment.
$shouldskip = true;
break;
}
}
}
}
if ($shouldskip) {
// Add this to the list of contexts to skip for parentage checks.
$pathstoskip[] = $context->path;
}
return !$shouldskip;
}
/**
* Deletes the expired contexts.
*
* @return int The number of deleted contexts.
* @return int[] The number of deleted contexts.
*/
public function delete() {
$numprocessed = 0;
public function process_approved_deletions() : array {
if (!$this->check_requirements()) {
return $numprocessed;
return [0, 0];
}
$privacymanager = new manager();
$privacymanager->set_observer(new \tool_dataprivacy\manager_observer());
$expiredcontexts = expired_context::get_records(['status' => expired_context::STATUS_APPROVED]);
$totalprocessed = 0;
$usercount = 0;
$coursecount = 0;
foreach ($expiredcontexts as $expiredctx) {
$context = \context::instance_by_id($expiredctx->get('contextid'), IGNORE_MISSING);
if (empty($context)) {
// Unable to process this request further.
// We have no context to delete.
$expiredctx->delete();
continue;
}
foreach ($this->get_context_levels() as $level) {
$expiredcontexts = expired_context::get_records_by_contextlevel($level, expired_context::STATUS_APPROVED);
foreach ($expiredcontexts as $expiredctx) {
if (!$this->delete_expired_context($privacymanager, $expiredctx)) {
continue;
if ($this->delete_expired_context($expiredctx)) {
if ($context instanceof \context_user) {
$usercount++;
} else {
$coursecount++;
}
$numprocessed += 1;
if ($numprocessed == self::DELETE_LIMIT) {
// Close the recordset.
$expiredcontexts->close();
break 2;
$totalprocessed++;
if ($totalprocessed >= $this->get_delete_limit()) {
break;
}
}
}
return $numprocessed;
return [$coursecount, $usercount];
}
/**
* Deletes user data from the provided context.
*
* @param manager $privacymanager
* @param expired_context $expiredctx
* @return \context|false
*/
protected function delete_expired_context(manager $privacymanager, expired_context $expiredctx) {
protected function delete_expired_context(expired_context $expiredctx) {
$context = \context::instance_by_id($expiredctx->get('contextid'));
$context = \context::instance_by_id($expiredctx->get('contextid'), IGNORE_MISSING);
if (!$context) {
api::delete_expired_context($expiredctx->get('contextid'));
$this->get_progress()->output("Deleting context {$context->id} - " . $context->get_context_name(true, true));
// Update the expired_context and verify that it is still ready for deletion.
$expiredctx = $this->update_expired_context($expiredctx);
if (empty($expiredctx)) {
$this->get_progress()->output("Context has changed since approval and is no longer pending approval. Skipping", 1);
return false;
}
if (!PHPUNIT_TEST) {
mtrace('Deleting context ' . $context->id . ' - ' .
shorten_text($context->get_context_name(true, true)));
if (!$expiredctx->can_process_deletion()) {
// This only happens if the record was updated after being first fetched.
$this->get_progress()->output("Context has changed since approval and must be re-approved. Skipping", 1);
$expiredctx->set('status', expired_context::STATUS_EXPIRED);
$expiredctx->save();
return false;
}
$privacymanager->delete_data_for_all_users_in_context($context);
api::set_expired_context_status($expiredctx, expired_context::STATUS_CLEANED);
$privacymanager = $this->get_privacy_manager();
if ($context instanceof \context_user) {
$this->delete_expired_user_context($expiredctx);
} else {
// This context is fully expired - that is that the default retention period has been reached.
$privacymanager->delete_data_for_all_users_in_context($context);
}
// Mark the record as cleaned.
$expiredctx->set('status', expired_context::STATUS_CLEANED);
$expiredctx->save();
return $context;
}
/**
* Deletes user data from the provided user context.
*
* @param expired_context $expiredctx
*/
protected function delete_expired_user_context(expired_context $expiredctx) {
global $DB;
$contextid = $expiredctx->get('contextid');
$context = \context::instance_by_id($contextid);
$user = \core_user::get_user($context->instanceid, '*', MUST_EXIST);
$privacymanager = $this->get_privacy_manager();
// Delete all child contexts of the user context.
$parentpath = $DB->sql_concat('ctxuser.path', "'/%'");
$params = [
'contextlevel' => CONTEXT_USER,
'contextid' => $expiredctx->get('contextid'),
];
$fields = \context_helper::get_preload_record_columns_sql('ctx');
$sql = "SELECT ctx.id, $fields
FROM {context} ctxuser
JOIN {context} ctx ON ctx.path LIKE {$parentpath}
WHERE ctxuser.contextlevel = :contextlevel AND ctxuser.id = :contextid
ORDER BY ctx.path DESC";
$children = $DB->get_recordset_sql($sql, $params);
foreach ($children as $child) {
\context_helper::preload_from_record($child);
$context = \context::instance_by_id($child->id);
$privacymanager->delete_data_for_all_users_in_context($context);
}
$children->close();
// Delete all unprotected data that the user holds.
$approvedlistcollection = new \core_privacy\local\request\contextlist_collection($user->id);
$contextlistcollection = $privacymanager->get_contexts_for_userid($user->id);
foreach ($contextlistcollection as $contextlist) {
$contextids = [];
$approvedlistcollection->add_contextlist(new \core_privacy\local\request\approved_contextlist(
$user,
$contextlist->get_component(),
$contextlist->get_contextids()
));
}
$privacymanager->delete_data_for_user($approvedlistcollection, $this->get_progress());
// Delete the user context.
$context = \context::instance_by_id($expiredctx->get('contextid'));
$privacymanager->delete_data_for_all_users_in_context($context);
// This user is now fully expired - finish by deleting the user.
delete_user($user);
}
/**
* Check that the requirements to start deleting contexts are satisified.
*
* @return bool
*/
protected function check_requirements() {
api::check_can_manage_data_registry(\context_system::instance()->id);
if (!data_registry::defaults_set()) {
return false;
}
return true;
}
/**
* Check whether a date is beyond the specified period.
*
* @param string $period The Expiry Period
* @param int $comparisondate The date for comparison
* @return bool
*/
protected static function has_expired(string $period, int $comparisondate) : bool {
$dt = new \DateTime();
$dt->setTimestamp($comparisondate);
$dt->add(new \DateInterval($period));
return (time() >= $dt->getTimestamp());
}
/**
* Get the expiry info object for the specified purpose and comparison date.
*
* @param purpose $purpose The purpose of this context
* @param int $comparisondate The date for comparison
* @return expiry_info
*/
protected static function get_expiry_info(purpose $purpose, int $comparisondate = 0) : expiry_info {
if (empty($comparisondate)) {
// The date is empty, therefore this context cannot be considered for automatic expiry.
$defaultexpired = false;
} else {
$defaultexpired = static::has_expired($purpose->get('retentionperiod'), $comparisondate);
}
return new expiry_info($defaultexpired);
}
/**
* Update or delete the expired_context from the expiry_info object.
* This function depends upon the data structure returned from get_nested_expiry_info.
*
* If the context is expired in any way, then an expired_context will be returned, otherwise null will be returned.
*
* @param \stdClass $expiryrecord
* @return expired_context|null
*/
protected function update_from_expiry_info(\stdClass $expiryrecord) {
if ($expiryrecord->info->is_any_expired()) {
// The context is expired in some fashion.
// Create or update as required.
if ($expiryrecord->record->expiredctxid) {
$expiredcontext = new expired_context(null, expired_context::extract_record($expiryrecord->record, 'expiredctx'));
$expiredcontext->update_from_expiry_info($expiryrecord->info);
if ($expiredcontext->is_complete()) {
return null;
}
} else {
$expiredcontext = expired_context::create_from_expiry_info($expiryrecord->context, $expiryrecord->info);
}
return $expiredcontext;
} else {
// The context is not expired.
if ($expiryrecord->record->expiredctxid) {
// There was previously an expired context record, but it is no longer relevant.
// Delete it to be on the safe side.
$expiredcontext = new expired_context(null, expired_context::extract_record($expiryrecord->record, 'expiredctx'));
$expiredcontext->delete();
}
return null;
}
}
/**
* Update the expired context record.
*
* Note: You should use the return value as the provided value will be used to fetch data only.
*
* @param expired_context $expiredctx The record to update
* @return expired_context|null
*/
protected function update_expired_context(expired_context $expiredctx) {
// Fetch the context from the expired_context record.
$context = \context::instance_by_id($expiredctx->get('contextid'));
// Fetch the current nested expiry data.
$expiryrecords = self::get_nested_expiry_info($context->path);
// Find the current record.
if (empty($expiryrecords[$context->path])) {
$expiredctx->delete();
return null;
}
// Refresh the record.
// Note: Use the returned expiredctx.
$expiredctx = $this->update_from_expiry_info($expiryrecords[$context->path]);
if (empty($expiredctx)) {
return null;
}
if (!$context instanceof \context_user) {
// Where the target context is not a user, we check all children of the context.
// The expiryrecords array only contains children, fetched from the get_nested_expiry_info call above.
// No need to check that these _are_ children.
foreach ($expiryrecords as $expiryrecord) {
if ($expiryrecord->context->id === $context->id) {
// This is record for the context being tested that we checked earlier.
continue;
}
if (empty($expiryrecord->record->expiredctxid)) {
// There is no expired context record for this context.
// If there is no record, then this context cannot have been approved for removal.
return null;
}
// Fetch the expired_context object for this record.
// This needs to be updated from the expiry_info data too as there may be child changes to consider.
$expiredcontext = new expired_context(null, expired_context::extract_record($expiryrecord->record, 'expiredctx'));
$expiredcontext->update_from_expiry_info($expiryrecord->info);
if (!$expiredcontext->is_complete()) {
return null;
}
}
}
return $expiredctx;
}
/**
* Create a new instance of the privacy manager.
*
* @return manager
*/
protected function get_privacy_manager() : manager {
if (null === $this->manager) {
$this->manager = new manager();
$this->manager->set_observer(new \tool_dataprivacy\manager_observer());
}
return $this->manager;
}
/**
* Fetch the limit for the maximum number of contexts to delete in one session.
*
* @return int
*/
protected function get_delete_limit() : int {
return self::DELETE_LIMIT;
}
/**
* Get the progress tracer.
*
* @return \progress_trace
*/
protected function get_progress() : \progress_trace {
if (null === $this->progresstracer) {
$this->set_progress(new \text_progress_trace());
}
return $this->progresstracer;
}
/**
* Set a specific tracer for the task.
*
* @param \progress_trace $trace
* @return $this
*/
public function set_progress(\progress_trace $trace) : expired_contexts_manager {
$this->progresstracer = $trace;
return $this;
}
}
admin/tool/dataprivacy/classes/expired_course_related_contexts.php
-135
View File
@@ -1,135 +0,0 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Expired contexts manager for CONTEXT_COURSE, CONTEXT_MODULE and CONTEXT_BLOCK.
*
* @package tool_dataprivacy
* @copyright 2018 David Monllao
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace tool_dataprivacy;
use tool_dataprivacy\purpose;
defined('MOODLE_INTERNAL') || die();
/**
* Expired contexts manager for CONTEXT_COURSE, CONTEXT_MODULE and CONTEXT_BLOCK.
*
* @copyright 2018 David Monllao
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class expired_course_related_contexts extends \tool_dataprivacy\expired_contexts_manager {
/**
* Course-related context levels.
*
* @return int[]
*/
protected function get_context_levels() {
return [CONTEXT_MODULE, CONTEXT_BLOCK, CONTEXT_COURSE];
}
/**
* Returns a recordset with user context instances that are possibly expired (to be confirmed by get_recordset_callback).
*
* @return \stdClass[]
*/
protected function get_expired_contexts() {
global $DB;
// Including context info + course end date + purposeid (this last one only if defined).
$fields = 'ctx.id AS id, ctxcourse.enddate AS courseenddate, dpctx.purposeid AS purposeid, ' .
\context_helper::get_preload_record_columns_sql('ctx');
// We want all contexts at course-dependant levels.
$parentpath = $DB->sql_concat('ctxcourse.path', "'/%'");
// This SQL query returns all course-dependant contexts (including the course context)
// which course end date already passed.
$sql = "SELECT $fields
FROM {context} ctx
JOIN (
SELECT c.enddate, subctx.path
FROM {context} subctx
JOIN {course} c
ON subctx.contextlevel = ? AND subctx.instanceid = c.id
WHERE c.enddate < ? AND c.enddate > 0
) ctxcourse
ON ctx.path LIKE {$parentpath} OR ctx.path = ctxcourse.path
LEFT JOIN {tool_dataprivacy_ctxinstance} dpctx
ON dpctx.contextid = ctx.id
LEFT JOIN {tool_dataprivacy_ctxexpired} expiredctx
ON ctx.id = expiredctx.contextid
WHERE expiredctx.id IS NULL
ORDER BY ctx.contextlevel DESC, ctx.path";
$possiblyexpired = $DB->get_recordset_sql($sql, [CONTEXT_COURSE, time()]);
$expiredcontexts = [];
$excludedcontextids = [];
foreach ($possiblyexpired as $record) {
\context_helper::preload_from_record($record);
// No strict checking as the context may already be deleted (e.g. we just deleted a course,
// module contexts below it will not exist).
$context = \context::instance_by_id($record->id, false);
if (!$context) {
continue;
}
// We pass the value we just got from SQL so get_effective_context_purpose don't need to query
// the db again to retrieve it. If there is no tool_dataprovider_ctxinstance record
// $record->purposeid will be null which is ok as it would force get_effective_context_purpose
// to return the default purpose for the context context level (no db queries involved).
$purposevalue = $record->purposeid !== null ? $record->purposeid : context_instance::NOTSET;
// It should be cheap as system purposes and context level purposes will be retrieved from a cache most of the time.
$purpose = api::get_effective_context_purpose($context, $purposevalue);
$dt = new \DateTime();
$dt->setTimestamp($record->courseenddate);
$di = new \DateInterval($purpose->get('retentionperiod'));
$dt->add($di);
if (time() < $dt->getTimestamp()) {
// Exclude this context ID as it has not reached the retention period yet.
$excludedcontextids[] = $context->id;
continue;
}
// Check if this context has children that have not yet expired.
$hasunexpiredchildren = false;
$children = $context->get_child_contexts();
foreach ($children as $child) {
if (in_array($child->id, $excludedcontextids)) {
$hasunexpiredchildren = true;
break;
}
}
if ($hasunexpiredchildren) {
// Exclude this context ID as it has children that have not yet expired.
$excludedcontextids[] = $context->id;
continue;
}
$expiredcontexts[$context->id] = $context;
}
return $expiredcontexts;
}
}
admin/tool/dataprivacy/classes/expired_user_contexts.php
-150
View File
@@ -1,150 +0,0 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Expired contexts manager for CONTEXT_USER.
*
* @package tool_dataprivacy
* @copyright 2018 David Monllao
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace tool_dataprivacy;
use core_privacy\manager;
defined('MOODLE_INTERNAL') || die();
/**
* Expired contexts manager for CONTEXT_USER.
*
* @copyright 2018 David Monllao
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class expired_user_contexts extends \tool_dataprivacy\expired_contexts_manager {
/**
* Only user level.
*
* @return int[]
*/
protected function get_context_levels() {
return [CONTEXT_USER];
}
/**
* Returns the user context instances that are expired.
*
* @return \stdClass[]
*/
protected function get_expired_contexts() {
global $DB;
// Including context info + last login timestamp.
$fields = 'ctx.id AS id, ' . \context_helper::get_preload_record_columns_sql('ctx');
$purpose = api::get_effective_contextlevel_purpose(CONTEXT_USER);
// Calculate what is considered expired according to the context level effective purpose (= now + retention period).
$expiredtime = new \DateTime();
$retention = new \DateInterval($purpose->get('retentionperiod'));
$expiredtime->sub($retention);
$sql = "SELECT $fields FROM {context} ctx
JOIN {user} u ON ctx.contextlevel = ? AND ctx.instanceid = u.id
LEFT JOIN {tool_dataprivacy_ctxexpired} expiredctx ON ctx.id = expiredctx.contextid
WHERE u.lastaccess <= ? AND u.lastaccess > 0 AND expiredctx.id IS NULL
ORDER BY ctx.path, ctx.contextlevel ASC";
$possiblyexpired = $DB->get_recordset_sql($sql, [CONTEXT_USER, $expiredtime->getTimestamp()]);
$expiredcontexts = [];
foreach ($possiblyexpired as $record) {
\context_helper::preload_from_record($record);
// No strict checking as the context may already be deleted (e.g. we just deleted a course,
// module contexts below it will not exist).
$context = \context::instance_by_id($record->id, false);
if (!$context) {
continue;
}
if (is_siteadmin($context->instanceid)) {
continue;
}
$courses = enrol_get_users_courses($context->instanceid, false, ['enddate']);
foreach ($courses as $course) {
if (!$course->enddate) {
// We can not know it what is going on here, so we prefer to be conservative.
continue 2;
}
if ($course->enddate >= time()) {
// Future or ongoing course.
continue 2;
}
}
$expiredcontexts[$context->id] = $context;
}
return $expiredcontexts;
}
/**
* Deletes user data from the provided context.
*
* Overwritten to delete the user.
*
* @param manager $privacymanager
* @param expired_context $expiredctx
* @return \context|false
*/
protected function delete_expired_context(manager $privacymanager, expired_context $expiredctx) {
$context = \context::instance_by_id($expiredctx->get('contextid'), IGNORE_MISSING);
if (!$context) {
api::delete_expired_context($expiredctx->get('contextid'));
return false;
}
if (!PHPUNIT_TEST) {
mtrace('Deleting context ' . $context->id . ' - ' .
shorten_text($context->get_context_name(true, true)));
}
// To ensure that all user data is deleted, instead of deleting by context, we run through and collect any stray
// contexts for the user that may still exist and call delete_data_for_user().
$user = \core_user::get_user($context->instanceid, '*', MUST_EXIST);
$approvedlistcollection = new \core_privacy\local\request\contextlist_collection($user->id);
$contextlistcollection = $privacymanager->get_contexts_for_userid($user->id);
foreach ($contextlistcollection as $contextlist) {
$approvedlistcollection->add_contextlist(new \core_privacy\local\request\approved_contextlist(
$user,
$contextlist->get_component(),
$contextlist->get_contextids()
));
}
$privacymanager->delete_data_for_user($approvedlistcollection);
api::set_expired_context_status($expiredctx, expired_context::STATUS_CLEANED);
// Delete the user.
delete_user($user);
return $context;
}
}
admin/tool/dataprivacy/classes/expiry_info.php
+93
View File
@@ -0,0 +1,93 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Expiry Data.
*
* @package tool_dataprivacy
* @copyright 2018 Andrew Nicols <[email protected]>
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace tool_dataprivacy;
use core_privacy\manager;
defined('MOODLE_INTERNAL') || die();
/**
* Expiry Data.
*
* @copyright 2018 Andrew Nicols <[email protected]>
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class expiry_info {
/** @var bool Whether this context is fully expired */
protected $isexpired = false;
/**
* Constructor for the expiry_info class.
*
* @param bool $isexpired Whether the retention period for this context has expired yet.
*/
public function __construct(bool $isexpired) {
$this->isexpired = $isexpired;
}
/**
* Whether this context has 'fully' expired.
* That is to say that the default retention period has been reached, and that there are no unexpired roles.
*
* @return bool
*/
public function is_fully_expired() : bool {
return $this->isexpired;
}
/**
* Whether any part of this context has expired.
*
* @return bool
*/
public function is_any_expired() : bool {
if ($this->is_fully_expired()) {
return true;
}
return false;
}
/**
* Merge this expiry_info object with another belonging to a child context in order to set the 'safest' heritage.
*
* It is not possible to delete any part of a context that is not deleted by a parent.
* So if a course's retention policy has been reached, then only parts where the children have also expired can be
* deleted.
*
* @param expiry_info $child The child record to merge with.
* @return $this
*/
public function merge_with_child(expiry_info $child) : expiry_info {
if ($child->is_fully_expired()) {
return $this;
}
// If the child is not fully expired, then none of the parents can be either.
$this->isexpired = false;
return $this;
}
}
admin/tool/dataprivacy/classes/external.php
+23 -20
View File
@@ -989,7 +989,6 @@ class external extends external_api {
$contextlevel = api::set_contextlevel($validateddata);
} else if ($errors = $mform->is_validated()) {
$warnings[] = json_encode($errors);
throw new moodle_exception('generalerror');
}
if ($contextlevel) {
@@ -1041,8 +1040,9 @@ class external extends external_api {
'jsonformdata' => $jsonformdata
]);
// Extra permission checkings are delegated to api::set_context_instance.
// Validate context and access to manage the registry.
self::validate_context(\context_system::instance());
api::check_can_manage_data_registry();
$serialiseddata = json_decode($params['jsonformdata']);
$data = array();
@@ -1188,24 +1188,27 @@ class external extends external_api {
$expiredcontext = new expired_context($id);
$targetcontext = context_helper::instance_by_id($expiredcontext->get('contextid'));
// Fetch this context's child contexts. Make sure that all of the child contexts are flagged for deletion.
$childcontexts = $targetcontext->get_child_contexts();
foreach ($childcontexts as $child) {
if ($expiredchildcontext = expired_context::get_record(['contextid' => $child->id])) {
// Add this child context to the list for approval.
$expiredcontextstoapprove[] = $expiredchildcontext;
} else {
// This context has not yet been flagged for deletion.
$result = false;
$message = get_string('errorcontexthasunexpiredchildren', 'tool_dataprivacy',
$targetcontext->get_context_name(false));
$warnings[] = [
'item' => 'tool_dataprivacy_ctxexpired',
'warningcode' => 'errorcontexthasunexpiredchildren',
'message' => $message
];
// Exit the process.
break 2;
if (!$targetcontext instanceof \context_user) {
// Fetch this context's child contexts. Make sure that all of the child contexts are flagged for deletion.
// User context children do not need to be considered.
$childcontexts = $targetcontext->get_child_contexts();
foreach ($childcontexts as $child) {
if ($expiredchildcontext = expired_context::get_record(['contextid' => $child->id])) {
// Add this child context to the list for approval.
$expiredcontextstoapprove[] = $expiredchildcontext;
} else {
// This context has not yet been flagged for deletion.
$result = false;
$message = get_string('errorcontexthasunexpiredchildren', 'tool_dataprivacy',
$targetcontext->get_context_name(false));
$warnings[] = [
'item' => 'tool_dataprivacy_ctxexpired',
'warningcode' => 'errorcontexthasunexpiredchildren',
'message' => $message
];
// Exit the process.
break 2;
}
}
}
admin/tool/dataprivacy/classes/task/delete_expired_contexts.php
+3 -11
View File
@@ -52,18 +52,10 @@ class delete_expired_contexts extends scheduled_task {
/**
* Run the task to delete context instances based on their retention periods.
*
*/
public function execute() {
$manager = new \tool_dataprivacy\expired_course_related_contexts();
$deleted = $manager->delete();
if ($deleted > 0) {
mtrace($deleted . ' course-related contexts have been deleted');
}
$manager = new \tool_dataprivacy\expired_user_contexts();
$deleted = $manager->delete();
if ($deleted > 0) {
mtrace($deleted . ' user contexts have been deleted');
}
$manager = new \tool_dataprivacy\expired_contexts_manager();
list($courses, $users) = $manager->process_approved_deletions();
mtrace("Processed deletions for {$courses} course contexts, and {$users} user contexts as expired");
}
}
admin/tool/dataprivacy/classes/task/expired_retention_period.php
+3 -10
View File
@@ -54,15 +54,8 @@ class expired_retention_period extends scheduled_task {
* Run the task to flag context instances as expired.
*/
public function execute() {
$manager = new \tool_dataprivacy\expired_course_related_contexts();
$flagged = $manager->flag_expired();
if ($flagged > 0) {
mtrace($flagged . ' course-related contexts have been flagged as expired');
}
$manager = new \tool_dataprivacy\expired_user_contexts();
$flagged = $manager->flag_expired();
if ($flagged > 0) {
mtrace($flagged . ' user contexts have been flagged as expired');
}
$manager = new \tool_dataprivacy\expired_contexts_manager();
list($courses, $users) = $manager->flag_expired_contexts();
mtrace("Flagged {$courses} course contexts, and {$users} user contexts as expired");
}
}
admin/tool/dataprivacy/tests/api_test.php
-47
View File
@@ -1320,53 +1320,6 @@ class tool_dataprivacy_api_testcase extends advanced_testcase {
$this->assertEquals($categories[1]->get('id'), $category->get('id'));
}
/**
* Tests the deletion of expired contexts.
*
* @return null
*/
public function test_expired_context_deletion() {
global $DB;
$this->setAdminUser();
list($purposes, $categories, $courses, $modules) = $this->add_purposes_and_categories();
$course0context = \context_course::instance($courses[0]->id);
$course1context = \context_course::instance($courses[1]->id);
$expiredcontext0 = api::create_expired_context($course0context->id);
$this->assertEquals(1, $DB->count_records('tool_dataprivacy_ctxexpired'));
$expiredcontext1 = api::create_expired_context($course1context->id);
$this->assertEquals(2, $DB->count_records('tool_dataprivacy_ctxexpired'));
api::delete_expired_context($expiredcontext0->get('id'));
$this->assertEquals(1, $DB->count_records('tool_dataprivacy_ctxexpired'));
}
/**
* Tests the status of expired contexts.
*
* @return null
*/
public function test_expired_context_status() {
global $DB;
$this->setAdminUser();
list($purposes, $categories, $courses, $modules) = $this->add_purposes_and_categories();
$course0context = \context_course::instance($courses[0]->id);
$expiredcontext = api::create_expired_context($course0context->id);
// Default status.
$this->assertEquals(expired_context::STATUS_EXPIRED, $expiredcontext->get('status'));
api::set_expired_context_status($expiredcontext, expired_context::STATUS_APPROVED);
$this->assertEquals(expired_context::STATUS_APPROVED, $expiredcontext->get('status'));
}
/**
* Creates test purposes and categories.
*
admin/tool/dataprivacy/tests/expired_contexts_test.php
+1331 -161
View File
File diff suppressed because it is too large Load Diff