From a53238a07b4e55e04f0fe5b44dbbc21912bbb7ae Mon Sep 17 00:00:00 2001 From: sam marshall Date: Thu, 10 Dec 2020 12:02:41 +0000 Subject: [PATCH] MDL-70446 search_solr: File indexing can fail due to multipart upload Due to Solr bug SOLR-15039, uploading files for indexing can fail if it uses multipart upload. This changes it to use direct binary upload. Unfortunately, the direct binary version in PHP curl only supports a string, so we have to load the file into memory. I added extra code to restrict the size of files indexed to (memory limit - 100MB), which is usually 284MB unless configured differently because cron runs under MEMORY_EXTRA. --- search/engine/solr/classes/engine.php | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/search/engine/solr/classes/engine.php b/search/engine/solr/classes/engine.php index d047694b784..08a45d129fb 100644 --- a/search/engine/solr/classes/engine.php +++ b/search/engine/solr/classes/engine.php @@ -1011,8 +1011,12 @@ class engine extends \core_search\engine { // A giant block of code that is really just error checking around the curl request. try { - // Now actually do the request. - $result = $curl->post($url->out(false), array('myfile' => $storedfile)); + // We have to post the file directly in binary data (not using multipart) to avoid + // Solr bug SOLR-15039 which can cause incorrect data when you use multipart upload. + // Note this loads the whole file into memory; see limit in file_is_indexable(). + $curl->setHeader('Content-Type: text/plain; charset=UTF-8'); + $result = $curl->post($url->out(false), $storedfile->get_content()); + $curl->resetHeader(); $code = $curl->get_errno(); $info = $curl->get_info(); @@ -1076,6 +1080,18 @@ class engine extends \core_search\engine { return false; } + // Because we now load files into memory to index them in Solr, we also have to ensure that + // we don't try to index anything bigger than the memory limit (less 100MB for safety). + // Memory limit in cron is MEMORY_EXTRA which is usually 256 or 384MB but can be increased + // in config, so this will allow files over 100MB to be indexed. + $limit = ini_get('memory_limit'); + if ($limit && $limit != -1) { + $limitbytes = get_real_size($limit); + if ($file->get_filesize() > $limitbytes) { + return false; + } + } + $mime = $file->get_mimetype(); if ($mime == 'application/vnd.moodle.backup') {