add mmap prefetch and unloading

2024-10-25 16:33:56 +04:00
parent ba5117581e
commit c97ea10617
7 changed files with 161 additions and 11 deletions
@@ -1039,6 +1039,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param
    cparams.n_world           = params.n_world;
    cparams.rank              = params.rank;
    cparams.n_layer_window    = params.n_layer_window;
+    cparams.unload            = params.unload;

    if (cparams.master_ip != nullptr) {
        delete[] cparams.master_ip;