add option --keep-inp-out-in-metal and fix bugs in unmap

2025-01-22 11:15:19 +04:00
parent ce2ef9699f
commit facb4ea736
5 changed files with 142 additions and 131 deletions
@@ -148,6 +148,7 @@ struct gpt_params {
    std::string master_ip         = "localhost"; // ip address of the master node
    std::string next_node_ip      = "localhost"; // ip address of my next node
    bool    unload                = false; // unload layer weights after use or not
+    bool    keep_inp_out_in_metal = false; // whether to keep input/output weight in metal, not by default
    int32_t cuda_mem              = 999.0; // cuda memory to use, in GiB
    int32_t n_predict             =    -1; // new tokens to predict
    int32_t n_ctx                 =     0; // context size