reduce kv cache from available memory

2024-11-28 20:21:21 +04:00
parent 9858d90ce4
commit 45a1e55eec
5 changed files with 25 additions and 7 deletions
@@ -914,7 +914,7 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
        dev_info_set = (struct device_info *)malloc(n_world * sizeof(struct device_info));
        dev_info_set[0] = dev_info;
        llama_gather_device_info(lctx, dev_info_set);
-        device_print_props(dev_info_set, n_world, model);
+        device_print_props(dev_info_set, n_world, model, cparams);
    } else {
        llama_send_device_info(lctx, &dev_info);
    }