electron-rare/prima-cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rename arg --keep-inp-out-in-metal to --keep-out-in-metal

Browse Source
This commit is contained in:
Lizonghang
2025-01-23 23:17:06 +04:00
parent 5fcf020cfb
commit 1c0087e919
5 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+3 -3
View File
@@ -739,10 +739,10 @@ gpt_params_context gpt_params_parser_init(gpt_params & params, llama_example ex,
).set_env("LLAMA_ARG_CUDA_MEM"));
#ifdef GGML_USE_METAL
add_opt(llama_arg(
{"--keep-inp-out-in-metal"},
format("whether to keep input and output weight in metal (default: %s)", params.keep_inp_out_in_metal ? "true" : "false"),
{"--keep-out-in-metal"},
format("whether to keep output weights in metal memory (default: %s)", params.keep_out_in_metal ? "true" : "false"),
[](gpt_params & params) {
params.keep_inp_out_in_metal = true;
params.keep_out_in_metal = true;
}
).set_env("LLAMA_ARG_KEEP_INP_OUT_IN_METAL"));
#endif
common/common.cpp
+3 -3
View File
@@ -1162,7 +1162,7 @@ static void assign_device(
if (dev.gpu_support.cuda || dev.gpu_support.metal) {
float reserved_mem = 0.1f; // reserved shared memory to avoid potential OOM, set to 100 MiB by default
vec_z_gpu[m] = (double)((dev.gpu_props.memory_free - reserved_mem) * GIGABYTE - c_gpu[m]) / (double)(n_layer * b_prime);
if (dev.gpu_support.metal && m == 0 && cparams.keep_inp_out_in_metal) {
if (dev.gpu_support.metal && m == 0 && cparams.keep_out_in_metal) {
vec_z_gpu[m] -= (double)(bi + bo) / (double)(n_layer * b_prime);
}
dev_gpu[m] = 1;
@@ -1624,7 +1624,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
mparams.use_mmap = params.use_mmap;
mparams.use_mlock = params.use_mlock;
mparams.check_tensors = params.check_tensors;
mparams.keep_inp_out_in_metal = params.keep_inp_out_in_metal;
mparams.keep_out_in_metal = params.keep_out_in_metal;
std::copy(std::begin(params.n_layer_window), std::end(params.n_layer_window), mparams.n_layer_window);
if (params.kv_overrides.empty()) {
mparams.kv_overrides = NULL;
@@ -1671,7 +1671,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param
cparams.n_world = params.n_world;
cparams.rank = params.rank;
cparams.unload = params.unload;
cparams.keep_inp_out_in_metal = params.keep_inp_out_in_metal;
cparams.keep_out_in_metal = params.keep_out_in_metal;
cparams.n_gpu_layers = params.n_gpu_layers;
std::copy(std::begin(params.n_layer_window), std::end(params.n_layer_window), cparams.n_layer_window);
common/common.h
+1 -1
View File
@@ -148,7 +148,7 @@ struct gpt_params {
std::string master_ip = "localhost"; // ip address of the master node
std::string next_node_ip = "localhost"; // ip address of my next node
bool unload = false; // unload layer weights after use or not
bool keep_inp_out_in_metal = false; // whether to keep input/output weight in metal, not by default
bool keep_out_in_metal = false; // whether to keep output weights in metal memory, not by default
int32_t gpu_mem = 999.0; // gpu memory to use, in GiB
int32_t n_predict = -1; // new tokens to predict
int32_t n_ctx = 0; // context size