feat: add args to allow user can control the communication datatype

common/arg.cpp

@@ -2118,6 +2118,13 @@ gpt_params_context gpt_params_parser_init(gpt_params & params, llama_example ex,
             params.enable_comm_compute_log = true;
         }
     ));
+    add_opt(llama_arg(
+        {"--comm-datatype"}, "TYPE",
+        format("Datatype for communication, currently support f32, q8_0, q4_0 (default: %s)", params.comm_datatype.c_str()),
+        [](gpt_params & params, const std::string & value) {
+            params.comm_datatype = value;
+        }
+    ));
     add_opt(llama_arg(
         {"--positive-file"}, "FNAME",
         format("positive prompts file, one prompt per line (default: '%s')", params.cvector_positive_file.c_str()),

common/common.cpp

@@ -2105,6 +2105,16 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param
 
     cparams.enable_comm_compute_log = params.enable_comm_compute_log;
 
+    if (cparams.comm_datatype != nullptr) {
+        delete[] cparams.comm_datatype;
+    }
+    if (!params.comm_datatype.empty()) {
+        cparams.comm_datatype = new char[params.comm_datatype.length() + 1];
+        std::strcpy(const_cast<char*>(cparams.comm_datatype), params.comm_datatype.c_str());
+    } else {
+        cparams.comm_datatype = nullptr;
+    }
+
     cparams.n_ctx             = params.n_ctx;
     cparams.n_predict         = params.n_predict;
     cparams.n_seq_max         = params.n_parallel;

common/common.h

@@ -379,6 +379,8 @@ struct gpt_params {
 
     // communication and computation logging
     bool enable_comm_compute_log = false; // enable/disable communication and computation logging
+
+    std::string comm_datatype = "f32"; // data type for communication
 };
 
 // call once at the start of a program if it uses libcommon