electron-rare/prima-cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

synchronize device info

Browse Source
This commit is contained in:
Lizonghang
2024-11-07 22:02:01 +04:00
parent ef7fdf70cc
commit 53cb3a6069
5 changed files with 408 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+24 -10
View File
@@ -833,10 +833,6 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
model = llama_load_model_from_file(params.model.c_str(), mparams);
}
// profile devices and determine the best setup
device_info dev_info;
llama_profile_device(&dev_info, model, params.model.c_str());
if (model == NULL) {
LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.c_str());
return iparams;
@@ -866,17 +862,35 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
}
}
auto cparams = llama_context_params_from_gpt_params(params);
// get device profile
device_info dev_info;
dev_info.rank = params.rank;
llama_profile_device(&dev_info, model, params.model.c_str());
llama_context * lctx = llama_new_context_with_model(model, cparams);
if (lctx == NULL) {
LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.c_str());
// create llama context
struct llama_context_params cparams = llama_context_params_from_gpt_params(params);
llama_context * lctx = llama_new_context_with_model(model, cparams);
// initialize sockets
llama_init_sockets(lctx, cparams.n_world, cparams.rank);
// sychronize device profile to the master node
struct device_info * dev_info_set = nullptr;
if (params.rank == 0) {
dev_info_set = (struct device_info *)malloc(cparams.n_world * sizeof(struct device_info));
dev_info_set[0] = dev_info;
llama_collect_device_info(dev_info_set, lctx);
device_print_props(dev_info_set, cparams.n_world);
} else {
llama_send_device_info(&dev_info, lctx);
}
if (llama_context_setup_backend(lctx) == nullptr) {
LOG_ERR("%s: failed to setup context with model '%s'\n", __func__, params.model.c_str());
llama_free_model(model);
return iparams;
}
llama_init_sockets(lctx, cparams.n_world, cparams.rank);
if (!params.control_vectors.empty()) {
if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_n_layer(model);
common/profiler.cpp
+280 -3
View File
@@ -300,8 +300,6 @@ int device_has_sycl(void) {
return ggml_cpu_has_sycl();
}
// ggml_backend_buffer_type_t llama_dev_buffer_type(const llama_model * model, int device)
void device_get_props(struct llama_model * model, int device, struct ggml_backend_dev_props * props) {
ggml_backend_buffer_type_t buft_type;
if (device == -1) { // type cpu
@@ -311,4 +309,283 @@ void device_get_props(struct llama_model * model, int device, struct ggml_backen
}
ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft_type);
ggml_backend_dev_get_props(dev, props);
}
}
void device_print_props(struct device_info * dev_info_set, int n) {
LOG_INF("\n-------------------------------------------------------------------------------------------\n");
LOG_INF("| Property ");
for (int i = 0; i < n; ++i) {
LOG_INF("| Rank %-8d", i);
GGML_ASSERT(dev_info_set[i].rank == i);
}
LOG_INF("\n-------------------------------------------------------------------------------------------\n");
LOG_INF("| Device Name ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.10s ", dev_info_set[i].device_name);
}
LOG_INF("\n");
LOG_INF("| CPU Name ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.10s ", dev_info_set[i].cpu_props.name);
}
LOG_INF("\n");
LOG_INF("| CPU Description ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.10s ", dev_info_set[i].cpu_props.description);
}
LOG_INF("\n");
LOG_INF("| Number of CPU cores ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10u ", dev_info_set[i].cpu_props.cores);
}
LOG_INF("\n");
LOG_INF("| Physical Mem Total (GB) ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.2f ", dev_info_set[i].memory.total_physical);
}
LOG_INF("\n");
LOG_INF("| Physical Mem Available (GB) ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.2f ", dev_info_set[i].memory.available_physical);
}
LOG_INF("\n");
LOG_INF("| Swap Mem Total (GB) ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.2f ", dev_info_set[i].memory.total_swap);
}
LOG_INF("\n");
LOG_INF("| Swap Mem Available (GB) ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.2f ", dev_info_set[i].memory.available_swap);
}
LOG_INF("\n");
LOG_INF("| Mem Bandwidth (GB/s) ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.2f ", dev_info_set[i].memory.bandwidth);
}
LOG_INF("\n");
LOG_INF("| Disk Read Bandwidth (GB/s) ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.2f ", dev_info_set[i].disk_read_bandwidth);
}
LOG_INF("\n");
LOG_INF("| GPU Metal ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10d ", dev_info_set[i].gpu_support.metal);
}
LOG_INF("\n");
LOG_INF("| GPU CUDA ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10d ", dev_info_set[i].gpu_support.cuda);
}
LOG_INF("\n");
LOG_INF("| GPU Vulkan ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10d ", dev_info_set[i].gpu_support.vulkan);
}
LOG_INF("\n");
LOG_INF("| GPU Kompute ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10d ", dev_info_set[i].gpu_support.kompute);
}
LOG_INF("\n");
LOG_INF("| GPU BLAS ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10d ", dev_info_set[i].gpu_support.gpublas);
}
LOG_INF("\n");
LOG_INF("| BLAS ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10d ", dev_info_set[i].gpu_support.blas);
}
LOG_INF("\n");
LOG_INF("| SYCL ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10d ", dev_info_set[i].gpu_support.sycl);
}
LOG_INF("\n");
LOG_INF("| GPU Name ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.10s ", dev_info_set[i].gpu_props.name);
}
LOG_INF("\n");
LOG_INF("| GPU Description ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.10s ", dev_info_set[i].gpu_props.description);
}
LOG_INF("\n");
LOG_INF("| GPU Mem Free (GB) ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.2f ", dev_info_set[i].gpu_props.memory_free);
}
LOG_INF("\n");
LOG_INF("| GPU Mem Total (GB) ");
for (int i = 0; i < n; ++i) {
LOG_INF("| %-10.2f ", dev_info_set[i].gpu_props.memory_total);
}
LOG_INF("\n");
LOG_INF("-------------------------------------------------------------------------------------------\n\n");
}
size_t serialize(const struct device_info * dev_info, char ** buffer) {
// calculate total size for serialized buffer
size_t device_name_len = strlen(dev_info->device_name) + 1;
size_t cpu_name_len = strlen(dev_info->cpu_props.name) + 1;
size_t cpu_description_len = strlen(dev_info->cpu_props.description) + 1;
size_t gpu_name_len = strlen(dev_info->gpu_props.name) + 1;
size_t gpu_description_len = strlen(dev_info->gpu_props.description) + 1;
size_t total_size = sizeof(uint32_t)
+ sizeof(size_t) * 5 // for lengths of strings
+ device_name_len
+ cpu_name_len
+ cpu_description_len
+ gpu_name_len
+ gpu_description_len
+ sizeof(float) // disk_read_bandwidth
+ sizeof(uint32_t) // cpu_props.cores
+ sizeof(struct memory_info)
+ sizeof(struct gpu_support)
+ sizeof(float) * 2; // gpu_props.memory_free and gpu_props.memory_total
*buffer = (char *)malloc(total_size);
char * ptr = *buffer;
// rank
memcpy(ptr, &dev_info->rank, sizeof(uint32_t));
ptr += sizeof(uint32_t);
// copy string lengths and string data
memcpy(ptr, &device_name_len, sizeof(size_t));
ptr += sizeof(size_t);
memcpy(ptr, dev_info->device_name, device_name_len);
ptr += device_name_len;
memcpy(ptr, &cpu_name_len, sizeof(size_t));
ptr += sizeof(size_t);
memcpy(ptr, dev_info->cpu_props.name, cpu_name_len);
ptr += cpu_name_len;
memcpy(ptr, &cpu_description_len, sizeof(size_t));
ptr += sizeof(size_t);
memcpy(ptr, dev_info->cpu_props.description, cpu_description_len);
ptr += cpu_description_len;
memcpy(ptr, &gpu_name_len, sizeof(size_t));
ptr += sizeof(size_t);
memcpy(ptr, dev_info->gpu_props.name, gpu_name_len);
ptr += gpu_name_len;
memcpy(ptr, &gpu_description_len, sizeof(size_t));
ptr += sizeof(size_t);
memcpy(ptr, dev_info->gpu_props.description, gpu_description_len);
ptr += gpu_description_len;
// copy the non-string members
memcpy(ptr, &dev_info->disk_read_bandwidth, sizeof(float));
ptr += sizeof(float);
memcpy(ptr, &dev_info->cpu_props.cores, sizeof(uint32_t));
ptr += sizeof(uint32_t);
memcpy(ptr, &dev_info->memory, sizeof(struct memory_info));
ptr += sizeof(struct memory_info);
memcpy(ptr, &dev_info->gpu_support, sizeof(struct gpu_support));
ptr += sizeof(struct gpu_support);
memcpy(ptr, &dev_info->gpu_props.memory_free, sizeof(float));
ptr += sizeof(float);
memcpy(ptr, &dev_info->gpu_props.memory_total, sizeof(float));
return total_size;
}
void deserialize(const char * buffer, struct device_info * dev_info) {
const char * ptr = buffer;
// rank
memcpy(&dev_info->rank, ptr, sizeof(uint32_t));
ptr += sizeof(uint32_t);
// device_name
size_t device_name_len;
memcpy(&device_name_len, ptr, sizeof(size_t));
ptr += sizeof(size_t);
dev_info->device_name = (char *)malloc(device_name_len);
memcpy((void *)dev_info->device_name, ptr, device_name_len);
ptr += device_name_len;
// cpu_props.name
size_t cpu_name_len;
memcpy(&cpu_name_len, ptr, sizeof(size_t));
ptr += sizeof(size_t);
dev_info->cpu_props.name = (char *)malloc(cpu_name_len);
memcpy((void *)dev_info->cpu_props.name, ptr, cpu_name_len);
ptr += cpu_name_len;
// cpu_props.description
size_t cpu_description_len;
memcpy(&cpu_description_len, ptr, sizeof(size_t));
ptr += sizeof(size_t);
dev_info->cpu_props.description = (char *)malloc(cpu_description_len);
memcpy((void *)dev_info->cpu_props.description, ptr, cpu_description_len);
ptr += cpu_description_len;
// gpu_props.name
size_t gpu_name_len;
memcpy(&gpu_name_len, ptr, sizeof(size_t));
ptr += sizeof(size_t);
dev_info->gpu_props.name = (char *)malloc(gpu_name_len);
memcpy((void *)dev_info->gpu_props.name, ptr, gpu_name_len);
ptr += gpu_name_len;
// gpu_props.description
size_t gpu_description_len;
memcpy(&gpu_description_len, ptr, sizeof(size_t));
ptr += sizeof(size_t);
dev_info->gpu_props.description = (char *)malloc(gpu_description_len);
memcpy((void *)dev_info->gpu_props.description, ptr, gpu_description_len);
ptr += gpu_description_len;
// other non-string members
memcpy(&dev_info->disk_read_bandwidth, ptr, sizeof(float));
ptr += sizeof(float);
memcpy(&dev_info->cpu_props.cores, ptr, sizeof(uint32_t));
ptr += sizeof(uint32_t);
memcpy(&dev_info->memory, ptr, sizeof(struct memory_info));
ptr += sizeof(struct memory_info);
memcpy(&dev_info->gpu_support, ptr, sizeof(struct gpu_support));
ptr += sizeof(struct gpu_support);
memcpy(&dev_info->gpu_props.memory_free, ptr, sizeof(float));
ptr += sizeof(float);
memcpy(&dev_info->gpu_props.memory_total, ptr, sizeof(float));
}
common/profiler.h
+11 -6
View File
@@ -35,6 +35,7 @@ struct gpu_props {
};
struct device_info {
uint32_t rank;
const char * device_name;
float disk_read_bandwidth; // in GB/s
struct cpu_props cpu_props;
@@ -50,14 +51,18 @@ uint64_t device_physical_memory(bool available);
uint64_t device_swap_memory (bool available);
uint64_t device_disk_read_bw (const char * test_file, size_t buffer_size_mb);
uint64_t device_memory_bw (size_t buffer_size_mb);
void device_get_props (struct llama_model * model, int device, struct ggml_backend_dev_props * props);
void device_get_props (struct llama_model * model, int device, struct ggml_backend_dev_props * props);
void device_print_props (struct device_info * dev_info_set, int n);
int device_has_metal(void);
int device_has_cuda(void);
int device_has_vulkan(void);
int device_has_metal (void);
int device_has_cuda (void);
int device_has_vulkan (void);
int device_has_kompute(void);
int device_has_gpublas(void);
int device_has_blas(void);
int device_has_sycl(void);
int device_has_blas (void);
int device_has_sycl (void);
size_t serialize (const struct device_info * dev_info, char ** buffer);
void deserialize(const char * buffer, struct device_info * dev_info);
#endif // PROFILER_H