electron-rare/prima-cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added support for iq1s and iq1m quantization type

Browse Source
This commit is contained in:
leeetao
2025-04-17 10:27:53 +00:00
parent b212d74dc3 45ec52c2cb
commit fc1e2d3fc6
8 changed files with 973 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+69 -55
View File
@@ -901,13 +901,19 @@ static bool assign_layers_to_device(
float t_read_ram_cpu = 0.0f;
float t_calc_cpu = (
master.model_flops.layer_f32_f32 / (dev.cpu_props.flops_f32_f32 * 1e9 + EPS) +
master.model_flops.layer_f16_f32 / (dev.cpu_props.flops_f16_f32 * 1e9 + EPS) +
master.model_flops.layer_q4k_f32 / (dev.cpu_props.flops_q4k_f32 * 1e9 + EPS) +
master.model_flops.layer_q50_f32 / (dev.cpu_props.flops_q50_f32 * 1e9 + EPS) +
master.model_flops.layer_q5k_f32 / (dev.cpu_props.flops_q5k_f32 * 1e9 + EPS) +
master.model_flops.layer_q6k_f32 / (dev.cpu_props.flops_q6k_f32 * 1e9 + EPS) +
master.model_flops.layer_q80_f32 / (dev.cpu_props.flops_q80_f32 * 1e9 + EPS)) * 1000; // in ms
master.model_flops.layer_f32_f32 / (dev.cpu_props.flops_f32_f32 * 1e9 + EPS) +
master.model_flops.layer_f16_f32 / (dev.cpu_props.flops_f16_f32 * 1e9 + EPS) +
master.model_flops.layer_q2k_f32 / (dev.cpu_props.flops_q2k_f32 * 1e9 + EPS) +
master.model_flops.layer_q4k_f32 / (dev.cpu_props.flops_q4k_f32 * 1e9 + EPS) +
master.model_flops.layer_q5k_f32 / (dev.cpu_props.flops_q5k_f32 * 1e9 + EPS) +
master.model_flops.layer_q6k_f32 / (dev.cpu_props.flops_q6k_f32 * 1e9 + EPS) +
master.model_flops.layer_iq2xxs_f32/ (dev.cpu_props.flops_iq2xxs_f32* 1e9 + EPS) +
master.model_flops.layer_q50_f32 / (dev.cpu_props.flops_q50_f32 * 1e9 + EPS) +
master.model_flops.layer_q80_f32 / (dev.cpu_props.flops_q80_f32 * 1e9 + EPS) +
master.model_flops.layer_iq1s_f32 / (dev.cpu_props.flops_iq1s_f32 * 1e9 + EPS) +
master.model_flops.layer_iq4nl_f32 / (dev.cpu_props.flops_iq4nl_f32 * 1e9 + EPS) +
master.model_flops.layer_iq1m_f32 / (dev.cpu_props.flops_iq1m_f32 * 1e9 + EPS) ) * 1000; // in ms
float t_kv_cpy_cpu = dev.memory.mem_cpy_delay; // in ms
// t_read_ram_cpu = b_prime / (dev.memory.cpu_read_ram_bw * 1e9) * 1000; // in ms
@@ -921,24 +927,36 @@ static bool assign_layers_to_device(
if (dev.gpu_support.metal) {
t_calc_gpu = (
master.model_flops.layer_f32_f32 / (dev.gpu_props.metal_flops_f32_f32 * 1e9 + EPS) +
master.model_flops.layer_f16_f32 / (dev.gpu_props.metal_flops_f16_f32 * 1e9 + EPS) +
master.model_flops.layer_q4k_f32 / (dev.gpu_props.metal_flops_q4k_f32 * 1e9 + EPS) +
master.model_flops.layer_q50_f32 / (dev.gpu_props.metal_flops_q50_f32 * 1e9 + EPS) +
master.model_flops.layer_q5k_f32 / (dev.gpu_props.metal_flops_q5k_f32 * 1e9 + EPS) +
master.model_flops.layer_q6k_f32 / (dev.gpu_props.metal_flops_q6k_f32 * 1e9 + EPS) +
master.model_flops.layer_q80_f32 / (dev.gpu_props.metal_flops_q80_f32 * 1e9 + EPS)) * 1000; // in ms
master.model_flops.layer_f32_f32 / (dev.gpu_props.metal_flops_f32_f32 * 1e9 + EPS) +
master.model_flops.layer_f16_f32 / (dev.gpu_props.metal_flops_f16_f32 * 1e9 + EPS) +
master.model_flops.layer_q2k_f32 / (dev.gpu_props.metal_flops_q2k_f32 * 1e9 + EPS) +
master.model_flops.layer_q4k_f32 / (dev.gpu_props.metal_flops_q4k_f32 * 1e9 + EPS) +
master.model_flops.layer_q5k_f32 / (dev.gpu_props.metal_flops_q5k_f32 * 1e9 + EPS) +
master.model_flops.layer_q6k_f32 / (dev.gpu_props.metal_flops_q6k_f32 * 1e9 + EPS) +
master.model_flops.layer_iq2xxs_f32 / (dev.gpu_props.metal_flops_iq2xxs_f32 * 1e9 + EPS) +
master.model_flops.layer_q50_f32 / (dev.gpu_props.metal_flops_q50_f32 * 1e9 + EPS) +
master.model_flops.layer_q80_f32 / (dev.gpu_props.metal_flops_q80_f32 * 1e9 + EPS) +
master.model_flops.layer_iq1s_f32 / (dev.gpu_props.metal_flops_iq1s_f32 * 1e9 + EPS) +
master.model_flops.layer_iq4nl_f32 / (dev.gpu_props.metal_flops_iq4nl_f32 * 1e9 + EPS) +
master.model_flops.layer_iq1m_f32 / (dev.gpu_props.metal_flops_iq1m_f32 * 1e9 + EPS) ) * 1000; // in ms
t_kv_cpy_gpu = dev.gpu_props.metal_mem_cpy_delay; // in ms
// t_read_ram_gpu = b_prime / (dev.gpu_props.metal_read_vram_bw * 1e9) * 1000; // in ms
} else {
t_calc_gpu = (
master.model_flops.layer_f32_f32 / (dev.gpu_props.cuda_flops_f32_f32 * 1e9 + EPS) +
master.model_flops.layer_f16_f32 / (dev.gpu_props.cuda_flops_f16_f32 * 1e9 + EPS) +
master.model_flops.layer_q4k_f32 / (dev.gpu_props.cuda_flops_q4k_f32 * 1e9 + EPS) +
master.model_flops.layer_q50_f32 / (dev.gpu_props.cuda_flops_q50_f32 * 1e9 + EPS) +
master.model_flops.layer_q5k_f32 / (dev.gpu_props.cuda_flops_q5k_f32 * 1e9 + EPS) +
master.model_flops.layer_q6k_f32 / (dev.gpu_props.cuda_flops_q6k_f32 * 1e9 + EPS) +
master.model_flops.layer_q80_f32 / (dev.gpu_props.cuda_flops_q80_f32 * 1e9 + EPS)) * 1000; // in ms
master.model_flops.layer_f32_f32 / (dev.gpu_props.cuda_flops_f32_f32 * 1e9 + EPS) +
master.model_flops.layer_f16_f32 / (dev.gpu_props.cuda_flops_f16_f32 * 1e9 + EPS) +
master.model_flops.layer_q2k_f32 / (dev.gpu_props.cuda_flops_q2k_f32 * 1e9 + EPS) +
master.model_flops.layer_q4k_f32 / (dev.gpu_props.cuda_flops_q4k_f32 * 1e9 + EPS) +
master.model_flops.layer_q5k_f32 / (dev.gpu_props.cuda_flops_q5k_f32 * 1e9 + EPS) +
master.model_flops.layer_q6k_f32 / (dev.gpu_props.cuda_flops_q6k_f32 * 1e9 + EPS) +
master.model_flops.layer_iq2xxs_f32 / (dev.gpu_props.cuda_flops_iq2xxs_f32 * 1e9 + EPS) +
master.model_flops.layer_q50_f32 / (dev.gpu_props.cuda_flops_q50_f32 * 1e9 + EPS) +
master.model_flops.layer_q80_f32 / (dev.gpu_props.cuda_flops_q80_f32 * 1e9 + EPS) +
master.model_flops.layer_iq1s_f32 / (dev.gpu_props.cuda_flops_iq1s_f32 * 1e9 + EPS) +
master.model_flops.layer_iq4nl_f32 / (dev.gpu_props.cuda_flops_iq4nl_f32 * 1e9 + EPS) +
master.model_flops.layer_iq1m_f32 / (dev.gpu_props.cuda_flops_iq1m_f32 * 1e9 + EPS) ) * 1000; // in ms
t_kv_cpy_gpu = dev.gpu_props.cuda_mem_cpy_delay; // in ms
// t_read_ram_gpu = b_prime / (dev.gpu_props.cuda_read_vram_bw * 1e9) * 1000; // in ms
}
@@ -1113,14 +1131,18 @@ static bool assign_layers_to_device(
if (m == 0) {
kappa = (
dev.model_flops.layer_f32_f32 / (dev.cpu_props.flops_f32_f32 * 1e9 + EPS) +
dev.model_flops.layer_f16_f32 / (dev.cpu_props.flops_f16_f32 * 1e9 + EPS) +
dev.model_flops.layer_q4k_f32 / (dev.cpu_props.flops_q4k_f32 * 1e9 + EPS) +
dev.model_flops.layer_q50_f32 / (dev.cpu_props.flops_q50_f32 * 1e9 + EPS) +
dev.model_flops.layer_q5k_f32 / (dev.cpu_props.flops_q5k_f32 * 1e9 + EPS) +
dev.model_flops.layer_q6k_f32 / (dev.cpu_props.flops_q6k_f32 * 1e9 + EPS) +
dev.model_flops.layer_q80_f32 / (dev.cpu_props.flops_q80_f32 * 1e9 + EPS)) * 1000; // in ms
dev.model_flops.layer_f32_f32 / (dev.cpu_props.flops_f32_f32 * 1e9 + EPS) +
dev.model_flops.layer_f16_f32 / (dev.cpu_props.flops_f16_f32 * 1e9 + EPS) +
dev.model_flops.layer_q2k_f32 / (dev.cpu_props.flops_q2k_f32 * 1e9 + EPS) +
dev.model_flops.layer_q4k_f32 / (dev.cpu_props.flops_q4k_f32 * 1e9 + EPS) +
dev.model_flops.layer_q5k_f32 / (dev.cpu_props.flops_q5k_f32 * 1e9 + EPS) +
dev.model_flops.layer_q6k_f32 / (dev.cpu_props.flops_q6k_f32 * 1e9 + EPS) +
dev.model_flops.layer_iq2xxs_f32 / (dev.cpu_props.flops_iq2xxs_f32 * 1e9 + EPS) +
dev.model_flops.layer_q50_f32 / (dev.cpu_props.flops_q50_f32 * 1e9 + EPS) +
dev.model_flops.layer_q80_f32 / (dev.cpu_props.flops_q80_f32 * 1e9 + EPS) +
dev.model_flops.layer_iq1s_f32 / (dev.cpu_props.flops_iq1s_f32 * 1e9 + EPS) +
dev.model_flops.layer_iq4nl_f32 / (dev.cpu_props.flops_iq4nl_f32 * 1e9 + EPS) +
dev.model_flops.layer_iq1m_f32 / (dev.cpu_props.flops_iq1m_f32 * 1e9 + EPS) ) * 1000; // in ms
// kappa += (bi / n_vocab + bo) / (dev.memory.cpu_read_ram_bw * 1e9) * 1000; // in ms
kappa += (bi / n_vocab) / (disk_speed[m] * 1e9) * 1000; // in ms
@@ -1766,33 +1788,25 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
return mparams;
}
static ggml_type kv_cache_type_from_str(const std::string & s) {
if (s == "f32") {
return GGML_TYPE_F32;
}
if (s == "f16") {
return GGML_TYPE_F16;
}
if (s == "q8_0") {
return GGML_TYPE_Q8_0;
}
if (s == "q4_0") {
return GGML_TYPE_Q4_0;
}
if (s == "q4_1") {
return GGML_TYPE_Q4_1;
}
if (s == "iq4_nl") {
return GGML_TYPE_IQ4_NL;
}
if (s == "q5_0") {
return GGML_TYPE_Q5_0;
}
if (s == "q5_1") {
return GGML_TYPE_Q5_1;
}
const std::vector<ggml_type> kv_cache_types = {
GGML_TYPE_F32,
GGML_TYPE_F16,
GGML_TYPE_BF16, // Added BF16 data type support
GGML_TYPE_Q8_0,
GGML_TYPE_Q4_0,
GGML_TYPE_Q4_1,
GGML_TYPE_IQ4_NL,
GGML_TYPE_Q5_0,
GGML_TYPE_Q5_1,
};
throw std::runtime_error("Invalid cache type: " + s);
static ggml_type kv_cache_type_from_str(const std::string & s) {
for (const auto & type : kv_cache_types) {
if (ggml_type_name(type) == s) {
return type;
}
}
throw std::runtime_error("Unsupported cache type: " + s);
}
struct llama_context_params llama_context_params_from_gpt_params(const gpt_params & params) {
common/profiler.cpp
+505 -170
View File
File diff suppressed because it is too large Load Diff
common/profiler.h
+184 -101
View File
@@ -15,25 +15,36 @@ struct cpu_props {
const char * name;
const char * description;
uint32_t cores;
float flops_f32_f32; // in GFLOPS
float flops_f16_f32; // in GFLOPS
float flops_q4k_f32; // in GFLOPS
float flops_q50_f32; // in GFLOPS
float flops_q5k_f32; // in GFLOPS
float flops_q6k_f32; // in GFLOPS
float flops_q80_f32; // in GFLOPS
float flops_f32_f32; // in GFLOPS
float flops_f16_f32; // in GFLOPS
float flops_q2k_f32; // in GFLOPS
float flops_q4k_f32; // in GFLOPS
float flops_q5k_f32; // in GFLOPS
float flops_q6k_f32; // in GFLOPS
float flops_iq2xxs_f32; // in GFLOPS
float flops_q50_f32; // in GFLOPS
float flops_q80_f32; // in GFLOPS
float flops_iq1s_f32; // in GFLOPS
float flops_iq4nl_f32; // in GFLOPS
float flops_iq1m_f32; // in GFLOPS
cpu_props() :
name(""),
description(""),
cores(0),
flops_f32_f32(0.0f),
flops_f16_f32(0.0f),
flops_q4k_f32(0.0f),
flops_q50_f32(0.0f),
flops_q5k_f32(0.0f),
flops_q6k_f32(0.0f),
flops_q80_f32(0.0f) {}
cpu_props()
: name (""),
description (""),
cores (0),
flops_f32_f32 (0.0f),
flops_f16_f32 (0.0f),
flops_q2k_f32 (0.0f),
flops_q4k_f32 (0.0f),
flops_q5k_f32 (0.0f),
flops_q6k_f32 (0.0f),
flops_iq2xxs_f32(0.0f),
flops_q50_f32 (0.0f),
flops_q80_f32 (0.0f),
flops_iq1s_f32 (0.0f),
flops_iq4nl_f32 (0.0f),
flops_iq1m_f32 (0.0f)
{}
};
struct memory_info {
@@ -77,132 +88,204 @@ struct gpu_support {
struct gpu_props {
const char * name;
const char * description;
float memory_free; // in GiB
float memory_total; // in GiB
float metal_read_vram_bw; // in GB/s
float metal_flops_f32_f32; // in GFLOPS
float metal_flops_f16_f32; // in GFLOPS
float metal_flops_q4k_f32; // in GFLOPS
float metal_flops_q50_f32; // in GFLOPS
float metal_flops_q5k_f32; // in GFLOPS
float metal_flops_q6k_f32; // in GFLOPS
float metal_flops_q80_f32; // in GFLOPS
float metal_mem_cpy_delay; // in ms
float cuda_read_vram_bw; // in GB/s
float cuda_flops_f32_f32; // in GFLOPS
float cuda_flops_f16_f32; // in GFLOPS
float cuda_flops_q4k_f32; // in GFLOPS
float cuda_flops_q50_f32; // in GFLOPS
float cuda_flops_q5k_f32; // in GFLOPS
float cuda_flops_q6k_f32; // in GFLOPS
float cuda_flops_q80_f32; // in GFLOPS
float cuda_mem_cpy_delay; // in ms
float memory_free; // in GiB
float memory_total; // in GiB
float metal_read_vram_bw; // in GB/s
float metal_flops_f32_f32; // in GFLOPS
float metal_flops_f16_f32; // in GFLOPS
float metal_flops_q2k_f32; // in GFLOPS
float metal_flops_q4k_f32; // in GFLOPS
float metal_flops_q5k_f32; // in GFLOPS
float metal_flops_q6k_f32; // in GFLOPS
float metal_flops_iq2xxs_f32; // in GFLOPS
float metal_flops_q50_f32; // in GFLOPS
float metal_flops_q80_f32; // in GFLOPS
float metal_flops_iq1s_f32; // in GFLOPS
float metal_flops_iq4nl_f32; // in GFLOPS
float metal_flops_iq1m_f32; // in GFLOPS
float metal_mem_cpy_delay; // in ms
float cuda_read_vram_bw; // in GB/s
float cuda_flops_f32_f32; // in GFLOPS
float cuda_flops_f16_f32; // in GFLOPS
float cuda_flops_q2k_f32; // in GFLOPS
float cuda_flops_q4k_f32; // in GFLOPS
float cuda_flops_q5k_f32; // in GFLOPS
float cuda_flops_q6k_f32; // in GFLOPS
float cuda_flops_iq2xxs_f32; // in GFLOPS
float cuda_flops_q50_f32; // in GFLOPS
float cuda_flops_q80_f32; // in GFLOPS
float cuda_flops_iq1s_f32; // in GFLOPS
float cuda_flops_iq4nl_f32; // in GFLOPS
float cuda_flops_iq1m_f32; // in GFLOPS
float cuda_mem_cpy_delay; // in ms
gpu_props() :
name(""),
description(""),
memory_free (0.0f),
memory_total (0.0f),
metal_read_vram_bw (0.0f),
metal_flops_f32_f32(0.0f),
metal_flops_f16_f32(0.0f),
metal_flops_q4k_f32(0.0f),
metal_flops_q50_f32(0.0f),
metal_flops_q5k_f32(0.0f),
metal_flops_q6k_f32(0.0f),
metal_flops_q80_f32(0.0f),
metal_mem_cpy_delay(0.0f),
cuda_read_vram_bw (0.0f),
cuda_flops_f32_f32 (0.0f),
cuda_flops_f16_f32 (0.0f),
cuda_flops_q4k_f32 (0.0f),
cuda_flops_q50_f32 (0.0f),
cuda_flops_q5k_f32 (0.0f),
cuda_flops_q6k_f32 (0.0f),
cuda_flops_q80_f32 (0.0f),
cuda_mem_cpy_delay (0.0f) {}
name (""),
description (""),
memory_free (0.0f),
memory_total (0.0f),
metal_read_vram_bw (0.0f),
metal_flops_f32_f32 (0.0f),
metal_flops_f16_f32 (0.0f),
metal_flops_q2k_f32 (0.0f),
metal_flops_q4k_f32 (0.0f),
metal_flops_q5k_f32 (0.0f),
metal_flops_q6k_f32 (0.0f),
metal_flops_iq2xxs_f32 (0.0f),
metal_flops_q50_f32 (0.0f),
metal_flops_q80_f32 (0.0f),
metal_flops_iq1s_f32 (0.0f),
metal_flops_iq4nl_f32 (0.0f),
metal_flops_iq1m_f32 (0.0f),
metal_mem_cpy_delay (0.0f),
cuda_read_vram_bw (0.0f),
cuda_flops_f32_f32 (0.0f),
cuda_flops_f16_f32 (0.0f),
cuda_flops_q2k_f32 (0.0f),
cuda_flops_q4k_f32 (0.0f),
cuda_flops_q5k_f32 (0.0f),
cuda_flops_q6k_f32 (0.0f),
cuda_flops_iq2xxs_f32 (0.0f),
cuda_flops_q50_f32 (0.0f),
cuda_flops_q80_f32 (0.0f),
cuda_flops_iq1s_f32 (0.0f),
cuda_flops_iq4nl_f32 (0.0f),
cuda_flops_iq1m_f32 (0.0f),
cuda_mem_cpy_delay (0.0f) {}
};
struct model_flops {
float inp_embd_ms;
int64_t output_f32_f32;
int64_t output_f16_f32;
int64_t output_q2k_f32;
int64_t output_q4k_f32;
int64_t output_q50_f32;
int64_t output_q5k_f32;
int64_t output_q6k_f32;
int64_t output_iq2xxs_f32;
int64_t output_q50_f32;
int64_t output_q80_f32;
int64_t output_iq1s_f32;
int64_t output_iq4nl_f32;
int64_t output_iq1m_f32;
int64_t layer_f32_f32;
int64_t layer_f16_f32;
int64_t layer_q2k_f32;
int64_t layer_q4k_f32;
int64_t layer_q50_f32;
int64_t layer_q5k_f32;
int64_t layer_q6k_f32;
int64_t layer_iq2xxs_f32;
int64_t layer_q50_f32;
int64_t layer_q80_f32;
int64_t layer_iq1s_f32;
int64_t layer_iq4nl_f32;
int64_t layer_iq1m_f32;
model_flops() :
inp_embd_ms(0.0f),
output_f32_f32(0),
output_f16_f32(0),
output_q4k_f32(0),
output_q50_f32(0),
output_q5k_f32(0),
output_q6k_f32(0),
output_q80_f32(0),
layer_f32_f32 (0),
layer_f16_f32 (0),
layer_q4k_f32 (0),
layer_q50_f32 (0),
layer_q5k_f32 (0),
layer_q6k_f32 (0),
layer_q80_f32 (0) {}
inp_embd_ms (0.0f),
output_f32_f32 (0),
output_f16_f32 (0),
output_q2k_f32 (0),
output_q4k_f32 (0),
output_q5k_f32 (0),
output_q6k_f32 (0),
output_iq2xxs_f32 (0),
output_q50_f32 (0),
output_q80_f32 (0),
output_iq1s_f32 (0),
output_iq4nl_f32 (0),
output_iq1m_f32 (0),
layer_f32_f32 (0),
layer_f16_f32 (0),
layer_q2k_f32 (0),
layer_q4k_f32 (0),
layer_q5k_f32 (0),
layer_q6k_f32 (0),
layer_iq2xxs_f32 (0),
layer_q50_f32 (0),
layer_q80_f32 (0),
layer_iq1s_f32 (0),
layer_iq4nl_f32 (0),
layer_iq1m_f32 (0)
{}
};
struct model_params {
int64_t input_f32;
int64_t input_f16;
int64_t input_q2k;
int64_t input_q4k;
int64_t input_q50;
int64_t input_q5k;
int64_t input_q6k;
int64_t input_iq2xxs;
int64_t input_q50;
int64_t input_q80;
int64_t input_iq1s;
int64_t input_iq4nl;
int64_t input_iq1m;
int64_t output_f32;
int64_t output_f16;
int64_t output_q2k;
int64_t output_q4k;
int64_t output_q50;
int64_t output_q5k;
int64_t output_q6k;
int64_t output_iq2xxs;
int64_t output_q50;
int64_t output_q80;
int64_t output_iq1s;
int64_t output_iq4nl;
int64_t output_iq1m;
int64_t layer_f32;
int64_t layer_f16;
int64_t layer_q2k;
int64_t layer_q4k;
int64_t layer_q50;
int64_t layer_q5k;
int64_t layer_q6k;
int64_t layer_iq2xxs;
int64_t layer_q50;
int64_t layer_q80;
int64_t layer_iq1s;
int64_t layer_iq4nl;
int64_t layer_iq1m;
model_params() :
input_f32 (0),
input_f16 (0),
input_q4k (0),
input_q50 (0),
input_q5k (0),
input_q6k (0),
input_q80 (0),
output_f32(0),
output_f16(0),
output_q4k(0),
output_q50(0),
output_q5k(0),
output_q6k(0),
output_q80(0),
layer_f32 (0),
layer_f16 (0),
layer_q4k (0),
layer_q50 (0),
layer_q5k (0),
layer_q6k (0),
layer_q80 (0) {}
input_f32 (0),
input_f16 (0),
input_q2k (0),
input_q4k (0),
input_q5k (0),
input_q6k (0),
input_iq2xxs (0),
input_q50 (0),
input_q80 (0),
input_iq1s (0),
input_iq4nl (0),
input_iq1m (0),
output_f32 (0),
output_f16 (0),
output_q2k (0),
output_q4k (0),
output_q5k (0),
output_q6k (0),
output_iq2xxs (0),
output_q50 (0),
output_q80 (0),
output_iq1s (0),
output_iq4nl (0),
output_iq1m (0),
layer_f32 (0),
layer_f16 (0),
layer_q2k (0),
layer_q4k (0),
layer_q5k (0),
layer_q6k (0),
layer_iq2xxs (0),
layer_q50 (0),
layer_q80 (0),
layer_iq1s (0),
layer_iq4nl (0),
layer_iq1m (0)
{}
};
struct model_bytes {