Gökdeniz Gülmez
1974376d70
* Add GLM4 MoE DSA model implementation with configurable parameters * Update Acknowledgments to include GLM4 MoE DSA support * format * update ackn. * Fixes * Update acknowledgments to include contributions for GLM MoE DSA and additional architectures * use dsv32 for glm5 * fix * Fix rope theta --------- Co-authored-by: Tarjei Mandt <kernelpool@gmail.com> Co-authored-by: Awni Hannun <awni@apple.com>
54 lines
1.3 KiB
Python
54 lines
1.3 KiB
Python
# Copyright © 2025 Apple Inc.
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any, Dict, Optional
|
|
|
|
from .base import BaseModelArgs
|
|
from .deepseek_v32 import Model as DSV32Model
|
|
|
|
|
|
@dataclass
|
|
class ModelArgs(BaseModelArgs):
|
|
model_type: str
|
|
vocab_size: int
|
|
hidden_size: int
|
|
index_head_dim: int
|
|
index_n_heads: int
|
|
index_topk: int
|
|
intermediate_size: int
|
|
moe_intermediate_size: int
|
|
num_hidden_layers: int
|
|
num_attention_heads: int
|
|
num_key_value_heads: int
|
|
n_shared_experts: Optional[int]
|
|
n_routed_experts: Optional[int]
|
|
routed_scaling_factor: float
|
|
kv_lora_rank: int
|
|
q_lora_rank: int
|
|
qk_rope_head_dim: int
|
|
v_head_dim: int
|
|
qk_nope_head_dim: int
|
|
topk_method: str
|
|
scoring_func: str
|
|
norm_topk_prob: bool
|
|
n_group: int
|
|
topk_group: int
|
|
num_experts_per_tok: int
|
|
moe_layer_freq: int
|
|
first_k_dense_replace: int
|
|
max_position_embeddings: int
|
|
rms_norm_eps: float
|
|
rope_parameters: Dict
|
|
attention_bias: bool
|
|
rope_scaling: Dict = None
|
|
rope_theta: Optional[float] = None
|
|
|
|
def __post_init__(self):
|
|
self.rope_scaling = self.rope_parameters
|
|
self.rope_theta = self.rope_parameters["rope_theta"]
|
|
|
|
|
|
class Model(DSV32Model):
|
|
def __init__(self, config: ModelArgs):
|
|
super().__init__(config)
|