Upload OLMo-2 model checkpoint

config.json

@@ -1 +1,32 @@
-{"run_name": "caia_olmo2_1b_d3", "launch": {"name": "caia_olmo2_1b_d3-train-95a8ba94", "cmd": ["src/scripts/train/OLMoE-2x1B-anneal.py", "train", "caia_olmo2_1b_d3", "ai2/jupiter-cirrascale-2", "--launch.num_nodes=1", "--launch.workspace=OLMo-modular", "--launch.priority=urgent", "--launch.beaker_image=petew/olmo-core-tch260cu124", "--trainer.callbacks.wandb.enabled=True", "--trainer.callbacks.comet.enabled=False", "--trainer.max_duration.value=200_000_000", "--trainer.max_duration.unit=tokens", "--dataset.mix_base_dir=/weka/oe-training-default/ai2-llm/preprocessed", "--dataset.mix=caia", "--trainer.load_path=/weka/oe-training-default/ai2-llm/checkpoints/swj0419/model/merge/olmo2_1b/moe2_random", "--train_module.dp_config.num_replicas=4", "--train_module.scheduler.warmup_steps=2000", "--train_module.optim.lr=4e-5", "--model.block.feed_forward_moe.router.top_k=2", "--train_module.rank_microbatch_size=4096", "--train_module.ep_config.degree=2"], "budget": "ai2/oe-base", "task_name": "train", "workspace": "OLMo-modular", "setup_steps": ["conda install gh --channel conda-forge", "gh repo clone \"$REPO_URL\" .", "git checkout \"$GIT_REF\"", "git submodule update --init --recursive", "conda shell.bash activate base", "pip install -e '.[dev,beaker,wandb,train]'", "pip freeze", "mkdir -p ~/.aws", "printenv AWS_CONFIG > ~/.aws/config", "printenv AWS_CREDENTIALS > ~/.aws/credentials"], "beaker_image": "petew/olmo-core-tch260cu124", "num_nodes": 1, "num_gpus": 8, "shared_memory": "10GiB", "clusters": ["ai2/jupiter-cirrascale-2"], "shared_filesystem": true, "priority": "urgent", "preemptible": true, "env_vars": [{"name": "NCCL_DEBUG", "value": "WARN", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvVar"}, {"name": "CUDA_LAUNCH_BLOCKING", "value": "0", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvVar"}], "env_secrets": [{"name": "GITHUB_TOKEN", "secret": "weijias_GITHUB_TOKEN", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "BEAKER_TOKEN", "secret": "weijias_BEAKER_TOKEN", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "WANDB_API_KEY", "secret": "weijias_WANDB_API_KEY", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "COMET_API_KEY", "secret": "weijias_COMET_API_KEY", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "AWS_CONFIG", "secret": "weijias_AWS_CONFIG", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "AWS_CREDENTIALS", "secret": "weijias_AWS_CREDENTIALS", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "R2_ENDPOINT_URL", "secret": "R2_ENDPOINT_URL", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "WEKA_ENDPOINT_URL", "secret": "WEKA_ENDPOINT_URL", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "SLACK_WEBHOOK_URL", "secret": "SLACK_WEBHOOK_URL", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}], "nfs": false, "weka_buckets": [{"bucket": "oe-training-default", "mount": "/weka/oe-training-default", "_CLASS_": "olmo_core.launch.beaker.BeakerWekaBucket"}], "allow_dirty": false, "_CLASS_": "olmo_core.launch.beaker.BeakerLaunchConfig"}, "model": {"d_model": 2048, "vocab_size": 100352, "n_layers": 16, "block": {"attention": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "use_flash": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.attention.AttentionConfig"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward_moe": {"name": "default", "num_experts": 2, "hidden_size": 8192, "capacity_factor": 1.2, "router": {"name": "default", "top_k": 2, "uniform_expert_assignment": false, "_CLASS_": "olmo_core.nn.moe.router.MoERouterConfig"}, "lb_loss_weight": 0.0, "z_loss_weight": 0.001, "dtype": "float32", "_CLASS_": "olmo_core.nn.moe.moe.MoEConfig"}, "name": "moe_reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "moe", "dtype": "float32", "init_method": "normal", "init_seed": 0, "freeze_params": ["embeddings.*", "blocks.*.attention*", "blocks.*.feed_forward_norm.*", "lm_head.*"], "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "name": "fsl", "sequence_length": 4096, "max_target_sequence_length": 8192, "mix": "caia", "mix_base_dir": "/weka/oe-training-default/ai2-llm", "include_instance_metadata": true, "generate_doc_lengths": false, "expand_glob": false, "work_dir": "/weka/oe-training-default/ai2-llm/checkpoints/weijias/caia_olmo2_1b_d3/dataset-cache", "_CLASS_": "olmo_core.data.numpy_dataset.NumpyDatasetConfig"}, "data_loader": {"global_batch_size": 4194304, "seed": 34521, "num_workers": 4, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig"}, "train_module": {"rank_microbatch_size": 4096, "max_sequence_length": 4096, "optim": {"compile": false, "fixed_fields": ["initial_lr"], "lr": 4e-05, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.0, "fused": true, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "warmup_steps": 2000, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "num_replicas": 4, "wrapping_strategy": "fine_grained", "prefetch_factor": 0, "_CLASS_": "olmo_core.train.train_module.transformer.TransformerDataParallelConfig"}, "ep_config": {"degree": 2, "_CLASS_": "olmo_core.train.train_module.transformer.TransformerExpertParallelConfig"}, "z_loss_multiplier": 1e-05, "label_ignore_index": -100, "_CLASS_": "olmo_modular.train.train_module.transfomer.FreezeTransformerTrainModuleConfig"}, "trainer": {"save_folder": "/weka/oe-training-default/ai2-llm/checkpoints/weijias/caia_olmo2_1b_d3", "load_path": "/weka/oe-training-default/ai2-llm/checkpoints/swj0419/model/merge/olmo2_1b/moe2_random", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": true, "max_duration": {"value": 200000000, "unit": "tokens", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 1, "metrics_collect_interval": 10, "callbacks": {"downstream_evaluator": {"tasks": ["piqa"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 500, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "enabled": true, "_CLASS_": "olmo_modular.eval.evaluator_callback.DownstreamEvaluatorUpdatedCallbackConfig"}, "checkpointer": {"save_interval": 10000, "ephemeral_save_interval": 250, "save_async": true, "remove": "ephemeral_only", "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "comet": {"enabled": false, "name": "caia_olmo2_1b_d3", "project": "OLMo-modular", "workspace": "ai2", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "notifications": "none", "failure_tag": "failed", "_CLASS_": "olmo_core.train.callbacks.comet.CometCallback"}, "wandb": {"enabled": true, "name": "caia_olmo2_1b_d3", "project": "OLMo-modular", "entity": "ai2-llm", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "profiler": {"skip_first": 0, "wait": 1, "warmup": 5, "active": 3, "repeat": 1, "enabled": false, "_CLASS_": "olmo_core.train.callbacks.profiler.ProfilerCallback"}, "garbage_collector": {"gc_interval": 1000, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.garbage_collector.GarbageCollectorCallback"}, "slack_notifier": {"name": "caia_olmo2_1b_d3", "notifications": "end_only", "enabled": false, "_CLASS_": "olmo_core.train.callbacks.slack_notifier.SlackNotifierCallback"}, "beaker": {"enabled": true, "_CLASS_": "olmo_core.train.callbacks.beaker.BeakerCallback"}, "gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "name": "padded_fsl", "sequence_length": 4096, "mix": "v3-small-ppl-validation", "mix_base_dir": "/weka/oe-training-default/ai2-llm", "include_instance_metadata": true, "generate_doc_lengths": false, "expand_glob": false, "work_dir": "/weka/oe-training-default/ai2-llm/checkpoints/weijias/dataset-cache", "_CLASS_": "olmo_core.data.numpy_dataset.NumpyDatasetConfig"}, "eval_interval": 1000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}}, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 12536, "_CLASS_": "olmo_modular.internal.freeze_experiment.ExperimentConfig"}
+{
+  "architectures": [
+    "Olmoe2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "clip_qkv": null,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 4096,
+  "model_type": "olmoe2",
+  "norm_topk_prob": false,
+  "num_attention_heads": 16,
+  "num_experts": 2,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "output_router_logits": false,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "router_aux_loss_coef": 0.01,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.50.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100352
+}

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.50.0.dev0"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06c127b3c484c0fe62a7aacad7a3bb5fc8ae94678f81383544a58bf4b9a60db7
+size 4983274792

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48e7e2fbe7be3055e776e30263c0c12aa51e36c6bf49c4bb537013e6d6d9f239
+size 4177908448

model.safetensors.index.json

@@ -0,0 +1,250 @@
+{
+  "metadata": {
+    "total_size": 9161154560
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.experts.0.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.mlp.experts.0.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.mlp.experts.0.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.mlp.experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.mlp.experts.1.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.mlp.experts.1.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.mlp.gate.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.mlp.experts.0.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.mlp.experts.0.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.mlp.experts.0.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.mlp.experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.mlp.experts.1.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.mlp.experts.1.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.mlp.gate.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.mlp.experts.0.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.mlp.experts.0.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.mlp.experts.0.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.mlp.experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.mlp.experts.1.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.mlp.experts.1.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.mlp.gate.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.mlp.experts.0.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.mlp.experts.0.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.mlp.experts.0.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.mlp.experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.mlp.experts.1.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.mlp.experts.1.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.mlp.gate.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.experts.0.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.experts.0.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.experts.0.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.experts.1.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.experts.1.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.gate.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.experts.0.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.experts.0.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.experts.0.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.experts.1.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.experts.1.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.gate.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.2.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.8.mlp.experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.experts.0.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.mlp.experts.0.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.mlp.experts.0.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.mlp.experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.mlp.experts.1.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.mlp.experts.1.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.mlp.gate.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}