{
  "tensors": {
    "h.3.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "wte.weight": {
      "type": "Distributed",
      "shape": [
        50257,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            50257,
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            384
          ],
          "shape": [
            50257,
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "ln_f.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "ln_f.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.7.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "wpe.weight": {
      "type": "Distributed",
      "shape": [
        1024,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1024,
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            384
          ],
          "shape": [
            1024,
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.attn.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        768,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384,
            0
          ],
          "shape": [
            384,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.0.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.9.mlp.c_fc.bias": {
      "type": "Distributed",
      "shape": [
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536
          ],
          "shape": [
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.ln_1.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.4.attn.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.6.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.2.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.attn.c_attn.weight": {
      "type": "Distributed",
      "shape": [
        768,
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1152
          ],
          "shape": [
            768,
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.ln_2.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.8.mlp.c_proj.weight": {
      "type": "Distributed",
      "shape": [
        3072,
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1536,
            0
          ],
          "shape": [
            1536,
            768
          ],
          "filename_index": 1
        }
      ]
    },
    "h.5.attn.c_attn.bias": {
      "type": "Distributed",
      "shape": [
        2304
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            1152
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            1152
          ],
          "shape": [
            1152
          ],
          "filename_index": 1
        }
      ]
    },
    "h.3.ln_2.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.11.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.ln_1.weight": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.10.attn.bias": {
      "type": "Distributed",
      "shape": [
        1,
        1,
        1024,
        1024
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0,
            0,
            0
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            0,
            0,
            512
          ],
          "shape": [
            1,
            1,
            1024,
            512
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.mlp.c_proj.bias": {
      "type": "Distributed",
      "shape": [
        768
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0
          ],
          "shape": [
            384
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            384
          ],
          "shape": [
            384
          ],
          "filename_index": 1
        }
      ]
    },
    "h.1.mlp.c_fc.weight": {
      "type": "Distributed",
      "shape": [
        768,
        3072
      ],
      "dtype": "F32",
      "chunks": [
        {
          "offsets": [
            0,
            0
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 0
        },
        {
          "offsets": [
            0,
            1536
          ],
          "shape": [
            768,
            1536
          ],
          "filename_index": 1
        }
      ]
    }
  },
  "filenames": [
    "rank0.safetensors",
    "rank1.safetensors"
  ],
  "n_ranks": 2
}