Upload topology.json

topology.json

@@ -1,54 +1,32 @@
 {
   "tensors": {
-    "h.2.ln_2.weight": {
+    "h.0.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
+        3072,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            1536,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
-          ],
-          "shape": [
-            384
-          ],
-          "filename_index": 1
-        }
-      ]
-    },
-    "h.5.mlp.c_proj.bias": {
-      "type": "Distributed",
-      "shape": [
-        768
-      ],
-      "dtype": "F32",
-      "chunks": [
-        {
-          "offsets": [
+            1536,
             0
           ],
           "shape": [
-            384
-          ],
-          "filename_index": 0
-        },
-        {
-          "offsets": [
-            384
-          ],
-          "shape": [
-            384
+            1536,
+            768
           ],
           "filename_index": 1
         }
@@ -86,52 +64,37 @@
         }
       ]
     },
-    "h.5.attn.bias": {
+    "h.2.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        3072
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
-            0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
+            1536
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.9.mlp.c_fc.bias": {
+    "h.8.ln_1.weight": {
       "type": "Distributed",
       "shape": [
-        3072
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -140,26 +103,26 @@
             0
           ],
           "shape": [
-            1536
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            384
           ],
           "shape": [
-            1536
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.mlp.c_fc.weight": {
+    "h.2.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
         768,
-        3072
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -170,24 +133,24 @@
           ],
           "shape": [
             768,
-            1536
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
             0,
-            1536
+            1152
           ],
           "shape": [
             768,
-            1536
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.ln_2.weight": {
+    "h.8.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -214,7 +177,7 @@
         }
       ]
     },
-    "h.4.ln_1.weight": {
+    "ln_f.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -241,7 +204,7 @@
         }
       ]
     },
-    "h.8.attn.c_proj.bias": {
+    "h.2.ln_2.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -268,7 +231,7 @@
         }
       ]
     },
-    "h.8.ln_1.weight": {
+    "h.6.ln_1.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -295,11 +258,11 @@
         }
       ]
     },
-    "h.10.attn.c_attn.weight": {
+    "h.2.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        768,
-        2304
+        3072,
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -309,52 +272,57 @@
             0
           ],
           "shape": [
-            768,
-            1152
+            1536,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1152
+            1536,
+            0
           ],
           "shape": [
-            768,
-            1152
+            1536,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "ln_f.bias": {
+    "h.10.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
-        768
+        768,
+        3072
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            768,
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            1536
           ],
           "shape": [
-            384
+            768,
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.3.ln_1.weight": {
+    "h.8.ln_1.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -381,74 +349,64 @@
         }
       ]
     },
-    "h.0.mlp.c_fc.weight": {
+    "h.9.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        768,
-        3072
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1536
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1536
+            1152
           ],
           "shape": [
-            768,
-            1536
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.10.attn.c_proj.weight": {
+    "h.5.ln_2.bias": {
       "type": "Distributed",
       "shape": [
-        768,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
-            0
+            384
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.1.attn.c_proj.weight": {
+    "h.6.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        768,
+        3072,
         768
       ],
       "dtype": "F32",
@@ -459,25 +417,25 @@
             0
           ],
           "shape": [
-            384,
+            1536,
             768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
+            1536,
             0
           ],
           "shape": [
-            384,
+            1536,
             768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.3.ln_2.bias": {
+    "h.7.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -504,10 +462,10 @@
         }
       ]
     },
-    "h.2.attn.c_proj.bias": {
+    "h.1.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -516,25 +474,25 @@
             0
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1536
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.ln_2.weight": {
+    "h.9.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -543,26 +501,26 @@
             0
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1536
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.3.attn.c_attn.weight": {
+    "h.5.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
         768,
-        2304
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -573,88 +531,78 @@
           ],
           "shape": [
             768,
-            1152
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
             0,
-            1152
+            1536
           ],
           "shape": [
             768,
-            1152
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.8.mlp.c_proj.weight": {
+    "h.7.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        3072,
-        768
+        3072
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            1536,
-            768
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            1536
           ],
           "shape": [
-            1536,
-            768
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.attn.c_attn.weight": {
+    "h.4.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
-        768,
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1152
+            384
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.attn.c_proj.weight": {
+    "h.1.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
         768,
@@ -686,49 +634,39 @@
         }
       ]
     },
-    "h.10.attn.bias": {
+    "h.0.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        768,
+        3072
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
             0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            768,
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
             0,
-            0,
-            0,
-            512
+            1536
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            768,
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.1.mlp.c_fc.weight": {
+    "h.8.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
         768,
@@ -760,10 +698,10 @@
         }
       ]
     },
-    "h.3.mlp.c_fc.bias": {
+    "h.5.ln_2.weight": {
       "type": "Distributed",
       "shape": [
-        3072
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -772,26 +710,26 @@
             0
           ],
           "shape": [
-            1536
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            384
           ],
           "shape": [
-            1536
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.8.attn.c_attn.weight": {
+    "h.11.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
         768,
-        2304
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -802,56 +740,51 @@
           ],
           "shape": [
             768,
-            1152
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
             0,
-            1152
+            1536
           ],
           "shape": [
             768,
-            1152
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.10.mlp.c_fc.weight": {
+    "h.9.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
-        768,
-        3072
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1536
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1536
+            384
           ],
           "shape": [
-            768,
-            1536
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.1.mlp.c_fc.bias": {
+    "h.6.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
         3072
@@ -878,10 +811,10 @@
         }
       ]
     },
-    "h.3.attn.c_proj.weight": {
+    "h.5.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        768,
+        3072,
         768
       ],
       "dtype": "F32",
@@ -892,25 +825,25 @@
             0
           ],
           "shape": [
-            384,
+            1536,
             768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
+            1536,
             0
           ],
           "shape": [
-            384,
+            1536,
             768
           ],
           "filename_index": 1
         }
       ]
     },
-    "ln_f.weight": {
+    "h.7.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -937,61 +870,71 @@
         }
       ]
     },
-    "h.7.attn.c_attn.bias": {
+    "h.6.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        2304
+        768,
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            1152
+            384,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1152
+            384,
+            0
           ],
           "shape": [
-            1152
+            384,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.attn.c_attn.bias": {
+    "wpe.weight": {
       "type": "Distributed",
       "shape": [
-        2304
+        1024,
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            1152
+            1024,
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1152
+            0,
+            384
           ],
           "shape": [
-            1152
+            1024,
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.ln_1.weight": {
+    "h.1.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -1018,10 +961,10 @@
         }
       ]
     },
-    "h.5.attn.c_proj.weight": {
+    "h.3.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        768,
+        3072,
         768
       ],
       "dtype": "F32",
@@ -1032,29 +975,29 @@
             0
           ],
           "shape": [
-            384,
+            1536,
             768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
+            1536,
             0
           ],
           "shape": [
-            384,
+            1536,
             768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.9.mlp.c_proj.weight": {
+    "h.2.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
-        3072,
-        768
+        768,
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -1064,52 +1007,67 @@
             0
           ],
           "shape": [
-            1536,
-            768
+            768,
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            0,
+            1536
           ],
           "shape": [
-            1536,
-            768
+            768,
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.4.mlp.c_fc.bias": {
+    "h.1.attn.bias": {
       "type": "Distributed",
       "shape": [
-        3072
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            1536
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            1536
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.3.ln_1.bias": {
+    "h.4.ln_1.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -1136,7 +1094,7 @@
         }
       ]
     },
-    "h.7.ln_1.weight": {
+    "h.0.ln_1.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -1163,91 +1121,66 @@
         }
       ]
     },
-    "h.8.attn.bias": {
+    "h.2.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
-            0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
+            384
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.attn.bias": {
+    "h.7.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        768,
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
             0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
+            384,
+            0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.5.ln_1.bias": {
+    "h.11.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -1274,10 +1207,10 @@
         }
       ]
     },
-    "h.7.mlp.c_proj.weight": {
+    "h.9.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        3072,
+        768,
         768
       ],
       "dtype": "F32",
@@ -1288,25 +1221,25 @@
             0
           ],
           "shape": [
-            1536,
+            384,
             768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
+            384,
             0
           ],
           "shape": [
-            1536,
+            384,
             768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.10.mlp.c_proj.bias": {
+    "h.11.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -1333,11 +1266,11 @@
         }
       ]
     },
-    "h.3.mlp.c_proj.weight": {
+    "h.1.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
-        3072,
-        768
+        768,
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -1347,25 +1280,25 @@
             0
           ],
           "shape": [
-            1536,
-            768
+            768,
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            0,
+            1536
           ],
           "shape": [
-            1536,
-            768
+            768,
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.attn.c_attn.weight": {
+    "h.11.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
         768,
@@ -1397,101 +1330,96 @@
         }
       ]
     },
-    "h.8.ln_2.weight": {
+    "h.3.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
-        768
+        768,
+        3072
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            768,
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            1536
           ],
           "shape": [
-            384
+            768,
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.7.attn.c_attn.weight": {
+    "h.9.ln_1.weight": {
       "type": "Distributed",
       "shape": [
-        768,
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1152
+            384
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.8.attn.c_proj.weight": {
+    "h.5.ln_1.weight": {
       "type": "Distributed",
       "shape": [
-        768,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
-            0
+            384
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.10.attn.c_proj.bias": {
+    "h.4.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -1500,25 +1428,25 @@
             0
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1152
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.7.ln_2.weight": {
+    "h.10.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -1527,49 +1455,54 @@
             0
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1152
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.8.mlp.c_fc.bias": {
+    "wte.weight": {
       "type": "Distributed",
       "shape": [
-        3072
+        50257,
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            1536
+            50257,
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            0,
+            384
           ],
           "shape": [
-            1536
+            50257,
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.2.mlp.c_fc.weight": {
+    "h.4.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
         768,
@@ -1601,7 +1534,7 @@
         }
       ]
     },
-    "h.4.ln_2.weight": {
+    "h.0.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -1628,10 +1561,10 @@
         }
       ]
     },
-    "h.9.attn.c_attn.bias": {
+    "h.10.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        2304
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -1640,48 +1573,70 @@
             0
           ],
           "shape": [
-            1152
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1152
+            1536
           ],
           "shape": [
-            1152
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.4.attn.c_proj.weight": {
+    "ln_f.weight": {
       "type": "Distributed",
       "shape": [
-        768,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
+            384
+          ],
+          "shape": [
+            384
+          ],
+          "filename_index": 1
+        }
+      ]
+    },
+    "h.2.ln_2.bias": {
+      "type": "Distributed",
+      "shape": [
+        768
+      ],
+      "dtype": "F32",
+      "chunks": [
+        {
+          "offsets": [
             0
           ],
           "shape": [
-            384,
-            768
+            384
+          ],
+          "filename_index": 0
+        },
+        {
+          "offsets": [
+            384
+          ],
+          "shape": [
+            384
           ],
           "filename_index": 1
         }
@@ -1714,7 +1669,7 @@
         }
       ]
     },
-    "h.10.mlp.c_proj.weight": {
+    "h.11.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
         3072,
@@ -1746,39 +1701,61 @@
         }
       ]
     },
-    "wte.weight": {
+    "h.7.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        50257,
-        768
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            50257,
-            384
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            384
+            1152
           ],
           "shape": [
-            50257,
-            384
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.10.ln_2.bias": {
+    "h.4.mlp.c_fc.bias": {
+      "type": "Distributed",
+      "shape": [
+        3072
+      ],
+      "dtype": "F32",
+      "chunks": [
+        {
+          "offsets": [
+            0
+          ],
+          "shape": [
+            1536
+          ],
+          "filename_index": 0
+        },
+        {
+          "offsets": [
+            1536
+          ],
+          "shape": [
+            1536
+          ],
+          "filename_index": 1
+        }
+      ]
+    },
+    "h.6.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -1805,11 +1782,11 @@
         }
       ]
     },
-    "h.9.attn.c_attn.weight": {
+    "h.10.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
         768,
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -1819,25 +1796,25 @@
             0
           ],
           "shape": [
-            768,
-            1152
+            384,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1152
+            384,
+            0
           ],
           "shape": [
-            768,
-            1152
+            384,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.2.mlp.c_proj.bias": {
+    "h.10.ln_1.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -1864,34 +1841,39 @@
         }
       ]
     },
-    "h.1.attn.c_proj.bias": {
+    "h.5.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
+        768,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            384,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            384,
+            0
           ],
           "shape": [
-            384
+            384,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.9.ln_2.weight": {
+    "h.8.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -1918,113 +1900,88 @@
         }
       ]
     },
-    "h.2.mlp.c_proj.weight": {
+    "h.10.ln_1.bias": {
       "type": "Distributed",
       "shape": [
-        3072,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            384
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.attn.bias": {
+    "h.4.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
-            0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
+            384
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.5.attn.c_attn.weight": {
+    "h.5.ln_1.bias": {
       "type": "Distributed",
       "shape": [
-        768,
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1152
+            384
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.3.attn.c_proj.bias": {
+    "h.9.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2051,113 +2008,88 @@
         }
       ]
     },
-    "h.7.attn.bias": {
+    "h.1.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
-            0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
+            1152
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.3.mlp.c_fc.weight": {
+    "h.0.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        768,
-        3072
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1536
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1536
+            1152
           ],
           "shape": [
-            768,
-            1536
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.8.mlp.c_fc.weight": {
+    "h.9.ln_2.weight": {
       "type": "Distributed",
       "shape": [
-        768,
-        3072
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1536
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1536
+            384
           ],
           "shape": [
-            768,
-            1536
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.attn.c_proj.bias": {
+    "h.6.ln_2.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -2184,7 +2116,7 @@
         }
       ]
     },
-    "h.8.mlp.c_proj.bias": {
+    "h.10.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2211,7 +2143,7 @@
         }
       ]
     },
-    "h.0.ln_1.weight": {
+    "h.10.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2238,76 +2170,66 @@
         }
       ]
     },
-    "h.3.attn.bias": {
+    "h.2.ln_1.bias": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
-            0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
+            384
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.ln_2.bias": {
+    "h.7.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
+        3072,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            1536,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1536,
+            0
           ],
           "shape": [
-            384
+            1536,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.3.ln_2.weight": {
+    "h.3.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2334,7 +2256,7 @@
         }
       ]
     },
-    "h.4.ln_1.bias": {
+    "h.1.ln_1.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2361,7 +2283,7 @@
         }
       ]
     },
-    "h.8.ln_2.bias": {
+    "h.0.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2388,7 +2310,7 @@
         }
       ]
     },
-    "h.1.ln_2.bias": {
+    "h.4.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2415,34 +2337,81 @@
         }
       ]
     },
-    "h.1.mlp.c_proj.bias": {
+    "h.6.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
-        768
+        768,
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            768,
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            1152
           ],
           "shape": [
-            384
+            768,
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.mlp.c_proj.weight": {
+    "h.6.attn.bias": {
+      "type": "Distributed",
+      "shape": [
+        1,
+        1,
+        1024,
+        1024
+      ],
+      "dtype": "F32",
+      "chunks": [
+        {
+          "offsets": [
+            0,
+            0,
+            0,
+            0
+          ],
+          "shape": [
+            1,
+            1,
+            1024,
+            512
+          ],
+          "filename_index": 0
+        },
+        {
+          "offsets": [
+            0,
+            0,
+            0,
+            512
+          ],
+          "shape": [
+            1,
+            1,
+            1024,
+            512
+          ],
+          "filename_index": 1
+        }
+      ]
+    },
+    "h.9.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
         3072,
@@ -2474,11 +2443,11 @@
         }
       ]
     },
-    "h.7.mlp.c_fc.weight": {
+    "h.7.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
         768,
-        3072
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -2489,56 +2458,51 @@
           ],
           "shape": [
             768,
-            1536
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
             0,
-            1536
+            1152
           ],
           "shape": [
             768,
-            1536
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.attn.c_proj.weight": {
+    "h.3.ln_2.weight": {
       "type": "Distributed",
       "shape": [
-        768,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
-            0
+            384
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.ln_1.weight": {
+    "h.0.ln_2.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -2565,10 +2529,10 @@
         }
       ]
     },
-    "h.5.attn.c_attn.bias": {
+    "h.3.ln_1.weight": {
       "type": "Distributed",
       "shape": [
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -2577,25 +2541,25 @@
             0
           ],
           "shape": [
-            1152
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1152
+            384
           ],
           "shape": [
-            1152
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.8.ln_1.bias": {
+    "h.11.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -2604,81 +2568,91 @@
             0
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1152
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.4.mlp.c_proj.weight": {
+    "h.1.ln_1.weight": {
       "type": "Distributed",
       "shape": [
-        3072,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            384
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.10.ln_1.bias": {
+    "h.8.attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.1.ln_2.weight": {
+    "h.2.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2705,7 +2679,7 @@
         }
       ]
     },
-    "h.10.ln_1.weight": {
+    "h.7.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -2732,147 +2706,182 @@
         }
       ]
     },
-    "h.11.mlp.c_fc.bias": {
+    "h.7.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
+        768,
         3072
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
+            768,
             1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
+            0,
             1536
           ],
           "shape": [
+            768,
             1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.10.mlp.c_fc.bias": {
+    "h.10.attn.bias": {
       "type": "Distributed",
       "shape": [
-        3072
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            1536
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            1536
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.5.mlp.c_proj.weight": {
+    "h.7.ln_1.bias": {
       "type": "Distributed",
       "shape": [
-        3072,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            384
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.10.attn.c_attn.bias": {
+    "h.0.attn.bias": {
       "type": "Distributed",
       "shape": [
-        2304
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            1152
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1152
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            1152
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.ln_2.bias": {
+    "h.6.mlp.c_fc.weight": {
       "type": "Distributed",
       "shape": [
-        768
+        768,
+        3072
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            768,
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            1536
           ],
           "shape": [
-            384
+            768,
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.mlp.c_proj.bias": {
+    "h.7.ln_1.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -2899,10 +2908,10 @@
         }
       ]
     },
-    "h.5.ln_2.bias": {
+    "h.3.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -2911,25 +2920,25 @@
             0
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1536
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.4.attn.c_attn.bias": {
+    "h.5.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -2938,76 +2947,96 @@
             0
           ],
           "shape": [
-            1152
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1152
+            384
           ],
           "shape": [
-            1152
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.4.ln_2.bias": {
+    "h.11.attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.2.attn.c_attn.bias": {
+    "h.8.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
+        768,
         2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
+            768,
             1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
+            0,
             1152
           ],
           "shape": [
+            768,
             1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.2.ln_1.weight": {
+    "h.9.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -3034,34 +3063,49 @@
         }
       ]
     },
-    "h.7.ln_1.bias": {
+    "h.7.attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.9.attn.c_proj.weight": {
+    "h.3.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
         768,
@@ -3093,7 +3137,7 @@
         }
       ]
     },
-    "h.0.mlp.c_proj.bias": {
+    "h.11.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -3120,7 +3164,7 @@
         }
       ]
     },
-    "h.7.attn.c_proj.bias": {
+    "h.3.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -3147,7 +3191,7 @@
         }
       ]
     },
-    "h.9.ln_1.bias": {
+    "h.7.ln_2.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -3174,74 +3218,64 @@
         }
       ]
     },
-    "h.1.mlp.c_proj.weight": {
+    "h.9.ln_1.bias": {
       "type": "Distributed",
       "shape": [
-        3072,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            384
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.mlp.c_proj.weight": {
+    "h.11.ln_1.weight": {
       "type": "Distributed",
       "shape": [
-        3072,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            384
           ],
           "shape": [
-            1536,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.ln_1.bias": {
+    "h.5.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -3250,22 +3284,22 @@
             0
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1536
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.7.mlp.c_proj.bias": {
+    "h.1.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -3292,10 +3326,10 @@
         }
       ]
     },
-    "h.5.ln_1.weight": {
+    "h.8.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -3304,58 +3338,53 @@
             0
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1152
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "wpe.weight": {
+    "h.8.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
-        1024,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            1024,
             384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
             384
           ],
           "shape": [
-            1024,
             384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.5.mlp.c_fc.weight": {
+    "h.5.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
         768,
-        3072
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -3366,24 +3395,24 @@
           ],
           "shape": [
             768,
-            1536
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
             0,
-            1536
+            1152
           ],
           "shape": [
             768,
-            1536
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.2.attn.bias": {
+    "h.4.attn.bias": {
       "type": "Distributed",
       "shape": [
         1,
@@ -3425,7 +3454,7 @@
         }
       ]
     },
-    "h.9.ln_2.bias": {
+    "h.5.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -3452,10 +3481,10 @@
         }
       ]
     },
-    "h.1.attn.c_attn.bias": {
+    "h.0.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        2304
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -3464,154 +3493,169 @@
             0
           ],
           "shape": [
-            1152
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1152
+            1536
           ],
           "shape": [
-            1152
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.9.attn.bias": {
+    "h.1.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        3072,
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
             0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            1536,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
+            1536,
+            0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            1536,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.attn.c_attn.weight": {
+    "h.6.ln_1.weight": {
       "type": "Distributed",
       "shape": [
-        768,
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1152
+            384
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.ln_2.weight": {
+    "h.5.attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.5.ln_2.weight": {
+    "h.2.attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            384
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.mlp.c_fc.weight": {
+    "h.8.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        768,
-        3072
+        3072,
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -3621,25 +3665,25 @@
             0
           ],
           "shape": [
-            768,
-            1536
+            1536,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1536
+            1536,
+            0
           ],
           "shape": [
-            768,
-            1536
+            1536,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.3.mlp.c_proj.bias": {
+    "h.0.ln_1.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -3666,10 +3710,10 @@
         }
       ]
     },
-    "h.10.ln_2.weight": {
+    "h.5.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -3678,150 +3722,140 @@
             0
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1152
           ],
           "shape": [
-            384
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.1.attn.bias": {
+    "h.11.ln_1.bias": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
-            0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
+            384
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.1.ln_1.bias": {
+    "h.11.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
+        768,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            384,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            384,
+            0
           ],
           "shape": [
-            384
+            384,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.4.attn.c_proj.bias": {
+    "h.10.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
-        768
+        768,
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            768,
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            1152
           ],
           "shape": [
-            384
+            768,
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.2.attn.c_attn.weight": {
+    "h.1.mlp.c_proj.bias": {
       "type": "Distributed",
       "shape": [
-        768,
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1152
+            384
           ],
           "shape": [
-            768,
-            1152
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.ln_2.bias": {
+    "h.3.ln_1.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -3848,7 +3882,7 @@
         }
       ]
     },
-    "h.9.attn.c_proj.bias": {
+    "h.6.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -3875,7 +3909,7 @@
         }
       ]
     },
-    "h.5.attn.c_proj.bias": {
+    "h.2.ln_1.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -3902,7 +3936,7 @@
         }
       ]
     },
-    "h.9.ln_1.weight": {
+    "h.4.ln_1.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -3929,7 +3963,7 @@
         }
       ]
     },
-    "h.4.mlp.c_proj.bias": {
+    "h.10.ln_2.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -3988,93 +4022,108 @@
         }
       ]
     },
-    "h.7.attn.c_proj.weight": {
+    "h.11.ln_2.weight": {
       "type": "Distributed",
       "shape": [
-        768,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
-            0
+            384
           ],
           "shape": [
-            384,
-            768
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.2.mlp.c_fc.bias": {
+    "h.8.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        3072
+        768,
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            1536
+            384,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            384,
+            0
           ],
           "shape": [
-            1536
+            384,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.7.mlp.c_fc.bias": {
+    "h.9.attn.bias": {
       "type": "Distributed",
       "shape": [
-        3072
+        1,
+        1,
+        1024,
+        1024
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
+            0,
+            0,
             0
           ],
           "shape": [
-            1536
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            0,
+            0,
+            0,
+            512
           ],
           "shape": [
-            1536
+            1,
+            1,
+            1024,
+            512
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.2.ln_2.bias": {
+    "h.1.ln_2.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -4101,10 +4150,10 @@
         }
       ]
     },
-    "h.6.attn.c_attn.bias": {
+    "h.4.ln_2.weight": {
       "type": "Distributed",
       "shape": [
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -4113,49 +4162,54 @@
             0
           ],
           "shape": [
-            1152
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1152
+            384
           ],
           "shape": [
-            1152
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.mlp.c_fc.bias": {
+    "h.3.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
-        3072
+        768,
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            1536
+            768,
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            0,
+            1152
           ],
           "shape": [
-            1536
+            768,
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.mlp.c_fc.bias": {
+    "h.8.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
         3072
@@ -4182,43 +4236,38 @@
         }
       ]
     },
-    "h.4.mlp.c_fc.weight": {
+    "h.10.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
-        768,
-        3072
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            768,
-            1536
+            384
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1536
+            384
           ],
           "shape": [
-            768,
-            1536
+            384
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.attn.c_proj.weight": {
+    "h.9.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
         768,
-        768
+        2304
       ],
       "dtype": "F32",
       "chunks": [
@@ -4228,25 +4277,25 @@
             0
           ],
           "shape": [
-            384,
-            768
+            768,
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384,
-            0
+            0,
+            1152
           ],
           "shape": [
-            384,
-            768
+            768,
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.ln_1.bias": {
+    "h.3.ln_2.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -4273,39 +4322,34 @@
         }
       ]
     },
-    "h.0.mlp.c_proj.weight": {
+    "h.6.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
-        3072,
-        768
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
             0
           ],
           "shape": [
-            1536,
-            768
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536,
-            0
+            1152
           ],
           "shape": [
-            1536,
-            768
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.8.attn.c_attn.bias": {
+    "h.2.attn.c_attn.bias": {
       "type": "Distributed",
       "shape": [
         2304
@@ -4332,88 +4376,103 @@
         }
       ]
     },
-    "h.7.ln_2.bias": {
+    "h.10.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
+        3072,
         768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            1536,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1536,
+            0
           ],
           "shape": [
-            384
+            1536,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.0.attn.c_attn.bias": {
+    "h.1.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
+        768,
         2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
+            768,
             1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
+            0,
             1152
           ],
           "shape": [
+            768,
             1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.9.mlp.c_proj.bias": {
+    "h.4.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
-        768
+        768,
+        2304
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            384
+            768,
+            1152
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            0,
+            1152
           ],
           "shape": [
-            384
+            768,
+            1152
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.ln_1.bias": {
+    "h.6.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -4440,79 +4499,42 @@
         }
       ]
     },
-    "h.4.attn.bias": {
+    "h.0.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        1,
-        1,
-        1024,
-        1024
+        768,
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
-            0,
-            0,
             0,
             0
           ],
           "shape": [
-            1,
-            1,
-            1024,
-            512
+            384,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            0,
-            0,
-            512
-          ],
-          "shape": [
-            1,
-            1,
-            1024,
-            512
-          ],
-          "filename_index": 1
-        }
-      ]
-    },
-    "h.1.ln_1.weight": {
-      "type": "Distributed",
-      "shape": [
-        768
-      ],
-      "dtype": "F32",
-      "chunks": [
-        {
-          "offsets": [
+            384,
             0
           ],
           "shape": [
-            384
-          ],
-          "filename_index": 0
-        },
-        {
-          "offsets": [
-            384
-          ],
-          "shape": [
-            384
+            384,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.6.attn.c_proj.bias": {
+    "h.11.mlp.c_fc.bias": {
       "type": "Distributed",
       "shape": [
-        768
+        3072
       ],
       "dtype": "F32",
       "chunks": [
@@ -4521,22 +4543,22 @@
             0
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            384
+            1536
           ],
           "shape": [
-            384
+            1536
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.1.attn.c_attn.weight": {
+    "h.0.attn.c_attn.weight": {
       "type": "Distributed",
       "shape": [
         768,
@@ -4568,7 +4590,7 @@
         }
       ]
     },
-    "h.6.attn.bias": {
+    "h.3.attn.bias": {
       "type": "Distributed",
       "shape": [
         1,
@@ -4610,38 +4632,11 @@
         }
       ]
     },
-    "h.6.mlp.c_proj.bias": {
-      "type": "Distributed",
-      "shape": [
-        768
-      ],
-      "dtype": "F32",
-      "chunks": [
-        {
-          "offsets": [
-            0
-          ],
-          "shape": [
-            384
-          ],
-          "filename_index": 0
-        },
-        {
-          "offsets": [
-            384
-          ],
-          "shape": [
-            384
-          ],
-          "filename_index": 1
-        }
-      ]
-    },
-    "h.4.attn.c_attn.weight": {
+    "h.4.attn.c_proj.weight": {
       "type": "Distributed",
       "shape": [
         768,
-        2304
+        768
       ],
       "dtype": "F32",
       "chunks": [
@@ -4651,25 +4646,25 @@
             0
           ],
           "shape": [
-            768,
-            1152
+            384,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            0,
-            1152
+            384,
+            0
           ],
           "shape": [
-            768,
-            1152
+            384,
+            768
           ],
           "filename_index": 1
         }
       ]
     },
-    "h.11.attn.c_proj.bias": {
+    "h.8.ln_2.weight": {
       "type": "Distributed",
       "shape": [
         768
@@ -4696,7 +4691,7 @@
         }
       ]
     },
-    "h.2.ln_1.bias": {
+    "h.0.attn.c_proj.bias": {
       "type": "Distributed",
       "shape": [
         768
@@ -4723,28 +4718,33 @@
         }
       ]
     },
-    "h.5.mlp.c_fc.bias": {
+    "h.4.mlp.c_proj.weight": {
       "type": "Distributed",
       "shape": [
-        3072
+        3072,
+        768
       ],
       "dtype": "F32",
       "chunks": [
         {
           "offsets": [
+            0,
             0
           ],
           "shape": [
-            1536
+            1536,
+            768
           ],
           "filename_index": 0
         },
         {
           "offsets": [
-            1536
+            1536,
+            0
           ],
           "shape": [
-            1536
+            1536,
+            768
           ],
           "filename_index": 1
         }