# Parameters Scaling Data-Constrained Language Models & Chinchilla paper with
# corrected n_heads for 3530M and 4084M and extensions from 
# "Scaling Data-Constrained Language Models"
# values are (d_model ffw_size kv_size n_heads n_layers)

PARAM_7M=(128 512 32 4 3)
PARAM_14M=(224 896 32 7 4)
PARAM_20M=(288 1152 32 7 5)
PARAM_35M=(448 1792 32 7 6)
PARAM_44M=(512 2048 64 8 8)
PARAM_57M=(576 2304 64 9 9)
PARAM_74M=(640 2560 64 10 10)
PARAM_90M=(640 2560 64 10 13)
PARAM_106M=(640 2560 64 10 16)
PARAM_117M=(768 3072 64 12 12)
PARAM_140M=(768 3072 64 12 15)
PARAM_163M=(768 3072 64 12 18)
PARAM_175M=(896 3584 64 14 14)
PARAM_196M=(896 3584 64 14 16)
PARAM_217M=(896 3584 64 14 18)
PARAM_251M=(1024 4096 64 16 16)
PARAM_278M=(1024 4096 64 16 18)
PARAM_306M=(1024 4096 64 16 20)
PARAM_425M=(1280 5120 128 10 18)
PARAM_489M=(1280 5120 128 10 21)
PARAM_509M=(1408 5632 128 11 18)
PARAM_552M=(1280 5120 128 10 24)
PARAM_587M=(1408 5632 128 11 21)
PARAM_632M=(1536 6144 128 12 19)
PARAM_664M=(1408 5632 128 11 24)
PARAM_724M=(1536 6144 128 12 22)
PARAM_816M=(1536 6144 128 12 25)
PARAM_893M=(1792 7168 128 14 20)
PARAM_1018M=(1792 7168 128 14 23)
PARAM_1143M=(1792 7168 128 14 26)
PARAM_1266M=(2048 8192 128 16 22)
PARAM_1424M=(2176 8704 128 17 22)
PARAM_1429M=(2048 8192 128 16 25)
PARAM_1593M=(2048 8192 128 16 28)
PARAM_1609M=(2176 8704 128 17 25)
PARAM_1731M=(2304 9216 128 18 24)
PARAM_1794M=(2176 8704 128 17 28)
PARAM_2007M=(2304 9216 128 18 28)
PARAM_2283M=(2304 9216 128 18 32)
PARAM_2298M=(2560 10240 128 20 26)
PARAM_2639M=(2560 10240 128 20 30)
PARAM_2980M=(2560 10240 128 20 34)
PARAM_3530M=(2688 10752 128 21 36)
PARAM_3802M=(2816 11264 128 22 36)
PARAM_4084M=(2944 11776 128 23 36)
PARAM_4516M=(3072 12288 128 24 36)
PARAM_6796M=(3584 14336 128 28 40)
PARAM_9293M=(4096 16384 128 32 42)
PARAM_11452M=(4352 17408 128 32 47)
PARAM_12295M=(4608 18432 128 36 44)
PARAM_12569M=(4608 18432 128 32 47)
PARAM_13735M=(4864 19456 128 32 47)
PARAM_14940M=(4992 19968 128 32 49)
PARAM_16183M=(5120 20480 128 40 47)
