diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,3975 @@ +{ + "metadata": { + "ParamSize": 313, + "ParamBytes": 3927297024.0, + "BitsPerParam": 4.12551973205239 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 272498688, + "records": [ + { + "name": "lm_head.q_weight", + "shape": [ + 448, + 152064 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 272498688, + "byteOffset": 0 + } + ], + "md5sum": "49207a101a8e2d29991f4e1f4d56f935" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 272498688, + "records": [ + { + "name": "model.embed_tokens.q_weight", + "shape": [ + 152064, + 448 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 272498688, + "byteOffset": 0 + } + ], + "md5sum": "f43d5d6a1758f059aa7778fd75e978ce" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.0.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "67c2900d1617dd7c9423c9a8d251df3a" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.0.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "64cc908710ec4b8179c9bd01445aa53e" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 28752896, + "records": [ + { + "name": "lm_head.q_scale", + "shape": [ + 28, + 152064 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8515584, + "byteOffset": 0 + }, + { + "name": "model.embed_tokens.q_scale", + "shape": [ + 152064, + 28 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8515584, + "byteOffset": 8515584 + }, + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 17031168 + }, + { + "name": "model.layers.0.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 17038336 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 18099200 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 20220928 + }, + { + "name": "model.layers.0.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 20228096 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 20237312 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 28494848 + } + ], + "md5sum": "35024592cf733a970f63a1e15c94bf7e" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.1.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "5cd4b126cda75ef2612c25eae5372172" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.1.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "7ba9f4c6945ebbe75bebe943909b661e" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.2.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "eed05677a06da072927bdc8ddc9511f6" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.2.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "26275a15d22caa1887fed68b563e3be1" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.0.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.1.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.1.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.2.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.2.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "151746a466d5aa03334d17a763614b3e" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.3.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "25d9daac858a8a0c2a18a7502334163c" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "fb9d6ef0355842dc0906861c92ef3af0" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.4.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "19b22a9e0ab8fada5950630d3884f941" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.2.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.3.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.3.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.3.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "8efc515ca2d6b39e90b9c1c5ca0ef5ea" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.4.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "a6f0c6c81af736a50279ffde3b152d7e" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.5.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "1ce8ec906dedf46644c8bcb7cf69ebba" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.5.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "e9b7921a86e5deb2c9a951aece96d920" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.4.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.4.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.5.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.5.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "aa16437217482e8c87f277d840679150" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.6.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "bc429bb62cdefa26a7d361e80b5302bf" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.6.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "4193cca14d5836d2a7fe08e565d74518" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.7.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "eed4820ad5adbeeea823e4c7c8031967" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "45f3d461974681a7f10d2fae4ad3a134" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.5.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.6.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.6.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.7.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.7.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.7.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "c8c9277b2ca471738532525df6135754" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.10.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "1cbccf5157baa1f8255735a2e1c99627" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.10.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "9d4b36ad0870356084351c065f03c3d5" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 33492992, + "records": [ + { + "name": "model.layers.7.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.8.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 15138816 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 15148032 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 23405568 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 23663616 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 30086144 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 30286848 + }, + { + "name": "model.layers.10.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 30294016 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 31354880 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33476608 + }, + { + "name": "model.layers.10.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 33483776 + } + ], + "md5sum": "858867fb318c199635be239694947974" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.11.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "a857119650edc45fa562710bca0077a0" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.11.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "be5be18232c0d51dd3b57bf374557d1d" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.12.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "a6671b8898aac70bc6c9a487d9951af0" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.10.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.11.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.11.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "16bf1cf85c24e9e5b4c87c03ad4dd21c" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.12.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "1de25be544a6630d46f21504ad7f54b5" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.13.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "f2539fcb6955fcc55d3e8b12d9b5190d" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.13.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "47eb1cdbe8a0c4a7b8e67884ac132c20" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.12.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.12.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.13.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.13.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "95413f66f61cec16989eb1461450b567" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.14.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "9b3708f775fdf0e3aa7f679249a1a7df" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.14.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "598ecfd9c684e6c78cb081dafef3cec5" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.15.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "42c184574b3a2b831a4b840743be9960" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "179309bf6fd03757eceb94294c7a64e3" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.13.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.14.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.14.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.15.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.15.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.15.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "dd91d59bc7d6c4200f59313067594845" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.16.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "67f95aec882cc6dadb6ff4c754f5bb51" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "9bd57c9b48905f3b34a5b58a8ead095d" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.17.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "b5f72ff25f41e0833b262222487f91a7" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.15.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.16.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.16.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.16.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "da300f854d030eb474572ee7a8df24bd" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.17.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "1632ed000479e129ae0da2f9806441ce" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.18.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "5700fafb921730e7926c117d0a3e8efb" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 28984320, + "records": [ + { + "name": "model.layers.17.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.17.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 18337792 + }, + { + "name": "model.layers.18.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 20459520 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 20468736 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 28726272 + } + ], + "md5sum": "42ca8008317720415f0c06d7f697f6d7" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.8.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "96418d146d6d0667fb90bed0a4fcdb4f" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "ca7a6536d2cc888c0f363416af94b92a" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.9.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "cee645bba5a96d60d27b733ef54dba64" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.9.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "2fbed61bc91fce70fbf3f77a4945e970" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.18.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "7ee1180c2d2a460ae612b4689b665226" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.19.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "77960f102aa3c125ae252a3c41dac71d" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "f4ba6ba55a435069ef2465ea6370e4e2" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 32446464, + "records": [ + { + "name": "model.layers.18.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.8.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.8.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9820160 + }, + { + "name": "model.layers.9.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 9827328 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 10888192 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 13009920 + }, + { + "name": "model.layers.9.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 13017088 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 13026304 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 21283840 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 21541888 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 27964416 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28165120 + }, + { + "name": "model.layers.18.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 28172288 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 29233152 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 29240320 + }, + { + "name": "model.layers.19.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 29247488 + }, + { + "name": "model.layers.19.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 30308352 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 32430080 + }, + { + "name": "model.layers.19.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 32437248 + } + ], + "md5sum": "75c1d5294db3ede5a7f2d85f90d22518" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.20.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "c3266c1ff9d1744018180d940221aa72" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.20.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "28e04110f03ec831b10fac02dd486baa" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.21.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "21a504851569dcb097865a259d236b7f" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.19.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.20.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.20.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "29375a5c083001f839a0a672c3a83f4c" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.21.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "8d61da9a13fe8631f2fc3df8b648e7d5" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.22.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "f0d19d1acb75565b8674b21c4bacf349" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.22.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "d786df214e0a65ace3a7c41ba664fbf2" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.21.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.21.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.22.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.22.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "784b7c1a4f685770ac02d568114d50f5" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.23.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "1ccbdb85e3737e4a9bfd50dd5a5f19fb" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "2c491693b4684736df728180f0325115" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.24.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "dca808a00de9d74f8382f39e0903a210" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.24.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "0fab9b7487a204c456158d6d30e96e65" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.22.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.23.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.23.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.23.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.24.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.24.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.24.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "8a620daec574fcdcf54b92cd577a95a4" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.25.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "6e9a410af9b51f479ba45556680d8f3e" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.25.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "ef9777eaeb5107b07d15639c555dfbb3" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "8372994fb0e7e09138a0720c6fae502a" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.24.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.24.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.25.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.25.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.25.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "11187cf232673b2df3123970aee1c426" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.26.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "84041ebb180d3ccbb0a52354c79e2606" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.27.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "cb9944443e845001290eb7a9bf9c983b" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "50d803cab5b6fd57dc847bdbd9b7a31e" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.26.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.26.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.27.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.27.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.27.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "bbb087f8ef51dd1dc0b6e7704142f6d9" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 6630400, + "records": [ + { + "name": "model.layers.27.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.norm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + } + ], + "md5sum": "219c5e6d29d8f3dfb1d47d9828ef17ac" + } + ] +} \ No newline at end of file