Hzfinfdu commited on
Commit
0318149
1 Parent(s): 03c4b47

Upload folder using huggingface_hub

Browse files
Llama3_1Base-L3A-32x/checkpoints/final.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81e142cc2c96956aee630d8f8cd6d75629ec6a71fb9bdffe0ef2c2652fd1effe
3
+ size 2148278848
Llama3_1Base-L3A-32x/hyperparams.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "hook_point_in": "blocks.3.hook_attn_out",
3
+ "hook_point_out": "blocks.3.hook_attn_out",
4
+ "use_decoder_bias": true,
5
+ "apply_decoder_bias_to_pre_encoder": false,
6
+ "expansion_factor": 32,
7
+ "d_model": 4096,
8
+ "d_sae": 131072,
9
+ "norm_activation": "token-wise",
10
+ "dataset_average_activation_norm": null,
11
+ "decoder_exactly_fixed_norm": false,
12
+ "sparsity_include_decoder_norm": true,
13
+ "use_glu_encoder": false,
14
+ "init_decoder_norm": null,
15
+ "init_encoder_norm": null,
16
+ "init_encoder_with_decoder_transpose": true,
17
+ "l1_coefficient": 1.6e-05,
18
+ "l1_coefficient_warmup_steps": 29296,
19
+ "lp": 1,
20
+ "use_ghost_grads": false,
21
+ "tp_size": 1,
22
+ "ddp_size": 1
23
+ }
Llama3_1Base-L3A-32x/lm_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "meta-llama/Meta-Llama-3.1-8B",
3
+ "use_flash_attn": false,
4
+ "cache_dir": null,
5
+ "d_model": 4096,
6
+ "local_files_only": false
7
+ }