Hugging Face's logo

Aleph-Alpha
/
umup-research-3b-bf16

Text Generation
scaling
Model card Files Community
GregorZiegltrumAA commited on
Commit
f0a7b1c
•
1 Parent(s): c7ac6ec

Add Model

Browse files
Files changed (29) hide show
  1. config.yml +81 -0
  2. model_state_layer_0_EmbeddingInput.pt +3 -0
  3. model_state_layer_10_TransformerLayer.pt +3 -0
  4. model_state_layer_11_TransformerLayer.pt +3 -0
  5. model_state_layer_12_TransformerLayer.pt +3 -0
  6. model_state_layer_13_TransformerLayer.pt +3 -0
  7. model_state_layer_14_TransformerLayer.pt +3 -0
  8. model_state_layer_15_TransformerLayer.pt +3 -0
  9. model_state_layer_16_TransformerLayer.pt +3 -0
  10. model_state_layer_17_TransformerLayer.pt +3 -0
  11. model_state_layer_18_TransformerLayer.pt +3 -0
  12. model_state_layer_19_TransformerLayer.pt +3 -0
  13. model_state_layer_1_TransformerLayer.pt +3 -0
  14. model_state_layer_20_TransformerLayer.pt +3 -0
  15. model_state_layer_21_TransformerLayer.pt +3 -0
  16. model_state_layer_22_TransformerLayer.pt +3 -0
  17. model_state_layer_23_TransformerLayer.pt +3 -0
  18. model_state_layer_24_TransformerLayer.pt +3 -0
  19. model_state_layer_25_LayerNormWrapper.pt +3 -0
  20. model_state_layer_26_TransformerLMHead.pt +3 -0
  21. model_state_layer_2_TransformerLayer.pt +3 -0
  22. model_state_layer_3_TransformerLayer.pt +3 -0
  23. model_state_layer_4_TransformerLayer.pt +3 -0
  24. model_state_layer_5_TransformerLayer.pt +3 -0
  25. model_state_layer_6_TransformerLayer.pt +3 -0
  26. model_state_layer_7_TransformerLayer.pt +3 -0
  27. model_state_layer_8_TransformerLayer.pt +3 -0
  28. model_state_layer_9_TransformerLayer.pt +3 -0
  29. vocab.json +0 -0
config.yml ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ optimizer:
2
+ allreduce_bucket_size: 500000000
3
+ beta1: 0.9
4
+ beta2: 0.95
5
+ debug_log: false
6
+ eps: 1e-08
7
+ gradient_clipping: 0.0
8
+ zero: true
9
+ zero_save_static: false
10
+ topology:
11
+ activation_checkpointing_type: disabled
12
+ global_batch_size: 1024
13
+ gradient_accumulation_steps: 2
14
+ micro_batch_size: 2
15
+ model_parallel_size: 1
16
+ pipe_parallel_size: 1
17
+ pipe_partition_method: balanced
18
+ pipe_partition_overwrite: null
19
+ sequence_parallel: false
20
+ trainer:
21
+ seed: 42
22
+ train_iterations: 72000
23
+ training:
24
+ allow_missing_params_in_optimizer: true
25
+ training_groups:
26
+ - group_name: param_group
27
+ independent_weight_decay: true
28
+ learning_rate_scheduler:
29
+ learning_rate: 11.313708498984761
30
+ learning_rate_decay_iters: 72000
31
+ learning_rate_decay_style: cosine
32
+ learning_rate_minimum: 1.131370849898476
33
+ learning_rate_warmup_steps: 500
34
+ parameters_exclude:
35
+ - norm
36
+ weight_decay: 0.0001221
37
+ transformer_architecture:
38
+ attention_bias: false
39
+ attention_num_kv_heads: null
40
+ attention_qkv_in_one: true
41
+ dropout_after_attention: 0.0
42
+ dropout_after_mlp: 0.0
43
+ dropout_attention_probs: 0.0
44
+ dropout_embedding: 0.0
45
+ dropout_image_encoder: 0.0
46
+ hidden_size: 3072
47
+ image_encoder: false
48
+ key_query_norm: false
49
+ layernorm:
50
+ layernorm_epsilon: 1e-05
51
+ optimization_type: torch
52
+ local_attention_window_size: null
53
+ masked_softmax:
54
+ kernel: flash_attention
55
+ scale: 1.0
56
+ softmax_in_fp32: false
57
+ mlp_bias: false
58
+ mlp_factor: 2.6666666666666665
59
+ mlp_type: swiglu
60
+ norm_type: rms
61
+ num_attention_heads: 24
62
+ num_layers: 24
63
+ num_local_attention_heads: 0
64
+ precision: bfloat16
65
+ relative_position_embedding_type: rotary_complex
66
+ reset_attention_mask: false
67
+ reset_position_ids: false
68
+ rotary_embedding_base: 10000
69
+ rotary_percentage: 1.0
70
+ sequence_length: 4096
71
+ umup:
72
+ act_mult: 1.0
73
+ attn_mult: 1.0
74
+ enable: true
75
+ loss_mult: 1.0
76
+ normalize_depth_to_num_layers: true
77
+ residual_attn_ratio: 0.25
78
+ residual_mult: 1.0
79
+ vocab_file: null
80
+ vocab_size: 65536
81
+ weight_tying: false
model_state_layer_0_EmbeddingInput.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7eb59178a59c6bd60c283c7aae20063d4409099b051276b17d18bf7075fb82bd
3
+ size 402654667
model_state_layer_10_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b76b724278f7932f8932bd003d8bb0ba13237ed51bd13236f14c1b94a9b8c8af
3
+ size 226507949
model_state_layer_11_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36f0ad01ff7004e4d3902eacbe924ce4d6850debfe23057437ff4c58ff0e32c8
3
+ size 226507949
model_state_layer_12_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2e89aa8a46aa2146f5f8a566188455a40f6e3bcb589d95a8299f1c507abbcb6
3
+ size 226507949
model_state_layer_13_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27d084732411fb000f3bcab4483ffbfeed18273b8460ad82f39548eb3a3d72f5
3
+ size 226507949
model_state_layer_14_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dff1ca67f3998c40562a4631664355f5fc81df2e082b590c622e7fafeab1b8a2
3
+ size 226507949
model_state_layer_15_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8b6625503aa861150b199ec1acdba8ae7897befa5daa4e45760e8a6998225e0
3
+ size 226507949
model_state_layer_16_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0f808d36556967ed61dc5b63e8686431f1658beffa0cbafc19287ff324dad31
3
+ size 226507949
model_state_layer_17_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b77b02c6a158e5fdd4578843c957a6cdceb8c5d9a548d7f9017b7190e10618e9
3
+ size 226507949
model_state_layer_18_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b967c176c104b72bffcfe0d9a127c94db91e95bffc766862d9b00f3f593923d
3
+ size 226507949
model_state_layer_19_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2fcd6e504cacf8e575b392fc2857d97b02bd4ae2e4e13f2f70664cbd751c98d
3
+ size 226507949
model_state_layer_1_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ef204c1732ab9aeeb2d890120c36491f3c6c06386df04e6ca628fe615559044
3
+ size 226507938
model_state_layer_20_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eac4cf2bf46c1c29fab1f51fa6f52edfe4cd00f14a8cb1ac9f4aadf40bb11899
3
+ size 226507949
model_state_layer_21_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bd127288a70718e6f4ce6b1036ed453fc7f512c3727676e4fc0ee128f7c085a
3
+ size 226507949
model_state_layer_22_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:91a83e99406e8bb36d7d70ed181fe6914f7a3d8825e7903ebd02c0827c8865f8
3
+ size 226507949
model_state_layer_23_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc449f35fc16520fbbd54b1e09b13176242f91926b9b4cc75e85a56c17254b51
3
+ size 226507949
model_state_layer_24_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58cc5e410e6d374eb97ed065b12f5d62b9c295b10366a8c17476ba276283255e
3
+ size 226507949
model_state_layer_25_LayerNormWrapper.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2b3c754dd51d475a2b59fe34562670ccffc8c8f005bffc176906fca2531106b
3
+ size 7602
model_state_layer_26_TransformerLMHead.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d1f3b838870704cc6277f67ce765a2dac84f00ecbd35422fe505da0a714d472d
3
+ size 402654632
model_state_layer_2_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd7ca22d7c2f635ae4ed4a1a598110ffe345d70ea2a293b4aa78c907a62a5555
3
+ size 226507938
model_state_layer_3_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7ea06cfc268eb25a362bfd7dc272a8ead793a6df64d4610f5465c9e5bb94800
3
+ size 226507938
model_state_layer_4_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46d2f4d16f5a7ed20336cf7d000c1d5ba5b0c39ffc3b601416f9099291c4a682
3
+ size 226507938
model_state_layer_5_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0dccd52c3734dbb6bc9c608d56c1402195c6e3e935c6a628a5e89f26e54ecb95
3
+ size 226507938
model_state_layer_6_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb621e13fcba352297ea4c11f1e088d0df9c99604863596ac5cb9360b940aa20
3
+ size 226507938
model_state_layer_7_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3fadae94ceacb76e4a80d36c19a3518a694e6a9047200a45f2aba681cb62b55
3
+ size 226507938
model_state_layer_8_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef9252162795f59e5a756750e4e8624683e55d7d7a24daf4f8b7c0f25e98d1a2
3
+ size 226507938
model_state_layer_9_TransformerLayer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32590a8f7dad813a417723a056b387ef9719418a8f13997c721976b5fd461379
3
+ size 226507938
vocab.json ADDED
The diff for this file is too large to render. See raw diff