slices: | |
- sources: | |
- model: RozGrov/NemoDori-v0.2-12B-MN-BT | |
layer_range: [0, 8] | |
- sources: | |
- model: RozGrov/NemoDori-v0.2-12B-MN-BT | |
layer_range: [8, 24] | |
parameters: | |
scale: | |
- filter: q_proj | |
value: 0.919 | |
- filter: k_proj | |
value: 0.919 | |
- value: 1.0 | |
- sources: | |
- model: RozGrov/NemoDori-v0.2-12B-MN-BT | |
layer_range: [16, 32] | |
parameters: | |
scale: | |
- filter: q_proj | |
value: 0.919 | |
- filter: k_proj | |
value: 0.919 | |
- value: 1.0 | |
- sources: | |
- model: RozGrov/NemoDori-v0.2-12B-MN-BT | |
layer_range: [32, 40] | |
merge_method: passthrough | |
dtype: bfloat16 | |