| models: | |
| - model: grimjim/mistralai-Mistral-Nemo-Instruct-2407 | |
| parameters: | |
| density: 0.2 | |
| weight: 0.23 | |
| - model: nbeerbower/mistral-nemo-bophades-12B | |
| parameters: | |
| density: 0.2 | |
| weight: 0.43 | |
| - model: nbeerbower/mistral-nemo-gutenberg-12B-v4 | |
| parameters: | |
| density: 0.2 | |
| weight: 0.43 | |
| - model: TheDrummer/UnslopNemo-12B-v4.1 | |
| parameters: | |
| density: 0.5 | |
| weight: 0.63 | |
| merge_method: task_arithmetic | |
| base_model: NewEden/nemo-erebus | |
| parameters: | |
| normalize: false | |
| int8_mask: true | |
| dtype: bfloat16 | |
| layer_parameters: | |
| - range: [0, 10] | |
| density_multiplier: 1.2 | |
| - range: [10, 20] | |
| density_multiplier: 1.0 | |
| - range: [20, 30] | |
| density_multiplier: 0.8 | |
| - range: [30, 40] | |
| density_multiplier: 0.6 | |
| regularization: | |
| - method: gradient_penalty | |
| scale: 0.05 | |
| - method: weight_clipping | |
| clip_range: [-0.2, 0.2] | |
| - method: random_noise | |
| scale: 0.01 | |
| - method: attention_dropout | |
| scale: 0.1 | |
| postprocessing: | |
| - operation: entropy_regularization | |
| scale: 0.05 | |
| - operation: non_linear_scaling | |
| parameters: | |
| function: tanh | |
| - operation: sharpening | |
| intensity: 0.5 | |
| - operation: gaussian_smoothing | |
| sigma: 1.5 | |
| - operation: normalize | |
| - operation: dynamic_scaling | |
| scale_range: [0.8, 1.2] | |
| - operation: smoothing | |
| parameters: | |
| adaptive: true | |
| range: [0.85, 1.15] | |
| kernel_size: 5 |