Update README.md
Browse files
README.md
CHANGED
|
@@ -38,19 +38,19 @@ Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.gi
|
|
| 38 |
lm_eval --model hf --model_args pretrained="Intel/bloom-7b1-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu --batch_size 32
|
| 39 |
```
|
| 40 |
|
| 41 |
-
| Metric |
|
| 42 |
| -------------- | ------ | ------ |
|
| 43 |
-
| Avg. | 0. | 0.4716 |
|
| 44 |
-
| mmlu | 0. | 0.2598 |
|
| 45 |
-
| lambada_openai | 0. | 0.5729 |
|
| 46 |
-
| hellaswag | 0. | 0.4619 |
|
| 47 |
-
| winogrande | 0. | 0.6369 |
|
| 48 |
-
| piqa | 0. | 0.7263 |
|
| 49 |
-
| truthfulqa_mc1 | 0. | 0.2350 |
|
| 50 |
-
| openbookqa | 0. | 0.2440 |
|
| 51 |
-
| boolq | 0. | 0.6294 |
|
| 52 |
-
| arc_easy | 0. | 0.6444 |
|
| 53 |
-
| arc_challenge | 0. | 0.3055 |
|
| 54 |
|
| 55 |
|
| 56 |
|
|
|
|
| 38 |
lm_eval --model hf --model_args pretrained="Intel/bloom-7b1-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu --batch_size 32
|
| 39 |
```
|
| 40 |
|
| 41 |
+
| Metric | FP16 | INT4 |
|
| 42 |
| -------------- | ------ | ------ |
|
| 43 |
+
| Avg. | 0.4736 | 0.4716 |
|
| 44 |
+
| mmlu | 0.2663 | 0.2598 |
|
| 45 |
+
| lambada_openai | 0.5758 | 0.5729 |
|
| 46 |
+
| hellaswag | 0.4648 | 0.4619 |
|
| 47 |
+
| winogrande | 0.6456 | 0.6369 |
|
| 48 |
+
| piqa | 0.7263 | 0.7263 |
|
| 49 |
+
| truthfulqa_mc1 | 0.2228 | 0.2350 |
|
| 50 |
+
| openbookqa | 0.2520 | 0.2440 |
|
| 51 |
+
| boolq | 0.6287 | 0.6294 |
|
| 52 |
+
| arc_easy | 0.6503 | 0.6444 |
|
| 53 |
+
| arc_challenge | 0.3038 | 0.3055 |
|
| 54 |
|
| 55 |
|
| 56 |
|