|
--- |
|
tags: |
|
- mteb |
|
- sentence-transformers |
|
- transformers |
|
- Qwen2 |
|
- sentence-similarity |
|
license: apache-2.0 |
|
model-index: |
|
- name: gte-qwen2-7B-instruct |
|
results: |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_counterfactual |
|
name: MTEB AmazonCounterfactualClassification (en) |
|
config: en |
|
split: test |
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
metrics: |
|
- type: accuracy |
|
value: 88.01492537313432 |
|
- type: ap |
|
value: 59.096217055359276 |
|
- type: f1 |
|
value: 83.2699173062069 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_polarity |
|
name: MTEB AmazonPolarityClassification |
|
config: default |
|
split: test |
|
revision: e2d317d38cd51312af73b3d32a06d1a08b442046 |
|
metrics: |
|
- type: accuracy |
|
value: 97.29805 |
|
- type: ap |
|
value: 95.97973142381882 |
|
- type: f1 |
|
value: 97.29773206176378 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (en) |
|
config: en |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 62.798 |
|
- type: f1 |
|
value: 61.33195375425034 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/arguana |
|
name: MTEB ArguAna |
|
config: default |
|
split: test |
|
revision: c22ab2a51041ffd869aaddef7af8d8215647e41a |
|
metrics: |
|
- type: map_at_1 |
|
value: 36.629 |
|
- type: map_at_10 |
|
value: 54.982 |
|
- type: map_at_100 |
|
value: 55.355 |
|
- type: map_at_1000 |
|
value: 55.355 |
|
- type: map_at_3 |
|
value: 50.036 |
|
- type: map_at_5 |
|
value: 53.25 |
|
- type: mrr_at_1 |
|
value: 37.624 |
|
- type: mrr_at_10 |
|
value: 55.376000000000005 |
|
- type: mrr_at_100 |
|
value: 55.749 |
|
- type: mrr_at_1000 |
|
value: 55.749 |
|
- type: mrr_at_3 |
|
value: 50.461999999999996 |
|
- type: mrr_at_5 |
|
value: 53.644999999999996 |
|
- type: ndcg_at_1 |
|
value: 36.629 |
|
- type: ndcg_at_10 |
|
value: 64.35499999999999 |
|
- type: ndcg_at_100 |
|
value: 65.778 |
|
- type: ndcg_at_1000 |
|
value: 65.778 |
|
- type: ndcg_at_3 |
|
value: 54.478 |
|
- type: ndcg_at_5 |
|
value: 60.260000000000005 |
|
- type: precision_at_1 |
|
value: 36.629 |
|
- type: precision_at_10 |
|
value: 9.381 |
|
- type: precision_at_100 |
|
value: 0.996 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_3 |
|
value: 22.451 |
|
- type: precision_at_5 |
|
value: 16.273 |
|
- type: recall_at_1 |
|
value: 36.629 |
|
- type: recall_at_10 |
|
value: 93.812 |
|
- type: recall_at_100 |
|
value: 99.644 |
|
- type: recall_at_1000 |
|
value: 99.644 |
|
- type: recall_at_3 |
|
value: 67.354 |
|
- type: recall_at_5 |
|
value: 81.366 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-p2p |
|
name: MTEB ArxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d |
|
metrics: |
|
- type: v_measure |
|
value: 56.30960182540703 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-s2s |
|
name: MTEB ArxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 |
|
metrics: |
|
- type: v_measure |
|
value: 51.858431775176975 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/askubuntudupquestions-reranking |
|
name: MTEB AskUbuntuDupQuestions |
|
config: default |
|
split: test |
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
|
metrics: |
|
- type: map |
|
value: 67.5678414928039 |
|
- type: mrr |
|
value: 79.56305236776153 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/biosses-sts |
|
name: MTEB BIOSSES |
|
config: default |
|
split: test |
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 82.32511136457549 |
|
- type: cos_sim_spearman |
|
value: 79.34518142776068 |
|
- type: euclidean_pearson |
|
value: 81.09762569927126 |
|
- type: euclidean_spearman |
|
value: 79.33554265391781 |
|
- type: manhattan_pearson |
|
value: 81.33942162521643 |
|
- type: manhattan_spearman |
|
value: 79.91206181439438 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/banking77 |
|
name: MTEB Banking77Classification |
|
config: default |
|
split: test |
|
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
|
metrics: |
|
- type: accuracy |
|
value: 85.99675324675324 |
|
- type: f1 |
|
value: 85.5564660877528 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-p2p |
|
name: MTEB BiorxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 |
|
metrics: |
|
- type: v_measure |
|
value: 50.413005916654384 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-s2s |
|
name: MTEB BiorxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 |
|
metrics: |
|
- type: v_measure |
|
value: 46.58170679922341 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackAndroidRetrieval |
|
config: default |
|
split: test |
|
revision: f46a197baaae43b4f621051089b82a364682dfeb |
|
metrics: |
|
- type: map_at_1 |
|
value: 34.588 |
|
- type: map_at_10 |
|
value: 47.851 |
|
- type: map_at_100 |
|
value: 49.484 |
|
- type: map_at_1000 |
|
value: 49.6 |
|
- type: map_at_3 |
|
value: 43.34 |
|
- type: map_at_5 |
|
value: 45.734 |
|
- type: mrr_at_1 |
|
value: 42.203 |
|
- type: mrr_at_10 |
|
value: 53.315999999999995 |
|
- type: mrr_at_100 |
|
value: 53.977 |
|
- type: mrr_at_1000 |
|
value: 54.001 |
|
- type: mrr_at_3 |
|
value: 50.381 |
|
- type: mrr_at_5 |
|
value: 52.198 |
|
- type: ndcg_at_1 |
|
value: 42.203 |
|
- type: ndcg_at_10 |
|
value: 55.143 |
|
- type: ndcg_at_100 |
|
value: 60.278 |
|
- type: ndcg_at_1000 |
|
value: 61.497 |
|
- type: ndcg_at_3 |
|
value: 48.9 |
|
- type: ndcg_at_5 |
|
value: 51.712 |
|
- type: precision_at_1 |
|
value: 42.203 |
|
- type: precision_at_10 |
|
value: 11.016 |
|
- type: precision_at_100 |
|
value: 1.718 |
|
- type: precision_at_1000 |
|
value: 0.219 |
|
- type: precision_at_3 |
|
value: 24.224999999999998 |
|
- type: precision_at_5 |
|
value: 17.711 |
|
- type: recall_at_1 |
|
value: 34.588 |
|
- type: recall_at_10 |
|
value: 69.91000000000001 |
|
- type: recall_at_100 |
|
value: 91.01700000000001 |
|
- type: recall_at_1000 |
|
value: 98.02199999999999 |
|
- type: recall_at_3 |
|
value: 51.9 |
|
- type: recall_at_5 |
|
value: 59.604 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackEnglishRetrieval |
|
config: default |
|
split: test |
|
revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 |
|
metrics: |
|
- type: map_at_1 |
|
value: 35.649 |
|
- type: map_at_10 |
|
value: 47.713 |
|
- type: map_at_100 |
|
value: 49.043 |
|
- type: map_at_1000 |
|
value: 49.178 |
|
- type: map_at_3 |
|
value: 44.355 |
|
- type: map_at_5 |
|
value: 46.152 |
|
- type: mrr_at_1 |
|
value: 44.268 |
|
- type: mrr_at_10 |
|
value: 53.403999999999996 |
|
- type: mrr_at_100 |
|
value: 54.035999999999994 |
|
- type: mrr_at_1000 |
|
value: 54.078 |
|
- type: mrr_at_3 |
|
value: 51.507000000000005 |
|
- type: mrr_at_5 |
|
value: 52.583999999999996 |
|
- type: ndcg_at_1 |
|
value: 44.268 |
|
- type: ndcg_at_10 |
|
value: 53.679 |
|
- type: ndcg_at_100 |
|
value: 57.794000000000004 |
|
- type: ndcg_at_1000 |
|
value: 59.74 |
|
- type: ndcg_at_3 |
|
value: 49.348 |
|
- type: ndcg_at_5 |
|
value: 51.266999999999996 |
|
- type: precision_at_1 |
|
value: 44.268 |
|
- type: precision_at_10 |
|
value: 10.120999999999999 |
|
- type: precision_at_100 |
|
value: 1.566 |
|
- type: precision_at_1000 |
|
value: 0.20600000000000002 |
|
- type: precision_at_3 |
|
value: 23.864 |
|
- type: precision_at_5 |
|
value: 16.650000000000002 |
|
- type: recall_at_1 |
|
value: 35.649 |
|
- type: recall_at_10 |
|
value: 64.152 |
|
- type: recall_at_100 |
|
value: 81.096 |
|
- type: recall_at_1000 |
|
value: 92.957 |
|
- type: recall_at_3 |
|
value: 51.498 |
|
- type: recall_at_5 |
|
value: 56.977 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGamingRetrieval |
|
config: default |
|
split: test |
|
revision: 4885aa143210c98657558c04aaf3dc47cfb54340 |
|
metrics: |
|
- type: map_at_1 |
|
value: 38.372 |
|
- type: map_at_10 |
|
value: 52.693 |
|
- type: map_at_100 |
|
value: 53.796 |
|
- type: map_at_1000 |
|
value: 53.836 |
|
- type: map_at_3 |
|
value: 48.818 |
|
- type: map_at_5 |
|
value: 51.052 |
|
- type: mrr_at_1 |
|
value: 44.013000000000005 |
|
- type: mrr_at_10 |
|
value: 55.769999999999996 |
|
- type: mrr_at_100 |
|
value: 56.415000000000006 |
|
- type: mrr_at_1000 |
|
value: 56.435 |
|
- type: mrr_at_3 |
|
value: 52.884 |
|
- type: mrr_at_5 |
|
value: 54.552 |
|
- type: ndcg_at_1 |
|
value: 44.013000000000005 |
|
- type: ndcg_at_10 |
|
value: 59.45 |
|
- type: ndcg_at_100 |
|
value: 63.422 |
|
- type: ndcg_at_1000 |
|
value: 64.214 |
|
- type: ndcg_at_3 |
|
value: 52.829 |
|
- type: ndcg_at_5 |
|
value: 56.079 |
|
- type: precision_at_1 |
|
value: 44.013000000000005 |
|
- type: precision_at_10 |
|
value: 9.912 |
|
- type: precision_at_100 |
|
value: 1.286 |
|
- type: precision_at_1000 |
|
value: 0.13899999999999998 |
|
- type: precision_at_3 |
|
value: 23.992 |
|
- type: precision_at_5 |
|
value: 16.803 |
|
- type: recall_at_1 |
|
value: 38.372 |
|
- type: recall_at_10 |
|
value: 76.279 |
|
- type: recall_at_100 |
|
value: 92.842 |
|
- type: recall_at_1000 |
|
value: 98.41 |
|
- type: recall_at_3 |
|
value: 58.738 |
|
- type: recall_at_5 |
|
value: 66.51899999999999 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGisRetrieval |
|
config: default |
|
split: test |
|
revision: 5003b3064772da1887988e05400cf3806fe491f2 |
|
metrics: |
|
- type: map_at_1 |
|
value: 26.784999999999997 |
|
- type: map_at_10 |
|
value: 37.152 |
|
- type: map_at_100 |
|
value: 38.371 |
|
- type: map_at_1000 |
|
value: 38.437 |
|
- type: map_at_3 |
|
value: 34.211999999999996 |
|
- type: map_at_5 |
|
value: 35.791000000000004 |
|
- type: mrr_at_1 |
|
value: 29.153000000000002 |
|
- type: mrr_at_10 |
|
value: 39.312999999999995 |
|
- type: mrr_at_100 |
|
value: 40.32 |
|
- type: mrr_at_1000 |
|
value: 40.367999999999995 |
|
- type: mrr_at_3 |
|
value: 36.760999999999996 |
|
- type: mrr_at_5 |
|
value: 38.083 |
|
- type: ndcg_at_1 |
|
value: 29.153000000000002 |
|
- type: ndcg_at_10 |
|
value: 42.785000000000004 |
|
- type: ndcg_at_100 |
|
value: 48.613 |
|
- type: ndcg_at_1000 |
|
value: 50.166 |
|
- type: ndcg_at_3 |
|
value: 37.255 |
|
- type: ndcg_at_5 |
|
value: 39.763999999999996 |
|
- type: precision_at_1 |
|
value: 29.153000000000002 |
|
- type: precision_at_10 |
|
value: 6.734 |
|
- type: precision_at_100 |
|
value: 1.0250000000000001 |
|
- type: precision_at_1000 |
|
value: 0.11900000000000001 |
|
- type: precision_at_3 |
|
value: 16.234 |
|
- type: precision_at_5 |
|
value: 11.232000000000001 |
|
- type: recall_at_1 |
|
value: 26.784999999999997 |
|
- type: recall_at_10 |
|
value: 57.915000000000006 |
|
- type: recall_at_100 |
|
value: 84.473 |
|
- type: recall_at_1000 |
|
value: 96.011 |
|
- type: recall_at_3 |
|
value: 43.105 |
|
- type: recall_at_5 |
|
value: 49.15 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackMathematicaRetrieval |
|
config: default |
|
split: test |
|
revision: 90fceea13679c63fe563ded68f3b6f06e50061de |
|
metrics: |
|
- type: map_at_1 |
|
value: 20.24 |
|
- type: map_at_10 |
|
value: 31.493 |
|
- type: map_at_100 |
|
value: 32.771 |
|
- type: map_at_1000 |
|
value: 32.883 |
|
- type: map_at_3 |
|
value: 27.062 |
|
- type: map_at_5 |
|
value: 29.421999999999997 |
|
- type: mrr_at_1 |
|
value: 25.622 |
|
- type: mrr_at_10 |
|
value: 35.729 |
|
- type: mrr_at_100 |
|
value: 36.613 |
|
- type: mrr_at_1000 |
|
value: 36.665 |
|
- type: mrr_at_3 |
|
value: 32.048 |
|
- type: mrr_at_5 |
|
value: 34.169 |
|
- type: ndcg_at_1 |
|
value: 25.622 |
|
- type: ndcg_at_10 |
|
value: 38.463 |
|
- type: ndcg_at_100 |
|
value: 43.909 |
|
- type: ndcg_at_1000 |
|
value: 46.21 |
|
- type: ndcg_at_3 |
|
value: 30.563000000000002 |
|
- type: ndcg_at_5 |
|
value: 34.178999999999995 |
|
- type: precision_at_1 |
|
value: 25.622 |
|
- type: precision_at_10 |
|
value: 7.7490000000000006 |
|
- type: precision_at_100 |
|
value: 1.1780000000000002 |
|
- type: precision_at_1000 |
|
value: 0.149 |
|
- type: precision_at_3 |
|
value: 15.049999999999999 |
|
- type: precision_at_5 |
|
value: 11.616999999999999 |
|
- type: recall_at_1 |
|
value: 20.24 |
|
- type: recall_at_10 |
|
value: 55.657000000000004 |
|
- type: recall_at_100 |
|
value: 78.803 |
|
- type: recall_at_1000 |
|
value: 94.801 |
|
- type: recall_at_3 |
|
value: 34.171 |
|
- type: recall_at_5 |
|
value: 43.16 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackPhysicsRetrieval |
|
config: default |
|
split: test |
|
revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 |
|
metrics: |
|
- type: map_at_1 |
|
value: 32.501000000000005 |
|
- type: map_at_10 |
|
value: 46.286 |
|
- type: map_at_100 |
|
value: 47.732 |
|
- type: map_at_1000 |
|
value: 47.814 |
|
- type: map_at_3 |
|
value: 41.957 |
|
- type: map_at_5 |
|
value: 44.506 |
|
- type: mrr_at_1 |
|
value: 39.75 |
|
- type: mrr_at_10 |
|
value: 51.285000000000004 |
|
- type: mrr_at_100 |
|
value: 52.051 |
|
- type: mrr_at_1000 |
|
value: 52.075 |
|
- type: mrr_at_3 |
|
value: 48.315999999999995 |
|
- type: mrr_at_5 |
|
value: 50.125 |
|
- type: ndcg_at_1 |
|
value: 39.75 |
|
- type: ndcg_at_10 |
|
value: 53.361999999999995 |
|
- type: ndcg_at_100 |
|
value: 58.703 |
|
- type: ndcg_at_1000 |
|
value: 59.962 |
|
- type: ndcg_at_3 |
|
value: 46.786 |
|
- type: ndcg_at_5 |
|
value: 50.169 |
|
- type: precision_at_1 |
|
value: 39.75 |
|
- type: precision_at_10 |
|
value: 10.154 |
|
- type: precision_at_100 |
|
value: 1.485 |
|
- type: precision_at_1000 |
|
value: 0.17600000000000002 |
|
- type: precision_at_3 |
|
value: 23.003 |
|
- type: precision_at_5 |
|
value: 16.766000000000002 |
|
- type: recall_at_1 |
|
value: 32.501000000000005 |
|
- type: recall_at_10 |
|
value: 68.901 |
|
- type: recall_at_100 |
|
value: 90.527 |
|
- type: recall_at_1000 |
|
value: 98.307 |
|
- type: recall_at_3 |
|
value: 51.056000000000004 |
|
- type: recall_at_5 |
|
value: 59.471 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackProgrammersRetrieval |
|
config: default |
|
split: test |
|
revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.962999999999997 |
|
- type: map_at_10 |
|
value: 41.434 |
|
- type: map_at_100 |
|
value: 42.961 |
|
- type: map_at_1000 |
|
value: 43.051 |
|
- type: map_at_3 |
|
value: 37.579 |
|
- type: map_at_5 |
|
value: 39.579 |
|
- type: mrr_at_1 |
|
value: 34.932 |
|
- type: mrr_at_10 |
|
value: 46.455999999999996 |
|
- type: mrr_at_100 |
|
value: 47.362 |
|
- type: mrr_at_1000 |
|
value: 47.398 |
|
- type: mrr_at_3 |
|
value: 43.855 |
|
- type: mrr_at_5 |
|
value: 45.322 |
|
- type: ndcg_at_1 |
|
value: 34.932 |
|
- type: ndcg_at_10 |
|
value: 48.323 |
|
- type: ndcg_at_100 |
|
value: 54.173 |
|
- type: ndcg_at_1000 |
|
value: 55.69 |
|
- type: ndcg_at_3 |
|
value: 42.498000000000005 |
|
- type: ndcg_at_5 |
|
value: 44.973 |
|
- type: precision_at_1 |
|
value: 34.932 |
|
- type: precision_at_10 |
|
value: 9.224 |
|
- type: precision_at_100 |
|
value: 1.429 |
|
- type: precision_at_1000 |
|
value: 0.172 |
|
- type: precision_at_3 |
|
value: 21.005 |
|
- type: precision_at_5 |
|
value: 15.0 |
|
- type: recall_at_1 |
|
value: 27.962999999999997 |
|
- type: recall_at_10 |
|
value: 63.563 |
|
- type: recall_at_100 |
|
value: 87.679 |
|
- type: recall_at_1000 |
|
value: 97.381 |
|
- type: recall_at_3 |
|
value: 47.205999999999996 |
|
- type: recall_at_5 |
|
value: 53.784 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackRetrieval |
|
config: default |
|
split: test |
|
revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.9895 |
|
- type: map_at_10 |
|
value: 39.67808333333333 |
|
- type: map_at_100 |
|
value: 41.05 |
|
- type: map_at_1000 |
|
value: 41.15800000000001 |
|
- type: map_at_3 |
|
value: 36.079499999999996 |
|
- type: map_at_5 |
|
value: 38.056749999999994 |
|
- type: mrr_at_1 |
|
value: 33.405583333333325 |
|
- type: mrr_at_10 |
|
value: 43.6965 |
|
- type: mrr_at_100 |
|
value: 44.568000000000005 |
|
- type: mrr_at_1000 |
|
value: 44.61208333333334 |
|
- type: mrr_at_3 |
|
value: 40.96574999999999 |
|
- type: mrr_at_5 |
|
value: 42.529833333333336 |
|
- type: ndcg_at_1 |
|
value: 33.405583333333325 |
|
- type: ndcg_at_10 |
|
value: 46.016 |
|
- type: ndcg_at_100 |
|
value: 51.39475 |
|
- type: ndcg_at_1000 |
|
value: 53.17333333333334 |
|
- type: ndcg_at_3 |
|
value: 40.166666666666664 |
|
- type: ndcg_at_5 |
|
value: 42.899750000000004 |
|
- type: precision_at_1 |
|
value: 33.405583333333325 |
|
- type: precision_at_10 |
|
value: 8.408999999999999 |
|
- type: precision_at_100 |
|
value: 1.3129166666666665 |
|
- type: precision_at_1000 |
|
value: 0.16583333333333336 |
|
- type: precision_at_3 |
|
value: 19.05825 |
|
- type: precision_at_5 |
|
value: 13.6845 |
|
- type: recall_at_1 |
|
value: 27.9895 |
|
- type: recall_at_10 |
|
value: 60.572416666666676 |
|
- type: recall_at_100 |
|
value: 83.63975 |
|
- type: recall_at_1000 |
|
value: 95.58775 |
|
- type: recall_at_3 |
|
value: 44.402750000000005 |
|
- type: recall_at_5 |
|
value: 51.40116666666666 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackStatsRetrieval |
|
config: default |
|
split: test |
|
revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a |
|
metrics: |
|
- type: map_at_1 |
|
value: 24.451 |
|
- type: map_at_10 |
|
value: 34.526 |
|
- type: map_at_100 |
|
value: 35.732 |
|
- type: map_at_1000 |
|
value: 35.824 |
|
- type: map_at_3 |
|
value: 31.503999999999998 |
|
- type: map_at_5 |
|
value: 33.241 |
|
- type: mrr_at_1 |
|
value: 28.221 |
|
- type: mrr_at_10 |
|
value: 37.34 |
|
- type: mrr_at_100 |
|
value: 38.389 |
|
- type: mrr_at_1000 |
|
value: 38.443 |
|
- type: mrr_at_3 |
|
value: 34.714 |
|
- type: mrr_at_5 |
|
value: 36.217 |
|
- type: ndcg_at_1 |
|
value: 28.221 |
|
- type: ndcg_at_10 |
|
value: 40.105000000000004 |
|
- type: ndcg_at_100 |
|
value: 45.619 |
|
- type: ndcg_at_1000 |
|
value: 47.597 |
|
- type: ndcg_at_3 |
|
value: 34.711 |
|
- type: ndcg_at_5 |
|
value: 37.38 |
|
- type: precision_at_1 |
|
value: 28.221 |
|
- type: precision_at_10 |
|
value: 6.7330000000000005 |
|
- type: precision_at_100 |
|
value: 1.0170000000000001 |
|
- type: precision_at_1000 |
|
value: 0.126 |
|
- type: precision_at_3 |
|
value: 15.798000000000002 |
|
- type: precision_at_5 |
|
value: 11.227 |
|
- type: recall_at_1 |
|
value: 24.451 |
|
- type: recall_at_10 |
|
value: 54.332 |
|
- type: recall_at_100 |
|
value: 78.842 |
|
- type: recall_at_1000 |
|
value: 92.868 |
|
- type: recall_at_3 |
|
value: 39.495999999999995 |
|
- type: recall_at_5 |
|
value: 46.198 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackTexRetrieval |
|
config: default |
|
split: test |
|
revision: 46989137a86843e03a6195de44b09deda022eec7 |
|
metrics: |
|
- type: map_at_1 |
|
value: 18.989 |
|
- type: map_at_10 |
|
value: 28.189999999999998 |
|
- type: map_at_100 |
|
value: 29.575000000000003 |
|
- type: map_at_1000 |
|
value: 29.705 |
|
- type: map_at_3 |
|
value: 25.406000000000002 |
|
- type: map_at_5 |
|
value: 26.851000000000003 |
|
- type: mrr_at_1 |
|
value: 23.400000000000002 |
|
- type: mrr_at_10 |
|
value: 32.231 |
|
- type: mrr_at_100 |
|
value: 33.239000000000004 |
|
- type: mrr_at_1000 |
|
value: 33.309 |
|
- type: mrr_at_3 |
|
value: 29.869 |
|
- type: mrr_at_5 |
|
value: 31.102999999999998 |
|
- type: ndcg_at_1 |
|
value: 23.400000000000002 |
|
- type: ndcg_at_10 |
|
value: 33.634 |
|
- type: ndcg_at_100 |
|
value: 39.772999999999996 |
|
- type: ndcg_at_1000 |
|
value: 42.385 |
|
- type: ndcg_at_3 |
|
value: 28.938999999999997 |
|
- type: ndcg_at_5 |
|
value: 30.913 |
|
- type: precision_at_1 |
|
value: 23.400000000000002 |
|
- type: precision_at_10 |
|
value: 6.366 |
|
- type: precision_at_100 |
|
value: 1.1159999999999999 |
|
- type: precision_at_1000 |
|
value: 0.153 |
|
- type: precision_at_3 |
|
value: 14.212 |
|
- type: precision_at_5 |
|
value: 10.151 |
|
- type: recall_at_1 |
|
value: 18.989 |
|
- type: recall_at_10 |
|
value: 45.837 |
|
- type: recall_at_100 |
|
value: 73.04899999999999 |
|
- type: recall_at_1000 |
|
value: 91.245 |
|
- type: recall_at_3 |
|
value: 32.309 |
|
- type: recall_at_5 |
|
value: 37.665 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackUnixRetrieval |
|
config: default |
|
split: test |
|
revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 |
|
metrics: |
|
- type: map_at_1 |
|
value: 30.595 |
|
- type: map_at_10 |
|
value: 42.286 |
|
- type: map_at_100 |
|
value: 43.586999999999996 |
|
- type: map_at_1000 |
|
value: 43.669000000000004 |
|
- type: map_at_3 |
|
value: 38.888 |
|
- type: map_at_5 |
|
value: 40.669 |
|
- type: mrr_at_1 |
|
value: 36.287000000000006 |
|
- type: mrr_at_10 |
|
value: 46.405 |
|
- type: mrr_at_100 |
|
value: 47.282999999999994 |
|
- type: mrr_at_1000 |
|
value: 47.327000000000005 |
|
- type: mrr_at_3 |
|
value: 43.874 |
|
- type: mrr_at_5 |
|
value: 45.414 |
|
- type: ndcg_at_1 |
|
value: 36.287000000000006 |
|
- type: ndcg_at_10 |
|
value: 48.407 |
|
- type: ndcg_at_100 |
|
value: 53.824000000000005 |
|
- type: ndcg_at_1000 |
|
value: 55.483000000000004 |
|
- type: ndcg_at_3 |
|
value: 42.9 |
|
- type: ndcg_at_5 |
|
value: 45.391999999999996 |
|
- type: precision_at_1 |
|
value: 36.287000000000006 |
|
- type: precision_at_10 |
|
value: 8.414000000000001 |
|
- type: precision_at_100 |
|
value: 1.232 |
|
- type: precision_at_1000 |
|
value: 0.147 |
|
- type: precision_at_3 |
|
value: 20.118 |
|
- type: precision_at_5 |
|
value: 13.993 |
|
- type: recall_at_1 |
|
value: 30.595 |
|
- type: recall_at_10 |
|
value: 62.656 |
|
- type: recall_at_100 |
|
value: 85.74199999999999 |
|
- type: recall_at_1000 |
|
value: 96.854 |
|
- type: recall_at_3 |
|
value: 47.413 |
|
- type: recall_at_5 |
|
value: 54.04 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWebmastersRetrieval |
|
config: default |
|
split: test |
|
revision: 160c094312a0e1facb97e55eeddb698c0abe3571 |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.236 |
|
- type: map_at_10 |
|
value: 39.751 |
|
- type: map_at_100 |
|
value: 41.435 |
|
- type: map_at_1000 |
|
value: 41.677 |
|
- type: map_at_3 |
|
value: 35.957 |
|
- type: map_at_5 |
|
value: 38.112 |
|
- type: mrr_at_1 |
|
value: 33.794000000000004 |
|
- type: mrr_at_10 |
|
value: 44.449 |
|
- type: mrr_at_100 |
|
value: 45.268 |
|
- type: mrr_at_1000 |
|
value: 45.311 |
|
- type: mrr_at_3 |
|
value: 41.502 |
|
- type: mrr_at_5 |
|
value: 43.142 |
|
- type: ndcg_at_1 |
|
value: 33.794000000000004 |
|
- type: ndcg_at_10 |
|
value: 46.787 |
|
- type: ndcg_at_100 |
|
value: 52.290000000000006 |
|
- type: ndcg_at_1000 |
|
value: 54.336 |
|
- type: ndcg_at_3 |
|
value: 40.78 |
|
- type: ndcg_at_5 |
|
value: 43.669999999999995 |
|
- type: precision_at_1 |
|
value: 33.794000000000004 |
|
- type: precision_at_10 |
|
value: 9.051 |
|
- type: precision_at_100 |
|
value: 1.7919999999999998 |
|
- type: precision_at_1000 |
|
value: 0.259 |
|
- type: precision_at_3 |
|
value: 19.368 |
|
- type: precision_at_5 |
|
value: 14.229 |
|
- type: recall_at_1 |
|
value: 28.236 |
|
- type: recall_at_10 |
|
value: 61.358000000000004 |
|
- type: recall_at_100 |
|
value: 85.028 |
|
- type: recall_at_1000 |
|
value: 97.813 |
|
- type: recall_at_3 |
|
value: 44.207 |
|
- type: recall_at_5 |
|
value: 51.885000000000005 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWordpressRetrieval |
|
config: default |
|
split: test |
|
revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 |
|
metrics: |
|
- type: map_at_1 |
|
value: 17.505000000000003 |
|
- type: map_at_10 |
|
value: 26.762000000000004 |
|
- type: map_at_100 |
|
value: 28.113 |
|
- type: map_at_1000 |
|
value: 28.222 |
|
- type: map_at_3 |
|
value: 23.876 |
|
- type: map_at_5 |
|
value: 25.572 |
|
- type: mrr_at_1 |
|
value: 19.224 |
|
- type: mrr_at_10 |
|
value: 28.660000000000004 |
|
- type: mrr_at_100 |
|
value: 29.863 |
|
- type: mrr_at_1000 |
|
value: 29.935000000000002 |
|
- type: mrr_at_3 |
|
value: 25.878 |
|
- type: mrr_at_5 |
|
value: 27.449 |
|
- type: ndcg_at_1 |
|
value: 19.224 |
|
- type: ndcg_at_10 |
|
value: 32.054 |
|
- type: ndcg_at_100 |
|
value: 38.339 |
|
- type: ndcg_at_1000 |
|
value: 40.8 |
|
- type: ndcg_at_3 |
|
value: 26.491 |
|
- type: ndcg_at_5 |
|
value: 29.298999999999996 |
|
- type: precision_at_1 |
|
value: 19.224 |
|
- type: precision_at_10 |
|
value: 5.434 |
|
- type: precision_at_100 |
|
value: 0.911 |
|
- type: precision_at_1000 |
|
value: 0.125 |
|
- type: precision_at_3 |
|
value: 11.83 |
|
- type: precision_at_5 |
|
value: 8.834999999999999 |
|
- type: recall_at_1 |
|
value: 17.505000000000003 |
|
- type: recall_at_10 |
|
value: 46.309 |
|
- type: recall_at_100 |
|
value: 74.579 |
|
- type: recall_at_1000 |
|
value: 92.384 |
|
- type: recall_at_3 |
|
value: 31.734 |
|
- type: recall_at_5 |
|
value: 38.361000000000004 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/emotion |
|
name: MTEB EmotionClassification |
|
config: default |
|
split: test |
|
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 |
|
metrics: |
|
- type: accuracy |
|
value: 67.085 |
|
- type: f1 |
|
value: 61.019909873305686 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/fiqa |
|
name: MTEB FiQA2018 |
|
config: default |
|
split: test |
|
revision: 27a168819829fe9bcd655c2df245fb19452e8e06 |
|
metrics: |
|
- type: map_at_1 |
|
value: 32.251999999999995 |
|
- type: map_at_10 |
|
value: 53.98500000000001 |
|
- type: map_at_100 |
|
value: 56.093 |
|
- type: map_at_1000 |
|
value: 56.198 |
|
- type: map_at_3 |
|
value: 46.765 |
|
- type: map_at_5 |
|
value: 50.739999999999995 |
|
- type: mrr_at_1 |
|
value: 60.956999999999994 |
|
- type: mrr_at_10 |
|
value: 69.38600000000001 |
|
- type: mrr_at_100 |
|
value: 69.877 |
|
- type: mrr_at_1000 |
|
value: 69.884 |
|
- type: mrr_at_3 |
|
value: 67.052 |
|
- type: mrr_at_5 |
|
value: 68.356 |
|
- type: ndcg_at_1 |
|
value: 60.956999999999994 |
|
- type: ndcg_at_10 |
|
value: 62.78399999999999 |
|
- type: ndcg_at_100 |
|
value: 68.743 |
|
- type: ndcg_at_1000 |
|
value: 69.92399999999999 |
|
- type: ndcg_at_3 |
|
value: 57.336 |
|
- type: ndcg_at_5 |
|
value: 59.121 |
|
- type: precision_at_1 |
|
value: 60.956999999999994 |
|
- type: precision_at_10 |
|
value: 17.346 |
|
- type: precision_at_100 |
|
value: 2.3689999999999998 |
|
- type: precision_at_1000 |
|
value: 0.259 |
|
- type: precision_at_3 |
|
value: 37.912 |
|
- type: precision_at_5 |
|
value: 27.900999999999996 |
|
- type: recall_at_1 |
|
value: 32.251999999999995 |
|
- type: recall_at_10 |
|
value: 71.616 |
|
- type: recall_at_100 |
|
value: 92.685 |
|
- type: recall_at_1000 |
|
value: 98.983 |
|
- type: recall_at_3 |
|
value: 52.064 |
|
- type: recall_at_5 |
|
value: 60.49099999999999 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/imdb |
|
name: MTEB ImdbClassification |
|
config: default |
|
split: test |
|
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
|
metrics: |
|
- type: accuracy |
|
value: 96.4592 |
|
- type: ap |
|
value: 94.57299077219179 |
|
- type: f1 |
|
value: 96.45842059801627 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (en) |
|
config: en |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 98.45873233014134 |
|
- type: f1 |
|
value: 98.38426074551533 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 90.01823985408116 |
|
- type: f1 |
|
value: 70.71419843084274 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 84.35104236718225 |
|
- type: f1 |
|
value: 82.50884520186432 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (en) |
|
config: en |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 88.0665770006725 |
|
- type: f1 |
|
value: 87.06928510969733 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-p2p |
|
name: MTEB MedrxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
|
metrics: |
|
- type: v_measure |
|
value: 46.053400985420204 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-s2s |
|
name: MTEB MedrxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
|
metrics: |
|
- type: v_measure |
|
value: 44.445957227318054 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/mind_small |
|
name: MTEB MindSmallReranking |
|
config: default |
|
split: test |
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
|
metrics: |
|
- type: map |
|
value: 33.277065197675775 |
|
- type: mrr |
|
value: 34.654704063060656 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/nfcorpus |
|
name: MTEB NFCorpus |
|
config: default |
|
split: test |
|
revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 |
|
metrics: |
|
- type: map_at_1 |
|
value: 6.728000000000001 |
|
- type: map_at_10 |
|
value: 15.052999999999999 |
|
- type: map_at_100 |
|
value: 19.901 |
|
- type: map_at_1000 |
|
value: 21.72 |
|
- type: map_at_3 |
|
value: 10.901 |
|
- type: map_at_5 |
|
value: 12.651000000000002 |
|
- type: mrr_at_1 |
|
value: 52.322 |
|
- type: mrr_at_10 |
|
value: 60.614999999999995 |
|
- type: mrr_at_100 |
|
value: 61.199000000000005 |
|
- type: mrr_at_1000 |
|
value: 61.227 |
|
- type: mrr_at_3 |
|
value: 58.977999999999994 |
|
- type: mrr_at_5 |
|
value: 59.907 |
|
- type: ndcg_at_1 |
|
value: 50.619 |
|
- type: ndcg_at_10 |
|
value: 40.278000000000006 |
|
- type: ndcg_at_100 |
|
value: 37.585 |
|
- type: ndcg_at_1000 |
|
value: 46.459 |
|
- type: ndcg_at_3 |
|
value: 46.143 |
|
- type: ndcg_at_5 |
|
value: 43.7 |
|
- type: precision_at_1 |
|
value: 52.012 |
|
- type: precision_at_10 |
|
value: 30.154999999999998 |
|
- type: precision_at_100 |
|
value: 9.87 |
|
- type: precision_at_1000 |
|
value: 2.343 |
|
- type: precision_at_3 |
|
value: 42.931000000000004 |
|
- type: precision_at_5 |
|
value: 37.771 |
|
- type: recall_at_1 |
|
value: 6.728000000000001 |
|
- type: recall_at_10 |
|
value: 19.372 |
|
- type: recall_at_100 |
|
value: 39.044000000000004 |
|
- type: recall_at_1000 |
|
value: 71.602 |
|
- type: recall_at_3 |
|
value: 12.328 |
|
- type: recall_at_5 |
|
value: 14.758 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/quora |
|
name: MTEB QuoraRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 72.421 |
|
- type: map_at_10 |
|
value: 86.648 |
|
- type: map_at_100 |
|
value: 87.258 |
|
- type: map_at_1000 |
|
value: 87.26899999999999 |
|
- type: map_at_3 |
|
value: 83.82 |
|
- type: map_at_5 |
|
value: 85.629 |
|
- type: mrr_at_1 |
|
value: 83.21 |
|
- type: mrr_at_10 |
|
value: 89.198 |
|
- type: mrr_at_100 |
|
value: 89.277 |
|
- type: mrr_at_1000 |
|
value: 89.277 |
|
- type: mrr_at_3 |
|
value: 88.428 |
|
- type: mrr_at_5 |
|
value: 88.98 |
|
- type: ndcg_at_1 |
|
value: 83.24000000000001 |
|
- type: ndcg_at_10 |
|
value: 90.067 |
|
- type: ndcg_at_100 |
|
value: 91.091 |
|
- type: ndcg_at_1000 |
|
value: 91.146 |
|
- type: ndcg_at_3 |
|
value: 87.6 |
|
- type: ndcg_at_5 |
|
value: 89.004 |
|
- type: precision_at_1 |
|
value: 83.24000000000001 |
|
- type: precision_at_10 |
|
value: 13.644 |
|
- type: precision_at_100 |
|
value: 1.542 |
|
- type: precision_at_1000 |
|
value: 0.157 |
|
- type: precision_at_3 |
|
value: 38.437 |
|
- type: precision_at_5 |
|
value: 25.194 |
|
- type: recall_at_1 |
|
value: 72.421 |
|
- type: recall_at_10 |
|
value: 96.49600000000001 |
|
- type: recall_at_100 |
|
value: 99.802 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_3 |
|
value: 89.31400000000001 |
|
- type: recall_at_5 |
|
value: 93.363 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering |
|
name: MTEB RedditClustering |
|
config: default |
|
split: test |
|
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb |
|
metrics: |
|
- type: v_measure |
|
value: 73.97491289906442 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering-p2p |
|
name: MTEB RedditClusteringP2P |
|
config: default |
|
split: test |
|
revision: 282350215ef01743dc01b456c7f5241fa8937f16 |
|
metrics: |
|
- type: v_measure |
|
value: 73.49590001712183 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/scidocs |
|
name: MTEB SCIDOCS |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 6.978 |
|
- type: map_at_10 |
|
value: 18.307000000000002 |
|
- type: map_at_100 |
|
value: 21.605 |
|
- type: map_at_1000 |
|
value: 21.965 |
|
- type: map_at_3 |
|
value: 12.642000000000001 |
|
- type: map_at_5 |
|
value: 15.453 |
|
- type: mrr_at_1 |
|
value: 34.300000000000004 |
|
- type: mrr_at_10 |
|
value: 46.886 |
|
- type: mrr_at_100 |
|
value: 47.78 |
|
- type: mrr_at_1000 |
|
value: 47.795 |
|
- type: mrr_at_3 |
|
value: 42.467 |
|
- type: mrr_at_5 |
|
value: 45.427 |
|
- type: ndcg_at_1 |
|
value: 34.300000000000004 |
|
- type: ndcg_at_10 |
|
value: 29.372999999999998 |
|
- type: ndcg_at_100 |
|
value: 40.355000000000004 |
|
- type: ndcg_at_1000 |
|
value: 45.221000000000004 |
|
- type: ndcg_at_3 |
|
value: 27.230999999999998 |
|
- type: ndcg_at_5 |
|
value: 24.352 |
|
- type: precision_at_1 |
|
value: 34.300000000000004 |
|
- type: precision_at_10 |
|
value: 15.36 |
|
- type: precision_at_100 |
|
value: 3.116 |
|
- type: precision_at_1000 |
|
value: 0.426 |
|
- type: precision_at_3 |
|
value: 25.367 |
|
- type: precision_at_5 |
|
value: 21.62 |
|
- type: recall_at_1 |
|
value: 6.978 |
|
- type: recall_at_10 |
|
value: 31.142999999999997 |
|
- type: recall_at_100 |
|
value: 63.27199999999999 |
|
- type: recall_at_1000 |
|
value: 86.512 |
|
- type: recall_at_3 |
|
value: 15.433 |
|
- type: recall_at_5 |
|
value: 21.918000000000003 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sickr-sts |
|
name: MTEB SICK-R |
|
config: default |
|
split: test |
|
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 81.90996932803432 |
|
- type: cos_sim_spearman |
|
value: 78.73848819688604 |
|
- type: euclidean_pearson |
|
value: 78.82008134820491 |
|
- type: euclidean_spearman |
|
value: 78.73797968799013 |
|
- type: manhattan_pearson |
|
value: 78.98817729907871 |
|
- type: manhattan_spearman |
|
value: 78.88989195290672 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts12-sts |
|
name: MTEB STS12 |
|
config: default |
|
split: test |
|
revision: a0d554a64d88156834ff5ae9920b964011b16384 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 86.9169693104017 |
|
- type: cos_sim_spearman |
|
value: 78.6067489618467 |
|
- type: euclidean_pearson |
|
value: 83.04545335395649 |
|
- type: euclidean_spearman |
|
value: 78.6070135484733 |
|
- type: manhattan_pearson |
|
value: 83.49435095447187 |
|
- type: manhattan_spearman |
|
value: 78.9690144080464 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts13-sts |
|
name: MTEB STS13 |
|
config: default |
|
split: test |
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 88.86389266236574 |
|
- type: cos_sim_spearman |
|
value: 88.88070867328447 |
|
- type: euclidean_pearson |
|
value: 88.52907860408021 |
|
- type: euclidean_spearman |
|
value: 88.88041097815055 |
|
- type: manhattan_pearson |
|
value: 88.65795865729802 |
|
- type: manhattan_spearman |
|
value: 89.09614539167227 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts14-sts |
|
name: MTEB STS14 |
|
config: default |
|
split: test |
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 85.90258145848692 |
|
- type: cos_sim_spearman |
|
value: 84.16679932371741 |
|
- type: euclidean_pearson |
|
value: 84.95294032883719 |
|
- type: euclidean_spearman |
|
value: 84.16781112349103 |
|
- type: manhattan_pearson |
|
value: 85.18004344325733 |
|
- type: manhattan_spearman |
|
value: 84.52374692147366 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts15-sts |
|
name: MTEB STS15 |
|
config: default |
|
split: test |
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 87.89191454971963 |
|
- type: cos_sim_spearman |
|
value: 88.44916193520294 |
|
- type: euclidean_pearson |
|
value: 87.85883738567667 |
|
- type: euclidean_spearman |
|
value: 88.44928880968476 |
|
- type: manhattan_pearson |
|
value: 88.1871454451139 |
|
- type: manhattan_spearman |
|
value: 88.94431200065807 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts16-sts |
|
name: MTEB STS16 |
|
config: default |
|
split: test |
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 85.36716703986853 |
|
- type: cos_sim_spearman |
|
value: 86.16132844716138 |
|
- type: euclidean_pearson |
|
value: 85.25811478217042 |
|
- type: euclidean_spearman |
|
value: 86.16215262183867 |
|
- type: manhattan_pearson |
|
value: 85.43281209842574 |
|
- type: manhattan_spearman |
|
value: 86.44640605346511 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-en) |
|
config: en-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 88.29794966742152 |
|
- type: cos_sim_spearman |
|
value: 88.27359278171622 |
|
- type: euclidean_pearson |
|
value: 88.06469525438956 |
|
- type: euclidean_spearman |
|
value: 88.28670070410784 |
|
- type: manhattan_pearson |
|
value: 87.89087342332212 |
|
- type: manhattan_spearman |
|
value: 88.11041644578535 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (en) |
|
config: en |
|
split: test |
|
revision: eea2b4fe26a775864c896887d910b76a8098ad3f |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 66.75199645389645 |
|
- type: cos_sim_spearman |
|
value: 66.20137384486978 |
|
- type: euclidean_pearson |
|
value: 68.622513186352 |
|
- type: euclidean_spearman |
|
value: 66.23640152769464 |
|
- type: manhattan_pearson |
|
value: 68.97988448341921 |
|
- type: manhattan_spearman |
|
value: 66.39142269154794 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/stsbenchmark-sts |
|
name: MTEB STSBenchmark |
|
config: default |
|
split: test |
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 86.48693548775047 |
|
- type: cos_sim_spearman |
|
value: 86.08823308674964 |
|
- type: euclidean_pearson |
|
value: 85.65692420470154 |
|
- type: euclidean_spearman |
|
value: 86.08859480677167 |
|
- type: manhattan_pearson |
|
value: 85.90164709250936 |
|
- type: manhattan_spearman |
|
value: 86.40785365360473 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/scidocs-reranking |
|
name: MTEB SciDocsRR |
|
config: default |
|
split: test |
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
|
metrics: |
|
- type: map |
|
value: 88.80093449044475 |
|
- type: mrr |
|
value: 97.02094655526028 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/scifact |
|
name: MTEB SciFact |
|
config: default |
|
split: test |
|
revision: 0228b52cf27578f30900b9e5271d331663a030d7 |
|
metrics: |
|
- type: map_at_1 |
|
value: 59.594 |
|
- type: map_at_10 |
|
value: 72.649 |
|
- type: map_at_100 |
|
value: 73.051 |
|
- type: map_at_1000 |
|
value: 73.056 |
|
- type: map_at_3 |
|
value: 69.667 |
|
- type: map_at_5 |
|
value: 71.528 |
|
- type: mrr_at_1 |
|
value: 62.666999999999994 |
|
- type: mrr_at_10 |
|
value: 73.625 |
|
- type: mrr_at_100 |
|
value: 73.956 |
|
- type: mrr_at_1000 |
|
value: 73.962 |
|
- type: mrr_at_3 |
|
value: 71.77799999999999 |
|
- type: mrr_at_5 |
|
value: 72.994 |
|
- type: ndcg_at_1 |
|
value: 62.666999999999994 |
|
- type: ndcg_at_10 |
|
value: 77.981 |
|
- type: ndcg_at_100 |
|
value: 79.474 |
|
- type: ndcg_at_1000 |
|
value: 79.569 |
|
- type: ndcg_at_3 |
|
value: 73.4 |
|
- type: ndcg_at_5 |
|
value: 75.806 |
|
- type: precision_at_1 |
|
value: 62.666999999999994 |
|
- type: precision_at_10 |
|
value: 10.567 |
|
- type: precision_at_100 |
|
value: 1.123 |
|
- type: precision_at_1000 |
|
value: 0.11299999999999999 |
|
- type: precision_at_3 |
|
value: 29.555999999999997 |
|
- type: precision_at_5 |
|
value: 19.467000000000002 |
|
- type: recall_at_1 |
|
value: 59.594 |
|
- type: recall_at_10 |
|
value: 93.167 |
|
- type: recall_at_100 |
|
value: 99.333 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_3 |
|
value: 80.72200000000001 |
|
- type: recall_at_5 |
|
value: 86.79400000000001 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/sprintduplicatequestions-pairclassification |
|
name: MTEB SprintDuplicateQuestions |
|
config: default |
|
split: test |
|
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 99.67920792079208 |
|
- type: cos_sim_ap |
|
value: 91.12451155203843 |
|
- type: cos_sim_f1 |
|
value: 82.7763496143959 |
|
- type: cos_sim_precision |
|
value: 85.18518518518519 |
|
- type: cos_sim_recall |
|
value: 80.5 |
|
- type: dot_accuracy |
|
value: 99.68019801980198 |
|
- type: dot_ap |
|
value: 91.12360077338997 |
|
- type: dot_f1 |
|
value: 82.81893004115227 |
|
- type: dot_precision |
|
value: 85.27542372881356 |
|
- type: dot_recall |
|
value: 80.5 |
|
- type: euclidean_accuracy |
|
value: 99.67920792079208 |
|
- type: euclidean_ap |
|
value: 91.12526537243333 |
|
- type: euclidean_f1 |
|
value: 82.7763496143959 |
|
- type: euclidean_precision |
|
value: 85.18518518518519 |
|
- type: euclidean_recall |
|
value: 80.5 |
|
- type: manhattan_accuracy |
|
value: 99.68613861386139 |
|
- type: manhattan_ap |
|
value: 91.52045550487428 |
|
- type: manhattan_f1 |
|
value: 83.38461538461539 |
|
- type: manhattan_precision |
|
value: 85.57894736842105 |
|
- type: manhattan_recall |
|
value: 81.3 |
|
- type: max_accuracy |
|
value: 99.68613861386139 |
|
- type: max_ap |
|
value: 91.52045550487428 |
|
- type: max_f1 |
|
value: 83.38461538461539 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering |
|
name: MTEB StackExchangeClustering |
|
config: default |
|
split: test |
|
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
|
metrics: |
|
- type: v_measure |
|
value: 79.90649023801956 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering-p2p |
|
name: MTEB StackExchangeClusteringP2P |
|
config: default |
|
split: test |
|
revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
|
metrics: |
|
- type: v_measure |
|
value: 49.681864218959205 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/stackoverflowdupquestions-reranking |
|
name: MTEB StackOverflowDupQuestions |
|
config: default |
|
split: test |
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
|
metrics: |
|
- type: map |
|
value: 55.89272881949486 |
|
- type: mrr |
|
value: 56.88128660555132 |
|
- task: |
|
type: Summarization |
|
dataset: |
|
type: mteb/summeval |
|
name: MTEB SummEval |
|
config: default |
|
split: test |
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 31.945233723225954 |
|
- type: cos_sim_spearman |
|
value: 31.361651389713284 |
|
- type: dot_pearson |
|
value: 31.96193321438737 |
|
- type: dot_spearman |
|
value: 31.37045148053791 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/trec-covid |
|
name: MTEB TRECCOVID |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 0.244 |
|
- type: map_at_10 |
|
value: 2.011 |
|
- type: map_at_100 |
|
value: 12.555 |
|
- type: map_at_1000 |
|
value: 30.386000000000003 |
|
- type: map_at_3 |
|
value: 0.718 |
|
- type: map_at_5 |
|
value: 1.118 |
|
- type: mrr_at_1 |
|
value: 94.0 |
|
- type: mrr_at_10 |
|
value: 97.0 |
|
- type: mrr_at_100 |
|
value: 97.0 |
|
- type: mrr_at_1000 |
|
value: 97.0 |
|
- type: mrr_at_3 |
|
value: 97.0 |
|
- type: mrr_at_5 |
|
value: 97.0 |
|
- type: ndcg_at_1 |
|
value: 93.0 |
|
- type: ndcg_at_10 |
|
value: 81.612 |
|
- type: ndcg_at_100 |
|
value: 63.468 |
|
- type: ndcg_at_1000 |
|
value: 56.508 |
|
- type: ndcg_at_3 |
|
value: 88.81599999999999 |
|
- type: ndcg_at_5 |
|
value: 85.599 |
|
- type: precision_at_1 |
|
value: 94.0 |
|
- type: precision_at_10 |
|
value: 84.0 |
|
- type: precision_at_100 |
|
value: 65.18 |
|
- type: precision_at_1000 |
|
value: 24.758 |
|
- type: precision_at_3 |
|
value: 93.333 |
|
- type: precision_at_5 |
|
value: 89.2 |
|
- type: recall_at_1 |
|
value: 0.244 |
|
- type: recall_at_10 |
|
value: 2.161 |
|
- type: recall_at_100 |
|
value: 15.862000000000002 |
|
- type: recall_at_1000 |
|
value: 53.146 |
|
- type: recall_at_3 |
|
value: 0.738 |
|
- type: recall_at_5 |
|
value: 1.167 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/toxic_conversations_50k |
|
name: MTEB ToxicConversationsClassification |
|
config: default |
|
split: test |
|
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c |
|
metrics: |
|
- type: accuracy |
|
value: 82.948 |
|
- type: ap |
|
value: 26.37282466987438 |
|
- type: f1 |
|
value: 66.9868680256644 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/tweet_sentiment_extraction |
|
name: MTEB TweetSentimentExtractionClassification |
|
config: default |
|
split: test |
|
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
|
metrics: |
|
- type: accuracy |
|
value: 73.78607809847199 |
|
- type: f1 |
|
value: 74.1324659804999 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/twentynewsgroups-clustering |
|
name: MTEB TwentyNewsgroupsClustering |
|
config: default |
|
split: test |
|
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
|
metrics: |
|
- type: v_measure |
|
value: 54.11838832136805 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twittersemeval2015-pairclassification |
|
name: MTEB TwitterSemEval2015 |
|
config: default |
|
split: test |
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 87.64975859808071 |
|
- type: cos_sim_ap |
|
value: 79.0918389936708 |
|
- type: cos_sim_f1 |
|
value: 72.18518052585232 |
|
- type: cos_sim_precision |
|
value: 68.98292858860303 |
|
- type: cos_sim_recall |
|
value: 75.69920844327177 |
|
- type: dot_accuracy |
|
value: 87.64379805686356 |
|
- type: dot_ap |
|
value: 79.09373814934631 |
|
- type: dot_f1 |
|
value: 72.18216318785579 |
|
- type: dot_precision |
|
value: 69.33171324422844 |
|
- type: dot_recall |
|
value: 75.27704485488127 |
|
- type: euclidean_accuracy |
|
value: 87.64975859808071 |
|
- type: euclidean_ap |
|
value: 79.09199976607417 |
|
- type: euclidean_f1 |
|
value: 72.17610062893083 |
|
- type: euclidean_precision |
|
value: 68.96634615384616 |
|
- type: euclidean_recall |
|
value: 75.69920844327177 |
|
- type: manhattan_accuracy |
|
value: 87.61399535077786 |
|
- type: manhattan_ap |
|
value: 78.91167634954901 |
|
- type: manhattan_f1 |
|
value: 72.0995176440721 |
|
- type: manhattan_precision |
|
value: 69.47162426614481 |
|
- type: manhattan_recall |
|
value: 74.93403693931398 |
|
- type: max_accuracy |
|
value: 87.64975859808071 |
|
- type: max_ap |
|
value: 79.09373814934631 |
|
- type: max_f1 |
|
value: 72.18518052585232 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twitterurlcorpus-pairclassification |
|
name: MTEB TwitterURLCorpus |
|
config: default |
|
split: test |
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 89.43415997205729 |
|
- type: cos_sim_ap |
|
value: 86.69200523144308 |
|
- type: cos_sim_f1 |
|
value: 79.16424418604652 |
|
- type: cos_sim_precision |
|
value: 74.95871180842279 |
|
- type: cos_sim_recall |
|
value: 83.86972590083154 |
|
- type: dot_accuracy |
|
value: 89.43415997205729 |
|
- type: dot_ap |
|
value: 86.69346224233253 |
|
- type: dot_f1 |
|
value: 79.15884340968833 |
|
- type: dot_precision |
|
value: 77.26139862190294 |
|
- type: dot_recall |
|
value: 81.15183246073299 |
|
- type: euclidean_accuracy |
|
value: 89.43221950556915 |
|
- type: euclidean_ap |
|
value: 86.69176407206174 |
|
- type: euclidean_f1 |
|
value: 79.16409231328366 |
|
- type: euclidean_precision |
|
value: 74.97074413161698 |
|
- type: euclidean_recall |
|
value: 83.85432707114259 |
|
- type: manhattan_accuracy |
|
value: 89.49237396670159 |
|
- type: manhattan_ap |
|
value: 86.72274876446832 |
|
- type: manhattan_f1 |
|
value: 79.18286510672633 |
|
- type: manhattan_precision |
|
value: 75.6058271466592 |
|
- type: manhattan_recall |
|
value: 83.1151832460733 |
|
- type: max_accuracy |
|
value: 89.49237396670159 |
|
- type: max_ap |
|
value: 86.72274876446832 |
|
- type: max_f1 |
|
value: 79.18286510672633 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/AFQMC |
|
name: MTEB AFQMC |
|
config: default |
|
split: validation |
|
revision: b44c3b011063adb25877c13823db83bb193913c4 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 65.7103214280117 |
|
- type: cos_sim_spearman |
|
value: 72.62249544256886 |
|
- type: euclidean_pearson |
|
value: 71.36812167041296 |
|
- type: euclidean_spearman |
|
value: 72.62325941111307 |
|
- type: manhattan_pearson |
|
value: 71.25613851615468 |
|
- type: manhattan_spearman |
|
value: 72.54244015155267 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/ATEC |
|
name: MTEB ATEC |
|
config: default |
|
split: test |
|
revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 59.903467713912974 |
|
- type: cos_sim_spearman |
|
value: 62.8205444560593 |
|
- type: euclidean_pearson |
|
value: 67.06329904158285 |
|
- type: euclidean_spearman |
|
value: 62.82051743557576 |
|
- type: manhattan_pearson |
|
value: 66.97943759454319 |
|
- type: manhattan_spearman |
|
value: 62.763028353169325 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (zh) |
|
config: zh |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 53.57399999999999 |
|
- type: f1 |
|
value: 50.57496370390049 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/BQ |
|
name: MTEB BQ |
|
config: default |
|
split: test |
|
revision: e3dda5e115e487b39ec7e618c0c6a29137052a55 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 79.09488668095824 |
|
- type: cos_sim_spearman |
|
value: 81.34731850197655 |
|
- type: euclidean_pearson |
|
value: 82.19030116395511 |
|
- type: euclidean_spearman |
|
value: 81.34699287691117 |
|
- type: manhattan_pearson |
|
value: 82.19510202220734 |
|
- type: manhattan_spearman |
|
value: 81.35888167395795 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringP2P |
|
name: MTEB CLSClusteringP2P |
|
config: default |
|
split: test |
|
revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476 |
|
metrics: |
|
- type: v_measure |
|
value: 48.60079470735067 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringS2S |
|
name: MTEB CLSClusteringS2S |
|
config: default |
|
split: test |
|
revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f |
|
metrics: |
|
- type: v_measure |
|
value: 46.125672623152155 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv1-reranking |
|
name: MTEB CMedQAv1 |
|
config: default |
|
split: test |
|
revision: 8d7f1e942507dac42dc58017c1a001c3717da7df |
|
metrics: |
|
- type: map |
|
value: 88.0714642862605 |
|
- type: mrr |
|
value: 90.17428571428572 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv2-reranking |
|
name: MTEB CMedQAv2 |
|
config: default |
|
split: test |
|
revision: 23d186750531a14a0357ca22cd92d712fd512ea0 |
|
metrics: |
|
- type: map |
|
value: 88.51263170426526 |
|
- type: mrr |
|
value: 90.53325396825396 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CmedqaRetrieval |
|
name: MTEB CmedqaRetrieval |
|
config: default |
|
split: dev |
|
revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301 |
|
metrics: |
|
- type: map_at_1 |
|
value: 29.610999999999997 |
|
- type: map_at_10 |
|
value: 42.832 |
|
- type: map_at_100 |
|
value: 44.639 |
|
- type: map_at_1000 |
|
value: 44.738 |
|
- type: map_at_3 |
|
value: 38.549 |
|
- type: map_at_5 |
|
value: 40.905 |
|
- type: mrr_at_1 |
|
value: 44.461 |
|
- type: mrr_at_10 |
|
value: 52.274 |
|
- type: mrr_at_100 |
|
value: 53.179 |
|
- type: mrr_at_1000 |
|
value: 53.213 |
|
- type: mrr_at_3 |
|
value: 49.917 |
|
- type: mrr_at_5 |
|
value: 51.13799999999999 |
|
- type: ndcg_at_1 |
|
value: 44.461 |
|
- type: ndcg_at_10 |
|
value: 49.557 |
|
- type: ndcg_at_100 |
|
value: 56.432 |
|
- type: ndcg_at_1000 |
|
value: 58.050000000000004 |
|
- type: ndcg_at_3 |
|
value: 44.419 |
|
- type: ndcg_at_5 |
|
value: 46.386 |
|
- type: precision_at_1 |
|
value: 44.461 |
|
- type: precision_at_10 |
|
value: 10.673 |
|
- type: precision_at_100 |
|
value: 1.6310000000000002 |
|
- type: precision_at_1000 |
|
value: 0.184 |
|
- type: precision_at_3 |
|
value: 24.656 |
|
- type: precision_at_5 |
|
value: 17.619 |
|
- type: recall_at_1 |
|
value: 29.610999999999997 |
|
- type: recall_at_10 |
|
value: 60.112 |
|
- type: recall_at_100 |
|
value: 88.346 |
|
- type: recall_at_1000 |
|
value: 98.993 |
|
- type: recall_at_3 |
|
value: 44.243 |
|
- type: recall_at_5 |
|
value: 50.64300000000001 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/CMNLI |
|
name: MTEB Cmnli |
|
config: default |
|
split: validation |
|
revision: 41bc36f332156f7adc9e38f53777c959b2ae9766 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 82.17678893565845 |
|
- type: cos_sim_ap |
|
value: 89.77899888165327 |
|
- type: cos_sim_f1 |
|
value: 83.03306727480046 |
|
- type: cos_sim_precision |
|
value: 81.0371689294458 |
|
- type: cos_sim_recall |
|
value: 85.1297638531681 |
|
- type: dot_accuracy |
|
value: 82.1647624774504 |
|
- type: dot_ap |
|
value: 89.78074283382892 |
|
- type: dot_f1 |
|
value: 83.03306727480046 |
|
- type: dot_precision |
|
value: 81.0371689294458 |
|
- type: dot_recall |
|
value: 85.1297638531681 |
|
- type: euclidean_accuracy |
|
value: 82.1888153938665 |
|
- type: euclidean_ap |
|
value: 89.77917362529757 |
|
- type: euclidean_f1 |
|
value: 83.03306727480046 |
|
- type: euclidean_precision |
|
value: 81.0371689294458 |
|
- type: euclidean_recall |
|
value: 85.1297638531681 |
|
- type: manhattan_accuracy |
|
value: 81.82802164762477 |
|
- type: manhattan_ap |
|
value: 89.56708721584408 |
|
- type: manhattan_f1 |
|
value: 82.72179938657275 |
|
- type: manhattan_precision |
|
value: 80.44631020768891 |
|
- type: manhattan_recall |
|
value: 85.1297638531681 |
|
- type: max_accuracy |
|
value: 82.1888153938665 |
|
- type: max_ap |
|
value: 89.78074283382892 |
|
- type: max_f1 |
|
value: 83.03306727480046 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CovidRetrieval |
|
name: MTEB CovidRetrieval |
|
config: default |
|
split: dev |
|
revision: 1271c7809071a13532e05f25fb53511ffce77117 |
|
metrics: |
|
- type: map_at_1 |
|
value: 66.807 |
|
- type: map_at_10 |
|
value: 75.47399999999999 |
|
- type: map_at_100 |
|
value: 75.837 |
|
- type: map_at_1000 |
|
value: 75.84 |
|
- type: map_at_3 |
|
value: 73.67399999999999 |
|
- type: map_at_5 |
|
value: 74.558 |
|
- type: mrr_at_1 |
|
value: 66.913 |
|
- type: mrr_at_10 |
|
value: 75.467 |
|
- type: mrr_at_100 |
|
value: 75.823 |
|
- type: mrr_at_1000 |
|
value: 75.82600000000001 |
|
- type: mrr_at_3 |
|
value: 73.67399999999999 |
|
- type: mrr_at_5 |
|
value: 74.586 |
|
- type: ndcg_at_1 |
|
value: 66.913 |
|
- type: ndcg_at_10 |
|
value: 79.591 |
|
- type: ndcg_at_100 |
|
value: 81.15 |
|
- type: ndcg_at_1000 |
|
value: 81.229 |
|
- type: ndcg_at_3 |
|
value: 75.83800000000001 |
|
- type: ndcg_at_5 |
|
value: 77.45 |
|
- type: precision_at_1 |
|
value: 66.913 |
|
- type: precision_at_10 |
|
value: 9.325999999999999 |
|
- type: precision_at_100 |
|
value: 1.0030000000000001 |
|
- type: precision_at_1000 |
|
value: 0.101 |
|
- type: precision_at_3 |
|
value: 27.432000000000002 |
|
- type: precision_at_5 |
|
value: 17.281 |
|
- type: recall_at_1 |
|
value: 66.807 |
|
- type: recall_at_10 |
|
value: 92.46600000000001 |
|
- type: recall_at_100 |
|
value: 99.262 |
|
- type: recall_at_1000 |
|
value: 99.895 |
|
- type: recall_at_3 |
|
value: 82.086 |
|
- type: recall_at_5 |
|
value: 85.985 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/DuRetrieval |
|
name: MTEB DuRetrieval |
|
config: default |
|
split: dev |
|
revision: a1a333e290fe30b10f3f56498e3a0d911a693ced |
|
metrics: |
|
- type: map_at_1 |
|
value: 26.599 |
|
- type: map_at_10 |
|
value: 81.577 |
|
- type: map_at_100 |
|
value: 84.368 |
|
- type: map_at_1000 |
|
value: 84.39999999999999 |
|
- type: map_at_3 |
|
value: 56.825 |
|
- type: map_at_5 |
|
value: 71.462 |
|
- type: mrr_at_1 |
|
value: 90.5 |
|
- type: mrr_at_10 |
|
value: 93.798 |
|
- type: mrr_at_100 |
|
value: 93.851 |
|
- type: mrr_at_1000 |
|
value: 93.853 |
|
- type: mrr_at_3 |
|
value: 93.5 |
|
- type: mrr_at_5 |
|
value: 93.672 |
|
- type: ndcg_at_1 |
|
value: 90.5 |
|
- type: ndcg_at_10 |
|
value: 88.633 |
|
- type: ndcg_at_100 |
|
value: 91.217 |
|
- type: ndcg_at_1000 |
|
value: 91.484 |
|
- type: ndcg_at_3 |
|
value: 87.29599999999999 |
|
- type: ndcg_at_5 |
|
value: 86.31299999999999 |
|
- type: precision_at_1 |
|
value: 90.5 |
|
- type: precision_at_10 |
|
value: 42.18 |
|
- type: precision_at_100 |
|
value: 4.839 |
|
- type: precision_at_1000 |
|
value: 0.49 |
|
- type: precision_at_3 |
|
value: 78.133 |
|
- type: precision_at_5 |
|
value: 65.82000000000001 |
|
- type: recall_at_1 |
|
value: 26.599 |
|
- type: recall_at_10 |
|
value: 90.137 |
|
- type: recall_at_100 |
|
value: 98.393 |
|
- type: recall_at_1000 |
|
value: 99.747 |
|
- type: recall_at_3 |
|
value: 59.199999999999996 |
|
- type: recall_at_5 |
|
value: 76.173 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/EcomRetrieval |
|
name: MTEB EcomRetrieval |
|
config: default |
|
split: dev |
|
revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9 |
|
metrics: |
|
- type: map_at_1 |
|
value: 55.2 |
|
- type: map_at_10 |
|
value: 64.925 |
|
- type: map_at_100 |
|
value: 65.446 |
|
- type: map_at_1000 |
|
value: 65.459 |
|
- type: map_at_3 |
|
value: 62.266999999999996 |
|
- type: map_at_5 |
|
value: 64.107 |
|
- type: mrr_at_1 |
|
value: 55.2 |
|
- type: mrr_at_10 |
|
value: 64.925 |
|
- type: mrr_at_100 |
|
value: 65.446 |
|
- type: mrr_at_1000 |
|
value: 65.459 |
|
- type: mrr_at_3 |
|
value: 62.266999999999996 |
|
- type: mrr_at_5 |
|
value: 64.107 |
|
- type: ndcg_at_1 |
|
value: 55.2 |
|
- type: ndcg_at_10 |
|
value: 69.85900000000001 |
|
- type: ndcg_at_100 |
|
value: 72.194 |
|
- type: ndcg_at_1000 |
|
value: 72.506 |
|
- type: ndcg_at_3 |
|
value: 64.538 |
|
- type: ndcg_at_5 |
|
value: 67.843 |
|
- type: precision_at_1 |
|
value: 55.2 |
|
- type: precision_at_10 |
|
value: 8.540000000000001 |
|
- type: precision_at_100 |
|
value: 0.959 |
|
- type: precision_at_1000 |
|
value: 0.098 |
|
- type: precision_at_3 |
|
value: 23.7 |
|
- type: precision_at_5 |
|
value: 15.82 |
|
- type: recall_at_1 |
|
value: 55.2 |
|
- type: recall_at_10 |
|
value: 85.39999999999999 |
|
- type: recall_at_100 |
|
value: 95.89999999999999 |
|
- type: recall_at_1000 |
|
value: 98.3 |
|
- type: recall_at_3 |
|
value: 71.1 |
|
- type: recall_at_5 |
|
value: 79.10000000000001 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/IFlyTek-classification |
|
name: MTEB IFlyTek |
|
config: default |
|
split: validation |
|
revision: 421605374b29664c5fc098418fe20ada9bd55f8a |
|
metrics: |
|
- type: accuracy |
|
value: 53.92843401308196 |
|
- type: f1 |
|
value: 40.44614048360205 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/JDReview-classification |
|
name: MTEB JDReview |
|
config: default |
|
split: test |
|
revision: b7c64bd89eb87f8ded463478346f76731f07bf8b |
|
metrics: |
|
- type: accuracy |
|
value: 86.22889305816133 |
|
- type: ap |
|
value: 55.542660925360835 |
|
- type: f1 |
|
value: 81.26964576055315 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/LCQMC |
|
name: MTEB LCQMC |
|
config: default |
|
split: test |
|
revision: 17f9b096f80380fce5ed12a9be8be7784b337daf |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 68.50234587951512 |
|
- type: cos_sim_spearman |
|
value: 73.04229322574785 |
|
- type: euclidean_pearson |
|
value: 71.76475440799503 |
|
- type: euclidean_spearman |
|
value: 73.04203161533454 |
|
- type: manhattan_pearson |
|
value: 71.75530397681868 |
|
- type: manhattan_spearman |
|
value: 73.01054099221574 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/Mmarco-reranking |
|
name: MTEB MMarcoReranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 22.67056873798454 |
|
- type: mrr |
|
value: 21.63888888888889 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MMarcoRetrieval |
|
name: MTEB MMarcoRetrieval |
|
config: default |
|
split: dev |
|
revision: 539bbde593d947e2a124ba72651aafc09eb33fc2 |
|
metrics: |
|
- type: map_at_1 |
|
value: 67.65 |
|
- type: map_at_10 |
|
value: 76.726 |
|
- type: map_at_100 |
|
value: 77.03 |
|
- type: map_at_1000 |
|
value: 77.042 |
|
- type: map_at_3 |
|
value: 74.924 |
|
- type: map_at_5 |
|
value: 76.08200000000001 |
|
- type: mrr_at_1 |
|
value: 69.87100000000001 |
|
- type: mrr_at_10 |
|
value: 77.238 |
|
- type: mrr_at_100 |
|
value: 77.492 |
|
- type: mrr_at_1000 |
|
value: 77.503 |
|
- type: mrr_at_3 |
|
value: 75.633 |
|
- type: mrr_at_5 |
|
value: 76.678 |
|
- type: ndcg_at_1 |
|
value: 69.87100000000001 |
|
- type: ndcg_at_10 |
|
value: 80.37100000000001 |
|
- type: ndcg_at_100 |
|
value: 81.658 |
|
- type: ndcg_at_1000 |
|
value: 81.94200000000001 |
|
- type: ndcg_at_3 |
|
value: 76.94 |
|
- type: ndcg_at_5 |
|
value: 78.926 |
|
- type: precision_at_1 |
|
value: 69.87100000000001 |
|
- type: precision_at_10 |
|
value: 9.681 |
|
- type: precision_at_100 |
|
value: 1.032 |
|
- type: precision_at_1000 |
|
value: 0.105 |
|
- type: precision_at_3 |
|
value: 28.906 |
|
- type: precision_at_5 |
|
value: 18.404 |
|
- type: recall_at_1 |
|
value: 67.65 |
|
- type: recall_at_10 |
|
value: 91.078 |
|
- type: recall_at_100 |
|
value: 96.767 |
|
- type: recall_at_1000 |
|
value: 98.933 |
|
- type: recall_at_3 |
|
value: 82.02000000000001 |
|
- type: recall_at_5 |
|
value: 86.771 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 79.7848016139879 |
|
- type: f1 |
|
value: 76.99189501152489 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 83.64492266308001 |
|
- type: f1 |
|
value: 82.84955852311293 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MedicalRetrieval |
|
name: MTEB MedicalRetrieval |
|
config: default |
|
split: dev |
|
revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6 |
|
metrics: |
|
- type: map_at_1 |
|
value: 54.400000000000006 |
|
- type: map_at_10 |
|
value: 60.529999999999994 |
|
- type: map_at_100 |
|
value: 61.114999999999995 |
|
- type: map_at_1000 |
|
value: 61.153999999999996 |
|
- type: map_at_3 |
|
value: 59.150000000000006 |
|
- type: map_at_5 |
|
value: 59.955000000000005 |
|
- type: mrr_at_1 |
|
value: 54.50000000000001 |
|
- type: mrr_at_10 |
|
value: 60.58 |
|
- type: mrr_at_100 |
|
value: 61.165000000000006 |
|
- type: mrr_at_1000 |
|
value: 61.204 |
|
- type: mrr_at_3 |
|
value: 59.199999999999996 |
|
- type: mrr_at_5 |
|
value: 60.004999999999995 |
|
- type: ndcg_at_1 |
|
value: 54.400000000000006 |
|
- type: ndcg_at_10 |
|
value: 63.522999999999996 |
|
- type: ndcg_at_100 |
|
value: 66.742 |
|
- type: ndcg_at_1000 |
|
value: 67.818 |
|
- type: ndcg_at_3 |
|
value: 60.702999999999996 |
|
- type: ndcg_at_5 |
|
value: 62.149 |
|
- type: precision_at_1 |
|
value: 54.400000000000006 |
|
- type: precision_at_10 |
|
value: 7.290000000000001 |
|
- type: precision_at_100 |
|
value: 0.8880000000000001 |
|
- type: precision_at_1000 |
|
value: 0.097 |
|
- type: precision_at_3 |
|
value: 21.733 |
|
- type: precision_at_5 |
|
value: 13.74 |
|
- type: recall_at_1 |
|
value: 54.400000000000006 |
|
- type: recall_at_10 |
|
value: 72.89999999999999 |
|
- type: recall_at_100 |
|
value: 88.8 |
|
- type: recall_at_1000 |
|
value: 97.39999999999999 |
|
- type: recall_at_3 |
|
value: 65.2 |
|
- type: recall_at_5 |
|
value: 68.7 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/MultilingualSentiment-classification |
|
name: MTEB MultilingualSentiment |
|
config: default |
|
split: validation |
|
revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a |
|
metrics: |
|
- type: accuracy |
|
value: 77.16000000000001 |
|
- type: f1 |
|
value: 76.97953105264186 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/OCNLI |
|
name: MTEB Ocnli |
|
config: default |
|
split: validation |
|
revision: 66e76a618a34d6d565d5538088562851e6daa7ec |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 79.48023822414727 |
|
- type: cos_sim_ap |
|
value: 84.483894203704 |
|
- type: cos_sim_f1 |
|
value: 80.55130168453292 |
|
- type: cos_sim_precision |
|
value: 77.96442687747036 |
|
- type: cos_sim_recall |
|
value: 83.31573389651531 |
|
- type: dot_accuracy |
|
value: 79.48023822414727 |
|
- type: dot_ap |
|
value: 84.49261973641154 |
|
- type: dot_f1 |
|
value: 80.55130168453292 |
|
- type: dot_precision |
|
value: 77.96442687747036 |
|
- type: dot_recall |
|
value: 83.31573389651531 |
|
- type: euclidean_accuracy |
|
value: 79.48023822414727 |
|
- type: euclidean_ap |
|
value: 84.48068994534293 |
|
- type: euclidean_f1 |
|
value: 80.55130168453292 |
|
- type: euclidean_precision |
|
value: 77.96442687747036 |
|
- type: euclidean_recall |
|
value: 83.31573389651531 |
|
- type: manhattan_accuracy |
|
value: 79.37195452084461 |
|
- type: manhattan_ap |
|
value: 84.45931914984077 |
|
- type: manhattan_f1 |
|
value: 80.53142565150742 |
|
- type: manhattan_precision |
|
value: 78.01980198019803 |
|
- type: manhattan_recall |
|
value: 83.21013727560718 |
|
- type: max_accuracy |
|
value: 79.48023822414727 |
|
- type: max_ap |
|
value: 84.49261973641154 |
|
- type: max_f1 |
|
value: 80.55130168453292 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/OnlineShopping-classification |
|
name: MTEB OnlineShopping |
|
config: default |
|
split: test |
|
revision: e610f2ebd179a8fda30ae534c3878750a96db120 |
|
metrics: |
|
- type: accuracy |
|
value: 94.3 |
|
- type: ap |
|
value: 92.84324255663363 |
|
- type: f1 |
|
value: 94.29275233313747 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/PAWSX |
|
name: MTEB PAWSX |
|
config: default |
|
split: test |
|
revision: 9c6a90e430ac22b5779fb019a23e820b11a8b5e1 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 50.25954594958544 |
|
- type: cos_sim_spearman |
|
value: 55.1554675848278 |
|
- type: euclidean_pearson |
|
value: 53.71113201288935 |
|
- type: euclidean_spearman |
|
value: 55.1558156481826 |
|
- type: manhattan_pearson |
|
value: 53.816355416293646 |
|
- type: manhattan_spearman |
|
value: 55.14310001157623 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/QBQTC |
|
name: MTEB QBQTC |
|
config: default |
|
split: test |
|
revision: 790b0510dc52b1553e8c49f3d2afb48c0e5c48b7 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 29.187751074660845 |
|
- type: cos_sim_spearman |
|
value: 30.889291180505868 |
|
- type: euclidean_pearson |
|
value: 28.73210543314964 |
|
- type: euclidean_spearman |
|
value: 30.889662787316784 |
|
- type: manhattan_pearson |
|
value: 29.21703764852649 |
|
- type: manhattan_spearman |
|
value: 31.47317743982721 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (zh) |
|
config: zh |
|
split: test |
|
revision: eea2b4fe26a775864c896887d910b76a8098ad3f |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 61.1898272680276 |
|
- type: cos_sim_spearman |
|
value: 64.93927648503598 |
|
- type: euclidean_pearson |
|
value: 61.11026474293018 |
|
- type: euclidean_spearman |
|
value: 64.94229072933243 |
|
- type: manhattan_pearson |
|
value: 62.19814132782434 |
|
- type: manhattan_spearman |
|
value: 65.2583560877569 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/STSB |
|
name: MTEB STSB |
|
config: default |
|
split: test |
|
revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 78.76122365246462 |
|
- type: cos_sim_spearman |
|
value: 78.68211802616669 |
|
- type: euclidean_pearson |
|
value: 77.17265704615994 |
|
- type: euclidean_spearman |
|
value: 78.68087191872655 |
|
- type: manhattan_pearson |
|
value: 77.61313585452194 |
|
- type: manhattan_spearman |
|
value: 79.22153641729726 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/T2Reranking |
|
name: MTEB T2Reranking |
|
config: default |
|
split: dev |
|
revision: 76631901a18387f85eaa53e5450019b87ad58ef9 |
|
metrics: |
|
- type: map |
|
value: 67.80243119237458 |
|
- type: mrr |
|
value: 78.00406497512118 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/T2Retrieval |
|
name: MTEB T2Retrieval |
|
config: default |
|
split: dev |
|
revision: 8731a845f1bf500a4f111cf1070785c793d10e64 |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.936 |
|
- type: map_at_10 |
|
value: 82.256 |
|
- type: map_at_100 |
|
value: 85.688 |
|
- type: map_at_1000 |
|
value: 85.727 |
|
- type: map_at_3 |
|
value: 57.655 |
|
- type: map_at_5 |
|
value: 71.05 |
|
- type: mrr_at_1 |
|
value: 92.548 |
|
- type: mrr_at_10 |
|
value: 94.586 |
|
- type: mrr_at_100 |
|
value: 94.64399999999999 |
|
- type: mrr_at_1000 |
|
value: 94.646 |
|
- type: mrr_at_3 |
|
value: 94.255 |
|
- type: mrr_at_5 |
|
value: 94.464 |
|
- type: ndcg_at_1 |
|
value: 92.548 |
|
- type: ndcg_at_10 |
|
value: 88.74600000000001 |
|
- type: ndcg_at_100 |
|
value: 91.58500000000001 |
|
- type: ndcg_at_1000 |
|
value: 91.953 |
|
- type: ndcg_at_3 |
|
value: 89.578 |
|
- type: ndcg_at_5 |
|
value: 88.584 |
|
- type: precision_at_1 |
|
value: 92.548 |
|
- type: precision_at_10 |
|
value: 43.954 |
|
- type: precision_at_100 |
|
value: 5.099 |
|
- type: precision_at_1000 |
|
value: 0.518 |
|
- type: precision_at_3 |
|
value: 78.213 |
|
- type: precision_at_5 |
|
value: 65.839 |
|
- type: recall_at_1 |
|
value: 28.936 |
|
- type: recall_at_10 |
|
value: 87.869 |
|
- type: recall_at_100 |
|
value: 97.286 |
|
- type: recall_at_1000 |
|
value: 99.173 |
|
- type: recall_at_3 |
|
value: 59.157000000000004 |
|
- type: recall_at_5 |
|
value: 74.02499999999999 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/TNews-classification |
|
name: MTEB TNews |
|
config: default |
|
split: validation |
|
revision: 317f262bf1e6126357bbe89e875451e4b0938fe4 |
|
metrics: |
|
- type: accuracy |
|
value: 53.269 |
|
- type: f1 |
|
value: 50.68236445411186 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringP2P |
|
name: MTEB ThuNewsClusteringP2P |
|
config: default |
|
split: test |
|
revision: 5798586b105c0434e4f0fe5e767abe619442cf93 |
|
metrics: |
|
- type: v_measure |
|
value: 86.47994658950259 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringS2S |
|
name: MTEB ThuNewsClusteringS2S |
|
config: default |
|
split: test |
|
revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d |
|
metrics: |
|
- type: v_measure |
|
value: 85.34791895793325 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/VideoRetrieval |
|
name: MTEB VideoRetrieval |
|
config: default |
|
split: dev |
|
revision: 58c2597a5943a2ba48f4668c3b90d796283c5639 |
|
metrics: |
|
- type: map_at_1 |
|
value: 65.5 |
|
- type: map_at_10 |
|
value: 74.134 |
|
- type: map_at_100 |
|
value: 74.49799999999999 |
|
- type: map_at_1000 |
|
value: 74.509 |
|
- type: map_at_3 |
|
value: 72.467 |
|
- type: map_at_5 |
|
value: 73.462 |
|
- type: mrr_at_1 |
|
value: 65.5 |
|
- type: mrr_at_10 |
|
value: 74.134 |
|
- type: mrr_at_100 |
|
value: 74.49799999999999 |
|
- type: mrr_at_1000 |
|
value: 74.509 |
|
- type: mrr_at_3 |
|
value: 72.467 |
|
- type: mrr_at_5 |
|
value: 73.462 |
|
- type: ndcg_at_1 |
|
value: 65.5 |
|
- type: ndcg_at_10 |
|
value: 78.144 |
|
- type: ndcg_at_100 |
|
value: 79.726 |
|
- type: ndcg_at_1000 |
|
value: 79.97800000000001 |
|
- type: ndcg_at_3 |
|
value: 74.735 |
|
- type: ndcg_at_5 |
|
value: 76.55999999999999 |
|
- type: precision_at_1 |
|
value: 65.5 |
|
- type: precision_at_10 |
|
value: 9.06 |
|
- type: precision_at_100 |
|
value: 0.976 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_3 |
|
value: 27.1 |
|
- type: precision_at_5 |
|
value: 17.16 |
|
- type: recall_at_1 |
|
value: 65.5 |
|
- type: recall_at_10 |
|
value: 90.60000000000001 |
|
- type: recall_at_100 |
|
value: 97.6 |
|
- type: recall_at_1000 |
|
value: 99.5 |
|
- type: recall_at_3 |
|
value: 81.3 |
|
- type: recall_at_5 |
|
value: 85.8 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/waimai-classification |
|
name: MTEB Waimai |
|
config: default |
|
split: test |
|
revision: 339287def212450dcaa9df8c22bf93e9980c7023 |
|
metrics: |
|
- type: accuracy |
|
value: 89.43999999999998 |
|
- type: ap |
|
value: 75.53653890653014 |
|
- type: f1 |
|
value: 87.91597334503136 |
|
--- |
|
|
|
## gte-Qwen2-7B-instruct |
|
|
|
**gte-Qwen2-7B-instruct** is the latest model in the gte (General Text Embedding) model family. |
|
|
|
Recently, the [**Qwen team**](https://huggingface.co/Qwen) released the Qwen2 series models, and we have trained the **gte-Qwen2-7B-instruct** model based on the [Qwen2-7B](https://huggingface.co/Qwen/Qwen2-7B) LLM model. Compared to the [gte-Qwen1.5-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen1.5-7B-instruct) model, the **gte-Qwen2-7B-instruct** model uses the same training data and training strategies during the finetuning stage, with the only difference being the upgraded base model to Qwen2-7B. Considering the improvements in the Qwen2 series models compared to the Qwen1.5 series, we can also expect consistent performance enhancements in the embedding models. |
|
|
|
The model incorporates several key advancements: |
|
|
|
- Integration of bidirectional attention mechanisms, enriching its contextual understanding. |
|
- Instruction tuning, applied solely on the query side for streamlined efficiency |
|
- Comprehensive training across a vast, multilingual text corpus spanning diverse domains and scenarios. This training leverages both weakly supervised and supervised data, ensuring the model's applicability across numerous languages and a wide array of downstream tasks. |
|
|
|
|
|
## Model Information |
|
- Model Size: 7B |
|
- Embedding Dimension: 4096 |
|
- Max Input Tokens: 32k |
|
|
|
## Requirements |
|
``` |
|
transformers>=4.39.2 |
|
flash_attn>=2.5.6 |
|
``` |
|
## Usage |
|
|
|
### Sentence Transformers |
|
|
|
```python |
|
from sentence_transformers import SentenceTransformer |
|
|
|
model = SentenceTransformer("Alibaba-NLP/gte-Qwen2-7B-instruct", trust_remote_code=True) |
|
# In case you want to reduce the maximum length: |
|
model.max_seq_length = 8192 |
|
|
|
queries = [ |
|
"how much protein should a female eat", |
|
"summit define", |
|
] |
|
documents = [ |
|
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", |
|
"Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments.", |
|
] |
|
|
|
query_embeddings = model.encode(queries, prompt_name="query") |
|
document_embeddings = model.encode(documents) |
|
|
|
scores = (query_embeddings @ document_embeddings.T) * 100 |
|
print(scores.tolist()) |
|
``` |
|
|
|
Observe the [config_sentence_transformers.json](config_sentence_transformers.json) to see all pre-built prompt names. Otherwise, you can use `model.encode(queries, prompt="Instruct: ...\nQuery: "` to use a custom prompt of your choice. |
|
|
|
### Transformers |
|
|
|
```python |
|
import torch |
|
import torch.nn.functional as F |
|
|
|
from torch import Tensor |
|
from transformers import AutoTokenizer, AutoModel |
|
|
|
|
|
def last_token_pool(last_hidden_states: Tensor, |
|
attention_mask: Tensor) -> Tensor: |
|
left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0]) |
|
if left_padding: |
|
return last_hidden_states[:, -1] |
|
else: |
|
sequence_lengths = attention_mask.sum(dim=1) - 1 |
|
batch_size = last_hidden_states.shape[0] |
|
return last_hidden_states[torch.arange(batch_size, device=last_hidden_states.device), sequence_lengths] |
|
|
|
|
|
def get_detailed_instruct(task_description: str, query: str) -> str: |
|
return f'Instruct: {task_description}\nQuery: {query}' |
|
|
|
|
|
# Each query must come with a one-sentence instruction that describes the task |
|
task = 'Given a web search query, retrieve relevant passages that answer the query' |
|
queries = [ |
|
get_detailed_instruct(task, 'how much protein should a female eat'), |
|
get_detailed_instruct(task, 'summit define') |
|
] |
|
# No need to add instruction for retrieval documents |
|
documents = [ |
|
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", |
|
"Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments." |
|
] |
|
input_texts = queries + documents |
|
|
|
tokenizer = AutoTokenizer.from_pretrained('Alibaba-NLP/gte-Qwen2-7B-instruct', trust_remote_code=True) |
|
model = AutoModel.from_pretrained('Alibaba-NLP/gte-Qwen2-7B-instruct', trust_remote_code=True) |
|
|
|
max_length = 8192 |
|
|
|
# Tokenize the input texts |
|
batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt') |
|
outputs = model(**batch_dict) |
|
embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask']) |
|
|
|
# normalize embeddings |
|
embeddings = F.normalize(embeddings, p=2, dim=1) |
|
scores = (embeddings[:2] @ embeddings[2:].T) * 100 |
|
print(scores.tolist()) |
|
``` |
|
|
|
## Evaluation |
|
|
|
### MTEB & C-MTEB |
|
|
|
You can use the [scripts/eval_mteb.py](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct/blob/main/scripts/eval_mteb.py) to reproduce the following result of **gte-Qwen2-7B-instruct** on MTEB(English)/C-MTEB(Chinese): |
|
|
|
| Model Name | MTEB(56) | C-MTEB(35) | |
|
|:----:|:---------:|:----------:| |
|
| [bge-base-en-1.5](https://huggingface.co/BAAI/bge-base-en-v1.5) | 64.23 | - | |
|
| [bge-large-en-1.5](https://huggingface.co/BAAI/bge-large-en-v1.5) | 63.55 | - | |
|
| [gte-large-en-v1.5](https://huggingface.co/Alibaba-NLP/gte-large-en-v1.5) | 65.39 | - | |
|
| [gte-base-en-v1.5](https://huggingface.co/Alibaba-NLP/gte-large-en-v1.5) | 64.11 | - | |
|
| [mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1) | 64.68 | - | |
|
| [acge_text_embedding](https://huggingface.co/aspire/acge_text_embedding) | - | 69.07 | |
|
| [stella-mrl-large-zh-v3.5-1792d](https://huggingface.co/infgrad/stella-mrl-large-zh-v3.5-1792d) | - | 68.55 | |
|
| [gte-large-zh](https://huggingface.co/thenlper/gte-large-zh) | - | 66.72 | |
|
| [multilingual-e5-base](https://huggingface.co/intfloat/multilingual-e5-base) | 59.45 | 56.21 | |
|
| [multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large) | 61.50 | 58.81 | |
|
| [e5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) | 66.63 | 60.81 | |
|
| [gte-Qwen1.5-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen1.5-7B-instruct) | 67.34 | 69.52 | |
|
| [NV-Embed-v1](https://huggingface.co/nvidia/NV-Embed-v1) | 69.32 | - | |
|
| [**gte-Qwen2-7B-instruct**](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) | **70.04** | **71.98** | |
|
|
|
### GTE Models |
|
|
|
The gte series models have consistently released two types of models: encoder-only models (based on the BERT architecture) and decode-only models (based on the LLM architecture). |
|
|
|
## Citation |
|
|
|
If you find our paper or models helpful, please consider cite: |
|
|
|
``` |
|
@article{li2023towards, |
|
title={Towards general text embeddings with multi-stage contrastive learning}, |
|
author={Li, Zehan and Zhang, Xin and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan}, |
|
journal={arXiv preprint arXiv:2308.03281}, |
|
year={2023} |
|
} |
|
``` |
