|
--- |
|
tags: |
|
- mteb |
|
model-index: |
|
- name: data1 |
|
results: |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/AFQMC |
|
name: MTEB AFQMC |
|
config: default |
|
split: validation |
|
revision: b44c3b011063adb25877c13823db83bb193913c4 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 53.66919706568301 |
|
- type: cos_sim_spearman |
|
value: 53.84074348656974 |
|
- type: euclidean_pearson |
|
value: 53.58226184439896 |
|
- type: euclidean_spearman |
|
value: 53.84074348656974 |
|
- type: manhattan_pearson |
|
value: 53.64565834381205 |
|
- type: manhattan_spearman |
|
value: 53.75526003581371 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/ATEC |
|
name: MTEB ATEC |
|
config: default |
|
split: test |
|
revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 58.123744893539495 |
|
- type: cos_sim_spearman |
|
value: 54.44277675493291 |
|
- type: euclidean_pearson |
|
value: 61.20550691770944 |
|
- type: euclidean_spearman |
|
value: 54.44277225170509 |
|
- type: manhattan_pearson |
|
value: 60.57835645653918 |
|
- type: manhattan_spearman |
|
value: 54.46153709699013 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (zh) |
|
config: zh |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 29.746 |
|
- type: f1 |
|
value: 29.039321522193585 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/BQ |
|
name: MTEB BQ |
|
config: default |
|
split: test |
|
revision: e3dda5e115e487b39ec7e618c0c6a29137052a55 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 70.7026320728244 |
|
- type: cos_sim_spearman |
|
value: 70.57218534128499 |
|
- type: euclidean_pearson |
|
value: 69.28488221289881 |
|
- type: euclidean_spearman |
|
value: 70.57218534192015 |
|
- type: manhattan_pearson |
|
value: 69.65344674392082 |
|
- type: manhattan_spearman |
|
value: 70.64136691477553 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringP2P |
|
name: MTEB CLSClusteringP2P |
|
config: default |
|
split: test |
|
revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476 |
|
metrics: |
|
- type: v_measure |
|
value: 38.87791994762536 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringS2S |
|
name: MTEB CLSClusteringS2S |
|
config: default |
|
split: test |
|
revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f |
|
metrics: |
|
- type: v_measure |
|
value: 39.09103599244803 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv1-reranking |
|
name: MTEB CMedQAv1 |
|
config: default |
|
split: test |
|
revision: 8d7f1e942507dac42dc58017c1a001c3717da7df |
|
metrics: |
|
- type: map |
|
value: 80.40249793910444 |
|
- type: mrr |
|
value: 82.96805555555555 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv2-reranking |
|
name: MTEB CMedQAv2 |
|
config: default |
|
split: test |
|
revision: 23d186750531a14a0357ca22cd92d712fd512ea0 |
|
metrics: |
|
- type: map |
|
value: 80.39046823499085 |
|
- type: mrr |
|
value: 83.22674603174602 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CmedqaRetrieval |
|
name: MTEB CmedqaRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 15.715000000000002 |
|
- type: map_at_10 |
|
value: 24.651 |
|
- type: map_at_100 |
|
value: 26.478 |
|
- type: map_at_1000 |
|
value: 26.648 |
|
- type: map_at_3 |
|
value: 21.410999999999998 |
|
- type: map_at_5 |
|
value: 23.233 |
|
- type: mrr_at_1 |
|
value: 24.806 |
|
- type: mrr_at_10 |
|
value: 32.336 |
|
- type: mrr_at_100 |
|
value: 33.493 |
|
- type: mrr_at_1000 |
|
value: 33.568999999999996 |
|
- type: mrr_at_3 |
|
value: 29.807 |
|
- type: mrr_at_5 |
|
value: 31.294 |
|
- type: ndcg_at_1 |
|
value: 24.806 |
|
- type: ndcg_at_10 |
|
value: 30.341 |
|
- type: ndcg_at_100 |
|
value: 38.329 |
|
- type: ndcg_at_1000 |
|
value: 41.601 |
|
- type: ndcg_at_3 |
|
value: 25.655 |
|
- type: ndcg_at_5 |
|
value: 27.758 |
|
- type: precision_at_1 |
|
value: 24.806 |
|
- type: precision_at_10 |
|
value: 7.119000000000001 |
|
- type: precision_at_100 |
|
value: 1.3679999999999999 |
|
- type: precision_at_1000 |
|
value: 0.179 |
|
- type: precision_at_3 |
|
value: 14.787 |
|
- type: precision_at_5 |
|
value: 11.208 |
|
- type: recall_at_1 |
|
value: 15.715000000000002 |
|
- type: recall_at_10 |
|
value: 39.519999999999996 |
|
- type: recall_at_100 |
|
value: 73.307 |
|
- type: recall_at_1000 |
|
value: 95.611 |
|
- type: recall_at_3 |
|
value: 26.026 |
|
- type: recall_at_5 |
|
value: 32.027 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/CMNLI |
|
name: MTEB Cmnli |
|
config: default |
|
split: validation |
|
revision: 41bc36f332156f7adc9e38f53777c959b2ae9766 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 66.89116055321708 |
|
- type: cos_sim_ap |
|
value: 75.66575745519994 |
|
- type: cos_sim_f1 |
|
value: 70.2448775612194 |
|
- type: cos_sim_precision |
|
value: 61.347765363128495 |
|
- type: cos_sim_recall |
|
value: 82.16039279869068 |
|
- type: dot_accuracy |
|
value: 66.89116055321708 |
|
- type: dot_ap |
|
value: 75.68262052264197 |
|
- type: dot_f1 |
|
value: 70.2448775612194 |
|
- type: dot_precision |
|
value: 61.347765363128495 |
|
- type: dot_recall |
|
value: 82.16039279869068 |
|
- type: euclidean_accuracy |
|
value: 66.89116055321708 |
|
- type: euclidean_ap |
|
value: 75.66576722188334 |
|
- type: euclidean_f1 |
|
value: 70.2448775612194 |
|
- type: euclidean_precision |
|
value: 61.347765363128495 |
|
- type: euclidean_recall |
|
value: 82.16039279869068 |
|
- type: manhattan_accuracy |
|
value: 67.03547805171377 |
|
- type: manhattan_ap |
|
value: 75.78816934864089 |
|
- type: manhattan_f1 |
|
value: 70.35407081416284 |
|
- type: manhattan_precision |
|
value: 61.4752665617899 |
|
- type: manhattan_recall |
|
value: 82.23053542202479 |
|
- type: max_accuracy |
|
value: 67.03547805171377 |
|
- type: max_ap |
|
value: 75.78816934864089 |
|
- type: max_f1 |
|
value: 70.35407081416284 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CovidRetrieval |
|
name: MTEB CovidRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 41.57 |
|
- type: map_at_10 |
|
value: 52.932 |
|
- type: map_at_100 |
|
value: 53.581999999999994 |
|
- type: map_at_1000 |
|
value: 53.61900000000001 |
|
- type: map_at_3 |
|
value: 50.066 |
|
- type: map_at_5 |
|
value: 51.735 |
|
- type: mrr_at_1 |
|
value: 41.623 |
|
- type: mrr_at_10 |
|
value: 52.964999999999996 |
|
- type: mrr_at_100 |
|
value: 53.6 |
|
- type: mrr_at_1000 |
|
value: 53.637 |
|
- type: mrr_at_3 |
|
value: 50.158 |
|
- type: mrr_at_5 |
|
value: 51.786 |
|
- type: ndcg_at_1 |
|
value: 41.623 |
|
- type: ndcg_at_10 |
|
value: 58.55200000000001 |
|
- type: ndcg_at_100 |
|
value: 61.824999999999996 |
|
- type: ndcg_at_1000 |
|
value: 62.854 |
|
- type: ndcg_at_3 |
|
value: 52.729000000000006 |
|
- type: ndcg_at_5 |
|
value: 55.696999999999996 |
|
- type: precision_at_1 |
|
value: 41.623 |
|
- type: precision_at_10 |
|
value: 7.692 |
|
- type: precision_at_100 |
|
value: 0.927 |
|
- type: precision_at_1000 |
|
value: 0.101 |
|
- type: precision_at_3 |
|
value: 20.162 |
|
- type: precision_at_5 |
|
value: 13.572000000000001 |
|
- type: recall_at_1 |
|
value: 41.57 |
|
- type: recall_at_10 |
|
value: 76.185 |
|
- type: recall_at_100 |
|
value: 91.728 |
|
- type: recall_at_1000 |
|
value: 99.895 |
|
- type: recall_at_3 |
|
value: 60.27400000000001 |
|
- type: recall_at_5 |
|
value: 67.46600000000001 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/DuRetrieval |
|
name: MTEB DuRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 21.071 |
|
- type: map_at_10 |
|
value: 65.093 |
|
- type: map_at_100 |
|
value: 69.097 |
|
- type: map_at_1000 |
|
value: 69.172 |
|
- type: map_at_3 |
|
value: 44.568000000000005 |
|
- type: map_at_5 |
|
value: 56.016999999999996 |
|
- type: mrr_at_1 |
|
value: 76.35 |
|
- type: mrr_at_10 |
|
value: 83.721 |
|
- type: mrr_at_100 |
|
value: 83.899 |
|
- type: mrr_at_1000 |
|
value: 83.904 |
|
- type: mrr_at_3 |
|
value: 82.958 |
|
- type: mrr_at_5 |
|
value: 83.488 |
|
- type: ndcg_at_1 |
|
value: 76.35 |
|
- type: ndcg_at_10 |
|
value: 75.05199999999999 |
|
- type: ndcg_at_100 |
|
value: 80.596 |
|
- type: ndcg_at_1000 |
|
value: 81.394 |
|
- type: ndcg_at_3 |
|
value: 73.298 |
|
- type: ndcg_at_5 |
|
value: 72.149 |
|
- type: precision_at_1 |
|
value: 76.35 |
|
- type: precision_at_10 |
|
value: 36.96 |
|
- type: precision_at_100 |
|
value: 4.688 |
|
- type: precision_at_1000 |
|
value: 0.48700000000000004 |
|
- type: precision_at_3 |
|
value: 66.2 |
|
- type: precision_at_5 |
|
value: 55.81 |
|
- type: recall_at_1 |
|
value: 21.071 |
|
- type: recall_at_10 |
|
value: 77.459 |
|
- type: recall_at_100 |
|
value: 94.425 |
|
- type: recall_at_1000 |
|
value: 98.631 |
|
- type: recall_at_3 |
|
value: 48.335 |
|
- type: recall_at_5 |
|
value: 63.227999999999994 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/EcomRetrieval |
|
name: MTEB EcomRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 36.3 |
|
- type: map_at_10 |
|
value: 46.888999999999996 |
|
- type: map_at_100 |
|
value: 47.789 |
|
- type: map_at_1000 |
|
value: 47.827999999999996 |
|
- type: map_at_3 |
|
value: 43.85 |
|
- type: map_at_5 |
|
value: 45.58 |
|
- type: mrr_at_1 |
|
value: 36.3 |
|
- type: mrr_at_10 |
|
value: 46.888999999999996 |
|
- type: mrr_at_100 |
|
value: 47.789 |
|
- type: mrr_at_1000 |
|
value: 47.827999999999996 |
|
- type: mrr_at_3 |
|
value: 43.85 |
|
- type: mrr_at_5 |
|
value: 45.58 |
|
- type: ndcg_at_1 |
|
value: 36.3 |
|
- type: ndcg_at_10 |
|
value: 52.539 |
|
- type: ndcg_at_100 |
|
value: 56.882 |
|
- type: ndcg_at_1000 |
|
value: 57.841 |
|
- type: ndcg_at_3 |
|
value: 46.303 |
|
- type: ndcg_at_5 |
|
value: 49.406 |
|
- type: precision_at_1 |
|
value: 36.3 |
|
- type: precision_at_10 |
|
value: 7.049999999999999 |
|
- type: precision_at_100 |
|
value: 0.907 |
|
- type: precision_at_1000 |
|
value: 0.098 |
|
- type: precision_at_3 |
|
value: 17.8 |
|
- type: precision_at_5 |
|
value: 12.18 |
|
- type: recall_at_1 |
|
value: 36.3 |
|
- type: recall_at_10 |
|
value: 70.5 |
|
- type: recall_at_100 |
|
value: 90.7 |
|
- type: recall_at_1000 |
|
value: 98.1 |
|
- type: recall_at_3 |
|
value: 53.400000000000006 |
|
- type: recall_at_5 |
|
value: 60.9 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/IFlyTek-classification |
|
name: MTEB IFlyTek |
|
config: default |
|
split: validation |
|
revision: 421605374b29664c5fc098418fe20ada9bd55f8a |
|
metrics: |
|
- type: accuracy |
|
value: 50.927279722970376 |
|
- type: f1 |
|
value: 39.57514582425314 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/JDReview-classification |
|
name: MTEB JDReview |
|
config: default |
|
split: test |
|
revision: b7c64bd89eb87f8ded463478346f76731f07bf8b |
|
metrics: |
|
- type: accuracy |
|
value: 84.93433395872421 |
|
- type: ap |
|
value: 50.35046267230439 |
|
- type: f1 |
|
value: 78.76452515604298 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/LCQMC |
|
name: MTEB LCQMC |
|
config: default |
|
split: test |
|
revision: 17f9b096f80380fce5ed12a9be8be7784b337daf |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 67.40319768112933 |
|
- type: cos_sim_spearman |
|
value: 74.9867527749418 |
|
- type: euclidean_pearson |
|
value: 74.08762625643878 |
|
- type: euclidean_spearman |
|
value: 74.98675720634276 |
|
- type: manhattan_pearson |
|
value: 73.86303861791671 |
|
- type: manhattan_spearman |
|
value: 75.0594224188492 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/Mmarco-reranking |
|
name: MTEB MMarcoReranking |
|
config: default |
|
split: dev |
|
revision: 8e0c766dbe9e16e1d221116a3f36795fbade07f6 |
|
metrics: |
|
- type: map |
|
value: 18.860945903258536 |
|
- type: mrr |
|
value: 17.686507936507937 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MMarcoRetrieval |
|
name: MTEB MMarcoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 49.16 |
|
- type: map_at_10 |
|
value: 57.992 |
|
- type: map_at_100 |
|
value: 58.638 |
|
- type: map_at_1000 |
|
value: 58.67 |
|
- type: map_at_3 |
|
value: 55.71 |
|
- type: map_at_5 |
|
value: 57.04900000000001 |
|
- type: mrr_at_1 |
|
value: 50.989 |
|
- type: mrr_at_10 |
|
value: 58.814 |
|
- type: mrr_at_100 |
|
value: 59.401 |
|
- type: mrr_at_1000 |
|
value: 59.431 |
|
- type: mrr_at_3 |
|
value: 56.726 |
|
- type: mrr_at_5 |
|
value: 57.955 |
|
- type: ndcg_at_1 |
|
value: 50.989 |
|
- type: ndcg_at_10 |
|
value: 62.259 |
|
- type: ndcg_at_100 |
|
value: 65.347 |
|
- type: ndcg_at_1000 |
|
value: 66.231 |
|
- type: ndcg_at_3 |
|
value: 57.78 |
|
- type: ndcg_at_5 |
|
value: 60.09100000000001 |
|
- type: precision_at_1 |
|
value: 50.989 |
|
- type: precision_at_10 |
|
value: 7.9479999999999995 |
|
- type: precision_at_100 |
|
value: 0.951 |
|
- type: precision_at_1000 |
|
value: 0.10200000000000001 |
|
- type: precision_at_3 |
|
value: 22.087 |
|
- type: precision_at_5 |
|
value: 14.479000000000001 |
|
- type: recall_at_1 |
|
value: 49.16 |
|
- type: recall_at_10 |
|
value: 74.792 |
|
- type: recall_at_100 |
|
value: 89.132 |
|
- type: recall_at_1000 |
|
value: 96.13199999999999 |
|
- type: recall_at_3 |
|
value: 62.783 |
|
- type: recall_at_5 |
|
value: 68.26100000000001 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 67.45796906523202 |
|
- type: f1 |
|
value: 65.97280169222601 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 71.59717552118359 |
|
- type: f1 |
|
value: 72.46681610207507 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MedicalRetrieval |
|
name: MTEB MedicalRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 40.5 |
|
- type: map_at_10 |
|
value: 46.892 |
|
- type: map_at_100 |
|
value: 47.579 |
|
- type: map_at_1000 |
|
value: 47.648 |
|
- type: map_at_3 |
|
value: 45.367000000000004 |
|
- type: map_at_5 |
|
value: 46.182 |
|
- type: mrr_at_1 |
|
value: 40.6 |
|
- type: mrr_at_10 |
|
value: 46.942 |
|
- type: mrr_at_100 |
|
value: 47.629 |
|
- type: mrr_at_1000 |
|
value: 47.698 |
|
- type: mrr_at_3 |
|
value: 45.417 |
|
- type: mrr_at_5 |
|
value: 46.232 |
|
- type: ndcg_at_1 |
|
value: 40.5 |
|
- type: ndcg_at_10 |
|
value: 50.078 |
|
- type: ndcg_at_100 |
|
value: 53.635999999999996 |
|
- type: ndcg_at_1000 |
|
value: 55.696999999999996 |
|
- type: ndcg_at_3 |
|
value: 46.847 |
|
- type: ndcg_at_5 |
|
value: 48.323 |
|
- type: precision_at_1 |
|
value: 40.5 |
|
- type: precision_at_10 |
|
value: 6.02 |
|
- type: precision_at_100 |
|
value: 0.773 |
|
- type: precision_at_1000 |
|
value: 0.094 |
|
- type: precision_at_3 |
|
value: 17.033 |
|
- type: precision_at_5 |
|
value: 10.94 |
|
- type: recall_at_1 |
|
value: 40.5 |
|
- type: recall_at_10 |
|
value: 60.199999999999996 |
|
- type: recall_at_100 |
|
value: 77.3 |
|
- type: recall_at_1000 |
|
value: 94.0 |
|
- type: recall_at_3 |
|
value: 51.1 |
|
- type: recall_at_5 |
|
value: 54.7 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: Shitao/MLDR |
|
name: MTEB MultiLongDocRetrieval (zh) |
|
config: zh |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 7.000000000000001 |
|
- type: map_at_10 |
|
value: 10.020999999999999 |
|
- type: map_at_100 |
|
value: 10.511 |
|
- type: map_at_1000 |
|
value: 10.595 |
|
- type: map_at_3 |
|
value: 9.042 |
|
- type: map_at_5 |
|
value: 9.654 |
|
- type: mrr_at_1 |
|
value: 6.875000000000001 |
|
- type: mrr_at_10 |
|
value: 9.958 |
|
- type: mrr_at_100 |
|
value: 10.449 |
|
- type: mrr_at_1000 |
|
value: 10.532 |
|
- type: mrr_at_3 |
|
value: 8.979 |
|
- type: mrr_at_5 |
|
value: 9.592 |
|
- type: ndcg_at_1 |
|
value: 7.000000000000001 |
|
- type: ndcg_at_10 |
|
value: 11.651 |
|
- type: ndcg_at_100 |
|
value: 14.580000000000002 |
|
- type: ndcg_at_1000 |
|
value: 17.183 |
|
- type: ndcg_at_3 |
|
value: 9.646 |
|
- type: ndcg_at_5 |
|
value: 10.738 |
|
- type: precision_at_1 |
|
value: 7.000000000000001 |
|
- type: precision_at_10 |
|
value: 1.687 |
|
- type: precision_at_100 |
|
value: 0.319 |
|
- type: precision_at_1000 |
|
value: 0.053 |
|
- type: precision_at_3 |
|
value: 3.7920000000000003 |
|
- type: precision_at_5 |
|
value: 2.8000000000000003 |
|
- type: recall_at_1 |
|
value: 7.000000000000001 |
|
- type: recall_at_10 |
|
value: 16.875 |
|
- type: recall_at_100 |
|
value: 31.874999999999996 |
|
- type: recall_at_1000 |
|
value: 53.25 |
|
- type: recall_at_3 |
|
value: 11.375 |
|
- type: recall_at_5 |
|
value: 14.000000000000002 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/MultilingualSentiment-classification |
|
name: MTEB MultilingualSentiment |
|
config: default |
|
split: validation |
|
revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a |
|
metrics: |
|
- type: accuracy |
|
value: 55.90333333333333 |
|
- type: f1 |
|
value: 55.291185234519546 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/OCNLI |
|
name: MTEB Ocnli |
|
config: default |
|
split: validation |
|
revision: 66e76a618a34d6d565d5538088562851e6daa7ec |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 59.01461829994585 |
|
- type: cos_sim_ap |
|
value: 61.84829541140869 |
|
- type: cos_sim_f1 |
|
value: 67.94150731158605 |
|
- type: cos_sim_precision |
|
value: 52.674418604651166 |
|
- type: cos_sim_recall |
|
value: 95.67053854276664 |
|
- type: dot_accuracy |
|
value: 59.01461829994585 |
|
- type: dot_ap |
|
value: 61.84829541140869 |
|
- type: dot_f1 |
|
value: 67.94150731158605 |
|
- type: dot_precision |
|
value: 52.674418604651166 |
|
- type: dot_recall |
|
value: 95.67053854276664 |
|
- type: euclidean_accuracy |
|
value: 59.01461829994585 |
|
- type: euclidean_ap |
|
value: 61.84829541140869 |
|
- type: euclidean_f1 |
|
value: 67.94150731158605 |
|
- type: euclidean_precision |
|
value: 52.674418604651166 |
|
- type: euclidean_recall |
|
value: 95.67053854276664 |
|
- type: manhattan_accuracy |
|
value: 59.06876015159719 |
|
- type: manhattan_ap |
|
value: 61.91217952354554 |
|
- type: manhattan_f1 |
|
value: 67.89059572873735 |
|
- type: manhattan_precision |
|
value: 52.613240418118465 |
|
- type: manhattan_recall |
|
value: 95.67053854276664 |
|
- type: max_accuracy |
|
value: 59.06876015159719 |
|
- type: max_ap |
|
value: 61.91217952354554 |
|
- type: max_f1 |
|
value: 67.94150731158605 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/OnlineShopping-classification |
|
name: MTEB OnlineShopping |
|
config: default |
|
split: test |
|
revision: e610f2ebd179a8fda30ae534c3878750a96db120 |
|
metrics: |
|
- type: accuracy |
|
value: 82.53 |
|
- type: ap |
|
value: 77.67591637020448 |
|
- type: f1 |
|
value: 82.39976599130478 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/PAWSX |
|
name: MTEB PAWSX |
|
config: default |
|
split: test |
|
revision: 9c6a90e430ac22b5779fb019a23e820b11a8b5e1 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 55.76388035743312 |
|
- type: cos_sim_spearman |
|
value: 58.34768166139753 |
|
- type: euclidean_pearson |
|
value: 57.971763429924074 |
|
- type: euclidean_spearman |
|
value: 58.34750745303424 |
|
- type: manhattan_pearson |
|
value: 58.044053497280245 |
|
- type: manhattan_spearman |
|
value: 58.61627719613188 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: paws-x |
|
name: MTEB PawsX (zh) |
|
config: zh |
|
split: test |
|
revision: 8a04d940a42cd40658986fdd8e3da561533a3646 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 75.75 |
|
- type: cos_sim_ap |
|
value: 78.80617392926526 |
|
- type: cos_sim_f1 |
|
value: 75.92417061611374 |
|
- type: cos_sim_precision |
|
value: 65.87171052631578 |
|
- type: cos_sim_recall |
|
value: 89.59731543624162 |
|
- type: dot_accuracy |
|
value: 75.75 |
|
- type: dot_ap |
|
value: 78.83768586994135 |
|
- type: dot_f1 |
|
value: 75.92417061611374 |
|
- type: dot_precision |
|
value: 65.87171052631578 |
|
- type: dot_recall |
|
value: 89.59731543624162 |
|
- type: euclidean_accuracy |
|
value: 75.75 |
|
- type: euclidean_ap |
|
value: 78.80617392926526 |
|
- type: euclidean_f1 |
|
value: 75.92417061611374 |
|
- type: euclidean_precision |
|
value: 65.87171052631578 |
|
- type: euclidean_recall |
|
value: 89.59731543624162 |
|
- type: manhattan_accuracy |
|
value: 75.75 |
|
- type: manhattan_ap |
|
value: 78.98640478955386 |
|
- type: manhattan_f1 |
|
value: 75.92954990215264 |
|
- type: manhattan_precision |
|
value: 67.47826086956522 |
|
- type: manhattan_recall |
|
value: 86.80089485458613 |
|
- type: max_accuracy |
|
value: 75.75 |
|
- type: max_ap |
|
value: 78.98640478955386 |
|
- type: max_f1 |
|
value: 75.92954990215264 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/QBQTC |
|
name: MTEB QBQTC |
|
config: default |
|
split: test |
|
revision: 790b0510dc52b1553e8c49f3d2afb48c0e5c48b7 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 74.40348414238575 |
|
- type: cos_sim_spearman |
|
value: 71.452270332177 |
|
- type: euclidean_pearson |
|
value: 72.62509231589097 |
|
- type: euclidean_spearman |
|
value: 71.45228258458943 |
|
- type: manhattan_pearson |
|
value: 73.03846856200839 |
|
- type: manhattan_spearman |
|
value: 71.43673225319574 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (zh) |
|
config: zh |
|
split: test |
|
revision: eea2b4fe26a775864c896887d910b76a8098ad3f |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 75.38335474357001 |
|
- type: cos_sim_spearman |
|
value: 74.92262892309807 |
|
- type: euclidean_pearson |
|
value: 73.93451693251345 |
|
- type: euclidean_spearman |
|
value: 74.92262892309807 |
|
- type: manhattan_pearson |
|
value: 74.55911294300788 |
|
- type: manhattan_spearman |
|
value: 74.89436791272614 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/STSB |
|
name: MTEB STSB |
|
config: default |
|
split: test |
|
revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 83.01687361650126 |
|
- type: cos_sim_spearman |
|
value: 82.74413230806265 |
|
- type: euclidean_pearson |
|
value: 81.50177295189083 |
|
- type: euclidean_spearman |
|
value: 82.74413230806265 |
|
- type: manhattan_pearson |
|
value: 81.90798387028589 |
|
- type: manhattan_spearman |
|
value: 82.65064251275778 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/T2Reranking |
|
name: MTEB T2Reranking |
|
config: default |
|
split: dev |
|
revision: 76631901a18387f85eaa53e5450019b87ad58ef9 |
|
metrics: |
|
- type: map |
|
value: 66.25459669294304 |
|
- type: mrr |
|
value: 76.76845224661744 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/T2Retrieval |
|
name: MTEB T2Retrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 22.515 |
|
- type: map_at_10 |
|
value: 63.63999999999999 |
|
- type: map_at_100 |
|
value: 67.67 |
|
- type: map_at_1000 |
|
value: 67.792 |
|
- type: map_at_3 |
|
value: 44.239 |
|
- type: map_at_5 |
|
value: 54.54599999999999 |
|
- type: mrr_at_1 |
|
value: 79.752 |
|
- type: mrr_at_10 |
|
value: 83.525 |
|
- type: mrr_at_100 |
|
value: 83.753 |
|
- type: mrr_at_1000 |
|
value: 83.763 |
|
- type: mrr_at_3 |
|
value: 82.65599999999999 |
|
- type: mrr_at_5 |
|
value: 83.192 |
|
- type: ndcg_at_1 |
|
value: 79.752 |
|
- type: ndcg_at_10 |
|
value: 72.699 |
|
- type: ndcg_at_100 |
|
value: 78.145 |
|
- type: ndcg_at_1000 |
|
value: 79.481 |
|
- type: ndcg_at_3 |
|
value: 74.401 |
|
- type: ndcg_at_5 |
|
value: 72.684 |
|
- type: precision_at_1 |
|
value: 79.752 |
|
- type: precision_at_10 |
|
value: 37.163000000000004 |
|
- type: precision_at_100 |
|
value: 4.769 |
|
- type: precision_at_1000 |
|
value: 0.508 |
|
- type: precision_at_3 |
|
value: 65.67399999999999 |
|
- type: precision_at_5 |
|
value: 55.105000000000004 |
|
- type: recall_at_1 |
|
value: 22.515 |
|
- type: recall_at_10 |
|
value: 71.816 |
|
- type: recall_at_100 |
|
value: 89.442 |
|
- type: recall_at_1000 |
|
value: 96.344 |
|
- type: recall_at_3 |
|
value: 46.208 |
|
- type: recall_at_5 |
|
value: 58.695 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/TNews-classification |
|
name: MTEB TNews |
|
config: default |
|
split: validation |
|
revision: 317f262bf1e6126357bbe89e875451e4b0938fe4 |
|
metrics: |
|
- type: accuracy |
|
value: 55.077999999999996 |
|
- type: f1 |
|
value: 53.2447237349446 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringP2P |
|
name: MTEB ThuNewsClusteringP2P |
|
config: default |
|
split: test |
|
revision: 5798586b105c0434e4f0fe5e767abe619442cf93 |
|
metrics: |
|
- type: v_measure |
|
value: 59.50582115422618 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringS2S |
|
name: MTEB ThuNewsClusteringS2S |
|
config: default |
|
split: test |
|
revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d |
|
metrics: |
|
- type: v_measure |
|
value: 54.71907850412647 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/VideoRetrieval |
|
name: MTEB VideoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 49.4 |
|
- type: map_at_10 |
|
value: 59.245999999999995 |
|
- type: map_at_100 |
|
value: 59.811 |
|
- type: map_at_1000 |
|
value: 59.836 |
|
- type: map_at_3 |
|
value: 56.733 |
|
- type: map_at_5 |
|
value: 58.348 |
|
- type: mrr_at_1 |
|
value: 49.4 |
|
- type: mrr_at_10 |
|
value: 59.245999999999995 |
|
- type: mrr_at_100 |
|
value: 59.811 |
|
- type: mrr_at_1000 |
|
value: 59.836 |
|
- type: mrr_at_3 |
|
value: 56.733 |
|
- type: mrr_at_5 |
|
value: 58.348 |
|
- type: ndcg_at_1 |
|
value: 49.4 |
|
- type: ndcg_at_10 |
|
value: 64.08 |
|
- type: ndcg_at_100 |
|
value: 67.027 |
|
- type: ndcg_at_1000 |
|
value: 67.697 |
|
- type: ndcg_at_3 |
|
value: 58.995 |
|
- type: ndcg_at_5 |
|
value: 61.891 |
|
- type: precision_at_1 |
|
value: 49.4 |
|
- type: precision_at_10 |
|
value: 7.93 |
|
- type: precision_at_100 |
|
value: 0.935 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_3 |
|
value: 21.833 |
|
- type: precision_at_5 |
|
value: 14.499999999999998 |
|
- type: recall_at_1 |
|
value: 49.4 |
|
- type: recall_at_10 |
|
value: 79.3 |
|
- type: recall_at_100 |
|
value: 93.5 |
|
- type: recall_at_1000 |
|
value: 98.8 |
|
- type: recall_at_3 |
|
value: 65.5 |
|
- type: recall_at_5 |
|
value: 72.5 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/waimai-classification |
|
name: MTEB Waimai |
|
config: default |
|
split: test |
|
revision: 339287def212450dcaa9df8c22bf93e9980c7023 |
|
metrics: |
|
- type: accuracy |
|
value: 81.16 |
|
- type: ap |
|
value: 60.864524843400616 |
|
- type: f1 |
|
value: 79.41246877404483 |
|
--- |
|
|
|
ZNV Embedding utilizes a 6B LLM (Large Language Model) for embedding, achieving excellent embedding results. |
|
|
|
In a single inference, we used two prompts to extract two different embeddings for a sentence, and then concatenated them. |
|
|
|
Model usage method: |
|
|
|
|
|
1. Define ZNVEmbeddingModel |
|
```python |
|
import os |
|
from transformers import ( |
|
LlamaForCausalLM, |
|
LlamaTokenizer, AutoConfig, |
|
) |
|
import torch |
|
import torch.nn.functional as F |
|
import numpy as np |
|
|
|
|
|
class ZNVEmbeddingModel(torch.nn.Module): |
|
def __init__(self, model_name_or_path): |
|
super(ZNVEmbeddingModel, self).__init__() |
|
self.prompt_prefix = "阅读下文，然后答题\n" |
|
self.prompt_suffixes = ["\n1.一个字总结上文的意思是:", |
|
"\n2.上文深层次的意思是:"] |
|
self.hidden_size = 4096 |
|
self.model_name_or_path = model_name_or_path |
|
self.linear_suffixes = torch.nn.ModuleList( |
|
[torch.nn.Linear(self.hidden_size, self.hidden_size//len(self.prompt_suffixes)) |
|
for _ in range(len(self.prompt_suffixes))]) |
|
self.tokenizer, self.llama = self.load_llama() |
|
|
|
self.tanh = torch.nn.Tanh() |
|
self.suffixes_ids = [] |
|
self.suffixes_ids_len = [] |
|
self.suffixes_len = 0 |
|
for suffix in self.prompt_suffixes: |
|
ids = self.tokenizer(suffix, return_tensors="pt")["input_ids"].tolist()[0] |
|
self.suffixes_ids += ids |
|
self.suffixes_ids_len.append(len(ids)) |
|
self.suffixes_len += len(ids) |
|
|
|
self.suffixes_ones = torch.ones(self.suffixes_len) |
|
self.suffixes_ids = torch.tensor(self.suffixes_ids) |
|
|
|
linear_file = os.path.join(model_name_or_path, "linears") |
|
load_layers = torch.load(linear_file) |
|
model_state = self.state_dict() |
|
model_state.update(load_layers) |
|
self.load_state_dict(model_state, strict=False) |
|
|
|
def load_llama(self): |
|
llm_path = os.path.join(self.model_name_or_path) |
|
config = AutoConfig.from_pretrained(llm_path) |
|
tokenizer = LlamaTokenizer.from_pretrained(self.model_name_or_path) |
|
tokenizer.padding_side = "left" |
|
model = LlamaForCausalLM.from_pretrained( |
|
llm_path, |
|
config=config, |
|
low_cpu_mem_usage=True |
|
) |
|
model.config.use_cache = False |
|
return tokenizer, model |
|
|
|
def forward(self, sentences): |
|
prompts_embeddings = [] |
|
sentences = [self.prompt_prefix + s for s in sentences] |
|
inputs = self.tokenizer(sentences, max_length=256, padding=True, truncation=True, |
|
return_tensors='pt') |
|
attention_mask = inputs["attention_mask"] |
|
input_ids = inputs["input_ids"] |
|
batch_size = len(sentences) |
|
suffixes_ones = self.suffixes_ones.unsqueeze(0) |
|
suffixes_ones = suffixes_ones.repeat(batch_size, 1) |
|
device = next(self.parameters()).device |
|
attention_mask = torch.cat([attention_mask, suffixes_ones], dim=-1).to(device) |
|
|
|
suffixes_ids = self.suffixes_ids.unsqueeze(0) |
|
suffixes_ids = suffixes_ids.repeat(batch_size, 1) |
|
input_ids = torch.cat([input_ids, suffixes_ids], dim=-1).to(device) |
|
last_hidden_state = self.llama.base_model.base_model(attention_mask=attention_mask, input_ids=input_ids).last_hidden_state |
|
index = -1 |
|
for i in range(len(self.suffixes_ids_len)): |
|
embedding = last_hidden_state[:, index, :] |
|
embedding = self.linear_suffixes[i](embedding) |
|
prompts_embeddings.append(embedding) |
|
index -= self.suffixes_ids_len[-i-1] |
|
|
|
output_embedding = torch.cat(prompts_embeddings, dim=-1) |
|
output_embedding = self.tanh(output_embedding) |
|
output_embedding = F.normalize(output_embedding, p=2, dim=1) |
|
return output_embedding |
|
|
|
def encode(self, sentences, batch_size=10, **kwargs): |
|
size = len(sentences) |
|
embeddings = None |
|
handled = 0 |
|
while handled < size: |
|
tokens = sentences[handled:handled + batch_size] |
|
output_embeddings = self.forward(tokens) |
|
result = output_embeddings.cpu().numpy() |
|
handled += result.shape[0] |
|
if embeddings is not None: |
|
embeddings = np.concatenate((embeddings, result), axis=0) |
|
else: |
|
embeddings = result |
|
return embeddings |
|
``` |
|
|
|
|
|
2. Use ZNVEmbeddingModel for Embedding. |
|
```python |
|
znv_model = ZNVEmbeddingModel("your_model_path") |
|
znv_model.eval() |
|
with torch.no_grad(): |
|
output = znv_model(["请问你的电话号码是多少？","可以告诉我你的手机号吗？"]) |
|
cos_sim = F.cosine_similarity(output[0],output[1],dim=0) |
|
print(cos_sim) |
|
``` |
