model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.8125396825396826
+      value: 0.844484126984127
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6844919786096256
+      value: 0.6764705882352942
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.685459940652819
+      value: 0.6735905044510386
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7804335742078933
+      value: 0.7926625903279599
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.96
+      value: 0.968
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6271929824561403
+      value: 0.6052631578947368
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.5995370370370371
+      value: 0.6226851851851852
   - task:
       name: Analogy Questions (ConceptNet Analogy)
       type: multiple-choice-qa
@@ -91,7 +91,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.4052013422818792
+      value: 0.39093959731543626
   - task:
       name: Analogy Questions (TREX Analogy)
       type: multiple-choice-qa
@@ -102,7 +102,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6229508196721312
+      value: 0.6612021857923497
   - task:
       name: Analogy Questions (NELL-ONE Analogy)
       type: multiple-choice-qa
@@ -113,7 +113,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6283333333333333
+      value: 0.6416666666666667
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -124,10 +124,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9168298930239566
+      value: 0.9171312339912611
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9141485326840691
+      value: 0.9144771714929822
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -138,10 +138,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8448356807511737
+      value: 0.8455399061032864
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6752034380630019
+      value: 0.6723727069325071
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -152,10 +152,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6722643553629469
+      value: 0.6793066088840737
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6535138350302286
+      value: 0.6676508193568021
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -166,10 +166,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9599360089031092
+      value: 0.95875356472143
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8799570036729836
+      value: 0.8812996852310372
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -183,7 +183,7 @@ model-index:
       value: 0.9072391099968662
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9048168410533853
+      value: 0.9041555105317078
 
 ---
 # relbert/relbert-roberta-large-iloob-d-semeval2012
@@ -191,23 +191,23 @@ model-index:
 RelBERT based on [roberta-large](https://huggingface.co/roberta-large) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
 This model achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-large-iloob-d-semeval2012/raw/main/analogy.forward.json)):
-    - Accuracy on SAT (full): 0.6844919786096256 
-    - Accuracy on SAT: 0.685459940652819
-    - Accuracy on BATS: 0.7804335742078933
-    - Accuracy on U2: 0.6271929824561403
-    - Accuracy on U4: 0.5995370370370371
-    - Accuracy on Google: 0.96
-    - Accuracy on ConceptNet Analogy: 0.4052013422818792
-    - Accuracy on T-Rex Analogy: 0.6229508196721312
-    - Accuracy on NELL-ONE Analogy: 0.6283333333333333
+    - Accuracy on SAT (full): 0.6764705882352942 
+    - Accuracy on SAT: 0.6735905044510386
+    - Accuracy on BATS: 0.7926625903279599
+    - Accuracy on U2: 0.6052631578947368
+    - Accuracy on U4: 0.6226851851851852
+    - Accuracy on Google: 0.968
+    - Accuracy on ConceptNet Analogy: 0.39093959731543626
+    - Accuracy on T-Rex Analogy: 0.6612021857923497
+    - Accuracy on NELL-ONE Analogy: 0.6416666666666667
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-large-iloob-d-semeval2012/raw/main/classification.json)):
-    - Micro F1 score on BLESS: 0.9168298930239566
-    - Micro F1 score on CogALexV: 0.8448356807511737
-    - Micro F1 score on EVALution: 0.6722643553629469
-    - Micro F1 score on K&H+N: 0.9599360089031092
+    - Micro F1 score on BLESS: 0.9171312339912611
+    - Micro F1 score on CogALexV: 0.8455399061032864
+    - Micro F1 score on EVALution: 0.6793066088840737
+    - Micro F1 score on K&H+N: 0.95875356472143
     - Micro F1 score on ROOT09: 0.9072391099968662
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-large-iloob-d-semeval2012/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.8125396825396826 
+    - Accuracy on Relation Mapping: 0.844484126984127 
 
 
 ### Usage

analogy.bidirection.json

@@ -1 +1 @@
-{"sat_full/test": 0.6871657754010695, "sat/test": 0.6913946587537092, "u2/test": 0.6622807017543859, "u4/test": 0.6527777777777778, "google/test": 0.968, "bats/test": 0.8349082823790995, "t_rex_relational_similarity/test": 0.6338797814207651, "conceptnet_relational_similarity/test": 0.45302013422818793, "nell_relational_similarity/test": 0.7333333333333333, "sat/validation": 0.6486486486486487, "u2/validation": 0.625, "u4/validation": 0.6875, "google/validation": 1.0, "bats/validation": 0.8944723618090452, "semeval2012_relational_similarity/validation": 0.7215189873417721, "t_rex_relational_similarity/validation": 0.31048387096774194, "conceptnet_relational_similarity/validation": 0.36151079136690645, "nell_relational_similarity/validation": 0.635}
+{"sat_full/test": 0.7085561497326203, "sat/test": 0.7091988130563798, "u2/test": 0.6666666666666666, "u4/test": 0.6666666666666666, "google/test": 0.978, "bats/test": 0.8293496386881601, "t_rex_relational_similarity/test": 0.6612021857923497, "conceptnet_relational_similarity/test": 0.4236577181208054, "nell_relational_similarity/test": 0.735, "sat/validation": 0.7027027027027027, "u2/validation": 0.625, "u4/validation": 0.6458333333333334, "google/validation": 1.0, "bats/validation": 0.8793969849246231, "semeval2012_relational_similarity/validation": 0.7215189873417721, "t_rex_relational_similarity/validation": 0.3165322580645161, "conceptnet_relational_similarity/validation": 0.3597122302158273, "nell_relational_similarity/validation": 0.6475}

analogy.forward.json

@@ -1 +1 @@
-{"semeval2012_relational_similarity/validation": 0.6962025316455697, "sat_full/test": 0.6844919786096256, "sat/test": 0.685459940652819, "u2/test": 0.6271929824561403, "u4/test": 0.5995370370370371, "google/test": 0.96, "bats/test": 0.7804335742078933, "t_rex_relational_similarity/test": 0.6229508196721312, "conceptnet_relational_similarity/test": 0.4052013422818792, "nell_relational_similarity/test": 0.6283333333333333, "sat/validation": 0.6756756756756757, "u2/validation": 0.4166666666666667, "u4/validation": 0.5208333333333334, "google/validation": 1.0, "bats/validation": 0.8391959798994975, "t_rex_relational_similarity/validation": 0.2903225806451613, "conceptnet_relational_similarity/validation": 0.32464028776978415, "nell_relational_similarity/validation": 0.6025}
+{"semeval2012_relational_similarity/validation": 0.759493670886076, "sat_full/test": 0.6764705882352942, "sat/test": 0.6735905044510386, "u2/test": 0.6052631578947368, "u4/test": 0.6226851851851852, "google/test": 0.968, "bats/test": 0.7926625903279599, "t_rex_relational_similarity/test": 0.6612021857923497, "conceptnet_relational_similarity/test": 0.39093959731543626, "nell_relational_similarity/test": 0.6416666666666667, "sat/validation": 0.7027027027027027, "u2/validation": 0.5833333333333334, "u4/validation": 0.5416666666666666, "google/validation": 1.0, "bats/validation": 0.8140703517587939, "t_rex_relational_similarity/validation": 0.3084677419354839, "conceptnet_relational_similarity/validation": 0.3300359712230216, "nell_relational_similarity/validation": 0.6125}

analogy.reverse.json

@@ -1 +1 @@
-{"sat_full/test": 0.6176470588235294, "sat/test": 0.6231454005934718, "u2/test": 0.6271929824561403, "u4/test": 0.6365740740740741, "google/test": 0.96, "bats/test": 0.8176764869371873, "t_rex_relational_similarity/test": 0.6338797814207651, "conceptnet_relational_similarity/test": 0.4077181208053691, "nell_relational_similarity/test": 0.7166666666666667, "sat/validation": 0.5675675675675675, "u2/validation": 0.7083333333333334, "u4/validation": 0.6458333333333334, "google/validation": 0.98, "bats/validation": 0.8743718592964824, "semeval2012_relational_similarity/validation": 0.6455696202531646, "t_rex_relational_similarity/validation": 0.2782258064516129, "conceptnet_relational_similarity/validation": 0.3471223021582734, "nell_relational_similarity/validation": 0.65}
+{"sat_full/test": 0.6256684491978609, "sat/test": 0.6409495548961425, "u2/test": 0.6535087719298246, "u4/test": 0.6620370370370371, "google/test": 0.96, "bats/test": 0.8148971650917176, "t_rex_relational_similarity/test": 0.6284153005464481, "conceptnet_relational_similarity/test": 0.3859060402684564, "nell_relational_similarity/test": 0.71, "sat/validation": 0.4864864864864865, "u2/validation": 0.625, "u4/validation": 0.6875, "google/validation": 1.0, "bats/validation": 0.8542713567839196, "semeval2012_relational_similarity/validation": 0.5949367088607594, "t_rex_relational_similarity/validation": 0.2963709677419355, "conceptnet_relational_similarity/validation": 0.31744604316546765, "nell_relational_similarity/validation": 0.655}

classification.json

@@ -1 +1 @@
-{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9168298930239566, "test/f1_macro": 0.9141485326840691, "test/f1_micro": 0.9168298930239566, "test/p_macro": 0.9122332727807994, "test/p_micro": 0.9168298930239566, "test/r_macro": 0.9167520407835513, "test/r_micro": 0.9168298930239566}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8448356807511737, "test/f1_macro": 0.6752034380630019, "test/f1_micro": 0.8448356807511737, "test/p_macro": 0.71054646454147, "test/p_micro": 0.8448356807511737, "test/r_macro": 0.6473529883092849, "test/r_micro": 0.8448356807511737}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6722643553629469, "test/f1_macro": 0.6535138350302286, "test/f1_micro": 0.6722643553629469, "test/p_macro": 0.6510526375182459, "test/p_micro": 0.6722643553629469, "test/r_macro": 0.6584808338479788, "test/r_micro": 0.6722643553629469}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9599360089031092, "test/f1_macro": 0.8799570036729836, "test/f1_micro": 0.9599360089031092, "test/p_macro": 0.8756723615945982, "test/p_micro": 0.9599360089031092, "test/r_macro": 0.8845880939554538, "test/r_micro": 0.9599360089031092}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9072391099968662, "test/f1_macro": 0.9048168410533853, "test/f1_micro": 0.9072391099968662, "test/p_macro": 0.904024414810284, "test/p_micro": 0.9072391099968662, "test/r_macro": 0.9059444075934731, "test/r_micro": 0.9072391099968662}}
+{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9171312339912611, "test/f1_macro": 0.9144771714929822, "test/f1_micro": 0.9171312339912611, "test/p_macro": 0.9140990494403654, "test/p_micro": 0.9171312339912611, "test/r_macro": 0.9153113450496887, "test/r_micro": 0.9171312339912611}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8455399061032863, "test/f1_macro": 0.6723727069325071, "test/f1_micro": 0.8455399061032864, "test/p_macro": 0.6945535744714355, "test/p_micro": 0.8455399061032863, "test/r_macro": 0.6549786745504992, "test/r_micro": 0.8455399061032863}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6793066088840737, "test/f1_macro": 0.6676508193568021, "test/f1_micro": 0.6793066088840737, "test/p_macro": 0.6791394688186779, "test/p_micro": 0.6793066088840737, "test/r_macro": 0.6641670647971617, "test/r_micro": 0.6793066088840737}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.95875356472143, "test/f1_macro": 0.8812996852310372, "test/f1_micro": 0.95875356472143, "test/p_macro": 0.9026237933224279, "test/p_micro": 0.95875356472143, "test/r_macro": 0.8634287514873775, "test/r_micro": 0.95875356472143}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9072391099968662, "test/f1_macro": 0.9041555105317078, "test/f1_micro": 0.9072391099968662, "test/p_macro": 0.9055653859573632, "test/p_micro": 0.9072391099968662, "test/r_macro": 0.9033586700399313, "test/r_micro": 0.9072391099968662}}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "relbert_output/ckpt/iloob_semeval2012/template-d/model",
+  "_name_or_path": "roberta-large",
   "architectures": [
     "RobertaModel"
   ],

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "relbert_output/ckpt/iloob_semeval2012/template-d/model",
+  "name_or_path": "roberta-large",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,