| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.7671625496286432, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7407051823403011, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7813083270941137, | |
| 5.741719519093736e-309 | |
| ], | |
| "spearman": [ | |
| 0.7888860081302335, | |
| 4.6194e-319 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7624448096598531, | |
| 1.3012886054724166e-262 | |
| ], | |
| "spearman": [ | |
| 0.7602771570534436, | |
| 2.908594263406545e-260 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7695053681260078, | |
| "mean": 0.7703052287942033, | |
| "wmean": 0.7688678002986993 | |
| }, | |
| "spearman": { | |
| "all": 0.7564014032837215, | |
| "mean": 0.7632894491746595, | |
| "wmean": 0.7522097015584654 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7772208691267224, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6978066630711928, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7909750341980787, | |
| 2.5296588439552555e-108 | |
| ], | |
| "spearman": [ | |
| 0.7277741174377886, | |
| 1.3244056458147093e-83 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.771314072761317, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6895861583188468, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7749070438706636, | |
| "mean": 0.7798366586953728, | |
| "wmean": 0.7749819547360027 | |
| }, | |
| "spearman": { | |
| "all": 0.6952695805138401, | |
| "mean": 0.7050556462759427, | |
| "wmean": 0.6952360274580659 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 80.83, | |
| "acc": 80.73, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 88.1, | |
| "acc": 86.04, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.32, | |
| "acc": 94.97, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 89.04, | |
| "acc": 88.84, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 84.06, | |
| "acc": 85.45, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 81.07, | |
| "acc": 88.0, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 70.68, | |
| "acc": 69.28, | |
| "f1": 80.87, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.44444679689851435, | |
| 1.1761912553301756e-37 | |
| ], | |
| "spearman": [ | |
| 0.44300447101923845, | |
| 2.1416387542051887e-37 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.8646599853522132, | |
| 7.68208333670891e-226 | |
| ], | |
| "spearman": [ | |
| 0.8699014177629724, | |
| 8.381559996022476e-232 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.47046175638197285, | |
| 1.1740073098461771e-26 | |
| ], | |
| "spearman": [ | |
| 0.6114586792865501, | |
| 2.0882913296724147e-48 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7178639582155328, | |
| 9.237684853571945e-120 | |
| ], | |
| "spearman": [ | |
| 0.6658532948970585, | |
| 3.0576891351321453e-97 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.611621280135737, | |
| 2.647843518922555e-42 | |
| ], | |
| "spearman": [ | |
| 0.6250107076256665, | |
| 1.2659259057241255e-44 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6378431863774492, | |
| "mean": 0.6218107553967942, | |
| "wmean": 0.6371322047307528 | |
| }, | |
| "spearman": { | |
| "all": 0.6163833612367604, | |
| "mean": 0.6430457141182971, | |
| "wmean": 0.6480399594255531 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.5562805069869132, | |
| 9.550380482948488e-17 | |
| ], | |
| "spearman": [ | |
| 0.5837123125601075, | |
| 1.19655468411138e-18 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7823638193897661, | |
| 5.690320525335935e-156 | |
| ], | |
| "spearman": [ | |
| 0.7917891963332858, | |
| 2.60264998377283e-162 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.7816458183409305, | |
| 1.083315842299476e-116 | |
| ], | |
| "spearman": [ | |
| 0.7796359177798075, | |
| 1.0258079303272485e-115 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7255022107614845, | |
| "mean": 0.7067633815725367, | |
| "wmean": 0.7536087896347422 | |
| }, | |
| "spearman": { | |
| "all": 0.7419430794508367, | |
| "mean": 0.7183791422244002, | |
| "wmean": 0.7610261827988645 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.5089575374073992, | |
| 5.023172357070916e-31 | |
| ], | |
| "spearman": [ | |
| 0.4963233135417476, | |
| 2.317944450085771e-29 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7651252415822635, | |
| 6.366264844550544e-59 | |
| ], | |
| "spearman": [ | |
| 0.7127419493226671, | |
| 8.180911780133503e-48 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.749244660243113, | |
| 5.438116092767328e-136 | |
| ], | |
| "spearman": [ | |
| 0.7099163551457461, | |
| 5.362434306324602e-116 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8463923099739724, | |
| 7.255366412502392e-207 | |
| ], | |
| "spearman": [ | |
| 0.8037324760119565, | |
| 7.818229780597217e-171 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8194865773850172, | |
| 5.086803489175948e-183 | |
| ], | |
| "spearman": [ | |
| 0.8214740885641392, | |
| 1.2146288498994132e-184 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.7298695292458673, | |
| 1.0631684722321918e-125 | |
| ], | |
| "spearman": [ | |
| 0.6599928354527725, | |
| 5.5057855924436005e-95 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7165932905627189, | |
| "mean": 0.7365126426396054, | |
| "wmean": 0.7512835391850631 | |
| }, | |
| "spearman": { | |
| "all": 0.6677462039090557, | |
| "mean": 0.7006968363398381, | |
| "wmean": 0.7156013046057461 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.7110270112832033, | |
| 5.18878369534419e-59 | |
| ], | |
| "spearman": [ | |
| 0.7180037029082824, | |
| 1.1507346620419953e-60 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.685403182699777, | |
| 3.763475670820268e-105 | |
| ], | |
| "spearman": [ | |
| 0.6892806170636011, | |
| 8.551548507203923e-107 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.7406221526966811, | |
| 2.1633293082135999e-66 | |
| ], | |
| "spearman": [ | |
| 0.7616812212998276, | |
| 2.7456783665122726e-72 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.8065884284038872, | |
| 5.856919545907542e-173 | |
| ], | |
| "spearman": [ | |
| 0.8065628750527286, | |
| 6.121300087095471e-173 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8734727058865335, | |
| 5.133694173289855e-236 | |
| ], | |
| "spearman": [ | |
| 0.8800760762065849, | |
| 3.7371804165528405e-244 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7786729190560969, | |
| "mean": 0.7634226961940165, | |
| "wmean": 0.7728222247450349 | |
| }, | |
| "spearman": { | |
| "all": 0.7899640319998164, | |
| "mean": 0.7711208985062049, | |
| "wmean": 0.7789405076067425 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.6932635451048339, | |
| 1.0228592987318116e-37 | |
| ], | |
| "spearman": [ | |
| 0.7021547155024479, | |
| 4.8011478191297344e-39 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7691218196080909, | |
| 6.220546451256778e-50 | |
| ], | |
| "spearman": [ | |
| 0.779843578127762, | |
| 3.643403874299668e-52 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.8038816275257141, | |
| 2.350047547816728e-53 | |
| ], | |
| "spearman": [ | |
| 0.8153039508029369, | |
| 5.088887617848758e-56 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.8113538363870572, | |
| 2.3510310736845822e-58 | |
| ], | |
| "spearman": [ | |
| 0.8303201531350315, | |
| 2.1906668671246972e-63 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7802351353605056, | |
| 4.64479928094145e-44 | |
| ], | |
| "spearman": [ | |
| 0.7884164498209355, | |
| 1.4564689988934998e-45 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7704350276676698, | |
| "mean": 0.7715711927972403, | |
| "wmean": 0.7702635136920273 | |
| }, | |
| "spearman": { | |
| "all": 0.7846388532549995, | |
| "mean": 0.7832077694778227, | |
| "wmean": 0.7819775824229623 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.6163833612367604, | |
| "eval_senteval-STS13": 0.7419430794508367, | |
| "eval_senteval-STS14": 0.6677462039090557, | |
| "eval_senteval-STS15": 0.7899640319998164, | |
| "eval_senteval-STS16": 0.7846388532549995, | |
| "eval_senteval-STSBenchmark": 0.7564014032837215, | |
| "eval_senteval-SICKRelatedness": 0.6952695805138401, | |
| "eval_senteval-avg_sts_7": 0.7217637876641472, | |
| "eval_senteval-MR": 80.83, | |
| "eval_senteval-CR": 88.1, | |
| "eval_senteval-SUBJ": 95.32, | |
| "eval_senteval-MPQA": 89.04, | |
| "eval_senteval-SST2": 84.06, | |
| "eval_senteval-TREC": 81.07, | |
| "eval_senteval-MRPC": 70.68, | |
| "eval_senteval-avg_transfer": 84.15714285714287 | |
| } | |