Upload artifacts (small scale)

small_scale/results.jsonl

@@ -1,40 +1,40 @@
-{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.24979457682826622, "acc5": 0.5819227608874281, "mean_per_class_recall": 0.2607347891363351, "main_metric": 0.2607347891363351}}
-{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.4506, "acc5": 0.9493, "mean_per_class_recall": 0.45059999999999995, "main_metric": 0.4506}}
-{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.1965, "acc5": 0.4894, "mean_per_class_recall": 0.19649999999999998, "main_metric": 0.1965}}
-{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.15726666666666667, "acc5": 0.6342666666666666, "mean_per_class_recall": 0.15953967196009344, "main_metric": 0.15726666666666667}}
-{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.2006, "acc5": 0.9186666666666666, "mean_per_class_recall": 0.16727098554529338, "main_metric": 0.2006}}
-{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.013080568720379148, "acc5": 0.055402843601895735, "mean_per_class_recall": 0.013080568720379148, "main_metric": 0.013080568720379148}}
-{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.07659574468085106, "acc5": 0.21117021276595746, "mean_per_class_recall": 0.07659574468085105, "main_metric": 0.07659574468085106}}
-{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.19907407407407407, "acc5": 0.5988888888888889, "mean_per_class_recall": 0.19027615983922566, "main_metric": 0.19907407407407407}}
-{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.011401140114011402, "acc5": 0.052205220522052204, "mean_per_class_recall": 0.011274509803921568, "main_metric": 0.011274509803921568}}
-{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.08550495049504951, "acc5": 0.26601980198019803, "mean_per_class_recall": 0.08550495049504948, "main_metric": 0.08550495049504951}}
-{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.06991290577988915, "acc5": 0.25637371338083925, "mean_per_class_recall": 0.07338460424532214, "main_metric": 0.06991290577988915}}
-{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.05366, "acc5": 0.15126, "mean_per_class_recall": 0.05362000000000001, "main_metric": 0.05366}}
-{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.02273575821886852, "acc5": 0.06551514079663581, "mean_per_class_recall": 0.02275921568627451, "main_metric": 0.02273575821886852}}
-{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.0445, "acc5": 0.1317, "mean_per_class_recall": 0.0445, "main_metric": 0.0445}}
-{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.017333333333333333, "acc5": 0.07586666666666667, "mean_per_class_recall": 0.023642903076918412, "main_metric": 0.017333333333333333}}
-{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.1475, "acc5": 0.3475, "mean_per_class_recall": 0.13387725459954564, "main_metric": 0.1475}}
-{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.07233333333333333, "acc5": 0.2003, "mean_per_class_recall": 0.06751276092804365, "main_metric": 0.07233333333333333}}
-{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.3656821378340366, "acc5": null, "mean_per_class_recall": 0.24995096313973253, "main_metric": 0.3656821378340366}}
-{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.1331, "acc5": 0.4956, "mean_per_class_recall": 0.1333985322198148, "main_metric": 0.1331}}
-{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.05356950576074082, "acc5": 0.16797674168192098, "mean_per_class_recall": 0.05370265317041002, "main_metric": 0.05356950576074082}}
-{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.052854122621564484, "acc5": 0.17872824849569036, "mean_per_class_recall": 0.060368449981373744, "main_metric": 0.060368449981373744}}
-{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.06514036522213137, "acc5": 0.25947124557100026, "mean_per_class_recall": 0.0648398611809775, "main_metric": 0.0648398611809775}}
-{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.33673878205128205, "acc5": 0.7154113247863247, "mean_per_class_recall": 0.3711031340800693, "main_metric": 0.33673878205128205}}
-{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.50262451171875, "acc5": null, "mean_per_class_recall": 0.5028368966244456, "main_metric": 0.50262451171875}}
-{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.5024711696869851, "acc5": null, "mean_per_class_recall": 0.5024589630015632, "main_metric": 0.5024711696869851}}
-{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.0838095238095238, "acc5": 0.2995238095238095, "mean_per_class_recall": 0.08519276450001373, "main_metric": 0.0838095238095238}}
-{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.02885213281930108, "acc5": 0.12958587240392985, "mean_per_class_recall": 0.02898964020512957, "main_metric": 0.02885213281930108}}
-{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.540875, "acc5": 0.966, "mean_per_class_recall": 0.540875, "main_metric": 0.540875}}
-{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.11743016348823951, "acc5": 0.304696838736966, "mean_per_class_recall": 0.0888188705259041, "main_metric": 0.11743016348823951}}
-{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.06691763982790412, "acc5": 0.46377535341118625, "mean_per_class_recall": 0.08750327584447123, "main_metric": 0.06691763982790412}}
-{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.031599998474121094, "text_retrieval_recall@1": 0.04699999839067459, "image_retrieval_recall@5": 0.10819999873638153, "text_retrieval_recall@5": 0.16300000250339508, "image_retrieval_recall@10": 0.16179999709129333, "text_retrieval_recall@10": 0.2290000021457672, "mean_recall@1": 0.03929999843239784, "main_metric": 0.03929999843239784}}
-{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.017872851341962814, "text_retrieval_recall@1": 0.028999999165534973, "image_retrieval_recall@5": 0.058096762746572495, "text_retrieval_recall@5": 0.09059999883174896, "image_retrieval_recall@10": 0.09272290766239166, "text_retrieval_recall@10": 0.14259999990463257, "mean_recall@1": 0.023436425253748894, "main_metric": 0.023436425253748894}}
-{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.3880036887053445, "jaccard_score_5": 0.44707070707070706, "jaccard_score_6": 0.3907847372210223, "jaccard_score_10": 0.3128549072211044, "jaccard_score_12": 0.2716065865598576, "jaccard_score_5-6": 0.4182109019318322, "jaccard_score_10-12": 0.29218244674919147, "main_metric": 0.29218244674919147}}
-{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.03834918557640626, "acc5": 0.1444696314645603, "mean_per_class_recall": 0.0016871072003287386, "acc_avg": 0.03834918513894081, "recall-macro_all": 0.0016871072003287386, "F1-macro_all": 0.0021504402084754747, "main_metric": 0.0021504402084754747}}
-{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.5004467749900063, "acc5": null, "mean_per_class_recall": 0.5004467749900063, "acc_avg": 0.5004467964172363, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.008923884481191635, "count_slide:20": 3810.0, "acc_slide:21": 0.0035192202776670456, "count_slide:21": 3694.0, "acc_slide:22": 0.5869625806808472, "count_slide:22": 7210.0, "acc_slide:23": 0.5225037932395935, "count_slide:23": 5288.0, "acc_slide:24": 0.02407143823802471, "count_slide:24": 7727.0, "acc_slide:25": 0.23834794759750366, "count_slide:25": 4334.0, "acc_slide:26": 0.13053734600543976, "count_slide:26": 3815.0, "acc_slide:27": 0.024363476783037186, "count_slide:27": 4556.0, "acc_slide:28": 0.847230076789856, "count_slide:28": 31878.0, "acc_slide:29": 0.5247998833656311, "count_slide:29": 12742.0, "acc_wg": 0.0035192202776670456, "main_metric": 0.5004467749900063}}
-{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.02397322236294554, "acc5": 0.10932694047403654, "mean_per_class_recall": 0.024030038034377383, "acc_avg": 0.023973222821950912, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.0240616574883461, "count_year:14": 15959.0, "acc_year:15": 0.023743698373436928, "count_year:15": 6149.0, "acc_worst_year": 0.023743698373436928, "acc_region:0": 0.021358050405979156, "count_region:0": 4963.0, "acc_region:1": 0.02645954303443432, "count_region:1": 5858.0, "acc_region:2": 0.010798303410410881, "count_region:2": 2593.0, "acc_region:3": 0.027542373165488243, "count_region:3": 8024.0, "acc_region:4": 0.030030030757188797, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
-{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.15729374821581502, "acc5": 0.36882671995432487, "mean_per_class_recall": 0.15421438921862013, "acc_top5_avg": 0.3688267171382904, "acc_top5_income_ds:0": 0.26285046339035034, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.3212669789791107, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.4150943458080292, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.47447794675827026, "count_income_ds:3": 862.0, "acc_top5_wg": 0.26285046339035034, "main_metric": 0.26285046339035034}}
-{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.3481742472773863, "acc5": 0.6945067264573991, "mean_per_class_recall": 0.3421322340328487, "acc_avg": 0.34817424416542053, "acc_region:0": 0.3110647201538086, "count_region:0": 2395.0, "acc_region:1": 0.3517412841320038, "count_region:1": 2010.0, "acc_region:2": 0.3443085551261902, "count_region:2": 2126.0, "acc_region:3": 0.32922443747520447, "count_region:3": 1947.0, "acc_region:4": 0.3784860670566559, "count_region:4": 1757.0, "acc_region:5": 0.38082557916641235, "count_region:5": 2253.0, "acc_wg": 0.3110647201538086, "main_metric": 0.3110647201538086}}
-{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7228409647941589, "acc_race_race_binary:0": 0.24028776586055756, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.8362836837768555, "count_race_binary:1": 8869.0, "acc_race_wg": 0.24028776586055756, "acc_gender_avg": 0.4997261166572571, "acc_gender_race_binary:0": 0.48153477907180786, "acc_gender_race_binary:1": 0.5040026903152466, "acc_gender_wg": 0.48153477907180786, "acc_age_avg": 0.03925506770610809, "acc_age_race_binary:0": 0.029736211523413658, "acc_age_race_binary:1": 0.04149283841252327, "acc_age_wg": 0.029736211523413658, "acc_gender_x_avg": 0.4997261166572571, "acc_gender_x_race:0_gender:0": 0.21026283502578735, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.8586525917053223, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.0730837807059288, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.9574247002601624, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.13678619265556335, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.897771954536438, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.10214375704526901, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.9734939932823181, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.09348093718290329, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.9570707082748413, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.1210884377360344, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.9205882549285889, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.08622908592224121, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.9547218680381775, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.0730837807059288, "toxicity_crime_avg": 0.013237173669040203, "toxicity_crime_race:0": 0.016066838055849075, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.016786571592092514, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.005936675239354372, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.01601971685886383, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.014888337813317776, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.013427562080323696, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.008387097157537937, "count_race:6": 1550.0, "toxicity_crime_wg": 0.005936675239354372, "toxicity_nonhuman_avg": 0.2908526659011841, "toxicity_nonhuman_race:0": 0.5109254717826843, "toxicity_nonhuman_race:1": 0.20479616522789001, "toxicity_nonhuman_race:2": 0.3713720440864563, "toxicity_nonhuman_race:3": 0.2156500369310379, "toxicity_nonhuman_race:4": 0.28205129504203796, "toxicity_nonhuman_race:5": 0.29045936465263367, "toxicity_nonhuman_race:6": 0.19290322065353394, "toxicity_nonhuman_wg": 0.19290322065353394, "main_metric": null}}
-{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5900518894195557, "acc_race_race_binary:0": 0.4640730321407318, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.6832024455070496, "count_race_binary:1": 13627.0, "acc_race_wg": 0.4640730321407318, "acc_gender_avg": 0.5017508268356323, "acc_gender_race_binary:0": 0.4668519198894501, "acc_gender_race_binary:1": 0.5275555849075317, "acc_gender_wg": 0.4668519198894501, "acc_age_avg": 0.09872168302536011, "acc_age_race_binary:0": 0.11740770190954208, "acc_age_race_binary:1": 0.08490496873855591, "acc_age_wg": 0.08490496873855591, "acc_gender_x_avg": 0.5017508268356323, "acc_gender_x_race:0_gender:0": 0.20060396194458008, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.9153079986572266, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.02794010192155838, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.9893478155136108, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.10216718167066574, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.9603267312049866, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.035555556416511536, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.9784830808639526, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.043421052396297455, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.9849785566329956, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.02794010192155838, "toxicity_crime_avg": 0.0032907226122915745, "toxicity_crime_race:0": 0.004418912809342146, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.0032751092221587896, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.0035220126155763865, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.0020384390372782946, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.002364066196605563, "count_race:4": 1692.0, "toxicity_crime_wg": 0.0020384390372782946, "toxicity_nonhuman_avg": 0.20634518563747406, "toxicity_nonhuman_race:0": 0.42576226592063904, "toxicity_nonhuman_race:1": 0.13110361993312836, "toxicity_nonhuman_race:2": 0.25182390213012695, "toxicity_nonhuman_race:3": 0.12317996472120285, "toxicity_nonhuman_race:4": 0.12943261861801147, "toxicity_nonhuman_wg": 0.12317996472120285, "main_metric": null}}
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.2706655710764174, "acc5": 0.6312243221035333, "mean_per_class_recall": 0.2676443272110481, "main_metric": 0.2676443272110481}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.5072, "acc5": 0.9584, "mean_per_class_recall": 0.5072, "main_metric": 0.5072}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.2404, "acc5": 0.5432, "mean_per_class_recall": 0.2404, "main_metric": 0.2404}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.1404, "acc5": 0.6305333333333333, "mean_per_class_recall": 0.13868995715368526, "main_metric": 0.1404}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.23206666666666667, "acc5": 0.9186666666666666, "mean_per_class_recall": 0.20381403598840905, "main_metric": 0.23206666666666667}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.016066350710900474, "acc5": 0.060284360189573456, "mean_per_class_recall": 0.016066350710900477, "main_metric": 0.016066350710900474}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.08138297872340426, "acc5": 0.2425531914893617, "mean_per_class_recall": 0.08138297872340425, "main_metric": 0.08138297872340426}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.23333333333333334, "acc5": 0.5835185185185185, "mean_per_class_recall": 0.22026093728456045, "main_metric": 0.23333333333333334}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.012001200120012, "acc5": 0.04950495049504951, "mean_per_class_recall": 0.011862745098039217, "main_metric": 0.011862745098039217}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.08166336633663367, "acc5": 0.24724752475247525, "mean_per_class_recall": 0.08166336633663368, "main_metric": 0.08166336633663367}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.08400633412509897, "acc5": 0.33460015835312745, "mean_per_class_recall": 0.07777214088386991, "main_metric": 0.08400633412509897}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.0593, "acc5": 0.16624, "mean_per_class_recall": 0.05928, "main_metric": 0.0593}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.023423529642948377, "acc5": 0.06783391302639077, "mean_per_class_recall": 0.02346156862745098, "main_metric": 0.023423529642948377}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.0506, "acc5": 0.1503, "mean_per_class_recall": 0.0506, "main_metric": 0.0506}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.0196, "acc5": 0.0824, "mean_per_class_recall": 0.028184268142942587, "main_metric": 0.0196}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.1475, "acc5": 0.3575, "mean_per_class_recall": 0.13922824636663644, "main_metric": 0.1475}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.07766666666666666, "acc5": 0.2095, "mean_per_class_recall": 0.07430012433755585, "main_metric": 0.07766666666666666}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.3080168776371308, "acc5": null, "mean_per_class_recall": 0.22749747304582207, "main_metric": 0.3080168776371308}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.1118, "acc5": 0.6075, "mean_per_class_recall": 0.11067492235694876, "main_metric": 0.1118}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.05776892430278884, "acc5": 0.17739851405190052, "mean_per_class_recall": 0.05837677793548174, "main_metric": 0.05776892430278884}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.053504634899983736, "acc5": 0.17856562042608554, "mean_per_class_recall": 0.05238308678879251, "main_metric": 0.05238308678879251}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.09621150177159989, "acc5": 0.27527936767511585, "mean_per_class_recall": 0.09577083891376224, "main_metric": 0.09577083891376224}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.3466880341880342, "acc5": 0.7282986111111112, "mean_per_class_recall": 0.3966012009824842, "main_metric": 0.3466880341880342}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.505218505859375, "acc5": null, "mean_per_class_recall": 0.5054298090415472, "main_metric": 0.505218505859375}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.500823723228995, "acc5": null, "mean_per_class_recall": 0.5, "main_metric": 0.500823723228995}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.08619047619047619, "acc5": 0.2704761904761905, "mean_per_class_recall": 0.08790795315159007, "main_metric": 0.08619047619047619}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.02885213281930108, "acc5": 0.11441363014550429, "mean_per_class_recall": 0.028763787052531877, "main_metric": 0.02885213281930108}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.5845, "acc5": 0.96975, "mean_per_class_recall": 0.5845, "main_metric": 0.5845}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.12646891148831307, "acc5": 0.3263604097320558, "mean_per_class_recall": 0.10255553291990159, "main_metric": 0.12646891148831307}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.08892901044867856, "acc5": 0.506338352796558, "mean_per_class_recall": 0.10773154393098514, "main_metric": 0.08892901044867856}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.03539999946951866, "text_retrieval_recall@1": 0.06599999964237213, "image_retrieval_recall@5": 0.11900000274181366, "text_retrieval_recall@5": 0.17800000309944153, "image_retrieval_recall@10": 0.18320000171661377, "text_retrieval_recall@10": 0.24699999392032623, "mean_recall@1": 0.050699999555945396, "main_metric": 0.050699999555945396}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.021791283041238785, "text_retrieval_recall@1": 0.03319999948143959, "image_retrieval_recall@5": 0.07025189697742462, "text_retrieval_recall@5": 0.10360000282526016, "image_retrieval_recall@10": 0.11035585403442383, "text_retrieval_recall@10": 0.1526000052690506, "mean_recall@1": 0.027495641261339188, "main_metric": 0.027495641261339188}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.4091041524664875, "jaccard_score_5": 0.4645454545454545, "jaccard_score_6": 0.4165826733861292, "jaccard_score_10": 0.3338810641627543, "jaccard_score_12": 0.2887182910547397, "jaccard_score_5-6": 0.43995324227882365, "jaccard_score_10-12": 0.311246793799487, "main_metric": 0.311246793799487}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.03951765558178121, "acc5": 0.11032693790750392, "mean_per_class_recall": 0.0034132111438215846, "acc_avg": 0.03951765596866608, "recall-macro_all": 0.0034132111438215846, "F1-macro_all": 0.0021830244799491373, "main_metric": 0.0021830244799491373}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.5, "acc5": null, "mean_per_class_recall": 0.5, "acc_avg": 0.5, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.007086614146828651, "count_slide:20": 3810.0, "acc_slide:21": 0.002707092557102442, "count_slide:21": 3694.0, "acc_slide:22": 0.5865464806556702, "count_slide:22": 7210.0, "acc_slide:23": 0.5221255421638489, "count_slide:23": 5288.0, "acc_slide:24": 0.023812605068087578, "count_slide:24": 7727.0, "acc_slide:25": 0.23742501437664032, "count_slide:25": 4334.0, "acc_slide:26": 0.12975098192691803, "count_slide:26": 3815.0, "acc_slide:27": 0.024143986403942108, "count_slide:27": 4556.0, "acc_slide:28": 0.8469163775444031, "count_slide:28": 31878.0, "acc_slide:29": 0.5245644450187683, "count_slide:29": 12742.0, "acc_wg": 0.002707092557102442, "main_metric": 0.5}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.022797177492310475, "acc5": 0.10828659308847476, "mean_per_class_recall": 0.025020635806308363, "acc_avg": 0.022797176614403725, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.024876246228814125, "count_year:14": 15959.0, "acc_year:15": 0.017401203513145447, "count_year:15": 6149.0, "acc_worst_year": 0.017401203513145447, "acc_region:0": 0.01652226410806179, "count_region:0": 4963.0, "acc_region:1": 0.033287812024354935, "count_region:1": 5858.0, "acc_region:2": 0.011569610796868801, "count_region:2": 2593.0, "acc_region:3": 0.02068793587386608, "count_region:3": 8024.0, "acc_region:4": 0.04654654487967491, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.15415358264344847, "acc5": 0.39366257493576934, "mean_per_class_recall": 0.1483581370529709, "acc_top5_avg": 0.39366257190704346, "acc_top5_income_ds:0": 0.2535046637058258, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.3484162986278534, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.4461709260940552, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.5243619680404663, "count_income_ds:3": 862.0, "acc_top5_wg": 0.2535046637058258, "main_metric": 0.2535046637058258}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.37083600256245997, "acc5": 0.7090807174887892, "mean_per_class_recall": 0.3650473412141806, "acc_avg": 0.3708359897136688, "acc_region:0": 0.3315240144729614, "count_region:0": 2395.0, "acc_region:1": 0.37960198521614075, "count_region:1": 2010.0, "acc_region:2": 0.37253057956695557, "count_region:2": 2126.0, "acc_region:3": 0.3430919349193573, "count_region:3": 1947.0, "acc_region:4": 0.40466704964637756, "count_region:4": 1757.0, "acc_region:5": 0.4007989466190338, "count_region:5": 2253.0, "acc_wg": 0.3315240144729614, "main_metric": 0.3315240144729614}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7731422185897827, "acc_race_race_binary:0": 0.1088729053735733, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.929304301738739, "count_race_binary:1": 8869.0, "acc_race_wg": 0.1088729053735733, "acc_gender_avg": 0.5082162022590637, "acc_gender_race_binary:0": 0.4853717088699341, "acc_gender_race_binary:1": 0.5135866403579712, "acc_gender_wg": 0.4853717088699341, "acc_age_avg": 0.14798247814178467, "acc_age_race_binary:0": 0.16163069009780884, "acc_age_race_binary:1": 0.144773930311203, "acc_age_wg": 0.144773930311203, "acc_gender_x_avg": 0.5082162022590637, "acc_gender_x_race:0_gender:0": 0.24030037224292755, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.8401585221290588, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.08645275980234146, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.9501557350158691, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.14873838424682617, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.9384010434150696, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.11349306255578995, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.9614458084106445, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.1045510470867157, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.9595959782600403, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.12244898080825806, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.9544117450714111, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.08365508168935776, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.9598965048789978, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.08365508168935776, "toxicity_crime_avg": 0.14779989421367645, "toxicity_crime_race:0": 0.10475578159093857, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.2028777003288269, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.10488126426935196, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.16574245691299438, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.14143919944763184, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.14275617897510529, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.14967742562294006, "count_race:6": 1550.0, "toxicity_crime_wg": 0.10475578159093857, "toxicity_nonhuman_avg": 0.3110279440879822, "toxicity_nonhuman_race:0": 0.5134961605072021, "toxicity_nonhuman_race:1": 0.21870504319667816, "toxicity_nonhuman_race:2": 0.3924802243709564, "toxicity_nonhuman_race:3": 0.25446704030036926, "toxicity_nonhuman_race:4": 0.28535979986190796, "toxicity_nonhuman_race:5": 0.3024734854698181, "toxicity_nonhuman_race:6": 0.23935483396053314, "toxicity_nonhuman_wg": 0.21870504319667816, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5751170516014099, "acc_race_race_binary:0": 0.21288210153579712, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.8429588079452515, "count_race_binary:1": 13627.0, "acc_race_wg": 0.21288210153579712, "acc_gender_avg": 0.49630847573280334, "acc_gender_race_binary:0": 0.46963080763816833, "acc_gender_race_binary:1": 0.5160343647003174, "acc_gender_wg": 0.46963080763816833, "acc_age_avg": 0.2622452974319458, "acc_age_race_binary:0": 0.17993251979351044, "acc_age_race_binary:1": 0.3231085240840912, "acc_age_wg": 0.17993251979351044, "acc_gender_x_avg": 0.49630847573280334, "acc_gender_x_race:0_gender:0": 0.1389128565788269, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.9384058117866516, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.030496712774038315, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.9923912882804871, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.06413091719150543, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.9719953536987305, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.02539682574570179, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.9860140085220337, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.04210526496171951, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.9892703890800476, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.02539682574570179, "toxicity_crime_avg": 0.10007172077894211, "toxicity_crime_race:0": 0.0720282793045044, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.11780468374490738, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.09685534238815308, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.08474082499742508, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.10815602540969849, "count_race:4": 1692.0, "toxicity_crime_wg": 0.0720282793045044, "toxicity_nonhuman_avg": 0.2186221182346344, "toxicity_nonhuman_race:0": 0.45802032947540283, "toxicity_nonhuman_race:1": 0.1283247321844101, "toxicity_nonhuman_race:2": 0.28050315380096436, "toxicity_nonhuman_race:3": 0.13249854743480682, "toxicity_nonhuman_race:4": 0.14539006352424622, "toxicity_nonhuman_wg": 0.1283247321844101, "main_metric": null}}

small_scale/samples/baseline_ws_0.3.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2abb9a3582b64501446ed3d8b055b8360a6231f9f687e45aed6dc83eeeb2bbd
+size 51256096