Spaces:
Running
on
Zero
Running
on
Zero
File size: 2,298 Bytes
9fa3d89 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
import pandas as pd
import json
import argparse
def load_jsonl(f):
lines = open(f, encoding='utf-8').readlines()
lines = [x.strip() for x in lines]
if lines[-1] == '':
lines = lines[:-1]
data = [json.loads(x) for x in lines]
return data
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--results_file", type=str, default="cv-bench_answer.jsonl")
args = parser.parse_args()
answers = load_jsonl(args.results_file)
data = {
"source": [],
"result": [],
"task": [],
}
import re
for a in answers:
data["source"].append(a["source"][0])
if "(" in a["prediction"]:
match = re.search(r'\(([A-Z])\)', a["prediction"])
if match:
pred = "(" + match.group(1) + ")"
else:
pred = "(" + a["prediction"][0] + ")"
data["result"].append(pred == a["answer"][0])
data["task"].append(a["task"][0])
df = pd.DataFrame(data)
def calculate_accuracy(df, source):
source_df = df[df['source'] == source]
accuracy = (source_df['result']).mean()
return accuracy
def calculate_task_accuracy(df, task):
source_df = df[df['task'] == task]
accuracy = (source_df['result']).mean()
return accuracy
accuracy_2d_ade = calculate_accuracy(df, 'ADE20K')
accuracy_2d_coco = calculate_accuracy(df, 'COCO')
accuracy_3d_omni = calculate_accuracy(df, 'Omni3D')
tasks = ["Count", "Depth", "Relation", "Distance"]
scores = {}
accuracy_2d = (accuracy_2d_ade + accuracy_2d_coco) / 2
accuracy_3d = accuracy_3d_omni
combined_accuracy = (accuracy_2d + accuracy_3d) / 2
scores["Overall"] = combined_accuracy
scores["3D"] = accuracy_3d
scores["2D"] = accuracy_2d
for t in tasks:
accuracy = calculate_task_accuracy(df, t)
scores[t] = accuracy
print("\n=========================CV-Bench Scores===============================")
for key, value in scores.items():
print(f"{key} -> {value}")
print("================================================================")
with open(args.results_file.replace('.jsonl', '_score.json'), "w") as f:
json.dump(scores, f, indent=2) |