Spaces:
Running
on
Zero
Running
on
Zero
| import fire | |
| import time | |
| import json | |
| from collections import defaultdict | |
| from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR | |
| from .utils import detect_language, get_time_stamp_from_date, get_input_image_path, load_image_from_path | |
| from tqdm import tqdm | |
| VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote", "chat"] | |
| def remove_html(raw): | |
| if raw.startswith("<h3>"): | |
| return raw[raw.find(": ") + 2 : -len("</h3>\n")] | |
| if raw.startswith("### Model A: ") or raw.startswith("### Model B: "): | |
| return raw[13:] | |
| return raw | |
| def read_file(filename): | |
| data = [] | |
| for retry in range(5): | |
| try: | |
| # lines = open(filename).readlines() | |
| for l in open(filename): | |
| row = json.loads(l) | |
| if row["type"] in VOTES: | |
| data.append(row) | |
| break | |
| except FileNotFoundError: | |
| time.sleep(2) | |
| return data | |
| def read_file_parallel(log_files, num_threads=16): | |
| data_all = [] | |
| from multiprocessing import Pool | |
| with Pool(num_threads) as p: | |
| ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files))) | |
| for ret in ret_all: | |
| data_all.extend(ret) | |
| return data_all | |
| def num_tokens(s:str): | |
| if s is None: | |
| return 0 | |
| return len(s) / 4 | |
| def main( | |
| ): | |
| log_files = get_log_files() | |
| data = read_file_parallel(log_files) | |
| all_model_counts = defaultdict(int) | |
| all_model_input_tokens_counts = defaultdict(list) | |
| all_model_output_tokens_counts = defaultdict(list) | |
| all_model_image_sizes = defaultdict(list) | |
| chat_battle_counts = defaultdict(int) | |
| for row in tqdm(data, desc="counting"): | |
| if row['type'] == "chat": | |
| chat_battle_counts["chat"] += 1 | |
| all_model_counts[row['model']] += 1 | |
| tstamp = row["tstamp"] | |
| conv_id = row["state"]["conv_id"] | |
| image = load_image_from_path(get_input_image_path(tstamp, conv_id)) | |
| if image is None: | |
| image_size = None | |
| else: | |
| image_size = load_image_from_path(get_input_image_path(tstamp, conv_id)).size | |
| all_model_image_sizes[row['model']].append(image_size) | |
| try: | |
| for message in row["state"]["messages"][row["state"]["offset"] :: 2]: | |
| all_model_input_tokens_counts[row['model']].append(num_tokens(message[1])) | |
| for message in row["state"]["messages"][row["state"]["offset"] + 1 :: 2]: | |
| all_model_output_tokens_counts[row['model']].append(num_tokens(message[1])) | |
| except Exception as e: | |
| print(row) | |
| raise e | |
| else: | |
| chat_battle_counts[row['type']] += 1 | |
| if row["models"][0] is None or row["models"][1] is None: | |
| continue | |
| # Resolve model names | |
| models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])] | |
| if "model_name" in row["states"][0]: | |
| models_hidden = [ | |
| row["states"][0]["model_name"], | |
| row["states"][1]["model_name"], | |
| ] | |
| if models_hidden[0] is None: | |
| models_hidden = models_public | |
| else: | |
| models_hidden = models_public | |
| if (models_public[0] == "" and models_public[1] != "") or ( | |
| models_public[1] == "" and models_public[0] != "" | |
| ): | |
| continue | |
| if models_public[0] == "" or models_public[0] == "Model A": | |
| anony = True | |
| models = models_hidden | |
| else: | |
| anony = False | |
| models = models_public | |
| if not models_public == models_hidden: | |
| continue | |
| all_model_counts[models[0]] += 1 | |
| all_model_counts[models[1]] += 1 | |
| tstamp = row["tstamp"] | |
| conv_id1 = row["states"][0]["conv_id"] | |
| conv_id2 = row["states"][1]["conv_id"] | |
| image1 = load_image_from_path(get_input_image_path(tstamp, conv_id1)) | |
| image2 = load_image_from_path(get_input_image_path(tstamp, conv_id2)) | |
| all_model_image_sizes[models[0]].append(None if image1 is None else image1.size) | |
| all_model_image_sizes[models[1]].append(None if image2 is None else image2.size) | |
| for message in row["states"][0]["messages"][row["states"][0]["offset"] :: 2]: | |
| all_model_input_tokens_counts[models[0]].append(num_tokens(message[1])) | |
| for message in row["states"][0]["messages"][row["states"][0]["offset"] + 1 :: 2]: | |
| all_model_output_tokens_counts[models[0]].append(num_tokens(message[1])) | |
| for message in row["states"][1]["messages"][row["states"][1]["offset"] :: 2]: | |
| all_model_input_tokens_counts[models[1]].append(num_tokens(message[1])) | |
| for message in row["states"][1]["messages"][row["states"][1]["offset"] + 1 :: 2]: | |
| all_model_output_tokens_counts[models[1]].append(num_tokens(message[1])) | |
| print("### Chat battle counts (requests)") | |
| print(json.dumps(chat_battle_counts, indent=4)) | |
| print("### Model counts (requests)") | |
| print(json.dumps(all_model_counts, indent=4)) | |
| print("### Model Avg input tokens counts (tokens)") | |
| average_input_tokens_counts = {} | |
| for model, counts in all_model_input_tokens_counts.items(): | |
| average_input_tokens_counts[model] = sum(counts) / len(counts) | |
| print(json.dumps(average_input_tokens_counts, indent=4)) | |
| print("### Model AVg output tokens counts (tokens)") | |
| average_output_tokens_counts = {} | |
| for model, counts in all_model_output_tokens_counts.items(): | |
| average_output_tokens_counts[model] = sum(counts) / len(counts) | |
| print(json.dumps(average_output_tokens_counts, indent=4)) | |
| print("### Model Avg image sizes (height, width)") | |
| average_image_sizes = {} | |
| for model, sizes in all_model_image_sizes.items(): | |
| avg_height = sum([size[0] for size in sizes if size is not None]) / len(sizes) | |
| avg_width = sum([size[1] for size in sizes if size is not None]) / len(sizes) | |
| average_image_sizes[model] = (avg_height, avg_width) | |
| print(json.dumps(average_image_sizes, indent=4)) | |
| print("### GPT-4V estimated cost (USD)") | |
| gpt_4v_name = "gpt-4-vision-preview" | |
| gpt_4v_cost = {} | |
| gpt_4v_cost['input'] = sum(all_model_input_tokens_counts[gpt_4v_name]) / 1000 * 0.01 | |
| gpt_4v_cost['output'] = sum(all_model_output_tokens_counts[gpt_4v_name]) / 1000 * 0.03 | |
| all_image_cost = 0 | |
| for size in all_model_image_sizes[gpt_4v_name]: | |
| if size is None: | |
| continue | |
| all_image_tokens = (size[0] // 512 + 1) * (size[1] // 512 + 1) * 170 + 85 | |
| all_image_cost += all_image_tokens / 1000 * 0.01 | |
| gpt_4v_cost['image'] = all_image_cost | |
| print(json.dumps(gpt_4v_cost, indent=4)) | |
| if __name__ == "__main__": | |
| fire.Fire(main) |