| { | |
| "base_current_gpu_type": "NVIDIA A100-PCIE-40GB", | |
| "base_current_gpu_total_memory": 40339.3125, | |
| "base_token_generation_latency_sync": 54.08949508666992, | |
| "base_token_generation_latency_async": 53.85234020650387, | |
| "base_token_generation_throughput_sync": 0.018487878254320123, | |
| "base_token_generation_throughput_async": 0.018569295153476502, | |
| "base_token_generation_CO2_emissions": null, | |
| "base_token_generation_energy_consumption": null, | |
| "base_inference_latency_sync": 52.99066848754883, | |
| "base_inference_latency_async": 52.35116481781006, | |
| "base_inference_throughput_sync": 0.018871247118442545, | |
| "base_inference_throughput_async": 0.019101771727145913, | |
| "base_inference_CO2_emissions": null, | |
| "base_inference_energy_consumption": null, | |
| "smashed_current_gpu_type": "NVIDIA A100-PCIE-40GB", | |
| "smashed_current_gpu_total_memory": 40339.3125, | |
| "smashed_token_generation_latency_sync": 172.1177947998047, | |
| "smashed_token_generation_latency_async": 172.1174782142043, | |
| "smashed_token_generation_throughput_sync": 0.0058099745070701704, | |
| "smashed_token_generation_throughput_async": 0.005809985193691231, | |
| "smashed_token_generation_CO2_emissions": null, | |
| "smashed_token_generation_energy_consumption": null, | |
| "smashed_inference_latency_sync": 178.31505889892577, | |
| "smashed_inference_latency_async": 155.96497058868408, | |
| "smashed_inference_throughput_sync": 0.005608051311958063, | |
| "smashed_inference_throughput_async": 0.0064116961406496375, | |
| "smashed_inference_CO2_emissions": null, | |
| "smashed_inference_energy_consumption": null | |
| } |