diff --git "a/runs.json" "b/runs.json" new file mode 100644--- /dev/null +++ "b/runs.json" @@ -0,0 +1,2406 @@ +[ + { + "model_name_or_path": "distilbert-base-uncased-distilled-squad", + "task": "question-answering", + "task_args": null, + "dataset": { + "path": "squad", + "eval_split": "validation", + "data_keys": { + "question": "question", + "context": "context" + }, + "ref_keys": [ + "answers" + ], + "name": null, + "calibration_split": "train" + }, + "quantization_approach": "static", + "operators_to_quantize": [ + "Add" + ], + "node_exclusion": [ + "layernorm", + "gelu", + "residual", + "gather", + "softmax" + ], + "aware_training": false, + "per_channel": false, + "calibration": { + "method": "minmax", + "num_calibration_samples": 100, + "calibration_histogram_percentile": null, + "calibration_moving_average": null, + "calibration_moving_average_constant": null + }, + "framework": "onnxruntime", + "framework_args": { + "opset": 11, + "optimization_level": 1 + }, + "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3100.278\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", + "versions": { + "transformers": "4.20.1", + "optimum": "1.2.3.dev0", + "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" + }, + "evaluation": { + "time": [ + { + "batch_size": 8, + "input_length": 32, + "baseline": { + "nb_forwards": 126, + "throughput": 8.4, + "latency_mean": 119.09526233333332, + "latency_std": 0.7552068299080471, + "latency_50": 119.159921, + "latency_90": 119.811235, + "latency_95": 120.0762565, + "latency_99": 121.2652685, + "latency_999": 121.44601075 + }, + "optimized": { + "nb_forwards": 132, + "throughput": 8.8, + "latency_mean": 114.4251281590909, + "latency_std": 0.5121636382229513, + "latency_50": 114.325617, + "latency_90": 114.8843064, + "latency_95": 115.2926259, + "latency_99": 116.21156513, + "latency_999": 116.902565364 + } + }, + { + "batch_size": 1, + "input_length": 32, + "baseline": { + "nb_forwards": 440, + "throughput": 29.33, + "latency_mean": 34.11132023409091, + "latency_std": 1.1428837223428832, + "latency_50": 34.0117225, + "latency_90": 35.7673362, + "latency_95": 36.214977399999995, + "latency_99": 36.84041992, + "latency_999": 37.28137764399999 + }, + "optimized": { + "nb_forwards": 780, + "throughput": 52.0, + "latency_mean": 19.232586415384617, + "latency_std": 0.19048831587735654, + "latency_50": 19.205587, + "latency_90": 19.3554198, + "latency_95": 19.4416857, + "latency_99": 19.747647200000003, + "latency_999": 21.31330125800001 + } + }, + { + "batch_size": 8, + "input_length": 64, + "baseline": { + "nb_forwards": 91, + "throughput": 6.07, + "latency_mean": 166.1344919010989, + "latency_std": 4.663006799959188, + "latency_50": 165.748607, + "latency_90": 167.573247, + "latency_95": 168.185849, + "latency_99": 176.2619040999998, + "latency_999": 204.17836050999986 + }, + "optimized": { + "nb_forwards": 68, + "throughput": 4.53, + "latency_mean": 220.92038330882352, + "latency_std": 17.990154740643728, + "latency_50": 229.036164, + "latency_90": 230.3515418, + "latency_95": 230.71870145, + "latency_99": 233.38270184, + "latency_999": 233.602886084 + } + }, + { + "batch_size": 4, + "input_length": 128, + "baseline": { + "nb_forwards": 69, + "throughput": 4.6, + "latency_mean": 220.57769956521742, + "latency_std": 0.6686484290829855, + "latency_50": 220.51593, + "latency_90": 221.12682819999998, + "latency_95": 221.66818940000002, + "latency_99": 223.12707279999998, + "latency_999": 223.50554278 + }, + "optimized": { + "nb_forwards": 73, + "throughput": 4.87, + "latency_mean": 207.50999308219178, + "latency_std": 27.815623148152877, + "latency_50": 189.774877, + "latency_90": 237.9391546, + "latency_95": 238.31236859999999, + "latency_99": 240.56186224, + "latency_999": 244.546144024 + } + }, + { + "batch_size": 1, + "input_length": 64, + "baseline": { + "nb_forwards": 347, + "throughput": 23.13, + "latency_mean": 43.31934634870317, + "latency_std": 1.0549048838002049, + "latency_50": 43.200915, + "latency_90": 44.882578200000005, + "latency_95": 45.2033363, + "latency_99": 46.21059290000001, + "latency_999": 46.872020362 + }, + "optimized": { + "nb_forwards": 536, + "throughput": 35.73, + "latency_mean": 27.99766281716418, + "latency_std": 1.0529425569523703, + "latency_50": 27.69748, + "latency_90": 29.514874, + "latency_95": 30.11867, + "latency_99": 30.962504349999996, + "latency_999": 31.912116325000028 + } + }, + { + "batch_size": 4, + "input_length": 64, + "baseline": { + "nb_forwards": 166, + "throughput": 11.07, + "latency_mean": 90.84950987349397, + "latency_std": 2.0682139308506917, + "latency_50": 90.8146185, + "latency_90": 93.020294, + "latency_95": 93.80372125, + "latency_99": 96.35470339999999, + "latency_999": 102.08768143500005 + }, + "optimized": { + "nb_forwards": 136, + "throughput": 9.07, + "latency_mean": 110.460723375, + "latency_std": 10.94678012157101, + "latency_50": 116.622233, + "latency_90": 117.13783, + "latency_95": 117.34992975, + "latency_99": 117.67875504999999, + "latency_999": 117.831356625 + } + }, + { + "batch_size": 8, + "input_length": 128, + "baseline": { + "nb_forwards": 49, + "throughput": 3.27, + "latency_mean": 309.91095881632657, + "latency_std": 3.608767284403935, + "latency_50": 310.015937, + "latency_90": 313.87800439999995, + "latency_95": 314.33423439999996, + "latency_99": 318.4142126, + "latency_999": 321.07916876 + }, + "optimized": { + "nb_forwards": 42, + "throughput": 2.8, + "latency_mean": 357.87234047619046, + "latency_std": 12.591861806274563, + "latency_50": 355.861331, + "latency_90": 371.2001084, + "latency_95": 392.5037146499999, + "latency_99": 396.58451314, + "latency_999": 398.517758014 + } + }, + { + "batch_size": 1, + "input_length": 128, + "baseline": { + "nb_forwards": 261, + "throughput": 17.4, + "latency_mean": 57.65446063601532, + "latency_std": 2.434519247994199, + "latency_50": 56.976535, + "latency_90": 60.541172, + "latency_95": 61.207556, + "latency_99": 68.33251279999983, + "latency_999": 73.13234924 + }, + "optimized": { + "nb_forwards": 259, + "throughput": 17.27, + "latency_mean": 57.94760827799228, + "latency_std": 5.454128073391148, + "latency_50": 62.715428, + "latency_90": 63.1070498, + "latency_95": 63.2122011, + "latency_99": 63.43313314, + "latency_999": 63.689983864000006 + } + }, + { + "batch_size": 4, + "input_length": 32, + "baseline": { + "nb_forwards": 241, + "throughput": 16.07, + "latency_mean": 62.38362809958507, + "latency_std": 6.7586580892753, + "latency_50": 58.222043, + "latency_90": 69.869798, + "latency_95": 70.271116, + "latency_99": 71.7271214, + "latency_999": 80.2065777999999 + }, + "optimized": { + "nb_forwards": 280, + "throughput": 18.67, + "latency_mean": 53.64114180357142, + "latency_std": 5.084170546990271, + "latency_50": 50.8657445, + "latency_90": 59.753021700000005, + "latency_95": 59.83479735, + "latency_99": 59.98286641, + "latency_999": 60.134200166 + } + } + ], + "others": { + "baseline": { + "exact_match": 82.3, + "f1": 87.2318519258519 + }, + "optimized": { + "exact_match": 76.9, + "f1": 83.01425661180923 + } + } + }, + "max_eval_samples": 1000, + "time_benchmark_args": { + "duration": 15, + "warmup_runs": 5 + }, + "model_type": "distilbert" + }, + { + "model_name_or_path": "distilbert-base-uncased-distilled-squad", + "task": "question-answering", + "task_args": null, + "dataset": { + "path": "squad", + "eval_split": "validation", + "data_keys": { + "question": "question", + "context": "context" + }, + "ref_keys": [ + "answers" + ], + "name": null, + "calibration_split": "train" + }, + "quantization_approach": "dynamic", + "operators_to_quantize": [ + "Add", + "MatMul" + ], + "node_exclusion": [ + "layernorm", + "gelu", + "residual", + "gather", + "softmax" + ], + "aware_training": false, + "per_channel": false, + "calibration": { + "method": "minmax", + "num_calibration_samples": 100, + "calibration_histogram_percentile": null, + "calibration_moving_average": null, + "calibration_moving_average_constant": null + }, + "framework": "onnxruntime", + "framework_args": { + "opset": 11, + "optimization_level": 1 + }, + "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.977\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", + "versions": { + "transformers": "4.20.1", + "optimum": "1.2.3.dev0", + "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" + }, + "evaluation": { + "time": [ + { + "batch_size": 1, + "input_length": 128, + "baseline": { + "nb_forwards": 205, + "throughput": 13.67, + "latency_mean": 73.39433734146341, + "latency_std": 1.0224081759969428, + "latency_50": 73.460104, + "latency_90": 74.518562, + "latency_95": 75.0664468, + "latency_99": 75.84203632, + "latency_999": 76.921990512 + }, + "optimized": { + "nb_forwards": 565, + "throughput": 37.67, + "latency_mean": 26.564435938053098, + "latency_std": 0.187471490836448, + "latency_50": 26.533942, + "latency_90": 26.71886, + "latency_95": 26.8078146, + "latency_99": 27.12104648, + "latency_999": 28.311193836000076 + } + }, + { + "batch_size": 4, + "input_length": 128, + "baseline": { + "nb_forwards": 88, + "throughput": 5.87, + "latency_mean": 172.14481681818182, + "latency_std": 1.6925453073042733, + "latency_50": 172.1907685, + "latency_90": 174.1415275, + "latency_95": 175.0375522, + "latency_99": 175.82339321, + "latency_999": 175.940543321 + }, + "optimized": { + "nb_forwards": 159, + "throughput": 10.6, + "latency_mean": 94.77951160377359, + "latency_std": 4.814673654949999, + "latency_50": 96.147419, + "latency_90": 97.289479, + "latency_95": 97.7011618, + "latency_99": 98.90126592, + "latency_999": 99.86395080400003 + } + }, + { + "batch_size": 1, + "input_length": 32, + "baseline": { + "nb_forwards": 314, + "throughput": 20.93, + "latency_mean": 47.86938606687898, + "latency_std": 0.5732448823818431, + "latency_50": 47.92357, + "latency_90": 48.562472299999996, + "latency_95": 48.774001, + "latency_99": 49.11157929, + "latency_999": 49.426655882 + }, + "optimized": { + "nb_forwards": 2076, + "throughput": 138.4, + "latency_mean": 7.228697234585741, + "latency_std": 0.31178062817765007, + "latency_50": 7.1645175, + "latency_90": 7.7391565, + "latency_95": 7.850395, + "latency_99": 7.99090625, + "latency_999": 8.272048450000009 + } + }, + { + "batch_size": 4, + "input_length": 64, + "baseline": { + "nb_forwards": 124, + "throughput": 8.27, + "latency_mean": 121.20055792741935, + "latency_std": 0.36103071773349715, + "latency_50": 121.1512925, + "latency_90": 121.64469659999999, + "latency_95": 121.8583837, + "latency_99": 121.94945611, + "latency_999": 122.865046776 + }, + "optimized": { + "nb_forwards": 374, + "throughput": 24.93, + "latency_mean": 40.12498197593583, + "latency_std": 1.289681830262187, + "latency_50": 39.74041, + "latency_90": 42.1230914, + "latency_95": 42.6942433, + "latency_99": 43.790769559999994, + "latency_999": 44.761699371 + } + }, + { + "batch_size": 8, + "input_length": 128, + "baseline": { + "nb_forwards": 37, + "throughput": 2.47, + "latency_mean": 407.8962844324324, + "latency_std": 1.0450568064054995, + "latency_50": 407.71793, + "latency_90": 409.2667942, + "latency_95": 409.5479284, + "latency_99": 410.92905384, + "latency_999": 411.387726384 + }, + "optimized": { + "nb_forwards": 100, + "throughput": 6.67, + "latency_mean": 151.48524175, + "latency_std": 2.974018003966197, + "latency_50": 151.4110505, + "latency_90": 155.56673659999998, + "latency_95": 156.83145130000003, + "latency_99": 159.26098265000002, + "latency_999": 159.893205065 + } + }, + { + "batch_size": 4, + "input_length": 32, + "baseline": { + "nb_forwards": 209, + "throughput": 13.93, + "latency_mean": 72.10892416267941, + "latency_std": 0.9442605052158691, + "latency_50": 72.232233, + "latency_90": 73.24587679999999, + "latency_95": 73.5688604, + "latency_99": 74.49961544, + "latency_999": 74.99024779999999 + }, + "optimized": { + "nb_forwards": 689, + "throughput": 45.93, + "latency_mean": 21.802329773584905, + "latency_std": 1.2356400802889618, + "latency_50": 21.462006, + "latency_90": 23.394015399999997, + "latency_95": 23.461499, + "latency_99": 23.60606868, + "latency_999": 23.790571160000002 + } + }, + { + "batch_size": 8, + "input_length": 32, + "baseline": { + "nb_forwards": 134, + "throughput": 8.93, + "latency_mean": 112.67185172388061, + "latency_std": 9.561218465080065, + "latency_50": 117.305159, + "latency_90": 118.676979, + "latency_95": 119.01818645, + "latency_99": 119.38788219, + "latency_999": 119.960128779 + }, + "optimized": { + "nb_forwards": 347, + "throughput": 23.13, + "latency_mean": 43.26179646974064, + "latency_std": 0.18815052466746143, + "latency_50": 43.253195, + "latency_90": 43.494735799999994, + "latency_95": 43.5794423, + "latency_99": 43.73514478, + "latency_999": 44.139542132 + } + }, + { + "batch_size": 1, + "input_length": 64, + "baseline": { + "nb_forwards": 251, + "throughput": 16.73, + "latency_mean": 59.92467478486056, + "latency_std": 0.5304618192835174, + "latency_50": 59.959358, + "latency_90": 60.55806, + "latency_95": 60.8509665, + "latency_99": 61.1526405, + "latency_999": 61.21307125 + }, + "optimized": { + "nb_forwards": 1191, + "throughput": 79.4, + "latency_mean": 12.6018126565911, + "latency_std": 0.417689097037675, + "latency_50": 12.531897, + "latency_90": 13.148671, + "latency_95": 13.431785, + "latency_99": 13.783848599999999, + "latency_999": 14.088670549999993 + } + }, + { + "batch_size": 8, + "input_length": 64, + "baseline": { + "nb_forwards": 70, + "throughput": 4.67, + "latency_mean": 215.0252773857143, + "latency_std": 8.088753831054484, + "latency_50": 216.4045865, + "latency_90": 218.0282024, + "latency_95": 218.17546385, + "latency_99": 219.1229224, + "latency_999": 219.88093984 + }, + "optimized": { + "nb_forwards": 193, + "throughput": 12.87, + "latency_mean": 78.03187068393781, + "latency_std": 5.702629213049006, + "latency_50": 75.158498, + "latency_90": 88.1716502, + "latency_95": 88.3947608, + "latency_99": 88.80716235999999, + "latency_999": 89.147765224 + } + } + ], + "others": { + "baseline": { + "exact_match": 82.3, + "f1": 87.2318519258519 + }, + "optimized": { + "exact_match": 80.6, + "f1": 86.09652042402038 + } + } + }, + "max_eval_samples": 1000, + "time_benchmark_args": { + "duration": 15, + "warmup_runs": 5 + }, + "model_type": "distilbert" + }, + { + "model_name_or_path": "distilbert-base-uncased-distilled-squad", + "task": "question-answering", + "task_args": null, + "dataset": { + "path": "squad", + "eval_split": "validation", + "data_keys": { + "question": "question", + "context": "context" + }, + "ref_keys": [ + "answers" + ], + "name": null, + "calibration_split": "train" + }, + "quantization_approach": "dynamic", + "operators_to_quantize": [ + "Add", + "MatMul" + ], + "node_exclusion": [], + "aware_training": false, + "per_channel": false, + "calibration": { + "method": "minmax", + "num_calibration_samples": 100, + "calibration_histogram_percentile": null, + "calibration_moving_average": null, + "calibration_moving_average_constant": null + }, + "framework": "onnxruntime", + "framework_args": { + "opset": 11, + "optimization_level": 1 + }, + "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3100.086\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", + "versions": { + "transformers": "4.20.1", + "optimum": "1.2.3.dev0", + "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" + }, + "evaluation": { + "time": [ + { + "batch_size": 8, + "input_length": 64, + "baseline": { + "nb_forwards": 70, + "throughput": 4.67, + "latency_mean": 214.7585842714286, + "latency_std": 1.0748282952739399, + "latency_50": 214.6217865, + "latency_90": 216.33038530000002, + "latency_95": 216.71350644999998, + "latency_99": 217.10804912999998, + "latency_999": 217.27182111300002 + }, + "optimized": { + "nb_forwards": 173, + "throughput": 11.53, + "latency_mean": 87.19321883815029, + "latency_std": 0.4127804706511856, + "latency_50": 87.115246, + "latency_90": 87.827362, + "latency_95": 87.9646276, + "latency_99": 88.20961412, + "latency_999": 88.528557128 + } + }, + { + "batch_size": 4, + "input_length": 128, + "baseline": { + "nb_forwards": 69, + "throughput": 4.6, + "latency_mean": 220.37778250724637, + "latency_std": 0.6723770956181739, + "latency_50": 220.31347, + "latency_90": 221.0006854, + "latency_95": 221.6680164, + "latency_99": 222.58888876, + "latency_999": 222.634509076 + }, + "optimized": { + "nb_forwards": 179, + "throughput": 11.93, + "latency_mean": 84.18192030726257, + "latency_std": 1.7207141010005933, + "latency_50": 84.029571, + "latency_90": 86.4943866, + "latency_95": 87.52624759999999, + "latency_99": 88.34396318, + "latency_999": 89.769148814 + } + }, + { + "batch_size": 4, + "input_length": 64, + "baseline": { + "nb_forwards": 165, + "throughput": 11.0, + "latency_mean": 90.97119612121213, + "latency_std": 1.6585097335652936, + "latency_50": 90.91036, + "latency_90": 93.046208, + "latency_95": 93.5801342, + "latency_99": 95.02802792, + "latency_999": 95.151213372 + }, + "optimized": { + "nb_forwards": 362, + "throughput": 24.13, + "latency_mean": 41.510697558011046, + "latency_std": 2.6850168617333043, + "latency_50": 40.5225655, + "latency_90": 45.749891299999994, + "latency_95": 45.84510315, + "latency_99": 46.05479407, + "latency_999": 46.304120276 + } + }, + { + "batch_size": 8, + "input_length": 32, + "baseline": { + "nb_forwards": 157, + "throughput": 10.47, + "latency_mean": 95.77657172611464, + "latency_std": 10.384333636895294, + "latency_50": 91.24492, + "latency_90": 118.03857640000001, + "latency_95": 119.1807458, + "latency_99": 120.53466472, + "latency_999": 121.96483704799999 + }, + "optimized": { + "nb_forwards": 369, + "throughput": 24.6, + "latency_mean": 40.65512325745258, + "latency_std": 2.8634757589258784, + "latency_50": 42.886645, + "latency_90": 43.2775952, + "latency_95": 43.3398654, + "latency_99": 43.578850839999994, + "latency_999": 43.896356176 + } + }, + { + "batch_size": 8, + "input_length": 128, + "baseline": { + "nb_forwards": 37, + "throughput": 2.47, + "latency_mean": 407.3405624594595, + "latency_std": 0.9446139169585147, + "latency_50": 407.370805, + "latency_90": 408.153985, + "latency_95": 408.5356864, + "latency_99": 410.3574352, + "latency_999": 411.19967482 + }, + "optimized": { + "nb_forwards": 98, + "throughput": 6.53, + "latency_mean": 154.55170329591837, + "latency_std": 3.2466049439500617, + "latency_50": 153.957612, + "latency_90": 159.2587284, + "latency_95": 159.9041657, + "latency_99": 163.49634651, + "latency_999": 165.108355851 + } + }, + { + "batch_size": 4, + "input_length": 32, + "baseline": { + "nb_forwards": 206, + "throughput": 13.73, + "latency_mean": 73.15383589805825, + "latency_std": 1.109006864241087, + "latency_50": 72.793732, + "latency_90": 75.0387975, + "latency_95": 75.40630425, + "latency_99": 76.2248304, + "latency_999": 80.54231973499994 + }, + "optimized": { + "nb_forwards": 725, + "throughput": 48.33, + "latency_mean": 20.701377126896553, + "latency_std": 0.5521964132279599, + "latency_50": 20.611305, + "latency_90": 21.5047578, + "latency_95": 21.702541800000002, + "latency_99": 22.15760376, + "latency_999": 24.283321767999993 + } + }, + { + "batch_size": 1, + "input_length": 32, + "baseline": { + "nb_forwards": 312, + "throughput": 20.8, + "latency_mean": 48.10309290064102, + "latency_std": 0.5452673615135677, + "latency_50": 48.1016005, + "latency_90": 48.788028, + "latency_95": 48.96113875, + "latency_99": 49.29684296, + "latency_999": 49.564814242 + }, + "optimized": { + "nb_forwards": 2102, + "throughput": 140.13, + "latency_mean": 7.138094656517603, + "latency_std": 0.28361271957621664, + "latency_50": 7.1017695, + "latency_90": 7.5524614, + "latency_95": 7.751003399999999, + "latency_99": 7.9702024399999996, + "latency_999": 8.201174685999996 + } + }, + { + "batch_size": 1, + "input_length": 64, + "baseline": { + "nb_forwards": 252, + "throughput": 16.8, + "latency_mean": 59.635192896825394, + "latency_std": 0.5367633151420198, + "latency_50": 59.6327055, + "latency_90": 60.299356, + "latency_95": 60.47347955, + "latency_99": 60.860342730000006, + "latency_999": 60.983657111 + }, + "optimized": { + "nb_forwards": 1132, + "throughput": 75.47, + "latency_mean": 13.252757100706713, + "latency_std": 0.7118720708722511, + "latency_50": 13.7066955, + "latency_90": 13.920642699999998, + "latency_95": 13.9857479, + "latency_99": 14.104317450000002, + "latency_999": 14.310801054999994 + } + }, + { + "batch_size": 1, + "input_length": 128, + "baseline": { + "nb_forwards": 261, + "throughput": 17.4, + "latency_mean": 57.64333686206896, + "latency_std": 1.0745535083532258, + "latency_50": 57.643633, + "latency_90": 59.099249, + "latency_95": 59.402868, + "latency_99": 59.834367, + "latency_999": 60.372973120000005 + }, + "optimized": { + "nb_forwards": 641, + "throughput": 42.73, + "latency_mean": 23.416976716068643, + "latency_std": 0.819280052981588, + "latency_50": 23.204464, + "latency_90": 24.644357, + "latency_95": 24.972543, + "latency_99": 25.292027, + "latency_999": 26.127226000000014 + } + } + ], + "others": { + "baseline": { + "exact_match": 82.3, + "f1": 87.2318519258519 + }, + "optimized": { + "exact_match": 80.6, + "f1": 86.09652042402038 + } + } + }, + "max_eval_samples": 1000, + "time_benchmark_args": { + "duration": 15, + "warmup_runs": 5 + }, + "model_type": "distilbert" + }, + { + "model_name_or_path": "distilbert-base-uncased-distilled-squad", + "task": "question-answering", + "task_args": null, + "dataset": { + "path": "squad", + "eval_split": "validation", + "data_keys": { + "question": "question", + "context": "context" + }, + "ref_keys": [ + "answers" + ], + "name": null, + "calibration_split": "train" + }, + "quantization_approach": "dynamic", + "operators_to_quantize": [ + "Add" + ], + "node_exclusion": [], + "aware_training": false, + "per_channel": false, + "calibration": { + "method": "minmax", + "num_calibration_samples": 100, + "calibration_histogram_percentile": null, + "calibration_moving_average": null, + "calibration_moving_average_constant": null + }, + "framework": "onnxruntime", + "framework_args": { + "opset": 11, + "optimization_level": 1 + }, + "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.804\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", + "versions": { + "transformers": "4.20.1", + "optimum": "1.2.3.dev0", + "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" + }, + "evaluation": { + "time": [ + { + "batch_size": 4, + "input_length": 128, + "baseline": { + "nb_forwards": 74, + "throughput": 4.93, + "latency_mean": 203.90076075675674, + "latency_std": 21.596870608446135, + "latency_50": 219.806601, + "latency_90": 222.28046669999998, + "latency_95": 222.6357457, + "latency_99": 223.52929084000002, + "latency_999": 223.571662084 + }, + "optimized": { + "nb_forwards": 86, + "throughput": 5.73, + "latency_mean": 175.15544305813953, + "latency_std": 18.874085885764828, + "latency_50": 167.724299, + "latency_90": 218.6726605, + "latency_95": 218.96398125, + "latency_99": 219.63268945, + "latency_999": 219.994796845 + } + }, + { + "batch_size": 1, + "input_length": 64, + "baseline": { + "nb_forwards": 252, + "throughput": 16.8, + "latency_mean": 59.620304126984124, + "latency_std": 0.9369837443196661, + "latency_50": 59.6408835, + "latency_90": 60.899260700000006, + "latency_95": 61.2446919, + "latency_99": 61.72324802, + "latency_999": 62.219581975000004 + }, + "optimized": { + "nb_forwards": 509, + "throughput": 33.93, + "latency_mean": 29.505452408644402, + "latency_std": 0.10477107837580692, + "latency_50": 29.494157, + "latency_90": 29.626543, + "latency_95": 29.6817152, + "latency_99": 29.80756116, + "latency_999": 30.165765552000007 + } + }, + { + "batch_size": 4, + "input_length": 64, + "baseline": { + "nb_forwards": 127, + "throughput": 8.47, + "latency_mean": 118.58148563779527, + "latency_std": 0.4419650117831178, + "latency_50": 118.534868, + "latency_90": 119.0686048, + "latency_95": 119.2944373, + "latency_99": 120.15261722, + "latency_999": 120.63352342200001 + }, + "optimized": { + "nb_forwards": 141, + "throughput": 9.4, + "latency_mean": 106.55073221985815, + "latency_std": 0.3944079713253884, + "latency_50": 106.498131, + "latency_90": 106.921763, + "latency_95": 107.019295, + "latency_99": 107.93368059999999, + "latency_999": 109.45881622000002 + } + }, + { + "batch_size": 1, + "input_length": 32, + "baseline": { + "nb_forwards": 440, + "throughput": 29.33, + "latency_mean": 34.126477825, + "latency_std": 1.0283869127760359, + "latency_50": 34.086549, + "latency_90": 35.5103139, + "latency_95": 35.82743535, + "latency_99": 36.42215259, + "latency_999": 36.908125651999995 + }, + "optimized": { + "nb_forwards": 882, + "throughput": 58.8, + "latency_mean": 17.017291861678004, + "latency_std": 0.08720510862403949, + "latency_50": 17.0091915, + "latency_90": 17.1355021, + "latency_95": 17.17376905, + "latency_99": 17.25574468, + "latency_999": 17.338018466 + } + }, + { + "batch_size": 8, + "input_length": 32, + "baseline": { + "nb_forwards": 168, + "throughput": 11.2, + "latency_mean": 89.80876792857143, + "latency_std": 2.724045354234363, + "latency_50": 89.23447, + "latency_90": 92.91518060000001, + "latency_95": 95.965778, + "latency_99": 99.1532608, + "latency_999": 100.916031267 + }, + "optimized": { + "nb_forwards": 165, + "throughput": 11.0, + "latency_mean": 91.00189352121211, + "latency_std": 11.013642722583302, + "latency_50": 85.592419, + "latency_90": 104.2900912, + "latency_95": 104.361947, + "latency_99": 104.4810248, + "latency_999": 104.49576398800001 + } + }, + { + "batch_size": 8, + "input_length": 64, + "baseline": { + "nb_forwards": 74, + "throughput": 4.93, + "latency_mean": 204.2916795945946, + "latency_std": 20.441719346331414, + "latency_50": 214.991102, + "latency_90": 216.5581006, + "latency_95": 217.0407147, + "latency_99": 217.41635044999998, + "latency_999": 217.686203345 + }, + "optimized": { + "nb_forwards": 71, + "throughput": 4.73, + "latency_mean": 212.32542294366195, + "latency_std": 0.531422234525691, + "latency_50": 212.331527, + "latency_90": 212.977908, + "latency_95": 213.082789, + "latency_99": 213.5128688, + "latency_999": 214.10746658000002 + } + }, + { + "batch_size": 8, + "input_length": 128, + "baseline": { + "nb_forwards": 49, + "throughput": 3.27, + "latency_mean": 309.52593997959184, + "latency_std": 3.142891825034627, + "latency_50": 309.382131, + "latency_90": 314.37877760000003, + "latency_95": 315.6069922, + "latency_99": 315.9048882, + "latency_999": 315.94479852 + }, + "optimized": { + "nb_forwards": 34, + "throughput": 2.27, + "latency_mean": 445.23998747058823, + "latency_std": 1.320393415283763, + "latency_50": 445.3137435, + "latency_90": 446.8903998, + "latency_95": 447.35347025, + "latency_99": 447.77492344999996, + "latency_999": 447.88785324500003 + } + }, + { + "batch_size": 4, + "input_length": 32, + "baseline": { + "nb_forwards": 268, + "throughput": 17.87, + "latency_mean": 55.97453225746269, + "latency_std": 1.0305715235126534, + "latency_50": 55.880397, + "latency_90": 57.251859200000006, + "latency_95": 57.9394124, + "latency_99": 58.42131721, + "latency_999": 58.653711982 + }, + "optimized": { + "nb_forwards": 280, + "throughput": 18.67, + "latency_mean": 53.5968861, + "latency_std": 0.09504994641745691, + "latency_50": 53.589719, + "latency_90": 53.727543700000005, + "latency_95": 53.763608700000006, + "latency_99": 53.84968344, + "latency_999": 53.900073552 + } + }, + { + "batch_size": 1, + "input_length": 128, + "baseline": { + "nb_forwards": 207, + "throughput": 13.8, + "latency_mean": 72.80511813526569, + "latency_std": 0.28605904617544076, + "latency_50": 72.792014, + "latency_90": 73.14956740000001, + "latency_95": 73.2242503, + "latency_99": 73.44227836, + "latency_999": 74.268179328 + }, + "optimized": { + "nb_forwards": 263, + "throughput": 17.53, + "latency_mean": 57.04931692775665, + "latency_std": 0.1852668604685806, + "latency_50": 57.008233, + "latency_90": 57.251367200000004, + "latency_95": 57.3371062, + "latency_99": 57.79880592, + "latency_999": 58.027793358000004 + } + } + ], + "others": { + "baseline": { + "exact_match": 82.3, + "f1": 87.2318519258519 + }, + "optimized": { + "exact_match": 82.3, + "f1": 87.2318519258519 + } + } + }, + "max_eval_samples": 1000, + "time_benchmark_args": { + "duration": 15, + "warmup_runs": 5 + }, + "model_type": "distilbert" + }, + { + "model_name_or_path": "distilbert-base-uncased-distilled-squad", + "task": "question-answering", + "task_args": null, + "dataset": { + "path": "squad", + "eval_split": "validation", + "data_keys": { + "question": "question", + "context": "context" + }, + "ref_keys": [ + "answers" + ], + "name": null, + "calibration_split": "train" + }, + "quantization_approach": "static", + "operators_to_quantize": [ + "Add" + ], + "node_exclusion": [], + "aware_training": false, + "per_channel": false, + "calibration": { + "method": "minmax", + "num_calibration_samples": 100, + "calibration_histogram_percentile": null, + "calibration_moving_average": null, + "calibration_moving_average_constant": null + }, + "framework": "onnxruntime", + "framework_args": { + "opset": 11, + "optimization_level": 1 + }, + "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.838\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", + "versions": { + "transformers": "4.20.1", + "optimum": "1.2.3.dev0", + "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" + }, + "evaluation": { + "time": [ + { + "batch_size": 1, + "input_length": 64, + "baseline": { + "nb_forwards": 340, + "throughput": 22.67, + "latency_mean": 44.18998163235294, + "latency_std": 1.4697810745421789, + "latency_50": 43.944576, + "latency_90": 46.3169695, + "latency_95": 46.85474685, + "latency_99": 47.600308, + "latency_999": 48.253145861 + }, + "optimized": { + "nb_forwards": 459, + "throughput": 30.6, + "latency_mean": 32.72214506100218, + "latency_std": 1.383428951575046, + "latency_50": 32.429874, + "latency_90": 34.6945812, + "latency_95": 35.1220015, + "latency_99": 36.2150666, + "latency_999": 36.859092831999995 + } + }, + { + "batch_size": 4, + "input_length": 128, + "baseline": { + "nb_forwards": 68, + "throughput": 4.53, + "latency_mean": 221.9408882647059, + "latency_std": 1.4998283620039232, + "latency_50": 221.727837, + "latency_90": 223.742314, + "latency_95": 224.84834684999998, + "latency_99": 226.48472676999998, + "latency_999": 227.941314877 + }, + "optimized": { + "nb_forwards": 61, + "throughput": 4.07, + "latency_mean": 246.87498426229507, + "latency_std": 28.403201366430647, + "latency_50": 265.371572, + "latency_90": 266.689959, + "latency_95": 266.835504, + "latency_99": 267.98315759999997, + "latency_999": 268.11036216 + } + }, + { + "batch_size": 1, + "input_length": 32, + "baseline": { + "nb_forwards": 310, + "throughput": 20.67, + "latency_mean": 48.538128174193545, + "latency_std": 1.0825802524617727, + "latency_50": 48.426445, + "latency_90": 49.9468469, + "latency_95": 50.608303299999996, + "latency_99": 51.1435016, + "latency_999": 51.579964874 + }, + "optimized": { + "nb_forwards": 709, + "throughput": 47.27, + "latency_mean": 21.175880187588152, + "latency_std": 1.3509200916441064, + "latency_50": 21.880383, + "latency_90": 22.094095199999998, + "latency_95": 22.1335476, + "latency_99": 22.22487572, + "latency_999": 22.332830584000003 + } + }, + { + "batch_size": 4, + "input_length": 32, + "baseline": { + "nb_forwards": 250, + "throughput": 16.67, + "latency_mean": 60.190009684, + "latency_std": 6.476253661495298, + "latency_50": 56.6032025, + "latency_90": 70.51879790000001, + "latency_95": 71.00682115, + "latency_99": 71.97614781, + "latency_999": 72.419652896 + }, + "optimized": { + "nb_forwards": 223, + "throughput": 14.87, + "latency_mean": 67.2927178340807, + "latency_std": 0.144766654312015, + "latency_50": 67.282017, + "latency_90": 67.4824698, + "latency_95": 67.53632420000001, + "latency_99": 67.65417208, + "latency_999": 67.74949128 + } + }, + { + "batch_size": 8, + "input_length": 64, + "baseline": { + "nb_forwards": 70, + "throughput": 4.67, + "latency_mean": 214.69050597142856, + "latency_std": 0.4979447468736716, + "latency_50": 214.625579, + "latency_90": 215.30233869999998, + "latency_95": 215.40087115, + "latency_99": 216.26991978, + "latency_999": 216.828470778 + }, + "optimized": { + "nb_forwards": 72, + "throughput": 4.8, + "latency_mean": 209.0101843472222, + "latency_std": 23.570857468465338, + "latency_50": 199.064305, + "latency_90": 255.36236, + "latency_95": 255.8589584, + "latency_99": 256.32438673, + "latency_999": 256.494448273 + } + }, + { + "batch_size": 1, + "input_length": 128, + "baseline": { + "nb_forwards": 210, + "throughput": 14.0, + "latency_mean": 71.66073837142856, + "latency_std": 0.48116978977616937, + "latency_50": 71.5734205, + "latency_90": 72.0439302, + "latency_95": 72.4196435, + "latency_99": 73.94004301999999, + "latency_999": 74.513449128 + }, + "optimized": { + "nb_forwards": 256, + "throughput": 17.07, + "latency_mean": 58.67059760546875, + "latency_std": 1.692148816240866, + "latency_50": 58.4892805, + "latency_90": 60.9907035, + "latency_95": 61.989116, + "latency_99": 62.95026975, + "latency_999": 63.22841489 + } + }, + { + "batch_size": 8, + "input_length": 128, + "baseline": { + "nb_forwards": 49, + "throughput": 3.27, + "latency_mean": 310.00153851020406, + "latency_std": 2.7851646055699177, + "latency_50": 309.754002, + "latency_90": 313.0454142, + "latency_95": 314.881432, + "latency_99": 317.12773244, + "latency_999": 317.61191674400004 + }, + "optimized": { + "nb_forwards": 37, + "throughput": 2.47, + "latency_mean": 406.5427697027027, + "latency_std": 6.185204572502198, + "latency_50": 406.487553, + "latency_90": 414.2124184, + "latency_95": 417.1470922, + "latency_99": 419.61633636, + "latency_999": 419.968637436 + } + }, + { + "batch_size": 4, + "input_length": 64, + "baseline": { + "nb_forwards": 125, + "throughput": 8.33, + "latency_mean": 120.571896528, + "latency_std": 0.47277616561376834, + "latency_50": 120.49457, + "latency_90": 121.2047576, + "latency_95": 121.5551152, + "latency_99": 121.83896872, + "latency_999": 122.05622189200001 + }, + "optimized": { + "nb_forwards": 145, + "throughput": 9.67, + "latency_mean": 103.6226343724138, + "latency_std": 2.9157268721681353, + "latency_50": 103.111948, + "latency_90": 107.64045859999999, + "latency_95": 108.98621659999999, + "latency_99": 112.292974, + "latency_999": 112.668847504 + } + }, + { + "batch_size": 8, + "input_length": 32, + "baseline": { + "nb_forwards": 126, + "throughput": 8.4, + "latency_mean": 119.27741188888889, + "latency_std": 0.8764285887799853, + "latency_50": 119.307692, + "latency_90": 120.473393, + "latency_95": 120.7184835, + "latency_99": 121.2451975, + "latency_999": 121.262551875 + }, + "optimized": { + "nb_forwards": 118, + "throughput": 7.87, + "latency_mean": 127.78634824576271, + "latency_std": 0.3648261747760306, + "latency_50": 127.7387245, + "latency_90": 128.2727152, + "latency_95": 128.37564195000002, + "latency_99": 128.60643422, + "latency_999": 129.373907065 + } + } + ], + "others": { + "baseline": { + "exact_match": 82.3, + "f1": 87.2318519258519 + }, + "optimized": { + "exact_match": 59.8, + "f1": 69.2166994698964 + } + } + }, + "max_eval_samples": 1000, + "time_benchmark_args": { + "duration": 15, + "warmup_runs": 5 + }, + "model_type": "distilbert" + }, + { + "model_name_or_path": "distilbert-base-uncased-distilled-squad", + "task": "question-answering", + "task_args": null, + "dataset": { + "path": "squad", + "eval_split": "validation", + "data_keys": { + "question": "question", + "context": "context" + }, + "ref_keys": [ + "answers" + ], + "name": null, + "calibration_split": "train" + }, + "quantization_approach": "static", + "operators_to_quantize": [ + "Add", + "MatMul" + ], + "node_exclusion": [], + "aware_training": false, + "per_channel": false, + "calibration": { + "method": "minmax", + "num_calibration_samples": 100, + "calibration_histogram_percentile": null, + "calibration_moving_average": null, + "calibration_moving_average_constant": null + }, + "framework": "onnxruntime", + "framework_args": { + "opset": 11, + "optimization_level": 1 + }, + "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3097.368\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", + "versions": { + "transformers": "4.20.1", + "optimum": "1.2.3.dev0", + "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" + }, + "evaluation": { + "time": [ + { + "batch_size": 1, + "input_length": 32, + "baseline": { + "nb_forwards": 311, + "throughput": 20.73, + "latency_mean": 48.27436349517685, + "latency_std": 1.040822281053826, + "latency_50": 48.187545, + "latency_90": 49.621862, + "latency_95": 50.0009575, + "latency_99": 50.911865399999996, + "latency_999": 51.49017933 + }, + "optimized": { + "nb_forwards": 1292, + "throughput": 86.13, + "latency_mean": 11.618419383126936, + "latency_std": 0.10444681939743777, + "latency_50": 11.602171, + "latency_90": 11.748442599999999, + "latency_95": 11.79717875, + "latency_99": 11.94069171, + "latency_999": 12.056363528000002 + } + }, + { + "batch_size": 4, + "input_length": 128, + "baseline": { + "nb_forwards": 69, + "throughput": 4.6, + "latency_mean": 220.32118042028983, + "latency_std": 0.9850709045782986, + "latency_50": 220.356704, + "latency_90": 221.59199759999998, + "latency_95": 221.9246008, + "latency_99": 223.13476428, + "latency_999": 223.77970762799998 + }, + "optimized": { + "nb_forwards": 123, + "throughput": 8.2, + "latency_mean": 122.36457313821138, + "latency_std": 10.835259832058163, + "latency_50": 121.072407, + "latency_90": 135.5895634, + "latency_95": 135.8899157, + "latency_99": 138.01084006, + "latency_999": 139.175728762 + } + }, + { + "batch_size": 8, + "input_length": 32, + "baseline": { + "nb_forwards": 162, + "throughput": 10.8, + "latency_mean": 92.77228116049382, + "latency_std": 9.463275951760943, + "latency_50": 89.6388245, + "latency_90": 117.9440383, + "latency_95": 119.128948, + "latency_99": 119.91939433999998, + "latency_999": 120.517662241 + }, + "optimized": { + "nb_forwards": 234, + "throughput": 15.6, + "latency_mean": 64.20727750854701, + "latency_std": 0.31098450167906233, + "latency_50": 64.201227, + "latency_90": 64.6116797, + "latency_95": 64.72937155, + "latency_99": 65.065812, + "latency_999": 65.162602081 + } + }, + { + "batch_size": 8, + "input_length": 128, + "baseline": { + "nb_forwards": 37, + "throughput": 2.47, + "latency_mean": 408.1440117567568, + "latency_std": 0.7971971555607033, + "latency_50": 408.020052, + "latency_90": 409.07254439999997, + "latency_95": 409.467417, + "latency_99": 410.72945427999997, + "latency_999": 411.308799328 + }, + "optimized": { + "nb_forwards": 64, + "throughput": 4.27, + "latency_mean": 236.944504875, + "latency_std": 20.551884369675392, + "latency_50": 227.635955, + "latency_90": 272.39387389999996, + "latency_95": 272.9781154, + "latency_99": 278.51420387999997, + "latency_999": 282.636704388 + } + }, + { + "batch_size": 1, + "input_length": 64, + "baseline": { + "nb_forwards": 248, + "throughput": 16.53, + "latency_mean": 60.48698351209678, + "latency_std": 0.5347970640782516, + "latency_50": 60.474966, + "latency_90": 61.153837700000004, + "latency_95": 61.448808, + "latency_99": 61.726770509999994, + "latency_999": 61.795131821000005 + }, + "optimized": { + "nb_forwards": 806, + "throughput": 53.73, + "latency_mean": 18.618812374689824, + "latency_std": 0.9875542199892051, + "latency_50": 18.4731305, + "latency_90": 20.025972, + "latency_95": 20.843049, + "latency_99": 21.69578405, + "latency_999": 22.082188700000003 + } + }, + { + "batch_size": 4, + "input_length": 32, + "baseline": { + "nb_forwards": 266, + "throughput": 17.73, + "latency_mean": 56.56638917669173, + "latency_std": 1.1636111458496279, + "latency_50": 56.435677, + "latency_90": 58.201028, + "latency_95": 58.79177375, + "latency_99": 59.38910765, + "latency_999": 59.7182885 + }, + "optimized": { + "nb_forwards": 486, + "throughput": 32.4, + "latency_mean": 30.90172244855967, + "latency_std": 1.7590904451420482, + "latency_50": 30.4496635, + "latency_90": 33.697194, + "latency_95": 34.585768, + "latency_99": 35.691296949999995, + "latency_999": 36.21122576 + } + }, + { + "batch_size": 8, + "input_length": 64, + "baseline": { + "nb_forwards": 72, + "throughput": 4.8, + "latency_mean": 209.6628640277778, + "latency_std": 13.976074504609217, + "latency_50": 214.2633815, + "latency_90": 215.4051871, + "latency_95": 215.8541272, + "latency_99": 218.25931918000003, + "latency_999": 221.387059318 + }, + "optimized": { + "nb_forwards": 141, + "throughput": 9.4, + "latency_mean": 106.43290490780142, + "latency_std": 3.23067147706519, + "latency_50": 106.433161, + "latency_90": 110.193956, + "latency_95": 112.048999, + "latency_99": 115.729673, + "latency_999": 116.92148872000001 + } + }, + { + "batch_size": 4, + "input_length": 64, + "baseline": { + "nb_forwards": 144, + "throughput": 9.6, + "latency_mean": 104.92533156944445, + "latency_std": 14.860284902135193, + "latency_50": 94.1759405, + "latency_90": 121.35962690000001, + "latency_95": 121.7929045, + "latency_99": 122.82086836, + "latency_999": 122.95308226099999 + }, + "optimized": { + "nb_forwards": 260, + "throughput": 17.33, + "latency_mean": 57.90458781923077, + "latency_std": 3.0707474582498206, + "latency_50": 57.1554325, + "latency_90": 62.939566, + "latency_95": 64.11821884999999, + "latency_99": 66.00585256000001, + "latency_999": 66.83383903299999 + } + }, + { + "batch_size": 1, + "input_length": 128, + "baseline": { + "nb_forwards": 212, + "throughput": 14.13, + "latency_mean": 71.04031975, + "latency_std": 1.2335423114384807, + "latency_50": 70.953013, + "latency_90": 72.93224040000001, + "latency_95": 73.2437023, + "latency_99": 74.43362789, + "latency_999": 75.253044574 + }, + "optimized": { + "nb_forwards": 396, + "throughput": 26.4, + "latency_mean": 37.94484309343434, + "latency_std": 0.16381138575361315, + "latency_50": 37.9182705, + "latency_90": 38.176733, + "latency_95": 38.23742675, + "latency_99": 38.305533200000006, + "latency_999": 38.36262916 + } + } + ], + "others": { + "baseline": { + "exact_match": 82.3, + "f1": 87.2318519258519 + }, + "optimized": { + "exact_match": 54.5, + "f1": 64.29164315407452 + } + } + }, + "max_eval_samples": 1000, + "time_benchmark_args": { + "duration": 15, + "warmup_runs": 5 + }, + "model_type": "distilbert" + }, + { + "model_name_or_path": "distilbert-base-uncased-distilled-squad", + "task": "question-answering", + "task_args": null, + "dataset": { + "path": "squad", + "eval_split": "validation", + "data_keys": { + "question": "question", + "context": "context" + }, + "ref_keys": [ + "answers" + ], + "name": null, + "calibration_split": "train" + }, + "quantization_approach": "static", + "operators_to_quantize": [ + "Add", + "MatMul" + ], + "node_exclusion": [ + "layernorm", + "gelu", + "residual", + "gather", + "softmax" + ], + "aware_training": false, + "per_channel": false, + "calibration": { + "method": "minmax", + "num_calibration_samples": 100, + "calibration_histogram_percentile": null, + "calibration_moving_average": null, + "calibration_moving_average_constant": null + }, + "framework": "onnxruntime", + "framework_args": { + "opset": 11, + "optimization_level": 1 + }, + "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.988\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", + "versions": { + "transformers": "4.20.1", + "optimum": "1.2.3.dev0", + "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" + }, + "evaluation": { + "time": [ + { + "batch_size": 8, + "input_length": 128, + "baseline": { + "nb_forwards": 37, + "throughput": 2.47, + "latency_mean": 407.5374461081081, + "latency_std": 0.9308811800770912, + "latency_50": 407.503129, + "latency_90": 408.4859978, + "latency_95": 409.1130008, + "latency_99": 410.21976224, + "latency_999": 410.505260024 + }, + "optimized": { + "nb_forwards": 67, + "throughput": 4.47, + "latency_mean": 224.46013014925373, + "latency_std": 1.2067813257810087, + "latency_50": 224.222622, + "latency_90": 225.39195980000002, + "latency_95": 226.41629740000002, + "latency_99": 228.78541636000003, + "latency_999": 230.883278236 + } + }, + { + "batch_size": 8, + "input_length": 64, + "baseline": { + "nb_forwards": 70, + "throughput": 4.67, + "latency_mean": 215.47036322857141, + "latency_std": 5.763796300712505, + "latency_50": 216.127969, + "latency_90": 218.3204449, + "latency_95": 218.88104615, + "latency_99": 220.6371353, + "latency_999": 223.19921363 + }, + "optimized": { + "nb_forwards": 144, + "throughput": 9.6, + "latency_mean": 104.44696947222222, + "latency_std": 0.4207091347287099, + "latency_50": 104.4406845, + "latency_90": 104.9043417, + "latency_95": 105.0561619, + "latency_99": 105.80782608999999, + "latency_999": 106.49383402599999 + } + }, + { + "batch_size": 4, + "input_length": 32, + "baseline": { + "nb_forwards": 213, + "throughput": 14.2, + "latency_mean": 70.45866511737088, + "latency_std": 0.2878721089411343, + "latency_50": 70.425888, + "latency_90": 70.864623, + "latency_95": 70.9372132, + "latency_99": 71.19328732, + "latency_999": 71.41624544400001 + }, + "optimized": { + "nb_forwards": 603, + "throughput": 40.2, + "latency_mean": 24.88167712437811, + "latency_std": 1.0019642978162417, + "latency_50": 24.758782, + "latency_90": 26.4062178, + "latency_95": 26.7184647, + "latency_99": 27.097667720000004, + "latency_999": 27.482882742000008 + } + }, + { + "batch_size": 1, + "input_length": 64, + "baseline": { + "nb_forwards": 255, + "throughput": 17.0, + "latency_mean": 58.937400823529416, + "latency_std": 0.9323535241532246, + "latency_50": 58.875322, + "latency_90": 59.8057036, + "latency_95": 60.399553299999994, + "latency_99": 61.85374206, + "latency_999": 62.514518306 + }, + "optimized": { + "nb_forwards": 992, + "throughput": 66.13, + "latency_mean": 15.12884672983871, + "latency_std": 0.9866967082432004, + "latency_50": 14.7669305, + "latency_90": 16.4293394, + "latency_95": 16.48365125, + "latency_99": 16.62896306, + "latency_999": 16.872082149999997 + } + }, + { + "batch_size": 1, + "input_length": 128, + "baseline": { + "nb_forwards": 213, + "throughput": 14.2, + "latency_mean": 70.56589055399061, + "latency_std": 0.8249341966205053, + "latency_50": 70.62431, + "latency_90": 71.616199, + "latency_95": 71.9370058, + "latency_99": 72.3793054, + "latency_999": 72.699766524 + }, + "optimized": { + "nb_forwards": 544, + "throughput": 36.27, + "latency_mean": 27.58882950367647, + "latency_std": 1.1939137183142827, + "latency_50": 27.347617, + "latency_90": 29.5275534, + "latency_95": 29.9602636, + "latency_99": 30.7295052, + "latency_999": 30.957031331 + } + }, + { + "batch_size": 4, + "input_length": 128, + "baseline": { + "nb_forwards": 78, + "throughput": 5.2, + "latency_mean": 192.63010338461538, + "latency_std": 24.23487329273834, + "latency_50": 173.7823215, + "latency_90": 222.2715464, + "latency_95": 223.3036971, + "latency_99": 225.55757293000002, + "latency_999": 226.406006593 + }, + "optimized": { + "nb_forwards": 132, + "throughput": 8.8, + "latency_mean": 113.81753161363636, + "latency_std": 0.6467249477070305, + "latency_50": 113.767756, + "latency_90": 114.6572987, + "latency_95": 114.88671805, + "latency_99": 115.46696548, + "latency_999": 115.9667929 + } + }, + { + "batch_size": 8, + "input_length": 32, + "baseline": { + "nb_forwards": 169, + "throughput": 11.27, + "latency_mean": 89.13836758579882, + "latency_std": 1.8355160566154043, + "latency_50": 89.192802, + "latency_90": 91.4133886, + "latency_95": 92.51820620000001, + "latency_99": 93.29504212, + "latency_999": 93.695905456 + }, + "optimized": { + "nb_forwards": 288, + "throughput": 19.2, + "latency_mean": 52.089132309027775, + "latency_std": 0.19887005817696832, + "latency_50": 52.075364, + "latency_90": 52.3625353, + "latency_95": 52.4649237, + "latency_99": 52.58880765, + "latency_999": 52.706969511 + } + }, + { + "batch_size": 1, + "input_length": 32, + "baseline": { + "nb_forwards": 428, + "throughput": 28.53, + "latency_mean": 35.0650053457944, + "latency_std": 1.9416905978175623, + "latency_50": 34.4812805, + "latency_90": 38.0941055, + "latency_95": 38.495843199999996, + "latency_99": 39.51390012, + "latency_999": 42.14187876599995 + }, + "optimized": { + "nb_forwards": 1628, + "throughput": 108.53, + "latency_mean": 9.214644947174447, + "latency_std": 0.0940311186126701, + "latency_50": 9.205694, + "latency_90": 9.3333551, + "latency_95": 9.36865875, + "latency_99": 9.4546762, + "latency_999": 9.629165349000006 + } + }, + { + "batch_size": 4, + "input_length": 64, + "baseline": { + "nb_forwards": 125, + "throughput": 8.33, + "latency_mean": 120.566423736, + "latency_std": 0.32773176890043826, + "latency_50": 120.535469, + "latency_90": 121.00340340000001, + "latency_95": 121.16425579999999, + "latency_99": 121.27308992, + "latency_999": 121.643879456 + }, + "optimized": { + "nb_forwards": 277, + "throughput": 18.47, + "latency_mean": 54.24849066787004, + "latency_std": 0.19047805788515895, + "latency_50": 54.224382, + "latency_90": 54.5044332, + "latency_95": 54.6109856, + "latency_99": 54.75149856, + "latency_999": 54.862251523999994 + } + } + ], + "others": { + "baseline": { + "exact_match": 82.3, + "f1": 87.2318519258519 + }, + "optimized": { + "exact_match": 72.9, + "f1": 79.96371998744281 + } + } + }, + "max_eval_samples": 1000, + "time_benchmark_args": { + "duration": 15, + "warmup_runs": 5 + }, + "model_type": "distilbert" + }, + { + "model_name_or_path": "distilbert-base-uncased-distilled-squad", + "task": "question-answering", + "task_args": null, + "dataset": { + "path": "squad", + "eval_split": "validation", + "data_keys": { + "question": "question", + "context": "context" + }, + "ref_keys": [ + "answers" + ], + "name": null, + "calibration_split": "train" + }, + "quantization_approach": "dynamic", + "operators_to_quantize": [ + "Add" + ], + "node_exclusion": [ + "layernorm", + "gelu", + "residual", + "gather", + "softmax" + ], + "aware_training": false, + "per_channel": false, + "calibration": { + "method": "minmax", + "num_calibration_samples": 100, + "calibration_histogram_percentile": null, + "calibration_moving_average": null, + "calibration_moving_average_constant": null + }, + "framework": "onnxruntime", + "framework_args": { + "opset": 11, + "optimization_level": 1 + }, + "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.997\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", + "versions": { + "transformers": "4.20.1", + "optimum": "1.2.3.dev0", + "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" + }, + "evaluation": { + "time": [ + { + "batch_size": 8, + "input_length": 64, + "baseline": { + "nb_forwards": 70, + "throughput": 4.67, + "latency_mean": 216.4796847285714, + "latency_std": 0.5319523840331125, + "latency_50": 216.4132115, + "latency_90": 217.2235808, + "latency_95": 217.43512135, + "latency_99": 217.93726658, + "latency_999": 218.004904658 + }, + "optimized": { + "nb_forwards": 93, + "throughput": 6.2, + "latency_mean": 162.63612432258063, + "latency_std": 6.7981229484609695, + "latency_50": 160.371024, + "latency_90": 175.9109966, + "latency_95": 178.2932724, + "latency_99": 180.45402764, + "latency_999": 181.166818064 + } + }, + { + "batch_size": 1, + "input_length": 64, + "baseline": { + "nb_forwards": 250, + "throughput": 16.67, + "latency_mean": 60.13216628, + "latency_std": 1.0214734052265138, + "latency_50": 60.110321, + "latency_90": 61.3823292, + "latency_95": 61.8370275, + "latency_99": 62.49582684, + "latency_999": 62.834720483999995 + }, + "optimized": { + "nb_forwards": 506, + "throughput": 33.73, + "latency_mean": 29.64927941897233, + "latency_std": 0.09914349596696566, + "latency_50": 29.640438, + "latency_90": 29.7649045, + "latency_95": 29.817656, + "latency_99": 29.927720100000002, + "latency_999": 30.19670687 + } + }, + { + "batch_size": 4, + "input_length": 64, + "baseline": { + "nb_forwards": 125, + "throughput": 8.33, + "latency_mean": 120.849639712, + "latency_std": 0.9668375634433722, + "latency_50": 120.813037, + "latency_90": 121.9735066, + "latency_95": 122.5098556, + "latency_99": 122.86986592, + "latency_999": 123.22419958 + }, + "optimized": { + "nb_forwards": 141, + "throughput": 9.4, + "latency_mean": 106.49951225531915, + "latency_std": 0.31470398129243726, + "latency_50": 106.442906, + "latency_90": 106.932199, + "latency_95": 107.086644, + "latency_99": 107.3766202, + "latency_999": 107.86804946000001 + } + }, + { + "batch_size": 1, + "input_length": 32, + "baseline": { + "nb_forwards": 343, + "throughput": 22.87, + "latency_mean": 43.83403140233236, + "latency_std": 6.777393138738352, + "latency_50": 47.974253, + "latency_90": 49.6771042, + "latency_95": 50.217503, + "latency_99": 50.915752839999996, + "latency_999": 52.175437058000014 + }, + "optimized": { + "nb_forwards": 875, + "throughput": 58.33, + "latency_mean": 17.15841726057143, + "latency_std": 0.08566829655094844, + "latency_50": 17.153516, + "latency_90": 17.2748228, + "latency_95": 17.3054081, + "latency_99": 17.36630658, + "latency_999": 17.49840138 + } + }, + { + "batch_size": 8, + "input_length": 32, + "baseline": { + "nb_forwards": 128, + "throughput": 8.53, + "latency_mean": 117.3791356640625, + "latency_std": 0.8413632402417973, + "latency_50": 117.250864, + "latency_90": 118.50406629999999, + "latency_95": 118.87116745, + "latency_99": 119.4692367, + "latency_999": 119.927835012 + }, + "optimized": { + "nb_forwards": 144, + "throughput": 9.6, + "latency_mean": 104.27652415277778, + "latency_std": 0.3048652909436058, + "latency_50": 104.223559, + "latency_90": 104.6723871, + "latency_95": 104.8562854, + "latency_99": 105.23841804999999, + "latency_999": 105.67265316 + } + }, + { + "batch_size": 4, + "input_length": 128, + "baseline": { + "nb_forwards": 68, + "throughput": 4.53, + "latency_mean": 221.2224431470588, + "latency_std": 1.118456788421114, + "latency_50": 221.1416945, + "latency_90": 222.8013713, + "latency_95": 223.27910730000002, + "latency_99": 223.56076078, + "latency_999": 223.828592878 + }, + "optimized": { + "nb_forwards": 68, + "throughput": 4.53, + "latency_mean": 221.36998825, + "latency_std": 0.4904801834733391, + "latency_50": 221.3431115, + "latency_90": 221.8755237, + "latency_95": 221.9769379, + "latency_99": 222.760048, + "latency_999": 223.048282 + } + }, + { + "batch_size": 4, + "input_length": 32, + "baseline": { + "nb_forwards": 209, + "throughput": 13.93, + "latency_mean": 72.04991935406699, + "latency_std": 0.9649173207131073, + "latency_50": 72.140642, + "latency_90": 73.1535642, + "latency_95": 73.46569579999999, + "latency_99": 74.7303538, + "latency_999": 75.40225632 + }, + "optimized": { + "nb_forwards": 280, + "throughput": 18.67, + "latency_mean": 53.682652257142855, + "latency_std": 0.2593595953484514, + "latency_50": 53.628275, + "latency_90": 53.8589829, + "latency_95": 54.04694715, + "latency_99": 55.05985277, + "latency_999": 55.324252431000005 + } + }, + { + "batch_size": 1, + "input_length": 128, + "baseline": { + "nb_forwards": 235, + "throughput": 15.67, + "latency_mean": 64.03706125957447, + "latency_std": 7.7729420260039, + "latency_50": 58.940644, + "latency_90": 73.7838802, + "latency_95": 74.4361305, + "latency_99": 75.54929478, + "latency_999": 76.00147653 + }, + "optimized": { + "nb_forwards": 300, + "throughput": 20.0, + "latency_mean": 50.13742374666667, + "latency_std": 4.800538669035651, + "latency_50": 47.3348575, + "latency_90": 57.0711271, + "latency_95": 57.1360735, + "latency_99": 57.42901637, + "latency_999": 57.585291638 + } + }, + { + "batch_size": 8, + "input_length": 128, + "baseline": { + "nb_forwards": 37, + "throughput": 2.47, + "latency_mean": 406.5056369189189, + "latency_std": 0.7541595003868546, + "latency_50": 406.359168, + "latency_90": 407.44441, + "latency_95": 408.065172, + "latency_99": 408.18207568, + "latency_999": 408.195881968 + }, + "optimized": { + "nb_forwards": 39, + "throughput": 2.6, + "latency_mean": 394.84622148717943, + "latency_std": 54.09154075921731, + "latency_50": 441.007729, + "latency_90": 446.218295, + "latency_95": 447.0315637, + "latency_99": 447.1409742, + "latency_999": 447.15734232 + } + } + ], + "others": { + "baseline": { + "exact_match": 82.3, + "f1": 87.2318519258519 + }, + "optimized": { + "exact_match": 82.3, + "f1": 87.2318519258519 + } + } + }, + "max_eval_samples": 1000, + "time_benchmark_args": { + "duration": 15, + "warmup_runs": 5 + }, + "model_type": "distilbert" + } +] \ No newline at end of file