[ { "model_name_or_path": "distilbert-base-uncased-distilled-squad", "task": "question-answering", "task_args": null, "dataset": { "path": "squad", "eval_split": "validation", "data_keys": { "question": "question", "context": "context" }, "ref_keys": [ "answers" ], "name": null, "calibration_split": "train" }, "quantization_approach": "static", "operators_to_quantize": [ "Add" ], "node_exclusion": [ "layernorm", "gelu", "residual", "gather", "softmax" ], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3100.278\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" }, "evaluation": { "time": [ { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 126, "throughput": 8.4, "latency_mean": 119.09526233333332, "latency_std": 0.7552068299080471, "latency_50": 119.159921, "latency_90": 119.811235, "latency_95": 120.0762565, "latency_99": 121.2652685, "latency_999": 121.44601075 }, "optimized": { "nb_forwards": 132, "throughput": 8.8, "latency_mean": 114.4251281590909, "latency_std": 0.5121636382229513, "latency_50": 114.325617, "latency_90": 114.8843064, "latency_95": 115.2926259, "latency_99": 116.21156513, "latency_999": 116.902565364 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 440, "throughput": 29.33, "latency_mean": 34.11132023409091, "latency_std": 1.1428837223428832, "latency_50": 34.0117225, "latency_90": 35.7673362, "latency_95": 36.214977399999995, "latency_99": 36.84041992, "latency_999": 37.28137764399999 }, "optimized": { "nb_forwards": 780, "throughput": 52.0, "latency_mean": 19.232586415384617, "latency_std": 0.19048831587735654, "latency_50": 19.205587, "latency_90": 19.3554198, "latency_95": 19.4416857, "latency_99": 19.747647200000003, "latency_999": 21.31330125800001 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 91, "throughput": 6.07, "latency_mean": 166.1344919010989, "latency_std": 4.663006799959188, "latency_50": 165.748607, "latency_90": 167.573247, "latency_95": 168.185849, "latency_99": 176.2619040999998, "latency_999": 204.17836050999986 }, "optimized": { "nb_forwards": 68, "throughput": 4.53, "latency_mean": 220.92038330882352, "latency_std": 17.990154740643728, "latency_50": 229.036164, "latency_90": 230.3515418, "latency_95": 230.71870145, "latency_99": 233.38270184, "latency_999": 233.602886084 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 69, "throughput": 4.6, "latency_mean": 220.57769956521742, "latency_std": 0.6686484290829855, "latency_50": 220.51593, "latency_90": 221.12682819999998, "latency_95": 221.66818940000002, "latency_99": 223.12707279999998, "latency_999": 223.50554278 }, "optimized": { "nb_forwards": 73, "throughput": 4.87, "latency_mean": 207.50999308219178, "latency_std": 27.815623148152877, "latency_50": 189.774877, "latency_90": 237.9391546, "latency_95": 238.31236859999999, "latency_99": 240.56186224, "latency_999": 244.546144024 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 347, "throughput": 23.13, "latency_mean": 43.31934634870317, "latency_std": 1.0549048838002049, "latency_50": 43.200915, "latency_90": 44.882578200000005, "latency_95": 45.2033363, "latency_99": 46.21059290000001, "latency_999": 46.872020362 }, "optimized": { "nb_forwards": 536, "throughput": 35.73, "latency_mean": 27.99766281716418, "latency_std": 1.0529425569523703, "latency_50": 27.69748, "latency_90": 29.514874, "latency_95": 30.11867, "latency_99": 30.962504349999996, "latency_999": 31.912116325000028 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 166, "throughput": 11.07, "latency_mean": 90.84950987349397, "latency_std": 2.0682139308506917, "latency_50": 90.8146185, "latency_90": 93.020294, "latency_95": 93.80372125, "latency_99": 96.35470339999999, "latency_999": 102.08768143500005 }, "optimized": { "nb_forwards": 136, "throughput": 9.07, "latency_mean": 110.460723375, "latency_std": 10.94678012157101, "latency_50": 116.622233, "latency_90": 117.13783, "latency_95": 117.34992975, "latency_99": 117.67875504999999, "latency_999": 117.831356625 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 49, "throughput": 3.27, "latency_mean": 309.91095881632657, "latency_std": 3.608767284403935, "latency_50": 310.015937, "latency_90": 313.87800439999995, "latency_95": 314.33423439999996, "latency_99": 318.4142126, "latency_999": 321.07916876 }, "optimized": { "nb_forwards": 42, "throughput": 2.8, "latency_mean": 357.87234047619046, "latency_std": 12.591861806274563, "latency_50": 355.861331, "latency_90": 371.2001084, "latency_95": 392.5037146499999, "latency_99": 396.58451314, "latency_999": 398.517758014 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 261, "throughput": 17.4, "latency_mean": 57.65446063601532, "latency_std": 2.434519247994199, "latency_50": 56.976535, "latency_90": 60.541172, "latency_95": 61.207556, "latency_99": 68.33251279999983, "latency_999": 73.13234924 }, "optimized": { "nb_forwards": 259, "throughput": 17.27, "latency_mean": 57.94760827799228, "latency_std": 5.454128073391148, "latency_50": 62.715428, "latency_90": 63.1070498, "latency_95": 63.2122011, "latency_99": 63.43313314, "latency_999": 63.689983864000006 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 241, "throughput": 16.07, "latency_mean": 62.38362809958507, "latency_std": 6.7586580892753, "latency_50": 58.222043, "latency_90": 69.869798, "latency_95": 70.271116, "latency_99": 71.7271214, "latency_999": 80.2065777999999 }, "optimized": { "nb_forwards": 280, "throughput": 18.67, "latency_mean": 53.64114180357142, "latency_std": 5.084170546990271, "latency_50": 50.8657445, "latency_90": 59.753021700000005, "latency_95": 59.83479735, "latency_99": 59.98286641, "latency_999": 60.134200166 } } ], "others": { "baseline": { "exact_match": 82.3, "f1": 87.2318519258519 }, "optimized": { "exact_match": 76.9, "f1": 83.01425661180923 } } }, "max_eval_samples": 1000, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "distilbert-base-uncased-distilled-squad", "task": "question-answering", "task_args": null, "dataset": { "path": "squad", "eval_split": "validation", "data_keys": { "question": "question", "context": "context" }, "ref_keys": [ "answers" ], "name": null, "calibration_split": "train" }, "quantization_approach": "dynamic", "operators_to_quantize": [ "Add", "MatMul" ], "node_exclusion": [ "layernorm", "gelu", "residual", "gather", "softmax" ], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.977\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 205, "throughput": 13.67, "latency_mean": 73.39433734146341, "latency_std": 1.0224081759969428, "latency_50": 73.460104, "latency_90": 74.518562, "latency_95": 75.0664468, "latency_99": 75.84203632, "latency_999": 76.921990512 }, "optimized": { "nb_forwards": 565, "throughput": 37.67, "latency_mean": 26.564435938053098, "latency_std": 0.187471490836448, "latency_50": 26.533942, "latency_90": 26.71886, "latency_95": 26.8078146, "latency_99": 27.12104648, "latency_999": 28.311193836000076 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 88, "throughput": 5.87, "latency_mean": 172.14481681818182, "latency_std": 1.6925453073042733, "latency_50": 172.1907685, "latency_90": 174.1415275, "latency_95": 175.0375522, "latency_99": 175.82339321, "latency_999": 175.940543321 }, "optimized": { "nb_forwards": 159, "throughput": 10.6, "latency_mean": 94.77951160377359, "latency_std": 4.814673654949999, "latency_50": 96.147419, "latency_90": 97.289479, "latency_95": 97.7011618, "latency_99": 98.90126592, "latency_999": 99.86395080400003 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 314, "throughput": 20.93, "latency_mean": 47.86938606687898, "latency_std": 0.5732448823818431, "latency_50": 47.92357, "latency_90": 48.562472299999996, "latency_95": 48.774001, "latency_99": 49.11157929, "latency_999": 49.426655882 }, "optimized": { "nb_forwards": 2076, "throughput": 138.4, "latency_mean": 7.228697234585741, "latency_std": 0.31178062817765007, "latency_50": 7.1645175, "latency_90": 7.7391565, "latency_95": 7.850395, "latency_99": 7.99090625, "latency_999": 8.272048450000009 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 124, "throughput": 8.27, "latency_mean": 121.20055792741935, "latency_std": 0.36103071773349715, "latency_50": 121.1512925, "latency_90": 121.64469659999999, "latency_95": 121.8583837, "latency_99": 121.94945611, "latency_999": 122.865046776 }, "optimized": { "nb_forwards": 374, "throughput": 24.93, "latency_mean": 40.12498197593583, "latency_std": 1.289681830262187, "latency_50": 39.74041, "latency_90": 42.1230914, "latency_95": 42.6942433, "latency_99": 43.790769559999994, "latency_999": 44.761699371 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 37, "throughput": 2.47, "latency_mean": 407.8962844324324, "latency_std": 1.0450568064054995, "latency_50": 407.71793, "latency_90": 409.2667942, "latency_95": 409.5479284, "latency_99": 410.92905384, "latency_999": 411.387726384 }, "optimized": { "nb_forwards": 100, "throughput": 6.67, "latency_mean": 151.48524175, "latency_std": 2.974018003966197, "latency_50": 151.4110505, "latency_90": 155.56673659999998, "latency_95": 156.83145130000003, "latency_99": 159.26098265000002, "latency_999": 159.893205065 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 209, "throughput": 13.93, "latency_mean": 72.10892416267941, "latency_std": 0.9442605052158691, "latency_50": 72.232233, "latency_90": 73.24587679999999, "latency_95": 73.5688604, "latency_99": 74.49961544, "latency_999": 74.99024779999999 }, "optimized": { "nb_forwards": 689, "throughput": 45.93, "latency_mean": 21.802329773584905, "latency_std": 1.2356400802889618, "latency_50": 21.462006, "latency_90": 23.394015399999997, "latency_95": 23.461499, "latency_99": 23.60606868, "latency_999": 23.790571160000002 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 134, "throughput": 8.93, "latency_mean": 112.67185172388061, "latency_std": 9.561218465080065, "latency_50": 117.305159, "latency_90": 118.676979, "latency_95": 119.01818645, "latency_99": 119.38788219, "latency_999": 119.960128779 }, "optimized": { "nb_forwards": 347, "throughput": 23.13, "latency_mean": 43.26179646974064, "latency_std": 0.18815052466746143, "latency_50": 43.253195, "latency_90": 43.494735799999994, "latency_95": 43.5794423, "latency_99": 43.73514478, "latency_999": 44.139542132 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 251, "throughput": 16.73, "latency_mean": 59.92467478486056, "latency_std": 0.5304618192835174, "latency_50": 59.959358, "latency_90": 60.55806, "latency_95": 60.8509665, "latency_99": 61.1526405, "latency_999": 61.21307125 }, "optimized": { "nb_forwards": 1191, "throughput": 79.4, "latency_mean": 12.6018126565911, "latency_std": 0.417689097037675, "latency_50": 12.531897, "latency_90": 13.148671, "latency_95": 13.431785, "latency_99": 13.783848599999999, "latency_999": 14.088670549999993 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 70, "throughput": 4.67, "latency_mean": 215.0252773857143, "latency_std": 8.088753831054484, "latency_50": 216.4045865, "latency_90": 218.0282024, "latency_95": 218.17546385, "latency_99": 219.1229224, "latency_999": 219.88093984 }, "optimized": { "nb_forwards": 193, "throughput": 12.87, "latency_mean": 78.03187068393781, "latency_std": 5.702629213049006, "latency_50": 75.158498, "latency_90": 88.1716502, "latency_95": 88.3947608, "latency_99": 88.80716235999999, "latency_999": 89.147765224 } } ], "others": { "baseline": { "exact_match": 82.3, "f1": 87.2318519258519 }, "optimized": { "exact_match": 80.6, "f1": 86.09652042402038 } } }, "max_eval_samples": 1000, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "distilbert-base-uncased-distilled-squad", "task": "question-answering", "task_args": null, "dataset": { "path": "squad", "eval_split": "validation", "data_keys": { "question": "question", "context": "context" }, "ref_keys": [ "answers" ], "name": null, "calibration_split": "train" }, "quantization_approach": "dynamic", "operators_to_quantize": [ "Add", "MatMul" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3100.086\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" }, "evaluation": { "time": [ { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 70, "throughput": 4.67, "latency_mean": 214.7585842714286, "latency_std": 1.0748282952739399, "latency_50": 214.6217865, "latency_90": 216.33038530000002, "latency_95": 216.71350644999998, "latency_99": 217.10804912999998, "latency_999": 217.27182111300002 }, "optimized": { "nb_forwards": 173, "throughput": 11.53, "latency_mean": 87.19321883815029, "latency_std": 0.4127804706511856, "latency_50": 87.115246, "latency_90": 87.827362, "latency_95": 87.9646276, "latency_99": 88.20961412, "latency_999": 88.528557128 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 69, "throughput": 4.6, "latency_mean": 220.37778250724637, "latency_std": 0.6723770956181739, "latency_50": 220.31347, "latency_90": 221.0006854, "latency_95": 221.6680164, "latency_99": 222.58888876, "latency_999": 222.634509076 }, "optimized": { "nb_forwards": 179, "throughput": 11.93, "latency_mean": 84.18192030726257, "latency_std": 1.7207141010005933, "latency_50": 84.029571, "latency_90": 86.4943866, "latency_95": 87.52624759999999, "latency_99": 88.34396318, "latency_999": 89.769148814 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 165, "throughput": 11.0, "latency_mean": 90.97119612121213, "latency_std": 1.6585097335652936, "latency_50": 90.91036, "latency_90": 93.046208, "latency_95": 93.5801342, "latency_99": 95.02802792, "latency_999": 95.151213372 }, "optimized": { "nb_forwards": 362, "throughput": 24.13, "latency_mean": 41.510697558011046, "latency_std": 2.6850168617333043, "latency_50": 40.5225655, "latency_90": 45.749891299999994, "latency_95": 45.84510315, "latency_99": 46.05479407, "latency_999": 46.304120276 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 157, "throughput": 10.47, "latency_mean": 95.77657172611464, "latency_std": 10.384333636895294, "latency_50": 91.24492, "latency_90": 118.03857640000001, "latency_95": 119.1807458, "latency_99": 120.53466472, "latency_999": 121.96483704799999 }, "optimized": { "nb_forwards": 369, "throughput": 24.6, "latency_mean": 40.65512325745258, "latency_std": 2.8634757589258784, "latency_50": 42.886645, "latency_90": 43.2775952, "latency_95": 43.3398654, "latency_99": 43.578850839999994, "latency_999": 43.896356176 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 37, "throughput": 2.47, "latency_mean": 407.3405624594595, "latency_std": 0.9446139169585147, "latency_50": 407.370805, "latency_90": 408.153985, "latency_95": 408.5356864, "latency_99": 410.3574352, "latency_999": 411.19967482 }, "optimized": { "nb_forwards": 98, "throughput": 6.53, "latency_mean": 154.55170329591837, "latency_std": 3.2466049439500617, "latency_50": 153.957612, "latency_90": 159.2587284, "latency_95": 159.9041657, "latency_99": 163.49634651, "latency_999": 165.108355851 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 206, "throughput": 13.73, "latency_mean": 73.15383589805825, "latency_std": 1.109006864241087, "latency_50": 72.793732, "latency_90": 75.0387975, "latency_95": 75.40630425, "latency_99": 76.2248304, "latency_999": 80.54231973499994 }, "optimized": { "nb_forwards": 725, "throughput": 48.33, "latency_mean": 20.701377126896553, "latency_std": 0.5521964132279599, "latency_50": 20.611305, "latency_90": 21.5047578, "latency_95": 21.702541800000002, "latency_99": 22.15760376, "latency_999": 24.283321767999993 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 312, "throughput": 20.8, "latency_mean": 48.10309290064102, "latency_std": 0.5452673615135677, "latency_50": 48.1016005, "latency_90": 48.788028, "latency_95": 48.96113875, "latency_99": 49.29684296, "latency_999": 49.564814242 }, "optimized": { "nb_forwards": 2102, "throughput": 140.13, "latency_mean": 7.138094656517603, "latency_std": 0.28361271957621664, "latency_50": 7.1017695, "latency_90": 7.5524614, "latency_95": 7.751003399999999, "latency_99": 7.9702024399999996, "latency_999": 8.201174685999996 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 252, "throughput": 16.8, "latency_mean": 59.635192896825394, "latency_std": 0.5367633151420198, "latency_50": 59.6327055, "latency_90": 60.299356, "latency_95": 60.47347955, "latency_99": 60.860342730000006, "latency_999": 60.983657111 }, "optimized": { "nb_forwards": 1132, "throughput": 75.47, "latency_mean": 13.252757100706713, "latency_std": 0.7118720708722511, "latency_50": 13.7066955, "latency_90": 13.920642699999998, "latency_95": 13.9857479, "latency_99": 14.104317450000002, "latency_999": 14.310801054999994 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 261, "throughput": 17.4, "latency_mean": 57.64333686206896, "latency_std": 1.0745535083532258, "latency_50": 57.643633, "latency_90": 59.099249, "latency_95": 59.402868, "latency_99": 59.834367, "latency_999": 60.372973120000005 }, "optimized": { "nb_forwards": 641, "throughput": 42.73, "latency_mean": 23.416976716068643, "latency_std": 0.819280052981588, "latency_50": 23.204464, "latency_90": 24.644357, "latency_95": 24.972543, "latency_99": 25.292027, "latency_999": 26.127226000000014 } } ], "others": { "baseline": { "exact_match": 82.3, "f1": 87.2318519258519 }, "optimized": { "exact_match": 80.6, "f1": 86.09652042402038 } } }, "max_eval_samples": 1000, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "distilbert-base-uncased-distilled-squad", "task": "question-answering", "task_args": null, "dataset": { "path": "squad", "eval_split": "validation", "data_keys": { "question": "question", "context": "context" }, "ref_keys": [ "answers" ], "name": null, "calibration_split": "train" }, "quantization_approach": "dynamic", "operators_to_quantize": [ "Add" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.804\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" }, "evaluation": { "time": [ { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 74, "throughput": 4.93, "latency_mean": 203.90076075675674, "latency_std": 21.596870608446135, "latency_50": 219.806601, "latency_90": 222.28046669999998, "latency_95": 222.6357457, "latency_99": 223.52929084000002, "latency_999": 223.571662084 }, "optimized": { "nb_forwards": 86, "throughput": 5.73, "latency_mean": 175.15544305813953, "latency_std": 18.874085885764828, "latency_50": 167.724299, "latency_90": 218.6726605, "latency_95": 218.96398125, "latency_99": 219.63268945, "latency_999": 219.994796845 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 252, "throughput": 16.8, "latency_mean": 59.620304126984124, "latency_std": 0.9369837443196661, "latency_50": 59.6408835, "latency_90": 60.899260700000006, "latency_95": 61.2446919, "latency_99": 61.72324802, "latency_999": 62.219581975000004 }, "optimized": { "nb_forwards": 509, "throughput": 33.93, "latency_mean": 29.505452408644402, "latency_std": 0.10477107837580692, "latency_50": 29.494157, "latency_90": 29.626543, "latency_95": 29.6817152, "latency_99": 29.80756116, "latency_999": 30.165765552000007 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 127, "throughput": 8.47, "latency_mean": 118.58148563779527, "latency_std": 0.4419650117831178, "latency_50": 118.534868, "latency_90": 119.0686048, "latency_95": 119.2944373, "latency_99": 120.15261722, "latency_999": 120.63352342200001 }, "optimized": { "nb_forwards": 141, "throughput": 9.4, "latency_mean": 106.55073221985815, "latency_std": 0.3944079713253884, "latency_50": 106.498131, "latency_90": 106.921763, "latency_95": 107.019295, "latency_99": 107.93368059999999, "latency_999": 109.45881622000002 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 440, "throughput": 29.33, "latency_mean": 34.126477825, "latency_std": 1.0283869127760359, "latency_50": 34.086549, "latency_90": 35.5103139, "latency_95": 35.82743535, "latency_99": 36.42215259, "latency_999": 36.908125651999995 }, "optimized": { "nb_forwards": 882, "throughput": 58.8, "latency_mean": 17.017291861678004, "latency_std": 0.08720510862403949, "latency_50": 17.0091915, "latency_90": 17.1355021, "latency_95": 17.17376905, "latency_99": 17.25574468, "latency_999": 17.338018466 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 168, "throughput": 11.2, "latency_mean": 89.80876792857143, "latency_std": 2.724045354234363, "latency_50": 89.23447, "latency_90": 92.91518060000001, "latency_95": 95.965778, "latency_99": 99.1532608, "latency_999": 100.916031267 }, "optimized": { "nb_forwards": 165, "throughput": 11.0, "latency_mean": 91.00189352121211, "latency_std": 11.013642722583302, "latency_50": 85.592419, "latency_90": 104.2900912, "latency_95": 104.361947, "latency_99": 104.4810248, "latency_999": 104.49576398800001 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 74, "throughput": 4.93, "latency_mean": 204.2916795945946, "latency_std": 20.441719346331414, "latency_50": 214.991102, "latency_90": 216.5581006, "latency_95": 217.0407147, "latency_99": 217.41635044999998, "latency_999": 217.686203345 }, "optimized": { "nb_forwards": 71, "throughput": 4.73, "latency_mean": 212.32542294366195, "latency_std": 0.531422234525691, "latency_50": 212.331527, "latency_90": 212.977908, "latency_95": 213.082789, "latency_99": 213.5128688, "latency_999": 214.10746658000002 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 49, "throughput": 3.27, "latency_mean": 309.52593997959184, "latency_std": 3.142891825034627, "latency_50": 309.382131, "latency_90": 314.37877760000003, "latency_95": 315.6069922, "latency_99": 315.9048882, "latency_999": 315.94479852 }, "optimized": { "nb_forwards": 34, "throughput": 2.27, "latency_mean": 445.23998747058823, "latency_std": 1.320393415283763, "latency_50": 445.3137435, "latency_90": 446.8903998, "latency_95": 447.35347025, "latency_99": 447.77492344999996, "latency_999": 447.88785324500003 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 268, "throughput": 17.87, "latency_mean": 55.97453225746269, "latency_std": 1.0305715235126534, "latency_50": 55.880397, "latency_90": 57.251859200000006, "latency_95": 57.9394124, "latency_99": 58.42131721, "latency_999": 58.653711982 }, "optimized": { "nb_forwards": 280, "throughput": 18.67, "latency_mean": 53.5968861, "latency_std": 0.09504994641745691, "latency_50": 53.589719, "latency_90": 53.727543700000005, "latency_95": 53.763608700000006, "latency_99": 53.84968344, "latency_999": 53.900073552 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 207, "throughput": 13.8, "latency_mean": 72.80511813526569, "latency_std": 0.28605904617544076, "latency_50": 72.792014, "latency_90": 73.14956740000001, "latency_95": 73.2242503, "latency_99": 73.44227836, "latency_999": 74.268179328 }, "optimized": { "nb_forwards": 263, "throughput": 17.53, "latency_mean": 57.04931692775665, "latency_std": 0.1852668604685806, "latency_50": 57.008233, "latency_90": 57.251367200000004, "latency_95": 57.3371062, "latency_99": 57.79880592, "latency_999": 58.027793358000004 } } ], "others": { "baseline": { "exact_match": 82.3, "f1": 87.2318519258519 }, "optimized": { "exact_match": 82.3, "f1": 87.2318519258519 } } }, "max_eval_samples": 1000, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "distilbert-base-uncased-distilled-squad", "task": "question-answering", "task_args": null, "dataset": { "path": "squad", "eval_split": "validation", "data_keys": { "question": "question", "context": "context" }, "ref_keys": [ "answers" ], "name": null, "calibration_split": "train" }, "quantization_approach": "static", "operators_to_quantize": [ "Add" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.838\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 340, "throughput": 22.67, "latency_mean": 44.18998163235294, "latency_std": 1.4697810745421789, "latency_50": 43.944576, "latency_90": 46.3169695, "latency_95": 46.85474685, "latency_99": 47.600308, "latency_999": 48.253145861 }, "optimized": { "nb_forwards": 459, "throughput": 30.6, "latency_mean": 32.72214506100218, "latency_std": 1.383428951575046, "latency_50": 32.429874, "latency_90": 34.6945812, "latency_95": 35.1220015, "latency_99": 36.2150666, "latency_999": 36.859092831999995 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 68, "throughput": 4.53, "latency_mean": 221.9408882647059, "latency_std": 1.4998283620039232, "latency_50": 221.727837, "latency_90": 223.742314, "latency_95": 224.84834684999998, "latency_99": 226.48472676999998, "latency_999": 227.941314877 }, "optimized": { "nb_forwards": 61, "throughput": 4.07, "latency_mean": 246.87498426229507, "latency_std": 28.403201366430647, "latency_50": 265.371572, "latency_90": 266.689959, "latency_95": 266.835504, "latency_99": 267.98315759999997, "latency_999": 268.11036216 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 310, "throughput": 20.67, "latency_mean": 48.538128174193545, "latency_std": 1.0825802524617727, "latency_50": 48.426445, "latency_90": 49.9468469, "latency_95": 50.608303299999996, "latency_99": 51.1435016, "latency_999": 51.579964874 }, "optimized": { "nb_forwards": 709, "throughput": 47.27, "latency_mean": 21.175880187588152, "latency_std": 1.3509200916441064, "latency_50": 21.880383, "latency_90": 22.094095199999998, "latency_95": 22.1335476, "latency_99": 22.22487572, "latency_999": 22.332830584000003 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 250, "throughput": 16.67, "latency_mean": 60.190009684, "latency_std": 6.476253661495298, "latency_50": 56.6032025, "latency_90": 70.51879790000001, "latency_95": 71.00682115, "latency_99": 71.97614781, "latency_999": 72.419652896 }, "optimized": { "nb_forwards": 223, "throughput": 14.87, "latency_mean": 67.2927178340807, "latency_std": 0.144766654312015, "latency_50": 67.282017, "latency_90": 67.4824698, "latency_95": 67.53632420000001, "latency_99": 67.65417208, "latency_999": 67.74949128 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 70, "throughput": 4.67, "latency_mean": 214.69050597142856, "latency_std": 0.4979447468736716, "latency_50": 214.625579, "latency_90": 215.30233869999998, "latency_95": 215.40087115, "latency_99": 216.26991978, "latency_999": 216.828470778 }, "optimized": { "nb_forwards": 72, "throughput": 4.8, "latency_mean": 209.0101843472222, "latency_std": 23.570857468465338, "latency_50": 199.064305, "latency_90": 255.36236, "latency_95": 255.8589584, "latency_99": 256.32438673, "latency_999": 256.494448273 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 210, "throughput": 14.0, "latency_mean": 71.66073837142856, "latency_std": 0.48116978977616937, "latency_50": 71.5734205, "latency_90": 72.0439302, "latency_95": 72.4196435, "latency_99": 73.94004301999999, "latency_999": 74.513449128 }, "optimized": { "nb_forwards": 256, "throughput": 17.07, "latency_mean": 58.67059760546875, "latency_std": 1.692148816240866, "latency_50": 58.4892805, "latency_90": 60.9907035, "latency_95": 61.989116, "latency_99": 62.95026975, "latency_999": 63.22841489 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 49, "throughput": 3.27, "latency_mean": 310.00153851020406, "latency_std": 2.7851646055699177, "latency_50": 309.754002, "latency_90": 313.0454142, "latency_95": 314.881432, "latency_99": 317.12773244, "latency_999": 317.61191674400004 }, "optimized": { "nb_forwards": 37, "throughput": 2.47, "latency_mean": 406.5427697027027, "latency_std": 6.185204572502198, "latency_50": 406.487553, "latency_90": 414.2124184, "latency_95": 417.1470922, "latency_99": 419.61633636, "latency_999": 419.968637436 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 125, "throughput": 8.33, "latency_mean": 120.571896528, "latency_std": 0.47277616561376834, "latency_50": 120.49457, "latency_90": 121.2047576, "latency_95": 121.5551152, "latency_99": 121.83896872, "latency_999": 122.05622189200001 }, "optimized": { "nb_forwards": 145, "throughput": 9.67, "latency_mean": 103.6226343724138, "latency_std": 2.9157268721681353, "latency_50": 103.111948, "latency_90": 107.64045859999999, "latency_95": 108.98621659999999, "latency_99": 112.292974, "latency_999": 112.668847504 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 126, "throughput": 8.4, "latency_mean": 119.27741188888889, "latency_std": 0.8764285887799853, "latency_50": 119.307692, "latency_90": 120.473393, "latency_95": 120.7184835, "latency_99": 121.2451975, "latency_999": 121.262551875 }, "optimized": { "nb_forwards": 118, "throughput": 7.87, "latency_mean": 127.78634824576271, "latency_std": 0.3648261747760306, "latency_50": 127.7387245, "latency_90": 128.2727152, "latency_95": 128.37564195000002, "latency_99": 128.60643422, "latency_999": 129.373907065 } } ], "others": { "baseline": { "exact_match": 82.3, "f1": 87.2318519258519 }, "optimized": { "exact_match": 59.8, "f1": 69.2166994698964 } } }, "max_eval_samples": 1000, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "distilbert-base-uncased-distilled-squad", "task": "question-answering", "task_args": null, "dataset": { "path": "squad", "eval_split": "validation", "data_keys": { "question": "question", "context": "context" }, "ref_keys": [ "answers" ], "name": null, "calibration_split": "train" }, "quantization_approach": "static", "operators_to_quantize": [ "Add", "MatMul" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3097.368\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 311, "throughput": 20.73, "latency_mean": 48.27436349517685, "latency_std": 1.040822281053826, "latency_50": 48.187545, "latency_90": 49.621862, "latency_95": 50.0009575, "latency_99": 50.911865399999996, "latency_999": 51.49017933 }, "optimized": { "nb_forwards": 1292, "throughput": 86.13, "latency_mean": 11.618419383126936, "latency_std": 0.10444681939743777, "latency_50": 11.602171, "latency_90": 11.748442599999999, "latency_95": 11.79717875, "latency_99": 11.94069171, "latency_999": 12.056363528000002 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 69, "throughput": 4.6, "latency_mean": 220.32118042028983, "latency_std": 0.9850709045782986, "latency_50": 220.356704, "latency_90": 221.59199759999998, "latency_95": 221.9246008, "latency_99": 223.13476428, "latency_999": 223.77970762799998 }, "optimized": { "nb_forwards": 123, "throughput": 8.2, "latency_mean": 122.36457313821138, "latency_std": 10.835259832058163, "latency_50": 121.072407, "latency_90": 135.5895634, "latency_95": 135.8899157, "latency_99": 138.01084006, "latency_999": 139.175728762 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 162, "throughput": 10.8, "latency_mean": 92.77228116049382, "latency_std": 9.463275951760943, "latency_50": 89.6388245, "latency_90": 117.9440383, "latency_95": 119.128948, "latency_99": 119.91939433999998, "latency_999": 120.517662241 }, "optimized": { "nb_forwards": 234, "throughput": 15.6, "latency_mean": 64.20727750854701, "latency_std": 0.31098450167906233, "latency_50": 64.201227, "latency_90": 64.6116797, "latency_95": 64.72937155, "latency_99": 65.065812, "latency_999": 65.162602081 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 37, "throughput": 2.47, "latency_mean": 408.1440117567568, "latency_std": 0.7971971555607033, "latency_50": 408.020052, "latency_90": 409.07254439999997, "latency_95": 409.467417, "latency_99": 410.72945427999997, "latency_999": 411.308799328 }, "optimized": { "nb_forwards": 64, "throughput": 4.27, "latency_mean": 236.944504875, "latency_std": 20.551884369675392, "latency_50": 227.635955, "latency_90": 272.39387389999996, "latency_95": 272.9781154, "latency_99": 278.51420387999997, "latency_999": 282.636704388 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 248, "throughput": 16.53, "latency_mean": 60.48698351209678, "latency_std": 0.5347970640782516, "latency_50": 60.474966, "latency_90": 61.153837700000004, "latency_95": 61.448808, "latency_99": 61.726770509999994, "latency_999": 61.795131821000005 }, "optimized": { "nb_forwards": 806, "throughput": 53.73, "latency_mean": 18.618812374689824, "latency_std": 0.9875542199892051, "latency_50": 18.4731305, "latency_90": 20.025972, "latency_95": 20.843049, "latency_99": 21.69578405, "latency_999": 22.082188700000003 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 266, "throughput": 17.73, "latency_mean": 56.56638917669173, "latency_std": 1.1636111458496279, "latency_50": 56.435677, "latency_90": 58.201028, "latency_95": 58.79177375, "latency_99": 59.38910765, "latency_999": 59.7182885 }, "optimized": { "nb_forwards": 486, "throughput": 32.4, "latency_mean": 30.90172244855967, "latency_std": 1.7590904451420482, "latency_50": 30.4496635, "latency_90": 33.697194, "latency_95": 34.585768, "latency_99": 35.691296949999995, "latency_999": 36.21122576 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 72, "throughput": 4.8, "latency_mean": 209.6628640277778, "latency_std": 13.976074504609217, "latency_50": 214.2633815, "latency_90": 215.4051871, "latency_95": 215.8541272, "latency_99": 218.25931918000003, "latency_999": 221.387059318 }, "optimized": { "nb_forwards": 141, "throughput": 9.4, "latency_mean": 106.43290490780142, "latency_std": 3.23067147706519, "latency_50": 106.433161, "latency_90": 110.193956, "latency_95": 112.048999, "latency_99": 115.729673, "latency_999": 116.92148872000001 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 144, "throughput": 9.6, "latency_mean": 104.92533156944445, "latency_std": 14.860284902135193, "latency_50": 94.1759405, "latency_90": 121.35962690000001, "latency_95": 121.7929045, "latency_99": 122.82086836, "latency_999": 122.95308226099999 }, "optimized": { "nb_forwards": 260, "throughput": 17.33, "latency_mean": 57.90458781923077, "latency_std": 3.0707474582498206, "latency_50": 57.1554325, "latency_90": 62.939566, "latency_95": 64.11821884999999, "latency_99": 66.00585256000001, "latency_999": 66.83383903299999 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 212, "throughput": 14.13, "latency_mean": 71.04031975, "latency_std": 1.2335423114384807, "latency_50": 70.953013, "latency_90": 72.93224040000001, "latency_95": 73.2437023, "latency_99": 74.43362789, "latency_999": 75.253044574 }, "optimized": { "nb_forwards": 396, "throughput": 26.4, "latency_mean": 37.94484309343434, "latency_std": 0.16381138575361315, "latency_50": 37.9182705, "latency_90": 38.176733, "latency_95": 38.23742675, "latency_99": 38.305533200000006, "latency_999": 38.36262916 } } ], "others": { "baseline": { "exact_match": 82.3, "f1": 87.2318519258519 }, "optimized": { "exact_match": 54.5, "f1": 64.29164315407452 } } }, "max_eval_samples": 1000, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "distilbert-base-uncased-distilled-squad", "task": "question-answering", "task_args": null, "dataset": { "path": "squad", "eval_split": "validation", "data_keys": { "question": "question", "context": "context" }, "ref_keys": [ "answers" ], "name": null, "calibration_split": "train" }, "quantization_approach": "static", "operators_to_quantize": [ "Add", "MatMul" ], "node_exclusion": [ "layernorm", "gelu", "residual", "gather", "softmax" ], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.988\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" }, "evaluation": { "time": [ { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 37, "throughput": 2.47, "latency_mean": 407.5374461081081, "latency_std": 0.9308811800770912, "latency_50": 407.503129, "latency_90": 408.4859978, "latency_95": 409.1130008, "latency_99": 410.21976224, "latency_999": 410.505260024 }, "optimized": { "nb_forwards": 67, "throughput": 4.47, "latency_mean": 224.46013014925373, "latency_std": 1.2067813257810087, "latency_50": 224.222622, "latency_90": 225.39195980000002, "latency_95": 226.41629740000002, "latency_99": 228.78541636000003, "latency_999": 230.883278236 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 70, "throughput": 4.67, "latency_mean": 215.47036322857141, "latency_std": 5.763796300712505, "latency_50": 216.127969, "latency_90": 218.3204449, "latency_95": 218.88104615, "latency_99": 220.6371353, "latency_999": 223.19921363 }, "optimized": { "nb_forwards": 144, "throughput": 9.6, "latency_mean": 104.44696947222222, "latency_std": 0.4207091347287099, "latency_50": 104.4406845, "latency_90": 104.9043417, "latency_95": 105.0561619, "latency_99": 105.80782608999999, "latency_999": 106.49383402599999 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 213, "throughput": 14.2, "latency_mean": 70.45866511737088, "latency_std": 0.2878721089411343, "latency_50": 70.425888, "latency_90": 70.864623, "latency_95": 70.9372132, "latency_99": 71.19328732, "latency_999": 71.41624544400001 }, "optimized": { "nb_forwards": 603, "throughput": 40.2, "latency_mean": 24.88167712437811, "latency_std": 1.0019642978162417, "latency_50": 24.758782, "latency_90": 26.4062178, "latency_95": 26.7184647, "latency_99": 27.097667720000004, "latency_999": 27.482882742000008 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 255, "throughput": 17.0, "latency_mean": 58.937400823529416, "latency_std": 0.9323535241532246, "latency_50": 58.875322, "latency_90": 59.8057036, "latency_95": 60.399553299999994, "latency_99": 61.85374206, "latency_999": 62.514518306 }, "optimized": { "nb_forwards": 992, "throughput": 66.13, "latency_mean": 15.12884672983871, "latency_std": 0.9866967082432004, "latency_50": 14.7669305, "latency_90": 16.4293394, "latency_95": 16.48365125, "latency_99": 16.62896306, "latency_999": 16.872082149999997 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 213, "throughput": 14.2, "latency_mean": 70.56589055399061, "latency_std": 0.8249341966205053, "latency_50": 70.62431, "latency_90": 71.616199, "latency_95": 71.9370058, "latency_99": 72.3793054, "latency_999": 72.699766524 }, "optimized": { "nb_forwards": 544, "throughput": 36.27, "latency_mean": 27.58882950367647, "latency_std": 1.1939137183142827, "latency_50": 27.347617, "latency_90": 29.5275534, "latency_95": 29.9602636, "latency_99": 30.7295052, "latency_999": 30.957031331 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 78, "throughput": 5.2, "latency_mean": 192.63010338461538, "latency_std": 24.23487329273834, "latency_50": 173.7823215, "latency_90": 222.2715464, "latency_95": 223.3036971, "latency_99": 225.55757293000002, "latency_999": 226.406006593 }, "optimized": { "nb_forwards": 132, "throughput": 8.8, "latency_mean": 113.81753161363636, "latency_std": 0.6467249477070305, "latency_50": 113.767756, "latency_90": 114.6572987, "latency_95": 114.88671805, "latency_99": 115.46696548, "latency_999": 115.9667929 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 169, "throughput": 11.27, "latency_mean": 89.13836758579882, "latency_std": 1.8355160566154043, "latency_50": 89.192802, "latency_90": 91.4133886, "latency_95": 92.51820620000001, "latency_99": 93.29504212, "latency_999": 93.695905456 }, "optimized": { "nb_forwards": 288, "throughput": 19.2, "latency_mean": 52.089132309027775, "latency_std": 0.19887005817696832, "latency_50": 52.075364, "latency_90": 52.3625353, "latency_95": 52.4649237, "latency_99": 52.58880765, "latency_999": 52.706969511 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 428, "throughput": 28.53, "latency_mean": 35.0650053457944, "latency_std": 1.9416905978175623, "latency_50": 34.4812805, "latency_90": 38.0941055, "latency_95": 38.495843199999996, "latency_99": 39.51390012, "latency_999": 42.14187876599995 }, "optimized": { "nb_forwards": 1628, "throughput": 108.53, "latency_mean": 9.214644947174447, "latency_std": 0.0940311186126701, "latency_50": 9.205694, "latency_90": 9.3333551, "latency_95": 9.36865875, "latency_99": 9.4546762, "latency_999": 9.629165349000006 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 125, "throughput": 8.33, "latency_mean": 120.566423736, "latency_std": 0.32773176890043826, "latency_50": 120.535469, "latency_90": 121.00340340000001, "latency_95": 121.16425579999999, "latency_99": 121.27308992, "latency_999": 121.643879456 }, "optimized": { "nb_forwards": 277, "throughput": 18.47, "latency_mean": 54.24849066787004, "latency_std": 0.19047805788515895, "latency_50": 54.224382, "latency_90": 54.5044332, "latency_95": 54.6109856, "latency_99": 54.75149856, "latency_999": 54.862251523999994 } } ], "others": { "baseline": { "exact_match": 82.3, "f1": 87.2318519258519 }, "optimized": { "exact_match": 72.9, "f1": 79.96371998744281 } } }, "max_eval_samples": 1000, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "distilbert-base-uncased-distilled-squad", "task": "question-answering", "task_args": null, "dataset": { "path": "squad", "eval_split": "validation", "data_keys": { "question": "question", "context": "context" }, "ref_keys": [ "answers" ], "name": null, "calibration_split": "train" }, "quantization_approach": "dynamic", "operators_to_quantize": [ "Add" ], "node_exclusion": [ "layernorm", "gelu", "residual", "gather", "softmax" ], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.997\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke avx512_vnni\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5c9af4e5f93c7e9bd523563230732b49603dc4d7" }, "evaluation": { "time": [ { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 70, "throughput": 4.67, "latency_mean": 216.4796847285714, "latency_std": 0.5319523840331125, "latency_50": 216.4132115, "latency_90": 217.2235808, "latency_95": 217.43512135, "latency_99": 217.93726658, "latency_999": 218.004904658 }, "optimized": { "nb_forwards": 93, "throughput": 6.2, "latency_mean": 162.63612432258063, "latency_std": 6.7981229484609695, "latency_50": 160.371024, "latency_90": 175.9109966, "latency_95": 178.2932724, "latency_99": 180.45402764, "latency_999": 181.166818064 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 250, "throughput": 16.67, "latency_mean": 60.13216628, "latency_std": 1.0214734052265138, "latency_50": 60.110321, "latency_90": 61.3823292, "latency_95": 61.8370275, "latency_99": 62.49582684, "latency_999": 62.834720483999995 }, "optimized": { "nb_forwards": 506, "throughput": 33.73, "latency_mean": 29.64927941897233, "latency_std": 0.09914349596696566, "latency_50": 29.640438, "latency_90": 29.7649045, "latency_95": 29.817656, "latency_99": 29.927720100000002, "latency_999": 30.19670687 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 125, "throughput": 8.33, "latency_mean": 120.849639712, "latency_std": 0.9668375634433722, "latency_50": 120.813037, "latency_90": 121.9735066, "latency_95": 122.5098556, "latency_99": 122.86986592, "latency_999": 123.22419958 }, "optimized": { "nb_forwards": 141, "throughput": 9.4, "latency_mean": 106.49951225531915, "latency_std": 0.31470398129243726, "latency_50": 106.442906, "latency_90": 106.932199, "latency_95": 107.086644, "latency_99": 107.3766202, "latency_999": 107.86804946000001 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 343, "throughput": 22.87, "latency_mean": 43.83403140233236, "latency_std": 6.777393138738352, "latency_50": 47.974253, "latency_90": 49.6771042, "latency_95": 50.217503, "latency_99": 50.915752839999996, "latency_999": 52.175437058000014 }, "optimized": { "nb_forwards": 875, "throughput": 58.33, "latency_mean": 17.15841726057143, "latency_std": 0.08566829655094844, "latency_50": 17.153516, "latency_90": 17.2748228, "latency_95": 17.3054081, "latency_99": 17.36630658, "latency_999": 17.49840138 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 128, "throughput": 8.53, "latency_mean": 117.3791356640625, "latency_std": 0.8413632402417973, "latency_50": 117.250864, "latency_90": 118.50406629999999, "latency_95": 118.87116745, "latency_99": 119.4692367, "latency_999": 119.927835012 }, "optimized": { "nb_forwards": 144, "throughput": 9.6, "latency_mean": 104.27652415277778, "latency_std": 0.3048652909436058, "latency_50": 104.223559, "latency_90": 104.6723871, "latency_95": 104.8562854, "latency_99": 105.23841804999999, "latency_999": 105.67265316 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 68, "throughput": 4.53, "latency_mean": 221.2224431470588, "latency_std": 1.118456788421114, "latency_50": 221.1416945, "latency_90": 222.8013713, "latency_95": 223.27910730000002, "latency_99": 223.56076078, "latency_999": 223.828592878 }, "optimized": { "nb_forwards": 68, "throughput": 4.53, "latency_mean": 221.36998825, "latency_std": 0.4904801834733391, "latency_50": 221.3431115, "latency_90": 221.8755237, "latency_95": 221.9769379, "latency_99": 222.760048, "latency_999": 223.048282 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 209, "throughput": 13.93, "latency_mean": 72.04991935406699, "latency_std": 0.9649173207131073, "latency_50": 72.140642, "latency_90": 73.1535642, "latency_95": 73.46569579999999, "latency_99": 74.7303538, "latency_999": 75.40225632 }, "optimized": { "nb_forwards": 280, "throughput": 18.67, "latency_mean": 53.682652257142855, "latency_std": 0.2593595953484514, "latency_50": 53.628275, "latency_90": 53.8589829, "latency_95": 54.04694715, "latency_99": 55.05985277, "latency_999": 55.324252431000005 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 235, "throughput": 15.67, "latency_mean": 64.03706125957447, "latency_std": 7.7729420260039, "latency_50": 58.940644, "latency_90": 73.7838802, "latency_95": 74.4361305, "latency_99": 75.54929478, "latency_999": 76.00147653 }, "optimized": { "nb_forwards": 300, "throughput": 20.0, "latency_mean": 50.13742374666667, "latency_std": 4.800538669035651, "latency_50": 47.3348575, "latency_90": 57.0711271, "latency_95": 57.1360735, "latency_99": 57.42901637, "latency_999": 57.585291638 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 37, "throughput": 2.47, "latency_mean": 406.5056369189189, "latency_std": 0.7541595003868546, "latency_50": 406.359168, "latency_90": 407.44441, "latency_95": 408.065172, "latency_99": 408.18207568, "latency_999": 408.195881968 }, "optimized": { "nb_forwards": 39, "throughput": 2.6, "latency_mean": 394.84622148717943, "latency_std": 54.09154075921731, "latency_50": 441.007729, "latency_90": 446.218295, "latency_95": 447.0315637, "latency_99": 447.1409742, "latency_999": 447.15734232 } } ], "others": { "baseline": { "exact_match": 82.3, "f1": 87.2318519258519 }, "optimized": { "exact_match": 82.3, "f1": 87.2318519258519 } } }, "max_eval_samples": 1000, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" } ]