[ { "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", "task": "token-classification", "dataset": { "path": "conll2003", "eval_split": "validation", "data_keys": { "primary": "tokens", "secondary": null }, "ref_keys": [ "ner_tags" ], "name": null, "calibration_split": "train" }, "quantization_approach": "static", "operators_to_quantize": [ "Add" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.998\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5ac9c0d9fd7e7cca55b2f9935b961ed5b6c50112" }, "evaluation": { "time": [ { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 130, "throughput": 8.67, "latency_mean": 115.81472209230769, "latency_std": 0.46895980796384223, "latency_50": 115.7693145, "latency_90": 116.2641674, "latency_95": 116.615633, "latency_99": 117.24392198999999, "latency_999": 118.33366130000002 }, "optimized": { "nb_forwards": 119, "throughput": 7.93, "latency_mean": 126.53206931932773, "latency_std": 0.21043917502896844, "latency_50": 126.51378, "latency_90": 126.8398328, "latency_95": 126.8832745, "latency_99": 127.04870174, "latency_999": 127.236541298 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 69, "throughput": 4.6, "latency_mean": 219.5741049275362, "latency_std": 0.425956641594441, "latency_50": 219.571816, "latency_90": 220.0821694, "latency_95": 220.1998948, "latency_99": 220.53578456, "latency_999": 220.90863515599997 }, "optimized": { "nb_forwards": 74, "throughput": 4.93, "latency_mean": 203.70586305405405, "latency_std": 4.723902925243521, "latency_50": 202.578965, "latency_90": 211.061159, "latency_95": 213.91898604999997, "latency_99": 216.57558472, "latency_999": 216.849314572 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 155, "throughput": 10.33, "latency_mean": 96.98536739354839, "latency_std": 13.276058953171962, "latency_50": 90.032648, "latency_90": 119.895212, "latency_95": 120.1166701, "latency_99": 121.32664904, "latency_999": 122.10800528 }, "optimized": { "nb_forwards": 148, "throughput": 9.87, "latency_mean": 102.02715479054055, "latency_std": 2.9594884849383276, "latency_50": 101.4796865, "latency_90": 105.83677709999999, "latency_95": 107.30721310000001, "latency_99": 110.73326877, "latency_999": 111.11521851100001 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 38, "throughput": 2.53, "latency_mean": 403.54761702631583, "latency_std": 0.7543435235070463, "latency_50": 403.5543235, "latency_90": 404.57511619999997, "latency_95": 404.74615289999997, "latency_99": 404.80311413, "latency_999": 404.825042513 }, "optimized": { "nb_forwards": 37, "throughput": 2.47, "latency_mean": 410.16972697297297, "latency_std": 7.6545384613060605, "latency_50": 409.333633, "latency_90": 420.4970922, "latency_95": 423.82144059999996, "latency_99": 426.282015, "latency_999": 426.663606 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 70, "throughput": 4.67, "latency_mean": 214.5513372142857, "latency_std": 0.44768920996368816, "latency_50": 214.507338, "latency_90": 215.1284881, "latency_95": 215.26822455, "latency_99": 215.65146749000002, "latency_999": 215.97797624900002 }, "optimized": { "nb_forwards": 75, "throughput": 5.0, "latency_mean": 200.16238184, "latency_std": 9.040248673052057, "latency_50": 196.89438, "latency_90": 214.99417680000002, "latency_95": 218.1261396, "latency_99": 223.02508824, "latency_999": 223.85004512400002 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 212, "throughput": 14.13, "latency_mean": 71.02946175, "latency_std": 0.3401815134935371, "latency_50": 71.0171625, "latency_90": 71.3381363, "latency_95": 71.41140920000001, "latency_99": 71.62187967, "latency_999": 74.00790798399997 }, "optimized": { "nb_forwards": 265, "throughput": 17.67, "latency_mean": 56.817175411320754, "latency_std": 3.6329155653183585, "latency_50": 55.596164, "latency_90": 60.593704, "latency_95": 66.950391, "latency_99": 67.13051659999999, "latency_999": 67.23656778399999 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 452, "throughput": 30.13, "latency_mean": 33.22745787610619, "latency_std": 1.0543225342336746, "latency_50": 33.078952, "latency_90": 35.073573100000004, "latency_95": 35.4359905, "latency_99": 35.666321880000005, "latency_999": 36.114601445999995 }, "optimized": { "nb_forwards": 746, "throughput": 49.73, "latency_mean": 20.109513683646114, "latency_std": 1.6406645419149348, "latency_50": 20.2270735, "latency_90": 21.831147, "latency_95": 21.8876735, "latency_99": 22.079700099999997, "latency_999": 22.4666168 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 340, "throughput": 22.67, "latency_mean": 44.21631953823529, "latency_std": 5.107156057378748, "latency_50": 42.547523, "latency_90": 45.71804530000001, "latency_95": 59.850769, "latency_99": 61.57001349, "latency_999": 62.189666854 }, "optimized": { "nb_forwards": 389, "throughput": 25.93, "latency_mean": 38.56483639331619, "latency_std": 0.11123736681980873, "latency_50": 38.556444, "latency_90": 38.712917, "latency_95": 38.7594766, "latency_99": 38.92545156, "latency_999": 39.024601432000004 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 207, "throughput": 13.8, "latency_mean": 72.4693660772947, "latency_std": 0.8661223930705324, "latency_50": 72.429662, "latency_90": 73.5786186, "latency_95": 74.0005967, "latency_99": 74.67420342, "latency_999": 74.911409554 }, "optimized": { "nb_forwards": 211, "throughput": 14.07, "latency_mean": 71.1041596492891, "latency_std": 0.28805137660931524, "latency_50": 71.03339, "latency_90": 71.471796, "latency_95": 71.633065, "latency_99": 72.0733595, "latency_999": 72.60335828 } } ], "others": { "baseline": { "precision": 0.9358012339503085, "recall": 0.9444631437226523, "f1": 0.9401122372057961, "accuracy": 0.9882013940267124 }, "optimized": { "precision": 0.05001838911364472, "recall": 0.16021541568495457, "f1": 0.07623623623623622, "accuracy": 0.31141700089560376 } } }, "max_eval_samples": null, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", "task": "token-classification", "dataset": { "path": "conll2003", "eval_split": "validation", "data_keys": { "primary": "tokens", "secondary": null }, "ref_keys": [ "ner_tags" ], "name": null, "calibration_split": "train" }, "quantization_approach": "dynamic", "operators_to_quantize": [ "Add", "MatMul" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3106.018\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5ac9c0d9fd7e7cca55b2f9935b961ed5b6c50112" }, "evaluation": { "time": [ { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 126, "throughput": 8.4, "latency_mean": 119.91479441269841, "latency_std": 0.2629922737356049, "latency_50": 119.9039375, "latency_90": 120.2240655, "latency_95": 120.3983675, "latency_99": 120.61099275, "latency_999": 120.725017125 }, "optimized": { "nb_forwards": 244, "throughput": 16.27, "latency_mean": 61.50554389344262, "latency_std": 0.9316797843470712, "latency_50": 61.3989675, "latency_90": 62.743792, "latency_95": 63.1351423, "latency_99": 64.33112306, "latency_999": 64.851473252 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 38, "throughput": 2.53, "latency_mean": 403.6921285, "latency_std": 0.8331604984169664, "latency_50": 403.615553, "latency_90": 404.5933473, "latency_95": 404.73024205, "latency_99": 405.73243518, "latency_999": 406.206855618 }, "optimized": { "nb_forwards": 49, "throughput": 3.27, "latency_mean": 307.35862414285714, "latency_std": 37.75323130347674, "latency_50": 328.22334, "latency_90": 329.4804032, "latency_95": 329.88989580000003, "latency_99": 331.99058364, "latency_999": 332.89210586400003 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 87, "throughput": 5.8, "latency_mean": 172.70831874712644, "latency_std": 16.93330536049502, "latency_50": 165.631011, "latency_90": 214.729821, "latency_95": 215.7495264, "latency_99": 219.2165974, "latency_999": 219.69907804 }, "optimized": { "nb_forwards": 128, "throughput": 8.53, "latency_mean": 117.8890106953125, "latency_std": 2.0181256632154128, "latency_50": 117.854545, "latency_90": 120.3485038, "latency_95": 121.1920466, "latency_99": 122.86635173, "latency_999": 123.87748661900001 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 69, "throughput": 4.6, "latency_mean": 219.78011085507248, "latency_std": 0.8383066531914855, "latency_50": 219.682452, "latency_90": 220.6735604, "latency_95": 221.179578, "latency_99": 222.15356576, "latency_999": 223.052069276 }, "optimized": { "nb_forwards": 122, "throughput": 8.13, "latency_mean": 123.70655378688525, "latency_std": 2.0832274392948285, "latency_50": 123.7493135, "latency_90": 126.5891177, "latency_95": 127.5693857, "latency_99": 128.41928172, "latency_999": 129.29519896800002 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 216, "throughput": 14.4, "latency_mean": 69.7580692962963, "latency_std": 0.7696003955372718, "latency_50": 69.544575, "latency_90": 70.7143, "latency_95": 71.6627545, "latency_99": 72.7330374, "latency_999": 73.26205913 }, "optimized": { "nb_forwards": 390, "throughput": 26.0, "latency_mean": 38.49885642307692, "latency_std": 3.3170930332820334, "latency_50": 40.230039, "latency_90": 40.3790612, "latency_95": 40.428531899999996, "latency_99": 40.510404130000005, "latency_999": 40.614662347 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 208, "throughput": 13.87, "latency_mean": 72.37936482211539, "latency_std": 0.24212622373713247, "latency_50": 72.3947265, "latency_90": 72.6858948, "latency_95": 72.75088670000001, "latency_99": 72.92894586, "latency_999": 73.01229876299999 }, "optimized": { "nb_forwards": 412, "throughput": 27.47, "latency_mean": 36.4657384684466, "latency_std": 3.4465598552729446, "latency_50": 34.966474, "latency_90": 43.4717706, "latency_95": 43.6103869, "latency_99": 43.74503971, "latency_999": 44.051604852 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 127, "throughput": 8.47, "latency_mean": 118.31688400787401, "latency_std": 1.1539171140975781, "latency_50": 118.215119, "latency_90": 119.8292888, "latency_95": 120.60338259999999, "latency_99": 121.44057798, "latency_999": 122.68986890599999 }, "optimized": { "nb_forwards": 254, "throughput": 16.93, "latency_mean": 59.2173072007874, "latency_std": 0.7459868576638755, "latency_50": 59.1816355, "latency_90": 60.196821299999996, "latency_95": 60.5587619, "latency_99": 60.966352210000004, "latency_999": 61.412101875 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 255, "throughput": 17.0, "latency_mean": 58.91992950980392, "latency_std": 0.8869192739714911, "latency_50": 58.910515, "latency_90": 59.9905808, "latency_95": 60.4006188, "latency_99": 61.36197958, "latency_999": 62.78803392800001 }, "optimized": { "nb_forwards": 763, "throughput": 50.87, "latency_mean": 19.684674589777195, "latency_std": 1.8042745579532666, "latency_50": 19.157213, "latency_90": 22.2412244, "latency_95": 22.3211958, "latency_99": 22.58112282, "latency_999": 26.754026579999863 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 324, "throughput": 21.6, "latency_mean": 46.3835297962963, "latency_std": 1.1343614396581607, "latency_50": 46.442469, "latency_90": 47.1782314, "latency_95": 47.4329425, "latency_99": 48.049914599999994, "latency_999": 50.92608084600004 }, "optimized": { "nb_forwards": 1507, "throughput": 100.47, "latency_mean": 9.958045324485735, "latency_std": 0.2901405767073424, "latency_50": 9.92078, "latency_90": 10.4131178, "latency_95": 10.5542065, "latency_99": 10.709247679999999, "latency_999": 10.978874069999991 } } ], "others": { "baseline": { "precision": 0.9358012339503085, "recall": 0.9444631437226523, "f1": 0.9401122372057961, "accuracy": 0.9882013940267124 }, "optimized": { "precision": 0.934913217623498, "recall": 0.9427802086839449, "f1": 0.9388302329478799, "accuracy": 0.9877341225030178 } } }, "max_eval_samples": null, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", "task": "token-classification", "dataset": { "path": "conll2003", "eval_split": "validation", "data_keys": { "primary": "tokens", "secondary": null }, "ref_keys": [ "ner_tags" ], "name": null, "calibration_split": "train" }, "quantization_approach": "static", "operators_to_quantize": [ "Add", "MatMul" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3101.721\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5ac9c0d9fd7e7cca55b2f9935b961ed5b6c50112" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 444, "throughput": 29.6, "latency_mean": 33.835281761261264, "latency_std": 1.1104691070719008, "latency_50": 33.7167165, "latency_90": 35.5099667, "latency_95": 35.79827469999999, "latency_99": 36.20388337, "latency_999": 36.403486599 }, "optimized": { "nb_forwards": 1038, "throughput": 69.2, "latency_mean": 14.456874846820808, "latency_std": 1.147945670896925, "latency_50": 15.261894, "latency_90": 15.4090676, "latency_95": 15.44100315, "latency_99": 15.5019354, "latency_999": 16.029519071999996 } }, { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 292, "throughput": 19.47, "latency_mean": 51.40666307191781, "latency_std": 7.904660749243075, "latency_50": 57.9910805, "latency_90": 59.4403903, "latency_95": 59.64796875, "latency_99": 60.01921038999999, "latency_999": 62.362501826000006 }, "optimized": { "nb_forwards": 511, "throughput": 34.07, "latency_mean": 29.36470456360078, "latency_std": 0.09446229792346662, "latency_50": 29.353048, "latency_90": 29.482019, "latency_95": 29.53371, "latency_99": 29.669687599999996, "latency_999": 29.82210045 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 81, "throughput": 5.4, "latency_mean": 186.3935255308642, "latency_std": 23.66920102974093, "latency_50": 170.663335, "latency_90": 219.886297, "latency_95": 220.412333, "latency_99": 221.20538580000002, "latency_999": 222.11007948 }, "optimized": { "nb_forwards": 85, "throughput": 5.67, "latency_mean": 176.9935626235294, "latency_std": 24.611926259834373, "latency_50": 162.989882, "latency_90": 205.825854, "latency_95": 206.062911, "latency_99": 206.69577235999998, "latency_999": 207.115859636 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 70, "throughput": 4.67, "latency_mean": 215.00074108571428, "latency_std": 0.9705650988667485, "latency_50": 214.867633, "latency_90": 215.72163830000002, "latency_95": 216.53323115, "latency_99": 218.93241332, "latency_999": 220.051872632 }, "optimized": { "nb_forwards": 101, "throughput": 6.73, "latency_mean": 148.9348962970297, "latency_std": 3.9263502299131137, "latency_50": 148.342845, "latency_90": 155.106506, "latency_95": 156.211825, "latency_99": 158.331491, "latency_999": 161.1917459 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 212, "throughput": 14.13, "latency_mean": 70.75742881132075, "latency_std": 1.2469446939770699, "latency_50": 70.660216, "latency_90": 72.6529741, "latency_95": 72.88815559999999, "latency_99": 73.11715371, "latency_999": 73.644905982 }, "optimized": { "nb_forwards": 312, "throughput": 20.8, "latency_mean": 48.23616327884615, "latency_std": 4.851493694643943, "latency_50": 45.751592, "latency_90": 55.8910255, "latency_95": 56.0878071, "latency_99": 56.63496909, "latency_999": 56.748669404 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 43, "throughput": 2.87, "latency_mean": 352.17799074418605, "latency_std": 48.791933253583736, "latency_50": 312.088024, "latency_90": 406.3363032, "latency_95": 407.0272163, "latency_99": 409.34203474000003, "latency_999": 410.012702374 }, "optimized": { "nb_forwards": 47, "throughput": 3.13, "latency_mean": 320.8497927446808, "latency_std": 37.045737718320964, "latency_50": 304.99127, "latency_90": 407.9567778, "latency_95": 408.6023321, "latency_99": 409.3466477, "latency_999": 409.53851807 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 273, "throughput": 18.2, "latency_mean": 55.05481489377289, "latency_std": 0.8315777141405638, "latency_50": 54.970724, "latency_90": 56.1484644, "latency_95": 56.430772600000005, "latency_99": 56.85986168, "latency_999": 57.515896824 }, "optimized": { "nb_forwards": 321, "throughput": 21.4, "latency_mean": 46.79807228660436, "latency_std": 4.841315127921891, "latency_50": 46.18684, "latency_90": 51.680476, "latency_95": 51.742804, "latency_99": 51.917291399999996, "latency_999": 52.1734384 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 129, "throughput": 8.6, "latency_mean": 116.59177039534883, "latency_std": 0.5224827890552791, "latency_50": 116.509951, "latency_90": 116.8391962, "latency_95": 117.141577, "latency_99": 119.15936884, "latency_999": 120.545900664 }, "optimized": { "nb_forwards": 180, "throughput": 12.0, "latency_mean": 83.54651290555556, "latency_std": 9.983735757677245, "latency_50": 78.13343, "latency_90": 97.8848908, "latency_95": 98.02936915000001, "latency_99": 98.11533426000001, "latency_999": 98.19551898499999 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 126, "throughput": 8.4, "latency_mean": 119.892617, "latency_std": 0.2462536509536135, "latency_50": 119.8820935, "latency_90": 120.218857, "latency_95": 120.33222325, "latency_99": 120.4118515, "latency_999": 120.443628 }, "optimized": { "nb_forwards": 173, "throughput": 11.53, "latency_mean": 86.75648606936416, "latency_std": 10.203153981382503, "latency_50": 81.471955, "latency_90": 100.5808368, "latency_95": 100.84913420000001, "latency_99": 101.67875240000001, "latency_999": 102.108927452 } } ], "others": { "baseline": { "precision": 0.9358012339503085, "recall": 0.9444631437226523, "f1": 0.9401122372057961, "accuracy": 0.9882013940267124 }, "optimized": { "precision": 0.06309066586113149, "recall": 0.24604510265903737, "f1": 0.10042933195947105, "accuracy": 0.34301623768544837 } } }, "max_eval_samples": null, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" }, { "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", "task": "token-classification", "dataset": { "path": "conll2003", "eval_split": "validation", "data_keys": { "primary": "tokens", "secondary": null }, "ref_keys": [ "ner_tags" ], "name": null, "calibration_split": "train" }, "quantization_approach": "dynamic", "operators_to_quantize": [ "Add" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3121.536\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "5ac9c0d9fd7e7cca55b2f9935b961ed5b6c50112" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 257, "throughput": 17.13, "latency_mean": 58.59407656031129, "latency_std": 0.5082693082988551, "latency_50": 58.62559, "latency_90": 59.226632, "latency_95": 59.3844464, "latency_99": 59.66381776, "latency_999": 60.193005848 }, "optimized": { "nb_forwards": 605, "throughput": 40.33, "latency_mean": 24.812764945454546, "latency_std": 1.8257073532356165, "latency_50": 24.192906, "latency_90": 29.2776762, "latency_95": 29.361327, "latency_99": 29.531946960000003, "latency_999": 29.634380991999997 } }, { "batch_size": 8, "input_length": 32, "baseline": { "nb_forwards": 129, "throughput": 8.6, "latency_mean": 116.51833388372093, "latency_std": 0.19355545540306737, "latency_50": 116.510617, "latency_90": 116.76812059999999, "latency_95": 116.8346944, "latency_99": 116.91619076, "latency_999": 117.070623216 }, "optimized": { "nb_forwards": 188, "throughput": 12.53, "latency_mean": 80.1677380851064, "latency_std": 1.8148568791642794, "latency_50": 79.986953, "latency_90": 82.70380990000001, "latency_95": 83.21852025, "latency_99": 84.57913724, "latency_999": 84.697106648 } }, { "batch_size": 1, "input_length": 32, "baseline": { "nb_forwards": 410, "throughput": 27.33, "latency_mean": 36.59453311707317, "latency_std": 5.201952859743346, "latency_50": 34.5124805, "latency_90": 46.8082206, "latency_95": 47.39390375, "latency_99": 47.86114868, "latency_999": 48.050237836 }, "optimized": { "nb_forwards": 1074, "throughput": 71.6, "latency_mean": 13.975352912476723, "latency_std": 0.2608975931098767, "latency_50": 13.9295265, "latency_90": 14.3837001, "latency_95": 14.4975223, "latency_99": 14.66426481, "latency_999": 14.912761724999998 } }, { "batch_size": 8, "input_length": 64, "baseline": { "nb_forwards": 91, "throughput": 6.07, "latency_mean": 166.0459372197802, "latency_std": 5.910350195698658, "latency_50": 165.351766, "latency_90": 168.122746, "latency_95": 169.064614, "latency_99": 189.41977489999985, "latency_999": 211.91127088999988 }, "optimized": { "nb_forwards": 96, "throughput": 6.4, "latency_mean": 156.98839121875, "latency_std": 3.8307406608519075, "latency_50": 156.713151, "latency_90": 161.920169, "latency_95": 162.8535025, "latency_99": 167.0273974, "latency_999": 169.60480174 } }, { "batch_size": 8, "input_length": 128, "baseline": { "nb_forwards": 41, "throughput": 2.73, "latency_mean": 372.8457580243902, "latency_std": 44.33863922960532, "latency_50": 403.71874, "latency_90": 407.483091, "latency_95": 409.321755, "latency_99": 414.9955262, "latency_999": 415.29942092 }, "optimized": { "nb_forwards": 48, "throughput": 3.2, "latency_mean": 317.5281361666667, "latency_std": 5.407804800179884, "latency_50": 317.912668, "latency_90": 323.9196422, "latency_95": 325.7003953, "latency_99": 328.84562712, "latency_999": 330.22228231200006 } }, { "batch_size": 4, "input_length": 128, "baseline": { "nb_forwards": 69, "throughput": 4.6, "latency_mean": 220.1285506231884, "latency_std": 0.9353033698467256, "latency_50": 220.192768, "latency_90": 221.36099019999997, "latency_95": 221.5007656, "latency_99": 222.25962384000002, "latency_999": 222.473907684 }, "optimized": { "nb_forwards": 81, "throughput": 5.4, "latency_mean": 187.20598314814814, "latency_std": 25.62748524575956, "latency_50": 168.810645, "latency_90": 215.939029, "latency_95": 216.125018, "latency_99": 217.0070662, "latency_999": 217.17069052000002 } }, { "batch_size": 4, "input_length": 32, "baseline": { "nb_forwards": 268, "throughput": 17.87, "latency_mean": 56.01774754104478, "latency_std": 1.5540596958913682, "latency_50": 55.6834485, "latency_90": 58.264829299999995, "latency_95": 59.282709049999994, "latency_99": 60.64031475, "latency_999": 62.074440195 }, "optimized": { "nb_forwards": 293, "throughput": 19.53, "latency_mean": 51.320777443686005, "latency_std": 3.776379337439246, "latency_50": 53.174213, "latency_90": 53.340690200000004, "latency_95": 53.408575, "latency_99": 53.4970542, "latency_999": 55.868342540000086 } }, { "batch_size": 1, "input_length": 128, "baseline": { "nb_forwards": 214, "throughput": 14.27, "latency_mean": 70.20732589719627, "latency_std": 0.8155710627132576, "latency_50": 70.365742, "latency_90": 71.1378453, "latency_95": 71.3491235, "latency_99": 71.89662288, "latency_999": 72.393311452 }, "optimized": { "nb_forwards": 324, "throughput": 21.6, "latency_mean": 46.297288277777774, "latency_std": 0.7972386433029148, "latency_50": 46.110238, "latency_90": 47.36061, "latency_95": 47.99375455, "latency_99": 48.857902409999994, "latency_999": 50.063431357000006 } }, { "batch_size": 4, "input_length": 64, "baseline": { "nb_forwards": 139, "throughput": 9.27, "latency_mean": 108.42828852517987, "latency_std": 13.16458394936121, "latency_50": 117.656529, "latency_90": 118.084375, "latency_95": 118.1652478, "latency_99": 118.36095123999999, "latency_999": 118.602071188 }, "optimized": { "nb_forwards": 142, "throughput": 9.47, "latency_mean": 105.65247052816902, "latency_std": 0.3934774499008662, "latency_50": 105.60453, "latency_90": 105.91063890000001, "latency_95": 106.2638084, "latency_99": 107.39340928, "latency_999": 107.831013231 } } ], "others": { "baseline": { "precision": 0.9358012339503085, "recall": 0.9444631437226523, "f1": 0.9401122372057961, "accuracy": 0.9882013940267124 }, "optimized": { "precision": 0.9358012339503085, "recall": 0.9444631437226523, "f1": 0.9401122372057961, "accuracy": 0.9882013940267124 } } }, "max_eval_samples": null, "time_benchmark_args": { "duration": 15, "warmup_runs": 5 }, "model_type": "distilbert" } ]