[ { "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", "task": "token-classification", "dataset": { "path": "conll2003", "eval_split": "validation", "data_keys": { "primary": "tokens", "secondary": null }, "ref_keys": [ "ner_tags" ], "name": null, "calibration_split": "train" }, "quantization_approach": "dynamic", "operators_to_quantize": [ "Add", "MatMul" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3098.713\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "65ad733f16d4e34fee21f98cb1b7edd33b8c1662" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 50, "throughput": 16.67, "latency_mean": 60.11770944, "latency_std": 1.5934818902628691, "latency_50": 60.264686, "latency_90": 61.2435885, "latency_95": 61.33683375, "latency_99": 65.10274001999998, "latency_999": 68.287157802 }, "optimized": { "nb_forwards": 166, "throughput": 55.33, "latency_mean": 18.127647289156627, "latency_std": 0.41933456288271403, "latency_50": 18.0557965, "latency_90": 18.814332, "latency_95": 18.98832, "latency_99": 19.21624665, "latency_999": 19.220398929999998 } } ], "others": { "baseline": { "precision": 0.9696969696969697, "recall": 0.9696969696969697, "f1": 0.9696969696969697, "accuracy": 0.993127147766323 }, "optimized": { "precision": 0.96875, "recall": 0.9393939393939394, "f1": 0.9538461538461539, "accuracy": 0.9896907216494846 } } }, "max_eval_samples": 10, "time_benchmark_args": { "duration": 3, "warmup_runs": 1 }, "model_type": "distilbert" }, { "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", "task": "token-classification", "dataset": { "path": "conll2003", "eval_split": "validation", "data_keys": { "primary": "tokens", "secondary": null }, "ref_keys": [ "ner_tags" ], "name": null, "calibration_split": "train" }, "quantization_approach": "static", "operators_to_quantize": [ "Add" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 2794.474\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "65ad733f16d4e34fee21f98cb1b7edd33b8c1662" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 70, "throughput": 23.33, "latency_mean": 43.18653295714286, "latency_std": 1.3760290540598898, "latency_50": 42.9665905, "latency_90": 45.0960815, "latency_95": 45.782922799999994, "latency_99": 46.19406632, "latency_999": 46.419223532 }, "optimized": { "nb_forwards": 79, "throughput": 26.33, "latency_mean": 38.12439892405064, "latency_std": 0.07978961670190722, "latency_50": 38.117262, "latency_90": 38.2253002, "latency_95": 38.2424445, "latency_99": 38.3146249, "latency_999": 38.35825069 } } ], "others": { "baseline": { "precision": 0.9696969696969697, "recall": 0.9696969696969697, "f1": 0.9696969696969697, "accuracy": 0.993127147766323 }, "optimized": { "precision": 0.037037037037037035, "recall": 0.12121212121212122, "f1": 0.05673758865248226, "accuracy": 0.10996563573883161 } } }, "max_eval_samples": 10, "time_benchmark_args": { "duration": 3, "warmup_runs": 1 }, "model_type": "distilbert" }, { "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", "task": "token-classification", "dataset": { "path": "conll2003", "eval_split": "validation", "data_keys": { "primary": "tokens", "secondary": null }, "ref_keys": [ "ner_tags" ], "name": null, "calibration_split": "train" }, "quantization_approach": "static", "operators_to_quantize": [ "Add", "MatMul" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3100.839\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "65ad733f16d4e34fee21f98cb1b7edd33b8c1662" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 51, "throughput": 17.0, "latency_mean": 58.88688531372549, "latency_std": 0.7101158934812859, "latency_50": 59.013485, "latency_90": 59.654999, "latency_95": 59.9591045, "latency_99": 60.169608, "latency_999": 60.262131600000004 }, "optimized": { "nb_forwards": 124, "throughput": 41.33, "latency_mean": 24.301893822580645, "latency_std": 1.3073645132666654, "latency_50": 23.906192, "latency_90": 26.371101600000003, "latency_95": 26.70486145, "latency_99": 26.872438550000002, "latency_999": 26.962627384 } } ], "others": { "baseline": { "precision": 0.9696969696969697, "recall": 0.9696969696969697, "f1": 0.9696969696969697, "accuracy": 0.993127147766323 }, "optimized": { "precision": 0.1044776119402985, "recall": 0.21212121212121213, "f1": 0.13999999999999999, "accuracy": 0.6907216494845361 } } }, "max_eval_samples": 10, "time_benchmark_args": { "duration": 3, "warmup_runs": 1 }, "model_type": "distilbert" }, { "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", "task": "token-classification", "dataset": { "path": "conll2003", "eval_split": "validation", "data_keys": { "primary": "tokens", "secondary": null }, "ref_keys": [ "ner_tags" ], "name": null, "calibration_split": "train" }, "quantization_approach": "dynamic", "operators_to_quantize": [ "Add" ], "node_exclusion": [], "aware_training": false, "per_channel": false, "calibration": { "method": "minmax", "num_calibration_samples": 100, "calibration_histogram_percentile": null, "calibration_moving_average": null, "calibration_moving_average_constant": null }, "framework": "onnxruntime", "framework_args": { "opset": 11, "optimization_level": 1 }, "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.924\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", "versions": { "transformers": "4.20.1", "optimum": "1.2.3.dev0", "optimum_hash": "65ad733f16d4e34fee21f98cb1b7edd33b8c1662" }, "evaluation": { "time": [ { "batch_size": 1, "input_length": 64, "baseline": { "nb_forwards": 51, "throughput": 17.0, "latency_mean": 59.487474725490195, "latency_std": 0.7968899973526024, "latency_50": 59.491817, "latency_90": 60.506018, "latency_95": 60.637663, "latency_99": 61.1182795, "latency_999": 61.46885605000001 }, "optimized": { "nb_forwards": 104, "throughput": 34.67, "latency_mean": 29.123214990384614, "latency_std": 0.08746558610670996, "latency_50": 29.1146975, "latency_90": 29.2330098, "latency_95": 29.28693925, "latency_99": 29.36537121, "latency_999": 29.454484605 } } ], "others": { "baseline": { "precision": 0.9696969696969697, "recall": 0.9696969696969697, "f1": 0.9696969696969697, "accuracy": 0.993127147766323 }, "optimized": { "precision": 0.9696969696969697, "recall": 0.9696969696969697, "f1": 0.9696969696969697, "accuracy": 0.993127147766323 } } }, "max_eval_samples": 10, "time_benchmark_args": { "duration": 3, "warmup_runs": 1 }, "model_type": "distilbert" } ]