Upload /phi-2/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

phi-2/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,827 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 985.8048,
+            "max_vram": 2585.788416,
+            "max_reserved": 2101.346304,
+            "max_allocated": 2014.912
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.0395912700095196,
+            "stdev": 0.0012688804043343982,
+            "values": [
+                0.04130633544921875,
+                0.039196670532226564,
+                0.0389285774230957,
+                0.038898689270019535,
+                0.03908217620849609,
+                0.038921215057373046,
+                0.03887731170654297,
+                0.039103488922119144,
+                0.0392540168762207,
+                0.04269158554077149,
+                0.03945471954345703,
+                0.03774156951904297,
+                0.03752140808105469,
+                0.03751116943359375,
+                0.037665985107421876,
+                0.037651454925537106,
+                0.03749990463256836,
+                0.037599231719970705,
+                0.03752243041992188,
+                0.037577728271484374,
+                0.03758489608764649,
+                0.037623809814453124,
+                0.03761459350585938,
+                0.03751731109619141,
+                0.03762483215332031,
+                0.03757689666748047,
+                0.03895296096801758,
+                0.04037529754638672,
+                0.040079360961914064,
+                0.04017264175415039,
+                0.04002099227905274,
+                0.040068096160888675,
+                0.03998720169067383,
+                0.0401715202331543,
+                0.0400373764038086,
+                0.04016156768798828,
+                0.040123390197753905,
+                0.040005760192871095,
+                0.04004044723510742,
+                0.04008345413208008,
+                0.040120319366455076,
+                0.040130561828613284,
+                0.040035327911376956,
+                0.04048896026611328,
+                0.04011929702758789,
+                0.04001177597045898,
+                0.04003635025024414,
+                0.04006092834472656,
+                0.040199169158935545,
+                0.04007321548461914,
+                0.04013772964477539,
+                0.04007321548461914,
+                0.04011008071899414,
+                0.04019200134277344,
+                0.04008038330078125,
+                0.040521728515625,
+                0.040180736541748044,
+                0.04015513610839844,
+                0.03999948883056641,
+                0.040089599609375,
+                0.04021964645385742,
+                0.04019507217407227,
+                0.04008038330078125,
+                0.04014387130737305,
+                0.04004249572753906,
+                0.04006604766845703,
+                0.04009574508666992,
+                0.0401080322265625,
+                0.040139774322509765,
+                0.040204288482666016,
+                0.040164352416992184,
+                0.04025455856323242,
+                0.04010905456542969,
+                0.04011520004272461,
+                0.04013568115234375,
+                0.040242176055908206,
+                0.04009983825683594,
+                0.040101886749267575,
+                0.04008038330078125,
+                0.04007321548461914,
+                0.040153087615966795,
+                0.04016844940185547,
+                0.040185855865478515,
+                0.04022476959228516,
+                0.04017561721801758,
+                0.040084480285644535,
+                0.0402503662109375,
+                0.04014387130737305,
+                0.04017663955688477,
+                0.04006399917602539,
+                0.040123390197753905,
+                0.04020940780639649,
+                0.04013158416748047,
+                0.04039884948730469,
+                0.040166400909423826,
+                0.0401162223815918,
+                0.040081409454345705,
+                0.04010905456542969,
+                0.04010700988769531,
+                0.04012134552001953,
+                0.04009881591796875,
+                0.040103935241699216,
+                0.040118270874023435,
+                0.040084480285644535,
+                0.040062976837158204,
+                0.04040499114990234,
+                0.040081409454345705,
+                0.04007424163818359,
+                0.04010496139526367,
+                0.04025241470336914,
+                0.040164352416992184,
+                0.04016844940185547,
+                0.04015427017211914,
+                0.040152126312255856,
+                0.0401715202331543,
+                0.04014080047607422,
+                0.04011724853515625,
+                0.04019200134277344,
+                0.04024729537963867,
+                0.04019814300537109,
+                0.04021145629882812,
+                0.04017049789428711,
+                0.04020940780639649,
+                0.04015923309326172,
+                0.040151039123535154,
+                0.04015923309326172,
+                0.04035174560546875,
+                0.04024627304077148,
+                0.04029337692260742,
+                0.040187904357910156,
+                0.04023100662231445,
+                0.040156158447265625,
+                0.040226814270019534,
+                0.0400711669921875,
+                0.040231937408447264,
+                0.0401797103881836,
+                0.04025446319580078,
+                0.04013158416748047,
+                0.0402606086730957,
+                0.04012953567504883,
+                0.040215553283691405,
+                0.04011929702758789,
+                0.04021452713012695,
+                0.040613887786865234,
+                0.040260673522949215,
+                0.04022784042358398,
+                0.04023503875732422,
+                0.040130657196044923,
+                0.040204288482666016,
+                0.040225791931152347,
+                0.040199169158935545,
+                0.04014387130737305,
+                0.04014518356323242,
+                0.04021990585327148,
+                0.04027801513671875,
+                0.040561664581298826,
+                0.040097793579101565,
+                0.04015513610839844,
+                0.040151039123535154,
+                0.04016332626342774,
+                0.04021452713012695,
+                0.04028108978271484,
+                0.04020019149780273,
+                0.04021680068969727,
+                0.04010086441040039,
+                0.040197120666503904,
+                0.040172542572021484,
+                0.04028518295288086,
+                0.040230049133300784,
+                0.04035583877563476,
+                0.04027391815185547,
+                0.04029747009277344,
+                0.040267070770263674,
+                0.04030783843994141,
+                0.040256511688232424,
+                0.04021145629882812,
+                0.04031488037109375,
+                0.04017868804931641,
+                0.04021145629882812,
+                0.040172542572021484,
+                0.040202239990234374,
+                0.04017049789428711,
+                0.0402606086730957,
+                0.04012646484375,
+                0.040153087615966795,
+                0.04008857727050781,
+                0.03771628952026367,
+                0.03776204681396484,
+                0.03787776184082031,
+                0.03766998291015625,
+                0.03782656097412109,
+                0.037820415496826174,
+                0.03776204681396484,
+                0.03774566268920899,
+                0.03817779159545898,
+                0.03770265579223633,
+                0.037748737335205076,
+                0.03772723388671875,
+                0.037663745880126956,
+                0.03777024078369141,
+                0.03778047943115234,
+                0.03773952102661133,
+                0.03775392150878906,
+                0.037749759674072264,
+                0.03916502380371094,
+                0.04332134246826172,
+                0.04401491165161133,
+                0.04322732925415039,
+                0.04298956680297852,
+                0.042708992004394535,
+                0.042501121520996096,
+                0.04294655990600586,
+                0.042656768798828126,
+                0.04047769546508789,
+                0.04006195068359375,
+                0.03965359878540039,
+                0.04196454238891602,
+                0.04081151962280274,
+                0.03944345474243164,
+                0.037792766571044925,
+                0.03770172882080078,
+                0.037695552825927736,
+                0.037766143798828124,
+                0.03765350341796875,
+                0.03768320083618164,
+                0.03773952102661133,
+                0.0378081283569336,
+                0.03789209747314453,
+                0.038095870971679685,
+                0.03766988754272461,
+                0.037822689056396484,
+                0.037781566619873044,
+                0.03784396743774414,
+                0.03779174423217774,
+                0.03767295837402344,
+                0.03775795364379883,
+                0.037735424041748046,
+                0.037746688842773435,
+                0.037719039916992186,
+                0.03772415924072266,
+                0.03782860946655273,
+                0.03778355026245117,
+                0.03777228927612305,
+                0.037766143798828124,
+                0.03772108840942383,
+                0.03775692749023438,
+                0.03823001480102539,
+                0.03817574310302734,
+                0.03778067016601563,
+                0.03769036865234375,
+                0.037694465637207034,
+                0.03770163345336914,
+                0.03773235321044922
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 6466.071938042038
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 985.8048,
+            "max_vram": 2858.418176,
+            "max_reserved": 2373.976064,
+            "max_allocated": 2326.085632
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 7.431077307701109,
+            "stdev": 0.013217331886290484,
+            "values": [
+                7.444294639587399,
+                7.417859975814818
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 34.31534748477637
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.029141479638043543,
+            "stdev": 0.0005050711192377097,
+            "values": [
+                0.029703168869018554,
+                0.0291778564453125,
+                0.02933452796936035,
+                0.029313024520874024,
+                0.029129728317260743,
+                0.029032447814941405,
+                0.02949836730957031,
+                0.02898124885559082,
+                0.029022207260131837,
+                0.029078527450561522,
+                0.028964864730834962,
+                0.029077503204345705,
+                0.02902118492126465,
+                0.029095935821533202,
+                0.029054975509643553,
+                0.029051904678344728,
+                0.029039615631103514,
+                0.029099008560180665,
+                0.029311264038085937,
+                0.028965599060058595,
+                0.029024255752563476,
+                0.029072383880615234,
+                0.02910207939147949,
+                0.029092863082885743,
+                0.029159423828125,
+                0.029129728317260743,
+                0.029054975509643553,
+                0.02899865531921387,
+                0.02894540786743164,
+                0.029014015197753908,
+                0.029047807693481444,
+                0.02897407913208008,
+                0.028985343933105468,
+                0.029139968872070314,
+                0.029106176376342774,
+                0.02910207939147949,
+                0.029076480865478517,
+                0.029108224868774416,
+                0.029034496307373047,
+                0.02917683219909668,
+                0.029055999755859374,
+                0.029123584747314454,
+                0.029088768005371093,
+                0.029100032806396486,
+                0.029112319946289062,
+                0.02916044807434082,
+                0.02897305679321289,
+                0.028948480606079102,
+                0.028964864730834962,
+                0.02897920036315918,
+                0.028888063430786134,
+                0.029284351348876952,
+                0.029041664123535156,
+                0.02917068862915039,
+                0.02909388732910156,
+                0.029018112182617187,
+                0.02901299285888672,
+                0.029030399322509767,
+                0.029236223220825194,
+                0.029176864624023437,
+                0.029003744125366212,
+                0.02901299285888672,
+                0.028999679565429686,
+                0.02898739242553711,
+                0.029082624435424805,
+                0.029055999755859374,
+                0.02894540786743164,
+                0.02898841667175293,
+                0.03653955078125,
+                0.03400688171386719,
+                0.03306291198730469,
+                0.032557056427001956,
+                0.030872575759887694,
+                0.029071359634399413,
+                0.02934886360168457,
+                0.02899865531921387,
+                0.029031423568725585,
+                0.029005823135375978,
+                0.029220127105712892,
+                0.029118175506591796,
+                0.029165567398071288,
+                0.029090816497802735,
+                0.029062143325805666,
+                0.029079551696777343,
+                0.029077503204345705,
+                0.02916147232055664,
+                0.02913484764099121,
+                0.029091840744018556,
+                0.029029375076293946,
+                0.029109247207641603,
+                0.02898636817932129,
+                0.029015039443969725,
+                0.029085695266723634,
+                0.029045759201049806,
+                0.029072383880615234,
+                0.029030399322509767,
+                0.02903654479980469,
+                0.02916966438293457,
+                0.029079551696777343,
+                0.029132799148559572,
+                0.02912665557861328,
+                0.029147136688232423,
+                0.02916659164428711,
+                0.02913484764099121,
+                0.02918502426147461,
+                0.02930790328979492,
+                0.02916761589050293,
+                0.02914406394958496,
+                0.029491199493408202,
+                0.02911846351623535,
+                0.02915635108947754,
+                0.029251583099365236,
+                0.029147136688232423,
+                0.029280256271362305,
+                0.03058892822265625,
+                0.02975948715209961,
+                0.029101055145263673,
+                0.02916454315185547,
+                0.029081600189208984,
+                0.02912563133239746,
+                0.029117439270019533,
+                0.029086719512939452,
+                0.029038591384887694,
+                0.02913587188720703,
+                0.02893824005126953,
+                0.029041664123535156,
+                0.029029375076293946,
+                0.02950553512573242,
+                0.029171712875366212,
+                0.02915123176574707,
+                0.02911027145385742,
+                0.029251583099365236,
+                0.029108224868774416,
+                0.029116416931152345,
+                0.029055999755859374,
+                0.029115392684936525,
+                0.029123584747314454,
+                0.029082624435424805,
+                0.02897715187072754,
+                0.0291276798248291,
+                0.029499391555786132,
+                0.029214719772338867,
+                0.02918604850769043,
+                0.029465599060058592,
+                0.02958438491821289,
+                0.02936729621887207,
+                0.029009920120239258,
+                0.029065216064453125,
+                0.029006847381591795,
+                0.029059072494506837,
+                0.029117439270019533,
+                0.0291409912109375,
+                0.029087743759155273,
+                0.029128704071044922,
+                0.029216768264770508,
+                0.029181951522827147,
+                0.02915443229675293,
+                0.029119359970092774,
+                0.02912665557861328,
+                0.02913587188720703,
+                0.0292096004486084,
+                0.029128704071044922,
+                0.029118560791015626,
+                0.029139871597290038,
+                0.029067264556884766,
+                0.029042688369750977,
+                0.02912563133239746,
+                0.029108224868774416,
+                0.029063167572021483,
+                0.029116416931152345,
+                0.028991487503051756,
+                0.02911846351623535,
+                0.029045759201049806,
+                0.029083648681640626,
+                0.029047807693481444,
+                0.02912563133239746,
+                0.02899456024169922,
+                0.029108224868774416,
+                0.029085695266723634,
+                0.02905292892456055,
+                0.029045759201049806,
+                0.02909491157531738,
+                0.029015039443969725,
+                0.029022207260131837,
+                0.029063167572021483,
+                0.029088768005371093,
+                0.029064191818237304,
+                0.028999679565429686,
+                0.029064191818237304,
+                0.029068288803100587,
+                0.029024255752563476,
+                0.029067264556884766,
+                0.02896691131591797,
+                0.02901094436645508,
+                0.02896281623840332,
+                0.02899046325683594,
+                0.029387775421142577,
+                0.02896895980834961,
+                0.028925952911376954,
+                0.02913484764099121,
+                0.029018112182617187,
+                0.029104127883911132,
+                0.02897203254699707,
+                0.029008895874023437,
+                0.029005823135375978,
+                0.029155328750610353,
+                0.029128704071044922,
+                0.02915328025817871,
+                0.028915712356567383,
+                0.02904473686218262,
+                0.02896691131591797,
+                0.029182975769042968,
+                0.029062143325805666,
+                0.02956800079345703,
+                0.029087743759155273,
+                0.029075456619262696,
+                0.02898227119445801,
+                0.02896895980834961,
+                0.029042688369750977,
+                0.029047807693481444,
+                0.02899660873413086,
+                0.029115392684936525,
+                0.02903654479980469,
+                0.029097984313964844,
+                0.02910310363769531,
+                0.029041664123535156,
+                0.029078527450561522,
+                0.029238271713256835,
+                0.029095935821533202,
+                0.029121536254882813,
+                0.029053951263427736,
+                0.029088768005371093,
+                0.02909491157531738,
+                0.0291727352142334,
+                0.029030399322509767,
+                0.02919219207763672,
+                0.029079551696777343,
+                0.029154304504394532,
+                0.02915839958190918,
+                0.02919219207763672,
+                0.029130752563476563,
+                0.02923417663574219,
+                0.02914201545715332,
+                0.029113344192504883,
+                0.029007871627807616,
+                0.02919321632385254,
+                0.02899046325683594,
+                0.029086719512939452,
+                0.02894643211364746,
+                0.028957696914672853,
+                0.02898841667175293,
+                0.02895871925354004,
+                0.029024255752563476,
+                0.029065216064453125,
+                0.02895564842224121,
+                0.02906403160095215,
+                0.029086719512939452,
+                0.029017087936401367,
+                0.029035520553588868,
+                0.029084672927856447,
+                0.029087743759155273,
+                0.029053951263427736,
+                0.029121536254882813,
+                0.029129728317260743,
+                0.029594623565673828,
+                0.02914508819580078,
+                0.02919321632385254,
+                0.02915225601196289,
+                0.02933964729309082,
+                0.029071359634399413,
+                0.02915123176574707,
+                0.029100032806396486,
+                0.029236223220825194,
+                0.029082624435424805,
+                0.029219839096069337,
+                0.029091840744018556,
+                0.02918809509277344,
+                0.02897715187072754,
+                0.029154304504394532,
+                0.029034496307373047,
+                0.029105152130126953,
+                0.029477888107299805,
+                0.029077503204345705,
+                0.029075456619262696,
+                0.029019136428833008,
+                0.02898227119445801,
+                0.029139968872070314,
+                0.029266944885253908,
+                0.0291276798248291,
+                0.029154304504394532,
+                0.029229055404663085,
+                0.02920140838623047,
+                0.02919424057006836,
+                0.029189119338989256,
+                0.02921779251098633,
+                0.029078527450561522,
+                0.029138944625854493,
+                0.029025279998779296,
+                0.02919628715515137,
+                0.02916454315185547,
+                0.02914406394958496,
+                0.029122560501098634,
+                0.02916659164428711,
+                0.029076480865478517,
+                0.029116416931152345,
+                0.029080575942993164,
+                0.029247488021850586,
+                0.029079551696777343,
+                0.029129728317260743,
+                0.029157375335693358,
+                0.02915328025817871,
+                0.02920243263244629,
+                0.029223936080932617,
+                0.02917683219909668,
+                0.02917068862915039,
+                0.029400224685668944,
+                0.029131616592407227,
+                0.029082624435424805,
+                0.029063167572021483,
+                0.029017087936401367,
+                0.029026304244995117,
+                0.029068288803100587,
+                0.02919219207763672,
+                0.029043712615966798,
+                0.029165567398071288,
+                0.029083648681640626,
+                0.0291276798248291,
+                0.02910207939147949,
+                0.029149183273315428,
+                0.02899865531921387,
+                0.029231103897094726,
+                0.029029375076293946,
+                0.029091840744018556,
+                0.02943712043762207,
+                0.029047615051269533,
+                0.029016063690185546,
+                0.029095935821533202,
+                0.029015039443969725,
+                0.029023231506347655,
+                0.028976127624511717,
+                0.029082624435424805,
+                0.029061279296875,
+                0.029178720474243164,
+                0.02899660873413086,
+                0.029117439270019533,
+                0.029037567138671876,
+                0.029130752563476563,
+                0.029022207260131837,
+                0.029048831939697265,
+                0.02896895980834961,
+                0.029452287673950195,
+                0.02896998405456543,
+                0.029077503204345705,
+                0.0289300479888916,
+                0.02899558448791504,
+                0.028926975250244142,
+                0.029007871627807616,
+                0.02892902374267578,
+                0.029058048248291016,
+                0.02894745635986328,
+                0.029051904678344728,
+                0.028960927963256836,
+                0.02899852752685547,
+                0.028940256118774415,
+                0.029017087936401367,
+                0.02896998405456543,
+                0.029065216064453125,
+                0.029243392944335936,
+                0.029146112442016602,
+                0.02900480079650879,
+                0.029051904678344728,
+                0.02892799949645996,
+                0.029050880432128907,
+                0.02902835273742676,
+                0.029006847381591795,
+                0.029072383880615234,
+                0.029067264556884766,
+                0.029043935775756837,
+                0.029120288848876952,
+                0.02896895980834961,
+                0.0290119686126709,
+                0.0289617919921875,
+                0.029015039443969725,
+                0.029045759201049806,
+                0.02899558448791504,
+                0.028957696914672853,
+                0.028941312789916993,
+                0.02895871925354004,
+                0.02894540786743164,
+                0.029001728057861328,
+                0.02901094436645508,
+                0.028951551437377928,
+                0.029107200622558595,
+                0.029039615631103514,
+                0.02913587188720703,
+                0.029035680770874022,
+                0.02903331184387207,
+                0.028983295440673826,
+                0.028956672668457032,
+                0.02895359992980957,
+                0.029023231506347655,
+                0.028983295440673826,
+                0.02898227119445801,
+                0.029480960845947264,
+                0.03250483322143555,
+                0.028959743499755858,
+                0.02896895980834961,
+                0.02898636817932129,
+                0.02896691131591797,
+                0.028968000411987306,
+                0.029171648025512694,
+                0.02905292892456055,
+                0.02911948776245117,
+                0.02917068862915039,
+                0.029138944625854493,
+                0.029088768005371093,
+                0.029063167572021483,
+                0.029110368728637696,
+                0.029097888946533205,
+                0.029478912353515626,
+                0.029038591384887694,
+                0.029057024002075195,
+                0.02902835273742676,
+                0.029113344192504883,
+                0.029059072494506837,
+                0.029032447814941405,
+                0.029063167572021483,
+                0.029061119079589845,
+                0.02914201545715332,
+                0.029089792251586914,
+                0.029123584747314454,
+                0.029106176376342774,
+                0.029083648681640626,
+                0.029048831939697265,
+                0.029035520553588868,
+                0.029009920120239258,
+                0.029025279998779296,
+                0.028999679565429686,
+                0.029120512008666992,
+                0.029019136428833008,
+                0.029057024002075195,
+                0.029073408126831055,
+                0.029057024002075195,
+                0.029000703811645507,
+                0.029123584747314454,
+                0.029105152130126953,
+                0.029097984313964844,
+                0.02909388732910156,
+                0.029074432373046875,
+                0.02909903907775879,
+                0.02901398468017578,
+                0.028993535995483398,
+                0.029047807693481444,
+                0.02893824005126953,
+                0.02898944091796875,
+                0.028878847122192384,
+                0.029147136688232423,
+                0.028968000411987306,
+                0.02906822395324707,
+                0.029130752563476563,
+                0.029211648941040037,
+                0.02900480079650879,
+                0.029088768005371093,
+                0.029019136428833008,
+                0.029092863082885743,
+                0.029033472061157226,
+                0.029163520812988283,
+                0.029055999755859374,
+                0.029085695266723634,
+                0.029027328491210938,
+                0.02919424057006836,
+                0.029090816497802735,
+                0.02952191925048828,
+                0.02898739242553711,
+                0.029113344192504883,
+                0.02896588706970215,
+                0.029072383880615234,
+                0.02891366386413574,
+                0.0291409912109375,
+                0.029059072494506837,
+                0.029042688369750977,
+                0.029022207260131837,
+                0.029121536254882813,
+                0.02900480079650879,
+                0.029046783447265623,
+                0.02892799949645996,
+                0.029079551696777343,
+                0.028906496047973632,
+                0.029180927276611326,
+                0.029186176300048827,
+                0.029064064025878907,
+                0.028892160415649414,
+                0.029034496307373047,
+                0.02899046325683594,
+                0.029034496307373047,
+                0.02896691131591797,
+                0.029122560501098634,
+                0.02894438362121582,
+                0.029070335388183592,
+                0.0289617919921875,
+                0.029029375076293946,
+                0.028914688110351562,
+                0.029051904678344728,
+                0.02892902374267578,
+                0.02897817611694336,
+                0.02894553565979004,
+                0.029031295776367188,
+                0.02896076774597168,
+                0.02902016067504883,
+                0.028964895248413086
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 34.315347484776396
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}