Upload /Mistral-7B-v0.1/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Mistral-7B-v0.1/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,745 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 2301.93152,
+            "max_vram": 5490.343936,
+            "max_reserved": 5005.901824,
+            "max_allocated": 4844.946944
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.05857614369977985,
+            "stdev": 0.0008302679798154835,
+            "values": [
+                0.06381891250610351,
+                0.06281932830810547,
+                0.06022348785400391,
+                0.05860966491699219,
+                0.0618342399597168,
+                0.0585615348815918,
+                0.05860249710083008,
+                0.05805072021484375,
+                0.05850726318359375,
+                0.058031105041503904,
+                0.05855027389526367,
+                0.05803007888793945,
+                0.05856051254272461,
+                0.058054656982421876,
+                0.05854207992553711,
+                0.0580208625793457,
+                0.05854719924926758,
+                0.05802204895019531,
+                0.05857894515991211,
+                0.05808230209350586,
+                0.05855539321899414,
+                0.0580423698425293,
+                0.05932032012939453,
+                0.059046913146972656,
+                0.05960396957397461,
+                0.05911859130859375,
+                0.059568126678466796,
+                0.05920767974853516,
+                0.06301081466674804,
+                0.06096403121948242,
+                0.060050430297851565,
+                0.05905408096313477,
+                0.05959987258911133,
+                0.05905100631713867,
+                0.059723808288574216,
+                0.05927731323242187,
+                0.05963289642333985,
+                0.05914419174194336,
+                0.059578369140625,
+                0.05913087844848633,
+                0.05960192108154297,
+                0.05910220718383789,
+                0.05963264083862305,
+                0.05902336120605469,
+                0.05855846405029297,
+                0.05803212738037109,
+                0.058598400115966794,
+                0.0580711669921875,
+                0.05856172943115234,
+                0.058028030395507815,
+                0.05850726318359375,
+                0.05857177734375,
+                0.05864243316650391,
+                0.05802598571777344,
+                0.05853081512451172,
+                0.05799628829956055,
+                0.05854003143310547,
+                0.057988094329833983,
+                0.05852671813964844,
+                0.05803212738037109,
+                0.05858614349365234,
+                0.05816320037841797,
+                0.05857593536376953,
+                0.05813145446777344,
+                0.05851136016845703,
+                0.0580761604309082,
+                0.05858816146850586,
+                0.05805363082885742,
+                0.058525760650634764,
+                0.058047489166259764,
+                0.05869465637207031,
+                0.05797273635864258,
+                0.058534912109375,
+                0.05804032135009766,
+                0.058517505645751956,
+                0.058041343688964846,
+                0.05847654342651367,
+                0.05807206344604492,
+                0.05860665512084961,
+                0.058014720916748044,
+                0.05856883239746094,
+                0.05803519821166992,
+                0.058552318572998044,
+                0.05805184173583984,
+                0.05860454559326172,
+                0.05807820892333984,
+                0.058534912109375,
+                0.05807923126220703,
+                0.05852262496948242,
+                0.05813043212890625,
+                0.05863759994506836,
+                0.05811513519287109,
+                0.05857791900634766,
+                0.058016895294189456,
+                0.058554367065429686,
+                0.058039295196533204,
+                0.058689537048339846,
+                0.05808127975463867,
+                0.05857177734375,
+                0.05810287857055664,
+                0.05849190521240234,
+                0.05806284713745117,
+                0.058573951721191404,
+                0.05795123291015625,
+                0.05870796966552734,
+                0.058074111938476565,
+                0.05857279968261719,
+                0.058011646270751956,
+                0.05849702453613281,
+                0.05799116897583008,
+                0.05852774429321289,
+                0.058044414520263675,
+                0.05851238250732422,
+                0.05804339218139649,
+                0.05851238250732422,
+                0.05805055999755859,
+                0.05854003143310547,
+                0.05801491165161133,
+                0.05850009536743164,
+                0.05796454238891602,
+                0.058548225402832034,
+                0.05805363082885742,
+                0.05862707138061524,
+                0.05803014373779297,
+                0.05860966491699219,
+                0.05808947372436524,
+                0.058665985107421874,
+                0.0581130256652832,
+                0.058554367065429686,
+                0.05805977630615235,
+                0.05861580657958984,
+                0.05815091323852539,
+                0.05855846405029297,
+                0.05809561538696289,
+                0.05858201599121094,
+                0.05802188873291016,
+                0.058552318572998044,
+                0.05803427124023437,
+                0.05859532928466797,
+                0.05815014266967773,
+                0.058510337829589844,
+                0.058039295196533204,
+                0.058534912109375,
+                0.058065921783447265,
+                0.058518527984619144,
+                0.05804339218139649,
+                0.05852687835693359,
+                0.057988094329833983,
+                0.0585533447265625,
+                0.05807513427734375,
+                0.05859635162353516,
+                0.05810073471069336,
+                0.05857689666748047,
+                0.058019840240478515,
+                0.058548225402832034,
+                0.05804851150512695,
+                0.05857689666748047,
+                0.058054656982421876,
+                0.05865574264526367,
+                0.05800447845458984,
+                0.058589183807373046,
+                0.05818368148803711,
+                0.05852569580078125,
+                0.05803427124023437,
+                0.05852671813964844,
+                0.058237953186035155,
+                0.058616832733154295,
+                0.05808947372436524,
+                0.05861478424072265,
+                0.05806796646118164,
+                0.05856665420532227
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 4370.379882159469
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 2301.93152,
+            "max_vram": 5710.544896,
+            "max_reserved": 5226.102784,
+            "max_allocated": 4911.274496
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 8.695093782424927,
+            "stdev": 0.007343614578248392,
+            "values": [
+                8.702437397003175,
+                8.687750167846678
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 29.326883226426162
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.034098406989901645,
+            "stdev": 0.00021590978745798326,
+            "values": [
+                0.03628236770629883,
+                0.03566080093383789,
+                0.034069694519042966,
+                0.034637760162353516,
+                0.03399769592285156,
+                0.03394271850585937,
+                0.03400582504272461,
+                0.034051071166992186,
+                0.03412684631347656,
+                0.03408486557006836,
+                0.033995777130126956,
+                0.034131233215332034,
+                0.0339719352722168,
+                0.034051071166992186,
+                0.03400703811645508,
+                0.033979393005371096,
+                0.03400089645385742,
+                0.034164737701416016,
+                0.034124801635742184,
+                0.03409100723266602,
+                0.03402137756347656,
+                0.03407974243164062,
+                0.03397836685180664,
+                0.034019359588623045,
+                0.033979358673095705,
+                0.03401955032348633,
+                0.03403855895996094,
+                0.03402547073364258,
+                0.033980415344238284,
+                0.03414220809936523,
+                0.03406131362915039,
+                0.03420159912109375,
+                0.033979393005371096,
+                0.03399782562255859,
+                0.0340049934387207,
+                0.033941505432128906,
+                0.03395174407958984,
+                0.03401852798461914,
+                0.03400271987915039,
+                0.034016254425048825,
+                0.03416582489013672,
+                0.034086112976074216,
+                0.03400265502929688,
+                0.03410054397583008,
+                0.03403744125366211,
+                0.034029823303222656,
+                0.033952510833740235,
+                0.033960960388183595,
+                0.034405376434326174,
+                0.03412275314331055,
+                0.034081790924072264,
+                0.03401334381103516,
+                0.03394339370727539,
+                0.03402137756347656,
+                0.03395072174072265,
+                0.033980415344238284,
+                0.03403673553466797,
+                0.03390566253662109,
+                0.034050048828125,
+                0.033933311462402346,
+                0.03404083251953125,
+                0.03403987121582031,
+                0.03453228759765625,
+                0.03408281707763672,
+                0.03415347290039063,
+                0.034164737701416016,
+                0.03414323043823242,
+                0.03395177459716797,
+                0.03406844711303711,
+                0.0340766716003418,
+                0.03393535995483398,
+                0.034083839416503905,
+                0.03400806427001953,
+                0.0340766716003418,
+                0.034103294372558594,
+                0.03399884796142578,
+                0.03405311965942383,
+                0.03410227203369141,
+                0.034154495239257815,
+                0.03422822570800781,
+                0.03407155227661133,
+                0.03411558532714844,
+                0.034181121826171876,
+                0.03395072174072265,
+                0.0339343376159668,
+                0.03463996887207031,
+                0.035277729034423826,
+                0.03420998382568359,
+                0.03403142547607422,
+                0.03399785614013672,
+                0.0341800651550293,
+                0.03399884796142578,
+                0.03392716979980469,
+                0.03418931198120117,
+                0.03441049575805664,
+                0.034067455291748046,
+                0.03419340896606445,
+                0.034181121826171876,
+                0.034081790924072264,
+                0.03391795349121094,
+                0.034121726989746096,
+                0.03409113693237305,
+                0.03407244873046875,
+                0.03398860931396484,
+                0.034108417510986325,
+                0.03392739105224609,
+                0.034040607452392575,
+                0.03440127944946289,
+                0.0341319694519043,
+                0.03408281707763672,
+                0.03409203338623047,
+                0.03407257461547852,
+                0.03420467376708984,
+                0.03425491333007812,
+                0.03420870590209961,
+                0.03405311965942383,
+                0.034200576782226565,
+                0.03402239990234375,
+                0.0341401596069336,
+                0.03440127944946289,
+                0.0341739501953125,
+                0.034528255462646484,
+                0.03402239990234375,
+                0.03421004867553711,
+                0.03417984008789063,
+                0.034136062622070314,
+                0.03405619049072266,
+                0.03408099365234375,
+                0.0339832649230957,
+                0.03407769775390625,
+                0.03402239990234375,
+                0.033997886657714846,
+                0.03397727966308594,
+                0.033999870300292966,
+                0.03434188842773438,
+                0.03419443130493164,
+                0.034141185760498044,
+                0.03399270248413086,
+                0.0341739501953125,
+                0.034081790924072264,
+                0.03394047927856445,
+                0.03403673553466797,
+                0.0341319694519043,
+                0.03405740737915039,
+                0.03393209457397461,
+                0.033942527770996093,
+                0.03403059387207031,
+                0.03404800033569336,
+                0.03400908660888672,
+                0.034331649780273435,
+                0.03421184158325195,
+                0.03435443115234375,
+                0.03420134353637695,
+                0.03426816177368164,
+                0.03393228912353516,
+                0.03400908660888672,
+                0.03412684631347656,
+                0.034129150390625,
+                0.03614796829223633,
+                0.03564851379394531,
+                0.03397529602050781,
+                0.034062335968017575,
+                0.03400422286987305,
+                0.03412863922119141,
+                0.03412582397460937,
+                0.034103294372558594,
+                0.03458560180664062,
+                0.03418956756591797,
+                0.034194175720214846,
+                0.03406335830688476,
+                0.03398553466796875,
+                0.0342724494934082,
+                0.034103103637695316,
+                0.03434086227416992,
+                0.03405516815185547,
+                0.03430201721191406,
+                0.03409913635253906,
+                0.03403673553466797,
+                0.033966079711914066,
+                0.03410432052612305,
+                0.03444224166870117,
+                0.03417001724243164,
+                0.03418505477905273,
+                0.03386265563964844,
+                0.03397017669677734,
+                0.034032638549804685,
+                0.0340582389831543,
+                0.03399168014526367,
+                0.034012161254882815,
+                0.034141185760498044,
+                0.03405721664428711,
+                0.03396915054321289,
+                0.034121726989746096,
+                0.03386188888549805,
+                0.03405696105957031,
+                0.03410124969482422,
+                0.034269184112548826,
+                0.03411382293701172,
+                0.034248416900634765,
+                0.034219009399414066,
+                0.03418214416503906,
+                0.034151424407958986,
+                0.0339046401977539,
+                0.03398963165283203,
+                0.033982463836669925,
+                0.03405209732055664,
+                0.03408793640136719,
+                0.03399168014526367,
+                0.034157569885253904,
+                0.034210975646972654,
+                0.03403145599365234,
+                0.03397753524780273,
+                0.03403859329223633,
+                0.0341319694519043,
+                0.03439411163330078,
+                0.03415347290039063,
+                0.034293758392333985,
+                0.03403673553466797,
+                0.03397017669677734,
+                0.03397324752807617,
+                0.03410636901855469,
+                0.03424665451049805,
+                0.034342910766601564,
+                0.03414745712280273,
+                0.03412057495117188,
+                0.03457228851318359,
+                0.03399782562255859,
+                0.03409920120239258,
+                0.03402547073364258,
+                0.03401113510131836,
+                0.03400396728515625,
+                0.03404201507568359,
+                0.03412054443359375,
+                0.034321407318115234,
+                0.03399168014526367,
+                0.03422518539428711,
+                0.03407356643676758,
+                0.03408614349365234,
+                0.03407846450805664,
+                0.03453440093994141,
+                0.033966079711914066,
+                0.03400294494628906,
+                0.0340766716003418,
+                0.0340398063659668,
+                0.03415347290039063,
+                0.03408281707763672,
+                0.03398144149780274,
+                0.03398963165283203,
+                0.03396915054321289,
+                0.034024574279785155,
+                0.03406118392944336,
+                0.03399270248413086,
+                0.03393024063110352,
+                0.03407769775390625,
+                0.033993728637695314,
+                0.03409100723266602,
+                0.03407772827148438,
+                0.034181087493896485,
+                0.03403878402709961,
+                0.033949695587158206,
+                0.034135040283203126,
+                0.033960960388183595,
+                0.03411056137084961,
+                0.03406326293945312,
+                0.03409100723266602,
+                0.03390262222290039,
+                0.034061279296875,
+                0.03401113510131836,
+                0.03412070465087891,
+                0.033941505432128906,
+                0.03399168014526367,
+                0.03433369445800781,
+                0.034135040283203126,
+                0.034249729156494144,
+                0.03402751922607422,
+                0.03406438446044922,
+                0.03402751922607422,
+                0.034086910247802735,
+                0.034106529235839844,
+                0.03399356842041015,
+                0.034092288970947265,
+                0.033952510833740235,
+                0.03446169662475586,
+                0.03402342224121094,
+                0.03408281707763672,
+                0.03402239990234375,
+                0.03410150527954101,
+                0.034180862426757816,
+                0.034127872467041014,
+                0.034127872467041014,
+                0.03403673553466797,
+                0.03419852828979492,
+                0.03410124969482422,
+                0.03399782562255859,
+                0.03407462310791016,
+                0.033979393005371096,
+                0.034492416381835936,
+                0.03398144149780274,
+                0.034055328369140624,
+                0.0340478401184082,
+                0.0340316162109375,
+                0.03405926513671875,
+                0.03417599868774414,
+                0.034097152709960936,
+                0.034233345031738284,
+                0.03410124969482422,
+                0.03407257461547852,
+                0.033928192138671875,
+                0.033993728637695314,
+                0.03400294494628906,
+                0.03401321411132813,
+                0.03391075134277344,
+                0.033941505432128906,
+                0.0339046401977539,
+                0.03389235305786133,
+                0.03415683364868164,
+                0.03391561508178711,
+                0.03406643295288086,
+                0.033915008544921875,
+                0.03405401611328125,
+                0.03401523208618164,
+                0.03400601577758789,
+                0.03399996948242188,
+                0.034083744049072266,
+                0.03423344039916992,
+                0.03428956985473633,
+                0.034285568237304685,
+                0.03408486557006836,
+                0.03400294494628906,
+                0.03427328109741211,
+                0.0341401596069336,
+                0.03427840042114258,
+                0.034034687042236327,
+                0.03410432052612305,
+                0.034202625274658206,
+                0.033977344512939454,
+                0.03398556900024414,
+                0.034028511047363284,
+                0.03398963165283203,
+                0.033857536315917966,
+                0.03380940628051758,
+                0.03450368118286133,
+                0.03392931365966797,
+                0.03396700668334961,
+                0.03400316619873047,
+                0.0340098876953125,
+                0.03404390335083008,
+                0.03405215835571289,
+                0.0340695686340332,
+                0.03408883285522461,
+                0.0339918098449707,
+                0.03410726547241211,
+                0.03415552139282227,
+                0.03422003173828125,
+                0.034081790924072264,
+                0.03435007858276367,
+                0.03429683303833008,
+                0.03409408187866211,
+                0.03406643295288086,
+                0.03416985702514649,
+                0.0342108154296875,
+                0.034151424407958986,
+                0.0341739501953125,
+                0.03408486557006836,
+                0.03400908660888672,
+                0.034136062622070314,
+                0.0341739501953125,
+                0.033947647094726564,
+                0.03400089645385742,
+                0.03400806427001953,
+                0.034118656158447266,
+                0.03423539352416992,
+                0.03395686340332031,
+                0.034132991790771484,
+                0.03408588790893555,
+                0.03402649688720703,
+                0.03408281707763672,
+                0.03407257461547852,
+                0.03407769775390625,
+                0.034097152709960936,
+                0.03401318359375,
+                0.034118656158447266,
+                0.03398553466796875,
+                0.03404800033569336,
+                0.03410124969482422,
+                0.034016254425048825,
+                0.03385139083862305,
+                0.03412582397460937,
+                0.0339159049987793,
+                0.034051071166992186,
+                0.03405209732055664,
+                0.03405516815185547,
+                0.034100223541259765,
+                0.0340316162109375,
+                0.03413401412963867,
+                0.03420467376708984,
+                0.034171009063720705,
+                0.0342342414855957,
+                0.03423660659790039,
+                0.034111297607421875,
+                0.034010112762451174,
+                0.03454873657226563,
+                0.03408816146850586,
+                0.03411740875244141,
+                0.03412070465087891,
+                0.03405414581298828,
+                0.034078720092773435,
+                0.03394559860229492,
+                0.03400089645385742,
+                0.03396710586547851,
+                0.03402239990234375,
+                0.033979393005371096,
+                0.033915969848632814,
+                0.033915840148925784,
+                0.03395276641845703,
+                0.03442073440551758,
+                0.03392419052124023,
+                0.03398953628540039,
+                0.03402678298950195,
+                0.03412041473388672,
+                0.03397836685180664,
+                0.03405926513671875,
+                0.034098175048828124,
+                0.034055233001708984,
+                0.03411347198486328,
+                0.03404390335083008,
+                0.03412188720703125,
+                0.034101089477539065,
+                0.033991935729980466,
+                0.03402931213378906,
+                0.034159713745117185,
+                0.034114463806152344,
+                0.03398963165283203,
+                0.03418316650390625,
+                0.034050048828125,
+                0.03405209732055664,
+                0.034123966217041016,
+                0.0341512336730957,
+                0.033997886657714846,
+                0.03404179382324219,
+                0.034022689819335934,
+                0.03412656021118164,
+                0.034103294372558594,
+                0.033999870300292966,
+                0.033821697235107424,
+                0.03398553466796875,
+                0.03391795349121094,
+                0.033854465484619144,
+                0.03394047927856445,
+                0.03391692733764649,
+                0.03389132690429687,
+                0.03437977600097656,
+                0.03417190551757812,
+                0.03403776168823242,
+                0.03397856140136719,
+                0.03391468811035156,
+                0.03411763381958008,
+                0.03391897583007813,
+                0.034062335968017575,
+                0.034083839416503905,
+                0.03448320007324219,
+                0.0339947509765625,
+                0.033877246856689455,
+                0.033944320678710935,
+                0.03401420974731445,
+                0.03403571319580078,
+                0.03406335830688476,
+                0.033979393005371096,
+                0.03424374389648437,
+                0.03400278472900391,
+                0.034034687042236327,
+                0.03404083251953125,
+                0.034141185760498044,
+                0.03418521499633789,
+                0.03460505676269531,
+                0.03421184158325195,
+                0.03398361587524414,
+                0.03405811309814453,
+                0.0340766716003418,
+                0.03408588790893555,
+                0.034078720092773435,
+                0.03402751922607422,
+                0.03404185485839844,
+                0.03400908660888672,
+                0.0341401596069336,
+                0.03412582397460937,
+                0.03403673553466797,
+                0.03415961456298828,
+                0.03410227203369141,
+                0.03419647979736328,
+                0.034081790924072264,
+                0.03415347290039063,
+                0.03402956771850586,
+                0.03396915054321289,
+                0.0339947509765625,
+                0.034049022674560545,
+                0.03402444839477539,
+                0.03402956771850586,
+                0.034067455291748046,
+                0.03402444839477539,
+                0.03407676696777344,
+                0.034051998138427735,
+                0.03416892623901367,
+                0.033885089874267575,
+                0.033993728637695314,
+                0.03404288101196289,
+                0.03388108825683594,
+                0.03398553466796875,
+                0.03392435073852539,
+                0.03385011291503906
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 29.326883226426187
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}