kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Mar 14

Commit

2456a8c

•

1 Parent(s): 3a9adf4

Upload /Mistral-7B-v0.1/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Mistral-7B-v0.1/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +692 -690

Mistral-7B-v0.1/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,192 +2,194 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 2301.93152,
             "max_vram": 5490.343936,
             "max_reserved": 5005.901824,
             "max_allocated": 4844.946944
         },
         "latency": {
             "unit": "s",
-            "mean": 0.05857614369977985,
-            "stdev": 0.0008302679798154835,
             "values": [
-                0.06381891250610351,
-                0.06281932830810547,
-                0.06022348785400391,
-                0.05860966491699219,
-                0.0618342399597168,
-                0.0585615348815918,
-                0.05860249710083008,
-                0.05805072021484375,
-                0.05850726318359375,
                 0.058031105041503904,
-                0.05855027389526367,
-                0.05803007888793945,
-                0.05856051254272461,
                 0.058054656982421876,
-                0.05854207992553711,
-                0.0580208625793457,
-                0.05854719924926758,
-                0.05802204895019531,
-                0.05857894515991211,
-                0.05808230209350586,
-                0.05855539321899414,
-                0.0580423698425293,
-                0.05932032012939453,
-                0.059046913146972656,
-                0.05960396957397461,
-                0.05911859130859375,
-                0.059568126678466796,
-                0.05920767974853516,
-                0.06301081466674804,
-                0.06096403121948242,
-                0.060050430297851565,
-                0.05905408096313477,
-                0.05959987258911133,
-                0.05905100631713867,
-                0.059723808288574216,
-                0.05927731323242187,
-                0.05963289642333985,
-                0.05914419174194336,
-                0.059578369140625,
-                0.05913087844848633,
-                0.05960192108154297,
-                0.05910220718383789,
-                0.05963264083862305,
-                0.05902336120605469,
-                0.05855846405029297,
-                0.05803212738037109,
-                0.058598400115966794,
-                0.0580711669921875,
-                0.05856172943115234,
-                0.058028030395507815,
-                0.05850726318359375,
-                0.05857177734375,
-                0.05864243316650391,
                 0.05802598571777344,
-                0.05853081512451172,
-                0.05799628829956055,
-                0.05854003143310547,
-                0.057988094329833983,
-                0.05852671813964844,
                 0.05803212738037109,
-                0.05858614349365234,
-                0.05816320037841797,
-                0.05857593536376953,
-                0.05813145446777344,
-                0.05851136016845703,
-                0.0580761604309082,
-                0.05858816146850586,
-                0.05805363082885742,
-                0.058525760650634764,
-                0.058047489166259764,
-                0.05869465637207031,
-                0.05797273635864258,
-                0.058534912109375,
-                0.05804032135009766,
-                0.058517505645751956,
-                0.058041343688964846,
-                0.05847654342651367,
-                0.05807206344604492,
-                0.05860665512084961,
-                0.058014720916748044,
-                0.05856883239746094,
-                0.05803519821166992,
-                0.058552318572998044,
-                0.05805184173583984,
-                0.05860454559326172,
-                0.05807820892333984,
-                0.058534912109375,
-                0.05807923126220703,
-                0.05852262496948242,
-                0.05813043212890625,
-                0.05863759994506836,
-                0.05811513519287109,
-                0.05857791900634766,
-                0.058016895294189456,
-                0.058554367065429686,
-                0.058039295196533204,
-                0.058689537048339846,
-                0.05808127975463867,
-                0.05857177734375,
-                0.05810287857055664,
-                0.05849190521240234,
-                0.05806284713745117,
-                0.058573951721191404,
-                0.05795123291015625,
-                0.05870796966552734,
-                0.058074111938476565,
-                0.05857279968261719,
-                0.058011646270751956,
-                0.05849702453613281,
-                0.05799116897583008,
-                0.05852774429321289,
-                0.058044414520263675,
-                0.05851238250732422,
-                0.05804339218139649,
-                0.05851238250732422,
-                0.05805055999755859,
-                0.05854003143310547,
-                0.05801491165161133,
-                0.05850009536743164,
-                0.05796454238891602,
-                0.058548225402832034,
-                0.05805363082885742,
-                0.05862707138061524,
-                0.05803014373779297,
-                0.05860966491699219,
-                0.05808947372436524,
-                0.058665985107421874,
-                0.0581130256652832,
-                0.058554367065429686,
-                0.05805977630615235,
-                0.05861580657958984,
-                0.05815091323852539,
-                0.05855846405029297,
-                0.05809561538696289,
-                0.05858201599121094,
-                0.05802188873291016,
-                0.058552318572998044,
-                0.05803427124023437,
-                0.05859532928466797,
-                0.05815014266967773,
-                0.058510337829589844,
-                0.058039295196533204,
-                0.058534912109375,
-                0.058065921783447265,
-                0.058518527984619144,
-                0.05804339218139649,
-                0.05852687835693359,
-                0.057988094329833983,
-                0.0585533447265625,
-                0.05807513427734375,
-                0.05859635162353516,
-                0.05810073471069336,
-                0.05857689666748047,
-                0.058019840240478515,
-                0.058548225402832034,
-                0.05804851150512695,
-                0.05857689666748047,
-                0.058054656982421876,
-                0.05865574264526367,
-                0.05800447845458984,
-                0.058589183807373046,
-                0.05818368148803711,
-                0.05852569580078125,
-                0.05803427124023437,
-                0.05852671813964844,
-                0.058237953186035155,
-                0.058616832733154295,
-                0.05808947372436524,
-                0.05861478424072265,
-                0.05806796646118164,
-                0.05856665420532227
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 4370.379882159469
         },
         "energy": null,
         "efficiency": null
@@ -195,23 +197,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 2301.93152,
             "max_vram": 5710.544896,
             "max_reserved": 5226.102784,
             "max_allocated": 4911.274496
         },
         "latency": {
             "unit": "s",
-            "mean": 8.695093782424927,
-            "stdev": 0.007343614578248392,
             "values": [
-                8.702437397003175,
-                8.687750167846678
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 29.326883226426162
         },
         "energy": null,
         "efficiency": null
@@ -220,524 +222,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.034098406989901645,
-            "stdev": 0.00021590978745798326,
             "values": [
-                0.03628236770629883,
-                0.03566080093383789,
-                0.034069694519042966,
-                0.034637760162353516,
-                0.03399769592285156,
-                0.03394271850585937,
-                0.03400582504272461,
-                0.034051071166992186,
-                0.03412684631347656,
-                0.03408486557006836,
-                0.033995777130126956,
-                0.034131233215332034,
-                0.0339719352722168,
-                0.034051071166992186,
-                0.03400703811645508,
-                0.033979393005371096,
-                0.03400089645385742,
-                0.034164737701416016,
-                0.034124801635742184,
-                0.03409100723266602,
-                0.03402137756347656,
-                0.03407974243164062,
-                0.03397836685180664,
-                0.034019359588623045,
-                0.033979358673095705,
-                0.03401955032348633,
-                0.03403855895996094,
-                0.03402547073364258,
-                0.033980415344238284,
-                0.03414220809936523,
-                0.03406131362915039,
-                0.03420159912109375,
-                0.033979393005371096,
-                0.03399782562255859,
-                0.0340049934387207,
-                0.033941505432128906,
-                0.03395174407958984,
-                0.03401852798461914,
-                0.03400271987915039,
-                0.034016254425048825,
-                0.03416582489013672,
-                0.034086112976074216,
-                0.03400265502929688,
-                0.03410054397583008,
-                0.03403744125366211,
-                0.034029823303222656,
-                0.033952510833740235,
-                0.033960960388183595,
-                0.034405376434326174,
-                0.03412275314331055,
-                0.034081790924072264,
-                0.03401334381103516,
-                0.03394339370727539,
-                0.03402137756347656,
-                0.03395072174072265,
-                0.033980415344238284,
-                0.03403673553466797,
-                0.03390566253662109,
-                0.034050048828125,
-                0.033933311462402346,
-                0.03404083251953125,
-                0.03403987121582031,
-                0.03453228759765625,
-                0.03408281707763672,
-                0.03415347290039063,
-                0.034164737701416016,
-                0.03414323043823242,
-                0.03395177459716797,
-                0.03406844711303711,
-                0.0340766716003418,
-                0.03393535995483398,
-                0.034083839416503905,
-                0.03400806427001953,
-                0.0340766716003418,
-                0.034103294372558594,
-                0.03399884796142578,
-                0.03405311965942383,
-                0.03410227203369141,
-                0.034154495239257815,
-                0.03422822570800781,
-                0.03407155227661133,
-                0.03411558532714844,
-                0.034181121826171876,
-                0.03395072174072265,
-                0.0339343376159668,
-                0.03463996887207031,
-                0.035277729034423826,
-                0.03420998382568359,
-                0.03403142547607422,
-                0.03399785614013672,
-                0.0341800651550293,
-                0.03399884796142578,
-                0.03392716979980469,
-                0.03418931198120117,
-                0.03441049575805664,
-                0.034067455291748046,
-                0.03419340896606445,
-                0.034181121826171876,
-                0.034081790924072264,
-                0.03391795349121094,
-                0.034121726989746096,
-                0.03409113693237305,
-                0.03407244873046875,
-                0.03398860931396484,
-                0.034108417510986325,
-                0.03392739105224609,
-                0.034040607452392575,
-                0.03440127944946289,
-                0.0341319694519043,
-                0.03408281707763672,
-                0.03409203338623047,
-                0.03407257461547852,
-                0.03420467376708984,
-                0.03425491333007812,
-                0.03420870590209961,
-                0.03405311965942383,
-                0.034200576782226565,
-                0.03402239990234375,
-                0.0341401596069336,
-                0.03440127944946289,
-                0.0341739501953125,
-                0.034528255462646484,
-                0.03402239990234375,
-                0.03421004867553711,
-                0.03417984008789063,
-                0.034136062622070314,
-                0.03405619049072266,
-                0.03408099365234375,
-                0.0339832649230957,
-                0.03407769775390625,
-                0.03402239990234375,
-                0.033997886657714846,
-                0.03397727966308594,
-                0.033999870300292966,
-                0.03434188842773438,
-                0.03419443130493164,
-                0.034141185760498044,
-                0.03399270248413086,
-                0.0341739501953125,
-                0.034081790924072264,
-                0.03394047927856445,
-                0.03403673553466797,
-                0.0341319694519043,
-                0.03405740737915039,
-                0.03393209457397461,
-                0.033942527770996093,
-                0.03403059387207031,
-                0.03404800033569336,
-                0.03400908660888672,
-                0.034331649780273435,
-                0.03421184158325195,
-                0.03435443115234375,
-                0.03420134353637695,
-                0.03426816177368164,
-                0.03393228912353516,
-                0.03400908660888672,
-                0.03412684631347656,
-                0.034129150390625,
-                0.03614796829223633,
-                0.03564851379394531,
-                0.03397529602050781,
-                0.034062335968017575,
-                0.03400422286987305,
-                0.03412863922119141,
-                0.03412582397460937,
-                0.034103294372558594,
-                0.03458560180664062,
-                0.03418956756591797,
-                0.034194175720214846,
-                0.03406335830688476,
-                0.03398553466796875,
-                0.0342724494934082,
-                0.034103103637695316,
-                0.03434086227416992,
-                0.03405516815185547,
-                0.03430201721191406,
-                0.03409913635253906,
-                0.03403673553466797,
-                0.033966079711914066,
-                0.03410432052612305,
-                0.03444224166870117,
-                0.03417001724243164,
-                0.03418505477905273,
-                0.03386265563964844,
-                0.03397017669677734,
-                0.034032638549804685,
-                0.0340582389831543,
-                0.03399168014526367,
-                0.034012161254882815,
-                0.034141185760498044,
-                0.03405721664428711,
-                0.03396915054321289,
-                0.034121726989746096,
-                0.03386188888549805,
-                0.03405696105957031,
-                0.03410124969482422,
-                0.034269184112548826,
-                0.03411382293701172,
-                0.034248416900634765,
-                0.034219009399414066,
-                0.03418214416503906,
-                0.034151424407958986,
-                0.0339046401977539,
-                0.03398963165283203,
-                0.033982463836669925,
-                0.03405209732055664,
-                0.03408793640136719,
-                0.03399168014526367,
-                0.034157569885253904,
-                0.034210975646972654,
-                0.03403145599365234,
-                0.03397753524780273,
-                0.03403859329223633,
-                0.0341319694519043,
-                0.03439411163330078,
-                0.03415347290039063,
-                0.034293758392333985,
-                0.03403673553466797,
-                0.03397017669677734,
-                0.03397324752807617,
-                0.03410636901855469,
-                0.03424665451049805,
-                0.034342910766601564,
-                0.03414745712280273,
-                0.03412057495117188,
-                0.03457228851318359,
-                0.03399782562255859,
-                0.03409920120239258,
-                0.03402547073364258,
-                0.03401113510131836,
-                0.03400396728515625,
-                0.03404201507568359,
-                0.03412054443359375,
-                0.034321407318115234,
-                0.03399168014526367,
-                0.03422518539428711,
-                0.03407356643676758,
-                0.03408614349365234,
-                0.03407846450805664,
-                0.03453440093994141,
-                0.033966079711914066,
-                0.03400294494628906,
-                0.0340766716003418,
-                0.0340398063659668,
-                0.03415347290039063,
-                0.03408281707763672,
-                0.03398144149780274,
-                0.03398963165283203,
-                0.03396915054321289,
-                0.034024574279785155,
-                0.03406118392944336,
-                0.03399270248413086,
-                0.03393024063110352,
-                0.03407769775390625,
-                0.033993728637695314,
-                0.03409100723266602,
-                0.03407772827148438,
-                0.034181087493896485,
-                0.03403878402709961,
-                0.033949695587158206,
-                0.034135040283203126,
-                0.033960960388183595,
-                0.03411056137084961,
-                0.03406326293945312,
-                0.03409100723266602,
-                0.03390262222290039,
-                0.034061279296875,
-                0.03401113510131836,
-                0.03412070465087891,
-                0.033941505432128906,
-                0.03399168014526367,
-                0.03433369445800781,
-                0.034135040283203126,
-                0.034249729156494144,
-                0.03402751922607422,
-                0.03406438446044922,
-                0.03402751922607422,
-                0.034086910247802735,
-                0.034106529235839844,
-                0.03399356842041015,
-                0.034092288970947265,
-                0.033952510833740235,
-                0.03446169662475586,
-                0.03402342224121094,
-                0.03408281707763672,
-                0.03402239990234375,
-                0.03410150527954101,
-                0.034180862426757816,
-                0.034127872467041014,
-                0.034127872467041014,
-                0.03403673553466797,
-                0.03419852828979492,
-                0.03410124969482422,
-                0.03399782562255859,
-                0.03407462310791016,
-                0.033979393005371096,
-                0.034492416381835936,
-                0.03398144149780274,
-                0.034055328369140624,
-                0.0340478401184082,
-                0.0340316162109375,
-                0.03405926513671875,
-                0.03417599868774414,
-                0.034097152709960936,
-                0.034233345031738284,
-                0.03410124969482422,
-                0.03407257461547852,
-                0.033928192138671875,
-                0.033993728637695314,
-                0.03400294494628906,
-                0.03401321411132813,
-                0.03391075134277344,
-                0.033941505432128906,
-                0.0339046401977539,
-                0.03389235305786133,
-                0.03415683364868164,
-                0.03391561508178711,
-                0.03406643295288086,
-                0.033915008544921875,
-                0.03405401611328125,
-                0.03401523208618164,
-                0.03400601577758789,
-                0.03399996948242188,
-                0.034083744049072266,
-                0.03423344039916992,
-                0.03428956985473633,
-                0.034285568237304685,
-                0.03408486557006836,
-                0.03400294494628906,
-                0.03427328109741211,
-                0.0341401596069336,
-                0.03427840042114258,
-                0.034034687042236327,
-                0.03410432052612305,
-                0.034202625274658206,
-                0.033977344512939454,
-                0.03398556900024414,
-                0.034028511047363284,
-                0.03398963165283203,
-                0.033857536315917966,
-                0.03380940628051758,
-                0.03450368118286133,
-                0.03392931365966797,
-                0.03396700668334961,
-                0.03400316619873047,
-                0.0340098876953125,
-                0.03404390335083008,
-                0.03405215835571289,
-                0.0340695686340332,
-                0.03408883285522461,
-                0.0339918098449707,
-                0.03410726547241211,
-                0.03415552139282227,
-                0.03422003173828125,
-                0.034081790924072264,
-                0.03435007858276367,
-                0.03429683303833008,
-                0.03409408187866211,
-                0.03406643295288086,
-                0.03416985702514649,
-                0.0342108154296875,
-                0.034151424407958986,
-                0.0341739501953125,
-                0.03408486557006836,
-                0.03400908660888672,
-                0.034136062622070314,
-                0.0341739501953125,
-                0.033947647094726564,
-                0.03400089645385742,
-                0.03400806427001953,
-                0.034118656158447266,
-                0.03423539352416992,
-                0.03395686340332031,
-                0.034132991790771484,
-                0.03408588790893555,
-                0.03402649688720703,
-                0.03408281707763672,
-                0.03407257461547852,
-                0.03407769775390625,
-                0.034097152709960936,
-                0.03401318359375,
-                0.034118656158447266,
-                0.03398553466796875,
-                0.03404800033569336,
-                0.03410124969482422,
-                0.034016254425048825,
-                0.03385139083862305,
-                0.03412582397460937,
-                0.0339159049987793,
-                0.034051071166992186,
-                0.03405209732055664,
-                0.03405516815185547,
-                0.034100223541259765,
-                0.0340316162109375,
-                0.03413401412963867,
-                0.03420467376708984,
-                0.034171009063720705,
-                0.0342342414855957,
-                0.03423660659790039,
-                0.034111297607421875,
-                0.034010112762451174,
-                0.03454873657226563,
-                0.03408816146850586,
-                0.03411740875244141,
-                0.03412070465087891,
-                0.03405414581298828,
-                0.034078720092773435,
-                0.03394559860229492,
-                0.03400089645385742,
-                0.03396710586547851,
-                0.03402239990234375,
-                0.033979393005371096,
-                0.033915969848632814,
-                0.033915840148925784,
-                0.03395276641845703,
-                0.03442073440551758,
-                0.03392419052124023,
-                0.03398953628540039,
-                0.03402678298950195,
-                0.03412041473388672,
-                0.03397836685180664,
-                0.03405926513671875,
-                0.034098175048828124,
-                0.034055233001708984,
-                0.03411347198486328,
-                0.03404390335083008,
-                0.03412188720703125,
-                0.034101089477539065,
-                0.033991935729980466,
-                0.03402931213378906,
-                0.034159713745117185,
-                0.034114463806152344,
-                0.03398963165283203,
-                0.03418316650390625,
-                0.034050048828125,
-                0.03405209732055664,
-                0.034123966217041016,
-                0.0341512336730957,
-                0.033997886657714846,
-                0.03404179382324219,
-                0.034022689819335934,
-                0.03412656021118164,
-                0.034103294372558594,
-                0.033999870300292966,
-                0.033821697235107424,
-                0.03398553466796875,
-                0.03391795349121094,
-                0.033854465484619144,
-                0.03394047927856445,
-                0.03391692733764649,
-                0.03389132690429687,
-                0.03437977600097656,
-                0.03417190551757812,
-                0.03403776168823242,
-                0.03397856140136719,
-                0.03391468811035156,
-                0.03411763381958008,
-                0.03391897583007813,
-                0.034062335968017575,
-                0.034083839416503905,
-                0.03448320007324219,
-                0.0339947509765625,
-                0.033877246856689455,
-                0.033944320678710935,
-                0.03401420974731445,
-                0.03403571319580078,
-                0.03406335830688476,
-                0.033979393005371096,
-                0.03424374389648437,
-                0.03400278472900391,
-                0.034034687042236327,
-                0.03404083251953125,
-                0.034141185760498044,
-                0.03418521499633789,
-                0.03460505676269531,
-                0.03421184158325195,
-                0.03398361587524414,
-                0.03405811309814453,
-                0.0340766716003418,
-                0.03408588790893555,
-                0.034078720092773435,
-                0.03402751922607422,
-                0.03404185485839844,
-                0.03400908660888672,
-                0.0341401596069336,
-                0.03412582397460937,
-                0.03403673553466797,
-                0.03415961456298828,
-                0.03410227203369141,
-                0.03419647979736328,
-                0.034081790924072264,
-                0.03415347290039063,
-                0.03402956771850586,
-                0.03396915054321289,
-                0.0339947509765625,
-                0.034049022674560545,
-                0.03402444839477539,
-                0.03402956771850586,
-                0.034067455291748046,
-                0.03402444839477539,
-                0.03407676696777344,
-                0.034051998138427735,
-                0.03416892623901367,
-                0.033885089874267575,
-                0.033993728637695314,
-                0.03404288101196289,
-                0.03388108825683594,
-                0.03398553466796875,
-                0.03392435073852539,
-                0.03385011291503906
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 29.326883226426187
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 2520.65792,
             "max_vram": 5490.343936,
             "max_reserved": 5005.901824,
             "max_allocated": 4844.946944
         },
         "latency": {
             "unit": "s",
+            "mean": 0.05782990103925586,
+            "stdev": 0.0005964412042560866,
             "values": [
+                0.06358777618408203,
+                0.06096588897705078,
+                0.05835059356689453,
+                0.05796454238891602,
+                0.06010879898071289,
+                0.05883596801757812,
+                0.05797478485107422,
+                0.057490432739257816,
+                0.058055679321289064,
+                0.057474048614501956,
+                0.05794508743286133,
+                0.057530399322509765,
+                0.058011646270751956,
+                0.057504768371582034,
+                0.05803519821166992,
+                0.05745471954345703,
+                0.058103809356689455,
+                0.05746585464477539,
+                0.058008575439453126,
+                0.05751603317260742,
+                0.05798297500610351,
+                0.05747711944580078,
+                0.05790617752075195,
+                0.05746585464477539,
+                0.05800755310058594,
+                0.057613311767578126,
+                0.05802905654907226,
+                0.057499870300292966,
+                0.0579317741394043,
+                0.057447425842285155,
+                0.05799628829956055,
+                0.057475070953369144,
+                0.05798297500610351,
+                0.05747097778320313,
+                0.05802598571777344,
+                0.05750886535644531,
+                0.057940990447998046,
+                0.05746790313720703,
+                0.057981952667236325,
+                0.05747945785522461,
+                0.05788671875,
+                0.05744144058227539,
+                0.057985023498535154,
+                0.05744435119628906,
+                0.05794406509399414,
+                0.057447425842285155,
+                0.05793382263183594,
+                0.05748223876953125,
+                0.05795635223388672,
+                0.05742601776123047,
+                0.05796761703491211,
+                0.05755392074584961,
+                0.057981952667236325,
+                0.05740851211547852,
+                0.05795123291015625,
+                0.05747097778320313,
+                0.05796761703491211,
+                0.05746905517578125,
+                0.05791846466064453,
+                0.057673728942871094,
+                0.0579317741394043,
+                0.05744876861572266,
                 0.058031105041503904,
+                0.05749657440185547,
+                0.058363903045654295,
+                0.05752627182006836,
+                0.05801062393188477,
+                0.0574730224609375,
+                0.0579317741394043,
+                0.05747817611694336,
+                0.05791846466064453,
+                0.05748534393310547,
+                0.05795840072631836,
+                0.05748428726196289,
+                0.05795328140258789,
+                0.05747711944580078,
+                0.0579317741394043,
+                0.05742182540893555,
+                0.05800447845458984,
+                0.05746380615234375,
+                0.05791743850708008,
+                0.057507839202880856,
+                0.05794713592529297,
+                0.05748428726196289,
+                0.057913345336914064,
+                0.05747711944580078,
+                0.05811404800415039,
+                0.05752115249633789,
+                0.05795123291015625,
+                0.05747609710693359,
+                0.05806182479858398,
+                0.057431041717529295,
+                0.058003551483154295,
+                0.057504768371582034,
+                0.05822873687744141,
+                0.05754470443725586,
+                0.057981952667236325,
+                0.05751193618774414,
+                0.057957374572753906,
+                0.057504768371582034,
+                0.05802598571777344,
+                0.05748428726196289,
+                0.05797785568237305,
+                0.05748428726196289,
+                0.05798912048339844,
+                0.05749248123168945,
+                0.05801369476318359,
+                0.057491455078125,
+                0.0580055046081543,
+                0.05748735809326172,
+                0.05794815826416016,
+                0.05745356750488281,
+                0.057985023498535154,
+                0.05748121643066406,
+                0.05798092651367188,
+                0.05744947052001953,
+                0.05799935913085937,
+                0.05751910400390625,
                 0.058054656982421876,
+                0.05749862289428711,
+                0.05798912048339844,
+                0.057724929809570315,
+                0.058025054931640625,
+                0.05746380615234375,
+                0.05796147155761719,
+                0.05753753662109375,
+                0.058057727813720705,
+                0.05752217483520508,
+                0.05796044921875,
+                0.05764198303222656,
+                0.05800960159301758,
+                0.05755904006958008,
+                0.05798096084594727,
+                0.05750374221801758,
+                0.05801369476318359,
+                0.057490432739257816,
+                0.05797683334350586,
+                0.05748633575439453,
+                0.05793088150024414,
+                0.05748553466796875,
+                0.058016769409179686,
+                0.05753548812866211,
+                0.058022911071777344,
+                0.057499870300292966,
+                0.058057727813720705,
+                0.057475070953369144,
+                0.05803007888793945,
+                0.05758771133422851,
+                0.057985023498535154,
+                0.05752115249633789,
+                0.05801574325561523,
+                0.05750681686401367,
+                0.05796454238891602,
+                0.05759078216552734,
                 0.05802598571777344,
+                0.05755084609985352,
+                0.05821440124511719,
+                0.057540607452392575,
+                0.05798400115966797,
+                0.057490432739257816,
+                0.05804646301269531,
+                0.057545726776123046,
+                0.05795644760131836,
+                0.057491455078125,
+                0.058218494415283206,
+                0.057534465789794924,
+                0.05807308959960938,
+                0.05757952117919922,
+                0.058028030395507815,
+                0.05753548812866211,
                 0.05803212738037109,
+                0.05754982376098633,
+                0.05800447845458984
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 4426.775688691273
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 2521.706496,
             "max_vram": 5710.544896,
             "max_reserved": 5226.102784,
             "max_allocated": 4911.274496
         },
         "latency": {
             "unit": "s",
+            "mean": 8.440941043853758,
+            "stdev": 0.011413997650144836,
             "values": [
+                8.452355041503903,
+                8.429527046203614
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 30.20990179592326
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.03310172958374023,
+            "stdev": 0.00021079061247777177,
             "values": [
+                0.03572844696044922,
+                0.035100608825683596,
+                0.03332723236083984,
+                0.033161407470703126,
+                0.03317523193359375,
+                0.03329228973388672,
+                0.03314790344238281,
+                0.033102848052978515,
+                0.033140735626220705,
+                0.0331387825012207,
+                0.0332327995300293,
+                0.033122337341308594,
+                0.03316016006469726,
+                0.03321548843383789,
+                0.03321651077270508,
+                0.03313151931762695,
+                0.033123615264892575,
+                0.03307388687133789,
+                0.03308544158935547,
+                0.03306291198730469,
+                0.033051647186279294,
+                0.033107967376708985,
+                0.03318374252319336,
+                0.03298406219482422,
+                0.03306496047973633,
+                0.032949249267578126,
+                0.03324415969848633,
+                0.03315507125854492,
+                0.033086463928222655,
+                0.033476608276367184,
+                0.03312639999389649,
+                0.03313663864135742,
+                0.03316838455200195,
+                0.03322470474243164,
+                0.03312947082519531,
+                0.033007614135742186,
+                0.033124351501464845,
+                0.0332492790222168,
+                0.03316428756713867,
+                0.033317886352539065,
+                0.033315040588378905,
+                0.033176353454589844,
+                0.03321036911010742,
+                0.033285247802734376,
+                0.033796993255615235,
+                0.033329151153564454,
+                0.03318067169189453,
+                0.03320729446411133,
+                0.03332505416870117,
+                0.03320012664794922,
+                0.03318374252319336,
+                0.03324415969848633,
+                0.03316438293457031,
+                0.033195934295654296,
+                0.03311820983886719,
+                0.03318374252319336,
+                0.03317452621459961,
+                0.03331379318237305,
+                0.03316249465942383,
+                0.03321932983398437,
+                0.033314815521240236,
+                0.03326976013183594,
+                0.033185791015625,
+                0.033154048919677735,
+                0.03321343994140625,
+                0.03322163009643555,
+                0.03335168075561523,
+                0.03324006271362305,
+                0.033123390197753906,
+                0.03322463989257812,
+                0.033182880401611325,
+                0.03321225738525391,
+                0.03322675323486328,
+                0.03322367858886719,
+                0.033095745086669924,
+                0.033196990966796874,
+                0.03314995193481445,
+                0.0331960334777832,
+                0.033209598541259766,
+                0.033299198150634764,
+                0.03322470474243164,
+                0.03321241760253906,
+                0.03313663864135742,
+                0.03336294555664063,
+                0.033037441253662106,
+                0.03419123077392578,
+                0.03421500778198242,
+                0.033410976409912106,
+                0.033175743103027344,
+                0.03322457504272461,
+                0.033476768493652345,
+                0.03319180679321289,
+                0.033048480987548826,
+                0.03319091033935547,
+                0.033099777221679685,
+                0.03322473526000977,
+                0.03324105453491211,
+                0.03316326522827148,
+                0.03314176177978516,
+                0.033277950286865234,
+                0.03313868713378906,
+                0.03322272109985352,
+                0.03307513427734375,
+                0.033260543823242186,
+                0.03368038558959961,
+                0.03302297592163086,
+                0.033067008972167966,
+                0.033122337341308594,
+                0.033328094482421876,
+                0.033113086700439456,
+                0.03310079956054687,
+                0.033173503875732424,
+                0.0330967025756836,
+                0.03319529724121094,
+                0.033060577392578124,
+                0.03325459289550781,
+                0.033119041442871096,
+                0.03300454330444336,
+                0.03299532699584961,
+                0.03313049697875976,
+                0.03316326522827148,
+                0.032993377685546874,
+                0.03323587036132813,
+                0.03308441543579101,
+                0.03307443237304687,
+                0.033071872711181644,
+                0.03312134552001953,
+                0.032997310638427736,
+                0.033071392059326174,
+                0.0330206413269043,
+                0.03294230270385742,
+                0.03292649459838867,
+                0.03303936004638672,
+                0.03300249481201172,
+                0.03306393432617188,
+                0.03303123092651367,
+                0.0329420166015625,
+                0.03299343872070312,
+                0.033041248321533205,
+                0.033091583251953126,
+                0.03301375961303711,
+                0.03321654510498047,
+                0.03327587127685547,
+                0.032876544952392575,
+                0.033081344604492184,
+                0.03303628921508789,
+                0.03301068878173828,
+                0.03292876815795898,
+                0.03303644943237305,
+                0.03296649551391601,
+                0.03305984115600586,
+                0.033483966827392575,
+                0.033032222747802736,
+                0.032884513854980466,
+                0.032895999908447264,
+                0.032985183715820314,
+                0.032986015319824216,
+                0.032955390930175785,
+                0.032912384033203124,
+                0.033081344604492184,
+                0.03307724761962891,
+                0.03327199935913086,
+                0.033079105377197264,
+                0.03313779067993164,
+                0.033027008056640626,
+                0.033469375610351564,
+                0.03287756729125976,
+                0.03308748626708984,
+                0.03295235061645508,
+                0.033172447204589846,
+                0.03306393432617188,
+                0.0331141128540039,
+                0.032939006805419925,
+                0.03323289489746094,
+                0.03303535842895508,
+                0.03311196899414062,
+                0.032935935974121096,
+                0.033086463928222655,
+                0.03290828704833984,
+                0.032909313201904294,
+                0.03294630432128906,
+                0.032944000244140625,
+                0.032927745819091796,
+                0.03303760147094727,
+                0.03321929550170898,
+                0.033099777221679685,
+                0.033108192443847655,
+                0.03321628952026367,
+                0.03309363174438477,
+                0.03308236694335937,
+                0.03294416046142578,
+                0.0331366081237793,
+                0.03298406219482422,
+                0.03304550552368164,
+                0.033030208587646485,
+                0.03293689727783203,
+                0.03298099136352539,
+                0.03285094451904297,
+                0.03304457473754883,
+                0.0329901123046875,
+                0.033124351501464845,
+                0.03314191818237305,
+                0.03306275177001953,
+                0.03311513519287109,
+                0.03307212829589844,
+                0.033007614135742186,
+                0.03311513519287109,
+                0.033165313720703124,
+                0.033040382385253905,
+                0.03298611068725586,
+                0.03293491363525391,
+                0.03302822494506836,
+                0.03345292663574219,
+                0.03327907180786133,
+                0.03291638565063477,
+                0.033116161346435545,
+                0.032914432525634765,
+                0.033037311553955076,
+                0.03298918533325195,
+                0.033058815002441407,
+                0.032919551849365236,
+                0.033051647186279294,
+                0.03299964904785156,
+                0.03303094482421875,
+                0.03297075271606445,
+                0.03306086349487305,
+                0.03337420654296875,
+                0.033068031311035154,
+                0.0329431037902832,
+                0.03307827377319336,
+                0.03304857635498047,
+                0.03309372711181641,
+                0.03320412826538086,
+                0.033113086700439456,
+                0.033181697845458984,
+                0.03314710235595703,
+                0.033155902862548825,
+                0.03309580612182617,
+                0.03299833679199219,
+                0.03306691360473633,
+                0.03307417678833008,
+                0.0330885124206543,
+                0.03295654296875,
+                0.033167232513427736,
+                0.03298611068725586,
+                0.03306601715087891,
+                0.03310895919799805,
+                0.03291033554077148,
+                0.03291340637207031,
+                0.033197246551513675,
+                0.033151809692382815,
+                0.03296672058105469,
+                0.03305161666870117,
+                0.03302601623535156,
+                0.03341107177734375,
+                0.0331673583984375,
+                0.03307724761962891,
+                0.033185791015625,
+                0.033075199127197266,
+                0.032939006805419925,
+                0.03308031845092774,
+                0.0333240966796875,
+                0.03308230209350586,
+                0.03307929611206055,
+                0.03302604675292969,
+                0.03306086349487305,
+                0.03299225616455078,
+                0.033046817779541014,
+                0.0331363525390625,
+                0.033320960998535154,
+                0.03310079956054687,
+                0.033565696716308595,
+                0.033091808319091795,
+                0.033137439727783206,
+                0.03304959869384766,
+                0.033058815002441407,
+                0.033068031311035154,
+                0.03316633605957031,
+                0.0330618896484375,
+                0.03309081649780273,
+                0.03296128082275391,
+                0.03320832061767578,
+                0.03308441543579101,
+                0.0332861442565918,
+                0.03323904037475586,
+                0.03363942337036133,
+                0.033037311553955076,
+                0.03306598281860352,
+                0.03304579162597656,
+                0.03302883148193359,
+                0.03300966262817383,
+                0.0331776008605957,
+                0.032996353149414064,
+                0.033170433044433595,
+                0.033067230224609376,
+                0.03310160064697266,
+                0.03306528091430664,
+                0.033051326751708986,
+                0.03304345703125,
+                0.033067008972167966,
+                0.033113086700439456,
+                0.03313663864135742,
+                0.03306496047973633,
+                0.03307628631591797,
+                0.03310688018798828,
+                0.0332042236328125,
+                0.033149185180664065,
+                0.03320499038696289,
+                0.033081375122070315,
+                0.03317679977416992,
+                0.03298892974853516,
+                0.03305472183227539,
+                0.03310182571411133,
+                0.03306496047973633,
+                0.03302707290649414,
+                0.03310182571411133,
+                0.03304652786254883,
+                0.03307622528076172,
+                0.03299737548828125,
+                0.033175552368164066,
+                0.0331776008605957,
+                0.0331141128540039,
+                0.03305267333984375,
+                0.03300966262817383,
+                0.033143806457519534,
+                0.03303734588623047,
+                0.03301801681518555,
+                0.03318764877319336,
+                0.03318067169189453,
+                0.03320217514038086,
+                0.033108192443847655,
+                0.03315894317626953,
+                0.03373673629760742,
+                0.03318268966674805,
+                0.033173503875732424,
+                0.033205249786376956,
+                0.03317891311645508,
+                0.03320291137695312,
+                0.03308236694335937,
+                0.0331069450378418,
+                0.03297814559936523,
+                0.033092384338378904,
+                0.032952320098876955,
+                0.03308441543579101,
+                0.033097793579101566,
+                0.033175487518310544,
+                0.03356159973144531,
+                0.03320012664794922,
+                0.03310182571411133,
+                0.03315814590454102,
+                0.03322060775756836,
+                0.0331673583984375,
+                0.03310079956054687,
+                0.033051647186279294,
+                0.03307334518432617,
+                0.033067840576171875,
+                0.0331409912109375,
+                0.0331220474243164,
+                0.033089534759521484,
+                0.033050624847412106,
+                0.033097728729248044,
+                0.033023998260498046,
+                0.03313356781005859,
+                0.032981056213378906,
+                0.03310995101928711,
+                0.03301171112060547,
+                0.03317046356201172,
+                0.03301577758789063,
+                0.03308348846435547,
+                0.033062816619873044,
+                0.03309283065795898,
+                0.03302889633178711,
+                0.03303936004638672,
+                0.03303424072265625,
+                0.03306905746459961,
+                0.03305779266357422,
+                0.03301171112060547,
+                0.03303219223022461,
+                0.03308748626708984,
+                0.03295129776000977,
+                0.03302912139892578,
+                0.03304652786254883,
+                0.0329238395690918,
+                0.033054527282714845,
+                0.03309363174438477,
+                0.032879615783691404,
+                0.032865409851074216,
+                0.0330463981628418,
+                0.032925918579101564,
+                0.03297257614135742,
+                0.03290726470947265,
+                0.03301593780517578,
+                0.032984031677246096,
+                0.03342959976196289,
+                0.032863040924072266,
+                0.03292364883422851,
+                0.03295948791503906,
+                0.03286220932006836,
+                0.03288576126098633,
+                0.03292876815795898,
+                0.03298611068725586,
+                0.033123519897460936,
+                0.032947200775146485,
+                0.03296553421020508,
+                0.03312015914916992,
+                0.033159168243408206,
+                0.03298713684082031,
+                0.03338854217529297,
+                0.032838878631591795,
+                0.03298502349853515,
+                0.03306480026245117,
+                0.03301200103759765,
+                0.03290083312988281,
+                0.032909313201904294,
+                0.032974048614501955,
+                0.03303504180908203,
+                0.032996353149414064,
+                0.032892929077148435,
+                0.0328172492980957,
+                0.03301279830932617,
+                0.03296352005004883,
+                0.03294198226928711,
+                0.03305574417114258,
+                0.03306905746459961,
+                0.033056766510009765,
+                0.03306291198730469,
+                0.03307724761962891,
+                0.033140960693359374,
+                0.033064735412597655,
+                0.03323104095458984,
+                0.032940097808837894,
+                0.03315276718139649,
+                0.03301273727416992,
+                0.03298406219482422,
+                0.03290009689331055,
+                0.03299532699584961,
+                0.03298303985595703,
+                0.03297894287109375,
+                0.03300873565673828,
+                0.032878559112548826,
+                0.03284883117675781,
+                0.03292364883422851,
+                0.032948513031005856,
+                0.032984798431396484,
+                0.033023998260498046,
+                0.03298924636840821,
+                0.03287340927124023,
+                0.032911361694335936,
+                0.03284400177001953,
+                0.03291027069091797,
+                0.033060703277587894,
+                0.033124351501464845,
+                0.03297382354736328,
+                0.03303014373779297,
+                0.03348992156982422,
+                0.03299235153198242,
+                0.033053791046142575,
+                0.03306399917602539,
+                0.032955135345458984,
+                0.032991359710693356,
+                0.032892799377441405,
+                0.0329881591796875,
+                0.03290828704833984,
+                0.03292160034179688,
+                0.03300454330444336,
+                0.033035263061523434,
+                0.033078369140625,
+                0.03300649642944336,
+                0.03344086456298828,
+                0.033134559631347656,
+                0.03300755310058594,
+                0.03307929611206055,
+                0.033031169891357424,
+                0.032952320098876955,
+                0.033089534759521484,
+                0.03298918533325195,
+                0.032939006805419925,
+                0.032966655731201173,
+                0.03299430465698242,
+                0.03299356842041016,
+                0.033051361083984376,
+                0.033113086700439456,
+                0.03297689437866211,
+                0.03301171112060547,
+                0.03291862487792969,
+                0.03299523162841797,
+                0.03288371276855469,
+                0.03292876815795898,
+                0.033075199127197266,
+                0.032998401641845705,
+                0.03321651077270508,
+                0.03300352096557617,
+                0.0330250244140625,
+                0.033081344604492184,
+                0.03307724761962891,
+                0.033104896545410156,
+                0.032911361694335936,
+                0.03300864028930664,
+                0.03287142562866211,
+                0.0329483528137207,
+                0.03305571365356445,
+                0.03292559814453125,
+                0.032968704223632815,
+                0.03297689437866211,
+                0.03288576126098633,
+                0.033040382385253905,
+                0.032901119232177735,
+                0.03300044631958008,
+                0.0329543685913086,
+                0.032833534240722655
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 30.209901795923255
         },
         "energy": null,
         "efficiency": null