kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 16

Commit

b6acf62

•

1 Parent(s): 6073480

Upload /Llama-2-7b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Llama-2-7b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +468 -759

Llama-2-7b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,257 +2,222 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1015.881728,
             "max_vram": 4905.238528,
             "max_reserved": 4420.796416,
             "max_allocated": 4252.667392
         },
         "latency": {
             "unit": "s",
-            "mean": 0.042467162730330105,
-            "stdev": 0.0006609647717465549,
             "values": [
-                0.048517601013183594,
-                0.04331315231323242,
-                0.043079681396484375,
-                0.04321279907226563,
-                0.04323123168945313,
-                0.043218944549560545,
-                0.043480064392089846,
-                0.04548710250854492,
-                0.04911718368530273,
-                0.0445296630859375,
-                0.042159103393554685,
-                0.04237823867797851,
-                0.04234444808959961,
-                0.0423741455078125,
-                0.04237619018554688,
-                0.04238336181640625,
-                0.04228633499145508,
-                0.042385406494140625,
-                0.04228409576416016,
-                0.04238336181640625,
-                0.04239462280273437,
-                0.04246220779418945,
-                0.042264575958251956,
-                0.042359809875488284,
-                0.04222873687744141,
-                0.042570751190185545,
-                0.04218265533447266,
-                0.042363903045654294,
-                0.04226764678955078,
-                0.04237209701538086,
-                0.04212940979003906,
-                0.042363903045654294,
-                0.04219612884521484,
-                0.04238156890869141,
-                0.04231782531738281,
-                0.04241408157348633,
-                0.042324031829833984,
-                0.042387454986572266,
-                0.04229529571533203,
-                0.04235776138305664,
-                0.04212838363647461,
-                0.04236800003051758,
-                0.04221440124511719,
-                0.04236800003051758,
-                0.04280115127563477,
-                0.04237619018554688,
-                0.04229430389404297,
-                0.0424007682800293,
-                0.04217139053344727,
-                0.04236185455322266,
-                0.04226867294311523,
-                0.042393600463867184,
-                0.042243072509765625,
-                0.04239257431030274,
-                0.04221952056884765,
-                0.042352638244628905,
-                0.042827934265136716,
-                0.04238643264770508,
-                0.042369022369384765,
-                0.0424161262512207,
-                0.04222873687744141,
-                0.042382335662841795,
-                0.042278911590576174,
-                0.04239052963256836,
-                0.042237953186035154,
-                0.04238147354125977,
-                0.042308609008789064,
-                0.04239052963256836,
-                0.04216934585571289,
-                0.042385406494140625,
-                0.04228710556030273,
-                0.0423741455078125,
-                0.04220927810668945,
-                0.04236185455322266,
-                0.04229119873046875,
-                0.04249103927612305,
-                0.042395648956298826,
-                0.04238131332397461,
-                0.04235673522949219,
-                0.04242124938964844,
-                0.04230144119262695,
-                0.04241116714477539,
-                0.042425342559814457,
-                0.04239052963256836,
-                0.042248191833496096,
-                0.04243865585327149,
-                0.04233126449584961,
-                0.04239769744873047,
-                0.04232191848754883,
-                0.042420223236083986,
-                0.042331134796142575,
-                0.042379264831542966,
-                0.0426506233215332,
-                0.04239462280273437,
-                0.04236697769165039,
-                0.042393600463867184,
-                0.042396671295166014,
-                0.04249292755126953,
-                0.04233011245727539,
-                0.042396671295166014,
-                0.04233011245727539,
-                0.042482688903808595,
-                0.042241024017333983,
-                0.04264243316650391,
-                0.04236288070678711,
-                0.042363903045654294,
-                0.04232191848754883,
-                0.04237311935424805,
-                0.042431488037109374,
-                0.04239052963256836,
-                0.0423741455078125,
-                0.042412033081054686,
-                0.04233523178100586,
-                0.04239462280273437,
-                0.042395648956298826,
-                0.04238950347900391,
-                0.04253696060180664,
-                0.04238131332397461,
-                0.042426368713378904,
-                0.04243865585327149,
-                0.042415103912353515,
-                0.042521598815917966,
-                0.042327041625976565,
-                0.04240588760375977,
-                0.042412033081054686,
-                0.04235059356689453,
-                0.042264575958251956,
-                0.042379264831542966,
-                0.04227174377441406,
-                0.042433761596679685,
-                0.04225331115722656,
-                0.04241222381591797,
-                0.04238131332397461,
-                0.04242943954467773,
-                0.04240691375732422,
-                0.042363903045654294,
-                0.04223004913330078,
-                0.042385406494140625,
-                0.04230758285522461,
-                0.04237004852294922,
-                0.042744831085205076,
-                0.04236697769165039,
-                0.04225228881835937,
-                0.04240793609619141,
-                0.04233830261230469,
-                0.04238336181640625,
-                0.042308609008789064,
-                0.042395648956298826,
-                0.04227993774414063,
-                0.04239567947387695,
-                0.04232499313354492,
-                0.042812416076660156,
-                0.0422083511352539,
-                0.04241100692749023,
-                0.042234878540039066,
-                0.042392894744873046,
-                0.04212940979003906,
-                0.04237823867797851,
-                0.04230451202392578,
-                0.042379264831542966,
-                0.04227174377441406,
-                0.04237619018554688,
-                0.04218982315063476,
-                0.04238131332397461,
-                0.04234854507446289,
-                0.04237516784667969,
-                0.04220927810668945,
-                0.042409984588623044,
-                0.042349567413330076,
-                0.04237004852294922,
-                0.04219903945922852,
-                0.04237516784667969,
-                0.04221440124511719,
-                0.042363903045654294,
-                0.04225228881835937,
-                0.042369022369384765,
-                0.04212428665161133,
-                0.04236800003051758,
-                0.0422737922668457,
-                0.04237107086181641,
-                0.04219903945922852,
-                0.0424161262512207,
-                0.04229529571533203,
-                0.04235366439819336,
-                0.042248191833496096,
-                0.04236800003051758,
-                0.04218572616577149,
-                0.04277248001098633,
-                0.042382335662841795,
-                0.042355712890625,
-                0.042428417205810545,
-                0.04238643264770508,
-                0.042352638244628905,
-                0.04240281677246094,
-                0.04240281677246094,
-                0.042387454986572266,
-                0.042262527465820314,
-                0.04239974212646484,
-                0.04257484817504883,
-                0.042591552734375,
-                0.04235673522949219,
-                0.042393600463867184,
-                0.04225024032592774,
-                0.04242345428466797,
-                0.04231782531738281,
-                0.04240691375732422,
-                0.04241222381591797,
-                0.04261171340942383,
-                0.042246337890625,
-                0.04240793609619141,
-                0.04248883056640625,
-                0.042380287170410154,
-                0.04228505706787109,
-                0.04241100692749023,
-                0.042417152404785156,
-                0.04240691375732422,
-                0.04239257431030274,
-                0.042386497497558594,
-                0.04246441650390625,
-                0.04236185455322266,
-                0.042369022369384765,
-                0.042417152404785156,
-                0.042482688903808595,
-                0.042385406494140625,
-                0.042297344207763675,
-                0.04238848114013672,
-                0.04245811080932617,
-                0.04240281677246094,
-                0.04224227142333985,
-                0.04238336181640625,
-                0.042420223236083986,
-                0.042389823913574216,
-                0.04247465515136719,
-                0.042393600463867184,
-                0.042292224884033204,
-                0.0428042221069336
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6028.187040081311
         },
         "energy": null,
         "efficiency": null
@@ -260,23 +225,22 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1015.881728,
             "max_vram": 5255.462912,
             "max_reserved": 4771.0208,
             "max_allocated": 4715.61216
         },
         "latency": {
             "unit": "s",
-            "mean": 8.599395771026611,
-            "stdev": 0.03203231048584154,
             "values": [
-                8.56736346054077,
-                8.631428081512453
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 29.653246203547816
         },
         "energy": null,
         "efficiency": null
@@ -285,524 +249,269 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.033723120670692594,
-            "stdev": 0.0007542579259829491,
             "values": [
-                0.03367903900146484,
-                0.034095104217529294,
-                0.03362924957275391,
-                0.033646526336669924,
-                0.033672191619873046,
-                0.03379916763305664,
-                0.03361587142944336,
-                0.03376639938354492,
-                0.03357798385620117,
-                0.03353190231323242,
-                0.033588222503662106,
-                0.033863937377929684,
-                0.03352755355834961,
-                0.033552383422851564,
-                0.03365494537353515,
-                0.033521503448486326,
-                0.03344486236572266,
-                0.03351980972290039,
-                0.0335296630859375,
-                0.03351244735717773,
-                0.033667072296142575,
-                0.03366400146484375,
-                0.03353702545166016,
-                0.03360665512084961,
-                0.033732608795166014,
-                0.033527809143066405,
-                0.033498111724853515,
-                0.033559776306152346,
-                0.033648414611816405,
-                0.03360153579711914,
-                0.03352678298950195,
-                0.03374694442749023,
-                0.0335781135559082,
-                0.03363315200805664,
-                0.033604606628417966,
-                0.03352371215820313,
-                0.033670143127441404,
-                0.03367424011230469,
-                0.03356979370117188,
-                0.03373056030273437,
-                0.033708030700683594,
-                0.03353395080566406,
-                0.033614849090576174,
-                0.03355852890014648,
-                0.03376332855224609,
-                0.03358617782592774,
-                0.033514495849609374,
-                0.033640735626220705,
-                0.033441600799560545,
-                0.03353900909423828,
-                0.03348188781738281,
-                0.03361465454101562,
-                0.03352268981933594,
-                0.033567745208740236,
-                0.03363759994506836,
-                0.033532703399658206,
-                0.0335175666809082,
-                0.0339835205078125,
-                0.03363119888305664,
-                0.0335728645324707,
-                0.033532928466796875,
-                0.03351347351074219,
-                0.03356979370117188,
-                0.03359231948852539,
-                0.03363225555419922,
-                0.03357807922363281,
-                0.03343247985839844,
-                0.03353497695922852,
-                0.03351347351074219,
-                0.0334202880859375,
-                0.03354009628295898,
-                0.033796127319335935,
-                0.03371942520141601,
-                0.03353379058837891,
-                0.03362918472290039,
-                0.03350140762329101,
-                0.03364742279052734,
-                0.03346121597290039,
-                0.0335022087097168,
-                0.0335175666809082,
-                0.033584129333496096,
-                0.033710079193115236,
-                0.03355574417114258,
-                0.03373641586303711,
-                0.03346944046020508,
-                0.033544193267822264,
-                0.03350352096557617,
-                0.03355516815185547,
-                0.033481727600097655,
-                0.03347359848022461,
-                0.0335082893371582,
-                0.033446910858154294,
-                0.03348992156982422,
-                0.03352883148193359,
-                0.033433727264404293,
-                0.03351744079589844,
-                0.0336445426940918,
-                0.03349708938598633,
-                0.03350527954101563,
-                0.033516544342041016,
-                0.033527935028076175,
-                0.03341606521606445,
-                0.03358003234863281,
-                0.03350527954101563,
-                0.03360563278198242,
-                0.0335093765258789,
-                0.03366604614257813,
-                0.03351574325561523,
-                0.03360847854614258,
-                0.03349913787841797,
-                0.033751041412353515,
-                0.03354214477539062,
-                0.03370086288452148,
-                0.03360153579711914,
-                0.033567745208740236,
-                0.03361280059814453,
-                0.03362508773803711,
-                0.03391795349121094,
-                0.03364863967895508,
-                0.033498111724853515,
-                0.033689697265625,
-                0.03361782455444336,
-                0.03367424011230469,
-                0.03368243026733399,
-                0.033576961517333984,
-                0.033584129333496096,
-                0.0337520637512207,
-                0.03362303924560547,
-                0.03363532638549805,
-                0.03349094390869141,
-                0.03344412612915039,
-                0.03370774459838867,
-                0.03375833511352539,
-                0.03355532836914062,
-                0.03360358428955078,
-                0.0337336311340332,
-                0.03346124649047852,
-                0.033522720336914065,
-                0.033508319854736325,
-                0.03347353744506836,
-                0.033693695068359376,
-                0.033535999298095705,
-                0.03356467056274414,
-                0.03382271957397461,
-                0.03361382293701172,
-                0.033459201812744144,
-                0.03358310317993164,
-                0.03362303924560547,
-                0.0335032958984375,
-                0.03348678588867188,
-                0.03362937545776367,
-                0.033528640747070314,
-                0.03379916763305664,
-                0.033650688171386715,
-                0.033576961517333984,
-                0.03346739196777344,
-                0.033620990753173825,
-                0.033701889038085936,
-                0.033667072296142575,
-                0.033514495849609374,
-                0.03357593536376953,
-                0.033729534149169925,
-                0.033511425018310545,
-                0.03358617782592774,
-                0.03355033493041992,
-                0.033583263397216796,
-                0.033570655822753905,
-                0.03360358428955078,
-                0.03356364822387695,
-                0.03360665512084961,
-                0.03374899291992187,
-                0.033718273162841796,
-                0.03359743881225586,
-                0.033584129333496096,
-                0.03356364822387695,
-                0.0336629753112793,
-                0.033620990753173825,
-                0.033919998168945316,
-                0.03372768020629883,
-                0.033554271697998045,
-                0.03358832168579102,
-                0.03361881637573242,
-                0.03367833709716797,
-                0.03359027099609375,
-                0.03363532638549805,
-                0.03375001525878906,
-                0.03360873413085937,
-                0.033424510955810546,
-                0.03377056121826172,
-                0.03363510513305664,
-                0.033546241760253906,
-                0.03377356719970703,
-                0.033805313110351565,
-                0.03364147186279297,
-                0.033600608825683595,
-                0.03353180694580078,
-                0.0335912971496582,
-                0.03360768127441406,
-                0.03358415985107422,
-                0.03361276626586914,
-                0.03357900619506836,
-                0.03366912078857422,
-                0.03369267272949219,
-                0.033539070129394534,
-                0.033519615173339845,
-                0.033642494201660156,
-                0.03354019165039063,
-                0.033525665283203124,
-                0.03361711883544922,
-                0.03361974334716797,
-                0.03349724960327148,
-                0.033764190673828125,
-                0.0334837760925293,
-                0.033565696716308595,
-                0.03359027099609375,
-                0.033501182556152344,
-                0.033549312591552735,
-                0.03354111862182617,
-                0.03348787307739258,
-                0.033649665832519535,
-                0.03352371215820313,
-                0.03361075210571289,
-                0.03359539031982422,
-                0.033685504913330076,
-                0.03357491302490234,
-                0.03379097747802735,
-                0.03350630569458008,
-                0.033620990753173825,
-                0.03346944046020508,
-                0.033530048370361325,
-                0.03350713729858398,
-                0.0335022087097168,
-                0.03343974304199219,
-                0.0335912971496582,
-                0.03343462371826172,
-                0.03355955123901367,
-                0.03342131042480469,
-                0.0340316162109375,
-                0.033539070129394534,
-                0.03346944046020508,
-                0.03349606323242187,
-                0.03350425720214844,
-                0.033500160217285156,
-                0.03361280059814453,
-                0.033909759521484374,
-                0.033649665832519535,
-                0.03361996841430664,
-                0.03359231948852539,
-                0.033546241760253906,
-                0.0335206413269043,
-                0.0335994873046875,
-                0.033827838897705076,
-                0.03380326461791992,
-                0.03357900619506836,
-                0.033546432495117184,
-                0.033893375396728515,
-                0.033667072296142575,
-                0.033667072296142575,
-                0.03363942337036133,
-                0.033555454254150394,
-                0.033598464965820314,
-                0.033509407043457035,
-                0.03363529586791992,
-                0.033582080841064454,
-                0.03367424011230469,
-                0.03362303924560547,
-                0.03373875045776367,
-                0.033576961517333984,
-                0.033649761199951174,
-                0.03349187088012695,
-                0.03364863967895508,
-                0.033584320068359375,
-                0.033551166534423825,
-                0.03348787307739258,
-                0.033719551086425784,
-                0.03360128021240234,
-                0.03377151870727539,
-                0.03352492904663086,
-                0.03367200088500977,
-                0.03359231948852539,
-                0.03360255813598633,
-                0.03355648040771484,
-                0.03363532638549805,
-                0.033584129333496096,
-                0.033532928466796875,
-                0.03352268981933594,
-                0.033827838897705076,
-                0.03350630569458008,
-                0.0335728645324707,
-                0.033484798431396484,
-                0.03364556884765625,
-                0.033462272644042966,
-                0.03363225555419922,
-                0.033565696716308595,
-                0.033704959869384765,
-                0.03361894226074219,
-                0.034223102569580076,
-                0.03355340957641602,
-                0.03362835311889648,
-                0.03361260986328125,
-                0.033585151672363284,
-                0.03353811264038086,
-                0.033615806579589846,
-                0.03343564987182617,
-                0.03364044952392578,
-                0.0336363525390625,
-                0.033729534149169925,
-                0.03359539031982422,
-                0.033632320404052736,
-                0.03352467346191406,
-                0.03403673553466797,
-                0.03348889541625977,
-                0.03359334564208984,
-                0.033519615173339845,
-                0.03355257415771484,
-                0.033492801666259765,
-                0.03364371109008789,
-                0.033594303131103516,
-                0.03369868850708008,
-                0.03351859283447266,
-                0.03353190231323242,
-                0.0335206413269043,
-                0.033588222503662106,
-                0.03347763061523437,
-                0.03348992156982422,
-                0.03362815856933594,
-                0.03365692901611328,
-                0.03376630401611328,
-                0.03358003234863281,
-                0.03367628860473633,
-                0.033670143127441404,
-                0.033584129333496096,
-                0.03367628860473633,
-                0.033584129333496096,
-                0.03358003234863281,
-                0.03368960189819336,
-                0.03364863967895508,
-                0.0335206413269043,
-                0.033588222503662106,
-                0.033568767547607424,
-                0.03355459213256836,
-                0.033698654174804686,
-                0.03367424011230469,
-                0.03360563278198242,
-                0.033555454254150394,
-                0.03350735855102539,
-                0.033648609161376956,
-                0.03350732803344727,
-                0.03362918472290039,
-                0.033598464965820314,
-                0.03358310317993164,
-                0.033928192138671875,
-                0.03349923324584961,
-                0.03360348892211914,
-                0.0335728645324707,
-                0.03362201690673828,
-                0.03391795349121094,
-                0.033555454254150394,
-                0.03356159973144531,
-                0.0336814079284668,
-                0.033651744842529294,
-                0.03361497497558594,
-                0.0335450553894043,
-                0.03351580810546875,
-                0.03373846435546875,
-                0.0335810546875,
-                0.033465343475341795,
-                0.03353702545166016,
-                0.0335013427734375,
-                0.03357782363891602,
-                0.03403571319580078,
-                0.033627166748046874,
-                0.03364767837524414,
-                0.03356047821044922,
-                0.033729534149169925,
-                0.03368054580688477,
-                0.033608543395996095,
-                0.033582080841064454,
-                0.03363532638549805,
-                0.03355955123901367,
-                0.03358310317993164,
-                0.03362508773803711,
-                0.03377766418457031,
-                0.03361820983886719,
-                0.033477344512939454,
-                0.033567745208740236,
-                0.03341033554077148,
-                0.03353673553466797,
-                0.03342131042480469,
-                0.0334837760925293,
-                0.03346636962890625,
-                0.03382601547241211,
-                0.033547039031982424,
-                0.033486881256103516,
-                0.03350012969970703,
-                0.03364147186279297,
-                0.03338751983642578,
-                0.033544193267822264,
-                0.03377356719970703,
-                0.03358617782592774,
-                0.03361996841430664,
-                0.03342745590209961,
-                0.03360358428955078,
-                0.03361587142944336,
-                0.0336732177734375,
-                0.03346432113647461,
-                0.033535999298095705,
-                0.033462272644042966,
-                0.03345817565917969,
-                0.033616897583007815,
-                0.033699840545654294,
-                0.03346124649047852,
-                0.03354009628295898,
-                0.033508350372314456,
-                0.03351248168945312,
-                0.033476638793945315,
-                0.03420975875854492,
-                0.03355542373657226,
-                0.0336097297668457,
-                0.03361177444458008,
-                0.03355033493041992,
-                0.033569950103759766,
-                0.03987760162353516,
-                0.039078048706054684,
-                0.03925167846679688,
-                0.0383559684753418,
-                0.038001953125,
-                0.03823791885375977,
-                0.03890585708618164,
-                0.03825151824951172,
-                0.03803955078125,
-                0.03559936141967773,
-                0.03567308807373047,
-                0.03725107192993164,
-                0.03808870315551758,
-                0.036789249420166016,
-                0.03564748764038086,
-                0.03366937637329102,
-                0.03364134216308594,
-                0.03357171249389648,
-                0.033396736145019534,
-                0.03352883148193359,
-                0.033667072296142575,
-                0.03363020706176758,
-                0.033617919921875,
-                0.033544193267822264,
-                0.033683456420898435,
-                0.03356979370117188,
-                0.03340508651733398,
-                0.03358294296264648,
-                0.033942527770996093,
-                0.03343769454956055,
-                0.03358924865722656,
-                0.03358323287963867,
-                0.03440729522705078,
-                0.03371212768554688,
-                0.03364761734008789,
-                0.03349932861328125,
-                0.033621822357177734,
-                0.033546241760253906,
-                0.03354217529296875,
-                0.03358307266235352,
-                0.03364863967895508,
-                0.03382089614868164,
-                0.0336146240234375,
-                0.03368960189819336,
-                0.03351039886474609,
-                0.03358937454223633,
-                0.03356659317016602,
-                0.03346124649047852,
-                0.03352166366577149,
-                0.03363532638549805,
-                0.03348787307739258,
-                0.033537311553955076,
-                0.03388079833984375,
-                0.033529857635498046,
-                0.033588222503662106,
-                0.03347251129150391,
-                0.03355875015258789,
-                0.03351836776733398,
-                0.03363020706176758,
-                0.033732769012451175,
-                0.03356143951416016,
-                0.03349094390869141,
-                0.03346022415161133,
-                0.03352390289306641,
-                0.033569599151611326,
-                0.033637374877929685,
-                0.034272254943847655,
-                0.0335810546875,
-                0.03355648040771484,
-                0.033538047790527346,
-                0.03346636962890625,
-                0.03358924865722656,
-                0.03346636962890625,
-                0.033432575225830076,
-                0.033492992401123044,
-                0.033538238525390625,
-                0.03342623901367187,
-                0.03347251129150391,
-                0.033470462799072266,
-                0.03345510482788086,
-                0.033585151672363284,
-                0.03352166366577149,
-                0.03354745483398437,
-                0.03356550216674805,
-                0.033538047790527346,
-                0.033567745208740236,
-                0.03365683364868164,
-                0.03360665512084961
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 29.653246203547816
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1045.4016,
             "max_vram": 4905.238528,
             "max_reserved": 4420.796416,
             "max_allocated": 4252.667392
         },
         "latency": {
             "unit": "s",
+            "mean": 0.04986847159162686,
+            "stdev": 0.0011566518505120138,
             "values": [
+                0.06372921752929687,
+                0.050318336486816405,
+                0.05015654373168945,
+                0.050157569885253904,
+                0.05006243133544922,
+                0.05029171371459961,
+                0.05021900939941406,
+                0.05005311965942383,
+                0.05022617721557617,
+                0.050337791442871094,
+                0.050283519744873044,
+                0.05015961456298828,
+                0.050119678497314454,
+                0.04968172836303711,
+                0.0495912971496582,
+                0.049530879974365234,
+                0.0496445426940918,
+                0.04979507064819336,
+                0.04986368179321289,
+                0.049704959869384766,
+                0.04969369506835938,
+                0.04954531097412109,
+                0.04972544097900391,
+                0.04965990447998047,
+                0.04981964874267578,
+                0.04965580749511719,
+                0.0495810546875,
+                0.04957593536376953,
+                0.049568767547607424,
+                0.04948070526123047,
+                0.04966912078857422,
+                0.04964044952392578,
+                0.04959743881225586,
+                0.04952371215820312,
+                0.0494919662475586,
+                0.04972032165527344,
+                0.049478656768798826,
+                0.04960768127441406,
+                0.049630496978759764,
+                0.04959539031982422,
+                0.0495728645324707,
+                0.04966604614257813,
+                0.04960563278198242,
+                0.04958720016479492,
+                0.04958310317993164,
+                0.04952166366577149,
+                0.049805313110351565,
+                0.04962815856933594,
+                0.04951859283447266,
+                0.04953417587280273,
+                0.04953497695922852,
+                0.049495040893554686,
+                0.049380352020263675,
+                0.04953395080566406,
+                0.04948070526123047,
+                0.049683456420898435,
+                0.04967628860473633,
+                0.049484798431396484,
+                0.049498111724853515,
+                0.049632545471191405,
+                0.049501182556152344,
+                0.0494919662475586,
+                0.04954316711425781,
+                0.04963840103149414,
+                0.04952473449707031,
+                0.049442817687988284,
+                0.0494837760925293,
+                0.04953497695922852,
+                0.049634304046630856,
+                0.049438720703125,
+                0.0496517105102539,
+                0.04968038558959961,
+                0.04953804779052735,
+                0.049430526733398435,
+                0.04953804779052735,
+                0.04955136108398438,
+                0.04950425720214844,
+                0.04949401473999023,
+                0.04981145477294922,
+                0.05230284881591797,
+                0.05170380783081055,
+                0.049993824005126954,
+                0.04965990447998047,
+                0.0496363525390625,
+                0.049870849609375,
+                0.050348033905029295,
+                0.05045248031616211,
+                0.05002035140991211,
+                0.04998246383666992,
+                0.049729534149169925,
+                0.049979393005371096,
+                0.049870849609375,
+                0.050083839416503906,
+                0.049840225219726565,
+                0.04976332855224609,
+                0.04979916763305664,
+                0.04975513458251953,
+                0.049710079193115236,
+                0.0497520637512207,
+                0.04974796676635742,
+                0.04954009628295898,
+                0.04965580749511719,
+                0.04993228912353516,
+                0.04983091354370117,
+                0.04981964874267578,
+                0.04952473449707031,
+                0.04956371307373047,
+                0.04962918472290039,
+                0.049549407958984375,
+                0.04975027084350586,
+                0.049658878326416016,
+                0.049710079193115236,
+                0.04968447875976562,
+                0.04983295822143555,
+                0.049734657287597656,
+                0.04956159973144531,
+                0.04963942337036133,
+                0.049600513458251956,
+                0.04993843078613281,
+                0.04988313674926758,
+                0.049999870300292966,
+                0.04972159957885742,
+                0.049855487823486325,
+                0.04960870361328125,
+                0.04965785598754883,
+                0.04967935943603516,
+                0.049871871948242184,
+                0.04967116928100586,
+                0.049896446228027344,
+                0.049646656036376954,
+                0.049563648223876954,
+                0.04984934234619141,
+                0.049686527252197264,
+                0.04962201690673828,
+                0.04972339248657227,
+                0.04955344009399414,
+                0.049516544342041016,
+                0.049552383422851565,
+                0.049786880493164064,
+                0.049721343994140625,
+                0.04983705520629883,
+                0.0496732177734375,
+                0.049582080841064455,
+                0.049809406280517575,
+                0.04988131332397461,
+                0.04965478515625,
+                0.0496363525390625,
+                0.04957491302490234,
+                0.049759231567382815,
+                0.04960255813598633,
+                0.049631233215332034,
+                0.04969267272949219,
+                0.049696769714355465,
+                0.049721343994140625,
+                0.04980428695678711,
+                0.049719295501708984,
+                0.049764575958251955,
+                0.04968972778320312,
+                0.049874942779541014,
+                0.04972032165527344,
+                0.04964044952392578,
+                0.057373695373535157,
+                0.05018009567260742,
+                0.04990771102905273,
+                0.049941505432128906,
+                0.04962508773803711,
+                0.049737728118896485,
+                0.049772544860839846,
+                0.049838081359863284,
+                0.04980121612548828,
+                0.04977356719970703,
+                0.04972032165527344,
+                0.049772544860839846,
+                0.04985958480834961,
+                0.0496732177734375,
+                0.049724414825439454,
+                0.04978585433959961,
+                0.04962815856933594,
+                0.049896446228027344,
+                0.049821697235107425,
+                0.04977766418457031,
+                0.04973158264160156,
+                0.049896446228027344,
+                0.049999870300292966,
+                0.049923072814941405,
+                0.04960870361328125,
+                0.04982681655883789,
+                0.04998246383666992,
+                0.05000396728515625,
+                0.04973056030273437,
+                0.04984320068359375,
+                0.04986368179321289,
+                0.04997017669677734,
+                0.04998368072509766,
+                0.049842174530029294,
+                0.04969574356079102,
+                0.04971110534667969,
+                0.04972851181030274,
+                0.049906688690185545,
+                0.04969267272949219,
+                0.04977151870727539
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 5133.504032294896
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1045.4016,
             "max_vram": 5255.462912,
             "max_reserved": 4771.0208,
             "max_allocated": 4715.61216
         },
         "latency": {
             "unit": "s",
+            "mean": 10.25303860855103,
+            "stdev": 0,
             "values": [
+                10.25303860855103
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 24.87067587820552
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.04020799454333737,
+            "stdev": 0.00017124566649101715,
             "values": [
+                0.039923934936523436,
+                0.040948513031005856,
+                0.04051865768432617,
+                0.04043775939941406,
+                0.040342529296875,
+                0.04033436965942383,
+                0.04048175811767578,
+                0.04037017440795899,
+                0.04020870590209961,
+                0.04028691101074219,
+                0.04045619201660156,
+                0.040343551635742186,
+                0.04041017532348633,
+                0.040244159698486326,
+                0.040269824981689455,
+                0.040594432830810545,
+                0.040237056732177735,
+                0.040474624633789064,
+                0.04022272109985352,
+                0.040204288482666016,
+                0.040153087615966795,
+                0.040342529296875,
+                0.04022988891601562,
+                0.04024422454833984,
+                0.040304641723632816,
+                0.04038348770141602,
+                0.040234081268310545,
+                0.04039158248901367,
+                0.04066611099243164,
+                0.040581119537353515,
+                0.04032716751098633,
+                0.040253440856933595,
+                0.04037734222412109,
+                0.040180736541748044,
+                0.0404398078918457,
+                0.040289279937744144,
+                0.04041318511962891,
+                0.04029337692260742,
+                0.04023091125488281,
+                0.040081409454345705,
+                0.04011315155029297,
+                0.040172542572021484,
+                0.04013875198364258,
+                0.04035686492919922,
+                0.04005580902099609,
+                0.03999859237670898,
+                0.04017139053344727,
+                0.04060671997070313,
+                0.0403507194519043,
+                0.040190975189208986,
+                0.04020326232910156,
+                0.04023199844360351,
+                0.040374401092529294,
+                0.040358718872070314,
+                0.04038143920898438,
+                0.04029158401489258,
+                0.04018457412719727,
+                0.04033331298828125,
+                0.04027084732055664,
+                0.040046592712402344,
+                0.040395774841308595,
+                0.04029644775390625,
+                0.040248321533203124,
+                0.040343551635742186,
+                0.04049407958984375,
+                0.04038143920898438,
+                0.040129791259765624,
+                0.040266494750976566,
+                0.0402790412902832,
+                0.040226814270019534,
+                0.04034457778930664,
+                0.04031488037109375,
+                0.040420352935791014,
+                0.04042444610595703,
+                0.04033740615844727,
+                0.04039680099487305,
+                0.040578048706054685,
+                0.04048076629638672,
+                0.040205310821533204,
+                0.04034457778930664,
+                0.04025753784179688,
+                0.04014796829223633,
+                0.04016537475585937,
+                0.040197120666503904,
+                0.040000511169433595,
+                0.04039680099487305,
+                0.040169471740722655,
+                0.04035276794433594,
+                0.040400894165039065,
+                0.04012953567504883,
+                0.04007628631591797,
+                0.040217601776123046,
+                0.040114177703857425,
+                0.04005990219116211,
+                0.04008652877807617,
+                0.0403056640625,
+                0.040436992645263674,
+                0.04030643081665039,
+                0.04026675033569336,
+                0.04036710357666016,
+                0.03996876907348633,
+                0.040243198394775394,
+                0.040444927215576174,
+                0.04007628631591797,
+                0.040085792541503906,
+                0.039993057250976564,
+                0.04015718460083008,
+                0.04008038330078125,
+                0.039965694427490234,
+                0.04016128158569336,
+                0.04013363265991211,
+                0.04014899063110351,
+                0.04012543869018555,
+                0.04005068969726563,
+                0.040610912322998044,
+                0.040493984222412106,
+                0.04039884948730469,
+                0.04029644775390625,
+                0.040172542572021484,
+                0.0400711669921875,
+                0.04024524688720703,
+                0.04021964645385742,
+                0.04020444869995117,
+                0.04001571273803711,
+                0.04008774566650391,
+                0.03995014572143555,
+                0.040188926696777344,
+                0.04022476959228516,
+                0.04027084732055664,
+                0.04010598373413086,
+                0.040025089263916014,
+                0.04022886276245117,
+                0.04034969711303711,
+                0.040180736541748044,
+                0.040286209106445314,
+                0.040130561828613284,
+                0.040360958099365234,
+                0.04036198425292969,
+                0.04007731246948242,
+                0.04023091125488281,
+                0.04017356872558594,
+                0.0401868782043457,
+                0.04005887985229492,
+                0.040018943786621096,
+                0.03984384155273438,
+                0.04030361557006836,
+                0.04027494430541992,
+                0.040049663543701174,
+                0.04029439926147461,
+                0.040202239990234374,
+                0.040180736541748044,
+                0.03995750427246094,
+                0.0402503662109375,
+                0.04022169494628906,
+                0.04030681610107422,
+                0.04034342575073242,
+                0.040220672607421876,
+                0.04028108978271484,
+                0.04012748718261719,
+                0.04002304077148437,
+                0.04020121765136719,
+                0.04022988891601562,
+                0.040414207458496096,
+                0.040390655517578124,
+                0.040197246551513674,
+                0.04011916732788086,
+                0.04021452713012695,
+                0.040158206939697266,
+                0.04013875198364258,
+                0.04044595336914063,
+                0.04072550582885742,
+                0.04021350479125976,
+                0.040597503662109374,
+                0.04010905456542969,
+                0.04013772964477539,
+                0.040018943786621096,
+                0.040256511688232424,
+                0.04033740615844727,
+                0.040199169158935545,
+                0.04010700988769531,
+                0.04024524688720703,
+                0.0403917121887207,
+                0.04025955200195312,
+                0.04010086441040039,
+                0.04028108978271484,
+                0.04032819366455078,
+                0.040325119018554685,
+                0.040068096160888675,
+                0.03986227035522461,
+                0.03998822402954102,
+                0.040376319885253906,
+                0.0403240966796875,
+                0.04001587295532227,
+                0.04003635025024414,
+                0.0401448974609375,
+                0.04009881591796875,
+                0.04043571090698242,
+                0.04011929702758789,
+                0.04024115371704102,
+                0.04000665664672851,
+                0.04009062576293945,
+                0.039997440338134765,
+                0.03993907165527344,
+                0.03988479995727539,
+                0.0398837776184082,
+                0.04017663955688477,
+                0.039981056213378906,
+                0.040010753631591796,
+                0.04002304077148437,
+                0.040207359313964845,
+                0.040041473388671874,
+                0.04018380737304687,
+                0.04019200134277344,
+                0.04014387130737305,
+                0.04008038330078125,
+                0.040008705139160154,
+                0.03998720169067383,
+                0.04004249572753906,
+                0.03994214248657227,
+                0.04009574508666992,
+                0.04006195068359375,
+                0.04002304077148437,
+                0.04005068969726563,
+                0.04020121765136719,
+                0.039962623596191404,
+                0.04037017440795899,
+                0.04046131134033203,
+                0.04007014465332031,
+                0.04009062576293945,
+                0.040123390197753905,
+                0.04011520004272461,
+                0.04015206527709961,
+                0.04003839874267578,
+                0.03986022567749024,
+                0.040139774322509765,
+                0.04010700988769531,
+                0.040035327911376956,
+                0.040043521881103515,
+                0.039929855346679685,
+                0.03989503860473633,
+                0.0399738883972168,
+                0.04001177597045898,
+                0.039962623596191404,
+                0.04008857727050781,
+                0.04000153732299805,
+                0.0401162223815918,
+                0.04003635025024414,
+                0.04026777648925781,
+                0.04012953567504883,
+                0.04001587295532227,
+                0.040103935241699216,
+                0.040013824462890625,
+                0.04007526397705078,
+                0.03989606475830078,
+                0.04017049789428711
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 24.87067587820552
         },
         "energy": null,
         "efficiency": null