kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 16

Commit

c5c7c9b

•

1 Parent(s): 98361e6

Upload /phi-2/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

phi-2/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +753 -787

phi-2/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,285 +2,251 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1010.712576,
             "max_vram": 2533.359616,
             "max_reserved": 2101.346304,
             "max_allocated": 2014.912
         },
         "latency": {
             "unit": "s",
-            "mean": 0.03793171258406208,
-            "stdev": 0.0010862268956460114,
             "values": [
-                0.0398897933959961,
-                0.03799859237670898,
-                0.03766681671142578,
-                0.037647361755371096,
-                0.037525505065917966,
-                0.03762790298461914,
-                0.038558719635009765,
-                0.04092825698852539,
-                0.03951513671875,
-                0.037187583923339845,
-                0.03666841506958008,
-                0.038029312133789066,
-                0.038714366912841795,
-                0.03859558486938477,
-                0.03851878356933594,
-                0.038525951385498046,
-                0.03846963119506836,
-                0.03868364715576172,
-                0.0384450569152832,
-                0.03861708831787109,
-                0.03861708831787109,
-                0.0386511344909668,
-                0.038757633209228516,
-                0.03889254379272461,
-                0.03980915069580078,
-                0.039874561309814455,
-                0.03890585708618164,
-                0.038724609375,
-                0.0387061767578125,
-                0.03874611282348633,
-                0.03874303817749023,
-                0.038803455352783206,
-                0.03906867218017578,
-                0.0388403205871582,
-                0.03867443084716797,
-                0.03879116821289062,
-                0.03875532913208008,
-                0.03856288146972656,
-                0.03857100677490234,
-                0.03863552093505859,
-                0.0386253776550293,
-                0.038637569427490234,
-                0.03862220764160156,
-                0.03858943939208984,
-                0.03922227096557617,
-                0.038612991333007815,
-                0.03868672180175781,
-                0.038708225250244144,
-                0.03874816131591797,
-                0.03868672180175781,
-                0.03870105743408203,
-                0.03872665786743164,
-                0.03868262481689453,
-                0.038708225250244144,
-                0.03877580642700195,
-                0.038740993499755856,
-                0.03884339141845703,
-                0.038793216705322264,
-                0.038740993499755856,
-                0.03881881713867188,
-                0.03879423904418945,
-                0.03927267074584961,
-                0.038740993499755856,
-                0.038790145874023435,
-                0.03867647933959961,
-                0.03871641540527344,
-                0.03865599822998047,
-                0.03858534240722656,
-                0.038609920501708986,
-                0.0387583999633789,
-                0.038768638610839845,
-                0.038661121368408206,
-                0.038719486236572266,
-                0.03879436874389648,
-                0.03860172653198242,
-                0.038703102111816406,
-                0.0385711669921875,
-                0.038747135162353515,
-                0.038711296081542966,
-                0.03880448150634765,
-                0.03876460647583008,
-                0.03876147079467773,
-                0.03883961486816406,
-                0.038760448455810545,
-                0.0390041618347168,
-                0.03865804672241211,
-                0.038757377624511716,
-                0.03864886474609375,
-                0.03870105743408203,
-                0.0386693115234375,
-                0.0387061767578125,
-                0.03872256088256836,
-                0.03873791885375977,
-                0.038665214538574216,
-                0.03869900894165039,
-                0.03882905578613281,
-                0.03930214309692383,
-                0.03874611282348633,
-                0.03858124923706055,
-                0.03872051239013672,
-                0.038599681854248044,
-                0.03873894500732422,
-                0.03849523162841797,
-                0.03885363388061523,
-                0.038865921020507815,
-                0.03865599822998047,
-                0.038578174591064454,
-                0.038752254486083985,
-                0.03864678573608398,
-                0.03861606216430664,
-                0.03868467330932617,
-                0.0387327995300293,
-                0.03892428970336914,
-                0.03975481414794922,
-                0.03985612869262695,
-                0.03977011108398437,
-                0.03977011108398437,
-                0.03998003387451172,
-                0.03923046493530274,
-                0.03878297424316406,
-                0.03883744049072266,
-                0.03894784164428711,
-                0.038708225250244144,
-                0.03890380859375,
-                0.0388403205871582,
-                0.038950912475585936,
-                0.03884646224975586,
-                0.03892633438110352,
-                0.03893862533569336,
-                0.038866943359375,
-                0.03886899185180664,
-                0.03891302490234375,
-                0.03878809738159179,
-                0.03940556716918946,
-                0.038849536895751956,
-                0.03879423904418945,
-                0.03926323318481445,
-                0.03869900894165039,
-                0.03890585708618164,
-                0.03881369781494141,
-                0.03879116821289062,
-                0.03868262481689453,
-                0.03875020980834961,
-                0.03873894500732422,
-                0.03887411117553711,
-                0.03878297424316406,
-                0.039008255004882815,
-                0.03891814422607422,
-                0.039801856994628904,
-                0.0398551025390625,
-                0.03893657684326172,
-                0.03884368133544922,
-                0.038863872528076174,
-                0.03882291030883789,
-                0.03881881713867188,
-                0.03671551895141602,
-                0.03657932662963867,
-                0.03680767822265625,
-                0.03676287841796875,
-                0.03680767822265625,
-                0.03663363265991211,
-                0.03680051040649414,
-                0.036770816802978515,
-                0.03680255889892578,
-                0.03676160049438477,
-                0.036741119384765625,
-                0.03666329574584961,
-                0.03683327865600586,
-                0.03656294250488281,
-                0.03676160049438477,
-                0.03660083389282227,
-                0.036674625396728514,
-                0.03671449661254883,
-                0.03681075286865235,
-                0.036770816802978515,
-                0.036790271759033204,
-                0.036760574340820314,
-                0.03668479919433594,
-                0.03671142578125,
-                0.03671142578125,
-                0.03673702239990234,
-                0.0367496337890625,
-                0.036734977722167966,
-                0.036601856231689454,
-                0.03677695846557617,
-                0.03661619186401367,
-                0.03668889617919922,
-                0.03671449661254883,
-                0.03664384078979492,
-                0.037158943176269534,
-                0.03665203094482422,
-                0.03664998245239258,
-                0.0366429443359375,
-                0.036713470458984376,
-                0.036751361846923826,
-                0.03668384170532227,
-                0.036751361846923826,
-                0.03670937728881836,
-                0.036772865295410156,
-                0.0366561279296875,
-                0.03666841506958008,
-                0.036631553649902344,
-                0.03710976028442383,
-                0.03670732879638672,
-                0.03665407943725586,
-                0.03657523345947265,
-                0.03661209487915039,
-                0.036640766143798825,
-                0.03665203094482422,
-                0.036572158813476564,
-                0.036636672973632815,
-                0.03667660903930664,
-                0.036603904724121096,
-                0.03683327865600586,
-                0.03670425415039062,
-                0.0367564811706543,
-                0.036596736907958984,
-                0.03688959884643555,
-                0.03712819290161133,
-                0.036708351135253905,
-                0.036744190216064454,
-                0.03668918228149414,
-                0.03660291290283203,
-                0.036547584533691405,
-                0.03664486312866211,
-                0.036729854583740236,
-                0.036672607421875,
-                0.036721664428710936,
-                0.03687628936767578,
-                0.036811775207519534,
-                0.03668377685546875,
-                0.036795391082763675,
-                0.03667763137817383,
-                0.03666534423828125,
-                0.03662643051147461,
-                0.036631553649902344,
-                0.03660287857055664,
-                0.036603904724121096,
-                0.03657523345947265,
-                0.03672063827514648,
-                0.03660595321655274,
-                0.036764766693115236,
-                0.03660800170898437,
-                0.036762622833251955,
-                0.03728998565673828,
-                0.03665203094482422,
-                0.03659468841552734,
-                0.03670937728881836,
-                0.036531360626220706,
-                0.036587520599365236,
-                0.03659980773925781,
-                0.03667763137817383,
-                0.036560897827148435,
-                0.036620288848876956,
-                0.03662847900390625,
-                0.03664896011352539,
-                0.036729854583740236,
-                0.03699097442626953,
-                0.03655372619628906,
-                0.036749313354492184,
-                0.03671654510498047,
-                0.03669504165649414,
-                0.03693056106567383,
-                0.036721664428710936
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6748.9702562906305
         },
         "energy": null,
         "efficiency": null
@@ -288,23 +254,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1010.712576,
             "max_vram": 2858.418176,
             "max_reserved": 2373.976064,
             "max_allocated": 2326.085632
         },
         "latency": {
             "unit": "s",
-            "mean": 7.231899645805356,
-            "stdev": 0.004840455055239001,
             "values": [
-                7.227059190750118,
-                7.236740100860596
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.26044504059248
         },
         "energy": null,
         "efficiency": null
@@ -313,524 +279,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.02836039076786413,
-            "stdev": 0.00015543314162218727,
             "values": [
-                0.02889727973937988,
-                0.028464128494262695,
-                0.028566560745239257,
-                0.028574687957763672,
-                0.02857881546020508,
-                0.028519424438476562,
-                0.028632192611694335,
-                0.028630943298339845,
-                0.028530656814575197,
-                0.02855014419555664,
-                0.028594400405883787,
-                0.028592927932739258,
-                0.028610559463500978,
-                0.028483583450317384,
-                0.028387519836425783,
-                0.02832876777648926,
-                0.02832793617248535,
-                0.02842835235595703,
-                0.028374975204467773,
-                0.028461055755615236,
-                0.02836172866821289,
-                0.028399616241455077,
-                0.02834841537475586,
-                0.02837196731567383,
-                0.028339391708374025,
-                0.028377920150756835,
-                0.028302335739135744,
-                0.028416000366210937,
-                0.02850201606750488,
-                0.028492799758911135,
-                0.02833203125,
-                0.028322816848754883,
-                0.028455936431884765,
-                0.02835456085205078,
-                0.028271615982055662,
-                0.028382207870483397,
-                0.02835558319091797,
-                0.028334079742431642,
-                0.028508159637451173,
-                0.028387327194213868,
-                0.028222591400146484,
-                0.028236671447753905,
-                0.028286975860595705,
-                0.02836582374572754,
-                0.02879692840576172,
-                0.028443647384643556,
-                0.02835763168334961,
-                0.02834841537475586,
-                0.028233760833740233,
-                0.028361696243286133,
-                0.028280832290649413,
-                0.028267520904541016,
-                0.02856755256652832,
-                0.028375295639038085,
-                0.028251903533935547,
-                0.028273664474487304,
-                0.028217344284057616,
-                0.02835968017578125,
-                0.02818252754211426,
-                0.0283240966796875,
-                0.028232448577880858,
-                0.02877542304992676,
-                0.028184576034545897,
-                0.028215295791625978,
-                0.02836172866821289,
-                0.02830745506286621,
-                0.02826767921447754,
-                0.02830524826049805,
-                0.02834124755859375,
-                0.028310527801513673,
-                0.028281856536865234,
-                0.028180479049682617,
-                0.028228607177734375,
-                0.028275711059570312,
-                0.02819891166687012,
-                0.02826153564453125,
-                0.02842425537109375,
-                0.028243743896484375,
-                0.02835456085205078,
-                0.028440576553344726,
-                0.028449792861938477,
-                0.02837196731567383,
-                0.028290048599243164,
-                0.028421119689941408,
-                0.02837401580810547,
-                0.028463104248046874,
-                0.028423168182373046,
-                0.028467199325561524,
-                0.028334079742431642,
-                0.028470272064208983,
-                0.028399679183959962,
-                0.028465087890625,
-                0.0283371524810791,
-                0.028469247817993162,
-                0.028370176315307617,
-                0.02839936065673828,
-                0.028383232116699218,
-                0.028420095443725587,
-                0.02838937568664551,
-                0.028399616241455077,
-                0.02851840019226074,
-                0.028298240661621094,
-                0.028391616821289063,
-                0.028290079116821288,
-                0.028343072891235353,
-                0.028434431076049805,
-                0.028407808303833007,
-                0.028398624420166017,
-                0.028337120056152344,
-                0.028427263259887696,
-                0.02840985679626465,
-                0.0285296630859375,
-                0.02835369682312012,
-                0.028356447219848632,
-                0.028270591735839845,
-                0.028742656707763672,
-                0.028229631423950196,
-                0.028448768615722656,
-                0.028470272064208983,
-                0.028412927627563478,
-                0.028430335998535155,
-                0.028430335998535155,
-                0.02841804885864258,
-                0.028455936431884765,
-                0.028471296310424804,
-                0.02857472038269043,
-                0.028555360794067383,
-                0.028402591705322267,
-                0.028370943069458008,
-                0.028362911224365236,
-                0.028229663848876953,
-                0.028408639907836913,
-                0.028663936614990233,
-                0.028305280685424806,
-                0.028332191467285155,
-                0.028282720565795897,
-                0.028428287506103517,
-                0.028265472412109374,
-                0.02825641632080078,
-                0.02841891288757324,
-                0.028293119430541993,
-                0.028286975860595705,
-                0.02830745506286621,
-                0.02833919906616211,
-                0.028236799240112305,
-                0.028269567489624024,
-                0.028225536346435546,
-                0.028270591735839845,
-                0.028237855911254883,
-                0.028358816146850586,
-                0.028229440689086914,
-                0.02831667137145996,
-                0.02833919906616211,
-                0.02833510398864746,
-                0.028445695877075194,
-                0.02826051139831543,
-                0.028312416076660157,
-                0.028472320556640625,
-                0.02827574348449707,
-                0.028472288131713867,
-                0.028404735565185548,
-                0.028456960678100586,
-                0.028512256622314453,
-                0.02851020812988281,
-                0.02856959915161133,
-                0.028448863983154295,
-                0.028454816818237305,
-                0.028411903381347657,
-                0.028161024093627928,
-                0.02835968017578125,
-                0.028298240661621094,
-                0.028276735305786133,
-                0.02800127983093262,
-                0.027842559814453126,
-                0.027865087509155274,
-                0.027910144805908203,
-                0.02793369674682617,
-                0.02829516792297363,
-                0.028288000106811522,
-                0.028406784057617186,
-                0.028370943069458008,
-                0.02844163131713867,
-                0.028296159744262694,
-                0.028354591369628906,
-                0.02837500762939453,
-                0.028441600799560547,
-                0.028850271224975587,
-                0.028435359954833983,
-                0.02809343910217285,
-                0.02795008087158203,
-                0.028382207870483397,
-                0.028536832809448243,
-                0.028274688720703125,
-                0.028269567489624024,
-                0.02834636878967285,
-                0.028399616241455077,
-                0.02815795135498047,
-                0.028342271804809572,
-                0.028244991302490235,
-                0.028201984405517577,
-                0.028278783798217775,
-                0.02830438423156738,
-                0.028411903381347657,
-                0.028682239532470705,
-                0.028203008651733398,
-                0.02831667137145996,
-                0.0281343994140625,
-                0.02830745506286621,
-                0.028317695617675782,
-                0.028305408477783203,
-                0.02835763168334961,
-                0.028263423919677736,
-                0.028269567489624024,
-                0.028327999114990236,
-                0.02823980712890625,
-                0.028282880783081055,
-                0.028212223052978515,
-                0.02835353660583496,
-                0.02817433547973633,
-                0.028423168182373046,
-                0.02854911994934082,
-                0.02859110450744629,
-                0.02816409683227539,
-                0.028043264389038085,
-                0.027923456192016603,
-                0.02832076835632324,
-                0.028268543243408203,
-                0.028448768615722656,
-                0.028398815155029296,
-                0.028332832336425782,
-                0.02837196731567383,
-                0.02836787223815918,
-                0.028389535903930663,
-                0.028278623580932617,
-                0.028296255111694337,
-                0.028357568740844726,
-                0.028168256759643556,
-                0.02832377624511719,
-                0.028200960159301756,
-                0.0283056640625,
-                0.028219135284423828,
-                0.02832691192626953,
-                0.028184576034545897,
-                0.02818374443054199,
-                0.028164928436279296,
-                0.028231679916381838,
-                0.027881471633911133,
-                0.027975679397583008,
-                0.02817535972595215,
-                0.027900928497314452,
-                0.027900928497314452,
-                0.02798182487487793,
-                0.02795724868774414,
-                0.02794393539428711,
-                0.0279736328125,
-                0.02794495964050293,
-                0.028493824005126952,
-                0.028597248077392577,
-                0.02855833625793457,
-                0.02836479949951172,
-                0.028449792861938477,
-                0.02837299156188965,
-                0.028438528060913085,
-                0.028452863693237306,
-                0.028406784057617186,
-                0.028440639495849608,
-                0.028445632934570312,
-                0.028489728927612305,
-                0.028449920654296874,
-                0.028534656524658204,
-                0.02855731201171875,
-                0.028634111404418947,
-                0.02852249526977539,
-                0.02889625549316406,
-                0.028468223571777345,
-                0.02854911994934082,
-                0.028564479827880858,
-                0.028508159637451173,
-                0.028415136337280274,
-                0.028507999420166016,
-                0.028421119689941408,
-                0.028468223571777345,
-                0.02835148811340332,
-                0.02832383918762207,
-                0.02840185546875,
-                0.028476224899291993,
-                0.028489728927612305,
-                0.028480512619018555,
-                0.028479488372802734,
-                0.02855423927307129,
-                0.028411903381347657,
-                0.028469247817993162,
-                0.028404767990112306,
-                0.0284150390625,
-                0.028509151458740233,
-                0.028517311096191406,
-                0.028464128494262695,
-                0.02855219268798828,
-                0.02835369682312012,
-                0.02845065689086914,
-                0.028519424438476562,
-                0.02850614356994629,
-                0.028421215057373047,
-                0.028519296646118165,
-                0.028463104248046874,
-                0.028480512619018555,
-                0.028251136779785156,
-                0.02854707145690918,
-                0.028503263473510742,
-                0.028447519302368163,
-                0.02838118362426758,
-                0.028467199325561524,
-                0.02850201606750488,
-                0.028483583450317384,
-                0.028445695877075194,
-                0.028494848251342773,
-                0.02840985679626465,
-                0.02853887939453125,
-                0.028413951873779295,
-                0.02838835144042969,
-                0.02854297637939453,
-                0.028398591995239256,
-                0.028432384490966797,
-                0.02840985679626465,
-                0.028473344802856446,
-                0.02838118362426758,
-                0.028495872497558594,
-                0.02894745635986328,
-                0.02839756774902344,
-                0.028487680435180664,
-                0.028451839447021485,
-                0.028391424179077147,
-                0.028504192352294924,
-                0.028441471099853517,
-                0.028464128494262695,
-                0.028411903381347657,
-                0.028485631942749022,
-                0.028465152740478516,
-                0.028548095703125,
-                0.028447744369506835,
-                0.028425216674804688,
-                0.02842624092102051,
-                0.028479488372802734,
-                0.02840575981140137,
-                0.028865535736083983,
-                0.028443840026855467,
-                0.028318527221679688,
-                0.02835558319091797,
-                0.028413951873779295,
-                0.028471296310424804,
-                0.028422143936157225,
-                0.028301311492919923,
-                0.028477439880371092,
-                0.02836387252807617,
-                0.028398496627807617,
-                0.028590080261230468,
-                0.028086271286010742,
-                0.027940959930419923,
-                0.02792745590209961,
-                0.028013568878173828,
-                0.02796550369262695,
-                0.02805241584777832,
-                0.028077056884765625,
-                0.028387327194213868,
-                0.028410911560058594,
-                0.028380128860473634,
-                0.028420095443725587,
-                0.02854604721069336,
-                0.028482559204101563,
-                0.028269567489624024,
-                0.028470272064208983,
-                0.028505088806152344,
-                0.028448928833007814,
-                0.028455936431884765,
-                0.028679008483886718,
-                0.028452863693237306,
-                0.028400640487670898,
-                0.028527616500854492,
-                0.028448768615722656,
-                0.02838528060913086,
-                0.028438528060913085,
-                0.028480575561523436,
-                0.028357568740844726,
-                0.02835763168334961,
-                0.028481536865234375,
-                0.02836275291442871,
-                0.028507135391235353,
-                0.028521472930908204,
-                0.028417024612426758,
-                0.028293119430541993,
-                0.02836089515686035,
-                0.028429311752319338,
-                0.028369728088378905,
-                0.028310527801513673,
-                0.028414976119995116,
-                0.02836185646057129,
-                0.028257152557373048,
-                0.02838937568664551,
-                0.028700672149658202,
-                0.02836787223815918,
-                0.028269567489624024,
-                0.028288000106811522,
-                0.02837401580810547,
-                0.028351680755615234,
-                0.028333887100219727,
-                0.028434431076049805,
-                0.02837606430053711,
-                0.02835264015197754,
-                0.028353408813476564,
-                0.028313600540161132,
-                0.028432384490966797,
-                0.0283371524810791,
-                0.02834022331237793,
-                0.02836275291442871,
-                0.028280832290649413,
-                0.028668928146362304,
-                0.028338176727294922,
-                0.02835148811340332,
-                0.028217344284057616,
-                0.028370943069458008,
-                0.028282880783081055,
-                0.02839347267150879,
-                0.0281343994140625,
-                0.028292095184326172,
-                0.028213504791259766,
-                0.028275455474853516,
-                0.028282880783081055,
-                0.028263423919677736,
-                0.028121088027954103,
-                0.028392448425292968,
-                0.02830438423156738,
-                0.028269567489624024,
-                0.028231679916381838,
-                0.028297216415405273,
-                0.028318880081176757,
-                0.028181375503540038,
-                0.028285919189453126,
-                0.028308479309082032,
-                0.02826451110839844,
-                0.028356544494628905,
-                0.028264448165893553,
-                0.028348512649536133,
-                0.028264352798461914,
-                0.02836070442199707,
-                0.028268543243408203,
-                0.028264448165893553,
-                0.028256256103515624,
-                0.028321792602539062,
-                0.028267520904541016,
-                0.028314624786376953,
-                0.028408832550048828,
-                0.02837299156188965,
-                0.02837708854675293,
-                0.028293119430541993,
-                0.028305408477783203,
-                0.02835456085205078,
-                0.028408832550048828,
-                0.028434431076049805,
-                0.028438528060913085,
-                0.028317695617675782,
-                0.028297279357910158,
-                0.02824799919128418,
-                0.02834022331237793,
-                0.028334079742431642,
-                0.028439552307128906,
-                0.02835068893432617,
-                0.028376863479614257,
-                0.028308479309082032,
-                0.028308479309082032,
-                0.028645376205444335,
-                0.028087295532226563,
-                0.028425216674804688,
-                0.02832383918762207,
-                0.02831155204772949,
-                0.02838015937805176,
-                0.0284816951751709,
-                0.02822127914428711,
-                0.028249088287353515,
-                0.028338176727294922,
-                0.028342271804809572,
-                0.02822047996520996,
-                0.02848249626159668,
-                0.028425216674804688,
-                0.028261375427246094,
-                0.02798080062866211,
-                0.027871231079101562,
-                0.028398752212524414,
-                0.027954015731811523,
-                0.02797056007385254,
-                0.02793779182434082,
-                0.028428287506103517,
-                0.02830335998535156,
-                0.02827894401550293,
-                0.0284619197845459,
-                0.028268800735473634,
-                0.028278528213500978,
-                0.02832713508605957,
-                0.02830518341064453,
-                0.028305408477783203,
-                0.028265472412109374,
-                0.02838118362426758,
-                0.02829929542541504,
-                0.028248031616210936,
-                0.028289024353027343,
-                0.028228607177734375,
-                0.028314624786376953,
-                0.028246015548706056,
-                0.02833510398864746,
-                0.02836275291442871,
-                0.02834534454345703
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.2604450405925
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 998.940672,
             "max_vram": 2533.359616,
             "max_reserved": 2101.346304,
             "max_allocated": 2014.912
         },
         "latency": {
             "unit": "s",
+            "mean": 0.04358285295237668,
+            "stdev": 0.0011312976009216172,
             "values": [
+                0.060005313873291014,
+                0.046088191986083986,
+                0.04404121780395508,
+                0.04407398223876953,
+                0.044042240142822264,
+                0.04398796844482422,
+                0.044096511840820314,
+                0.0440709114074707,
+                0.043960319519042966,
+                0.04405145645141602,
+                0.04407603073120117,
+                0.044066814422607424,
+                0.044221439361572266,
+                0.04668723297119141,
+                0.04373708724975586,
+                0.04333158493041992,
+                0.043246593475341794,
+                0.043379711151123046,
+                0.043289600372314455,
+                0.04339199829101562,
+                0.04352819061279297,
+                0.043450366973876955,
+                0.04334694290161133,
+                0.04347391891479492,
+                0.04351795196533203,
+                0.04333184051513672,
+                0.0433704948425293,
+                0.04354969787597656,
+                0.043324417114257815,
+                0.04343603134155274,
+                0.043394046783447264,
+                0.043509761810302736,
+                0.04348518371582031,
+                0.043409408569335936,
+                0.04373811340332031,
+                0.04346879959106445,
+                0.04343500900268555,
+                0.04351385498046875,
+                0.043442176818847655,
+                0.04352511978149414,
+                0.04334796905517578,
+                0.043442176818847655,
+                0.043458560943603515,
+                0.04348416137695312,
+                0.043565055847167966,
+                0.04338483047485352,
+                0.043507713317871094,
+                0.043437057495117185,
+                0.04332032012939453,
+                0.04348416137695312,
+                0.04341862487792969,
+                0.04338995361328125,
+                0.04325068664550781,
+                0.043358207702636715,
+                0.043509822845458984,
+                0.04344627380371094,
+                0.043445247650146485,
+                0.04337868881225586,
+                0.04325580978393555,
+                0.04345139312744141,
+                0.04334592056274414,
+                0.043453441619873044,
+                0.04331932830810547,
+                0.04338687896728516,
+                0.04346572875976563,
+                0.043379711151123046,
+                0.043363327026367186,
+                0.043391231536865235,
+                0.04327526473999024,
+                0.04338380813598633,
+                0.043218944549560545,
+                0.043561985015869144,
+                0.043319297790527345,
+                0.04359395217895508,
+                0.043358207702636715,
+                0.04346777725219726,
+                0.043350177764892577,
+                0.04333977508544922,
+                0.04348723220825195,
+                0.04357222366333008,
+                0.043458560943603515,
+                0.043232257843017576,
+                0.04331622314453125,
+                0.04341452789306641,
+                0.043390975952148435,
+                0.0433623046875,
+                0.04352511978149414,
+                0.04335308837890625,
+                0.04335103988647461,
+                0.04323123168945313,
+                0.0432803840637207,
+                0.04330393600463867,
+                0.043494400024414064,
+                0.04326297760009765,
+                0.04330188751220703,
+                0.04320278549194336,
+                0.043227134704589845,
+                0.04327423858642578,
+                0.04346265411376953,
+                0.04339129638671875,
+                0.04335001754760742,
+                0.04341862487792969,
+                0.04336947250366211,
+                0.04332953643798828,
+                0.043496448516845705,
+                0.043202560424804685,
+                0.04329062271118164,
+                0.043170814514160154,
+                0.043401214599609376,
+                0.043483135223388675,
+                0.043312126159667966,
+                0.043393024444580076,
+                0.043308032989501956,
+                0.043368671417236326,
+                0.043410430908203124,
+                0.0433704948425293,
+                0.04336947250366211,
+                0.04336537551879883,
+                0.04327423858642578,
+                0.043753471374511715,
+                0.04357529449462891,
+                0.043622398376464845,
+                0.04343299102783203,
+                0.04351295852661133,
+                0.04348723220825195,
+                0.043422721862792966,
+                0.04354048156738281,
+                0.04361523056030273,
+                0.04343500900268555,
+                0.04354048156738281,
+                0.04352511978149414,
+                0.04345471954345703,
+                0.04335615921020508,
+                0.04370534515380859,
+                0.04352102279663086,
+                0.043423744201660154,
+                0.043390975952148435,
+                0.04353036880493164,
+                0.043512832641601565,
+                0.04339507293701172,
+                0.04361852645874023,
+                0.043442176818847655,
+                0.04343091201782227,
+                0.04339712142944336,
+                0.04333158493041992,
+                0.04348518371582031,
+                0.04340019226074219,
+                0.04350156784057617,
+                0.04361011123657227,
+                0.04348825454711914,
+                0.04335923385620117,
+                0.043425792694091796,
+                0.043579391479492184,
+                0.04350566482543945,
+                0.043412704467773434,
+                0.04362956619262695,
+                0.04350361633300781,
+                0.043512832641601565,
+                0.04362956619262695,
+                0.04357222366333008,
+                0.043422721862792966,
+                0.043668479919433595,
+                0.04331622314453125,
+                0.043514881134033206,
+                0.04347513580322265,
+                0.04348121643066406,
+                0.043496448516845705,
+                0.043578369140625,
+                0.043461631774902344,
+                0.04357734298706055,
+                0.043463680267333986,
+                0.04360396957397461,
+                0.043581439971923826,
+                0.04357324981689453,
+                0.04348108673095703,
+                0.043730945587158204,
+                0.043646976470947264,
+                0.04357529449462891,
+                0.04361625671386719,
+                0.04351500701904297,
+                0.04349542236328125,
+                0.04343603134155274,
+                0.04354662322998047,
+                0.043361438751220706,
+                0.04365548706054687,
+                0.04348620986938476,
+                0.04341452789306641,
+                0.04354457473754883,
+                0.04359270477294922,
+                0.04355686569213867,
+                0.043507713317871094,
+                0.043548671722412106,
+                0.04344319915771484,
+                0.04354969787597656,
+                0.04340224075317383,
+                0.04353747177124023,
+                0.04345052719116211,
+                0.04338995361328125,
+                0.043600894927978515,
+                0.043496448516845705,
+                0.043524097442626954,
+                0.043518177032470705,
+                0.04366233444213867,
+                0.043573375701904296,
+                0.04357324981689453,
+                0.043597824096679685,
+                0.043676673889160154,
+                0.043584510803222655,
+                0.04342169570922851,
+                0.04358758544921875,
+                0.04350156784057617,
+                0.043501663208007815,
+                0.04353948974609375,
+                0.04351385498046875,
+                0.043377662658691404,
+                0.04350259017944336,
+                0.0435230712890625,
+                0.04349849700927735,
+                0.043417728424072266,
+                0.043415550231933595,
+                0.04348620986938476,
+                0.04352614212036133,
+                0.043579391479492184,
+                0.04358553695678711,
+                0.04354150390625,
+                0.04356095886230469,
+                0.04335923385620117,
+                0.04352000045776367,
+                0.04349849700927735,
+                0.04351078414916992
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 5873.869713846709
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 998.940672,
             "max_vram": 2858.418176,
             "max_reserved": 2373.976064,
             "max_allocated": 2326.085632
         },
         "latency": {
             "unit": "s",
+            "mean": 8.749820934295656,
+            "stdev": 0.004574188232418841,
             "values": [
+                8.754395122528074,
+                8.745246746063236
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 29.14345355349001
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.03431302327174767,
+            "stdev": 0.0001471226828680532,
             "values": [
+                0.03473100662231445,
+                0.03461119842529297,
+                0.03435724639892578,
+                0.03457535934448242,
+                0.034272254943847655,
+                0.03426611328125,
+                0.03434393692016602,
+                0.03422412872314453,
+                0.034129920959472655,
+                0.03416371154785156,
+                0.03421286392211914,
+                0.03423846435546875,
+                0.03415859222412109,
+                0.034326526641845705,
+                0.03412377548217774,
+                0.03419750213623047,
+                0.03419955062866211,
+                0.03450368118286133,
+                0.03429478454589844,
+                0.034508800506591795,
+                0.03427942276000977,
+                0.03440639877319336,
+                0.03443199920654297,
+                0.03435520172119141,
+                0.03431321716308594,
+                0.03450163269042969,
+                0.03418236923217773,
+                0.034309921264648435,
+                0.03448320007324219,
+                0.034315265655517575,
+                0.0343296012878418,
+                0.03439308929443359,
+                0.034105342864990236,
+                0.03420159912109375,
+                0.034274528503417966,
+                0.034144031524658204,
+                0.034049022674560545,
+                0.03424665451049805,
+                0.034482177734375,
+                0.03439820861816406,
+                0.03425484848022461,
+                0.03434086227416992,
+                0.03423846435546875,
+                0.03430912017822266,
+                0.03445145416259766,
+                0.034252799987792966,
+                0.034307071685791016,
+                0.034283519744873044,
+                0.034179073333740234,
+                0.03441356658935547,
+                0.03405619049072266,
+                0.034459648132324217,
+                0.034516990661621096,
+                0.03435007858276367,
+                0.03431628799438476,
+                0.034339839935302735,
+                0.03422719955444336,
+                0.03449139022827148,
+                0.03443302536010742,
+                0.03445145416259766,
+                0.034310142517089845,
+                0.03428659057617187,
+                0.0342210578918457,
+                0.03441664123535156,
+                0.034269184112548826,
+                0.034492416381835936,
+                0.034423809051513675,
+                0.03469823837280273,
+                0.03415039825439453,
+                0.034223102569580076,
+                0.03431935882568359,
+                0.03443814468383789,
+                0.034252799987792966,
+                0.03419750213623047,
+                0.03418009567260742,
+                0.034402305603027344,
+                0.034523136138916014,
+                0.034402305603027344,
+                0.03441971206665039,
+                0.034361343383789066,
+                0.03436236953735351,
+                0.034385921478271485,
+                0.03440435028076172,
+                0.03456716918945312,
+                0.03427942276000977,
+                0.034353153228759765,
+                0.03433574295043945,
+                0.034476032257080076,
+                0.03437158584594727,
+                0.03422617721557617,
+                0.03421388626098633,
+                0.0342476806640625,
+                0.034339839935302735,
+                0.034377727508544925,
+                0.0341739501953125,
+                0.03407257461547852,
+                0.03424153518676758,
+                0.03430403137207031,
+                0.0343326416015625,
+                0.034353153228759765,
+                0.034476032257080076,
+                0.03424460983276367,
+                0.034187263488769534,
+                0.034293758392333985,
+                0.03434598541259765,
+                0.034326526641845705,
+                0.034477054595947264,
+                0.03434905624389648,
+                0.034435073852539064,
+                0.034328575134277346,
+                0.0342999038696289,
+                0.03424870300292969,
+                0.034315265655517575,
+                0.03433267211914062,
+                0.0342457275390625,
+                0.03439401626586914,
+                0.03422208023071289,
+                0.034411518096923825,
+                0.034477054595947264,
+                0.03437363052368164,
+                0.03418931198120117,
+                0.034369537353515625,
+                0.03431436920166016,
+                0.03432230377197266,
+                0.03428147125244141,
+                0.03425484848022461,
+                0.034531326293945314,
+                0.03444326400756836,
+                0.03431935882568359,
+                0.034405376434326174,
+                0.034356353759765625,
+                0.034350975036621094,
+                0.03430809783935547,
+                0.034587646484375,
+                0.034305023193359374,
+                0.03446579360961914,
+                0.03431219100952149,
+                0.03445145416259766,
+                0.0345241584777832,
+                0.03460710525512695,
+                0.03424358367919922,
+                0.03447097778320313,
+                0.03433567810058594,
+                0.03449139022827148,
+                0.034356224060058595,
+                0.03460505676269531,
+                0.03427328109741211,
+                0.03421388626098633,
+                0.03427840042114258,
+                0.03418624114990235,
+                0.03428147125244141,
+                0.03421286392211914,
+                0.03404390335083008,
+                0.034138111114501955,
+                0.0341104621887207,
+                0.03425894546508789,
+                0.03476070404052734,
+                0.0343818244934082,
+                0.03417702484130859,
+                0.034388992309570314,
+                0.03423539352416992,
+                0.03424256134033203,
+                0.03422719955444336,
+                0.03430604934692383,
+                0.03441254425048828,
+                0.03455692672729492,
+                0.03434905624389648,
+                0.03451596832275391,
+                0.034348033905029295,
+                0.034563072204589845,
+                0.03443711853027344,
+                0.0346951675415039,
+                0.034277374267578126,
+                0.03448831939697266,
+                0.034530303955078126,
+                0.03445657730102539,
+                0.03451903915405274,
+                0.03437875366210937,
+                0.034482177734375,
+                0.03442585754394531,
+                0.034418689727783204,
+                0.03452620697021484,
+                0.03431219100952149,
+                0.03433881759643555,
+                0.03412684631347656,
+                0.034255870819091795,
+                0.03420979309082031,
+                0.03446476745605469,
+                0.03459993743896484,
+                0.034459648132324217,
+                0.03442892837524414,
+                0.034759681701660154,
+                0.03448524856567383,
+                0.034358272552490236,
+                0.034237438201904294,
+                0.03433881759643555,
+                0.03434598541259765,
+                0.03416883087158203,
+                0.03405619049072266,
+                0.0342108154296875,
+                0.03427635192871094,
+                0.03435520172119141,
+                0.034374656677246096,
+                0.03448831939697266,
+                0.03422617721557617,
+                0.034334720611572264,
+                0.034132991790771484,
+                0.034315265655517575,
+                0.03445555114746094,
+                0.034530303955078126,
+                0.034353153228759765,
+                0.03441356658935547,
+                0.03433369445800781,
+                0.034574337005615234,
+                0.03423539352416992,
+                0.03432447814941406,
+                0.034184192657470705,
+                0.03424358367919922,
+                0.03409408187866211,
+                0.03450982284545898,
+                0.03427020645141601,
+                0.03432243347167969,
+                0.034176063537597653,
+                0.03434796905517578,
+                0.03426303863525391,
+                0.034272254943847655,
+                0.03409612655639648,
+                0.034135040283203126,
+                0.03403776168823242,
+                0.03437363052368164,
+                0.034402305603027344,
+                0.034298881530761716,
+                0.034356224060058595,
+                0.03439308929443359,
+                0.03421388626098633,
+                0.03420774459838867,
+                0.03414527893066406,
+                0.03433369445800781,
+                0.034203647613525394,
+                0.03442073440551758,
+                0.034446334838867186,
+                0.03440639877319336,
+                0.03416985702514649,
+                0.03419647979736328,
+                0.033995777130126956,
+                0.03419443130493164,
+                0.03410124969482422,
+                0.0342108154296875,
+                0.034326526641845705,
+                0.03426201629638672,
+                0.03419750213623047,
+                0.03429785537719727,
+                0.03427328109741211,
+                0.03425689697265625,
+                0.034067455291748046,
+                0.0338524169921875,
+                0.03434700775146484,
+                0.03428966522216797,
+                0.034508800506591795,
+                0.034369537353515625,
+                0.03467059326171875,
+                0.03440639877319336,
+                0.034427902221679685,
+                0.03465420913696289,
+                0.03436646270751953,
+                0.03440127944946289,
+                0.03441459274291992,
+                0.0343111686706543,
+                0.03428659057617187,
+                0.034288639068603514,
+                0.03426406478881836,
+                0.03443302536010742,
+                0.03483955383300781,
+                0.03446476745605469,
+                0.034252799987792966,
+                0.03408281707763672,
+                0.0341473274230957,
+                0.03415244674682617,
+                0.03422617721557617,
+                0.03422822570800781,
+                0.03419852828979492,
+                0.03439923095703125,
+                0.03462451171875,
+                0.03439923095703125,
+                0.034304000854492187,
+                0.034518081665039065,
+                0.03444934463500977,
+                0.03420569610595703,
+                0.034282497406005856,
+                0.03439923095703125,
+                0.034511871337890625,
+                0.0343296012878418,
+                0.03441254425048828,
+                0.03435417556762695,
+                0.034440193176269535,
+                0.03466342544555664,
+                0.034460670471191404,
+                0.034513919830322266,
+                0.03451801681518555,
+                0.034385921478271485,
+                0.03422412872314453,
+                0.03438387298583984,
+                0.03442278289794922,
+                0.03426303863525391,
+                0.034385921478271485,
+                0.03438284683227539,
+                0.03422003173828125,
+                0.03427840042114258,
+                0.03444736099243164,
+                0.03431321716308594,
+                0.034252799987792966,
+                0.03425177764892578,
+                0.03429478454589844,
+                0.03426508712768555,
+                0.03427123260498047,
+                0.0342845458984375,
+                0.03420467376708984,
+                0.034127872467041014,
+                0.03421491241455078,
+                0.03425177764892578,
+                0.03413401412963867,
+                0.0343111686706543,
+                0.03425689697265625,
+                0.034118656158447266,
+                0.034290687561035156,
+                0.03423027038574219,
+                0.034255870819091795,
+                0.03428659057617187,
+                0.03439308929443359,
+                0.034351104736328124,
+                0.034375679016113284,
+                0.03421286392211914,
+                0.03437977600097656,
+                0.03445145416259766,
+                0.03473715209960938,
+                0.03423129653930664,
+                0.03435212707519531,
+                0.034146305084228515,
+                0.034353153228759765,
+                0.0342927360534668,
+                0.03411455917358398,
+                0.03420467376708984,
+                0.034462718963623046,
+                0.034552833557128904,
+                0.03448524856567383,
+                0.03435520172119141,
+                0.03426201629638672,
+                0.03406438446044922,
+                0.03430912017822266,
+                0.034181121826171876,
+                0.034236415863037106,
+                0.034124801635742184,
+                0.03446886444091797,
+                0.03427635192871094,
+                0.03422208023071289,
+                0.03406131362915039,
+                0.03421184158325195,
+                0.034034687042236327,
+                0.03415654373168946,
+                0.03398758316040039,
+                0.034342910766601564,
+                0.0344268798828125,
+                0.03427123260498047,
+                0.0341923828125,
+                0.03434905624389648,
+                0.0343633918762207,
+                0.03442892837524414,
+                0.03430092620849609,
+                0.034334720611572264,
+                0.034385921478271485,
+                0.03445145416259766,
+                0.03430297470092773,
+                0.03431219100952149,
+                0.03450470352172851,
+                0.03455692672729492,
+                0.03415347290039063,
+                0.034255870819091795,
+                0.03429478454589844,
+                0.03434905624389648,
+                0.0342108154296875,
+                0.034249729156494144,
+                0.03408588790893555,
+                0.03438079833984375,
+                0.03462451171875,
+                0.03432447814941406,
+                0.03429171371459961,
+                0.03438796615600586,
+                0.0343633918762207,
+                0.03438387298583984,
+                0.034249729156494144,
+                0.034444286346435544,
+                0.03428761672973633,
+                0.034520065307617184,
+                0.034095104217529294,
+                0.03424563217163086,
+                0.03408588790893555,
+                0.03412684631347656,
+                0.034129920959472655,
+                0.03416883087158203,
+                0.03413913726806641,
+                0.0342476806640625,
+                0.03416678237915039,
+                0.034105342864990236,
+                0.03441049575805664,
+                0.034255870819091795,
+                0.03398963165283203,
+                0.03408588790893555,
+                0.03412275314331055,
+                0.03422003173828125,
+                0.03419340896606445,
+                0.03417804718017578,
+                0.034181121826171876,
+                0.03438796615600586,
+                0.034282497406005856,
+                0.03437158584594727,
+                0.03426713562011719,
+                0.034229248046875,
+                0.03417190551757812,
+                0.034216960906982424,
+                0.03421286392211914,
+                0.034239486694335936,
+                0.034154495239257815,
+                0.03441049575805664,
+                0.03434086227416992,
+                0.03434598541259765,
+                0.03450368118286133,
+                0.034385921478271485,
+                0.034136062622070314,
+                0.03424665451049805,
+                0.03423436737060547,
+                0.034342910766601564,
+                0.0342476806640625,
+                0.03417702484130859,
+                0.03401318359375,
+                0.03431423950195313,
+                0.03448729705810547,
+                0.03428147125244141,
+                0.034274303436279296,
+                0.034121726989746096,
+                0.03399782562255859,
+                0.034108417510986325,
+                0.0339681282043457,
+                0.03406131362915039,
+                0.03393638229370117,
+                0.03403673553466797,
+                0.03425996780395508,
+                0.03418624114990235,
+                0.03433779144287109,
+                0.034367488861083983,
+                0.0341401596069336,
+                0.03411455917358398,
+                0.03400396728515625,
+                0.034108417510986325,
+                0.0340766716003418,
+                0.034170879364013675,
+                0.0342999038696289,
+                0.034460670471191404,
+                0.034408447265625,
+                0.03475558471679688,
+                0.03426713562011719,
+                0.03424051284790039,
+                0.03416166305541992,
+                0.03409920120239258,
+                0.03409408187866211,
+                0.034388992309570314,
+                0.03424153518676758,
+                0.03425894546508789,
+                0.034164737701416016,
+                0.03453849411010742,
+                0.03436236953735351,
+                0.03434188842773438,
+                0.0342210578918457,
+                0.034321407318115234,
+                0.03423539352416992,
+                0.034405376434326174,
+                0.034372608184814454,
+                0.03435007858276367,
+                0.03416384124755859,
+                0.03466432189941406,
+                0.03418624114990235,
+                0.03422003173828125,
+                0.03422822570800781,
+                0.034374656677246096,
+                0.034351104736328124,
+                0.03443916702270508,
+                0.03428659057617187,
+                0.03418624114990235,
+                0.03426611328125,
+                0.034179073333740234,
+                0.03401113510131836,
+                0.03414527893066406,
+                0.03422208023071289,
+                0.034149375915527344,
+                0.03418931198120117,
+                0.03425177764892578,
+                0.03419136047363281,
+                0.03457843017578125,
+                0.03430604934692383,
+                0.034513919830322266,
+                0.03446783828735352,
+                0.03431731033325195,
+                0.03418931198120117,
+                0.03435007858276367,
+                0.03421184158325195,
+                0.034427902221679685,
+                0.034361343383789066,
+                0.03470438385009766,
+                0.034477054595947264,
+                0.03461017608642578,
+                0.03449958419799805
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 29.14345355349001
         },
         "energy": null,
         "efficiency": null