kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Mar 14

Commit

274f00d

•

1 Parent(s): 1b4e6a2

Upload /gemma-7b/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

gemma-7b/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +681 -681

gemma-7b/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,202 +2,202 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1092.5056,
             "max_vram": 6796.869632,
             "max_reserved": 6312.42752,
             "max_allocated": 6191.16288
         },
         "latency": {
             "unit": "s",
-            "mean": 0.0554454770588743,
-            "stdev": 0.0004663816241365346,
             "values": [
-                0.06086252975463867,
-                0.057556991577148435,
-                0.05733478546142578,
-                0.05619302368164063,
-                0.055826431274414064,
-                0.05585203170776367,
-                0.05542195129394531,
-                0.05526425552368164,
-                0.055316478729248046,
-                0.0552806396484375,
-                0.05524991989135742,
-                0.05527142333984375,
-                0.05532160186767578,
-                0.05532364654541016,
-                0.05526732635498047,
-                0.0552540168762207,
-                0.05535948944091797,
-                0.05530214309692383,
-                0.05532262420654297,
-                0.05526732635498047,
-                0.05538508987426758,
-                0.05527865600585938,
-                0.055395328521728515,
-                0.05530112075805664,
-                0.055370880126953126,
-                0.05529395294189453,
-                0.055354366302490236,
-                0.05530419158935547,
-                0.055410785675048826,
-                0.055329887390136716,
-                0.05536767959594727,
-                0.05533497619628906,
-                0.05533798217773438,
-                0.05532793426513672,
-                0.05535027313232422,
-                0.05534515380859375,
                 0.055346176147460936,
-                0.055387134552001956,
-                0.05535334396362305,
-                0.05540556716918945,
-                0.05538816070556641,
-                0.05535129547119141,
-                0.05537484741210937,
-                0.05542195129394531,
-                0.05540454483032226,
-                0.05540249633789063,
-                0.05536972808837891,
-                0.055444480895996094,
-                0.055378944396972656,
-                0.055462913513183595,
-                0.055411712646484375,
                 0.05542195129394531,
-                0.055416831970214846,
-                0.05541388702392578,
-                0.055414783477783204,
-                0.055430145263671876,
-                0.05539126586914062,
-                0.05540454483032226,
-                0.05533184051513672,
-                0.05538918304443359,
-                0.05534822463989258,
-                0.05540153503417969,
-                0.05540761566162109,
                 0.05541888046264649,
-                0.05538508987426758,
-                0.05533695983886719,
                 0.05538816070556641,
-                0.05535232162475586,
-                0.055390209197998044,
                 0.05537484741210937,
-                0.05534003067016602,
-                0.05540966415405273,
-                0.05534822463989258,
-                0.05533388900756836,
-                0.055419902801513675,
-                0.05529203033447266,
-                0.05536153411865234,
                 0.055375873565673826,
-                0.055392257690429686,
-                0.05540569686889649,
-                0.055400447845458986,
-                0.055406593322753904,
-                0.05535027313232422,
-                0.05541580963134766,
-                0.055362560272216796,
-                0.055406593322753904,
-                0.05544243240356445,
-                0.05542502212524414,
-                0.05542092895507812,
-                0.05539126586914062,
-                0.05543423843383789,
-                0.05539123153686523,
-                0.05543526458740235,
-                0.05542092895507812,
-                0.055403518676757815,
                 0.05536870574951172,
                 0.055387134552001956,
-                0.05540556716918945,
-                0.05535232162475586,
-                0.05537996673583984,
-                0.05544460678100586,
-                0.05541273498535156,
-                0.05544140625,
-                0.05540780639648438,
-                0.05541580963134766,
-                0.0553994255065918,
-                0.05532057571411133,
-                0.05541273498535156,
-                0.05542195129394531,
-                0.055392513275146486,
                 0.05536972808837891,
-                0.05535027313232422,
-                0.0553994255065918,
                 0.05535539245605469,
-                0.055411712646484375,
-                0.05534342575073242,
                 0.055393280029296874,
-                0.055346176147460936,
                 0.05536665725708008,
-                0.055431167602539064,
                 0.05539737701416016,
-                0.055354366302490236,
-                0.055375873565673826,
-                0.055362560272216796,
-                0.055382015228271485,
-                0.05544038391113281,
-                0.055413761138916016,
-                0.05538918304443359,
-                0.05534310531616211,
-                0.05535846328735351,
-                0.05537177658081055,
-                0.05534310531616211,
-                0.055662593841552734,
                 0.05537177658081055,
-                0.05538918304443359,
-                0.055300094604492187,
                 0.055373825073242185,
-                0.055400447845458986,
-                0.0555335693359375,
-                0.055416831970214846,
-                0.05536153411865234,
                 0.05539737701416016,
-                0.05540556716918945,
-                0.05542604827880859,
-                0.05539228820800781,
                 0.055384063720703126,
                 0.05541273498535156,
-                0.05540985488891602,
-                0.05543244934082031,
-                0.05541888046264649,
                 0.055416831970214846,
-                0.05532979202270508,
-                0.055419166564941405,
-                0.05542502212524414,
-                0.05542604827880859,
-                0.055446529388427736,
-                0.055408641815185546,
-                0.05539430236816406,
-                0.055387134552001956,
-                0.055465984344482425,
-                0.05535129547119141,
-                0.05542412948608398,
-                0.05538304138183594,
-                0.05541888046264649,
-                0.0553994255065918,
-                0.05537484741210937,
-                0.05544345474243164,
-                0.0553994255065918,
-                0.055354366302490236,
                 0.0553809928894043,
-                0.05542092895507812,
                 0.055363582611083983,
-                0.055454814910888675,
-                0.055408641815185546,
-                0.05529395294189453,
-                0.055400447845458986,
-                0.0553994255065918,
-                0.05542915344238281,
-                0.05540249633789063,
-                0.05542707061767578,
-                0.05538524627685547
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 4617.14847774091
         },
         "energy": null,
         "efficiency": null
@@ -205,23 +205,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1092.5056,
             "max_vram": 7212.105728,
             "max_reserved": 6727.663616,
             "max_allocated": 6570.75456
         },
         "latency": {
             "unit": "s",
-            "mean": 7.895504375457767,
-            "stdev": 0.03317504310608088,
             "values": [
-                7.928679418563847,
-                7.862329332351686
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 32.2968600704772
         },
         "energy": null,
         "efficiency": null
@@ -230,524 +230,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.03096276225669712,
-            "stdev": 0.0007418630006546437,
             "values": [
-                0.03081625556945801,
-                0.031161344528198243,
-                0.030646272659301758,
-                0.03083673667907715,
-                0.030887935638427736,
-                0.030846975326538087,
-                0.03084492874145508,
-                0.030793727874755858,
-                0.03078246307373047,
-                0.03079475212097168,
-                0.030785535812377928,
-                0.030809087753295897,
-                0.030712831497192384,
-                0.03077017593383789,
-                0.03467468643188477,
-                0.035454975128173825,
-                0.03563008117675781,
-                0.03514572906494141,
-                0.0348487663269043,
-                0.03461939239501953,
-                0.03455692672729492,
-                0.03460710525512695,
-                0.03475251388549805,
-                0.034625537872314455,
-                0.03277926254272461,
-                0.032715774536132815,
-                0.03133132743835449,
-                0.033539070129394534,
-                0.032826465606689455,
-                0.03347241592407227,
-                0.032421886444091795,
-                0.031038463592529295,
-                0.03073023986816406,
-                0.030729215621948244,
-                0.030690303802490236,
-                0.030704639434814454,
-                0.0307957763671875,
-                0.030838783264160157,
-                0.030740480422973632,
-                0.030740480422973632,
-                0.03074764823913574,
-                0.030766080856323243,
-                0.03078348731994629,
-                0.03077631950378418,
-                0.03134668731689453,
-                0.031099903106689454,
-                0.03076198387145996,
-                0.030817279815673827,
-                0.030736383438110353,
-                0.031029247283935548,
-                0.030942207336425782,
-                0.03074662399291992,
-                0.030700544357299804,
-                0.030785535812377928,
-                0.030657535552978517,
-                0.030769151687622072,
-                0.030698495864868162,
-                0.030673919677734376,
-                0.03076812744140625,
-                0.03081625556945801,
-                0.03083263969421387,
-                0.03079270362854004,
-                0.030847999572753908,
-                0.03072204780578613,
-                0.030718975067138672,
-                0.03119001579284668,
-                0.03085312080383301,
-                0.03077324867248535,
-                0.03082444763183594,
-                0.030843904495239258,
-                0.03078041648864746,
-                0.030716928482055664,
-                0.03076198387145996,
-                0.030907392501831055,
-                0.030675968170166015,
-                0.030741504669189453,
-                0.030874624252319335,
-                0.031073280334472656,
-                0.030836832046508788,
-                0.030646175384521485,
-                0.030741504669189453,
-                0.030703615188598633,
-                0.030765056610107422,
-                0.030855167388916017,
-                0.030675968170166015,
-                0.03079884719848633,
-                0.030819328308105468,
-                0.03077222442626953,
-                0.030721023559570314,
-                0.030728191375732423,
-                0.030712831497192384,
-                0.03077120018005371,
-                0.030698495864868162,
-                0.030657535552978517,
-                0.030687231063842774,
-                0.030693376541137695,
-                0.030731391906738282,
-                0.0306932487487793,
-                0.030877695083618165,
-                0.030657535552978517,
-                0.030698495864868162,
-                0.03082342338562012,
-                0.03077939224243164,
-                0.030850048065185546,
-                0.030744575500488282,
-                0.030687231063842774,
-                0.030749696731567383,
-                0.030879743576049806,
-                0.03078758430480957,
-                0.03075481605529785,
-                0.030676992416381835,
-                0.03072204780578613,
-                0.030850048065185546,
-                0.030676992416381835,
-                0.030814207077026368,
-                0.03122790336608887,
-                0.030826496124267577,
-                0.030756864547729492,
-                0.030823488235473633,
-                0.030783424377441405,
-                0.030742528915405274,
-                0.03079987144470215,
-                0.03079987144470215,
-                0.030901248931884766,
-                0.030727167129516602,
-                0.03080601692199707,
-                0.030846975326538087,
-                0.030818304061889647,
-                0.030757888793945313,
-                0.03078758430480957,
-                0.03100876808166504,
-                0.031054847717285155,
-                0.03077120018005371,
-                0.03081318473815918,
-                0.030737407684326173,
-                0.030785535812377928,
-                0.030838783264160157,
-                0.03077631950378418,
-                0.03715584182739258,
-                0.035434497833251956,
-                0.03479449462890625,
-                0.03339571380615235,
-                0.031710208892822264,
-                0.030769151687622072,
-                0.030708736419677734,
-                0.030838783264160157,
-                0.03073244857788086,
-                0.030737247467041016,
-                0.03075993537902832,
-                0.03075481605529785,
-                0.030882816314697265,
-                0.0307589111328125,
-                0.030900224685668946,
-                0.030785535812377928,
-                0.03082444763183594,
-                0.030835712432861328,
-                0.03077017593383789,
-                0.030830591201782227,
-                0.030880767822265624,
-                0.030858240127563476,
-                0.03077836799621582,
-                0.03082137680053711,
-                0.030860288619995117,
-                0.030826496124267577,
-                0.030736383438110353,
-                0.03080294418334961,
-                0.030740480422973632,
-                0.030834688186645507,
-                0.030833663940429686,
-                0.03079680061340332,
-                0.03081625556945801,
-                0.03077222442626953,
-                0.03082444763183594,
-                0.030745599746704103,
-                0.03078665542602539,
-                0.030721952438354492,
-                0.030744575500488282,
-                0.030710784912109376,
-                0.030850048065185546,
-                0.030749696731567383,
-                0.03125760078430176,
-                0.03080499267578125,
-                0.032191680908203124,
-                0.0311662712097168,
-                0.030852096557617188,
-                0.03080601692199707,
-                0.030819328308105468,
-                0.03077222442626953,
-                0.030774272918701173,
-                0.03074662399291992,
-                0.03076812744140625,
-                0.030708736419677734,
-                0.03078348731994629,
-                0.03077120018005371,
-                0.03078246307373047,
-                0.031164415359497072,
-                0.030726144790649414,
-                0.030769151687622072,
-                0.030845951080322266,
-                0.03079987144470215,
-                0.03061759948730469,
-                0.03062579154968262,
-                0.030766080856323243,
-                0.030656511306762696,
-                0.030877695083618165,
-                0.030880767822265624,
-                0.030741600036621092,
-                0.03079465675354004,
-                0.030851072311401367,
-                0.030839807510375978,
-                0.030701568603515625,
-                0.03078246307373047,
-                0.03073244857788086,
-                0.03083247947692871,
-                0.03093926429748535,
-                0.030780288696289064,
-                0.030809087753295897,
-                0.030760959625244142,
-                0.03082854461669922,
-                0.030793727874755858,
-                0.030810111999511718,
-                0.030852096557617188,
-                0.03077017593383789,
-                0.03077222442626953,
-                0.030911487579345705,
-                0.030766080856323243,
-                0.03080499267578125,
-                0.030711807250976563,
-                0.030811168670654296,
-                0.030770143508911132,
-                0.030753952026367187,
-                0.030772064208984377,
-                0.030732288360595703,
-                0.030712831497192384,
-                0.03079270362854004,
-                0.030878719329833985,
-                0.030817279815673827,
-                0.0307957763671875,
-                0.030712831497192384,
-                0.030740480422973632,
-                0.030895103454589845,
-                0.030740480422973632,
-                0.030841856002807616,
-                0.030765056610107422,
-                0.030818304061889647,
-                0.03121664047241211,
-                0.030819328308105468,
-                0.030715904235839843,
-                0.030817279815673827,
-                0.03079167938232422,
-                0.03074355125427246,
-                0.030868480682373047,
-                0.03077939224243164,
-                0.03079475212097168,
-                0.03080089569091797,
-                0.03084492874145508,
-                0.030757888793945313,
-                0.03078963279724121,
-                0.03078860855102539,
-                0.031169536590576172,
-                0.030860288619995117,
-                0.030740480422973632,
-                0.030822399139404297,
-                0.03075584030151367,
-                0.030855167388916017,
-                0.030705663681030275,
-                0.030954496383666992,
-                0.03075174331665039,
-                0.0307640323638916,
-                0.030744575500488282,
-                0.030860288619995117,
-                0.03080703926086426,
-                0.03079884719848633,
-                0.030749696731567383,
-                0.03078348731994629,
-                0.030732288360595703,
-                0.030903295516967775,
-                0.030689279556274415,
-                0.03079884719848633,
-                0.03075993537902832,
-                0.030895103454589845,
-                0.030865407943725585,
-                0.030849023818969725,
-                0.0307589111328125,
-                0.030850048065185546,
-                0.03077120018005371,
-                0.03079167938232422,
-                0.030729215621948244,
-                0.030857215881347655,
-                0.03082444763183594,
-                0.030873600006103515,
-                0.030769151687622072,
-                0.030765056610107422,
-                0.030684160232543944,
-                0.030870527267456056,
-                0.03076300811767578,
-                0.03081625556945801,
-                0.03077939224243164,
-                0.030769151687622072,
-                0.030827520370483398,
-                0.030879743576049806,
-                0.030699520111083983,
-                0.030873600006103515,
-                0.030873600006103515,
-                0.030930944442749023,
-                0.03074662399291992,
-                0.03077529525756836,
-                0.030870527267456056,
-                0.0307589111328125,
                 0.03132825660705566,
-                0.03078758430480957,
-                0.0307640323638916,
-                0.03077836799621582,
-                0.031096832275390625,
-                0.03077939224243164,
-                0.03077836799621582,
-                0.03080499267578125,
-                0.030863359451293947,
-                0.030882816314697265,
-                0.030911487579345705,
-                0.030827520370483398,
-                0.030704639434814454,
-                0.03079680061340332,
-                0.03082444763183594,
                 0.03127910423278808,
-                0.03076812744140625,
-                0.030865407943725585,
-                0.030745599746704103,
-                0.03082956886291504,
-                0.03079270362854004,
-                0.030860288619995117,
-                0.03073843193054199,
-                0.03079475212097168,
-                0.030833663940429686,
-                0.030826496124267577,
-                0.030744575500488282,
-                0.030737407684326173,
-                0.030678016662597656,
-                0.030865407943725585,
-                0.03075481605529785,
-                0.03075993537902832,
-                0.03094118309020996,
-                0.030649343490600587,
-                0.03078348731994629,
-                0.030849023818969725,
-                0.03095756721496582,
-                0.030733312606811523,
-                0.03078144073486328,
-                0.030864383697509764,
-                0.030793727874755858,
-                0.03075071907043457,
-                0.030826496124267577,
-                0.03073843193054199,
-                0.030728191375732423,
-                0.030857215881347655,
-                0.03082956886291504,
-                0.030809087753295897,
-                0.03073023986816406,
-                0.030943231582641603,
-                0.030929920196533203,
-                0.030892032623291016,
-                0.030856191635131838,
-                0.030845951080322266,
-                0.030867456436157226,
-                0.030856191635131838,
-                0.03080703926086426,
-                0.0307906551361084,
-                0.03079270362854004,
-                0.030810111999511718,
-                0.03081216049194336,
-                0.030879743576049806,
-                0.030850048065185546,
-                0.030882816314697265,
-                0.03085312080383301,
-                0.031268863677978515,
-                0.030932992935180665,
-                0.030842880249023437,
-                0.03079475212097168,
-                0.030857215881347655,
-                0.030814207077026368,
-                0.030883840560913086,
-                0.03078144073486328,
-                0.03077017593383789,
-                0.030869504928588868,
-                0.030830591201782227,
-                0.03077017593383789,
-                0.030895103454589845,
-                0.03080396842956543,
-                0.03079782485961914,
-                0.030867456436157226,
-                0.031406080245971676,
-                0.030826496124267577,
-                0.030826496124267577,
-                0.030801919937133788,
-                0.030830591201782227,
-                0.03093708801269531,
-                0.03073945617675781,
-                0.03082956886291504,
-                0.030864383697509764,
-                0.030818304061889647,
-                0.030830591201782227,
-                0.030710784912109376,
-                0.030757888793945313,
-                0.030861312866210938,
-                0.030801919937133788,
-                0.030862335205078126,
-                0.030756864547729492,
-                0.03076300811767578,
-                0.03079782485961914,
-                0.030866432189941406,
-                0.031041536331176758,
-                0.03082342338562012,
-                0.030741504669189453,
-                0.03080806350708008,
-                0.030833663940429686,
-                0.030908416748046875,
-                0.03074764823913574,
-                0.030870527267456056,
-                0.03078758430480957,
-                0.03080294418334961,
-                0.030748672485351562,
-                0.030680063247680665,
-                0.030690303802490236,
-                0.03077734375,
-                0.030736383438110353,
-                0.03082854461669922,
-                0.030716928482055664,
-                0.03084492874145508,
-                0.030932992935180665,
-                0.030822399139404297,
-                0.030934015274047853,
-                0.030841856002807616,
-                0.030839807510375978,
-                0.030843904495239258,
-                0.03080703926086426,
-                0.030983167648315428,
-                0.030835712432861328,
-                0.030757888793945313,
-                0.030887935638427736,
-                0.031033344268798828,
-                0.03079270362854004,
-                0.03082342338562012,
-                0.03081318473815918,
-                0.03084492874145508,
-                0.03078451156616211,
-                0.030814207077026368,
-                0.03081625556945801,
-                0.030729215621948244,
-                0.030866432189941406,
-                0.030885887145996094,
-                0.030685184478759765,
-                0.03096575927734375,
-                0.03083263969421387,
-                0.030883840560913086,
-                0.030912511825561522,
-                0.0310118408203125,
-                0.030694400787353516,
-                0.03077939224243164,
-                0.030895103454589845,
-                0.03079987144470215,
-                0.030879743576049806,
-                0.03073023986816406,
-                0.03076812744140625,
-                0.030737407684326173,
-                0.030946304321289062,
-                0.03079270362854004,
-                0.03079167938232422,
-                0.03072928047180176,
-                0.030773183822631837,
-                0.030817279815673827,
-                0.03077631950378418,
-                0.03072204780578613,
-                0.03082342338562012,
-                0.030838783264160157,
-                0.03098111915588379,
-                0.03082035255432129,
-                0.03080703926086426,
-                0.030739551544189454,
-                0.03076291275024414,
-                0.030855167388916017,
-                0.030793727874755858,
-                0.030733312606811523,
-                0.03077017593383789,
-                0.03081523132324219,
-                0.03080703926086426,
-                0.03077836799621582,
-                0.030658559799194338,
-                0.0324640007019043,
-                0.031488895416259766,
-                0.03078041648864746,
-                0.03075584030151367,
-                0.030697471618652345,
-                0.030693376541137695,
-                0.03078246307373047,
-                0.03075584030151367,
-                0.03080601692199707,
-                0.030621696472167968,
-                0.030688255310058594,
-                0.030740480422973632,
-                0.030710784912109376,
-                0.03080499267578125,
-                0.03076812744140625,
-                0.030735359191894532,
-                0.030817279815673827,
-                0.03128422355651855,
-                0.031218687057495118,
-                0.030834688186645507,
-                0.03079884719848633,
-                0.030842880249023437,
-                0.03077631950378418
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 32.29686007047721
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1088.110592,
             "max_vram": 6796.869632,
             "max_reserved": 6312.42752,
             "max_allocated": 6191.16288
         },
         "latency": {
             "unit": "s",
+            "mean": 0.05544503396387261,
+            "stdev": 0.0004891661242245622,
             "values": [
+                0.060918048858642576,
+                0.057869312286376956,
+                0.057816062927246094,
+                0.056248321533203124,
+                0.05567488098144531,
+                0.055400447845458986,
+                0.05543945693969726,
+                0.055229438781738284,
+                0.05531852722167969,
+                0.0552355842590332,
+                0.055294975280761716,
+                0.055228416442871096,
+                0.05527654266357422,
+                0.05522022247314453,
                 0.055346176147460936,
+                0.055196800231933595,
+                0.055360511779785154,
+                0.05526220703125,
+                0.05544038391113281,
+                0.05528678512573242,
+                0.05537996673583984,
+                0.05530624008178711,
                 0.05542195129394531,
+                0.05531238555908203,
+                0.055373825073242185,
+                0.055362560272216796,
+                0.05541273498535156,
+                0.0553072624206543,
+                0.05542092895507812,
+                0.05531955337524414,
+                0.055392574310302735,
+                0.05545471954345703,
                 0.05541888046264649,
+                0.055330814361572264,
+                0.055376895904541014,
+                0.05536870574951172,
+                0.055373825073242185,
+                0.05538611221313477,
+                0.05538111877441406,
+                0.05536767959594727,
+                0.05557555389404297,
+                0.05539737701416016,
+                0.05548646545410156,
+                0.055411712646484375,
+                0.055382015228271485,
                 0.05538816070556641,
+                0.055377185821533205,
+                0.055387134552001956,
+                0.055365631103515625,
+                0.05539123153686523,
+                0.05536368179321289,
+                0.0553963508605957,
+                0.05538304138183594,
+                0.055393280029296874,
+                0.05535948944091797,
+                0.05537996673583984,
+                0.05538611221313477,
+                0.05538611221313477,
+                0.055378944396972656,
+                0.0553902702331543,
+                0.0553779182434082,
+                0.055362560272216796,
                 0.05537484741210937,
+                0.0553809928894043,
+                0.05535948944091797,
                 0.055375873565673826,
+                0.05537279891967774,
+                0.055384063720703126,
+                0.055387134552001956,
+                0.055379039764404295,
+                0.05538816070556641,
+                0.05541785430908203,
+                0.055370750427246096,
+                0.05538304138183594,
+                0.05535334396362305,
                 0.05536870574951172,
+                0.05537484741210937,
                 0.055387134552001956,
                 0.05536972808837891,
+                0.055375873565673826,
                 0.05535539245605469,
+                0.05540454483032226,
+                0.05536870574951172,
+                0.05538844680786133,
+                0.0553779182434082,
+                0.055632896423339843,
+                0.05552537536621094,
+                0.05537484741210937,
+                0.05536767959594727,
+                0.05541580963134766,
+                0.0553779182434082,
+                0.05538816070556641,
+                0.055375873565673826,
                 0.055393280029296874,
+                0.055387134552001956,
+                0.05540249633789063,
+                0.0553809928894043,
+                0.05540454483032226,
+                0.05537996673583984,
+                0.055398399353027344,
                 0.05536665725708008,
+                0.05538611221313477,
+                0.05536972808837891,
                 0.05539737701416016,
+                0.05539049530029297,
+                0.0553963508605957,
                 0.05537177658081055,
+                0.05536675262451172,
                 0.055373825073242185,
+                0.05538515090942383,
+                0.05539123153686523,
+                0.055375873565673826,
+                0.055398399353027344,
+                0.05538611221313477,
+                0.0553809928894043,
+                0.055433216094970705,
+                0.055378944396972656,
+                0.055411712646484375,
+                0.0553779182434082,
                 0.05539737701416016,
+                0.05537996673583984,
+                0.055398399353027344,
+                0.055357440948486325,
+                0.055400447845458986,
+                0.05537996673583984,
+                0.05538614273071289,
+                0.05537996673583984,
+                0.055386432647705076,
+                0.055395328521728515,
                 0.055384063720703126,
+                0.05537590408325195,
+                0.055390209197998044,
+                0.05539123153686523,
+                0.055401470184326174,
+                0.05539430236816406,
                 0.05541273498535156,
                 0.055416831970214846,
+                0.05538614273071289,
+                0.055393280029296874,
+                0.05538531112670898,
                 0.0553809928894043,
+                0.05537996673583984,
+                0.05537279891967774,
+                0.05541785430908203,
+                0.05536460876464844,
+                0.055376895904541014,
+                0.05537177658081055,
+                0.05536153411865234,
+                0.05536870574951172,
+                0.05537484741210937,
+                0.05537996673583984,
+                0.055378944396972656,
+                0.05537484741210937,
+                0.055382015228271485,
+                0.05537807846069336,
+                0.055378944396972656,
+                0.05538508987426758,
+                0.055373825073242185,
+                0.05537305450439453,
+                0.055380001068115234,
+                0.0553779182434082,
+                0.055360511779785154,
                 0.055363582611083983,
+                0.05538508987426758,
+                0.055387134552001956,
+                0.055401470184326174,
+                0.05537484741210937,
+                0.05539430236816406,
+                0.05537996673583984,
+                0.055382015228271485,
+                0.05562060928344727,
+                0.0553963508605957,
+                0.055387134552001956,
+                0.05539452743530274,
+                0.055382015228271485,
+                0.05540787124633789,
+                0.055393280029296874,
+                0.055395328521728515,
+                0.05537279891967774,
+                0.055389408111572266,
+                0.055362560272216796
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 4617.185376183678
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1088.110592,
             "max_vram": 7212.105728,
             "max_reserved": 6727.663616,
             "max_allocated": 6570.75456
         },
         "latency": {
             "unit": "s",
+            "mean": 8.01907046890259,
+            "stdev": 0.0005186519622775165,
             "values": [
+                8.018551816940313,
+                8.019589120864868
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 31.799196800785406
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.031447335172167065,
+            "stdev": 0.00021283639578114806,
             "values": [
+                0.03141222381591797,
+                0.03129343986511231,
+                0.03156172752380371,
+                0.031410175323486327,
+                0.03126272010803223,
+                0.031343711853027346,
+                0.0313723201751709,
+                0.031293472290039065,
+                0.031290367126464845,
+                0.0317479362487793,
+                0.031246335983276367,
+                0.031352832794189454,
+                0.03143065643310547,
+                0.03143286323547363,
+                0.031374176025390624,
+                0.03134464073181152,
+                0.031320064544677735,
+                0.03136000061035156,
+                0.03122585678100586,
+                0.03136204719543457,
+                0.03128220748901367,
+                0.03132719993591308,
+                0.031236255645751953,
+                0.031357791900634764,
+                0.031323135375976564,
+                0.03128537559509277,
+                0.031196128845214843,
+                0.03133347129821777,
+                0.031164352416992187,
+                0.03135987281799316,
+                0.0313753604888916,
+                0.031514623641967776,
+                0.03130598449707031,
+                0.03147967910766602,
+                0.03132403182983398,
+                0.03139276885986328,
+                0.0313702392578125,
+                0.031497215270996096,
+                0.03137433624267578,
+                0.03134464073181152,
+                0.03133235168457031,
+                0.03134771156311035,
+                0.03131391906738281,
+                0.03137945556640625,
+                0.03135408020019531,
+                0.031364896774291995,
+                0.03141529655456543,
+                0.03139507293701172,
+                0.03128518486022949,
+                0.03129120063781738,
+                0.031329343795776364,
+                0.03160979270935058,
+                0.03142348861694336,
+                0.03140812873840332,
+                0.03139788818359375,
+                0.031415327072143555,
+                0.031344608306884764,
+                0.03140105628967285,
+                0.031836063385009765,
+                0.03136307144165039,
+                0.03156070327758789,
+                0.03141427230834961,
+                0.03138047981262207,
+                0.03141222381591797,
+                0.03134873580932617,
+                0.03146751976013184,
+                0.03137228775024414,
+                0.03138787269592285,
+                0.03119491195678711,
+                0.03135487937927246,
                 0.03132825660705566,
+                0.031322208404541016,
+                0.03146435165405274,
+                0.03138460731506348,
+                0.03154863929748535,
+                0.03148262405395508,
                 0.03127910423278808,
+                0.03138457679748535,
+                0.031454208374023435,
+                0.03138047981262207,
+                0.031308799743652346,
+                0.03130982398986817,
+                0.031254528045654296,
+                0.03141750335693359,
+                0.031375200271606445,
+                0.0313702392578125,
+                0.03151587104797363,
+                0.03131062316894531,
+                0.031270912170410156,
+                0.031356927871704104,
+                0.03136000061035156,
+                0.03139299201965332,
+                0.03135158348083496,
+                0.031328384399414065,
+                0.031320959091186526,
+                0.031388671875,
+                0.03142963218688965,
+                0.03149619293212891,
+                0.03138150405883789,
+                0.03137747192382812,
+                0.03128313636779785,
+                0.03130483245849609,
+                0.031438720703125,
+                0.03143577575683594,
+                0.03144908714294434,
+                0.03149619293212891,
+                0.03138457679748535,
+                0.03142473602294922,
+                0.031447839736938474,
+                0.03150540733337402,
+                0.03133337593078613,
+                0.031508480072021484,
+                0.03143987274169922,
+                0.0315361270904541,
+                0.03138150405883789,
+                0.03141529655456543,
+                0.0314019832611084,
+                0.031325183868408206,
+                0.031336448669433595,
+                0.031441055297851565,
+                0.031362911224365235,
+                0.03142860794067383,
+                0.031460351943969726,
+                0.03156499290466309,
+                0.03142636871337891,
+                0.031455232620239255,
+                0.031426559448242186,
+                0.03156377601623535,
+                0.03132825660705566,
+                0.03149516868591309,
+                0.03146342468261719,
+                0.031497215270996096,
+                0.03143270492553711,
+                0.0314337272644043,
+                0.031409151077270506,
+                0.031492095947265625,
+                0.0314204158782959,
+                0.03155148887634277,
+                0.031715328216552735,
+                0.03145011138916016,
+                0.03157113647460937,
+                0.03146425628662109,
+                0.03140505599975586,
+                0.03141632080078125,
+                0.03129651260375976,
+                0.03133132743835449,
+                0.03137945556640625,
+                0.03136614418029785,
+                0.03134771156311035,
+                0.03139507293701172,
+                0.031551231384277345,
+                0.03141836738586426,
+                0.031410175323486327,
+                0.03141632080078125,
+                0.031445119857788084,
+                0.03143667221069336,
+                0.03148595237731933,
+                0.031437824249267575,
+                0.031460351943969726,
+                0.03138559913635254,
+                0.031422719955444336,
+                0.03139865684509277,
+                0.03149539184570312,
+                0.031380256652832034,
+                0.03146751976013184,
+                0.03142758369445801,
+                0.03146137619018555,
+                0.0314780158996582,
+                0.034412288665771486,
+                0.03351561737060547,
+                0.033315967559814454,
+                0.031565664291381836,
+                0.031426496505737304,
+                0.0314521598815918,
+                0.031480831146240236,
+                0.031545343399047854,
+                0.031497215270996096,
+                0.031494144439697266,
+                0.031454208374023435,
+                0.03146751976013184,
+                0.031476736068725586,
+                0.03164998435974121,
+                0.031419456481933596,
+                0.03148467254638672,
+                0.031474687576293944,
+                0.031494144439697266,
+                0.03179212760925293,
+                0.0315228157043457,
+                0.031542272567749025,
+                0.03138995170593262,
+                0.031463167190551755,
+                0.03147980880737305,
+                0.031442943572998046,
+                0.03138662338256836,
+                0.03138355255126953,
+                0.031510528564453126,
+                0.03141529655456543,
+                0.03142451286315918,
+                0.0314102725982666,
+                0.03148380851745605,
+                0.031437824249267575,
+                0.03141222381591797,
+                0.03176038360595703,
+                0.031455232620239255,
+                0.031320064544677735,
+                0.03141324806213379,
+                0.031422464370727536,
+                0.031474687576293944,
+                0.031373311996459964,
+                0.03142969512939453,
+                0.031431615829467775,
+                0.031459583282470706,
+                0.03147443199157715,
+                0.03151769638061523,
+                0.031471616744995115,
+                0.031475967407226565,
+                0.031406848907470704,
+                0.03142860794067383,
+                0.031470592498779294,
+                0.031509504318237305,
+                0.03149619293212891,
+                0.03156480026245117,
+                0.031409151077270506,
+                0.03140096092224121,
+                0.03138764762878418,
+                0.0314071044921875,
+                0.031475711822509765,
+                0.03143987274169922,
+                0.03134771156311035,
+                0.03145113563537598,
+                0.03138355255126953,
+                0.0313702392578125,
+                0.03144908714294434,
+                0.03140403175354004,
+                0.03136944007873535,
+                0.031431455612182614,
+                0.031388671875,
+                0.031456415176391604,
+                0.03134959983825684,
+                0.031306751251220705,
+                0.031323360443115233,
+                0.03143862342834473,
+                0.03141324806213379,
+                0.03148287963867188,
+                0.03137638473510742,
+                0.03140006446838379,
+                0.031398784637451174,
+                0.03156582450866699,
+                0.031422464370727536,
+                0.03150150489807129,
+                0.03158713531494141,
+                0.031591424942016604,
+                0.03146342468261719,
+                0.03138559913635254,
+                0.031340543746948245,
+                0.03135897636413574,
+                0.0313702392578125,
+                0.031322111129760744,
+                0.03134873580932617,
+                0.031324159622192385,
+                0.031318016052246093,
+                0.03139993667602539,
+                0.03127398490905762,
+                0.03138559913635254,
+                0.03194777679443359,
+                0.031246559143066406,
+                0.03140918350219726,
+                0.03128400039672852,
+                0.031293472290039065,
+                0.03137017631530762,
+                0.0313118724822998,
+                0.031380640029907224,
+                0.0313024959564209,
+                0.03125964736938477,
+                0.03129766464233399,
+                0.031350656509399416,
+                0.03129558372497559,
+                0.03145411109924316,
+                0.03136819267272949,
+                0.03120947265625,
+                0.031339519500732424,
+                0.03129651260375976,
+                0.03137843132019043,
+                0.031351808547973634,
+                0.03127193641662598,
+                0.03121664047241211,
+                0.031263744354248044,
+                0.03123200035095215,
+                0.031319040298461914,
+                0.031281152725219724,
+                0.03136819267272949,
+                0.03127305603027344,
+                0.03132534408569336,
+                0.031287296295166016,
+                0.03139967918395996,
+                0.03128118324279785,
+                0.031385568618774416,
+                0.031322111129760744,
+                0.031410175323486327,
+                0.03121894454956055,
+                0.03132697677612305,
+                0.03140403175354004,
+                0.03134780883789062,
+                0.031201408386230468,
+                0.03138640022277832,
+                0.03123200035095215,
+                0.03137228775024414,
+                0.031226879119873048,
+                0.03139174461364746,
+                0.03127824020385742,
+                0.0313558406829834,
+                0.03128003120422363,
+                0.03135897636413574,
+                0.031680511474609374,
+                0.03137241554260254,
+                0.03142438316345215,
+                0.03143680000305176,
+                0.03130572891235352,
+                0.031341567993164066,
+                0.031453184127807614,
+                0.03143270492553711,
+                0.031440895080566404,
+                0.03142166328430176,
+                0.031348512649536135,
+                0.03154435157775879,
+                0.031350751876831055,
+                0.031453184127807614,
+                0.03138768005371094,
+                0.03173884773254394,
+                0.031498239517211916,
+                0.031307775497436525,
+                0.031529983520507815,
+                0.0314521598815918,
+                0.0314654712677002,
+                0.0314337272644043,
+                0.031369216918945314,
+                0.03147878456115723,
+                0.031457279205322264,
+                0.031466623306274415,
+                0.031501184463500975,
+                0.03143065643310547,
+                0.03152179145812988,
+                0.03139276885986328,
+                0.03147878456115723,
+                0.03139379119873047,
+                0.03146854400634766,
+                0.03136832046508789,
+                0.03137420845031738,
+                0.03140931129455567,
+                0.031414112091064456,
+                0.03150643157958984,
+                0.03142348861694336,
+                0.03143475151062012,
+                0.03148902320861816,
+                0.03133132743835449,
+                0.03139276885986328,
+                0.031438848495483396,
+                0.03143270492553711,
+                0.03139084815979004,
+                0.03144607925415039,
+                0.031433696746826174,
+                0.03145916748046875,
+                0.03136428833007812,
+                0.031419200897216795,
+                0.03126681518554687,
+                0.031480831146240236,
+                0.03142758369445801,
+                0.03139072036743164,
+                0.03138252830505371,
+                0.03138047981262207,
+                0.031456256866455076,
+                0.03148192024230957,
+                0.03150841522216797,
+                0.031322111129760744,
+                0.03135487937927246,
+                0.03169075202941894,
+                0.03137238311767578,
+                0.03190569686889649,
+                0.03136307144165039,
+                0.03142758369445801,
+                0.031356927871704104,
+                0.03158732795715332,
+                0.03129241561889649,
+                0.03140505599975586,
+                0.031286272048950195,
+                0.03136211204528808,
+                0.03145004844665528,
+                0.031389696121215824,
+                0.03157196807861328,
+                0.03157401657104492,
+                0.031529983520507815,
+                0.031488000869750975,
+                0.03202252960205078,
+                0.03160576057434082,
+                0.031676416397094724,
+                0.031460351943969726,
+                0.031509504318237305,
+                0.031497215270996096,
+                0.03148287963867188,
+                0.03155046463012695,
+                0.031528959274291994,
+                0.03159347152709961,
+                0.03157321548461914,
+                0.03157481575012207,
+                0.03152179145812988,
+                0.03156889533996582,
+                0.03167231941223145,
+                0.03152179145812988,
+                0.031611904144287106,
+                0.031480831146240236,
+                0.031665151596069335,
+                0.031514623641967776,
+                0.03143577575683594,
+                0.031491071701049804,
+                0.03159654426574707,
+                0.031453184127807614,
+                0.031425535202026365,
+                0.031459327697753905,
+                0.03168972778320313,
+                0.0315863037109375,
+                0.03158118438720703,
+                0.03157708740234375,
+                0.03161497688293457,
+                0.0314450569152832,
+                0.03155881690979004,
+                0.031433504104614256,
+                0.031643648147583005,
+                0.031496416091918944,
+                0.03159939193725586,
+                0.031409151077270506,
+                0.03165184020996094,
+                0.0314594554901123,
+                0.031472511291503906,
+                0.03156812858581543,
+                0.03156147193908691,
+                0.03142963218688965,
+                0.03154351997375488,
+                0.03153078460693359,
+                0.03174604797363281,
+                0.031550687789916994,
+                0.031682559967041016,
+                0.031872800827026364,
+                0.03165798377990723,
+                0.031485055923461916,
+                0.0315994873046875,
+                0.031663103103637694,
+                0.03156480026245117,
+                0.031442943572998046,
+                0.031604736328125,
+                0.0314521598815918,
+                0.03142860794067383,
+                0.03143475151062012,
+                0.03147392082214356,
+                0.03153689575195313,
+                0.03143577575683594,
+                0.031488000869750975,
+                0.03179116821289062,
+                0.031449024200439456,
+                0.03143987274169922,
+                0.03130684852600098,
+                0.031404960632324216,
+                0.03144806480407715,
+                0.031355903625488284,
+                0.03139481544494629,
+                0.03134771156311035,
+                0.03141529655456543,
+                0.03153919982910156,
+                0.031425535202026365,
+                0.031426559448242186,
+                0.03153408050537109,
+                0.03138787269592285,
+                0.03143657684326172,
+                0.03145113563537598,
+                0.03144806480407715,
+                0.03143168067932129,
+                0.03136716842651367,
+                0.031369216918945314,
+                0.0314521598815918,
+                0.03146137619018555,
+                0.03139583969116211,
+                0.0313702392578125,
+                0.03143270492553711,
+                0.03150643157958984,
+                0.03139174461364746,
+                0.03145011138916016,
+                0.03144499206542969,
+                0.031355903625488284,
+                0.031572223663330075,
+                0.03150422477722168,
+                0.03152272033691406,
+                0.03139583969116211,
+                0.03158323287963867,
+                0.03146342468261719,
+                0.031531007766723636,
+                0.031527936935424807,
+                0.03149619293212891,
+                0.03135408020019531,
+                0.03147855949401855,
+                0.03158220863342285,
+                0.03149004745483398,
+                0.031562847137451173,
+                0.03153603172302246,
+                0.031409151077270506,
+                0.0314204158782959,
+                0.031440895080566404,
+                0.03183001518249512,
+                0.031475711822509765,
+                0.03141427230834961,
+                0.03144704055786133,
+                0.03156889533996582
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 31.799196800785364
         },
         "energy": null,
         "efficiency": null