kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 16

Commit

89862d5

•

1 Parent(s): 7db2403

Upload /phi-2/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

phi-2/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +757 -795

phi-2/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,292 +2,254 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1009.913856,
-            "max_vram": 2585.788416,
             "max_reserved": 2101.346304,
             "max_allocated": 2014.912
         },
         "latency": {
             "unit": "s",
-            "mean": 0.03703229379126066,
-            "stdev": 0.0005140512718572568,
             "values": [
-                0.041216480255126954,
-                0.03949055862426758,
-                0.038371326446533204,
-                0.03802521514892578,
-                0.037797889709472655,
-                0.03784089660644531,
-                0.04021657562255859,
-                0.04187136077880859,
-                0.038798336029052735,
-                0.037601280212402347,
-                0.03723583984375,
-                0.03724627304077149,
-                0.03697663879394531,
-                0.03689574432373047,
-                0.03692031860351563,
-                0.036977664947509765,
-                0.03687014389038086,
-                0.03688755035400391,
-                0.03679334259033203,
-                0.03679155349731445,
-                0.03726847839355469,
-                0.03683225631713867,
-                0.036825088500976565,
-                0.036843521118164066,
-                0.03674009704589844,
-                0.03679846572875976,
-                0.037228801727294925,
-                0.03674828720092774,
-                0.03709337615966797,
-                0.03716505432128906,
-                0.036773887634277344,
-                0.03713433456420898,
-                0.03712307357788086,
-                0.037580799102783204,
-                0.03706070327758789,
-                0.0371701774597168,
-                0.037101566314697264,
-                0.03711385726928711,
-                0.037160961151123044,
-                0.03709747314453125,
-                0.03717324829101563,
-                0.03698483276367188,
-                0.0367710075378418,
-                0.03686809539794922,
-                0.036918270111083985,
-                0.03681398391723633,
-                0.03687628936767578,
-                0.03683020782470703,
-                0.036959232330322264,
-                0.03681382369995117,
-                0.036908031463623044,
-                0.03686118316650391,
-                0.036891841888427736,
-                0.03693260955810547,
-                0.03685273742675781,
-                0.03679452896118164,
-                0.03683430480957031,
-                0.03686707305908203,
-                0.036831230163574216,
-                0.036945919036865234,
-                0.036811775207519534,
-                0.036953086853027346,
-                0.03687014389038086,
-                0.036910079956054685,
-                0.037101566314697264,
-                0.03745792007446289,
-                0.03725321578979492,
-                0.03730124664306641,
-                0.036999168395996096,
-                0.03678003311157227,
-                0.03679641723632812,
-                0.03693056106567383,
-                0.03686297607421875,
-                0.036847614288330076,
-                0.03693772888183594,
-                0.03724185562133789,
-                0.03679948806762695,
-                0.037147647857666014,
-                0.03702374267578125,
-                0.036915199279785156,
-                0.03711081695556641,
-                0.03719887924194336,
-                0.03720499038696289,
-                0.03711695861816406,
-                0.03706982421875,
-                0.03690086364746094,
-                0.03696537780761719,
-                0.03718265533447265,
-                0.03718963241577149,
-                0.03692544174194336,
-                0.0368702392578125,
-                0.03684659194946289,
-                0.03687014389038086,
-                0.03685580825805664,
-                0.03693670272827149,
-                0.037010433197021485,
-                0.036942848205566405,
-                0.03688140869140625,
-                0.0369244155883789,
-                0.036857856750488284,
-                0.03677798461914063,
-                0.03681792068481445,
-                0.036782081604003904,
-                0.03681075286865235,
-                0.03684044647216797,
-                0.036738048553466796,
-                0.03684454345703125,
-                0.03688345718383789,
-                0.03696230316162109,
-                0.037085182189941404,
-                0.03685887908935547,
-                0.037093505859375,
-                0.03690291213989258,
-                0.036915199279785156,
-                0.03684259033203125,
-                0.036890625,
-                0.03690291213989258,
-                0.03696640014648438,
-                0.037144577026367184,
-                0.037308414459228514,
-                0.037348350524902346,
-                0.037379070281982424,
-                0.03705036926269531,
-                0.03698688125610351,
-                0.03698688125610351,
-                0.03722649765014648,
-                0.0372674560546875,
-                0.03702067184448242,
-                0.037010433197021485,
-                0.03747020721435547,
-                0.03693260955810547,
-                0.03698175811767578,
-                0.0370063362121582,
-                0.03688755035400391,
-                0.03696464157104492,
-                0.036926464080810545,
-                0.03693670272827149,
-                0.03685683059692383,
-                0.03699513626098633,
-                0.03699302291870117,
-                0.036918270111083985,
-                0.036985855102539066,
-                0.037375999450683595,
-                0.036898815155029296,
-                0.036927646636962894,
-                0.036913150787353514,
-                0.036982784271240236,
-                0.036920543670654296,
-                0.03700838470458984,
-                0.036988929748535154,
-                0.03687628936767578,
-                0.03690291213989258,
-                0.03681689453125,
-                0.03689686584472656,
-                0.03679846572875976,
-                0.036880382537841795,
-                0.03692134475708008,
-                0.03697356796264648,
-                0.036905086517333985,
-                0.03706572723388672,
-                0.03681894302368164,
-                0.036921600341796874,
-                0.03681280136108398,
-                0.03688652801513672,
-                0.03684659194946289,
-                0.03687833786010742,
-                0.0368353271484375,
-                0.036887870788574216,
-                0.036863998413085936,
-                0.036945919036865234,
-                0.03673190307617188,
-                0.03686912155151367,
-                0.03685990524291992,
-                0.0370247688293457,
-                0.03696537780761719,
-                0.03691622543334961,
-                0.036877311706542966,
-                0.03682304000854492,
-                0.03684966278076172,
-                0.036883487701416015,
-                0.03692134475708008,
-                0.0368353271484375,
-                0.036880382537841795,
-                0.03688447952270508,
-                0.037407745361328126,
-                0.03691622543334961,
-                0.03685171127319336,
-                0.03703091049194336,
-                0.03697459030151367,
-                0.03688550567626953,
-                0.03682304000854492,
-                0.03688550567626953,
-                0.036787200927734375,
-                0.03684454345703125,
-                0.03678412628173828,
-                0.03676803207397461,
-                0.03691417694091797,
-                0.03723775863647461,
-                0.036841472625732424,
-                0.03687116622924805,
-                0.036934654235839845,
-                0.036880382537841795,
-                0.0369541130065918,
-                0.03690188980102539,
-                0.036915199279785156,
-                0.03685990524291992,
-                0.03683327865600586,
-                0.03685887908935547,
-                0.037002239227294925,
-                0.03694182586669922,
-                0.036962558746337894,
-                0.03683225631713867,
-                0.036918270111083985,
-                0.03703398513793945,
-                0.03692246246337891,
-                0.036959232330322264,
-                0.03681587219238281,
-                0.036945919036865234,
-                0.03688243103027344,
-                0.036959232330322264,
-                0.03692544174194336,
-                0.03696230316162109,
-                0.036853759765625,
-                0.03692031860351563,
-                0.036907009124755856,
-                0.0370247688293457,
-                0.036860927581787106,
-                0.036969470977783206,
-                0.03687628936767578,
-                0.036915199279785156,
-                0.036754688262939456,
-                0.036893695831298826,
-                0.03686604690551758,
-                0.03684454345703125,
-                0.0369172477722168,
-                0.03696332931518555,
-                0.03690086364746094,
-                0.03701657485961914,
-                0.037238784790039066,
-                0.03691622543334961,
-                0.03681382369995117,
-                0.036994049072265625,
-                0.03682918548583984,
-                0.03694079971313476,
-                0.03696028900146484,
-                0.03695833587646485,
-                0.036923393249511716,
-                0.036988929748535154,
-                0.036972545623779295,
-                0.03706572723388672,
-                0.036931583404541016,
-                0.03733708953857422,
-                0.036874240875244144,
-                0.0368721923828125,
-                0.036916385650634764,
-                0.03696844863891602,
-                0.0369183349609375,
-                0.036946048736572264,
-                0.03693363189697266,
-                0.03698284912109375,
-                0.036947967529296875,
-                0.03700735855102539,
-                0.036992000579833983,
-                0.03701145553588867,
-                0.03693875122070313,
-                0.036953086853027346,
-                0.03674041748046875,
-                0.0369736328125,
-                0.03684048080444336,
-                0.03690822219848633,
-                0.03685273742675781
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6912.88531688021
         },
         "energy": null,
         "efficiency": null
@@ -295,23 +257,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1009.913856,
             "max_vram": 2858.418176,
             "max_reserved": 2373.976064,
             "max_allocated": 2326.085632
         },
         "latency": {
             "unit": "s",
-            "mean": 7.244024317741396,
-            "stdev": 0.004989447593689356,
             "values": [
-                7.249013765335086,
-                7.239034870147707
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.20142793771102
         },
         "energy": null,
         "efficiency": null
@@ -320,524 +282,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.02840793850094664,
-            "stdev": 0.00019315407561114144,
             "values": [
-                0.029053951263427736,
-                0.028392448425292968,
-                0.028463104248046874,
-                0.028463104248046874,
-                0.02838528060913086,
-                0.0283986873626709,
-                0.028428192138671874,
-                0.028367008209228516,
-                0.028359519958496095,
-                0.028430335998535155,
-                0.02840166473388672,
-                0.028322816848754883,
-                0.02834329605102539,
-                0.02832691192626953,
-                0.028408832550048828,
-                0.028498943328857423,
-                0.028465152740478516,
-                0.028429311752319338,
-                0.028297216415405273,
-                0.02834329605102539,
-                0.02837606430053711,
-                0.028421119689941408,
-                0.028333248138427733,
-                0.02850592041015625,
-                0.028374143600463867,
-                0.02839347267150879,
-                0.02842201614379883,
-                0.02875494384765625,
-                0.028363807678222656,
-                0.02842313575744629,
-                0.02835763168334961,
-                0.028366847991943358,
-                0.02837731170654297,
-                0.028475168228149415,
-                0.028412960052490235,
-                0.028468191146850588,
-                0.02831257629394531,
-                0.02841804885864258,
-                0.02835660743713379,
-                0.028419071197509766,
-                0.028685440063476564,
-                0.028428159713745117,
-                0.02835353660583496,
-                0.028428287506103517,
-                0.028785663604736327,
-                0.028375040054321288,
-                0.02834943962097168,
-                0.028439552307128906,
-                0.028455936431884765,
-                0.02840575981140137,
-                0.028338176727294922,
-                0.02877542304992676,
-                0.028358655929565428,
-                0.028476415634155275,
-                0.028492799758911135,
-                0.028387327194213868,
-                0.028468223571777345,
-                0.02840166473388672,
-                0.028440832138061523,
-                0.028384031295776366,
-                0.028553184509277345,
-                0.028410879135131836,
-                0.03211775970458984,
-                0.028556480407714843,
-                0.028443456649780274,
-                0.028383232116699218,
-                0.02838630485534668,
-                0.02855936050415039,
-                0.028443647384643556,
-                0.028481664657592772,
-                0.02838003158569336,
-                0.02852454376220703,
-                0.028449792861938477,
-                0.02834432029724121,
-                0.02837708854675293,
-                0.028356672286987305,
-                0.02842412757873535,
-                0.02843974494934082,
-                0.028511039733886717,
-                0.028443647384643556,
-                0.028400640487670898,
-                0.028478464126586913,
-                0.02834943962097168,
-                0.028367071151733397,
-                0.028371776580810547,
-                0.02843440055847168,
-                0.02846313667297363,
-                0.028473312377929688,
-                0.028483583450317384,
-                0.0283371524810791,
-                0.028400640487670898,
-                0.02849488067626953,
-                0.0284671688079834,
-                0.02836582374572754,
-                0.028358655929565428,
-                0.02835660743713379,
-                0.02836275291442871,
-                0.028474367141723633,
-                0.028836000442504884,
-                0.028378976821899413,
-                0.0283504638671875,
-                0.028473344802856446,
-                0.028402687072753906,
-                0.028473344802856446,
-                0.028468223571777345,
-                0.028496896743774414,
-                0.028423168182373046,
-                0.02843145561218262,
-                0.028446624755859375,
-                0.028390399932861327,
-                0.028431360244750976,
-                0.028417024612426758,
-                0.028308479309082032,
-                0.028528640747070313,
-                0.028342271804809572,
-                0.028818559646606446,
-                0.02842838478088379,
-                0.028426015853881836,
-                0.028427263259887696,
-                0.028458112716674804,
-                0.028477312088012696,
-                0.028442623138427735,
-                0.02839449691772461,
-                0.028499967575073244,
-                0.028404735565185548,
-                0.028482559204101563,
-                0.028407808303833007,
-                0.028410879135131836,
-                0.028484607696533205,
-                0.028481536865234375,
-                0.028461055755615236,
-                0.028444671630859376,
-                0.02836582374572754,
-                0.028404735565185548,
-                0.02852249526977539,
-                0.028507135391235353,
-                0.028419071197509766,
-                0.028472320556640625,
-                0.028443647384643556,
-                0.028404895782470702,
-                0.02864726448059082,
-                0.028556287765502928,
-                0.028437503814697264,
-                0.028462080001831053,
-                0.028370943069458008,
-                0.02834342384338379,
-                0.028390304565429687,
-                0.02841804885864258,
-                0.0285644474029541,
-                0.02837196731567383,
-                0.02834534454345703,
-                0.028468223571777345,
-                0.028430400848388673,
-                0.02846201515197754,
-                0.028539903640747072,
-                0.028413951873779295,
-                0.02835353660583496,
-                0.028482559204101563,
-                0.02832076835632324,
-                0.02838444709777832,
-                0.02827552032470703,
-                0.02834124755859375,
-                0.0283637752532959,
-                0.028454912185668944,
-                0.028437503814697264,
-                0.028407808303833007,
-                0.02838118362426758,
-                0.02838937568664551,
-                0.028309503555297853,
-                0.028862464904785157,
-                0.02851139259338379,
-                0.028392288208007814,
-                0.028421119689941408,
-                0.028422143936157225,
-                0.028391424179077147,
-                0.02835148811340332,
-                0.02835660743713379,
-                0.028377344131469726,
-                0.02839116859436035,
-                0.028471296310424804,
-                0.02836070442199707,
-                0.02837606430053711,
-                0.028399616241455077,
-                0.02834534454345703,
-                0.028303455352783204,
-                0.028364704132080077,
-                0.028820480346679687,
-                0.02838118362426758,
-                0.028358848571777343,
-                0.028378944396972656,
-                0.02829516792297363,
-                0.028408832550048828,
-                0.028251327514648438,
-                0.028241727828979494,
-                0.02841200065612793,
-                0.028364704132080077,
-                0.02830745506286621,
-                0.028416000366210937,
-                0.02832486343383789,
-                0.028497919082641602,
-                0.028309503555297853,
-                0.028399616241455077,
-                0.02838937568664551,
-                0.028403711318969727,
-                0.028446720123291015,
-                0.028404735565185548,
-                0.02830335998535156,
-                0.02840166473388672,
-                0.028344352722167968,
-                0.02835964775085449,
-                0.02832896041870117,
-                0.02836992073059082,
-                0.02834739112854004,
-                0.028321792602539062,
-                0.028239871978759764,
-                0.028261375427246094,
-                0.028269567489624024,
-                0.02829516792297363,
-                0.028292095184326172,
-                0.028280832290649413,
-                0.028257280349731444,
-                0.028394752502441407,
-                0.028260095596313477,
-                0.028417184829711915,
-                0.028330848693847655,
-                0.0283156795501709,
-                0.028314592361450196,
-                0.02836787223815918,
-                0.028286975860595705,
-                0.02828607940673828,
-                0.028283775329589842,
-                0.02862387275695801,
-                0.028473344802856446,
-                0.02835968017578125,
-                0.02839347267150879,
-                0.028368896484375,
-                0.02835660743713379,
-                0.028329984664916992,
-                0.028262399673461915,
-                0.028370943069458008,
-                0.028691455841064452,
-                0.02836070442199707,
-                0.028375040054321288,
-                0.028420095443725587,
-                0.02837708854675293,
-                0.02835660743713379,
-                0.028233728408813476,
-                0.028281024932861328,
-                0.02833286476135254,
-                0.028402687072753906,
-                0.028263423919677736,
-                0.028331008911132813,
-                0.0283504638671875,
-                0.02829516792297363,
-                0.028177408218383788,
-                0.028719104766845704,
-                0.02847551918029785,
-                0.02838924789428711,
-                0.028374176025390625,
-                0.028345184326171877,
-                0.02850201606750488,
-                0.028445695877075194,
-                0.02839449691772461,
-                0.028371103286743166,
-                0.028382047653198243,
-                0.028526655197143556,
-                0.028366783142089843,
-                0.028421119689941408,
-                0.0283637752532959,
-                0.02838528060913086,
-                0.028375263214111327,
-                0.028448543548583984,
-                0.02839449691772461,
-                0.02856038475036621,
-                0.028449792861938477,
-                0.028479488372802734,
-                0.02834534454345703,
-                0.02831987190246582,
-                0.028410751342773436,
-                0.028457984924316407,
-                0.028470272064208983,
-                0.02842748832702637,
-                0.028431135177612303,
-                0.02835558319091797,
-                0.02835148811340332,
-                0.028292095184326172,
-                0.028395519256591797,
-                0.028476415634155275,
-                0.028383232116699218,
-                0.028387327194213868,
-                0.02842233657836914,
-                0.028399423599243166,
-                0.028383232116699218,
-                0.028408832550048828,
-                0.028411903381347657,
-                0.028411903381347657,
-                0.02836787223815918,
-                0.02838835144042969,
-                0.02838528060913086,
-                0.028413951873779295,
-                0.028474367141723633,
-                0.028511295318603514,
-                0.028382335662841797,
-                0.028393280029296874,
-                0.028373088836669922,
-                0.028394399642944337,
-                0.028284927368164063,
-                0.028430335998535155,
-                0.028423168182373046,
-                0.028652544021606444,
-                0.028681215286254884,
-                0.028465152740478516,
-                0.028387327194213868,
-                0.028425216674804688,
-                0.028387327194213868,
-                0.028441600799560547,
-                0.02858598327636719,
-                0.028498943328857423,
-                0.028461055755615236,
-                0.028472320556640625,
-                0.028481536865234375,
-                0.02856345558166504,
-                0.028428287506103517,
-                0.028395519256591797,
-                0.028425216674804688,
-                0.028453887939453124,
-                0.02879078483581543,
-                0.02836582374572754,
-                0.028353759765625,
-                0.02839241600036621,
-                0.028436288833618165,
-                0.028276735305786133,
-                0.02840575981140137,
-                0.028400640487670898,
-                0.02849398422241211,
-                0.028372831344604492,
-                0.028413951873779295,
-                0.028440576553344726,
-                0.02837606430053711,
-                0.028382207870483397,
-                0.028390399932861327,
-                0.028382240295410158,
-                0.02836172866821289,
-                0.028453855514526366,
-                0.028475391387939454,
-                0.028404735565185548,
-                0.028391424179077147,
-                0.02834329605102539,
-                0.028427263259887696,
-                0.028416000366210937,
-                0.0283536319732666,
-                0.02832579231262207,
-                0.02836992073059082,
-                0.028366975784301758,
-                0.028426111221313475,
-                0.028368896484375,
-                0.028292095184326172,
-                0.02832793617248535,
-                0.028473344802856446,
-                0.02836479949951172,
-                0.02839756774902344,
-                0.028379167556762695,
-                0.028479455947875976,
-                0.028305408477783203,
-                0.02836479949951172,
-                0.02831667137145996,
-                0.028325887680053712,
-                0.028296192169189452,
-                0.02837196731567383,
-                0.028294143676757814,
-                0.02833612823486328,
-                0.028234752655029297,
-                0.02836992073059082,
-                0.028209152221679686,
-                0.0283371524810791,
-                0.02822675132751465,
-                0.028374847412109376,
-                0.02832076835632324,
-                0.02837606430053711,
-                0.028416000366210937,
-                0.028641279220581056,
-                0.028425216674804688,
-                0.028416000366210937,
-                0.028472320556640625,
-                0.028457984924316407,
-                0.028430335998535155,
-                0.028391424179077147,
-                0.028375040054321288,
-                0.028279903411865235,
-                0.028236831665039062,
-                0.02830732727050781,
-                0.02831667137145996,
-                0.028330175399780274,
-                0.0283123836517334,
-                0.028429311752319338,
-                0.028556287765502928,
-                0.02850624084472656,
-                0.028791807174682618,
-                0.02832579231262207,
-                0.02838217544555664,
-                0.028464128494262695,
-                0.028398591995239256,
-                0.028396543502807618,
-                0.02834841537475586,
-                0.02838425636291504,
-                0.02837708854675293,
-                0.028447872161865236,
-                0.028418943405151366,
-                0.028431360244750976,
-                0.02842643165588379,
-                0.028483392715454102,
-                0.02838630485534668,
-                0.028313600540161132,
-                0.028277759552001954,
-                0.02854400062561035,
-                0.028270591735839845,
-                0.028271615982055662,
-                0.028357824325561522,
-                0.02831955146789551,
-                0.028306655883789063,
-                0.028366624832153322,
-                0.028283903121948242,
-                0.028298431396484375,
-                0.028388160705566406,
-                0.028314687728881835,
-                0.028319679260253906,
-                0.028314624786376953,
-                0.02834841537475586,
-                0.028278783798217775,
-                0.02831974411010742,
-                0.028209184646606444,
-                0.028265439987182617,
-                0.02827270317077637,
-                0.028287935256958007,
-                0.028434431076049805,
-                0.028313695907592775,
-                0.0283288631439209,
-                0.02833612823486328,
-                0.02840985679626465,
-                0.028386367797851562,
-                0.028263359069824218,
-                0.02840073585510254,
-                0.028315711975097656,
-                0.028368736267089845,
-                0.02834022331237793,
-                0.02838425636291504,
-                0.02838015937805176,
-                0.028420095443725587,
-                0.02834124755859375,
-                0.028298240661621094,
-                0.02838243293762207,
-                0.02853660774230957,
-                0.02844281578063965,
-                0.02834127998352051,
-                0.028352287292480467,
-                0.028203168869018556,
-                0.0283851203918457,
-                0.028429471969604492,
-                0.028372831344604492,
-                0.028271615982055662,
-                0.02862387275695801,
-                0.028215423583984375,
-                0.028498815536499023,
-                0.028403711318969727,
-                0.02840575981140137,
-                0.02833510398864746,
-                0.02834124755859375,
-                0.028305408477783203,
-                0.028829696655273438,
-                0.02830745506286621,
-                0.028261375427246094,
-                0.028290048599243164,
-                0.028422239303588868,
-                0.028380064010620116,
-                0.028422143936157225,
-                0.02832896041870117,
-                0.028313600540161132,
-                0.02836479949951172,
-                0.0283637752532959,
-                0.028235776901245117,
-                0.02834329605102539,
-                0.028300287246704102,
-                0.028451839447021485,
-                0.02831564712524414,
-                0.028484607696533205,
-                0.028422143936157225,
-                0.02834943962097168,
-                0.028281856536865234,
-                0.028266496658325195,
-                0.028297216415405273,
-                0.028455968856811523,
-                0.028338144302368164,
-                0.028422208786010743,
-                0.028363712310791017,
-                0.028345407485961913,
-                0.028309440612792967,
-                0.028444671630859376,
-                0.028345439910888674,
-                0.02834223937988281,
-                0.02831660842895508,
-                0.02839347267150879,
-                0.02839449691772461,
-                0.028424192428588867,
-                0.0283371524810791,
-                0.02833510398864746,
-                0.028302335739135744,
-                0.028375040054321288,
-                0.028282880783081055,
-                0.028521472930908204,
-                0.028213247299194336
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.20142793771104
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 988.24192,
+            "max_vram": 2533.359616,
             "max_reserved": 2101.346304,
             "max_allocated": 2014.912
         },
         "latency": {
             "unit": "s",
+            "mean": 0.04293074952481638,
+            "stdev": 0.0002911280619323985,
             "values": [
+                0.046001983642578126,
+                0.04444672012329102,
+                0.04340428924560547,
+                0.04333260726928711,
+                0.043170814514160154,
+                0.043199489593505856,
+                0.04324147033691406,
+                0.043292671203613284,
+                0.043289600372314455,
+                0.04327040100097656,
+                0.04345753479003906,
+                0.043431934356689454,
+                0.043412479400634765,
+                0.0433807373046875,
+                0.043448318481445314,
+                0.04290867233276367,
+                0.04275609588623047,
+                0.042813438415527344,
+                0.04270694351196289,
+                0.042684673309326175,
+                0.042825790405273435,
+                0.042810367584228515,
+                0.04271430587768555,
+                0.042797054290771484,
+                0.0427336311340332,
+                0.042676223754882815,
+                0.04290867233276367,
+                0.04286566543579102,
+                0.04300595092773438,
+                0.04339321517944336,
+                0.0427407341003418,
+                0.04276633453369141,
+                0.04274486541748047,
+                0.04289535903930664,
+                0.042780670166015625,
+                0.042761215209960936,
+                0.04278579330444336,
+                0.042703872680664064,
+                0.04273971176147461,
+                0.04270284652709961,
+                0.042793983459472655,
+                0.042782718658447266,
+                0.0428328971862793,
+                0.042739967346191406,
+                0.042651649475097655,
+                0.042883071899414066,
+                0.04271513748168945,
+                0.04275711822509766,
+                0.04274816131591797,
+                0.04274607849121094,
+                0.04276633453369141,
+                0.0427407341003418,
+                0.04264780807495117,
+                0.042708992004394535,
+                0.04280934524536133,
+                0.04272435379028321,
+                0.04288716888427734,
+                0.04279507064819336,
+                0.04293222427368164,
+                0.04278681564331055,
+                0.04268236923217773,
+                0.04270694351196289,
+                0.042603519439697264,
+                0.042799102783203126,
+                0.04267827224731445,
+                0.04278374481201172,
+                0.04268851089477539,
+                0.042828800201416016,
+                0.042851329803466794,
+                0.04289023971557617,
+                0.0430909423828125,
+                0.0431646728515625,
+                0.042687488555908204,
+                0.042842113494873046,
+                0.0427509765625,
+                0.04283903884887695,
+                0.04273356628417969,
+                0.04287692642211914,
+                0.04269363021850586,
+                0.04277862548828125,
+                0.042979328155517575,
+                0.04276224136352539,
+                0.04273356628417969,
+                0.04259145736694336,
+                0.04273356628417969,
+                0.042966014862060545,
+                0.04284415817260742,
+                0.042812416076660156,
+                0.042777599334716795,
+                0.04292633438110351,
+                0.04281753540039063,
+                0.042845184326171876,
+                0.04282572937011719,
+                0.04278579330444336,
+                0.04275609588623047,
+                0.042823680877685545,
+                0.042787841796875,
+                0.042759166717529294,
+                0.0427325439453125,
+                0.04309811019897461,
+                0.04295167922973633,
+                0.04283203125,
+                0.04267520141601563,
+                0.04275814437866211,
+                0.042777599334716795,
+                0.042867710113525394,
+                0.042823680877685545,
+                0.0427407341003418,
+                0.04293939208984375,
+                0.04287078475952148,
+                0.04265369415283203,
+                0.04275199890136719,
+                0.042831871032714845,
+                0.04298854446411133,
+                0.042780670166015625,
+                0.04280831909179687,
+                0.042859519958496094,
+                0.04290252685546875,
+                0.042906688690185546,
+                0.042891265869140625,
+                0.04288716888427734,
+                0.04295987319946289,
+                0.04274176025390625,
+                0.042866687774658206,
+                0.042885185241699215,
+                0.04290252685546875,
+                0.04281139373779297,
+                0.043207679748535156,
+                0.04314521789550781,
+                0.04284723281860352,
+                0.04288716888427734,
+                0.04294041442871094,
+                0.04284928131103516,
+                0.04283596801757812,
+                0.042889217376708984,
+                0.04301926422119141,
+                0.042799102783203126,
+                0.04291484832763672,
+                0.042968063354492186,
+                0.04289945602416992,
+                0.0428974723815918,
+                0.043251712799072264,
+                0.04310630416870117,
+                0.04312268829345703,
+                0.04298137664794922,
+                0.04278988647460937,
+                0.04318719863891601,
+                0.04324761581420898,
+                0.04329983901977539,
+                0.04341862487792969,
+                0.04290150451660156,
+                0.04308892822265625,
+                0.04297830581665039,
+                0.043046913146972655,
+                0.04331315231323242,
+                0.04327219009399414,
+                0.04303155136108398,
+                0.04297625732421875,
+                0.042812416076660156,
+                0.04285468673706055,
+                0.04275711822509766,
+                0.04278476715087891,
+                0.042861568450927735,
+                0.04309196853637695,
+                0.043087871551513675,
+                0.04287913513183594,
+                0.04264755249023437,
+                0.042775550842285154,
+                0.04304908752441406,
+                0.04313600158691406,
+                0.042943489074707034,
+                0.04279808044433594,
+                0.043053054809570314,
+                0.04303257751464844,
+                0.04288716888427734,
+                0.04318515014648437,
+                0.043251712799072264,
+                0.04299264144897461,
+                0.04303769683837891,
+                0.042867710113525394,
+                0.04275199890136719,
+                0.04275302505493164,
+                0.043192577362060544,
+                0.04299468612670899,
+                0.04301004791259765,
+                0.04286054229736328,
+                0.042812416076660156,
+                0.042705921173095705,
+                0.04291891098022461,
+                0.04283830261230469,
+                0.04288000106811524,
+                0.04298956680297852,
+                0.04281958389282227,
+                0.042783775329589845,
+                0.042893310546875,
+                0.04287385559082031,
+                0.042867710113525394,
+                0.0429639663696289,
+                0.043259902954101564,
+                0.04301311874389648,
+                0.043104255676269534,
+                0.04284940719604492,
+                0.04301619338989258,
+                0.042845184326171876,
+                0.04277350234985351,
+                0.04290374374389649,
+                0.04277350234985351,
+                0.04280144119262695,
+                0.042738689422607425,
+                0.04287180709838867,
+                0.042943489074707034,
+                0.043081729888916016,
+                0.042867710113525394,
+                0.04280627059936523,
+                0.042805248260498044,
+                0.04286675262451172,
+                0.042866687774658206,
+                0.043123710632324216,
+                0.04312678527832031,
+                0.04308889770507812,
+                0.04314419174194336,
+                0.043087871551513675,
+                0.04278374481201172,
+                0.04295065689086914,
+                0.043038719177246096,
+                0.04290867233276367,
+                0.04290867233276367,
+                0.043000831604003906,
+                0.042910720825195314,
+                0.04293222427368164,
+                0.04318719863891601,
+                0.043184127807617184,
+                0.043038719177246096
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 5963.09178930169
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 988.24192,
             "max_vram": 2858.418176,
             "max_reserved": 2373.976064,
             "max_allocated": 2326.085632
         },
         "latency": {
             "unit": "s",
+            "mean": 8.607044107437137,
+            "stdev": 0.0006589412689184115,
             "values": [
+                8.607703048706055,
+                8.606385166168218
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 29.62689592582205
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.03375311414681231,
+            "stdev": 0.00012139704096911313,
             "values": [
+                0.03426611328125,
+                0.03396198272705078,
+                0.03381760025024414,
+                0.03385139083862305,
+                0.03381865692138672,
+                0.033925086975097656,
+                0.033966079711914066,
+                0.03388313674926758,
+                0.03379507064819336,
+                0.033800193786621094,
+                0.03376639938354492,
+                0.033800193786621094,
+                0.033726463317871096,
+                0.03383910369873047,
+                0.033739776611328126,
+                0.033701889038085936,
+                0.03391897583007813,
+                0.03387494277954101,
+                0.033742847442626955,
+                0.033756160736083986,
+                0.03369062423706055,
+                0.03377459335327149,
+                0.03365273666381836,
+                0.033742847442626955,
+                0.03389753723144531,
+                0.0338144645690918,
+                0.03360870361328125,
+                0.033688575744628906,
+                0.033724414825439454,
+                0.033750080108642576,
+                0.033605567932128905,
+                0.03383910369873047,
+                0.03366809463500976,
+                0.033811454772949216,
+                0.03383193588256836,
+                0.03371110534667969,
+                0.03363430404663086,
+                0.03373056030273437,
+                0.03360563278198242,
+                0.03370086288452148,
+                0.03364044952392578,
+                0.03370716857910156,
+                0.03387580871582031,
+                0.03373567962646484,
+                0.03373158264160156,
+                0.033718273162841796,
+                0.03360870361328125,
+                0.03376332855224609,
+                0.033844223022460936,
+                0.03371724700927734,
+                0.03363225555419922,
+                0.03413199996948242,
+                0.03370800018310547,
+                0.03391385650634766,
+                0.03379097747802735,
+                0.033827838897705076,
+                0.03363942337036133,
+                0.03366195297241211,
+                0.033702911376953124,
+                0.03400806427001953,
+                0.03388620758056641,
+                0.033751041412353515,
+                0.03362713623046875,
+                0.03373567962646484,
+                0.03382374572753906,
+                0.03369779205322266,
+                0.03380633544921875,
+                0.033683456420898435,
+                0.033655807495117186,
+                0.033693695068359376,
+                0.033552383422851564,
+                0.033598464965820314,
+                0.03368352127075196,
+                0.03385440063476562,
+                0.03356467056274414,
+                0.03363225555419922,
+                0.03360563278198242,
+                0.03391795349121094,
+                0.03400294494628906,
+                0.033702911376953124,
+                0.03367119979858398,
+                0.03372540664672852,
+                0.03366204833984375,
+                0.03363932800292969,
+                0.0335810546875,
+                0.03378688049316406,
+                0.03376639938354492,
+                0.033650688171386715,
+                0.03372236633300781,
+                0.03372032165527344,
+                0.0336743049621582,
+                0.03381343841552734,
+                0.03368243026733399,
+                0.03366092681884766,
+                0.033701889038085936,
+                0.034012161254882815,
+                0.033703041076660153,
+                0.03370278549194336,
+                0.033552383422851564,
+                0.033708030700683594,
+                0.03369267272949219,
+                0.0336445426940918,
+                0.033693695068359376,
+                0.03367731094360352,
+                0.03363430404663086,
+                0.033898494720458985,
+                0.033655807495117186,
+                0.03365990447998047,
+                0.03381151962280273,
+                0.033589183807373045,
+                0.03369267272949219,
+                0.033775615692138675,
+                0.03360255813598633,
+                0.03426816177368164,
+                0.03384320068359375,
+                0.03368755340576172,
+                0.03363430404663086,
+                0.033699840545654294,
+                0.03372851181030274,
+                0.03381452941894531,
+                0.03378278350830078,
+                0.03376844787597656,
+                0.033745918273925785,
+                0.03382374572753906,
+                0.033931262969970705,
+                0.03371929550170898,
+                0.033926143646240234,
+                0.03382067108154297,
+                0.03378483200073242,
+                0.03383295822143555,
+                0.03392716979980469,
+                0.03391078567504883,
+                0.03371212768554688,
+                0.03378278350830078,
+                0.03386880111694336,
+                0.03380940628051758,
+                0.03377151870727539,
+                0.03372851181030274,
+                0.03366400146484375,
+                0.03360563278198242,
+                0.033995777130126956,
+                0.03385548782348633,
+                0.033825790405273434,
+                0.0337367057800293,
+                0.03372032165527344,
+                0.03361587142944336,
+                0.03372032165527344,
+                0.03375513458251953,
+                0.03401830291748047,
+                0.033960960388183595,
+                0.03370700836181641,
+                0.03364147186279297,
+                0.03370393753051758,
+                0.03376537704467773,
+                0.03412070465087891,
+                0.03370086288452148,
+                0.03375820922851563,
+                0.033827838897705076,
+                0.03383091354370117,
+                0.033800193786621094,
+                0.033642494201660156,
+                0.033701889038085936,
+                0.03370598220825195,
+                0.03377766418457031,
+                0.0337367057800293,
+                0.03384012985229492,
+                0.034051071166992186,
+                0.033842174530029294,
+                0.03369372940063477,
+                0.03370902252197266,
+                0.03371212768554688,
+                0.03379507064819336,
+                0.033693695068359376,
+                0.03373875045776367,
+                0.03374899291992187,
+                0.033775615692138675,
+                0.033800193786621094,
+                0.033672191619873046,
+                0.03381248092651367,
+                0.03386163330078125,
+                0.03374387359619141,
+                0.03369881439208984,
+                0.03385548782348633,
+                0.03390873718261719,
+                0.03401113510131836,
+                0.033827838897705076,
+                0.03378483200073242,
+                0.033896446228027344,
+                0.033686527252197264,
+                0.03373168182373047,
+                0.03367619323730469,
+                0.033767425537109375,
+                0.03379507064819336,
+                0.033685504913330076,
+                0.03366604614257813,
+                0.033759231567382815,
+                0.03365478515625,
+                0.033759231567382815,
+                0.033713153839111325,
+                0.03389440155029297,
+                0.03359436798095703,
+                0.03386163330078125,
+                0.03379302215576172,
+                0.033670143127441404,
+                0.033539070129394534,
+                0.03364556884765625,
+                0.03360563278198242,
+                0.03392102432250976,
+                0.03376332855224609,
+                0.03375820922851563,
+                0.03368960189819336,
+                0.034032638549804685,
+                0.03380428695678711,
+                0.03378995132446289,
+                0.03367833709716797,
+                0.03395276641845703,
+                0.0336732177734375,
+                0.03361177444458008,
+                0.033524734497070316,
+                0.033895423889160156,
+                0.033710079193115236,
+                0.03362406539916992,
+                0.03360153579711914,
+                0.03379609680175781,
+                0.033538047790527346,
+                0.03361587142944336,
+                0.033552383422851564,
+                0.03378176116943359,
+                0.0335728645324707,
+                0.033675262451171875,
+                0.03353190231323242,
+                0.03382067108154297,
+                0.03364659118652344,
+                0.03361075210571289,
+                0.03366502380371094,
+                0.03379404830932617,
+                0.03368259048461914,
+                0.03406934356689453,
+                0.033800193786621094,
+                0.03380940628051758,
+                0.03383500671386719,
+                0.03378176116943359,
+                0.0337367057800293,
+                0.03402035140991211,
+                0.03367424011230469,
+                0.033702911376953124,
+                0.033807361602783206,
+                0.033632320404052736,
+                0.03351340866088867,
+                0.0337520637512207,
+                0.033770496368408204,
+                0.033800193786621094,
+                0.033600513458251956,
+                0.03376537704467773,
+                0.033688575744628906,
+                0.033633281707763675,
+                0.033964031219482424,
+                0.03394867324829102,
+                0.03384832000732422,
+                0.03373056030273437,
+                0.03382476806640625,
+                0.033772544860839845,
+                0.03369267272949219,
+                0.033714206695556644,
+                0.03389334487915039,
+                0.033844223022460936,
+                0.033825790405273434,
+                0.03380326461791992,
+                0.03379404830932617,
+                0.03377151870727539,
+                0.03381760025024414,
+                0.033955841064453124,
+                0.033903617858886716,
+                0.033893375396728515,
+                0.033926143646240234,
+                0.033807361602783206,
+                0.033721343994140625,
+                0.03373567962646484,
+                0.03368447875976562,
+                0.03384320068359375,
+                0.03370086288452148,
+                0.03376025772094727,
+                0.03377664184570312,
+                0.03375513458251953,
+                0.03395276641845703,
+                0.03387494277954101,
+                0.033870849609375,
+                0.03386880111694336,
+                0.0337786865234375,
+                0.03401830291748047,
+                0.03401523208618164,
+                0.03385343933105469,
+                0.03378790283203125,
+                0.033882110595703126,
+                0.03405209732055664,
+                0.03387289428710937,
+                0.033827838897705076,
+                0.033833984375,
+                0.03377356719970703,
+                0.03383910369873047,
+                0.03375513458251953,
+                0.0337448959350586,
+                0.03377766418457031,
+                0.03385548782348633,
+                0.03365683364868164,
+                0.03371724700927734,
+                0.03369881439208984,
+                0.03416985702514649,
+                0.033827838897705076,
+                0.03377356719970703,
+                0.033737728118896484,
+                0.0337336311340332,
+                0.03366604614257813,
+                0.03367731094360352,
+                0.033723392486572266,
+                0.033827838897705076,
+                0.033895423889160156,
+                0.03395686340332031,
+                0.033675262451171875,
+                0.03371110534667969,
+                0.03387494277954101,
+                0.033585151672363284,
+                0.03367935943603516,
+                0.03360563278198242,
+                0.03366195297241211,
+                0.03379529571533203,
+                0.03369551849365234,
+                0.03384627151489258,
+                0.033672191619873046,
+                0.03363443374633789,
+                0.03372236633300781,
+                0.03365670394897461,
+                0.033617919921875,
+                0.03378278350830078,
+                0.033568767547607424,
+                0.03361996841430664,
+                0.033653759002685545,
+                0.03367628860473633,
+                0.03366604614257813,
+                0.03357900619506836,
+                0.03362713623046875,
+                0.03361382293701172,
+                0.03378585433959961,
+                0.033860607147216795,
+                0.03363840103149414,
+                0.03379507064819336,
+                0.03365785598754883,
+                0.03368457412719727,
+                0.033922977447509765,
+                0.033593505859375,
+                0.0336976318359375,
+                0.03354739379882812,
+                0.03364646530151367,
+                0.033614849090576174,
+                0.03365990447998047,
+                0.03363840103149414,
+                0.033649665832519535,
+                0.033532928466796875,
+                0.03365785598754883,
+                0.03367424011230469,
+                0.03390566253662109,
+                0.033655807495117186,
+                0.033873920440673826,
+                0.03366502380371094,
+                0.03375820922851563,
+                0.03362713623046875,
+                0.03364147186279297,
+                0.03360255813598633,
+                0.033726463317871096,
+                0.033532928466796875,
+                0.0335810546875,
+                0.03363532638549805,
+                0.033701889038085936,
+                0.033547264099121094,
+                0.033842174530029294,
+                0.03380121612548828,
+                0.03365990447998047,
+                0.033653759002685545,
+                0.03397529602050781,
+                0.03384012985229492,
+                0.03381964874267578,
+                0.03391385650634766,
+                0.0337520637512207,
+                0.033963008880615236,
+                0.03395481491088867,
+                0.03383603286743164,
+                0.03367833709716797,
+                0.03362918472290039,
+                0.03359231948852539,
+                0.033650688171386715,
+                0.03361280059814453,
+                0.03358310317993164,
+                0.033726463317871096,
+                0.0339343376159668,
+                0.0337448959350586,
+                0.03365478515625,
+                0.033980415344238284,
+                0.03391795349121094,
+                0.03368038558959961,
+                0.03367731094360352,
+                0.03362406539916992,
+                0.03360665512084961,
+                0.03355340957641602,
+                0.033854465484619144,
+                0.03370822525024414,
+                0.033707839965820316,
+                0.0337336311340332,
+                0.033702110290527346,
+                0.03364227294921875,
+                0.03371110534667969,
+                0.03369062423706055,
+                0.03371724700927734,
+                0.03382271957397461,
+                0.033893375396728515,
+                0.034034687042236327,
+                0.03386880111694336,
+                0.033756160736083986,
+                0.03380633544921875,
+                0.03376332855224609,
+                0.03371110534667969,
+                0.03389132690429687,
+                0.03387801742553711,
+                0.0337786865234375,
+                0.03395993423461914,
+                0.03379814529418945,
+                0.033879039764404296,
+                0.03372748947143555,
+                0.033928192138671875,
+                0.03378585433959961,
+                0.03374899291992187,
+                0.033783809661865234,
+                0.03375820922851563,
+                0.034067455291748046,
+                0.03370905685424805,
+                0.03376128005981445,
+                0.03365478515625,
+                0.03391385650634766,
+                0.03367424011230469,
+                0.033742847442626955,
+                0.03371110534667969,
+                0.03385958480834961,
+                0.03378073501586914,
+                0.03376025772094727,
+                0.03370089721679687,
+                0.033669086456298826,
+                0.033726463317871096,
+                0.033614849090576174,
+                0.03381964874267578,
+                0.033742847442626955,
+                0.034233345031738284,
+                0.03388620758056641,
+                0.033756160736083986,
+                0.03379302215576172,
+                0.03367833709716797,
+                0.03374387359619141,
+                0.033683456420898435,
+                0.03379507064819336,
+                0.0337655029296875,
+                0.03376012802124023,
+                0.033870849609375,
+                0.03378995132446289,
+                0.03364659118652344,
+                0.03379609680175781,
+                0.033708030700683594,
+                0.03373894500732422,
+                0.033920833587646484,
+                0.033658878326416015,
+                0.033909759521484374,
+                0.03381043243408203,
+                0.03361075210571289,
+                0.03382067108154297,
+                0.03376233673095703,
+                0.03363427352905273,
+                0.033751041412353515,
+                0.033923072814941405,
+                0.03373056030273437,
+                0.03362815856933594,
+                0.03361996841430664,
+                0.03363225555419922,
+                0.03381452941894531,
+                0.0335994873046875,
+                0.03371929550170898,
+                0.033756160736083986,
+                0.03368960189819336,
+                0.03359334564208984,
+                0.03357491302490234,
+                0.03361382293701172,
+                0.03375718307495117,
+                0.033710079193115236,
+                0.03360768127441406,
+                0.033538047790527346,
+                0.033554431915283206,
+                0.03383500671386719,
+                0.033532928466796875,
+                0.03384320068359375,
+                0.03364556884765625,
+                0.03364147186279297,
+                0.03354220962524414,
+                0.03364755249023438,
+                0.03363020706176758,
+                0.03364556884765625,
+                0.033633281707763675,
+                0.03379507064819336,
+                0.03388620758056641,
+                0.03369676971435547,
+                0.03372032165527344,
+                0.03369267272949219,
+                0.03376639938354492,
+                0.033721343994140625,
+                0.03353702545166016
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 29.62689592582204
         },
         "energy": null,
         "efficiency": null