Upload /Mistral-7B-v0.1-GPTQ-4bit/gptq4bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Mistral-7B-v0.1-GPTQ-4bit/gptq4bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,775 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 7099.604992,
+            "max_vram": 5544.869888,
+            "max_reserved": 5058.330624,
+            "max_allocated": 4906.976256
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.04978826615822252,
+            "stdev": 0.00024757412295624384,
+            "values": [
+                0.05262575912475586,
+                0.05165670394897461,
+                0.049909793853759765,
+                0.04978483200073242,
+                0.04984832000732422,
+                0.04983295822143555,
+                0.04985651016235351,
+                0.049857536315917966,
+                0.04984524917602539,
+                0.0498790397644043,
+                0.04994047927856445,
+                0.04988927841186523,
+                0.04987289428710937,
+                0.04981248092651367,
+                0.04984454345703125,
+                0.04975308990478516,
+                0.04978176116943359,
+                0.04976025772094726,
+                0.04979507064819336,
+                0.04974387359619141,
+                0.04981350326538086,
+                0.04977971267700195,
+                0.04978278350830078,
+                0.049751041412353515,
+                0.04977356719970703,
+                0.049719295501708984,
+                0.04977971267700195,
+                0.04972748947143555,
+                0.04981862258911133,
+                0.04964966583251953,
+                0.049772544860839846,
+                0.04968960189819336,
+                0.04977356719970703,
+                0.049729534149169925,
+                0.04974591827392578,
+                0.04967833709716797,
+                0.049748992919921874,
+                0.04970598220825195,
+                0.049756160736083986,
+                0.04965683364868164,
+                0.049742847442626956,
+                0.04971417617797851,
+                0.04972544097900391,
+                0.04971212768554688,
+                0.049710174560546876,
+                0.04966502380371094,
+                0.04974796676635742,
+                0.04969472122192383,
+                0.049718273162841796,
+                0.049642494201660156,
+                0.04975001525878906,
+                0.049675262451171875,
+                0.04982886505126953,
+                0.04965580749511719,
+                0.049793022155761715,
+                0.04964966583251953,
+                0.04978073501586914,
+                0.04971417617797851,
+                0.04974489593505859,
+                0.04968764877319336,
+                0.049716224670410154,
+                0.04968038558959961,
+                0.049729534149169925,
+                0.049670143127441405,
+                0.0497151985168457,
+                0.04969881439208984,
+                0.04975513458251953,
+                0.04965683364868164,
+                0.04976947021484375,
+                0.04964659118652344,
+                0.04976537704467773,
+                0.049675262451171875,
+                0.04976844787597656,
+                0.04965683364868164,
+                0.049757183074951174,
+                0.04971212768554688,
+                0.04973567962646484,
+                0.049740798950195314,
+                0.04975308990478516,
+                0.04972236633300781,
+                0.049772865295410154,
+                0.04969472122192383,
+                0.049786880493164064,
+                0.04969881439208984,
+                0.049805313110351565,
+                0.04967628860473633,
+                0.049767425537109375,
+                0.04973056030273437,
+                0.049757183074951174,
+                0.049724414825439454,
+                0.04978073501586914,
+                0.049757183074951174,
+                0.04975001525878906,
+                0.049726463317871096,
+                0.04974387359619141,
+                0.04969779205322265,
+                0.04984627151489258,
+                0.04971212768554688,
+                0.04983091354370117,
+                0.0497151985168457,
+                0.04976639938354492,
+                0.049708030700683595,
+                0.04990566253662109,
+                0.04971023941040039,
+                0.04976639938354492,
+                0.049767425537109375,
+                0.04978073501586914,
+                0.04974489593505859,
+                0.04992512130737305,
+                0.049716224670410154,
+                0.04984934234619141,
+                0.049718273162841796,
+                0.049792129516601565,
+                0.049756160736083986,
+                0.04982067108154297,
+                0.049756160736083986,
+                0.049827838897705076,
+                0.049767425537109375,
+                0.04983910369873047,
+                0.04973567962646484,
+                0.04973875045776367,
+                0.04978585433959961,
+                0.049767425537109375,
+                0.049729534149169925,
+                0.049809406280517575,
+                0.04970700836181641,
+                0.04973567962646484,
+                0.04979814529418945,
+                0.04983603286743164,
+                0.04972851181030274,
+                0.04986675262451172,
+                0.04979609680175781,
+                0.04979916763305664,
+                0.04976128005981445,
+                0.049713153839111325,
+                0.04972032165527344,
+                0.04976947021484375,
+                0.04972339248657227,
+                0.04986265563964844,
+                0.04972851181030274,
+                0.049833984375,
+                0.04978995132446289,
+                0.04984115219116211,
+                0.04979916763305664,
+                0.04979609680175781,
+                0.04975513458251953,
+                0.04979097747802735,
+                0.049724414825439454,
+                0.04983705520629883,
+                0.0497786865234375,
+                0.04988518524169922,
+                0.049708030700683595,
+                0.04984115219116211,
+                0.04976435089111328,
+                0.049783809661865235,
+                0.049775745391845705,
+                0.049825790405273435,
+                0.04972748947143555,
+                0.04977356719970703,
+                0.049734718322753904,
+                0.04976639938354492,
+                0.049754112243652344,
+                0.049821697235107425,
+                0.049770496368408204,
+                0.04979404830932617,
+                0.04976947021484375,
+                0.049874942779541014,
+                0.04967731094360352,
+                0.04982681655883789,
+                0.04976128005981445,
+                0.049858558654785154,
+                0.04969267272949219,
+                0.04977664184570312,
+                0.04970905685424805,
+                0.04983295822143555,
+                0.04973567962646484,
+                0.04974694442749023,
+                0.049746143341064454,
+                0.04977369689941406,
+                0.049757183074951174,
+                0.04980223846435547,
+                0.049751041412353515,
+                0.04981964874267578,
+                0.049721343994140625,
+                0.04979404830932617,
+                0.04970905685424805,
+                0.04980950546264649,
+                0.04977971267700195,
+                0.04984320068359375,
+                0.049757183074951174,
+                0.04981452941894531,
+                0.04976332855224609,
+                0.04981248092651367,
+                0.04974694442749023,
+                0.04976537704467773,
+                0.049754112243652344,
+                0.04978278350830078,
+                0.04981760025024414,
+                0.04983910369873047,
+                0.049732608795166014,
+                0.04978483200073242
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 5141.773750193582
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 7099.604992,
+            "max_vram": 5714.7392,
+            "max_reserved": 5228.199936,
+            "max_allocated": 5030.959616
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 5.667942934036259,
+            "stdev": 0.022348278045655867,
+            "values": [
+                5.690291212081915,
+                5.645594655990603
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 44.98986721773665
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.02222722719229906,
+            "stdev": 0.0004308662127795926,
+            "values": [
+                0.024837120056152344,
+                0.02454015922546387,
+                0.022458368301391602,
+                0.0222423038482666,
+                0.022146272659301757,
+                0.022225791931152344,
+                0.022161407470703123,
+                0.022094751358032228,
+                0.02214297676086426,
+                0.02212998390197754,
+                0.022113983154296874,
+                0.02211942481994629,
+                0.02215936088562012,
+                0.02211737632751465,
+                0.022191104888916017,
+                0.022191104888916017,
+                0.022132959365844727,
+                0.022175519943237305,
+                0.0221265926361084,
+                0.02213478469848633,
+                0.022180864334106445,
+                0.022119712829589844,
+                0.022091487884521484,
+                0.022106111526489256,
+                0.022141151428222657,
+                0.022032159805297852,
+                0.022130687713623046,
+                0.02210223960876465,
+                0.022119199752807617,
+                0.022141952514648438,
+                0.02212761688232422,
+                0.02208153533935547,
+                0.022165504455566407,
+                0.02214297676086426,
+                0.02209791946411133,
+                0.02208870315551758,
+                0.022184959411621095,
+                0.022160383224487306,
+                0.022165504455566407,
+                0.022152191162109376,
+                0.022133760452270508,
+                0.022153343200683594,
+                0.02209881591796875,
+                0.02211737632751465,
+                0.02215116882324219,
+                0.022067264556884767,
+                0.022162368774414062,
+                0.02221670341491699,
+                0.022156288146972656,
+                0.022172704696655273,
+                0.022235103607177734,
+                0.02217884826660156,
+                0.022212575912475586,
+                0.022163455963134765,
+                0.022177919387817383,
+                0.02212441635131836,
+                0.022235136032104492,
+                0.02209382438659668,
+                0.022180864334106445,
+                0.022136831283569337,
+                0.022197248458862305,
+                0.022198272705078126,
+                0.02214297676086426,
+                0.022207487106323243,
+                0.022079488754272462,
+                0.02213478469848633,
+                0.022220800399780274,
+                0.022149120330810547,
+                0.022219776153564453,
+                0.022202688217163084,
+                0.022175424575805663,
+                0.02211123275756836,
+                0.022199296951293947,
+                0.02205388832092285,
+                0.022163455963134765,
+                0.022042623519897463,
+                0.022120447158813478,
+                0.022082559585571288,
+                0.022188032150268554,
+                0.02229350471496582,
+                0.022141952514648438,
+                0.02204364776611328,
+                0.02219024085998535,
+                0.022081375122070312,
+                0.021976064682006836,
+                0.024197120666503907,
+                0.023562240600585937,
+                0.022541311264038084,
+                0.022137855529785155,
+                0.02214713668823242,
+                0.02220025634765625,
+                0.022156448364257814,
+                0.02217353630065918,
+                0.022120447158813478,
+                0.02209382438659668,
+                0.022162431716918944,
+                0.02225155258178711,
+                0.022171615600585937,
+                0.022184959411621095,
+                0.022128639221191407,
+                0.022158336639404298,
+                0.022190080642700196,
+                0.022158336639404298,
+                0.022090911865234375,
+                0.02210902404785156,
+                0.02214816093444824,
+                0.022106048583984374,
+                0.02209791946411133,
+                0.022184959411621095,
+                0.022180864334106445,
+                0.022197439193725587,
+                0.022136640548706055,
+                0.022190080642700196,
+                0.022152191162109376,
+                0.022166528701782227,
+                0.02208367919921875,
+                0.02222172737121582,
+                0.0221265926361084,
+                0.0221265926361084,
+                0.02220134353637695,
+                0.022152191162109376,
+                0.02213580894470215,
+                0.022114400863647462,
+                0.022172576904296876,
+                0.02223849678039551,
+                0.02218057632446289,
+                0.022168575286865236,
+                0.022176767349243166,
+                0.022523168563842774,
+                0.022159072875976564,
+                0.022139904022216796,
+                0.022207487106323243,
+                0.022137855529785155,
+                0.0220897274017334,
+                0.022158336639404298,
+                0.022082752227783203,
+                0.022173568725585936,
+                0.022134719848632814,
+                0.022175743103027345,
+                0.022098207473754884,
+                0.022174432754516603,
+                0.022100255966186522,
+                0.022089439392089842,
+                0.02210304069519043,
+                0.022143007278442383,
+                0.022043743133544923,
+                0.022203264236450197,
+                0.022091808319091796,
+                0.022358240127563475,
+                0.022207487106323243,
+                0.022207231521606446,
+                0.022114336013793947,
+                0.022205408096313477,
+                0.02211123275756836,
+                0.022167552947998048,
+                0.022145023345947267,
+                0.022165504455566407,
+                0.022071296691894532,
+                0.02212179183959961,
+                0.022160064697265624,
+                0.022124544143676757,
+                0.022074367523193358,
+                0.022192127227783204,
+                0.022120447158813478,
+                0.022230016708374024,
+                0.02212761688232422,
+                0.022140127182006836,
+                0.022198047637939453,
+                0.022196224212646484,
+                0.02226495933532715,
+                0.022190208435058593,
+                0.02207513618469238,
+                0.022321151733398437,
+                0.02225049591064453,
+                0.022026239395141603,
+                0.022131935119628906,
+                0.02208438491821289,
+                0.022012928009033202,
+                0.022108160018920898,
+                0.022082559585571288,
+                0.022079488754272462,
+                0.022050975799560547,
+                0.022062944412231444,
+                0.022811935424804686,
+                0.024096479415893556,
+                0.02451353645324707,
+                0.024466432571411133,
+                0.02445635223388672,
+                0.024449888229370116,
+                0.024359935760498046,
+                0.024391679763793944,
+                0.024410272598266603,
+                0.024415359497070313,
+                0.02446614456176758,
+                0.02450022315979004,
+                0.024391679763793944,
+                0.024385536193847656,
+                0.02410905647277832,
+                0.0237260799407959,
+                0.02361142349243164,
+                0.022265823364257812,
+                0.022116352081298828,
+                0.021984479904174806,
+                0.02208950424194336,
+                0.022166528701782227,
+                0.022156288146972656,
+                0.02210918426513672,
+                0.022137088775634764,
+                0.02213657569885254,
+                0.02205388832092285,
+                0.022042720794677735,
+                0.02205174446105957,
+                0.02208153533935547,
+                0.022170719146728517,
+                0.022055839538574217,
+                0.02215545654296875,
+                0.022062911987304687,
+                0.022112255096435548,
+                0.02212563133239746,
+                0.02207539176940918,
+                0.022117311477661133,
+                0.022156288146972656,
+                0.02207744026184082,
+                0.02204979133605957,
+                0.02207744026184082,
+                0.022086719512939453,
+                0.022137792587280273,
+                0.022010879516601564,
+                0.02230784034729004,
+                0.022098943710327147,
+                0.022071296691894532,
+                0.02205388832092285,
+                0.022462528228759767,
+                0.022174655914306642,
+                0.02215116882324219,
+                0.021998592376708984,
+                0.022147071838378905,
+                0.02208665657043457,
+                0.022124544143676757,
+                0.02206515121459961,
+                0.02210508728027344,
+                0.02211020851135254,
+                0.022120447158813478,
+                0.022136831283569337,
+                0.02217296028137207,
+                0.02211299133300781,
+                0.022122495651245116,
+                0.022091775894165038,
+                0.022053056716918946,
+                0.022111167907714845,
+                0.022236032485961912,
+                0.022064128875732423,
+                0.022037599563598635,
+                0.02203536033630371,
+                0.022047744750976563,
+                0.02225868797302246,
+                0.02239897537231445,
+                0.022169599533081053,
+                0.022225984573364256,
+                0.022270912170410155,
+                0.022223871231079103,
+                0.02215852737426758,
+                0.022141759872436523,
+                0.022145088195800782,
+                0.022124479293823242,
+                0.022199296951293947,
+                0.02211327934265137,
+                0.022192127227783204,
+                0.022189056396484375,
+                0.022182912826538087,
+                0.02205183982849121,
+                0.022110240936279297,
+                0.02211734390258789,
+                0.022191104888916017,
+                0.022164480209350586,
+                0.022140928268432617,
+                0.022150432586669922,
+                0.02218671989440918,
+                0.0220446720123291,
+                0.022160383224487306,
+                0.022166784286499024,
+                0.02212428855895996,
+                0.022220800399780274,
+                0.022214656829833986,
+                0.02209791946411133,
+                0.022173696517944336,
+                0.022260736465454102,
+                0.022184959411621095,
+                0.022187007904052734,
+                0.022137855529785155,
+                0.022154239654541014,
+                0.022155263900756835,
+                0.022128639221191407,
+                0.02220134353637695,
+                0.02211327934265137,
+                0.022155263900756835,
+                0.022214656829833986,
+                0.022146175384521485,
+                0.02209686470031738,
+                0.022136735916137695,
+                0.022161407470703123,
+                0.022139904022216796,
+                0.02213478469848633,
+                0.02204569625854492,
+                0.022167648315429687,
+                0.022104032516479494,
+                0.02205900764465332,
+                0.022072256088256834,
+                0.022132736206054687,
+                0.022180864334106445,
+                0.022015071868896483,
+                0.02203228759765625,
+                0.022138879776000975,
+                0.02205183982849121,
+                0.022188032150268554,
+                0.02208358383178711,
+                0.0221265926361084,
+                0.022071296691894532,
+                0.02209280014038086,
+                0.022131711959838866,
+                0.02215936088562012,
+                0.02206719970703125,
+                0.02205286407470703,
+                0.022120479583740235,
+                0.022180831909179688,
+                0.02204364776611328,
+                0.02281881523132324,
+                0.022228992462158204,
+                0.02226278305053711,
+                0.022237184524536133,
+                0.022179840087890625,
+                0.022161407470703123,
+                0.022208608627319337,
+                0.022141855239868165,
+                0.022221824645996095,
+                0.02211020851135254,
+                0.022232063293457033,
+                0.02224742317199707,
+                0.02208358383178711,
+                0.022090751647949217,
+                0.02208358383178711,
+                0.022106111526489256,
+                0.022122495651245116,
+                0.0221265926361084,
+                0.022175840377807617,
+                0.02224425506591797,
+                0.022331392288208008,
+                0.022167552947998048,
+                0.02206003189086914,
+                0.022199296951293947,
+                0.022164480209350586,
+                0.02213478469848633,
+                0.022108448028564452,
+                0.022112159729003905,
+                0.022116159439086912,
+                0.022148256301879884,
+                0.02216329574584961,
+                0.022162431716918944,
+                0.02208995246887207,
+                0.022185951232910155,
+                0.022152000427246094,
+                0.022063104629516602,
+                0.02209587287902832,
+                0.02246348762512207,
+                0.022174720764160157,
+                0.022189056396484375,
+                0.022175743103027345,
+                0.022185152053833007,
+                0.022137664794921876,
+                0.022129663467407225,
+                0.02215116882324219,
+                0.02215116882324219,
+                0.022165504455566407,
+                0.022125568389892578,
+                0.022094079971313477,
+                0.02214374351501465,
+                0.022155263900756835,
+                0.022137855529785155,
+                0.02213478469848633,
+                0.02216886329650879,
+                0.022093727111816407,
+                0.02219398307800293,
+                0.022054912567138672,
+                0.022416383743286132,
+                0.022193151473999022,
+                0.022212608337402344,
+                0.022144287109375,
+                0.022091487884521484,
+                0.022146047592163084,
+                0.02218115234375,
+                0.02216419219970703,
+                0.022215679168701173,
+                0.022120447158813478,
+                0.022107135772705077,
+                0.022174720764160157,
+                0.022114303588867186,
+                0.022112255096435548,
+                0.02207257652282715,
+                0.021996288299560546,
+                0.022167552947998048,
+                0.02202726364135742,
+                0.02207539176940918,
+                0.02208153533935547,
+                0.02230169677734375,
+                0.022161407470703123,
+                0.022170623779296874,
+                0.02208153533935547,
+                0.021985279083251954,
+                0.02206924819946289,
+                0.021992448806762696,
+                0.02206822395324707,
+                0.02201907157897949,
+                0.022125568389892578,
+                0.022116352081298828,
+                0.022064128875732423,
+                0.02203647994995117,
+                0.022033407211303712,
+                0.02216886329650879,
+                0.022149856567382813,
+                0.022037504196166992,
+                0.02202828788757324,
+                0.022001951217651368,
+                0.022064863204956056,
+                0.022091775894165038,
+                0.021955583572387697,
+                0.022146047592163084,
+                0.022133760452270508,
+                0.022390783309936522,
+                0.0221265926361084,
+                0.022127744674682617,
+                0.022120384216308592,
+                0.022052799224853516,
+                0.02205286407470703,
+                0.02206719970703125,
+                0.022091775894165038,
+                0.02206822395324707,
+                0.022200511932373046,
+                0.02223187255859375,
+                0.022033407211303712,
+                0.022076416015625,
+                0.0221265926361084,
+                0.02207539176940918,
+                0.02210508728027344,
+                0.022141952514648438,
+                0.02208870315551758,
+                0.022076416015625,
+                0.022090751647949217,
+                0.022124832153320312,
+                0.022084320068359375,
+                0.02214297676086426,
+                0.02208870315551758,
+                0.022076416015625,
+                0.02209587287902832,
+                0.022092992782592774,
+                0.022121376037597656,
+                0.022432832717895507,
+                0.02216636848449707,
+                0.022140928268432617,
+                0.02223209571838379,
+                0.02211324882507324,
+                0.022210559844970702,
+                0.02209814453125,
+                0.02226361656188965,
+                0.02208867263793945,
+                0.022116352081298828,
+                0.022108160018920898,
+                0.022164480209350586,
+                0.022137920379638673,
+                0.022054847717285157,
+                0.022063104629516602,
+                0.022167552947998048,
+                0.022152191162109376,
+                0.0220579833984375,
+                0.02212268829345703,
+                0.02203628730773926,
+                0.022185983657836913,
+                0.022130687713623046,
+                0.022188032150268554,
+                0.022090751647949217,
+                0.02213478469848633,
+                0.02210918426513672,
+                0.022191232681274414,
+                0.022129535675048828,
+                0.022181888580322266,
+                0.02210099220275879,
+                0.02215116882324219,
+                0.02206822395324707,
+                0.022492160797119142,
+                0.022124544143676757,
+                0.022179840087890625,
+                0.02221772766113281,
+                0.02211840057373047,
+                0.022143999099731446,
+                0.02210111999511719,
+                0.02216643142700195,
+                0.02209071922302246,
+                0.02209280014038086,
+                0.022114303588867186,
+                0.02208153533935547,
+                0.022153215408325197,
+                0.022082559585571288,
+                0.022047744750976563,
+                0.02204569625854492,
+                0.022136831283569337,
+                0.022143999099731446,
+                0.022148096084594726,
+                0.022023168563842774,
+                0.022172672271728516,
+                0.02204979133605957,
+                0.02198121643066406
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 44.989867217736645
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}