Upload /Mistral-7B-v0.1/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Mistral-7B-v0.1/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,852 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 985.341952,
+            "max_vram": 15755.902976,
+            "max_reserved": 15271.460864,
+            "max_allocated": 15114.469888
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.035987248907843924,
+            "stdev": 0.00021574759418921462,
+            "values": [
+                0.03843862533569336,
+                0.03772723388671875,
+                0.03614720153808594,
+                0.036101119995117184,
+                0.036176895141601564,
+                0.03613798522949219,
+                0.03643904113769531,
+                0.03620249557495117,
+                0.036176895141601564,
+                0.03693772888183594,
+                0.03680460739135742,
+                0.036363552093505856,
+                0.03591680145263672,
+                0.03592192077636719,
+                0.0358922233581543,
+                0.03592192077636719,
+                0.03593318557739258,
+                0.035937278747558594,
+                0.035958782196044925,
+                0.035972095489501955,
+                0.03597312164306641,
+                0.035866622924804685,
+                0.03590569686889648,
+                0.03593011093139648,
+                0.03598233413696289,
+                0.03594854354858398,
+                0.035937278747558594,
+                0.03593625640869141,
+                0.03592806243896484,
+                0.035958782196044925,
+                0.035952640533447267,
+                0.0359444465637207,
+                0.035888126373291016,
+                0.03601203155517578,
+                0.0359024658203125,
+                0.03589017486572266,
+                0.035963905334472655,
+                0.03593523025512695,
+                0.035947521209716796,
+                0.036975616455078124,
+                0.0359813117980957,
+                0.035945472717285154,
+                0.03592806243896484,
+                0.03592192077636719,
+                0.035929088592529294,
+                0.03596083068847656,
+                0.03594035339355469,
+                0.035955745697021486,
+                0.03589120101928711,
+                0.03592499160766602,
+                0.03595673751831055,
+                0.0359106559753418,
+                0.035980289459228515,
+                0.035983360290527344,
+                0.03590553665161133,
+                0.03590963363647461,
+                0.035942401885986325,
+                0.035931137084960936,
+                0.03593952178955078,
+                0.03594035339355469,
+                0.0359659538269043,
+                0.035942401885986325,
+                0.03612499237060547,
+                0.03594956970214844,
+                0.035939327239990236,
+                0.03592806243896484,
+                0.035958782196044925,
+                0.03593625640869141,
+                0.03592704010009766,
+                0.03592499160766602,
+                0.03597824096679687,
+                0.035952640533447267,
+                0.035929088592529294,
+                0.03593625640869141,
+                0.03595775985717774,
+                0.03603558349609375,
+                0.03607551956176758,
+                0.035953662872314454,
+                0.03595673751831055,
+                0.03597625732421875,
+                0.035922943115234376,
+                0.0359628791809082,
+                0.03595161437988281,
+                0.036015392303466794,
+                0.03597824096679687,
+                0.03595161437988281,
+                0.035929088592529294,
+                0.035969024658203126,
+                0.03593011093139648,
+                0.03597619247436523,
+                0.035963905334472655,
+                0.03594137573242188,
+                0.03594649505615234,
+                0.035999744415283204,
+                0.03595775985717774,
+                0.03597619247436523,
+                0.035958782196044925,
+                0.03592416000366211,
+                0.035912704467773435,
+                0.035920894622802735,
+                0.03590860748291016,
+                0.035969024658203126,
+                0.03592704010009766,
+                0.03602329635620117,
+                0.03599359893798828,
+                0.0359251823425293,
+                0.03596492767333984,
+                0.03601408004760742,
+                0.035970046997070314,
+                0.03595596694946289,
+                0.035937278747558594,
+                0.035945472717285154,
+                0.03594137573242188,
+                0.035932159423828124,
+                0.035963905334472655,
+                0.03594137573242188,
+                0.03599257659912109,
+                0.035966976165771485,
+                0.03612364959716797,
+                0.03595161437988281,
+                0.035945472717285154,
+                0.03600281524658203,
+                0.03595775985717774,
+                0.035963905334472655,
+                0.03594035339355469,
+                0.035950721740722655,
+                0.03597926330566406,
+                0.03600896072387695,
+                0.035942401885986325,
+                0.03597926330566406,
+                0.035963905334472655,
+                0.03598233413696289,
+                0.03606425476074219,
+                0.03592704010009766,
+                0.03594460678100586,
+                0.03594137573242188,
+                0.035952640533447267,
+                0.03596291351318359,
+                0.035922943115234376,
+                0.03592601776123047,
+                0.03592396926879883,
+                0.03594137573242188,
+                0.03594342422485351,
+                0.035929088592529294,
+                0.0358922233581543,
+                0.036050945281982424,
+                0.03590553665161133,
+                0.035922943115234376,
+                0.03592396926879883,
+                0.035915775299072264,
+                0.035912704467773435,
+                0.03593625640869141,
+                0.03595372772216797,
+                0.03594956970214844,
+                0.03599155044555664,
+                0.03593523025512695,
+                0.03589529418945313,
+                0.035915775299072264,
+                0.03595161437988281,
+                0.03598144149780273,
+                0.0359106559753418,
+                0.03590348815917969,
+                0.03593318557739258,
+                0.035980289459228515,
+                0.035901439666748046,
+                0.035939327239990236,
+                0.0359106559753418,
+                0.035953662872314454,
+                0.03592601776123047,
+                0.035970046997070314,
+                0.03588915252685547,
+                0.035896320343017575,
+                0.035904510498046875,
+                0.03593011093139648,
+                0.03606016159057617,
+                0.03591088104248047,
+                0.03589427185058594,
+                0.0359628791809082,
+                0.035934207916259765,
+                0.03592601776123047,
+                0.03591705703735352,
+                0.0359466552734375,
+                0.035884033203125,
+                0.03591987228393555,
+                0.035969024658203126,
+                0.035896320343017575,
+                0.03590963363647461,
+                0.03588710403442383,
+                0.036071422576904294,
+                0.035915775299072264,
+                0.03594956970214844,
+                0.035914752960205076,
+                0.035915775299072264,
+                0.03592806243896484,
+                0.03593830490112305,
+                0.035901439666748046,
+                0.03592396926879883,
+                0.03593625640869141,
+                0.035904510498046875,
+                0.03586969757080078,
+                0.035883007049560545,
+                0.035988479614257815,
+                0.03588608169555664,
+                0.03592396926879883,
+                0.03595161437988281,
+                0.03593625640869141,
+                0.035975166320800785,
+                0.03593318557739258,
+                0.035899391174316404,
+                0.03592499160766602,
+                0.0359106559753418,
+                0.03592396926879883,
+                0.03593830490112305,
+                0.03595673751831055,
+                0.035942401885986325,
+                0.03592806243896484,
+                0.03592499160766602,
+                0.0359598388671875,
+                0.035901439666748046,
+                0.03594854354858398,
+                0.035947521209716796,
+                0.03593011093139648,
+                0.035915775299072264,
+                0.035942401885986325,
+                0.035901439666748046,
+                0.0358922233581543,
+                0.03592704010009766,
+                0.035931137084960936,
+                0.03590758514404297,
+                0.03590655899047852,
+                0.03608473587036133,
+                0.035961856842041014,
+                0.03594140625,
+                0.03595980834960937,
+                0.035934207916259765,
+                0.035955711364746096,
+                0.03592704010009766,
+                0.03594137573242188,
+                0.036029441833496094,
+                0.03604275131225586,
+                0.036001792907714845,
+                0.03600076675415039,
+                0.036055038452148434,
+                0.0360447998046875,
+                0.03612979125976563,
+                0.036038654327392575,
+                0.035999744415283204,
+                0.03601715087890625,
+                0.03602022552490235,
+                0.036050945281982424,
+                0.035975166320800785,
+                0.03603968048095703,
+                0.03599792098999023,
+                0.036021247863769534,
+                0.03602841567993164,
+                0.035974334716796875,
+                0.035945472717285154,
+                0.03602329635620117,
+                0.03597747039794922,
+                0.03599052810668945,
+                0.03600815963745117,
+                0.035999744415283204,
+                0.036103168487548826,
+                0.03600691223144531,
+                0.03599462509155273,
+                0.03599462509155273,
+                0.03597619247436523,
+                0.03599462509155273,
+                0.03597107315063477,
+                0.03599462509155273,
+                0.03599155044555664,
+                0.03597312164306641,
+                0.036001792907714845,
+                0.03600080108642578,
+                0.0360079345703125,
+                0.03595468902587891,
+                0.03594956970214844,
+                0.0359628791809082
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 7113.630737808391
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 985.341952,
+            "max_vram": 15923.675136,
+            "max_reserved": 15439.233024,
+            "max_allocated": 15238.453248
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 7.1229505767822285,
+            "stdev": 0.00452053070068148,
+            "values": [
+                7.12747110748291,
+                7.118430046081547
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 35.79977107116128
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.027933139516793035,
+            "stdev": 0.00013863113423522085,
+            "values": [
+                0.02992742347717285,
+                0.029331775665283204,
+                0.02788319969177246,
+                0.027845632553100585,
+                0.027849727630615235,
+                0.027805696487426756,
+                0.027852800369262694,
+                0.027850751876831056,
+                0.027831296920776367,
+                0.027850751876831056,
+                0.027958431243896485,
+                0.02785161590576172,
+                0.0278569278717041,
+                0.027859935760498045,
+                0.02779648017883301,
+                0.02790297508239746,
+                0.027836416244506838,
+                0.027822080612182616,
+                0.027811840057373048,
+                0.027823104858398437,
+                0.027835391998291017,
+                0.027995328903198242,
+                0.027817792892456054,
+                0.027900928497314452,
+                0.02787046432495117,
+                0.02780953598022461,
+                0.027860992431640624,
+                0.028075008392333983,
+                0.027883520126342775,
+                0.027865087509155274,
+                0.027867136001586915,
+                0.027848703384399414,
+                0.027900928497314452,
+                0.027790336608886718,
+                0.0279418888092041,
+                0.027858943939208985,
+                0.02785513687133789,
+                0.02784227180480957,
+                0.02786623954772949,
+                0.02795916748046875,
+                0.027914304733276368,
+                0.02787424087524414,
+                0.027900928497314452,
+                0.027838464736938476,
+                0.02788476753234863,
+                0.02789455986022949,
+                0.027903200149536133,
+                0.02792835235595703,
+                0.027928607940673828,
+                0.027907039642333983,
+                0.027910144805908203,
+                0.02795929527282715,
+                0.027897855758666993,
+                0.02790297508239746,
+                0.027855871200561523,
+                0.027850751876831056,
+                0.027824127197265625,
+                0.02788761520385742,
+                0.027913215637207032,
+                0.027872447967529298,
+                0.02782598304748535,
+                0.027838464736938476,
+                0.02785206413269043,
+                0.02787708854675293,
+                0.027843584060668947,
+                0.027884544372558592,
+                0.027873279571533204,
+                0.02791116714477539,
+                0.027892736434936522,
+                0.02793471908569336,
+                0.02794495964050293,
+                0.028044288635253906,
+                0.027882816314697266,
+                0.02790060806274414,
+                0.02790809631347656,
+                0.027889663696289063,
+                0.027888639450073242,
+                0.027866111755371094,
+                0.02791116714477539,
+                0.02797875213623047,
+                0.027861087799072266,
+                0.02795817565917969,
+                0.02812313652038574,
+                0.027864063262939453,
+                0.02778828811645508,
+                0.0286167049407959,
+                0.028886016845703126,
+                0.028100608825683594,
+                0.027857952117919922,
+                0.027945951461791994,
+                0.027838720321655273,
+                0.027878143310546874,
+                0.02792755126953125,
+                0.027923456192016603,
+                0.02797260856628418,
+                0.02795212745666504,
+                0.02794086456298828,
+                0.02792959976196289,
+                0.027882816314697266,
+                0.02810643196105957,
+                0.027881696701049806,
+                0.027904800415039063,
+                0.02787843132019043,
+                0.027882463455200197,
+                0.02795315170288086,
+                0.027926528930664062,
+                0.027879615783691407,
+                0.027866943359375,
+                0.027881471633911133,
+                0.027865087509155274,
+                0.02788380813598633,
+                0.02785456085205078,
+                0.027880640029907228,
+                0.027861824035644533,
+                0.027875328063964845,
+                0.027867136001586915,
+                0.02792448043823242,
+                0.027880447387695313,
+                0.02791116714477539,
+                0.02793164825439453,
+                0.02794803237915039,
+                0.027913215637207032,
+                0.02795827293395996,
+                0.027869184494018553,
+                0.027904319763183593,
+                0.02791289520263672,
+                0.02793267250061035,
+                0.027873279571533204,
+                0.027849727630615235,
+                0.02793471908569336,
+                0.027868160247802733,
+                0.027874303817749024,
+                0.02790505599975586,
+                0.027891679763793944,
+                0.02789580726623535,
+                0.02791116714477539,
+                0.027889663696289063,
+                0.02790809631347656,
+                0.02789580726623535,
+                0.027913215637207032,
+                0.0279418888092041,
+                0.027975679397583008,
+                0.02792140769958496,
+                0.0279552001953125,
+                0.027906208038330077,
+                0.027929439544677734,
+                0.02793984031677246,
+                0.027905023574829102,
+                0.027898015975952147,
+                0.027917152404785157,
+                0.02791935920715332,
+                0.02793267250061035,
+                0.027915264129638673,
+                0.02795622444152832,
+                0.028094463348388672,
+                0.02792550468444824,
+                0.02794905662536621,
+                0.027963392257690428,
+                0.028026016235351562,
+                0.027937631607055664,
+                0.02791731262207031,
+                0.027987968444824218,
+                0.027918336868286132,
+                0.027930624008178712,
+                0.02795315170288086,
+                0.02792448043823242,
+                0.027892736434936522,
+                0.027936767578125,
+                0.02792755126953125,
+                0.02792959976196289,
+                0.02797260856628418,
+                0.028213247299194336,
+                0.027881568908691406,
+                0.027962272644042968,
+                0.027909120559692382,
+                0.02794598388671875,
+                0.02795212745666504,
+                0.027910144805908203,
+                0.0279685115814209,
+                0.02792857551574707,
+                0.02798489570617676,
+                0.02794086456298828,
+                0.027979776382446288,
+                0.02792448043823242,
+                0.027992063522338868,
+                0.02791219139099121,
+                0.027988191604614257,
+                0.02792732810974121,
+                0.027991071701049804,
+                0.027942880630493164,
+                0.027944992065429688,
+                0.02795516777038574,
+                0.02799001693725586,
+                0.02798182487487793,
+                0.02795008087158203,
+                0.0279736328125,
+                0.027910144805908203,
+                0.02794803237915039,
+                0.027998207092285156,
+                0.027982847213745117,
+                0.02796031951904297,
+                0.027992063522338868,
+                0.027930624008178712,
+                0.02797260856628418,
+                0.02799843215942383,
+                0.027974431991577148,
+                0.027931743621826172,
+                0.0279837760925293,
+                0.027997215270996093,
+                0.02807907295227051,
+                0.028030975341796875,
+                0.027982847213745117,
+                0.02796544075012207,
+                0.027987968444824218,
+                0.028022783279418945,
+                0.02795827293395996,
+                0.028037120819091797,
+                0.027997184753417968,
+                0.028047359466552735,
+                0.027999231338500977,
+                0.028000255584716797,
+                0.02799308776855469,
+                0.02797670364379883,
+                0.028011520385742186,
+                0.028018688201904295,
+                0.027982847213745117,
+                0.02818079948425293,
+                0.028029632568359376,
+                0.027998207092285156,
+                0.028014591217041016,
+                0.02800230407714844,
+                0.02800230407714844,
+                0.028045312881469726,
+                0.028025856018066408,
+                0.028013568878173828,
+                0.027995136260986327,
+                0.028000255584716797,
+                0.028000255584716797,
+                0.02795827293395996,
+                0.027966463088989257,
+                0.028037439346313475,
+                0.027996864318847656,
+                0.028018688201904295,
+                0.028227584838867188,
+                0.028049407958984376,
+                0.028036096572875976,
+                0.027994176864624024,
+                0.027978687286376952,
+                0.02794393539428711,
+                0.028012544631958007,
+                0.028040191650390626,
+                0.02795724868774414,
+                0.027936767578125,
+                0.02795622444152832,
+                0.02794291114807129,
+                0.0278822078704834,
+                0.027841535568237305,
+                0.027809823989868164,
+                0.02782716751098633,
+                0.02785001564025879,
+                0.0278504638671875,
+                0.027863040924072265,
+                0.027849727630615235,
+                0.027792383193969726,
+                0.027856895446777344,
+                0.027827199935913087,
+                0.027827199935913087,
+                0.027794624328613283,
+                0.027830080032348634,
+                0.027824127197265625,
+                0.027859968185424806,
+                0.027836416244506838,
+                0.027894880294799803,
+                0.027906240463256834,
+                0.027821887969970704,
+                0.027813791275024414,
+                0.027883520126342775,
+                0.02792550468444824,
+                0.02776678466796875,
+                0.0278703670501709,
+                0.02787004852294922,
+                0.027865087509155274,
+                0.027892736434936522,
+                0.02784390449523926,
+                0.027822784423828125,
+                0.027849727630615235,
+                0.027910144805908203,
+                0.027849760055541992,
+                0.027846624374389648,
+                0.027857919692993165,
+                0.027869184494018553,
+                0.02785408020019531,
+                0.027915008544921877,
+                0.027837440490722655,
+                0.027782144546508788,
+                0.027819007873535157,
+                0.027820032119750978,
+                0.027991039276123047,
+                0.027807743072509765,
+                0.027923583984375,
+                0.027935615539550783,
+                0.02791628837585449,
+                0.027881471633911133,
+                0.027856096267700196,
+                0.02788534355163574,
+                0.027859136581420897,
+                0.02787308883666992,
+                0.027873279571533204,
+                0.027820032119750978,
+                0.02780467224121094,
+                0.027869184494018553,
+                0.027883712768554687,
+                0.02786105537414551,
+                0.027901695251464843,
+                0.028049407958984376,
+                0.027893760681152343,
+                0.027838464736938476,
+                0.027864063262939453,
+                0.02788761520385742,
+                0.027801599502563477,
+                0.02791219139099121,
+                0.027893760681152343,
+                0.027882495880126954,
+                0.027837440490722655,
+                0.027824127197265625,
+                0.027892736434936522,
+                0.027833343505859375,
+                0.027897855758666993,
+                0.027885568618774413,
+                0.02794905662536621,
+                0.02791731262207031,
+                0.02790297508239746,
+                0.027872255325317383,
+                0.02790399932861328,
+                0.028000255584716797,
+                0.027893760681152343,
+                0.028014591217041016,
+                0.027886592864990234,
+                0.027891712188720705,
+                0.02779648017883301,
+                0.027890687942504884,
+                0.02779363250732422,
+                0.027860767364501954,
+                0.027867359161376955,
+                0.027907871246337892,
+                0.028010496139526365,
+                0.027847679138183593,
+                0.027883615493774414,
+                0.02791926383972168,
+                0.02791219139099121,
+                0.02793369674682617,
+                0.027888639450073242,
+                0.027982847213745117,
+                0.027886592864990234,
+                0.027938816070556642,
+                0.027819007873535157,
+                0.02790399932861328,
+                0.027875328063964845,
+                0.02793574333190918,
+                0.027898015975952147,
+                0.02782908821105957,
+                0.027868160247802733,
+                0.027852800369262694,
+                0.0278702392578125,
+                0.02787222480773926,
+                0.027911327362060548,
+                0.02783420753479004,
+                0.02796441650390625,
+                0.027878400802612304,
+                0.028066816329956053,
+                0.027840511322021484,
+                0.027883520126342775,
+                0.02790297508239746,
+                0.027821311950683592,
+                0.027857664108276368,
+                0.027849760055541992,
+                0.027896799087524415,
+                0.027869184494018553,
+                0.02792959976196289,
+                0.02794086456298828,
+                0.02791219139099121,
+                0.028007423400878906,
+                0.02792959976196289,
+                0.027900224685668946,
+                0.027863744735717774,
+                0.027909120559692382,
+                0.0279552001953125,
+                0.027983871459960938,
+                0.027842559814453126,
+                0.027886592864990234,
+                0.027918336868286132,
+                0.027851776123046876,
+                0.02795315170288086,
+                0.027909120559692382,
+                0.02793779182434082,
+                0.02790399932861328,
+                0.027915264129638673,
+                0.027867136001586915,
+                0.027894784927368164,
+                0.027889888763427736,
+                0.027906848907470704,
+                0.027909120559692382,
+                0.027936767578125,
+                0.027839487075805663,
+                0.027922431945800782,
+                0.0279418888092041,
+                0.02793267250061035,
+                0.027851776123046876,
+                0.027923456192016603,
+                0.02790297508239746,
+                0.027897855758666993,
+                0.027876575469970702,
+                0.027903776168823242,
+                0.027872447967529298,
+                0.027949888229370116,
+                0.027848928451538087,
+                0.028020511627197264,
+                0.027884544372558592,
+                0.02792550468444824,
+                0.02789580726623535,
+                0.027923456192016603,
+                0.02795417594909668,
+                0.027884544372558592,
+                0.02793164825439453,
+                0.02790809631347656,
+                0.027940128326416017,
+                0.02785558319091797,
+                0.027998207092285156,
+                0.02795417594909668,
+                0.0279418888092041,
+                0.027876352310180662,
+                0.02791731262207031,
+                0.027889663696289063,
+                0.02787942314147949,
+                0.027854848861694335,
+                0.027897855758666993,
+                0.02787942314147949,
+                0.02789990425109863,
+                0.027892736434936522,
+                0.02793164825439453,
+                0.027906047821044923,
+                0.028054527282714844,
+                0.028037120819091797,
+                0.02792959976196289,
+                0.02799411201477051,
+                0.028045440673828127,
+                0.02797657585144043,
+                0.028091615676879882,
+                0.0279467830657959,
+                0.028004415512084962,
+                0.027964351654052734,
+                0.02795827293395996,
+                0.02792959976196289,
+                0.027910144805908203,
+                0.027900928497314452,
+                0.027920671463012695,
+                0.027931360244750975,
+                0.028000255584716797,
+                0.027963392257690428,
+                0.02811801528930664,
+                0.02795724868774414,
+                0.02793471908569336,
+                0.028017663955688478,
+                0.027987968444824218,
+                0.02796544075012207,
+                0.028044288635253906,
+                0.028071935653686524,
+                0.028028032302856446,
+                0.02802777671813965,
+                0.028034048080444338,
+                0.028017663955688478,
+                0.02799308776855469,
+                0.02798080062866211,
+                0.027991039276123047,
+                0.027995136260986327,
+                0.027984064102172853,
+                0.027921215057373047,
+                0.02797670364379883,
+                0.02800230407714844,
+                0.02795724868774414,
+                0.02792550468444824,
+                0.027991039276123047,
+                0.02795622444152832,
+                0.02798489570617676,
+                0.028110847473144532,
+                0.027997184753417968,
+                0.028050432205200194,
+                0.028066816329956053,
+                0.028067840576171874,
+                0.02802617645263672,
+                0.027970239639282225,
+                0.02797260856628418,
+                0.028064767837524415,
+                0.02799001693725586,
+                0.027983871459960938,
+                0.027952159881591797,
+                0.02797667121887207,
+                0.02795110321044922,
+                0.027966463088989257,
+                0.02800230407714844,
+                0.027982847213745117,
+                0.028027904510498046,
+                0.027974655151367187,
+                0.02799827194213867,
+                0.027931583404541015,
+                0.028000320434570312,
+                0.02801043128967285,
+                0.027996160507202147,
+                0.02796953582763672,
+                0.027999359130859373
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 35.79977107116131
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}