Upload /Llama-2-7b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Llama-2-7b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,811 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1014.673408,
+            "max_vram": 4905.238528,
+            "max_reserved": 4420.796416,
+            "max_allocated": 4252.667392
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.04235845992333779,
+            "stdev": 0.000575210706458403,
+            "values": [
+                0.04604630279541016,
+                0.04459417724609375,
+                0.044017696380615236,
+                0.043,
+                0.04298649597167969,
+                0.044181758880615235,
+                0.04794060897827149,
+                0.045223072052001954,
+                0.04260659027099609,
+                0.04237311935424805,
+                0.04183859252929688,
+                0.04231884765625,
+                0.04192665481567383,
+                0.04237004852294922,
+                0.04188876724243164,
+                0.04235776138305664,
+                0.042006526947021484,
+                0.04234137725830078,
+                0.04183567810058594,
+                0.04234649658203125,
+                0.04187033462524414,
+                0.04235878372192383,
+                0.04184985733032227,
+                0.04235878372192383,
+                0.041902080535888675,
+                0.04236083221435547,
+                0.04187340927124023,
+                0.04234137725830078,
+                0.041842689514160154,
+                0.04272127914428711,
+                0.041869312286376956,
+                0.04237823867797851,
+                0.041842689514160154,
+                0.042352638244628905,
+                0.041847808837890625,
+                0.042379264831542966,
+                0.04184783935546875,
+                0.04233830261230469,
+                0.041853950500488284,
+                0.04236083221435547,
+                0.04184371185302734,
+                0.042344703674316406,
+                0.04214886474609375,
+                0.04237331390380859,
+                0.04184985733032227,
+                0.04237516784667969,
+                0.0418682861328125,
+                0.04235878372192383,
+                0.041821182250976564,
+                0.04236697769165039,
+                0.041850879669189454,
+                0.04236288070678711,
+                0.04184371185302734,
+                0.04235059356689453,
+                0.04187363052368164,
+                0.042355712890625,
+                0.042249214172363284,
+                0.04237619018554688,
+                0.04187955093383789,
+                0.04239257431030274,
+                0.0418785285949707,
+                0.04235366439819336,
+                0.04186531066894531,
+                0.042355712890625,
+                0.041917537689208986,
+                0.04249190521240234,
+                0.04222566223144531,
+                0.042352638244628905,
+                0.04187855911254883,
+                0.04234649658203125,
+                0.04184064102172851,
+                0.042349567413330076,
+                0.04185497665405274,
+                0.04238438415527344,
+                0.04185190582275391,
+                0.04234985733032227,
+                0.042038272857666016,
+                0.04237107086181641,
+                0.042256385803222656,
+                0.042438751220703126,
+                0.04217958450317383,
+                0.0424089584350586,
+                0.04188671875,
+                0.04234137725830078,
+                0.04185804748535156,
+                0.04234137725830078,
+                0.04192256164550781,
+                0.04234649658203125,
+                0.04183254241943359,
+                0.04236492919921875,
+                0.04184985733032227,
+                0.0423454704284668,
+                0.04186111831665039,
+                0.042359809875488284,
+                0.04212758255004883,
+                0.04239974212646484,
+                0.04209766387939453,
+                0.04239769744873047,
+                0.0418600959777832,
+                0.042365951538085936,
+                0.04186111831665039,
+                0.042379264831542966,
+                0.041869312286376956,
+                0.04236492919921875,
+                0.04219190216064453,
+                0.042369022369384765,
+                0.04218572616577149,
+                0.042369022369384765,
+                0.04184473419189453,
+                0.04236492919921875,
+                0.04185702514648437,
+                0.04235366439819336,
+                0.041853950500488284,
+                0.04245708847045898,
+                0.04187750244140625,
+                0.042382335662841795,
+                0.0421662712097168,
+                0.04239257431030274,
+                0.04235161590576172,
+                0.04247552108764648,
+                0.04256563186645508,
+                0.04246220779418945,
+                0.04232601547241211,
+                0.04243046569824219,
+                0.04306022262573242,
+                0.042395648956298826,
+                0.042397758483886716,
+                0.042433536529541016,
+                0.04243558502197266,
+                0.042396961212158205,
+                0.042479713439941405,
+                0.04241305541992187,
+                0.04252262496948242,
+                0.042379264831542966,
+                0.042336254119873046,
+                0.042444801330566405,
+                0.04249497604370117,
+                0.0424161262512207,
+                0.04234649658203125,
+                0.04237823867797851,
+                0.04191129684448242,
+                0.04237516784667969,
+                0.04225024032592774,
+                0.0424192008972168,
+                0.04230656051635742,
+                0.04242329788208008,
+                0.04240588760375977,
+                0.04239180755615234,
+                0.04234672164916992,
+                0.042385406494140625,
+                0.04242432022094727,
+                0.042499073028564455,
+                0.04243558502197266,
+                0.0424192008972168,
+                0.042444801330566405,
+                0.04238950347900391,
+                0.04229529571533203,
+                0.042387454986572266,
+                0.042387454986572266,
+                0.04235488128662109,
+                0.042378303527832034,
+                0.04238950347900391,
+                0.04216115188598633,
+                0.04242432022094727,
+                0.042482688903808595,
+                0.04240793609619141,
+                0.04228915023803711,
+                0.04238643264770508,
+                0.04234137725830078,
+                0.04237516784667969,
+                0.04225228881835937,
+                0.04238643264770508,
+                0.04278988647460937,
+                0.04238336181640625,
+                0.04233523178100586,
+                0.042380287170410154,
+                0.04213555145263672,
+                0.0424051513671875,
+                0.04213452911376953,
+                0.042353759765625,
+                0.041837825775146484,
+                0.042349567413330076,
+                0.0418785285949707,
+                0.04242329788208008,
+                0.04229017639160156,
+                0.04263628768920898,
+                0.042299678802490234,
+                0.04239974212646484,
+                0.042503360748291016,
+                0.04242943954467773,
+                0.042280960083007815,
+                0.04239974212646484,
+                0.042363903045654294,
+                0.042439678192138675,
+                0.042352638244628905,
+                0.042404895782470704,
+                0.04229836654663086,
+                0.04256972885131836,
+                0.04243046569824219,
+                0.042403839111328126,
+                0.04239769744873047,
+                0.04239257431030274,
+                0.04236083221435547,
+                0.04242432022094727,
+                0.04230553436279297,
+                0.042396671295166014,
+                0.04221644973754883,
+                0.042379264831542966,
+                0.04259430313110352,
+                0.04235776138305664,
+                0.04236800003051758,
+                0.04237311935424805,
+                0.042365951538085936,
+                0.04242227172851563,
+                0.042560672760009764,
+                0.042385406494140625,
+                0.04235776138305664,
+                0.04237619018554688,
+                0.04247347259521484,
+                0.04280934524536133,
+                0.042359809875488284,
+                0.04239769744873047,
+                0.04246323013305664,
+                0.042395648956298826,
+                0.042229759216308595,
+                0.04241305541992187,
+                0.042379264831542966,
+                0.042379264831542966,
+                0.04227686309814453,
+                0.04240486526489258,
+                0.04219190216064453,
+                0.04236288070678711,
+                0.0423004150390625,
+                0.042365951538085936,
+                0.042259456634521485,
+                0.04236697769165039,
+                0.042383392333984374
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 6043.656933309665
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1014.673408,
+            "max_vram": 5255.462912,
+            "max_reserved": 4771.0208,
+            "max_allocated": 4715.611648
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 8.496251308441167,
+            "stdev": 0.006892139434813593,
+            "values": [
+                8.489359169006352,
+                8.50314344787598
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 30.01323651369084
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.033318632582122205,
+            "stdev": 0.0001715019741849696,
+            "values": [
+                0.03346720123291016,
+                0.03380940628051758,
+                0.033414142608642575,
+                0.03335782241821289,
+                0.03330867385864258,
+                0.033291263580322264,
+                0.03315609741210938,
+                0.0333199348449707,
+                0.03316940689086914,
+                0.0333383674621582,
+                0.0332861442565918,
+                0.03326566314697266,
+                0.03335168075561523,
+                0.03326873779296875,
+                0.033446910858154294,
+                0.033567745208740236,
+                0.033219585418701174,
+                0.033255424499511715,
+                0.033175552368164066,
+                0.03319193649291992,
+                0.03323392105102539,
+                0.03323801422119141,
+                0.033708030700683594,
+                0.0332861442565918,
+                0.03327897644042969,
+                0.033274879455566404,
+                0.03344384002685547,
+                0.033271808624267575,
+                0.033372161865234375,
+                0.033377281188964845,
+                0.03337420654296875,
+                0.033328128814697267,
+                0.03321753692626953,
+                0.03337011337280273,
+                0.033259521484375,
+                0.03322470474243164,
+                0.03324825668334961,
+                0.03370086288452148,
+                0.033255424499511715,
+                0.03335782241821289,
+                0.033307647705078124,
+                0.033285118103027346,
+                0.03321446228027344,
+                0.033290241241455076,
+                0.033334270477294925,
+                0.033416191101074216,
+                0.03401830291748047,
+                0.03400908660888672,
+                0.03389132690429687,
+                0.03330559921264648,
+                0.03326976013183594,
+                0.03334368133544922,
+                0.03332281494140625,
+                0.03337932968139649,
+                0.033296382904052735,
+                0.035366912841796876,
+                0.03342335891723633,
+                0.033235969543457033,
+                0.03321139144897461,
+                0.03332198333740234,
+                0.03336294555664063,
+                0.03337625503540039,
+                0.033347583770751955,
+                0.033261566162109374,
+                0.033377281188964845,
+                0.03352678298950195,
+                0.03329536056518555,
+                0.033296382904052735,
+                0.033291263580322264,
+                0.03328409576416016,
+                0.03331174468994141,
+                0.033258495330810545,
+                0.033219585418701174,
+                0.033219585418701174,
+                0.03324415969848633,
+                0.03327590560913086,
+                0.033263614654541016,
+                0.033285118103027346,
+                0.03314483261108398,
+                0.03323494338989258,
+                0.0332861442565918,
+                0.0332492790222168,
+                0.03318067169189453,
+                0.033701889038085936,
+                0.03316121673583984,
+                0.0332492790222168,
+                0.03317452621459961,
+                0.033314815521240236,
+                0.03318783950805664,
+                0.033288192749023435,
+                0.03322265625,
+                0.03326873779296875,
+                0.03327590560913086,
+                0.03323187255859375,
+                0.03310182571411133,
+                0.033255424499511715,
+                0.033142784118652346,
+                0.033650688171386715,
+                0.03315097427368164,
+                0.033307647705078124,
+                0.03320012664794922,
+                0.03325235366821289,
+                0.03338240051269531,
+                0.033293312072753906,
+                0.03321753692626953,
+                0.03321465682983398,
+                0.03318764877319336,
+                0.03331686401367188,
+                0.03322367858886719,
+                0.03321855926513672,
+                0.0332042236328125,
+                0.03322675323486328,
+                0.03317657470703125,
+                0.0332677116394043,
+                0.033380352020263675,
+                0.033414142608642575,
+                0.03347455978393555,
+                0.03338137435913086,
+                0.03322265625,
+                0.033468414306640625,
+                0.0333568000793457,
+                0.03338444900512695,
+                0.03336703872680664,
+                0.03322470474243164,
+                0.033470462799072266,
+                0.03316121673583984,
+                0.033160190582275394,
+                0.033225727081298825,
+                0.03322675323486328,
+                0.03319014358520508,
+                0.033279743194580075,
+                0.03321855926513672,
+                0.03325337600708008,
+                0.03334656143188477,
+                0.03335782241821289,
+                0.03322470474243164,
+                0.03336908721923828,
+                0.03313356781005859,
+                0.033172481536865236,
+                0.03320012664794922,
+                0.0332410888671875,
+                0.03342335891723633,
+                0.03331891250610351,
+                0.033825790405273434,
+                0.03311820983886719,
+                0.03343360137939453,
+                0.033291263580322264,
+                0.03317279815673828,
+                0.03330835342407226,
+                0.03316428756713867,
+                0.03321241760253906,
+                0.03336601638793945,
+                0.033107967376708985,
+                0.03324313735961914,
+                0.033320960998535154,
+                0.033124351501464845,
+                0.03309056091308594,
+                0.033544193267822264,
+                0.033555454254150394,
+                0.03325747299194336,
+                0.03314585494995117,
+                0.033423583984375,
+                0.03320195388793945,
+                0.03326668930053711,
+                0.033263614654541016,
+                0.03348582458496094,
+                0.03338444900512695,
+                0.033186817169189455,
+                0.03367628860473633,
+                0.03368038558959961,
+                0.03340083312988281,
+                0.033301502227783206,
+                0.03327283096313476,
+                0.0331038703918457,
+                0.0331141128540039,
+                0.033296382904052735,
+                0.03316223907470703,
+                0.03312639999389649,
+                0.033113086700439456,
+                0.03316428756713867,
+                0.03325030517578125,
+                0.03342233657836914,
+                0.03314585494995117,
+                0.0331673583984375,
+                0.03301273727416992,
+                0.03329232025146484,
+                0.0331734733581543,
+                0.03319705581665039,
+                0.03306496047973633,
+                0.033276927947998046,
+                0.033083393096923826,
+                0.033170433044433595,
+                0.03320832061767578,
+                0.03324518585205078,
+                0.03321548843383789,
+                0.03318272018432617,
+                0.03312844848632813,
+                0.0331960334777832,
+                0.03331686401367188,
+                0.03302195358276367,
+                0.03327078247070313,
+                0.033020927429199216,
+                0.03313971328735352,
+                0.03311001586914063,
+                0.03360665512084961,
+                0.033329151153564454,
+                0.033143806457519534,
+                0.03317782211303711,
+                0.033045280456542966,
+                0.03308544158935547,
+                0.0331141128540039,
+                0.03321241760253906,
+                0.03304959869384766,
+                0.033111038208007815,
+                0.03309465789794922,
+                0.03321241760253906,
+                0.03324415969848633,
+                0.03318486404418945,
+                0.03342633438110352,
+                0.033380352020263675,
+                0.03314688110351562,
+                0.033235969543457033,
+                0.03321343994140625,
+                0.033331199645996096,
+                0.03318272018432617,
+                0.03340595245361328,
+                0.03321855926513672,
+                0.033263614654541016,
+                0.03313049697875976,
+                0.03328409576416016,
+                0.03324006271362305,
+                0.03340697479248047,
+                0.033242111206054685,
+                0.03329536056518555,
+                0.03313663864135742,
+                0.03324518585205078,
+                0.03329536056518555,
+                0.033291263580322264,
+                0.03312844848632813,
+                0.03336191940307617,
+                0.03318067169189453,
+                0.03320832061767578,
+                0.033323009490966796,
+                0.03341107177734375,
+                0.03322880172729492,
+                0.03324518585205078,
+                0.033230846405029296,
+                0.03338137435913086,
+                0.0330618896484375,
+                0.03305574417114258,
+                0.033113086700439456,
+                0.033364990234375,
+                0.03319807815551758,
+                0.03337420654296875,
+                0.03324825668334961,
+                0.033560577392578124,
+                0.0333383674621582,
+                0.03336806488037109,
+                0.033272064208984375,
+                0.03331865692138672,
+                0.03336703872680664,
+                0.03326876831054688,
+                0.03331683349609375,
+                0.03364863967895508,
+                0.03325439834594727,
+                0.033377281188964845,
+                0.033259521484375,
+                0.03315609741210938,
+                0.03319091033935547,
+                0.03331174468994141,
+                0.033337345123291014,
+                0.0330885124206543,
+                0.03322880172729492,
+                0.03315814590454102,
+                0.033154048919677735,
+                0.03321036911010742,
+                0.033137664794921876,
+                0.03323801422119141,
+                0.03346636962890625,
+                0.033058815002441407,
+                0.03313971328735352,
+                0.03320627212524414,
+                0.0331776008605957,
+                0.03320115280151367,
+                0.03321651077270508,
+                0.033285118103027346,
+                0.03320217514038086,
+                0.03319398498535156,
+                0.03333324813842774,
+                0.03325030517578125,
+                0.03329945755004883,
+                0.03313868713378906,
+                0.033148929595947264,
+                0.033159168243408206,
+                0.03326464080810547,
+                0.033230846405029296,
+                0.03321446228027344,
+                0.033276927947998046,
+                0.033344512939453126,
+                0.03339775848388672,
+                0.0332564468383789,
+                0.033276927947998046,
+                0.033326175689697264,
+                0.03324918365478516,
+                0.0333834228515625,
+                0.03355340957641602,
+                0.03336806488037109,
+                0.03344793701171875,
+                0.03344384002685547,
+                0.033290241241455076,
+                0.033430526733398434,
+                0.0331253776550293,
+                0.03326367950439453,
+                0.033176513671875,
+                0.033307647705078124,
+                0.03341516876220703,
+                0.03333017730712891,
+                0.033277950286865234,
+                0.0333568000793457,
+                0.03335372924804687,
+                0.033455265045166015,
+                0.033401695251464844,
+                0.033288192749023435,
+                0.033329151153564454,
+                0.033827838897705076,
+                0.033544193267822264,
+                0.033500160217285156,
+                0.033445888519287106,
+                0.03355648040771484,
+                0.03351859283447266,
+                0.033858558654785154,
+                0.03339468765258789,
+                0.03349094390869141,
+                0.03340185546875,
+                0.033432575225830076,
+                0.033538047790527346,
+                0.033470462799072266,
+                0.03341926574707031,
+                0.03385651016235352,
+                0.03338649749755859,
+                0.0335093765258789,
+                0.03346432113647461,
+                0.03339263916015625,
+                0.03343564987182617,
+                0.03352166366577149,
+                0.033414142608642575,
+                0.03345100784301758,
+                0.033337345123291014,
+                0.033454078674316406,
+                0.03328716659545899,
+                0.033503231048583985,
+                0.03328307342529297,
+                0.03344076919555664,
+                0.033410049438476565,
+                0.03337932968139649,
+                0.0335810546875,
+                0.03328409576416016,
+                0.03346636962890625,
+                0.03332819366455078,
+                0.03340486526489258,
+                0.03334883117675781,
+                0.03342006301879883,
+                0.033514495849609374,
+                0.03355750274658203,
+                0.03342745590209961,
+                0.03348787307739258,
+                0.03352576065063476,
+                0.03352268981933594,
+                0.033358016967773435,
+                0.033468223571777346,
+                0.03339775848388672,
+                0.033430526733398434,
+                0.03353190231323242,
+                0.033342464447021485,
+                0.03337113571166992,
+                0.033320960998535154,
+                0.03354009628295898,
+                0.03343667221069336,
+                0.03338751983642578,
+                0.03338956832885742,
+                0.033463294982910154,
+                0.0332861442565918,
+                0.03337932968139649,
+                0.03340390396118164,
+                0.03373875045776367,
+                0.03335475158691406,
+                0.03340595245361328,
+                0.03340083312988281,
+                0.03331584167480469,
+                0.03341209411621094,
+                0.03326873779296875,
+                0.033306625366210936,
+                0.033342464447021485,
+                0.03333529663085937,
+                0.033393665313720705,
+                0.03337113571166992,
+                0.03336908721923828,
+                0.03347558212280274,
+                0.03350732803344727,
+                0.0336445426940918,
+                0.03340902328491211,
+                0.03340288162231445,
+                0.03340185546875,
+                0.03337420654296875,
+                0.03329740905761719,
+                0.03334348678588867,
+                0.033375232696533204,
+                0.033413120269775394,
+                0.03320832061767578,
+                0.03330252838134766,
+                0.03333855819702149,
+                0.033346366882324216,
+                0.03326566314697266,
+                0.03337113571166992,
+                0.03346944046020508,
+                0.033393665313720705,
+                0.033498111724853515,
+                0.0334284782409668,
+                0.03336191940307617,
+                0.033323009490966796,
+                0.03340697479248047,
+                0.03341110229492188,
+                0.0332789421081543,
+                0.03335372924804687,
+                0.03322060775756836,
+                0.03331071853637695,
+                0.033242271423339846,
+                0.03330134582519531,
+                0.033205249786376956,
+                0.03320115280151367,
+                0.03326566314697266,
+                0.03329536056518555,
+                0.033342464447021485,
+                0.03318476867675781,
+                0.0333496322631836,
+                0.03325337600708008,
+                0.033328128814697267,
+                0.03326566314697266,
+                0.033203201293945314,
+                0.033329151153564454,
+                0.033285118103027346,
+                0.03334041595458984,
+                0.033342464447021485,
+                0.03336191940307617,
+                0.033380352020263675,
+                0.03370086288452148,
+                0.03334764862060547,
+                0.033365951538085935,
+                0.03345305633544922,
+                0.033372161865234375,
+                0.03334860610961914,
+                0.0332861442565918,
+                0.03339468765258789,
+                0.03320115280151367,
+                0.033328128814697267,
+                0.03332710266113281,
+                0.03342335891723633,
+                0.033372161865234375,
+                0.033271808624267575,
+                0.03356979370117188,
+                0.03353497695922852,
+                0.03361894226074219,
+                0.033181697845458984,
+                0.033328128814697267,
+                0.03331174468994141,
+                0.03318476867675781,
+                0.03319500732421875,
+                0.033261566162109374,
+                0.033519615173339845,
+                0.03323392105102539,
+                0.033285118103027346,
+                0.03306905746459961,
+                0.03324313735961914,
+                0.033328128814697267,
+                0.033113086700439456,
+                0.033312767028808594,
+                0.0332492790222168,
+                0.03352371215820313,
+                0.03318179321289062,
+                0.03322051239013672,
+                0.03315302276611328,
+                0.03323699188232422,
+                0.033152000427246094,
+                0.03322777557373047,
+                0.03332419204711914,
+                0.03319075012207031,
+                0.03314688110351562,
+                0.03334348678588867,
+                0.03321343994140625,
+                0.033181697845458984,
+                0.03316857528686523,
+                0.03333100891113281,
+                0.03328921508789062,
+                0.03334041595458984,
+                0.03347763061523437,
+                0.03329740905761719,
+                0.03316633605957031,
+                0.03334656143188477,
+                0.03329945755004883,
+                0.033334270477294925,
+                0.033111038208007815,
+                0.03321855926513672,
+                0.033140735626220705,
+                0.03334656143188477,
+                0.03337324905395508,
+                0.03357382583618164,
+                0.033154048919677735,
+                0.03328204727172852,
+                0.033148929595947264,
+                0.033307647705078124
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 30.013236513690856
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}