Upload /Mistral-7B-v0.1/int8_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Mistral-7B-v0.1/int8_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,472 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 4899.352576,
+            "max_vram": 8757.706752,
+            "max_reserved": 8300.527616,
+            "max_allocated": 8141.046272
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.06498407784994545,
+            "stdev": 0.0012725798540905208,
+            "values": [
+                0.07094246673583984,
+                0.06825062561035156,
+                0.06625587463378907,
+                0.06964224243164062,
+                0.07119462585449218,
+                0.06544895935058594,
+                0.06484480285644531,
+                0.06624460601806641,
+                0.06468402862548828,
+                0.06472191619873047,
+                0.06462258911132812,
+                0.06466969299316407,
+                0.0648253402709961,
+                0.06485708618164063,
+                0.06478131103515625,
+                0.06481919860839844,
+                0.06479872131347657,
+                0.06470861053466796,
+                0.06476083374023438,
+                0.0647014389038086,
+                0.064468994140625,
+                0.06457344055175782,
+                0.06462054443359375,
+                0.06466867065429688,
+                0.06438912200927735,
+                0.06441165161132813,
+                0.0644485092163086,
+                0.06455296325683593,
+                0.0646297607421875,
+                0.06463488006591797,
+                0.06486323547363282,
+                0.06442803192138671,
+                0.06460313415527344,
+                0.06466047668457031,
+                0.06472089385986328,
+                0.06455296325683593,
+                0.064574462890625,
+                0.06447309112548828,
+                0.06504345703125,
+                0.06460620880126954,
+                0.06446694183349609,
+                0.06450994873046875,
+                0.06444751739501953,
+                0.0647383041381836,
+                0.06471475219726562,
+                0.06452838134765625,
+                0.06454681396484375,
+                0.064648193359375,
+                0.06457855987548829,
+                0.06477516937255859,
+                0.06461337280273438,
+                0.06448435211181641,
+                0.06449152374267578,
+                0.06454579162597657,
+                0.0645038070678711,
+                0.06446694183349609,
+                0.06449152374267578,
+                0.0644136962890625,
+                0.06437478637695312,
+                0.06454271697998047,
+                0.06440652465820312,
+                0.06463897705078125,
+                0.06486732482910157,
+                0.06457651519775391,
+                0.06454271697998047,
+                0.06441292572021484,
+                0.06450387573242188,
+                0.06448947143554687,
+                0.06458573150634765,
+                0.06476902770996094,
+                0.06449356842041015,
+                0.06458777618408203,
+                0.06443417358398437,
+                0.06463590240478516,
+                0.06469222259521484,
+                0.06456217956542969,
+                0.06440447998046875,
+                0.06440140533447265,
+                0.06439628601074218,
+                0.06453350067138672,
+                0.06457855987548829,
+                0.06439628601074218,
+                0.0643594207763672,
+                0.06456012725830078,
+                0.06453043365478515,
+                0.06456934356689453,
+                0.0646297607421875,
+                0.06454905700683594,
+                0.06456012725830078,
+                0.06454886627197266,
+                0.06442291259765626,
+                0.06443724822998047,
+                0.06456626892089844,
+                0.0648980484008789,
+                0.0646645736694336,
+                0.064611328125,
+                0.06437888336181641,
+                0.06456832122802734,
+                0.06453478240966797,
+                0.06498099517822266,
+                0.0649574432373047,
+                0.0645038070678711,
+                0.06426930999755859,
+                0.06442803192138671,
+                0.06462873840332031,
+                0.06453247833251953,
+                0.06440857696533203,
+                0.0643737564086914,
+                0.06434611511230469,
+                0.06456114959716797,
+                0.06445260620117188,
+                0.0646983642578125,
+                0.06459699249267578,
+                0.06488063812255859,
+                0.06463897705078125,
+                0.06488985443115235,
+                0.0647352294921875,
+                0.0648089599609375,
+                0.06459187316894531,
+                0.06459699249267578,
+                0.06456320190429687,
+                0.06465331268310547,
+                0.06464307403564454,
+                0.06456114959716797,
+                0.06512537384033203,
+                0.06454402923583985,
+                0.06483455657958985,
+                0.0670730209350586,
+                0.06992588806152343,
+                0.06921830749511719,
+                0.06901350402832031,
+                0.06951628875732421,
+                0.06914559936523437,
+                0.06901964569091797,
+                0.06459187316894531,
+                0.06472908782958985,
+                0.0646266860961914,
+                0.06460108947753906,
+                0.06512230682373046,
+                0.06468915557861328,
+                0.06465740966796875,
+                0.06479666900634766,
+                0.06475059509277344,
+                0.06482841491699219,
+                0.06464335632324218,
+                0.0648622055053711,
+                0.06478336334228516,
+                0.06468915557861328,
+                0.06449766540527344,
+                0.06479257965087891,
+                0.06481305694580078,
+                0.06494412994384766,
+                0.06491340637207031,
+                0.06461644744873046
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 3939.4265252348255
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 4899.352576,
+            "max_vram": 8959.033344,
+            "max_reserved": 8466.202624,
+            "max_allocated": 8264.640512
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 16.6724708480835,
+            "stdev": 0,
+            "values": [
+                16.6724708480835
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 15.294673616377159
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.06538223861993529,
+            "stdev": 0.0002781621419081298,
+            "values": [
+                0.06762483215332031,
+                0.06704640197753907,
+                0.06520832061767579,
+                0.0651714859008789,
+                0.06487548828125,
+                0.06525151824951173,
+                0.06513337707519531,
+                0.06553600311279296,
+                0.06511615753173829,
+                0.06526668548583985,
+                0.06533529663085938,
+                0.06514073944091797,
+                0.06533222198486328,
+                0.06537324523925782,
+                0.06544786834716797,
+                0.06525647735595703,
+                0.06542230224609374,
+                0.06531702423095703,
+                0.06525628662109376,
+                0.06509670257568359,
+                0.06520320129394531,
+                0.06529142761230469,
+                0.06517948913574219,
+                0.06516326141357422,
+                0.06506803131103515,
+                0.06535167694091797,
+                0.06541311645507812,
+                0.06532710266113281,
+                0.06523494720458985,
+                0.06521036529541016,
+                0.06549811553955077,
+                0.0654366683959961,
+                0.06522879791259766,
+                0.06520524597167969,
+                0.06532608032226563,
+                0.06534758758544922,
+                0.06536192321777344,
+                0.06514892578125,
+                0.06570700836181641,
+                0.06523289489746094,
+                0.06521343994140626,
+                0.06533222198486328,
+                0.06529638671875,
+                0.06514073944091797,
+                0.06515312194824219,
+                0.0652225570678711,
+                0.06515814208984375,
+                0.0651878433227539,
+                0.0652195816040039,
+                0.06536089324951172,
+                0.06540493011474609,
+                0.0653322525024414,
+                0.0654345932006836,
+                0.06535577392578125,
+                0.065080322265625,
+                0.06531686401367187,
+                0.0653117446899414,
+                0.06521244812011719,
+                0.06517040252685546,
+                0.0655617904663086,
+                0.06513645172119141,
+                0.06590771484375,
+                0.06514892578125,
+                0.06524432373046875,
+                0.06522557067871093,
+                0.06532505798339844,
+                0.06511615753173829,
+                0.06538137817382812,
+                0.06524825286865234,
+                0.06594371032714844,
+                0.06514876556396484,
+                0.06517964935302735,
+                0.0651673583984375,
+                0.06523910522460938,
+                0.06521337890625,
+                0.06530150604248047,
+                0.06514073944091797,
+                0.06531993865966797,
+                0.06534349060058593,
+                0.06525440216064453,
+                0.06528205108642578,
+                0.06546636962890626,
+                0.0655994873046875,
+                0.06517657470703125,
+                0.0652195816040039,
+                0.06653453063964844,
+                0.06650150299072266,
+                0.06548684692382813,
+                0.06526566314697266,
+                0.06539263916015625,
+                0.06532505798339844,
+                0.06512230682373046,
+                0.06594662475585937,
+                0.06531686401367187,
+                0.06568959808349609,
+                0.0654543685913086,
+                0.06544764709472656,
+                0.06538137817382812,
+                0.0654202880859375,
+                0.06544588470458984,
+                0.06574899291992188,
+                0.06559539031982421,
+                0.06564147186279297,
+                0.0653803482055664,
+                0.0653486099243164,
+                0.06536396789550782,
+                0.0654172134399414,
+                0.06518374633789062,
+                0.06549298858642578,
+                0.06541110229492188,
+                0.06528816223144532,
+                0.06514585876464844,
+                0.06548172760009766,
+                0.06582784271240234,
+                0.0651673583984375,
+                0.0655626220703125,
+                0.06563123321533203,
+                0.06521753692626953,
+                0.06543154907226563,
+                0.06539798736572265,
+                0.06518761444091797,
+                0.06530252838134766,
+                0.06542233276367188,
+                0.06601830291748047,
+                0.06546227264404297,
+                0.0652933120727539,
+                0.065544189453125,
+                0.06536089324951172,
+                0.06534963226318359,
+                0.06544802856445313,
+                0.0656536636352539,
+                0.06505779266357421,
+                0.06537830352783203,
+                0.06531686401367187,
+                0.06549504089355469,
+                0.06539366149902344,
+                0.06528717041015625,
+                0.06548377227783203,
+                0.06567833709716797,
+                0.0653834228515625,
+                0.06545101165771484,
+                0.06531378936767578,
+                0.06521446228027344,
+                0.06529862213134766,
+                0.06516432189941407,
+                0.06522550201416015,
+                0.0649318389892578,
+                0.06545919799804688,
+                0.06518374633789062,
+                0.06527385711669922,
+                0.06544076538085937,
+                0.0652759017944336,
+                0.06514380645751953,
+                0.06511430358886719,
+                0.06576518249511719,
+                0.0652011489868164,
+                0.06531072235107421,
+                0.06541619110107422,
+                0.06532300567626953,
+                0.06530764770507813,
+                0.06522163391113281,
+                0.06577049255371094,
+                0.06522982025146484,
+                0.06542848205566407,
+                0.06550220489501953,
+                0.06544281768798828,
+                0.06558515167236328,
+                0.0654202880859375,
+                0.06538137817382812,
+                0.06533529663085938,
+                0.0655257568359375,
+                0.06571724700927735,
+                0.06536704254150391,
+                0.06547875213623047,
+                0.06547650909423829,
+                0.06550326538085938,
+                0.06534345245361328,
+                0.06527897644042968,
+                0.06557081604003906,
+                0.06523494720458985,
+                0.0654755859375,
+                0.06553497314453124,
+                0.0654366683959961,
+                0.06553804779052734,
+                0.06546022033691407,
+                0.06621705627441406,
+                0.0654591064453125,
+                0.06548070526123047,
+                0.06533529663085938,
+                0.06509056091308593,
+                0.06556877136230468,
+                0.06517759704589844,
+                0.0657940444946289,
+                0.06517759704589844,
+                0.0652779541015625,
+                0.06526054382324219,
+                0.06545919799804688,
+                0.06518886566162109,
+                0.06528307342529296,
+                0.06530662536621094,
+                0.06529126739501953,
+                0.06509670257568359,
+                0.06508748626708985,
+                0.06531788635253906,
+                0.0652759017944336,
+                0.06528428649902343,
+                0.06515487670898437,
+                0.06521446228027344,
+                0.06553804779052734,
+                0.06530764770507813,
+                0.06534963226318359,
+                0.06533529663085938,
+                0.06534246063232421,
+                0.06545439910888672,
+                0.06537490844726562,
+                0.0655494384765625,
+                0.06549593353271485,
+                0.0652779541015625,
+                0.06527999877929687,
+                0.06521548461914063,
+                0.06546534729003907,
+                0.06517686462402343,
+                0.06531452941894532,
+                0.06571532440185547,
+                0.06525325012207031,
+                0.0652042236328125,
+                0.0652390365600586,
+                0.06543782043457032,
+                0.06538841247558594,
+                0.06537216186523437,
+                0.06537216186523437,
+                0.06557183837890625,
+                0.06528409576416015,
+                0.06531584167480468,
+                0.06535987091064453,
+                0.0654366683959961,
+                0.06534143829345704,
+                0.06544691467285156,
+                0.06544179534912109,
+                0.06558924865722657,
+                0.06534143829345704,
+                0.06554637145996094,
+                0.06571302032470704,
+                0.06539574432373046,
+                0.06537315368652344,
+                0.06562201690673829,
+                0.06593536376953125,
+                0.06536089324951172,
+                0.06548992156982422,
+                0.06525567626953124,
+                0.06522137451171875,
+                0.06509260559082031,
+                0.06487757110595703,
+                0.06510415649414063,
+                0.0656944351196289
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 15.294673616377159
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}