Upload /Mistral-7B-v0.1-GPTQ-3bit/gptq3bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Mistral-7B-v0.1-GPTQ-3bit/gptq3bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,600 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 6528.4096,
+            "max_vram": 5339.348992,
+            "max_reserved": 4854.90688,
+            "max_allocated": 4578.492928
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.39816815772423375,
+            "stdev": 0.002303332450814792,
+            "values": [
+                0.4079566650390625,
+                0.4037754821777344,
+                0.3976529846191406,
+                0.39734170532226565,
+                0.3976304626464844,
+                0.3972608032226562,
+                0.39766220092773436,
+                0.39732736206054686,
+                0.3976365966796875,
+                0.3974502258300781,
+                0.397623291015625,
+                0.3975618591308594,
+                0.39749017333984377,
+                0.3972812805175781,
+                0.3980052490234375,
+                0.39739801025390625,
+                0.3976386413574219,
+                0.39732864379882815,
+                0.3978076171875,
+                0.3973171081542969,
+                0.3976079406738281,
+                0.39740826416015623,
+                0.3976837158203125,
+                0.39737548828125,
+                0.3977031555175781,
+                0.39744717407226565
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 642.9444319786676
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 6528.4096,
+            "max_vram": 5509.218304,
+            "max_reserved": 5024.776192,
+            "max_allocated": 4645.47584
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 7.997448039054872,
+            "stdev": 0.011532164573672343,
+            "values": [
+                8.008980203628544,
+                7.9859158744812
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 31.885171213958344
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.031362541329626945,
+            "stdev": 0.00022266343846452815,
+            "values": [
+                0.03446857452392578,
+                0.03357388687133789,
+                0.0317388801574707,
+                0.031476736068725586,
+                0.031341856002807617,
+                0.03127881622314453,
+                0.03127398490905762,
+                0.03122380828857422,
+                0.03141324806213379,
+                0.0313753604888916,
+                0.03141734313964844,
+                0.031510528564453126,
+                0.031320064544677735,
+                0.03136204719543457,
+                0.0313384952545166,
+                0.03127193641662598,
+                0.03147878456115723,
+                0.03139686393737793,
+                0.031322111129760744,
+                0.03127398490905762,
+                0.03136409568786621,
+                0.031386720657348634,
+                0.03139267158508301,
+                0.03139993667602539,
+                0.031507455825805664,
+                0.031403072357177736,
+                0.03159136009216309,
+                0.031221759796142577,
+                0.03141324806213379,
+                0.031438848495483396,
+                0.031303680419921875,
+                0.03132524871826172,
+                0.0313896312713623,
+                0.03132928085327148,
+                0.03139379119873047,
+                0.03129651260375976,
+                0.03141939163208008,
+                0.03134480094909668,
+                0.031377248764038086,
+                0.031251455307006834,
+                0.03129875183105469,
+                0.03133113670349121,
+                0.03150643157958984,
+                0.031287296295166016,
+                0.03125964736938477,
+                0.03139788818359375,
+                0.031318016052246093,
+                0.03136716842651367,
+                0.031335424423217774,
+                0.0313436164855957,
+                0.03133872032165527,
+                0.03145712089538574,
+                0.03146745681762695,
+                0.031425535202026365,
+                0.03130060768127441,
+                0.0315228157043457,
+                0.03136031913757324,
+                0.03146310424804687,
+                0.03136102485656738,
+                0.0315043830871582,
+                0.03148287963867188,
+                0.03150028800964356,
+                0.03138150405883789,
+                0.0314337272644043,
+                0.03141836738586426,
+                0.03127193641662598,
+                0.03128220748901367,
+                0.031411327362060544,
+                0.03129840087890625,
+                0.03170816040039062,
+                0.03136716842651367,
+                0.031513599395751955,
+                0.03140505599975586,
+                0.03146137619018555,
+                0.032069633483886716,
+                0.03126796722412109,
+                0.03132403182983398,
+                0.03146649551391602,
+                0.03158937644958496,
+                0.03140812873840332,
+                0.031415519714355467,
+                0.03149398422241211,
+                0.03149407958984375,
+                0.031460351943969726,
+                0.031251455307006834,
+                0.0321976318359375,
+                0.03282944107055664,
+                0.03155046463012695,
+                0.03142451286315918,
+                0.03140812873840332,
+                0.0312956485748291,
+                0.031407968521118164,
+                0.03128422355651855,
+                0.031389696121215824,
+                0.031284479141235354,
+                0.03131161689758301,
+                0.03123404884338379,
+                0.03134876823425293,
+                0.03133340835571289,
+                0.031415231704711916,
+                0.031324159622192385,
+                0.03162419128417969,
+                0.031281152725219724,
+                0.031275007247924806,
+                0.031244287490844725,
+                0.03135190391540527,
+                0.031262624740600584,
+                0.03137228775024414,
+                0.0315545597076416,
+                0.03155763244628906,
+                0.03116851234436035,
+                0.031388671875,
+                0.03126681518554687,
+                0.031396928787231444,
+                0.031176639556884767,
+                0.031287296295166016,
+                0.031140863418579103,
+                0.03134566307067871,
+                0.03121561622619629,
+                0.031336448669433595,
+                0.03115007972717285,
+                0.0313118724822998,
+                0.03118489646911621,
+                0.031243263244628908,
+                0.03118694305419922,
+                0.03134259223937988,
+                0.03121049690246582,
+                0.031245311737060546,
+                0.031341567993164066,
+                0.03139891242980957,
+                0.03127603149414063,
+                0.03134259223937988,
+                0.03122483253479004,
+                0.03135206413269043,
+                0.031340383529663084,
+                0.03131279945373535,
+                0.03131596755981445,
+                0.03138764762878418,
+                0.03132620811462403,
+                0.03133337593078613,
+                0.03139686393737793,
+                0.03133337593078613,
+                0.031340543746948245,
+                0.031357952117919925,
+                0.03127603149414063,
+                0.03143475151062012,
+                0.03127910423278808,
+                0.03133235168457031,
+                0.03147878456115723,
+                0.03141632080078125,
+                0.03133235168457031,
+                0.03143987274169922,
+                0.03141324806213379,
+                0.03139993667602539,
+                0.03141222381591797,
+                0.03143577575683594,
+                0.03145536041259766,
+                0.03138751983642578,
+                0.03131699180603027,
+                0.031410207748413084,
+                0.03143984031677246,
+                0.031509504318237305,
+                0.031493120193481446,
+                0.0314019832611084,
+                0.03142758369445801,
+                0.03140403175354004,
+                0.031509504318237305,
+                0.03140505599975586,
+                0.03126067161560059,
+                0.031459327697753905,
+                0.03138047981262207,
+                0.031475967407226565,
+                0.03166080093383789,
+                0.03151900863647461,
+                0.031423200607299806,
+                0.03152400016784668,
+                0.03150422477722168,
+                0.031306751251220705,
+                0.031263744354248044,
+                0.03133247947692871,
+                0.0314069766998291,
+                0.03144704055786133,
+                0.03139583969116211,
+                0.03138047981262207,
+                0.03150131225585937,
+                0.03136102485656738,
+                0.03141427230834961,
+                0.03136310386657715,
+                0.03143984031677246,
+                0.031285247802734374,
+                0.031275007247924806,
+                0.03136307144165039,
+                0.03119001579284668,
+                0.031410175323486327,
+                0.03136409568786621,
+                0.031406080245971676,
+                0.03126169586181641,
+                0.031321088790893556,
+                0.031363199234008786,
+                0.03139468765258789,
+                0.031643648147583005,
+                0.03158016014099121,
+                0.031337472915649416,
+                0.03135814476013184,
+                0.03142848014831543,
+                0.031386783599853516,
+                0.03139161682128906,
+                0.03135791969299316,
+                0.0313240966796875,
+                0.03141427230834961,
+                0.031226879119873048,
+                0.03134976005554199,
+                0.03148492813110351,
+                0.03134566307067871,
+                0.03146137619018555,
+                0.03140505599975586,
+                0.03142758369445801,
+                0.03146137619018555,
+                0.03133440017700195,
+                0.03137433624267578,
+                0.031355903625488284,
+                0.03141222381591797,
+                0.03132233619689941,
+                0.03125334358215332,
+                0.03135276794433594,
+                0.031307775497436525,
+                0.031369216918945314,
+                0.031408416748046876,
+                0.031371999740600585,
+                0.03139891242980957,
+                0.03137638473510742,
+                0.03127193641662598,
+                0.031735807418823245,
+                0.03167231941223145,
+                0.03142348861694336,
+                0.03138252830505371,
+                0.031425535202026365,
+                0.03142758369445801,
+                0.03142348861694336,
+                0.03138252830505371,
+                0.03132825660705566,
+                0.0314583683013916,
+                0.03131385612487793,
+                0.03139993667602539,
+                0.031252479553222655,
+                0.031508480072021484,
+                0.03144601631164551,
+                0.031152128219604492,
+                0.031351808547973634,
+                0.031274208068847655,
+                0.03148784065246582,
+                0.031200191497802735,
+                0.03102003288269043,
+                0.03141836738586426,
+                0.0311592960357666,
+                0.03169779205322266,
+                0.031855615615844726,
+                0.03142451286315918,
+                0.03126278305053711,
+                0.0314080638885498,
+                0.03131391906738281,
+                0.03133440017700195,
+                0.031322111129760744,
+                0.031308799743652346,
+                0.031287296295166016,
+                0.03132620811462403,
+                0.03136102485656738,
+                0.03130572891235352,
+                0.03135398483276367,
+                0.0312544002532959,
+                0.031185920715332032,
+                0.03134873580932617,
+                0.03125971221923828,
+                0.031224767684936525,
+                0.03123302459716797,
+                0.031328384399414065,
+                0.03125747108459473,
+                0.03134259223937988,
+                0.03126169586181641,
+                0.031220735549926756,
+                0.03127910423278808,
+                0.03136614418029785,
+                0.031188287734985352,
+                0.0312736644744873,
+                0.031213855743408202,
+                0.03124502372741699,
+                0.03127004814147949,
+                0.03135574340820312,
+                0.0313384952545166,
+                0.0313436164855957,
+                0.031310880661010745,
+                0.031335391998291016,
+                0.03117465591430664,
+                0.03127398490905762,
+                0.031199264526367187,
+                0.031217632293701173,
+                0.031236095428466795,
+                0.03128832054138184,
+                0.03130982398986817,
+                0.031252479553222655,
+                0.03123302459716797,
+                0.031098880767822266,
+                0.03121664047241211,
+                0.0311297607421875,
+                0.03125436782836914,
+                0.031245311737060546,
+                0.031270912170410156,
+                0.03126272010803223,
+                0.03128217506408691,
+                0.03120025634765625,
+                0.03135603141784668,
+                0.031210367202758788,
+                0.03126681518554687,
+                0.03134668731689453,
+                0.03140412712097168,
+                0.03124857521057129,
+                0.031338207244873045,
+                0.031169536590576172,
+                0.03133337593078613,
+                0.031269887924194335,
+                0.031169536590576172,
+                0.031245344161987303,
+                0.031178720474243166,
+                0.03119820785522461,
+                0.031110143661499022,
+                0.031246591567993164,
+                0.03128499221801758,
+                0.03135385513305664,
+                0.031144960403442383,
+                0.03129446411132813,
+                0.031116287231445314,
+                0.03114195251464844,
+                0.0313691520690918,
+                0.03127398490905762,
+                0.03124940872192383,
+                0.03130473518371582,
+                0.031185152053833008,
+                0.031396703720092775,
+                0.031230016708374022,
+                0.031101951599121092,
+                0.031221567153930666,
+                0.03125657653808594,
+                0.03130060768127441,
+                0.03126790428161621,
+                0.03112851142883301,
+                0.03129548835754394,
+                0.031357952117919925,
+                0.03137638473510742,
+                0.031251455307006834,
+                0.03126188850402832,
+                0.031183679580688475,
+                0.031308799743652346,
+                0.03121971130371094,
+                0.03126067161560059,
+                0.031224063873291016,
+                0.031221759796142577,
+                0.03123583984375,
+                0.031263744354248044,
+                0.03132928085327148,
+                0.03136511993408203,
+                0.03130982398986817,
+                0.03128323173522949,
+                0.03138147163391113,
+                0.03256329727172851,
+                0.0315380802154541,
+                0.03126684761047363,
+                0.031351776123046876,
+                0.031340543746948245,
+                0.031442943572998046,
+                0.03129958343505859,
+                0.03134687995910645,
+                0.03141305541992188,
+                0.03139174461364746,
+                0.03133337593078613,
+                0.031356927871704104,
+                0.0311592960357666,
+                0.031313024520874026,
+                0.03136297607421875,
+                0.03155475234985351,
+                0.03135977554321289,
+                0.031410175323486327,
+                0.031297536849975584,
+                0.03140403175354004,
+                0.03164672088623047,
+                0.03158016014099121,
+                0.031441919326782225,
+                0.03142451286315918,
+                0.03139276885986328,
+                0.031437824249267575,
+                0.0314839038848877,
+                0.03139072036743164,
+                0.03134771156311035,
+                0.03135487937927246,
+                0.031336448669433595,
+                0.03139174461364746,
+                0.03136307144165039,
+                0.03140300750732422,
+                0.03140096092224121,
+                0.03138457679748535,
+                0.031433984756469725,
+                0.03143571281433105,
+                0.031265600204467774,
+                0.03135078430175781,
+                0.031508480072021484,
+                0.03157939147949219,
+                0.031373056411743164,
+                0.031290367126464845,
+                0.03129958343505859,
+                0.03126272010803223,
+                0.03142963218688965,
+                0.03121366310119629,
+                0.031301536560058595,
+                0.03140505599975586,
+                0.031226911544799806,
+                0.031200223922729493,
+                0.031204351425170897,
+                0.03125862312316895,
+                0.03127705574035645,
+                0.031307775497436525,
+                0.03132825660705566,
+                0.03139993667602539,
+                0.03129958343505859,
+                0.03133235168457031,
+                0.031232032775878906,
+                0.031296480178833005,
+                0.03126383972167969,
+                0.031196063995361328,
+                0.03154867172241211,
+                0.03141606330871582,
+                0.03137433624267578,
+                0.031205440521240236,
+                0.031250591278076174,
+                0.03121027183532715,
+                0.03121664047241211,
+                0.03125555229187012,
+                0.031162368774414063,
+                0.031236095428466795,
+                0.03131391906738281,
+                0.03135206413269043,
+                0.031340288162231444,
+                0.0313702392578125,
+                0.03138991928100586,
+                0.03132803153991699,
+                0.031247360229492187,
+                0.03133235168457031,
+                0.031197183609008788,
+                0.031325183868408206,
+                0.03130572891235352,
+                0.03127705574035645,
+                0.031308799743652346,
+                0.03127705574035645,
+                0.031220735549926756,
+                0.031302656173706055,
+                0.031346815109252926,
+                0.0313752326965332,
+                0.03158323287963867,
+                0.03134668731689453,
+                0.03129548835754394,
+                0.03143987274169922,
+                0.0312740478515625,
+                0.031430591583251954,
+                0.031322111129760744,
+                0.03137843132019043,
+                0.031437824249267575,
+                0.03128934478759766,
+                0.03131494331359863,
+                0.0313753604888916,
+                0.03129964828491211,
+                0.03131385612487793,
+                0.03138457679748535,
+                0.031287296295166016,
+                0.031422464370727536,
+                0.031325183868408206,
+                0.03135385513305664,
+                0.031198368072509766,
+                0.031358816146850585,
+                0.03139174461364746,
+                0.031476736068725586,
+                0.031337472915649416,
+                0.03129343986511231,
+                0.03132825660705566,
+                0.03129548835754394,
+                0.03139072036743164,
+                0.03125657653808594,
+                0.03133030319213867,
+                0.031285247802734374,
+                0.03133440017700195,
+                0.03160985565185547,
+                0.03133440017700195,
+                0.03136614418029785,
+                0.03136307144165039,
+                0.03127516746520996,
+                0.03143561553955078,
+                0.03126169586181641,
+                0.0313275203704834,
+                0.03133001518249512,
+                0.03125555229187012,
+                0.03126905632019043,
+                0.031284032821655275,
+                0.0312729606628418,
+                0.03139174461364746,
+                0.031441055297851565,
+                0.03129756736755371,
+                0.031143743515014647,
+                0.031213567733764647,
+                0.031263744354248044,
+                0.031143936157226562,
+                0.03125760078430176,
+                0.031321088790893556,
+                0.031165536880493165
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 31.885171213958348
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}