{
  "student_ckpt": "/root/bitnet1/ckpt/v29_50M_50K_last.pt",
  "student_config": {
    "vocab_size": 128,
    "d_model": 768,
    "n_layers": 10,
    "n_heads": 12,
    "d_ff": 1280,
    "seq_len": 256
  },
  "student_step": 50000,
  "student_val_bpc": 1.2349195671077682,
  "timestamp": "2026-04-22 17:05:59",
  "layer_ablation": {
    "baseline_bpc": 1.2372383790923875,
    "per_layer": [
      {
        "layer": 0,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 2.658024141629142,
        "delta_bpc": 1.4207857625367544
      },
      {
        "layer": 1,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.791177114375058,
        "delta_bpc": 0.5539387352826706
      },
      {
        "layer": 2,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.40893652335544,
        "delta_bpc": 0.17169814426305252
      },
      {
        "layer": 3,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.3546692810857077,
        "delta_bpc": 0.11743090199332018
      },
      {
        "layer": 4,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.4076403331115497,
        "delta_bpc": 0.17040195401916214
      },
      {
        "layer": 5,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.3853021601250528,
        "delta_bpc": 0.14806378103266526
      },
      {
        "layer": 6,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.4080221517947302,
        "delta_bpc": 0.17078377270234268
      },
      {
        "layer": 7,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.5111466640773288,
        "delta_bpc": 0.27390828498494124
      },
      {
        "layer": 8,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.5338596560531164,
        "delta_bpc": 0.2966212769607288
      },
      {
        "layer": 9,
        "baseline_bpc": 1.2372383790923875,
        "ablated_bpc": 1.6496860416056456,
        "delta_bpc": 0.41244766251325804
      }
    ]
  },
  "weight_saturation": [
    {
      "name": "out_codebook",
      "shape": [
        128,
        768
      ],
      "n": 98304,
      "mean": 0.32484379410743713,
      "median": 0.26955974102020264,
      "q10": 0.022471295669674873,
      "q90": 0.6936785578727722,
      "q99": 1.0008821487426758,
      "frac_below_0.01": 0.077606201171875,
      "frac_below_0.05": 0.1428019255399704,
      "frac_above_0.5": 0.2465413510799408,
      "max": 2.1756932735443115
    },
    {
      "name": "embed.weight",
      "shape": [
        128,
        768
      ],
      "n": 98304,
      "mean": 0.025270771235227585,
      "median": 0.01747666671872139,
      "q10": 0.0027019940316677094,
      "q90": 0.06000998988747597,
      "q99": 0.1124815121293068,
      "frac_below_0.01": 0.322998046875,
      "frac_below_0.05": 0.8575338125228882,
      "frac_above_0.5": 0.0,
      "max": 0.20307880640029907
    },
    {
      "name": "blocks.0.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.020984457805752754,
      "median": 0.01630321331322193,
      "q10": 0.002880966058000922,
      "q90": 0.04540819674730301,
      "q99": 0.08269572257995605,
      "frac_below_0.01": 0.327667236328125,
      "frac_below_0.05": 0.9232143759727478,
      "frac_above_0.5": 0.0,
      "max": 0.17969276010990143
    },
    {
      "name": "blocks.0.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.021982185542583466,
      "median": 0.016868969425559044,
      "q10": 0.0029649189673364162,
      "q90": 0.04804832115769386,
      "q99": 0.08712130039930344,
      "frac_below_0.01": 0.318261057138443,
      "frac_below_0.05": 0.9096696972846985,
      "frac_above_0.5": 0.0,
      "max": 0.17866352200508118
    },
    {
      "name": "blocks.0.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.02966231107711792,
      "median": 0.02414115145802498,
      "q10": 0.0041543589904904366,
      "q90": 0.0629270151257515,
      "q99": 0.10377822816371918,
      "frac_below_0.01": 0.2300957590341568,
      "frac_below_0.05": 0.8167402744293213,
      "frac_above_0.5": 0.0,
      "max": 0.21888598799705505
    },
    {
      "name": "blocks.0.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.03943490982055664,
      "median": 0.024108367040753365,
      "q10": 0.0036507798358798027,
      "q90": 0.09731841832399368,
      "q99": 0.2101808488368988,
      "frac_below_0.01": 0.2511308491230011,
      "frac_below_0.05": 0.7431420087814331,
      "frac_above_0.5": 3.3908420391526306e-06,
      "max": 0.5122478604316711
    },
    {
      "name": "blocks.0.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.029497429728507996,
      "median": 0.024814119562506676,
      "q10": 0.004512307234108448,
      "q90": 0.06109948456287384,
      "q99": 0.09630865603685379,
      "frac_below_0.01": 0.21697694063186646,
      "frac_below_0.05": 0.8223521113395691,
      "frac_above_0.5": 0.0,
      "max": 0.17949974536895752
    },
    {
      "name": "blocks.0.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.029491588473320007,
      "median": 0.024759948253631592,
      "q10": 0.004544392228126526,
      "q90": 0.06112118810415268,
      "q99": 0.09623175859451294,
      "frac_below_0.01": 0.21637777984142303,
      "frac_below_0.05": 0.8225098252296448,
      "frac_above_0.5": 0.0,
      "max": 0.17936795949935913
    },
    {
      "name": "blocks.0.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.030015649273991585,
      "median": 0.025115979835391045,
      "q10": 0.004507269244641066,
      "q90": 0.06252331286668777,
      "q99": 0.09870631247758865,
      "frac_below_0.01": 0.21663717925548553,
      "frac_below_0.05": 0.8136902451515198,
      "frac_above_0.5": 0.0,
      "max": 0.19937986135482788
    },
    {
      "name": "blocks.1.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.021738752722740173,
      "median": 0.016651637852191925,
      "q10": 0.0030065574683248997,
      "q90": 0.04737270250916481,
      "q99": 0.0873987078666687,
      "frac_below_0.01": 0.3206566870212555,
      "frac_below_0.05": 0.9128333330154419,
      "frac_above_0.5": 0.0,
      "max": 0.20024967193603516
    },
    {
      "name": "blocks.1.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.022124914452433586,
      "median": 0.016716480255126953,
      "q10": 0.0029788485262542963,
      "q90": 0.0486590750515461,
      "q99": 0.09064910560846329,
      "frac_below_0.01": 0.3203616738319397,
      "frac_below_0.05": 0.9063364863395691,
      "frac_above_0.5": 0.0,
      "max": 0.21088646352291107
    },
    {
      "name": "blocks.1.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.032271161675453186,
      "median": 0.02542409859597683,
      "q10": 0.004067442379891872,
      "q90": 0.07015787810087204,
      "q99": 0.11874839663505554,
      "frac_below_0.01": 0.2300516813993454,
      "frac_below_0.05": 0.7786475419998169,
      "frac_above_0.5": 0.0,
      "max": 0.29738032817840576
    },
    {
      "name": "blocks.1.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.0365101583302021,
      "median": 0.026378890499472618,
      "q10": 0.004064246546477079,
      "q90": 0.08268115669488907,
      "q99": 0.15872563421726227,
      "frac_below_0.01": 0.2282664030790329,
      "frac_below_0.05": 0.7426249384880066,
      "frac_above_0.5": 0.0,
      "max": 0.32939204573631287
    },
    {
      "name": "blocks.1.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.030397042632102966,
      "median": 0.0249449722468853,
      "q10": 0.0042805117554962635,
      "q90": 0.06440477073192596,
      "q99": 0.10299010574817657,
      "frac_below_0.01": 0.22425639629364014,
      "frac_below_0.05": 0.8044993281364441,
      "frac_above_0.5": 0.0,
      "max": 0.20056255161762238
    },
    {
      "name": "blocks.1.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03043045848608017,
      "median": 0.02498842217028141,
      "q10": 0.00427834689617157,
      "q90": 0.06437431275844574,
      "q99": 0.1029980257153511,
      "frac_below_0.01": 0.22370098531246185,
      "frac_below_0.05": 0.8039296865463257,
      "frac_above_0.5": 0.0,
      "max": 0.20558297634124756
    },
    {
      "name": "blocks.1.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.030961716547608376,
      "median": 0.025217225775122643,
      "q10": 0.004261382389813662,
      "q90": 0.06597873568534851,
      "q99": 0.10618423670530319,
      "frac_below_0.01": 0.22413738071918488,
      "frac_below_0.05": 0.7958465814590454,
      "frac_above_0.5": 0.0,
      "max": 0.20832186937332153
    },
    {
      "name": "blocks.2.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.021832674741744995,
      "median": 0.016725117340683937,
      "q10": 0.003007261548191309,
      "q90": 0.04768085852265358,
      "q99": 0.08757057785987854,
      "frac_below_0.01": 0.31890869140625,
      "frac_below_0.05": 0.9113871455192566,
      "frac_above_0.5": 0.0,
      "max": 0.19008490443229675
    },
    {
      "name": "blocks.2.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.02204788476228714,
      "median": 0.01679176464676857,
      "q10": 0.003010682761669159,
      "q90": 0.04842023551464081,
      "q99": 0.08911668509244919,
      "frac_below_0.01": 0.3183034360408783,
      "frac_below_0.05": 0.907684326171875,
      "frac_above_0.5": 0.0,
      "max": 0.19714821875095367
    },
    {
      "name": "blocks.2.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.03241870179772377,
      "median": 0.025510499253869057,
      "q10": 0.0039276741445064545,
      "q90": 0.07076237350702286,
      "q99": 0.11824272572994232,
      "frac_below_0.01": 0.2332882434129715,
      "frac_below_0.05": 0.7741004228591919,
      "frac_above_0.5": 0.0,
      "max": 0.25875869393348694
    },
    {
      "name": "blocks.2.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.0360061414539814,
      "median": 0.026566587388515472,
      "q10": 0.003982395865023136,
      "q90": 0.08094409108161926,
      "q99": 0.15047675371170044,
      "frac_below_0.01": 0.2296006977558136,
      "frac_below_0.05": 0.7429623007774353,
      "frac_above_0.5": 0.0,
      "max": 0.3236503005027771
    },
    {
      "name": "blocks.2.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.030454641208052635,
      "median": 0.024827929213643074,
      "q10": 0.004183255601674318,
      "q90": 0.06483904272317886,
      "q99": 0.10405371338129044,
      "frac_below_0.01": 0.2272491604089737,
      "frac_below_0.05": 0.8031179308891296,
      "frac_above_0.5": 0.0,
      "max": 0.21293406188488007
    },
    {
      "name": "blocks.2.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.030473940074443817,
      "median": 0.024836573749780655,
      "q10": 0.0041793277487158775,
      "q90": 0.0648508369922638,
      "q99": 0.10394246876239777,
      "frac_below_0.01": 0.22719117999076843,
      "frac_below_0.05": 0.8021494746208191,
      "frac_above_0.5": 0.0,
      "max": 0.20337601006031036
    },
    {
      "name": "blocks.2.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.03145662695169449,
      "median": 0.025447478517889977,
      "q10": 0.004232366569340229,
      "q90": 0.0673917829990387,
      "q99": 0.10850941389799118,
      "frac_below_0.01": 0.22418315708637238,
      "frac_below_0.05": 0.7883524894714355,
      "frac_above_0.5": 0.0,
      "max": 0.2242489606142044
    },
    {
      "name": "blocks.3.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.021820461377501488,
      "median": 0.016704333946108818,
      "q10": 0.0030075714457780123,
      "q90": 0.047725576907396317,
      "q99": 0.08704143017530441,
      "frac_below_0.01": 0.3202582597732544,
      "frac_below_0.05": 0.9108836054801941,
      "frac_above_0.5": 0.0,
      "max": 0.1996636688709259
    },
    {
      "name": "blocks.3.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.02202785760164261,
      "median": 0.016765017062425613,
      "q10": 0.003018269781023264,
      "q90": 0.04830968379974365,
      "q99": 0.08893266320228577,
      "frac_below_0.01": 0.319023996591568,
      "frac_below_0.05": 0.9081844687461853,
      "frac_above_0.5": 0.0,
      "max": 0.2032247930765152
    },
    {
      "name": "blocks.3.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.03259618952870369,
      "median": 0.02546798251569271,
      "q10": 0.0037475833669304848,
      "q90": 0.07166170328855515,
      "q99": 0.1207074448466301,
      "frac_below_0.01": 0.2383185476064682,
      "frac_below_0.05": 0.7702789306640625,
      "frac_above_0.5": 0.0,
      "max": 0.308125764131546
    },
    {
      "name": "blocks.3.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.036724548786878586,
      "median": 0.026713047176599503,
      "q10": 0.0038133268244564533,
      "q90": 0.08330091834068298,
      "q99": 0.1569368839263916,
      "frac_below_0.01": 0.233211949467659,
      "frac_below_0.05": 0.7358195185661316,
      "frac_above_0.5": 0.0,
      "max": 0.35014036297798157
    },
    {
      "name": "blocks.3.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03041911870241165,
      "median": 0.024767549708485603,
      "q10": 0.004097540397197008,
      "q90": 0.06491811573505402,
      "q99": 0.10423436015844345,
      "frac_below_0.01": 0.23008118569850922,
      "frac_below_0.05": 0.8022247552871704,
      "frac_above_0.5": 0.0,
      "max": 0.2128787487745285
    },
    {
      "name": "blocks.3.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.030419766902923584,
      "median": 0.024740705266594887,
      "q10": 0.004079720471054316,
      "q90": 0.06493699550628662,
      "q99": 0.10483106225728989,
      "frac_below_0.01": 0.2303914576768875,
      "frac_below_0.05": 0.802680492401123,
      "frac_above_0.5": 0.0,
      "max": 0.21313220262527466
    },
    {
      "name": "blocks.3.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.031688056886196136,
      "median": 0.025530001148581505,
      "q10": 0.004116586875170469,
      "q90": 0.06815119832754135,
      "q99": 0.11011636257171631,
      "frac_below_0.01": 0.22733664512634277,
      "frac_below_0.05": 0.7837514877319336,
      "frac_above_0.5": 0.0,
      "max": 0.22840875387191772
    },
    {
      "name": "blocks.4.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.021806135773658752,
      "median": 0.016604429110884666,
      "q10": 0.0029621128924191,
      "q90": 0.047807518392801285,
      "q99": 0.0883331298828125,
      "frac_below_0.01": 0.3223758339881897,
      "frac_below_0.05": 0.9105224609375,
      "frac_above_0.5": 0.0,
      "max": 0.19498515129089355
    },
    {
      "name": "blocks.4.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.0219953004270792,
      "median": 0.01666717790067196,
      "q10": 0.002971069421619177,
      "q90": 0.04836598038673401,
      "q99": 0.08983262628316879,
      "frac_below_0.01": 0.320641428232193,
      "frac_below_0.05": 0.9077809453010559,
      "frac_above_0.5": 0.0,
      "max": 0.19498971104621887
    },
    {
      "name": "blocks.4.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.03282567113637924,
      "median": 0.025203978642821312,
      "q10": 0.0033933515660464764,
      "q90": 0.07318802922964096,
      "q99": 0.12469246238470078,
      "frac_below_0.01": 0.2480655312538147,
      "frac_below_0.05": 0.7664320468902588,
      "frac_above_0.5": 0.0,
      "max": 0.28078585863113403
    },
    {
      "name": "blocks.4.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.03867517039179802,
      "median": 0.026929715648293495,
      "q10": 0.0034530421253293753,
      "q90": 0.09060992300510406,
      "q99": 0.1718837320804596,
      "frac_below_0.01": 0.2434438019990921,
      "frac_below_0.05": 0.7186652421951294,
      "frac_above_0.5": 0.0,
      "max": 0.3775966465473175
    },
    {
      "name": "blocks.4.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.030496973544359207,
      "median": 0.024575216695666313,
      "q10": 0.003859907388687134,
      "q90": 0.06571313738822937,
      "q99": 0.10632941871881485,
      "frac_below_0.01": 0.23670147359371185,
      "frac_below_0.05": 0.7992218136787415,
      "frac_above_0.5": 0.0,
      "max": 0.236494243144989
    },
    {
      "name": "blocks.4.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03050924651324749,
      "median": 0.02466735430061817,
      "q10": 0.003866195445880294,
      "q90": 0.06563732028007507,
      "q99": 0.10621653497219086,
      "frac_below_0.01": 0.23652751743793488,
      "frac_below_0.05": 0.799262523651123,
      "frac_above_0.5": 0.0,
      "max": 0.22307677567005157
    },
    {
      "name": "blocks.4.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.03216877207159996,
      "median": 0.025634581223130226,
      "q10": 0.003866716753691435,
      "q90": 0.07008880376815796,
      "q99": 0.11355020850896835,
      "frac_below_0.01": 0.23405660688877106,
      "frac_below_0.05": 0.7747518420219421,
      "frac_above_0.5": 0.0,
      "max": 0.2466614544391632
    },
    {
      "name": "blocks.5.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.021769991144537926,
      "median": 0.016515638679265976,
      "q10": 0.0029437255579978228,
      "q90": 0.04784300550818443,
      "q99": 0.08883869647979736,
      "frac_below_0.01": 0.3243001401424408,
      "frac_below_0.05": 0.9103173017501831,
      "frac_above_0.5": 0.0,
      "max": 0.1770746260881424
    },
    {
      "name": "blocks.5.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.02204853482544422,
      "median": 0.016590725630521774,
      "q10": 0.002919677644968033,
      "q90": 0.04862433299422264,
      "q99": 0.09129194915294647,
      "frac_below_0.01": 0.3242102861404419,
      "frac_below_0.05": 0.9064585566520691,
      "frac_above_0.5": 0.0,
      "max": 0.2087617963552475
    },
    {
      "name": "blocks.5.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.0333058126270771,
      "median": 0.025002961978316307,
      "q10": 0.002964397193863988,
      "q90": 0.07551417499780655,
      "q99": 0.13100847601890564,
      "frac_below_0.01": 0.2606692910194397,
      "frac_below_0.05": 0.7589297890663147,
      "frac_above_0.5": 0.0,
      "max": 0.3152570426464081
    },
    {
      "name": "blocks.5.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.041803035885095596,
      "median": 0.02722357213497162,
      "q10": 0.0029212224762886763,
      "q90": 0.10118373483419418,
      "q99": 0.20479606091976166,
      "frac_below_0.01": 0.2576039731502533,
      "frac_below_0.05": 0.6999596357345581,
      "frac_above_0.5": 5.086263172415784e-06,
      "max": 0.5692521929740906
    },
    {
      "name": "blocks.5.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.030752409249544144,
      "median": 0.0244681965559721,
      "q10": 0.0035621533170342445,
      "q90": 0.06708699464797974,
      "q99": 0.10938585549592972,
      "frac_below_0.01": 0.2462412714958191,
      "frac_below_0.05": 0.7924683094024658,
      "frac_above_0.5": 0.0,
      "max": 0.2319944053888321
    },
    {
      "name": "blocks.5.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.030752165243029594,
      "median": 0.024411072954535484,
      "q10": 0.0035458123311400414,
      "q90": 0.06726153194904327,
      "q99": 0.10948219895362854,
      "frac_below_0.01": 0.24592998623847961,
      "frac_below_0.05": 0.7923950552940369,
      "frac_above_0.5": 0.0,
      "max": 0.228058323264122
    },
    {
      "name": "blocks.5.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.03299623355269432,
      "median": 0.02572372555732727,
      "q10": 0.003507574088871479,
      "q90": 0.07318581640720367,
      "q99": 0.12024357914924622,
      "frac_below_0.01": 0.24329835176467896,
      "frac_below_0.05": 0.7614848017692566,
      "frac_above_0.5": 0.0,
      "max": 0.27965444326400757
    },
    {
      "name": "blocks.6.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.0215331818908453,
      "median": 0.016295911744236946,
      "q10": 0.0028458735905587673,
      "q90": 0.04729600250720978,
      "q99": 0.08885350078344345,
      "frac_below_0.01": 0.329857736825943,
      "frac_below_0.05": 0.9127739667892456,
      "frac_above_0.5": 0.0,
      "max": 0.19640463590621948
    },
    {
      "name": "blocks.6.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.02174237370491028,
      "median": 0.01627686247229576,
      "q10": 0.0028480947948992252,
      "q90": 0.047940269112586975,
      "q99": 0.0914376974105835,
      "frac_below_0.01": 0.3300510048866272,
      "frac_below_0.05": 0.9095560908317566,
      "frac_above_0.5": 0.0,
      "max": 0.21430690586566925
    },
    {
      "name": "blocks.6.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.03459187224507332,
      "median": 0.02503044530749321,
      "q10": 0.0024408106692135334,
      "q90": 0.08047246932983398,
      "q99": 0.14331185817718506,
      "frac_below_0.01": 0.2751024067401886,
      "frac_below_0.05": 0.7422892451286316,
      "frac_above_0.5": 0.0,
      "max": 0.3117149770259857
    },
    {
      "name": "blocks.6.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.04699027165770531,
      "median": 0.02844177559018135,
      "q10": 0.0022870029788464308,
      "q90": 0.1200011819601059,
      "q99": 0.23704038560390472,
      "frac_below_0.01": 0.2714368999004364,
      "frac_below_0.05": 0.6671345829963684,
      "frac_above_0.5": 3.3908420391526306e-06,
      "max": 0.518464207649231
    },
    {
      "name": "blocks.6.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03131778910756111,
      "median": 0.02425358071923256,
      "q10": 0.0030630475375801325,
      "q90": 0.07003403455018997,
      "q99": 0.11553935706615448,
      "frac_below_0.01": 0.2605092525482178,
      "frac_below_0.05": 0.7811055779457092,
      "frac_above_0.5": 0.0,
      "max": 0.24002909660339355
    },
    {
      "name": "blocks.6.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.031340092420578,
      "median": 0.024272354319691658,
      "q10": 0.0030343779362738132,
      "q90": 0.07006758451461792,
      "q99": 0.11566709727048874,
      "frac_below_0.01": 0.2608998715877533,
      "frac_below_0.05": 0.780102550983429,
      "frac_above_0.5": 0.0,
      "max": 0.2660861313343048
    },
    {
      "name": "blocks.6.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.034312065690755844,
      "median": 0.025947941467165947,
      "q10": 0.0029694088734686375,
      "q90": 0.07810769975185394,
      "q99": 0.13014021515846252,
      "frac_below_0.01": 0.2571146786212921,
      "frac_below_0.05": 0.7417826652526855,
      "frac_above_0.5": 0.0,
      "max": 0.26336076855659485
    },
    {
      "name": "blocks.7.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.022099530324339867,
      "median": 0.016269685700535774,
      "q10": 0.0027676254976540804,
      "q90": 0.04940343275666237,
      "q99": 0.0952984169125557,
      "frac_below_0.01": 0.3332875669002533,
      "frac_below_0.05": 0.9026760458946228,
      "frac_above_0.5": 0.0,
      "max": 0.21799291670322418
    },
    {
      "name": "blocks.7.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.022439096122980118,
      "median": 0.016338741406798363,
      "q10": 0.002755739027634263,
      "q90": 0.050511594861745834,
      "q99": 0.09869782626628876,
      "frac_below_0.01": 0.3318294882774353,
      "frac_below_0.05": 0.8977118730545044,
      "frac_above_0.5": 0.0,
      "max": 0.219570130109787
    },
    {
      "name": "blocks.7.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.03680150955915451,
      "median": 0.024955160915851593,
      "q10": 0.0015987252118065953,
      "q90": 0.08944220095872879,
      "q99": 0.16461941599845886,
      "frac_below_0.01": 0.3004574179649353,
      "frac_below_0.05": 0.7197520136833191,
      "frac_above_0.5": 0.0,
      "max": 0.3584654927253723
    },
    {
      "name": "blocks.7.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.05452443286776543,
      "median": 0.029793892055749893,
      "q10": 0.0014873046893626451,
      "q90": 0.14702990651130676,
      "q99": 0.28597983717918396,
      "frac_below_0.01": 0.2919921875,
      "frac_below_0.05": 0.6334109902381897,
      "frac_above_0.5": 8.646646892884746e-05,
      "max": 0.7674525380134583
    },
    {
      "name": "blocks.7.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03266124054789543,
      "median": 0.024235490709543228,
      "q10": 0.0022881708573549986,
      "q90": 0.07565723359584808,
      "q99": 0.1269170194864273,
      "frac_below_0.01": 0.2822510004043579,
      "frac_below_0.05": 0.757544994354248,
      "frac_above_0.5": 0.0,
      "max": 0.28104186058044434
    },
    {
      "name": "blocks.7.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.032623130828142166,
      "median": 0.0241586584597826,
      "q10": 0.002281878376379609,
      "q90": 0.07564595341682434,
      "q99": 0.1268070936203003,
      "frac_below_0.01": 0.281890869140625,
      "frac_below_0.05": 0.7584859728813171,
      "frac_above_0.5": 0.0,
      "max": 0.267304927110672
    },
    {
      "name": "blocks.7.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.036399319767951965,
      "median": 0.026400091126561165,
      "q10": 0.0022754990495741367,
      "q90": 0.08571262657642365,
      "q99": 0.14497342705726624,
      "frac_below_0.01": 0.27541911602020264,
      "frac_below_0.05": 0.7168355584144592,
      "frac_above_0.5": 0.0,
      "max": 0.30715522170066833
    },
    {
      "name": "blocks.8.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.02204364724457264,
      "median": 0.01618419960141182,
      "q10": 0.0027065007016062737,
      "q90": 0.04928082227706909,
      "q99": 0.09612085670232773,
      "frac_below_0.01": 0.3353847861289978,
      "frac_below_0.05": 0.9031490683555603,
      "frac_above_0.5": 0.0,
      "max": 0.2505124807357788
    },
    {
      "name": "blocks.8.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.022502297535538673,
      "median": 0.01621812768280506,
      "q10": 0.0026969979517161846,
      "q90": 0.05086328089237213,
      "q99": 0.1007092148065567,
      "frac_below_0.01": 0.3353763222694397,
      "frac_below_0.05": 0.8963436484336853,
      "frac_above_0.5": 0.0,
      "max": 0.23622968792915344
    },
    {
      "name": "blocks.8.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.03888942673802376,
      "median": 0.0260167196393013,
      "q10": 0.001105984323658049,
      "q90": 0.0957922413945198,
      "q99": 0.17475828528404236,
      "frac_below_0.01": 0.3079240620136261,
      "frac_below_0.05": 0.6965315341949463,
      "frac_above_0.5": 0.0,
      "max": 0.49975377321243286
    },
    {
      "name": "blocks.8.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.058232564479112625,
      "median": 0.03141973167657852,
      "q10": 0.0010921643115580082,
      "q90": 0.15505672991275787,
      "q99": 0.33220720291137695,
      "frac_below_0.01": 0.2909308671951294,
      "frac_below_0.05": 0.6194356083869934,
      "frac_above_0.5": 0.0006086561479605734,
      "max": 0.8799290657043457
    },
    {
      "name": "blocks.8.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03446163237094879,
      "median": 0.024365149438381195,
      "q10": 0.001547358464449644,
      "q90": 0.08275697380304337,
      "q99": 0.14115624129772186,
      "frac_below_0.01": 0.30199891328811646,
      "frac_below_0.05": 0.733826756477356,
      "frac_above_0.5": 0.0,
      "max": 0.2949505150318146
    },
    {
      "name": "blocks.8.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03450019657611847,
      "median": 0.024399472400546074,
      "q10": 0.0015466592740267515,
      "q90": 0.08281046152114868,
      "q99": 0.14151407778263092,
      "frac_below_0.01": 0.3011057674884796,
      "frac_below_0.05": 0.733771800994873,
      "frac_above_0.5": 0.0,
      "max": 0.279220312833786
    },
    {
      "name": "blocks.8.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.03970775753259659,
      "median": 0.027399426326155663,
      "q10": 0.0016542956000193954,
      "q90": 0.0966249480843544,
      "q99": 0.16643178462982178,
      "frac_below_0.01": 0.28803712129592896,
      "frac_below_0.05": 0.6849355697631836,
      "frac_above_0.5": 0.0,
      "max": 0.3608091473579407
    },
    {
      "name": "blocks.9.attn.q_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.022359108552336693,
      "median": 0.016284987330436707,
      "q10": 0.00270974263548851,
      "q90": 0.05029057338833809,
      "q99": 0.09790308773517609,
      "frac_below_0.01": 0.3334842324256897,
      "frac_below_0.05": 0.8987799882888794,
      "frac_above_0.5": 0.0,
      "max": 0.2203265279531479
    },
    {
      "name": "blocks.9.attn.k_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.02273663878440857,
      "median": 0.016311366111040115,
      "q10": 0.0026683020405471325,
      "q90": 0.05166053771972656,
      "q99": 0.10231450200080872,
      "frac_below_0.01": 0.3345811665058136,
      "frac_below_0.05": 0.8929036259651184,
      "frac_above_0.5": 0.0,
      "max": 0.24579688906669617
    },
    {
      "name": "blocks.9.attn.v_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.042768146842718124,
      "median": 0.028541607782244682,
      "q10": 0.0006766233709640801,
      "q90": 0.10649372637271881,
      "q99": 0.18854376673698425,
      "frac_below_0.01": 0.303490549325943,
      "frac_below_0.05": 0.6605818271636963,
      "frac_above_0.5": 0.0,
      "max": 0.4518583118915558
    },
    {
      "name": "blocks.9.attn.o_proj.raw.weight",
      "shape": [
        768,
        768
      ],
      "n": 589824,
      "mean": 0.0570216067135334,
      "median": 0.03520055115222931,
      "q10": 0.0008812142186798155,
      "q90": 0.14035534858703613,
      "q99": 0.3157142400741577,
      "frac_below_0.01": 0.2710367739200592,
      "frac_below_0.05": 0.59759521484375,
      "frac_above_0.5": 0.0011274550342932343,
      "max": 1.018288016319275
    },
    {
      "name": "blocks.9.ffn.gate.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03622754290699959,
      "median": 0.024802587926387787,
      "q10": 0.0009529480012133718,
      "q90": 0.08900798112154007,
      "q99": 0.15433676540851593,
      "frac_below_0.01": 0.31352946162223816,
      "frac_below_0.05": 0.7148427963256836,
      "frac_above_0.5": 0.0,
      "max": 0.3419005572795868
    },
    {
      "name": "blocks.9.ffn.up.raw.weight",
      "shape": [
        1280,
        768
      ],
      "n": 983040,
      "mean": 0.03622148931026459,
      "median": 0.024851901456713676,
      "q10": 0.000944304745644331,
      "q90": 0.08894885331392288,
      "q99": 0.15408946573734283,
      "frac_below_0.01": 0.31365662813186646,
      "frac_below_0.05": 0.7145477533340454,
      "frac_above_0.5": 0.0,
      "max": 0.3313311040401459
    },
    {
      "name": "blocks.9.ffn.down.raw.weight",
      "shape": [
        768,
        1280
      ],
      "n": 983040,
      "mean": 0.0442914180457592,
      "median": 0.02870362251996994,
      "q10": 0.0009061374003067613,
      "q90": 0.11138416081666946,
      "q99": 0.1992027461528778,
      "frac_below_0.01": 0.3033294975757599,
      "frac_below_0.05": 0.6538391709327698,
      "frac_above_0.5": 0.0,
      "max": 0.44191890954971313
    }
  ],
  "attention_entropy": [
    {
      "layer": 0,
      "entropy_per_head": [
        0.31201067566871643,
        0.25528043508529663,
        0.17442357540130615,
        0.08394858986139297,
        0.03744904324412346,
        0.004117322154343128,
        6.864327682654103e-12,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.8789216876029968,
        0.8898435831069946,
        0.9274678230285645,
        0.9629272222518921,
        0.9853159785270691,
        0.9986785054206848,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.07226914167404175,
      "mean_max_prob": 0.9702628254890442
    },
    {
      "layer": 1,
      "entropy_per_head": [
        0.2243039309978485,
        0.1652330458164215,
        0.11748333275318146,
        0.055319301784038544,
        0.026457656174898148,
        0.005040682852268219,
        1.5577959115806866e-14,
        1.0707181436059495e-40,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.9096110463142395,
        0.9270795583724976,
        0.9465883374214172,
        0.9738114476203918,
        0.9879803657531738,
        0.9976929426193237,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.049486491829156876,
      "mean_max_prob": 0.9785636067390442
    },
    {
      "layer": 2,
      "entropy_per_head": [
        0.2058943510055542,
        0.16393162310123444,
        0.10231588780879974,
        0.06609746068716049,
        0.023581046611070633,
        0.009287012740969658,
        7.763824034622696e-14,
        4.0588610019168326e-41,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.9163063168525696,
        0.9273470640182495,
        0.9537574052810669,
        0.9697834849357605,
        0.9889707565307617,
        0.9955309629440308,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.04759228229522705,
      "mean_max_prob": 0.9793079495429993
    },
    {
      "layer": 3,
      "entropy_per_head": [
        0.209762841463089,
        0.1171560063958168,
        0.09794622659683228,
        0.06588968634605408,
        0.026721913367509842,
        0.006907845847308636,
        3.580622567379583e-15,
        9.686195374952433e-41,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.9159234166145325,
        0.94795161485672,
        0.9551283717155457,
        0.9698406457901001,
        0.9875055551528931,
        0.9967637062072754,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.04369870945811272,
      "mean_max_prob": 0.9810927510261536
    },
    {
      "layer": 4,
      "entropy_per_head": [
        0.21481843292713165,
        0.15817253291606903,
        0.10903926193714142,
        0.06867136061191559,
        0.03494655713438988,
        0.01739848032593727,
        5.683253889200712e-13,
        9.1925179259708e-43,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.9135462045669556,
        0.9296945333480835,
        0.9507255554199219,
        0.9684354662895203,
        0.9833245277404785,
        0.9917436838150024,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.050253886729478836,
      "mean_max_prob": 0.9781224727630615
    },
    {
      "layer": 5,
      "entropy_per_head": [
        0.20507557690143585,
        0.1427854597568512,
        0.10421045869588852,
        0.04479587450623512,
        0.01711101457476616,
        0.005781891755759716,
        7.97702019456753e-13,
        2.397621672459762e-42,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.917660117149353,
        0.9356474876403809,
        0.9525343179702759,
        0.9798707962036133,
        0.9924705624580383,
        0.997445285320282,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.04331335797905922,
      "mean_max_prob": 0.9813024401664734
    },
    {
      "layer": 6,
      "entropy_per_head": [
        0.1552124321460724,
        0.10944928973913193,
        0.08590570837259293,
        0.03441280126571655,
        0.01842230185866356,
        0.007083357311785221,
        1.0530790649221355e-13,
        1.4939242928166875e-41,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.9378371238708496,
        0.9510696530342102,
        0.9609823226928711,
        0.983575165271759,
        0.9912484288215637,
        0.9965624809265137,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.03420715779066086,
      "mean_max_prob": 0.9851062297821045
    },
    {
      "layer": 7,
      "entropy_per_head": [
        0.20514321327209473,
        0.12170679867267609,
        0.0744284838438034,
        0.051210127770900726,
        0.019812947139143944,
        0.017020035535097122,
        6.806287304594871e-13,
        5.125829270832249e-39,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.9193431735038757,
        0.9457699060440063,
        0.9668407440185547,
        0.976292610168457,
        0.9907869100570679,
        0.9917880892753601,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.040776800364255905,
      "mean_max_prob": 0.9825684428215027
    },
    {
      "layer": 8,
      "entropy_per_head": [
        0.17893101274967194,
        0.07504458725452423,
        0.059000205248594284,
        0.031063487753272057,
        0.025076285004615784,
        0.007854106836020947,
        1.886881761947734e-10,
        3.0688436368713494e-43,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.9290326833724976,
        0.9654616117477417,
        0.9727820158004761,
        0.9861835241317749,
        0.9882169961929321,
        0.9964586496353149,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.03141414001584053,
      "mean_max_prob": 0.9865112900733948
    },
    {
      "layer": 9,
      "entropy_per_head": [
        0.22920922935009003,
        0.13090595602989197,
        0.07475412636995316,
        0.05009441822767258,
        0.02543170377612114,
        0.006821715272963047,
        7.358213973025773e-13,
        1.4680002712266784e-41,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "max_prob_per_head": [
        0.909065842628479,
        0.9412500262260437,
        0.9665333032608032,
        0.9774504899978638,
        0.9881967306137085,
        0.9967927932739258,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0,
        1.0
      ],
      "mean_entropy": 0.04310142993927002,
      "mean_max_prob": 0.9816074371337891
    }
  ],
  "error_breakdown": {
    "overall_accuracy": 0.727154541015625,
    "per_class": {
      "space": {
        "accuracy": 0.933965844402277,
        "n": 31620
      },
      "newline": {
        "accuracy": 0.5960154952960708,
        "n": 1807
      },
      "lowercase": {
        "accuracy": 0.699845422511801,
        "n": 120328
      },
      "uppercase": {
        "accuracy": 0.34219190968955787,
        "n": 4252
      },
      "digit": {
        "accuracy": 0.0,
        "n": 9
      },
      "punct": {
        "accuracy": 0.49141483516483514,
        "n": 5824
      }
    }
  },
  "teacher_ckpt": "/root/bitnet1/ckpt/fp32_ref_50M_last.pt",
  "teacher_val_bpc": 0.8601819578579681,
  "student_teacher_similarity": [
    {
      "layer": 0,
      "sign_agreement": 0.577347195148468
    },
    {
      "layer": 1,
      "sign_agreement": 0.5173985481262207
    },
    {
      "layer": 2,
      "sign_agreement": 0.5035130262374878
    },
    {
      "layer": 3,
      "sign_agreement": 0.5001080393791199
    },
    {
      "layer": 4,
      "sign_agreement": 0.5003906428813935
    },
    {
      "layer": 5,
      "sign_agreement": 0.5008003354072571
    },
    {
      "layer": 6,
      "sign_agreement": 0.5006855249404907
    },
    {
      "layer": 7,
      "sign_agreement": 0.5008371591567993
    },
    {
      "layer": 8,
      "sign_agreement": 0.49889018535614016
    },
    {
      "layer": 9,
      "sign_agreement": 0.49474666118621824
    }
  ]
}