aggregate_synth_seeds.py

"""Aggregate synth+ALiBi+n=8 results across seeds and placements.

Reads logs in /tmp/synth_short_dist/ and produces a markdown table.
"""
import re
from pathlib import Path

LOG_DIR = Path("/tmp/synth_short_dist")


def parse_log(p: Path):
    if not p.exists():
        return None
    text = p.read_text()
    # last copy_acc
    accs = re.findall(r"copy_acc=([\d.]+)%", text)
    ipmrs = re.findall(r"Global IPMR \(any-head match\): ([\d.]+)%", text)
    if not accs or not ipmrs:
        return None
    return {
        "copy_acc": float(accs[-1]),
        "ipmr": float(ipmrs[-1]),
    }


def main():
    seeds = sorted({int(m.group(1)) for f in LOG_DIR.glob("n8_seed*_L*.log")
                    if (m := re.match(r"n8_seed(\d+)_L\d+\.log", f.name))})
    layers = sorted({int(m.group(1)) for f in LOG_DIR.glob("n8_seed*_L*.log")
                     if (m := re.search(r"_L(\d+)\.log", f.name))})
    print(f"Found seeds: {seeds}, layers: {layers}\n")

    # Build header
    print("| seed |" + "".join(f" L{L} copy / IPMR |" for L in layers))
    print("|---|" + "|".join("---" for _ in layers) + "|")
    for seed in seeds:
        line = f"| {seed} |"
        for L in layers:
            r = parse_log(LOG_DIR / f"n8_seed{seed}_L{L}.log")
            if r:
                line += f" {r['copy_acc']:.1f}% / {r['ipmr']:.1f}% |"
            else:
                line += " (missing) |"
        print(line)

    # Per-layer means
    print("\n### Per-layer means")
    print("| L | mean copy | mean IPMR | seeds with >50% IPMR |")
    print("|---|---|---|---|")
    for L in layers:
        copies, ipmrs, hi_count = [], [], 0
        for seed in seeds:
            r = parse_log(LOG_DIR / f"n8_seed{seed}_L{L}.log")
            if r:
                copies.append(r['copy_acc'])
                ipmrs.append(r['ipmr'])
                if r['ipmr'] > 50:
                    hi_count += 1
        if copies:
            mc = sum(copies)/len(copies)
            mi = sum(ipmrs)/len(ipmrs)
            print(f"| L{L} | {mc:.1f}% | {mi:.1f}% | {hi_count}/{len(copies)} |")


if __name__ == "__main__":
    main()