"""Aggregate synth+ALiBi+n=8 results across seeds and placements. Reads logs in /tmp/synth_short_dist/ and produces a markdown table. """ import re from pathlib import Path LOG_DIR = Path("/tmp/synth_short_dist") def parse_log(p: Path): if not p.exists(): return None text = p.read_text() # last copy_acc accs = re.findall(r"copy_acc=([\d.]+)%", text) ipmrs = re.findall(r"Global IPMR \(any-head match\): ([\d.]+)%", text) if not accs or not ipmrs: return None return { "copy_acc": float(accs[-1]), "ipmr": float(ipmrs[-1]), } def main(): seeds = sorted({int(m.group(1)) for f in LOG_DIR.glob("n8_seed*_L*.log") if (m := re.match(r"n8_seed(\d+)_L\d+\.log", f.name))}) layers = sorted({int(m.group(1)) for f in LOG_DIR.glob("n8_seed*_L*.log") if (m := re.search(r"_L(\d+)\.log", f.name))}) print(f"Found seeds: {seeds}, layers: {layers}\n") # Build header print("| seed |" + "".join(f" L{L} copy / IPMR |" for L in layers)) print("|---|" + "|".join("---" for _ in layers) + "|") for seed in seeds: line = f"| {seed} |" for L in layers: r = parse_log(LOG_DIR / f"n8_seed{seed}_L{L}.log") if r: line += f" {r['copy_acc']:.1f}% / {r['ipmr']:.1f}% |" else: line += " (missing) |" print(line) # Per-layer means print("\n### Per-layer means") print("| L | mean copy | mean IPMR | seeds with >50% IPMR |") print("|---|---|---|---|") for L in layers: copies, ipmrs, hi_count = [], [], 0 for seed in seeds: r = parse_log(LOG_DIR / f"n8_seed{seed}_L{L}.log") if r: copies.append(r['copy_acc']) ipmrs.append(r['ipmr']) if r['ipmr'] > 50: hi_count += 1 if copies: mc = sum(copies)/len(copies) mi = sum(ipmrs)/len(ipmrs) print(f"| L{L} | {mc:.1f}% | {mi:.1f}% | {hi_count}/{len(copies)} |") if __name__ == "__main__": main()