Upload jobs/occ_cheap_baselines.py

jobs/occ_cheap_baselines.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+OCC Cheap Baselines — Additional conditions for workshop-paper strength.
+======================================================================
+Adds 6 baselines without new infrastructure:
+  1. Confidence-gated debate (drop low-confidence turns)
+  2. Disagreement-gated debate (extra turns only when agents disagree)
+  3. Round-robin capped (equal turns but hard token cap)
+  4. Single-agent best-of-N (same budget, no multi-agent)
+  5. No-adversary 4-agent (all honest — does collapse still happen?)
+  6. Reputation-only allocator (earn/lose score, no decay/transfer)
+
+30 topics, 2 seeds, Qwen3-Coder-30B-A3B-Instruct.
+"""
+
+import json, os, sys, time, random, torch
+from pathlib import Path
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+MODEL = os.environ.get("MODEL", "Qwen/Qwen3-Coder-30B-A3B-Instruct")
+SEEDS = [int(s) for s in os.environ.get("SEEDS", "42,123").split(",")]
+OUT = Path(os.environ.get("OUT_DIR", "/app/results"))
+OUT.mkdir(parents=True, exist_ok=True)
+START = time.time()
+
+def log(msg):
+    print(f"[+{time.time()-START:5.0f}s] {msg}", flush=True)
+
+DEBATE_TOPICS = [
+    {"q": "Is Python faster than C for numerical computation?", "truth": "no"},
+    {"q": "Does water boil at 100C at all altitudes?", "truth": "no"},
+    {"q": "Can quantum computers break RSA-2048 today (2026)?", "truth": "no"},
+    {"q": "Is the Earth core hotter than the surface of the Sun?", "truth": "yes"},
+    {"q": "Does the Moon have an atmosphere?", "truth": "no"},
+    {"q": "Can sound travel through a vacuum?", "truth": "no"},
+    {"q": "Is JavaScript single-threaded by default in browsers?", "truth": "yes"},
+    {"q": "Does DNA replication occur in the nucleus of eukaryotic cells?", "truth": "yes"},
+    {"q": "Can a protein structure be determined with 100% certainty from X-ray?", "truth": "no"},
+    {"q": "Is gradient descent guaranteed to find global min for convex functions?", "truth": "yes"},
+    {"q": "Can GPT-4 reliably solve novel math proofs without supervision?", "truth": "no"},
+    {"q": "Is P vs NP solved as of 2026?", "truth": "no"},
+    {"q": "Do all metals expand when heated?", "truth": "no"},
+    {"q": "Is the speed of light constant in all reference frames?", "truth": "yes"},
+    {"q": "Can a program determine if an arbitrary program halts?", "truth": "no"},
+    {"q": "Is the Earth flat?", "truth": "no"},
+    {"q": "Does CO2 make up more than 1 percent of Earth atmosphere?", "truth": "no"},
+    {"q": "Can classical computers efficiently simulate quantum?", "truth": "no"},
+    {"q": "Is the golden ratio exactly (1+sqrt5)/2?", "truth": "yes"},
+    {"q": "Can 1-hidden-layer NN approximate any continuous function?", "truth": "yes"},
+    {"q": "Does entropy always increase in isolated systems?", "truth": "yes"},
+    {"q": "Is Python GIL removed in CPython 3.13+?", "truth": "yes"},
+    {"q": "Do sharks get cancer?", "truth": "yes"},
+    {"q": "Is Antarctica a country?", "truth": "no"},
+    {"q": "Can humans survive without gut bacteria?", "truth": "yes"},
+    {"q": "Do all birds fly?", "truth": "no"},
+    {"q": "Is lightning hotter than the Sun surface?", "truth": "yes"},
+    {"q": "Can finite-tape TM recognize all recursive languages?", "truth": "no"},
+    {"q": "Is the Riemann Hypothesis proved as of 2026?", "truth": "no"},
+    {"q": "Does gravitational lensing confirm GR?", "truth": "yes"},
+]
+
+_model = None
+_tok = None
+
+def get_model():
+    global _model, _tok
+    if _model is None:
+        log(f"Loading {MODEL}...")
+        _tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
+        _tok.pad_token = _tok.eos_token
+        _model = AutoModelForCausalLM.from_pretrained(
+            MODEL, trust_remote_code=True, torch_dtype=torch.bfloat16, device_map="auto")
+        log(f"Loaded. Device: {_model.device}")
+    return _model, _tok
+
+def generate(prompt, max_tokens=512, temperature=0.7):
+    model, tok = get_model()
+    inputs = tok(prompt, return_tensors="pt", truncation=True, max_length=2048).to(model.device)
+    ilen = inputs.input_ids.shape[1]
+    with torch.no_grad():
+        out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True,
+                            temperature=temperature, top_p=0.9, pad_token_id=tok.eos_token_id)
+    ntok = out.shape[1] - ilen
+    return tok.decode(out[0][ilen:], skip_special_tokens=True), ntok
+
+def extract_position(text):
+    t = text.strip()
+    fl = t.split("\n")[0].lower()
+    if fl.startswith("yes:") or fl.startswith("yes "): return "yes"
+    if fl.startswith("no:") or fl.startswith("no "): return "no"
+    for w in ["clearly yes", "definitely yes", "answer is yes"]:
+        if w in fl: return "yes"
+    for w in ["clearly no", "definitely no", "answer is no"]:
+        if w in fl: return "no"
+    tc = t.lower()
+    return "yes" if tc.count(" yes ") > tc.count(" no ") else "no" if tc.count(" no ") > tc.count(" yes ") else "unclear"
+
+def extract_confidence(text):
+    tlow = text.lower()
+    if "high confidence" in tlow or "confidence: high" in tlow: return 1.0
+    if "medium confidence" in tlow or "confidence: medium" in tlow: return 0.6
+    if "low confidence" in tlow or "confidence: low" in tlow: return 0.3
+    return 0.5
+
+def accuracy(results):
+    n = len(results); c = sum(1 for r in results if r["correct"]); return c/n, c, n
+
+def run_confidence_gated():
+    results = []
+    for t in DEBATE_TOPICS:
+        positions = []
+        for rnd in range(3):
+            for a in range(4):
+                persona = "honest" if a < 3 else "adversarial"
+                text, _ = generate(
+                    f"Question: {t['q']}\nYou are a {persona} debater. Round {rnd+1}/3. "
+                    f"Start with YES: or NO:. State your confidence as HIGH, MEDIUM, or LOW.\n", 512)
+                if extract_confidence(text) >= 0.5:
+                    positions.append(extract_position(text))
+        votes = [p for p in positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner, "correct": winner == t["truth"]})
+    return results
+
+def run_disagreement_gated():
+    results = []
+    for t in DEBATE_TOPICS:
+        r1_positions = []
+        for a in range(4):
+            persona = "honest" if a < 3 else "adversarial"
+            text, _ = generate(
+                f"Question: {t['q']}\nYou are a {persona} debater. Round 1/3. "
+                f"Start with YES: or NO: followed by brief explanation.\n", 512)
+            r1_positions.append(extract_position(text))
+        yes_c = sum(1 for p in r1_positions if p == "yes")
+        no_c = sum(1 for p in r1_positions if p == "no")
+        disagreed = yes_c > 0 and no_c > 0
+        all_positions = list(r1_positions)
+        if disagreed:
+            for rnd in [2, 3]:
+                for a in range(4):
+                    persona = "honest" if a < 3 else "adversarial"
+                    text, _ = generate(
+                        f"Question: {t['q']}\nYou are a {persona} debater. Round {rnd}/3. "
+                        f"Start with YES: or NO: followed by brief explanation.\n", 512)
+                    all_positions.append(extract_position(text))
+        votes = [p for p in all_positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner,
+                       "correct": winner == t["truth"], "disagreed": disagreed})
+    return results
+
+def run_capped_debate(max_tok=2000):
+    results = []
+    for t in DEBATE_TOPICS:
+        all_positions = []; tok_used = 0
+        for rnd in range(3):
+            for a in range(4):
+                if tok_used >= max_tok: break
+                persona = "honest" if a < 3 else "adversarial"
+                rem = max(50, max_tok - tok_used)
+                text, nt = generate(
+                    f"Question: {t['q']}\nYou are a {persona} debater. Round {rnd+1}/3. "
+                    f"Be very concise. Start with YES: or NO:.\n", max_tokens=min(128, rem))
+                all_positions.append(extract_position(text)); tok_used += nt
+            if tok_used >= max_tok: break
+        votes = [p for p in all_positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner,
+                       "correct": winner == t["truth"], "tokens_used": tok_used})
+    return results
+
+def run_single_agent_best_of_n(n=12):
+    results = []
+    for t in DEBATE_TOPICS:
+        positions = []
+        for _ in range(n):
+            text, _ = generate(
+                f"Question: {t['q']}\nYou are a honest debater. "
+                f"Start with YES: or NO: followed by brief explanation.\n", 512)
+            positions.append(extract_position(text))
+        votes = [p for p in positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner, "correct": winner == t["truth"]})
+    return results
+
+def run_no_adversary_3round():
+    results = []
+    for t in DEBATE_TOPICS:
+        all_positions = []
+        for rnd in range(3):
+            for a in range(4):
+                text, _ = generate(
+                    f"Question: {t['q']}\nYou are a honest debater. Round {rnd+1}/3. "
+                    f"Start with YES: or NO: followed by brief explanation.\n", 512)
+                all_positions.append(extract_position(text))
+        votes = [p for p in all_positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner, "correct": winner == t["truth"]})
+    return results
+
+def run_reputation_only():
+    results = []
+    for t in DEBATE_TOPICS:
+        rep = [1.0, 1.0, 1.0, 1.0]; all_positions = []; rnd_positions = []
+        for rnd in range(3):
+            rp = []
+            for a in range(4):
+                persona = "honest" if a < 3 else "adversarial"
+                text, _ = generate(
+                    f"Question: {t['q']}\nYou are a {persona} debater. Round {rnd+1}/3. "
+                    f"Start with YES: or NO: followed by brief explanation.\n", 512)
+                pos = extract_position(text); rp.append(pos); all_positions.append(pos)
+            rnd_positions.append(rp)
+            votes = [p for p in rp if p != "unclear"]
+            if votes:
+                w = max(set(votes), key=votes.count)
+                for a in range(4): rep[a] += 0.1 if rp[a] == w else -0.1; rep[a] = max(0, rep[a])
+        yes_w = sum(rep[a] for a in range(4) if rnd_positions[-1][a] == "yes")
+        no_w = sum(rep[a] for a in range(4) if rnd_positions[-1][a] == "no")
+        winner = "yes" if yes_w > no_w else "no" if no_w > yes_w else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner, "correct": winner == t["truth"]})
+    return results
+
+CONDITIONS = [
+    ("confidence_gated", run_confidence_gated),
+    ("disagreement_gated", run_disagreement_gated),
+    ("capped_debate", run_capped_debate),
+    ("single_agent_best_of_n", run_single_agent_best_of_n),
+    ("no_adversary_3round", run_no_adversary_3round),
+    ("reputation_only", run_reputation_only),
+]
+
+all_results = {"model": MODEL, "seeds": {}, "conditions": [c[0] for c in CONDITIONS]}
+
+for seed in SEEDS:
+    torch.manual_seed(seed); random.seed(seed)
+    if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed)
+    log(f"\n{'='*60}\nSEED {seed}\n{'='*60}")
+    get_model()
+    seed_results = {}
+    for name, fn in CONDITIONS:
+        log(f"--- {name} ---"); t0 = time.time()
+        try:
+            res = fn(); acc, corr, total = accuracy(res)
+            extra = {}
+            if name == "disagreement_gated":
+                extra = {"disagreed": sum(1 for r in res if r.get("disagreed"))}
+            elif name == "capped_debate" and res:
+                extra = {"avg_tokens": round(sum(r.get("tokens_used", 0) for r in res)/len(res))}
+            seed_results[name] = {"accuracy": acc, "correct": corr, "total": total, **extra}
+            log(f"  {corr}/{total} ({acc:.3f}) ({time.time()-t0:.0f}s)")
+        except Exception as e:
+            log(f"  ERROR: {e}"); seed_results[name] = {"accuracy": None, "error": str(e)}
+    all_results["seeds"][str(seed)] = seed_results
+
+summary = {}
+for name, _ in CONDITIONS:
+    accs = [all_results["seeds"][str(s)][name].get("accuracy", 0) or 0 for s in SEEDS
+            if all_results["seeds"].get(str(s), {}).get(name, {}).get("accuracy") is not None]
+    if accs:
+        mn, mx = min(accs), max(accs); mean = sum(accs)/len(accs)
+        log(f"  {name:<25} {mean:7.3f} [{mn:.3f}, {mx:.3f}]")
+        summary[name] = {"mean": mean, "min": mn, "max": mx}
+all_results["summary"] = summary
+
+path = OUT / "cheap_baselines_results.json"
+path.write_text(json.dumps(all_results, indent=2))
+log(f"\nSaved -> {path}")
+
+try:
+    from huggingface_hub import HfApi
+    HfApi().upload_file(path_or_fileobj=str(path), path_in_repo="reports/cheap_baselines_results.json",
+                       repo_id="narcolepticchicken/occ-stack", repo_type="model",
+                       commit_message="Cheap baselines results")
+    log("Pushed to Hub")
+except Exception as e:
+    log(f"Push failed: {e}")
+log(f"Total elapsed: {time.time()-START:.0f}s")