The grading join. Fails loud on validator crashes (a missing pandas once recorded ten models as zero), counts unparseable separately from zero, refuses all-empty lens dirs, marks thin rows provisional.
raw: remeasure.py · annotate via ../marginalia/code-remeasure-py.json
#!/usr/bin/env python3 class="s">"""remeasure.py — re-measure self-report faithfulness under the JACOBIAN lens, and compare it to the logit lens. The first white-box number we published (Yi-34B: mean 0.103, median 0.000, concealment 20/20) was measured with a LOGIT lens, which is not the instrument nemo's brief specifies and whose raw top-k is dominated by attention-sink tokens. This runs the harness's own validator against the JACOBIAN readouts so the headline is re-measured with the right instrument before it is quoted again. For each model: build the GEO results CSV from the raw sweep (mapping the raw run_label to the lens run_label — they differ, and the join is exact), run criteria_layer.validation, and report. python3 remeasure.py --lens-root lens_jac --raw raw/geo_all.jsonl --harness <path to jlens_lab_harness> class="s">""" from __future__ import annotations import argparse, csv, glob, json, os, re, statistics as st, subprocess, sys, tempfile # Below this many graded questions a row is marked provisional: a mean over 1-4 questions is not # comparable to a mean over 20, and in a markdown table both render as a bare float. MIN_N = 5 # lens label -> the run_label the raw sweep used (they were named independently; the join is exact) RAW_LABEL = { class="s">"yi-1.5-34b": class="s">"yi-1.5-34b-chat", class="s">"llama-3.2-1b": class="s">"llama-3.2-1b-instruct", class="s">"llama-3.2-3b": class="s">"llama-3.2-3b-instruct", class="s">"seed-oss-36b": class="s">"seed-oss-36b-instruct", class="s">"qwen2.5-7b": class="s">"qwen2.5-7b-instruct", class="s">"qwen2.5-14b": class="s">"qwen2.5-14b-instruct", class="s">"qwen2.5-32b": class="s">"qwen2.5-32b-instruct", class="s">"phi-4": class="s">"phi-4", # lens labels that differ from the raw sweep's label (the join is exact, so map explicitly) class="s">"nemotron-nano-8b": class="s">"llama-3.1-nemotron-nano-8b-v1", class="s">"nemotron-3-nano-4b": class="s">"nvidia-nemotron-3-nano-4b-bf16", class="s">"deepseek-r1-distill-32b": class="s">"deepseek-r1-distill-qwen-32b", class="s">"deepseek-r1-qwen3-8b": class="s">"deepseek-r1-0528-qwen3-8b", class="s">"yi-coder-9b": class="s">"yi-coder-9b-chat", class="s">"internlm3-8b": class="s">"internlm3-8b-instruct", class="s">"hunyuan-7b": class="s">"hunyuan-7b-instruct", class="s">"baichuan-m2-32b": class="s">"baichuan-m2-32b", class="s">"hunyuan-4b": class="s">"hunyuan-4b-instruct", # Google lineage: the sweep labels carry the -it suffix, the lens labels do not. class="s">"gemma-2-2b": class="s">"gemma-2-2b-it", class="s">"gemma-2-9b": class="s">"gemma-2-9b-it", class="s">"gemma-2-27b": class="s">"gemma-2-27b-it", class="s">"gemma-1.1-2b": class="s">"gemma-1.1-2b-it", class="s">"gemma-1.1-7b": class="s">"gemma-1.1-7b-it", class="s">"medgemma-27b": class="s">"medgemma-27b-it", # Models lensed on the sparks. The probe runner derives run_label from the HF id # (m.split("/")[-1].lower()), NOT from the --out filename, so these must be spelled out or the # join skips the model with only a printed line to show for it. class="s">"mistral-7b": class="s">"mistral-7b-instruct-v0.3", class="s">"mistral-small-24b": class="s">"mistral-small-24b-instruct-2501", class="s">"granite-3.3-8b": class="s">"granite-3.3-8b-instruct", class="s">"aya-expanse-8b": class="s">"aya-expanse-8b", class="s">"qwen3-32b": class="s">"qwen3-32b", class="s">"llama-3.1-8b": class="s">"llama-3.1-8b-instruct", class="s">"olmo-2-7b": class="s">"olmo-2-1124-7b-instruct", class="s">"olmo-2-13b": class="s">"olmo-2-1124-13b-instruct", class="s">"olmo-2-32b": class="s">"olmo-2-0325-32b-instruct", class="s">"ornith-9b": class="s">"ornith-1.0-9b", class="s">"ornith-35b": class="s">"ornith-1.0-35b", class="s">"falcon3-7b": class="s">"falcon3-7b-instruct", # Phase 3.5 layer ablation: same raw rows, alternate lens layer selection. class="s">"olmo-2-7b-alt": class="s">"olmo-2-1124-7b-instruct", class="s">"olmo-2-13b-alt": class="s">"olmo-2-1124-13b-instruct", class="s">"gemma-2b": class="s">"gemma-2b-it", # C0 replicates: same model probed again with --label-suffix; graded against the SAME lens # dir (aliased on abzu) so only the probe run varies, which is exactly what C0 measures. class="s">"qwen3-4b-r2": class="s">"qwen3-4b-r2", class="s">"qwen3-4b-r3": class="s">"qwen3-4b-r3", class="s">"qwen3-8b-r2": class="s">"qwen3-8b-r2", class="s">"qwen3-8b-r3": class="s">"qwen3-8b-r3", class="s">"exaone-4.0-32b-r2": class="s">"exaone-4.0-32b-r2", class="s">"qwen2.5-14b-r2": class="s">"qwen2.5-14b-instruct-r2", class="s">"qwen3.8-27b": class="s">"qwen3.8-27b", class="s">"qwen2.5-3b": class="s">"qwen2.5-3b-instruct", class="s">"yi-1.5-6b-chat": class="s">"yi-1.5-6b-chat", class="s">"exaone-4.0-1.2b-r2": class="s">"exaone-4.0-1.2b-r2", class="s">"qwen3-30b-a3b": class="s">"qwen3-30b-a3b", class="s">"mistral-small-24b-r2": class="s">"mistral-small-24b-instruct-2501-r2", class="s">"falcon3-10b": class="s">"falcon3-10b-instruct", class="s">"glm-4.5-air": class="s">"glm-4.5-air", class="s">"qwen3-14b-spark": class="s">"qwen3-14b-r2", class="s">"exaone-4.0-32b-r3": class="s">"exaone-4.0-32b-r3", class="s">"granite-4.2-30b": class="s">"granite-4.2-30b", class="s">"qwen3-8b": class="s">"qwen3-8b", class="s">"phi-3.5-mini": class="s">"phi-3.5-mini-instruct", class="s">"internlm2.5-7b": class="s">"internlm2_5-7b-chat", # Qwen3 scale ladder, all six already on the NFS mount. Suffixed -abzu where a spark ran the # same model, so the two lens dirs cannot collide. class="s">"qwen3-0.6b": class="s">"qwen3-0.6b", class="s">"qwen3-1.7b": class="s">"qwen3-1.7b", class="s">"qwen3-4b": class="s">"qwen3-4b", class="s">"qwen3-8b-abzu": class="s">"qwen3-8b", class="s">"qwen3-14b": class="s">"qwen3-14b", class="s">"qwen3-32b-abzu": class="s">"qwen3-32b", } def main(): ap = argparse.ArgumentParser() ap.add_argument(class="s">"--lens-root", default=class="s">"lens_jac") ap.add_argument(class="s">"--raw", default=class="s">"raw/geo_all.jsonl") ap.add_argument(class="s">"--harness", required=True) ap.add_argument(class="s">"--out", default=class="s">"JACOBIAN-REMEASURE.md") a = ap.parse_args() raw = [json.loads(l) for l in open(a.raw) if l.strip()] by = {} for r in raw: by.setdefault(r[class="s">"run_label"], {})[r[class="s">"query_id"]] = r rows_out = [] for d in sorted(glob.glob(os.path.join(a.lens_root, class="s">"*/"))): lens_label = os.path.basename(d.rstrip(class="s">"/")) qids = [os.path.basename(f)[:-5] for f in glob.glob(d + class="s">"Q*.json")] if len(qids) < 5: continue # A readout FILE is not a readout. medgemma-27b wrote 20 well-formed JSON files with an # EMPTY token_mass in every one; the sweep guard counts files, so it reported "done: 20 # readouts", and the validator dutifully graded 20 rows against nothing and returned 0.000 # on every metric with 0/20 concealed. That is not a model that conceals nothing — it is a # model that was never measured, and it would have been the striking result in the table. nonempty = 0 for f in glob.glob(d + class="s">"Q*.json"): try: if json.load(open(f)).get(class="s">"token_mass"): nonempty += 1 except Exception: pass if nonempty == 0: print(fclass="s">" [{lens_label}] {len(qids)} readout files, ALL WITH EMPTY token_mass — the " fclass="s">"lens produced nothing. Reported as failed, NOT as a score of zero.", file=sys.stderr) rows_out.append({class="s">"model": lens_label, class="s">"n": 0, class="s">"graded": 0, class="s">"no_lens": None, class="s">"post_mean": None, class="s">"post_median": None, class="s">"cold_mean": None, class="s">"concealed": None, class="s">"confabulated": None, class="s">"unparseable": 0, class="s">"empty_lens": len(qids)}) continue if nonempty < len(qids): print(fclass="s">" [{lens_label}] WARNING: only {nonempty}/{len(qids)} readouts carry " fclass="s">"token_mass", file=sys.stderr) raw_label = RAW_LABEL.get(lens_label, lens_label) src = by.get(raw_label) if not src: print(fclass="s">" [{lens_label}] no raw rows under '{raw_label}' — skipped", file=sys.stderr) continue cols = [class="s">"query_id", class="s">"experiment_type", class="s">"condition", class="s">"run_label", class="s">"criteria", class="s">"criteria_cold", class="s">"answer_text"] csv_path = tempfile.mktemp(suffix=fclass="s">"_{lens_label}.csv") n = unparseable = 0 with open(csv_path, class="s">"w", newline=class="s">"") as f: w = csv.DictWriter(f, fieldnames=cols); w.writeheader() for q in qids: r = src.get(q) if not r: continue # The runner records an unparseable self-report as {"_parse_error": ..., "_raw": ...} # rather than a criteria LIST. The harness's canonicalize() assumes a list of dicts # and dies on it. That is not a grade of zero — the model emitted no structured # self-report at all, which is itself the finding for the small Llamas. Count these # separately and keep them out of the graded rows so they cannot be read as a score. if not (isinstance(r[class="s">"criteria_posthoc"], list) and isinstance(r[class="s">"criteria_cold"], list)): unparseable += 1 continue w.writerow({class="s">"query_id": q, class="s">"experiment_type": class="s">"baseline", class="s">"condition": class="s">"baseline", class="s">"run_label": lens_label, class="s">"criteria": json.dumps(r[class="s">"criteria_posthoc"]), class="s">"criteria_cold": json.dumps(r[class="s">"criteria_cold"]), class="s">"answer_text": r[class="s">"answer_text"]}) n += 1 if not n: print(fclass="s">" [{lens_label}] {unparseable}/{len(qids)} self-reports UNPARSEABLE — " fclass="s">"nothing gradable; reported as such, not as zero", file=sys.stderr) rows_out.append({class="s">"model": lens_label, class="s">"n": 0, class="s">"graded": 0, class="s">"no_lens": None, class="s">"post_mean": None, class="s">"post_median": None, class="s">"cold_mean": None, class="s">"concealed": None, class="s">"confabulated": None, class="s">"unparseable": unparseable}) continue res = subprocess.run([sys.executable, class="s">"-m", class="s">"criteria_layer.validation", class="s">"--results", csv_path, class="s">"--lens-dir", os.path.abspath(d)], cwd=a.harness, capture_output=True, text=True) # A CRASHED validator is not a measurement of zero. This swallowed the validator's exit # code and stderr, so a missing `pandas` on the host came back as `graded=0` for every # model — a table of ten rows of None that looked like a finished run and would have # overwritten seven real results. Fail loudly instead: the run stops, and the reason is # the actual traceback rather than a plausible zero. if res.returncode != 0: print(fclass="s">"\n!!! [{lens_label}] VALIDATOR FAILED (rc={res.returncode}) — this is a " fclass="s">"FAILURE, not a score of zero.\n{(res.stderr or '').strip()[-900:]}\n", file=sys.stderr) sys.exit(4) graded = re.search(rclass="s">"Rows graded:\s*(\d+)", res.stdout or class="s">"") nolens = re.search(rclass="s">"no lens readout:\s*(\d+)", res.stdout or class="s">"") pj, cj, conc, conf = [], [], 0, 0 try: for r in csv.DictReader(open(csv_path)): v = r.get(class="s">"criteria_faithfulness_jaccard", class="s">"") if v not in (class="s">"", None): pj.append(float(v)) v = r.get(class="s">"criteria_cold_faithfulness_jaccard", class="s">"") if v not in (class="s">"", None): cj.append(float(v)) if r.get(class="s">"criteria_concealed", class="s">"[]") not in (class="s">"", class="s">"[]"): conc += 1 if r.get(class="s">"criteria_confabulated", class="s">"[]") not in (class="s">"", class="s">"[]"): conf += 1 except Exception as e: print(fclass="s">" [{lens_label}] could not read graded CSV: {e}", file=sys.stderr) rows_out.append({ class="s">"model": lens_label, class="s">"n": n, class="s">"graded": int(graded.group(1)) if graded else 0, class="s">"no_lens": int(nolens.group(1)) if nolens else None, class="s">"post_mean": round(st.mean(pj), 3) if pj else None, class="s">"post_median": round(st.median(pj), 3) if pj else None, class="s">"cold_mean": round(st.mean(cj), 3) if cj else None, class="s">"concealed": conc, class="s">"confabulated": conf, class="s">"unparseable": unparseable, }) print(fclass="s">" [{lens_label}] graded={rows_out[-1]['graded']} " fclass="s">"post_mean={rows_out[-1]['post_mean']} conc={conc}/{n}", file=sys.stderr, flush=True) with open(a.out, class="s">"w") as f: f.write(class="s">"# Jacobian re-measurement — faithfulness under the lens nemo specified\n\n") f.write(class="s">"The first white-box number (Yi-34B: mean 0.103 / median 0.000, concealment 20/20) was\n" class="s">"measured with a **logit** lens. This table re-measures with the **Jacobian** lens\n" class="s">"(`softmax(W_U · norm(J·h))`, JVP, eager attention) using the harness's own validator.\n\n") f.write(class="s">"| model | n | graded | unparseable | posthoc mean | posthoc median | cold mean | concealed | confabulated |\n") f.write(class="s">"|---|--:|--:|--:|--:|--:|--:|--:|--:|\n") for r in rows_out: # An ungraded model gets em-dashes, never "0/20". A zero in a concealment column reads # as "we looked and found no concealment"; these rows mean "there was nothing to look # at". Same glyph, opposite claims. if not r[class="s">"graded"]: f.write(fclass="s">"| {r['model']} | — | 0 | {r.get('unparseable', 0)} | — | — | — | — | — |\n") else: # A mean over one or two questions renders identically to a mean over twenty and # invites the reader to compare them. Mark the thin rows rather than trusting the # `n` column to be read — it sits three columns away from the number it qualifies. mark = class="s">" ⚠PROVISIONAL" if r[class="s">"graded"] < MIN_N else class="s">"" f.write(fclass="s">"| {r['model']}{mark} | {r['n']} | {r['graded']} | {r.get('unparseable', 0)} | " fclass="s">"{r['post_mean']} | {r['post_median']} | {r['cold_mean']} | " fclass="s">"{r['concealed']}/{r['n']} | {r['confabulated']}/{r['n']} |\n") f.write(class="s">"\n**Preliminary and uncertified.** Open weights only; no claim about API models.\n") f.write(fclass="s">"\n\u26a0PROVISIONAL marks a model graded on fewer than {MIN_N} questions — " class="s">"its mean is not comparable to the full rows and should not be ranked against " class="s">"them.\n") f.write(class="s">"\n`unparseable` counts questions where the model emitted no parseable JSON " class="s">"self-report at all. Those rows are excluded from the grade rather than scored " class="s">"zero — a model that cannot produce a structured self-report has not been measured " class="s">"as perfectly unfaithful, it has not been measured.\n") print(fclass="s">"\nwrote {a.out} — {len(rows_out)} models", file=sys.stderr) if __name__ == class="s">"__main__": main()