{
  "$schema": "https://smokingmirror.ai/schema/experiment-catalog-v1",
  "catalog_version": 1,
  "experiments": [
    {
      "slug": "v2",
      "title": "Binary preference observatory",
      "status": "published",
      "route": "/v2/",
      "data_route": "/v2/data.json",
      "snapshot": "2026-06-10",
      "models": 40,
      "probes": 863,
      "envelopes": 6,
      "tested": "Whether forced choices form stable, comparable model-preference fingerprints, and how prompt envelope and word order change what can be measured.",
      "headline": "Model families and effort levels leave distinct fingerprints; abstention is part of the signal, not missing data.",
      "generator": "site5/regen_v2.sh",
      "source": "smokingmirror-backup"
    },
    {
      "slug": "opus-45-48",
      "title": "Opus 4.5–4.8 · v2 default set",
      "status": "published",
      "route": "/opus-45-48/",
      "data_route": "/opus-45-48/data.json",
      "snapshot": "2026-05-29",
      "models": 6,
      "probes": 863,
      "envelopes": 6,
      "tested": "Whether Claude Opus 4.5, 4.6, 4.7 and 4.8 hold stable forced-choice preferences on the frozen v2 grid (863 pairs × 6 envelopes × 2 word orders), at default sampling with 5 samples per cell and, for 4.5 and 4.6, at temperature 0.",
      "headline": "Real cross-envelope preferences on 445 (4.5), 553 (4.6), 608 (4.7) and 572 (4.8) of 863 pairs; the temperature-0 rows of 4.5 and 4.6 agree with their 5-sample rows on every pair both answered (428 and 535).",
      "generator": "site5/regen_opus45_48.sh",
      "source": "site5/opus45_48 (export of research.public.traces)"
    },
    {
      "slug": "opus-45-48-coding",
      "title": "Opus 4.5–4.8 · v2 + coding set",
      "status": "published",
      "route": "/opus-45-48-coding/",
      "data_route": "/opus-45-48-coding/data.json",
      "snapshot": "2026-10-12",
      "models": 12,
      "probes": 863,
      "envelopes": 20,
      "tested": "Whether the same models give the same preferences when each v2 question is asked as a code completion: 14 code envelopes (7 languages, minimal and full), 24,164 cells per model, one sample, run 2026-10-12, shown beside the six v2 rows.",
      "headline": "Where both sets give a real preference, each model's coding answer matches its v2 answer on 96.5–98.9% of pairs; Opus 4.8 refused 1,519 of 24,164 coding prompts.",
      "generator": "site5/regen_opus45_48.sh",
      "source": "site5/opus45_48 (export of research.public.traces)"
    }
  ]
}