{
 "generated_for": "PSF-Med frontier-model arms, binary population",
 "population": "results/uai/revision/psf_binary_inclusion_manifest.jsonl (1,539 mimic / 8,445 padchest / 2,807 vindr); hardest-500 re-ranked WITHIN it",
 "protocol": {
  "prompt": "unchanged from PSF-Med API arms (byte-identical)",
  "claude_max_tokens": 64,
  "openai_max_output_tokens": 256,
  "kimi_max_tokens": 256,
  "kimi_note": "kimi-k3 has always-on thinking and cannot answer at max_tokens=64",
  "images": "16-bit-safe percentile windowing via scripts/uai/image_utils.load_image_rgb",
  "ground_truth": "manifest `ref`; the prior scripts forged every reference to 'yes'",
  "parser": "head-first yes/no; unreadable responses excluded, never coerced"
 },
 "caveats": [
  "flip rate is pairwise (denominator = paraphrase pair), not query-level",
  "500-hardest upper-tail arms; NOT benchmark-representative rates",
  "kimi-k3 mimic parses only 90.2% (always-on thinking exceeds max_tokens=256)",
  "the gpt-5.6-sol cells here are the CORRECTED 2026-08-07 arms; the 2026-08-05 version of this file carried gpt-5-mini responses under the Sol label",
  "image-reliance controls complete for all frontier configs (2026-08-07 OpenAI; 2026-08-09 Claude Opus 5 both reasoning settings and kimi-k3)",
  "only gpt-5-mini has a same-model replicate; Opus 5 / Sol / Kimi control cells have no sampling-noise ceiling",
  "kimi-k3 text-only arms lose 5-15.6% of responses to always-on-thinking truncation at max_tokens=256 (dropped as unreadable, never coerced); the loss selects the longest-reasoned questions"
 ],
 "frontier_pooled": {
  "n_pairs": 24336,
  "n_flips": 2040,
  "pooled_flip_pct": 8.38,
  "pooled_ci95": [
   8.04,
   8.74
  ]
 },
 "per_model": [
  {
   "model": "claude-haiku-4-5-20251001",
   "config": "",
   "is_frontier": false,
   "datasets": [
    "mimic",
    "padchest",
    "vindr"
   ],
   "n_datasets": 3,
   "n_pairs": 6199,
   "n_flips": 690,
   "pooled_flip_pct": 11.13,
   "pooled_ci95": [
    10.37,
    11.94
   ],
   "macro_flip_pct": 11.05,
   "complete": true
  },
  {
   "model": "claude-opus-5",
   "config": "adaptive",
   "is_frontier": true,
   "datasets": [
    "mimic",
    "padchest",
    "vindr"
   ],
   "n_datasets": 3,
   "n_pairs": 6209,
   "n_flips": 592,
   "pooled_flip_pct": 9.53,
   "pooled_ci95": [
    8.83,
    10.29
   ],
   "macro_flip_pct": 9.67,
   "complete": true
  },
  {
   "model": "claude-opus-5",
   "config": "disabled",
   "is_frontier": true,
   "datasets": [
    "mimic",
    "padchest",
    "vindr"
   ],
   "n_datasets": 3,
   "n_pairs": 6194,
   "n_flips": 518,
   "pooled_flip_pct": 8.36,
   "pooled_ci95": [
    7.7,
    9.08
   ],
   "macro_flip_pct": 8.56,
   "complete": true
  },
  {
   "model": "gpt-5-mini-2025-08-07",
   "config": "low",
   "is_frontier": false,
   "datasets": [
    "mimic",
    "padchest",
    "vindr"
   ],
   "n_datasets": 3,
   "n_pairs": 6192,
   "n_flips": 367,
   "pooled_flip_pct": 5.93,
   "pooled_ci95": [
    5.37,
    6.54
   ],
   "macro_flip_pct": 6.24,
   "complete": true
  },
  {
   "model": "gpt-5.6-sol",
   "config": "low",
   "is_frontier": true,
   "datasets": [
    "mimic",
    "padchest",
    "vindr"
   ],
   "n_datasets": 3,
   "n_pairs": 6208,
   "n_flips": 603,
   "pooled_flip_pct": 9.71,
   "pooled_ci95": [
    9.0,
    10.48
   ],
   "macro_flip_pct": 10.15,
   "complete": true
  },
  {
   "model": "kimi-k3",
   "config": "low",
   "is_frontier": true,
   "datasets": [
    "mimic",
    "padchest",
    "vindr"
   ],
   "n_datasets": 3,
   "n_pairs": 5725,
   "n_flips": 327,
   "pooled_flip_pct": 5.71,
   "pooled_ci95": [
    5.14,
    6.34
   ],
   "macro_flip_pct": 5.97,
   "complete": true
  }
 ],
 "per_cell": [
  {
   "model": "claude-haiku-4-5-20251001",
   "dataset": "mimic",
   "config": "",
   "is_frontier": false,
   "n_questions": 499,
   "n_pairs": 1901,
   "n_flips": 99,
   "flip_rate": 5.21,
   "flip_ci95": [
    4.3,
    6.3
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 61.92,
   "pred_yes_rate": 89.18,
   "ref_yes_rate": 58.72,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_mimic_b2_haiku.json"
  },
  {
   "model": "claude-haiku-4-5-20251001",
   "dataset": "padchest",
   "config": "",
   "is_frontier": false,
   "n_questions": 500,
   "n_pairs": 2370,
   "n_flips": 279,
   "flip_rate": 11.77,
   "flip_ci95": [
    10.54,
    13.13
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 87.4,
   "pred_yes_rate": 33.4,
   "ref_yes_rate": 40.4,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_padchest_b2_haiku.json"
  },
  {
   "model": "claude-haiku-4-5-20251001",
   "dataset": "vindr",
   "config": "",
   "is_frontier": false,
   "n_questions": 499,
   "n_pairs": 1928,
   "n_flips": 312,
   "flip_rate": 16.18,
   "flip_ci95": [
    14.61,
    17.89
   ],
   "parse_clean_pct": 99.79,
   "accuracy": 30.06,
   "pred_yes_rate": 30.06,
   "ref_yes_rate": 100.0,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_vindr_b2_haiku.json"
  },
  {
   "model": "claude-opus-5",
   "dataset": "mimic",
   "config": "adaptive",
   "is_frontier": true,
   "n_questions": 500,
   "n_pairs": 1904,
   "n_flips": 150,
   "flip_rate": 7.88,
   "flip_ci95": [
    6.75,
    9.17
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 63.8,
   "pred_yes_rate": 69.2,
   "ref_yes_rate": 58.6,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_mimic_b2_opus5_on.json"
  },
  {
   "model": "claude-opus-5",
   "dataset": "padchest",
   "config": "adaptive",
   "is_frontier": true,
   "n_questions": 500,
   "n_pairs": 2370,
   "n_flips": 180,
   "flip_rate": 7.59,
   "flip_ci95": [
    6.6,
    8.73
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 72.0,
   "pred_yes_rate": 27.6,
   "ref_yes_rate": 40.4,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_padchest_b2_opus5_on.json"
  },
  {
   "model": "claude-opus-5",
   "dataset": "vindr",
   "config": "adaptive",
   "is_frontier": true,
   "n_questions": 500,
   "n_pairs": 1935,
   "n_flips": 262,
   "flip_rate": 13.54,
   "flip_ci95": [
    12.09,
    15.14
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 32.8,
   "pred_yes_rate": 32.8,
   "ref_yes_rate": 100.0,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_vindr_b2_opus5_on.json"
  },
  {
   "model": "claude-opus-5",
   "dataset": "mimic",
   "config": "disabled",
   "is_frontier": true,
   "n_questions": 498,
   "n_pairs": 1892,
   "n_flips": 176,
   "flip_rate": 9.3,
   "flip_ci95": [
    8.07,
    10.69
   ],
   "parse_clean_pct": 99.79,
   "accuracy": 61.24,
   "pred_yes_rate": 58.23,
   "ref_yes_rate": 58.43,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_mimic_b2_opus5_off.json"
  },
  {
   "model": "claude-opus-5",
   "dataset": "padchest",
   "config": "disabled",
   "is_frontier": true,
   "n_questions": 500,
   "n_pairs": 2368,
   "n_flips": 137,
   "flip_rate": 5.79,
   "flip_ci95": [
    4.91,
    6.8
   ],
   "parse_clean_pct": 99.92,
   "accuracy": 72.0,
   "pred_yes_rate": 14.8,
   "ref_yes_rate": 40.4,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_padchest_b2_opus5_off.json"
  },
  {
   "model": "claude-opus-5",
   "dataset": "vindr",
   "config": "disabled",
   "is_frontier": true,
   "n_questions": 500,
   "n_pairs": 1934,
   "n_flips": 205,
   "flip_rate": 10.6,
   "flip_ci95": [
    9.31,
    12.05
   ],
   "parse_clean_pct": 99.95,
   "accuracy": 15.0,
   "pred_yes_rate": 15.0,
   "ref_yes_rate": 100.0,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_claude_batch/results_vindr_b2_opus5_off.json"
  },
  {
   "model": "gpt-5-mini-2025-08-07",
   "dataset": "mimic",
   "config": "low",
   "is_frontier": false,
   "n_questions": 497,
   "n_pairs": 1887,
   "n_flips": 211,
   "flip_rate": 11.18,
   "flip_ci95": [
    9.84,
    12.68
   ],
   "parse_clean_pct": 99.63,
   "accuracy": 68.21,
   "pred_yes_rate": 58.75,
   "ref_yes_rate": 58.75,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_gpt5mini_batch/results_mimic_b2_gpt5mini.json"
  },
  {
   "model": "gpt-5-mini-2025-08-07",
   "dataset": "padchest",
   "config": "low",
   "is_frontier": false,
   "n_questions": 500,
   "n_pairs": 2370,
   "n_flips": 57,
   "flip_rate": 2.41,
   "flip_ci95": [
    1.86,
    3.1
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 80.4,
   "pred_yes_rate": 20.8,
   "ref_yes_rate": 40.4,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_gpt5mini_batch/results_padchest_b2_gpt5mini.json"
  },
  {
   "model": "gpt-5-mini-2025-08-07",
   "dataset": "vindr",
   "config": "low",
   "is_frontier": false,
   "n_questions": 500,
   "n_pairs": 1935,
   "n_flips": 99,
   "flip_rate": 5.12,
   "flip_ci95": [
    4.22,
    6.19
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 4.2,
   "pred_yes_rate": 4.2,
   "ref_yes_rate": 100.0,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_gpt5mini_batch/results_vindr_b2_gpt5mini.json"
  },
  {
   "model": "gpt-5.6-sol",
   "dataset": "mimic",
   "config": "low",
   "is_frontier": true,
   "n_questions": 500,
   "n_pairs": 1903,
   "n_flips": 170,
   "flip_rate": 8.93,
   "flip_ci95": [
    7.73,
    10.3
   ],
   "parse_clean_pct": 99.95,
   "accuracy": 70.6,
   "pred_yes_rate": 72.4,
   "ref_yes_rate": 58.6,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_gpt5mini_batch/results_mimic_b2_sol.json"
  },
  {
   "model": "gpt-5.6-sol",
   "dataset": "padchest",
   "config": "low",
   "is_frontier": true,
   "n_questions": 500,
   "n_pairs": 2370,
   "n_flips": 91,
   "flip_rate": 3.84,
   "flip_ci95": [
    3.14,
    4.69
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 87.6,
   "pred_yes_rate": 28.4,
   "ref_yes_rate": 40.4,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_gpt5mini_batch/results_padchest_b2_sol.json"
  },
  {
   "model": "gpt-5.6-sol",
   "dataset": "vindr",
   "config": "low",
   "is_frontier": true,
   "n_questions": 500,
   "n_pairs": 1935,
   "n_flips": 342,
   "flip_rate": 17.67,
   "flip_ci95": [
    16.04,
    19.44
   ],
   "parse_clean_pct": 100.0,
   "accuracy": 35.8,
   "pred_yes_rate": 35.8,
   "ref_yes_rate": 100.0,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_gpt5mini_batch/results_vindr_b2_sol.json"
  },
  {
   "model": "kimi-k3",
   "dataset": "mimic",
   "config": "low",
   "is_frontier": true,
   "n_questions": 474,
   "n_pairs": 1634,
   "n_flips": 112,
   "flip_rate": 6.85,
   "flip_ci95": [
    5.73,
    8.18
   ],
   "parse_clean_pct": 90.23,
   "accuracy": 66.88,
   "pred_yes_rate": 48.52,
   "ref_yes_rate": 58.86,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_kimi/results_mimic_b2_kimi3.json"
  },
  {
   "model": "kimi-k3",
   "dataset": "padchest",
   "config": "low",
   "is_frontier": true,
   "n_questions": 490,
   "n_pairs": 2274,
   "n_flips": 70,
   "flip_rate": 3.08,
   "flip_ci95": [
    2.44,
    3.87
   ],
   "parse_clean_pct": 97.76,
   "accuracy": 75.31,
   "pred_yes_rate": 14.49,
   "ref_yes_rate": 39.18,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_kimi/results_padchest_b2_kimi3.json"
  },
  {
   "model": "kimi-k3",
   "dataset": "vindr",
   "config": "low",
   "is_frontier": true,
   "n_questions": 483,
   "n_pairs": 1817,
   "n_flips": 145,
   "flip_rate": 7.98,
   "flip_ci95": [
    6.82,
    9.32
   ],
   "parse_clean_pct": 96.75,
   "accuracy": 9.32,
   "pred_yes_rate": 9.32,
   "ref_yes_rate": 100.0,
   "population": "psf_binary_inclusion_manifest.jsonl",
   "source_file": "data/psf_kimi/results_vindr_b2_kimi3.json"
  }
 ],
 "image_reliance": {
  "population": "psf_binary_inclusion_manifest.jsonl",
  "selection": "500 hardest binary questions per dataset",
  "flip_arms": {
   "gpt-5-mini": {
    "mimic": {
     "n_questions": 497,
     "n_pairs": 1887,
     "n_flips": 211,
     "flip_rate": 11.18,
     "flip_ci95": [
      9.84,
      12.68
     ],
     "flip_ci95_cluster": [
      9.09,
      13.28
     ],
     "accuracy": 68.21,
     "ref_yes_rate": 58.75,
     "majority_class_baseline": 58.75,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 58.75,
     "questions_dropped": 3,
     "pairs_dropped": 7,
     "pairs_abstained": 0,
     "parse_clean_pct": 99.63
    },
    "padchest": {
     "n_questions": 500,
     "n_pairs": 2370,
     "n_flips": 57,
     "flip_rate": 2.41,
     "flip_ci95": [
      1.86,
      3.1
     ],
     "flip_ci95_cluster": [
      1.44,
      3.46
     ],
     "accuracy": 80.4,
     "ref_yes_rate": 40.4,
     "majority_class_baseline": 59.6,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 20.8,
     "questions_dropped": 0,
     "pairs_dropped": 0,
     "pairs_abstained": 0,
     "parse_clean_pct": 100.0
    },
    "vindr": {
     "n_questions": 500,
     "n_pairs": 1935,
     "n_flips": 99,
     "flip_rate": 5.12,
     "flip_ci95": [
      4.22,
      6.19
     ],
     "flip_ci95_cluster": [
      3.84,
      6.56
     ],
     "accuracy": 4.2,
     "ref_yes_rate": 100.0,
     "majority_class_baseline": 100.0,
     "ref_is_degenerate": true,
     "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
     "pred_yes_rate": 4.2,
     "questions_dropped": 0,
     "pairs_dropped": 0,
     "pairs_abstained": 0,
     "parse_clean_pct": 100.0
    }
   },
   "gpt-5.6-sol": {
    "mimic": {
     "n_questions": 500,
     "n_pairs": 1903,
     "n_flips": 170,
     "flip_rate": 8.93,
     "flip_ci95": [
      7.73,
      10.3
     ],
     "flip_ci95_cluster": [
      7.01,
      11.08
     ],
     "accuracy": 70.6,
     "ref_yes_rate": 58.6,
     "majority_class_baseline": 58.6,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 72.4,
     "questions_dropped": 0,
     "pairs_dropped": 1,
     "pairs_abstained": 0,
     "parse_clean_pct": 99.95
    },
    "padchest": {
     "n_questions": 500,
     "n_pairs": 2370,
     "n_flips": 91,
     "flip_rate": 3.84,
     "flip_ci95": [
      3.14,
      4.69
     ],
     "flip_ci95_cluster": [
      2.66,
      5.12
     ],
     "accuracy": 87.6,
     "ref_yes_rate": 40.4,
     "majority_class_baseline": 59.6,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 28.4,
     "questions_dropped": 0,
     "pairs_dropped": 0,
     "pairs_abstained": 0,
     "parse_clean_pct": 100.0
    },
    "vindr": {
     "n_questions": 500,
     "n_pairs": 1935,
     "n_flips": 342,
     "flip_rate": 17.67,
     "flip_ci95": [
      16.04,
      19.44
     ],
     "flip_ci95_cluster": [
      15.43,
      20.1
     ],
     "accuracy": 35.8,
     "ref_yes_rate": 100.0,
     "majority_class_baseline": 100.0,
     "ref_is_degenerate": true,
     "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
     "pred_yes_rate": 35.8,
     "questions_dropped": 0,
     "pairs_dropped": 0,
     "pairs_abstained": 0,
     "parse_clean_pct": 100.0
    }
   },
   "claude-opus-5-adaptive": {
    "mimic": {
     "n_questions": 500,
     "n_pairs": 1904,
     "n_flips": 150,
     "flip_rate": 7.88,
     "flip_ci95": [
      6.75,
      9.17
     ],
     "flip_ci95_cluster": [
      6.13,
      9.69
     ],
     "accuracy": 63.8,
     "ref_yes_rate": 58.6,
     "majority_class_baseline": 58.6,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 69.2,
     "questions_dropped": 0,
     "pairs_dropped": 0,
     "pairs_abstained": 0,
     "parse_clean_pct": 100.0
    },
    "padchest": {
     "n_questions": 500,
     "n_pairs": 2370,
     "n_flips": 180,
     "flip_rate": 7.59,
     "flip_ci95": [
      6.6,
      8.73
     ],
     "flip_ci95_cluster": [
      5.87,
      9.35
     ],
     "accuracy": 72.0,
     "ref_yes_rate": 40.4,
     "majority_class_baseline": 59.6,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 27.6,
     "questions_dropped": 0,
     "pairs_dropped": 0,
     "pairs_abstained": 0,
     "parse_clean_pct": 100.0
    },
    "vindr": {
     "n_questions": 500,
     "n_pairs": 1935,
     "n_flips": 262,
     "flip_rate": 13.54,
     "flip_ci95": [
      12.09,
      15.14
     ],
     "flip_ci95_cluster": [
      11.38,
      15.75
     ],
     "accuracy": 32.8,
     "ref_yes_rate": 100.0,
     "majority_class_baseline": 100.0,
     "ref_is_degenerate": true,
     "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
     "pred_yes_rate": 32.8,
     "questions_dropped": 0,
     "pairs_dropped": 0,
     "pairs_abstained": 0,
     "parse_clean_pct": 100.0
    }
   },
   "claude-opus-5-disabled": {
    "mimic": {
     "n_questions": 498,
     "n_pairs": 1892,
     "n_flips": 176,
     "flip_rate": 9.3,
     "flip_ci95": [
      8.07,
      10.69
     ],
     "flip_ci95_cluster": [
      7.49,
      11.35
     ],
     "accuracy": 61.24,
     "ref_yes_rate": 58.43,
     "majority_class_baseline": 58.43,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 58.23,
     "questions_dropped": 2,
     "pairs_dropped": 4,
     "pairs_abstained": 0,
     "parse_clean_pct": 99.79
    },
    "padchest": {
     "n_questions": 500,
     "n_pairs": 2368,
     "n_flips": 137,
     "flip_rate": 5.79,
     "flip_ci95": [
      4.91,
      6.8
     ],
     "flip_ci95_cluster": [
      4.34,
      7.39
     ],
     "accuracy": 72.0,
     "ref_yes_rate": 40.4,
     "majority_class_baseline": 59.6,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 14.8,
     "questions_dropped": 0,
     "pairs_dropped": 2,
     "pairs_abstained": 0,
     "parse_clean_pct": 99.92
    },
    "vindr": {
     "n_questions": 500,
     "n_pairs": 1934,
     "n_flips": 205,
     "flip_rate": 10.6,
     "flip_ci95": [
      9.31,
      12.05
     ],
     "flip_ci95_cluster": [
      8.78,
      12.64
     ],
     "accuracy": 15.0,
     "ref_yes_rate": 100.0,
     "majority_class_baseline": 100.0,
     "ref_is_degenerate": true,
     "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
     "pred_yes_rate": 15.0,
     "questions_dropped": 0,
     "pairs_dropped": 1,
     "pairs_abstained": 0,
     "parse_clean_pct": 99.95
    }
   },
   "kimi-k3": {
    "mimic": {
     "n_questions": 474,
     "n_pairs": 1634,
     "n_flips": 112,
     "flip_rate": 6.85,
     "flip_ci95": [
      5.73,
      8.18
     ],
     "flip_ci95_cluster": [
      5.27,
      8.61
     ],
     "accuracy": 66.88,
     "ref_yes_rate": 58.86,
     "majority_class_baseline": 58.86,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 48.52,
     "questions_dropped": 26,
     "pairs_dropped": 177,
     "pairs_abstained": 0,
     "parse_clean_pct": 90.23
    },
    "padchest": {
     "n_questions": 490,
     "n_pairs": 2274,
     "n_flips": 70,
     "flip_rate": 3.08,
     "flip_ci95": [
      2.44,
      3.87
     ],
     "flip_ci95_cluster": [
      2.07,
      4.22
     ],
     "accuracy": 75.31,
     "ref_yes_rate": 39.18,
     "majority_class_baseline": 60.82,
     "ref_is_degenerate": false,
     "accuracy_meaning": "accuracy",
     "pred_yes_rate": 14.49,
     "questions_dropped": 10,
     "pairs_dropped": 52,
     "pairs_abstained": 0,
     "parse_clean_pct": 97.76
    },
    "vindr": {
     "n_questions": 483,
     "n_pairs": 1817,
     "n_flips": 145,
     "flip_rate": 7.98,
     "flip_ci95": [
      6.82,
      9.32
     ],
     "flip_ci95_cluster": [
      6.35,
      9.78
     ],
     "accuracy": 9.32,
     "ref_yes_rate": 100.0,
     "majority_class_baseline": 100.0,
     "ref_is_degenerate": true,
     "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
     "pred_yes_rate": 9.32,
     "questions_dropped": 17,
     "pairs_dropped": 61,
     "pairs_abstained": 0,
     "parse_clean_pct": 96.75
    }
   }
  },
  "image_reliance": {
   "gpt-5-mini": {
    "mimic": [
     {
      "control": "replicate",
      "n_scored": 496,
      "n_distinct_question_texts": 149,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 1,
      "agreement_with_real_image": 92.54,
      "agreement_ci95": [
       89.89,
       94.54
      ],
      "agreement_ci95_cluster_by_question_text": [
       89.57,
       94.17
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 7.46,
      "chance_agreement": 51.18,
      "cohens_kappa": 0.847,
      "cohens_kappa_ci95": [
       0.8,
       0.892
      ],
      "ctl_pred_yes_rate": 56.65,
      "real_pred_yes_rate": 58.87,
      "ctl_accuracy": 66.73,
      "real_accuracy": 68.15,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 191,
       "no->yes": 13,
       "yes->no": 24,
       "yes->yes": 268
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 15,
       "real_right_ctl_wrong": 22,
       "mcnemar_p": 0.3240086000878364
      }
     },
     {
      "control": "swap",
      "n_scored": 495,
      "n_distinct_question_texts": 148,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 2,
      "agreement_with_real_image": 58.38,
      "agreement_ci95": [
       53.99,
       62.65
      ],
      "agreement_ci95_cluster_by_question_text": [
       53.31,
       63.14
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 399,
       "agreement": 55.39,
       "ci95": [
        50.48,
        60.19
       ],
       "ci95_cluster": [
        51.2,
        63.4
       ],
       "swap_partition": {
        "undetermined": 354,
        "changing": 49,
        "preserving": 97
       }
      },
      "answer_change_rate": 41.62,
      "chance_agreement": 51.33,
      "cohens_kappa": 0.145,
      "cohens_kappa_ci95": [
       0.056,
       0.23
      ],
      "ctl_pred_yes_rate": 57.58,
      "real_pred_yes_rate": 58.79,
      "ctl_accuracy": 56.97,
      "real_accuracy": 68.28,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 104,
       "no->yes": 100,
       "yes->no": 106,
       "yes->yes": 185
      },
      "vs_replicate_ceiling": {
       "ctl_agrees_rep_not": 18,
       "rep_agrees_ctl_not": 187,
       "mcnemar_p": 1.2727936412119112e-36,
       "mcnemar_p_cluster_by_question_text": 5e-05,
       "n_discordant_question_texts": 66,
       "direction": "below_ceiling",
       "significant": true,
       "significant_clustered": true
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 75,
       "real_right_ctl_wrong": 131,
       "mcnemar_p": 0.00011629368668712308
      }
     },
     {
      "control": "textonly",
      "n_scored": 493,
      "n_distinct_question_texts": 147,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 4,
      "agreement_with_real_image": 59.03,
      "agreement_ci95": [
       54.63,
       63.28
      ],
      "agreement_ci95_cluster_by_question_text": [
       52.45,
       64.1
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 40.97,
      "chance_agreement": 51.62,
      "cohens_kappa": 0.153,
      "cohens_kappa_ci95": [
       0.064,
       0.241
      ],
      "ctl_pred_yes_rate": 59.63,
      "real_pred_yes_rate": 58.42,
      "ctl_accuracy": 54.97,
      "real_accuracy": 68.36,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 101,
       "no->yes": 104,
       "yes->no": 98,
       "yes->yes": 190
      },
      "vs_replicate_ceiling": {
       "ctl_agrees_rep_not": 19,
       "rep_agrees_ctl_not": 183,
       "mcnemar_p": 7.536096414309635e-35,
       "mcnemar_p_cluster_by_question_text": 5e-05,
       "n_discordant_question_texts": 64,
       "direction": "below_ceiling",
       "significant": true,
       "significant_clustered": true
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 68,
       "real_right_ctl_wrong": 134,
       "mcnemar_p": 3.9796172586991415e-06
      }
     }
    ],
    "padchest": [
     {
      "control": "replicate",
      "n_scored": 500,
      "n_distinct_question_texts": 47,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 98.0,
      "agreement_ci95": [
       96.36,
       98.91
      ],
      "agreement_ci95_cluster_by_question_text": [
       92.68,
       99.44
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 2.0,
      "chance_agreement": 66.82,
      "cohens_kappa": 0.94,
      "cohens_kappa_ci95": [
       0.9,
       0.974
      ],
      "ctl_pred_yes_rate": 21.2,
      "real_pred_yes_rate": 20.8,
      "ctl_accuracy": 80.8,
      "real_accuracy": 80.4,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 390,
       "no->yes": 6,
       "yes->no": 4,
       "yes->yes": 100
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 6,
       "real_right_ctl_wrong": 4,
       "mcnemar_p": 0.75390625
      }
     },
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 47,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 85.4,
      "agreement_ci95": [
       82.04,
       88.23
      ],
      "agreement_ci95_cluster_by_question_text": [
       59.38,
       94.16
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 258,
       "agreement": 72.87,
       "ci95": [
        67.14,
        77.93
       ],
       "ci95_cluster": [
        59.41,
        83.38
       ],
       "swap_partition": {
        "undetermined": 258,
        "preserving": 242
       }
      },
      "answer_change_rate": 14.6,
      "chance_agreement": 70.21,
      "cohens_kappa": 0.51,
      "cohens_kappa_ci95": [
       0.407,
       0.607
      ],
      "ctl_pred_yes_rate": 15.4,
      "real_pred_yes_rate": 20.8,
      "ctl_accuracy": 73.4,
      "real_accuracy": 80.4,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 373,
       "no->yes": 23,
       "yes->no": 50,
       "yes->yes": 54
      },
      "vs_replicate_ceiling": {
       "ctl_agrees_rep_not": 6,
       "rep_agrees_ctl_not": 69,
       "mcnemar_p": 1.1641661378765454e-14,
       "mcnemar_p_cluster_by_question_text": 5e-05,
       "n_discordant_question_texts": 22,
       "direction": "below_ceiling",
       "significant": true,
       "significant_clustered": true
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 19,
       "real_right_ctl_wrong": 54,
       "mcnemar_p": 5.0622659111780655e-05
      }
     },
     {
      "control": "textonly",
      "n_scored": 499,
      "n_distinct_question_texts": 47,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 1,
      "agreement_with_real_image": 81.76,
      "agreement_ci95": [
       78.14,
       84.9
      ],
      "agreement_ci95_cluster_by_question_text": [
       54.77,
       90.55
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 18.24,
      "chance_agreement": 61.98,
      "cohens_kappa": 0.52,
      "cohens_kappa_ci95": [
       0.436,
       0.603
      ],
      "ctl_pred_yes_rate": 29.46,
      "real_pred_yes_rate": 20.84,
      "ctl_accuracy": 82.97,
      "real_accuracy": 80.36,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 328,
       "no->yes": 67,
       "yes->no": 24,
       "yes->yes": 80
      },
      "vs_replicate_ceiling": {
       "ctl_agrees_rep_not": 5,
       "rep_agrees_ctl_not": 86,
       "mcnemar_p": 3.982648467244083e-20,
       "mcnemar_p_cluster_by_question_text": 5e-05,
       "n_discordant_question_texts": 22,
       "direction": "below_ceiling",
       "significant": true,
       "significant_clustered": true
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 52,
       "real_right_ctl_wrong": 39,
       "mcnemar_p": 0.20817442238117587
      }
     }
    ],
    "vindr": [
     {
      "control": "replicate",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 97.0,
      "agreement_ci95": [
       95.11,
       98.17
      ],
      "agreement_ci95_cluster_by_question_text": [
       93.41,
       98.55
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 3.0,
      "chance_agreement": 92.5,
      "cohens_kappa": 0.6,
      "cohens_kappa_ci95": [
       0.396,
       0.77
      ],
      "ctl_pred_yes_rate": 3.6,
      "real_pred_yes_rate": 4.2,
      "ctl_accuracy": 3.6,
      "real_accuracy": 4.2,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 473,
       "no->yes": 6,
       "yes->no": 9,
       "yes->yes": 12
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 6,
       "real_right_ctl_wrong": 9,
       "mcnemar_p": 0.60723876953125
      }
     },
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 92.8,
      "agreement_ci95": [
       90.19,
       94.75
      ],
      "agreement_ci95_cluster_by_question_text": [
       85.8,
       95.01
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 331,
       "agreement": 92.75,
       "ci95": [
        89.44,
        95.08
       ],
       "ci95_cluster": [
        84.67,
        95.53
       ],
       "swap_partition": {
        "undetermined": 331,
        "preserving": 169
       }
      },
      "answer_change_rate": 7.2,
      "chance_agreement": 91.22,
      "cohens_kappa": 0.18,
      "cohens_kappa_ci95": [
       0.02,
       0.344
      ],
      "ctl_pred_yes_rate": 5.0,
      "real_pred_yes_rate": 4.2,
      "ctl_accuracy": 5.0,
      "real_accuracy": 4.2,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 459,
       "no->yes": 20,
       "yes->no": 16,
       "yes->yes": 5
      },
      "vs_replicate_ceiling": {
       "ctl_agrees_rep_not": 5,
       "rep_agrees_ctl_not": 26,
       "mcnemar_p": 0.00019219517707824707,
       "mcnemar_p_cluster_by_question_text": 0.06995,
       "n_discordant_question_texts": 8,
       "direction": "below_ceiling",
       "significant": true,
       "significant_clustered": false
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 20,
       "real_right_ctl_wrong": 16,
       "mcnemar_p": 0.617719317029696
      }
     },
     {
      "control": "textonly",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 74.8,
      "agreement_ci95": [
       70.82,
       78.41
      ],
      "agreement_ci95_cluster_by_question_text": [
       51.19,
       91.21
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 25.2,
      "chance_agreement": 75.1,
      "cohens_kappa": -0.012,
      "cohens_kappa_ci95": [
       -0.062,
       0.05
      ],
      "ctl_pred_yes_rate": 22.6,
      "real_pred_yes_rate": 4.2,
      "ctl_accuracy": 22.6,
      "real_accuracy": 4.2,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 370,
       "no->yes": 109,
       "yes->no": 17,
       "yes->yes": 4
      },
      "vs_replicate_ceiling": {
       "ctl_agrees_rep_not": 6,
       "rep_agrees_ctl_not": 117,
       "mcnemar_p": 8.41628443011684e-28,
       "mcnemar_p_cluster_by_question_text": 0.0017,
       "n_discordant_question_texts": 10,
       "direction": "below_ceiling",
       "significant": true,
       "significant_clustered": true
      },
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 109,
       "real_right_ctl_wrong": 17,
       "mcnemar_p": 1.2816437724907014e-17
      }
     }
    ]
   },
   "gpt-5.6-sol": {
    "mimic": [
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 151,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 68.0,
      "agreement_ci95": [
       63.79,
       71.94
      ],
      "agreement_ci95_cluster_by_question_text": [
       55.86,
       73.58
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 403,
       "agreement": 64.52,
       "ci95": [
        59.73,
        69.03
       ],
       "ci95_cluster": [
        56.4,
        69.4
       ],
       "swap_partition": {
        "undetermined": 354,
        "changing": 49,
        "preserving": 97
       }
      },
      "answer_change_rate": 32.0,
      "chance_agreement": 57.17,
      "cohens_kappa": 0.253,
      "cohens_kappa_ci95": [
       0.163,
       0.337
      ],
      "ctl_pred_yes_rate": 66.0,
      "real_pred_yes_rate": 72.4,
      "ctl_accuracy": 61.8,
      "real_accuracy": 70.6,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 74,
       "no->yes": 64,
       "yes->no": 96,
       "yes->yes": 266
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 58,
       "real_right_ctl_wrong": 102,
       "mcnemar_p": 0.0006293470747506888
      }
     },
     {
      "control": "textonly",
      "n_scored": 500,
      "n_distinct_question_texts": 151,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 37.8,
      "agreement_ci95": [
       33.66,
       42.13
      ],
      "agreement_ci95_cluster_by_question_text": [
       34.37,
       45.38
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 62.2,
      "chance_agreement": 35.4,
      "cohens_kappa": 0.037,
      "cohens_kappa_ci95": [
       -0.008,
       0.081
      ],
      "ctl_pred_yes_rate": 17.4,
      "real_pred_yes_rate": 72.4,
      "ctl_accuracy": 44.4,
      "real_accuracy": 70.6,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 120,
       "no->yes": 18,
       "yes->no": 293,
       "yes->yes": 69
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 90,
       "real_right_ctl_wrong": 221,
       "mcnemar_p": 7.193818045116729e-14
      }
     }
    ],
    "padchest": [
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 47,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 77.0,
      "agreement_ci95": [
       73.11,
       80.47
      ],
      "agreement_ci95_cluster_by_question_text": [
       42.77,
       88.33
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 258,
       "agreement": 57.75,
       "ci95": [
        51.65,
        63.62
       ],
       "ci95_cluster": [
        41.86,
        69.23
       ],
       "swap_partition": {
        "undetermined": 258,
        "preserving": 242
       }
      },
      "answer_change_rate": 23.0,
      "chance_agreement": 64.08,
      "cohens_kappa": 0.36,
      "cohens_kappa_ci95": [
       0.267,
       0.445
      ],
      "ctl_pred_yes_rate": 17.4,
      "real_pred_yes_rate": 28.4,
      "ctl_accuracy": 71.0,
      "real_accuracy": 87.6,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 328,
       "no->yes": 30,
       "yes->no": 85,
       "yes->yes": 57
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 16,
       "real_right_ctl_wrong": 99,
       "mcnemar_p": 8.564981783013103e-16
      }
     },
     {
      "control": "textonly",
      "n_scored": 500,
      "n_distinct_question_texts": 47,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 74.2,
      "agreement_ci95": [
       70.19,
       77.84
      ],
      "agreement_ci95_cluster_by_question_text": [
       31.33,
       90.76
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 25.8,
      "chance_agreement": 70.3,
      "cohens_kappa": 0.131,
      "cohens_kappa_ci95": [
       0.068,
       0.201
      ],
      "ctl_pred_yes_rate": 3.0,
      "real_pred_yes_rate": 28.4,
      "ctl_accuracy": 62.6,
      "real_accuracy": 87.6,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 357,
       "no->yes": 1,
       "yes->no": 128,
       "yes->yes": 14
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 2,
       "real_right_ctl_wrong": 127,
       "mcnemar_p": 2.464423906498926e-35
      }
     }
    ],
    "vindr": [
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 58.2,
      "agreement_ci95": [
       53.83,
       62.44
      ],
      "agreement_ci95_cluster_by_question_text": [
       51.52,
       74.36
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 331,
       "agreement": 58.31,
       "ci95": [
        52.93,
        63.49
       ],
       "ci95_cluster": [
        49.46,
        73.78
       ],
       "swap_partition": {
        "undetermined": 331,
        "preserving": 169
       }
      },
      "answer_change_rate": 41.8,
      "chance_agreement": 55.23,
      "cohens_kappa": 0.066,
      "cohens_kappa_ci95": [
       -0.02,
       0.159
      ],
      "ctl_pred_yes_rate": 31.6,
      "real_pred_yes_rate": 35.8,
      "ctl_accuracy": 31.6,
      "real_accuracy": 35.8,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 227,
       "no->yes": 94,
       "yes->no": 115,
       "yes->yes": 64
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 94,
       "real_right_ctl_wrong": 115,
       "mcnemar_p": 0.1663794203232106
      }
     },
     {
      "control": "textonly",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 64.2,
      "agreement_ci95": [
       59.9,
       68.28
      ],
      "agreement_ci95_cluster_by_question_text": [
       55.79,
       82.5
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 35.8,
      "chance_agreement": 64.09,
      "cohens_kappa": 0.003,
      "cohens_kappa_ci95": [
       -0.009,
       0.021
      ],
      "ctl_pred_yes_rate": 0.4,
      "real_pred_yes_rate": 35.8,
      "ctl_accuracy": 0.4,
      "real_accuracy": 35.8,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 320,
       "no->yes": 1,
       "yes->no": 178,
       "yes->yes": 1
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 1,
       "real_right_ctl_wrong": 178,
       "mcnemar_p": 4.698219216958938e-52
      }
     }
    ]
   },
   "claude-opus-5-adaptive": {
    "mimic": [
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 151,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 72.0,
      "agreement_ci95": [
       67.91,
       75.76
      ],
      "agreement_ci95_cluster_by_question_text": [
       55.4,
       80.37
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 403,
       "agreement": 67.74,
       "ci95": [
        63.03,
        72.12
       ],
       "ci95_cluster": [
        55.6,
        75.66
       ],
       "swap_partition": {
        "undetermined": 354,
        "changing": 49,
        "preserving": 97
       }
      },
      "answer_change_rate": 28.0,
      "chance_agreement": 55.22,
      "cohens_kappa": 0.375,
      "cohens_kappa_ci95": [
       0.288,
       0.458
      ],
      "ctl_pred_yes_rate": 63.6,
      "real_pred_yes_rate": 69.2,
      "ctl_accuracy": 56.2,
      "real_accuracy": 63.8,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 98,
       "no->yes": 56,
       "yes->no": 84,
       "yes->yes": 262
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 51,
       "real_right_ctl_wrong": 89,
       "mcnemar_p": 0.0016675138859024257
      }
     },
     {
      "control": "textonly",
      "n_scored": 478,
      "n_distinct_question_texts": 151,
      "n_abstained": 22,
      "abstain_rate": 4.4,
      "n_unreadable": 0,
      "agreement_with_real_image": 72.38,
      "agreement_ci95": [
       68.21,
       76.2
      ],
      "agreement_ci95_cluster_by_question_text": [
       53.03,
       83.2
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 27.62,
      "chance_agreement": 58.61,
      "cohens_kappa": 0.333,
      "cohens_kappa_ci95": [
       0.236,
       0.423
      ],
      "ctl_pred_yes_rate": 73.64,
      "real_pred_yes_rate": 68.2,
      "ctl_accuracy": 62.76,
      "real_accuracy": 62.76,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 73,
       "no->yes": 79,
       "yes->no": 53,
       "yes->yes": 273
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 66,
       "real_right_ctl_wrong": 66,
       "mcnemar_p": 1.0
      }
     }
    ],
    "padchest": [
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 47,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 65.6,
      "agreement_ci95": [
       61.33,
       69.63
      ],
      "agreement_ci95_cluster_by_question_text": [
       46.25,
       71.63
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 258,
       "agreement": 53.49,
       "ci95": [
        47.4,
        59.48
       ],
       "ci95_cluster": [
        44.67,
        62.42
       ],
       "swap_partition": {
        "undetermined": 258,
        "preserving": 242
       }
      },
      "answer_change_rate": 34.4,
      "chance_agreement": 63.08,
      "cohens_kappa": 0.068,
      "cohens_kappa_ci95": [
       -0.022,
       0.16
      ],
      "ctl_pred_yes_rate": 20.8,
      "real_pred_yes_rate": 27.6,
      "ctl_accuracy": 57.6,
      "real_accuracy": 72.0,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 293,
       "no->yes": 69,
       "yes->no": 103,
       "yes->yes": 35
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 50,
       "real_right_ctl_wrong": 122,
       "mcnemar_p": 3.952460425923357e-08
      }
     },
     {
      "control": "textonly",
      "n_scored": 486,
      "n_distinct_question_texts": 47,
      "n_abstained": 14,
      "abstain_rate": 2.8,
      "n_unreadable": 0,
      "agreement_with_real_image": 75.1,
      "agreement_ci95": [
       71.07,
       78.74
      ],
      "agreement_ci95_cluster_by_question_text": [
       51.02,
       83.02
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 24.9,
      "chance_agreement": 65.4,
      "cohens_kappa": 0.28,
      "cohens_kappa_ci95": [
       0.187,
       0.376
      ],
      "ctl_pred_yes_rate": 15.02,
      "real_pred_yes_rate": 27.98,
      "ctl_accuracy": 73.46,
      "real_accuracy": 71.6,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 321,
       "no->yes": 29,
       "yes->no": 92,
       "yes->yes": 44
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 65,
       "real_right_ctl_wrong": 56,
       "mcnemar_p": 0.4672071020769661
      }
     }
    ],
    "vindr": [
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 61.8,
      "agreement_ci95": [
       57.47,
       65.95
      ],
      "agreement_ci95_cluster_by_question_text": [
       56.53,
       82.11
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 331,
       "agreement": 64.65,
       "ci95": [
        59.36,
        69.61
       ],
       "ci95_cluster": [
        57.07,
        82.65
       ],
       "swap_partition": {
        "undetermined": 331,
        "preserving": 169
       }
      },
      "answer_change_rate": 38.2,
      "chance_agreement": 57.5,
      "cohens_kappa": 0.101,
      "cohens_kappa_ci95": [
       0.011,
       0.188
      ],
      "ctl_pred_yes_rate": 28.2,
      "real_pred_yes_rate": 32.8,
      "ctl_accuracy": 28.2,
      "real_accuracy": 32.8,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 252,
       "no->yes": 84,
       "yes->no": 107,
       "yes->yes": 57
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 84,
       "real_right_ctl_wrong": 107,
       "mcnemar_p": 0.11117576581299024
      }
     },
     {
      "control": "textonly",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 50.0,
      "agreement_ci95": [
       45.63,
       54.37
      ],
      "agreement_ci95_cluster_by_question_text": [
       37.78,
       83.01
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 50.0,
      "chance_agreement": 41.88,
      "cohens_kappa": 0.14,
      "cohens_kappa_ci95": [
       0.082,
       0.198
      ],
      "ctl_pred_yes_rate": 73.6,
      "real_pred_yes_rate": 32.8,
      "ctl_accuracy": 73.6,
      "real_accuracy": 32.8,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 109,
       "no->yes": 227,
       "yes->no": 23,
       "yes->yes": 141
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 227,
       "real_right_ctl_wrong": 23,
       "mcnemar_p": 2.3782766514017757e-43
      }
     }
    ]
   },
   "claude-opus-5-disabled": {
    "mimic": [
     {
      "control": "swap",
      "n_scored": 495,
      "n_distinct_question_texts": 149,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 3,
      "agreement_with_real_image": 71.31,
      "agreement_ci95": [
       67.18,
       75.12
      ],
      "agreement_ci95_cluster_by_question_text": [
       62.5,
       75.03
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 398,
       "agreement": 68.59,
       "ci95": [
        63.87,
        72.96
       ],
       "ci95_cluster": [
        62.55,
        72.7
       ],
       "swap_partition": {
        "undetermined": 354,
        "changing": 49,
        "preserving": 97
       }
      },
      "answer_change_rate": 28.69,
      "chance_agreement": 51.2,
      "cohens_kappa": 0.412,
      "cohens_kappa_ci95": [
       0.329,
       0.492
      ],
      "ctl_pred_yes_rate": 57.17,
      "real_pred_yes_rate": 58.38,
      "ctl_accuracy": 53.74,
      "real_accuracy": 61.41,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 138,
       "no->yes": 68,
       "yes->no": 74,
       "yes->yes": 215
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 52,
       "real_right_ctl_wrong": 90,
       "mcnemar_p": 0.001801710219750172
      }
     },
     {
      "control": "textonly",
      "n_scored": 498,
      "n_distinct_question_texts": 150,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 72.89,
      "agreement_ci95": [
       68.82,
       76.61
      ],
      "agreement_ci95_cluster_by_question_text": [
       57.42,
       80.11
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 27.11,
      "chance_agreement": 53.37,
      "cohens_kappa": 0.419,
      "cohens_kappa_ci95": [
       0.339,
       0.498
      ],
      "ctl_pred_yes_rate": 70.48,
      "real_pred_yes_rate": 58.23,
      "ctl_accuracy": 59.04,
      "real_accuracy": 61.24,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 110,
       "no->yes": 98,
       "yes->no": 37,
       "yes->yes": 253
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 62,
       "real_right_ctl_wrong": 73,
       "mcnemar_p": 0.3895026289761405
      }
     }
    ],
    "padchest": [
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 47,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 81.0,
      "agreement_ci95": [
       77.33,
       84.2
      ],
      "agreement_ci95_cluster_by_question_text": [
       61.83,
       86.92
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 258,
       "agreement": 67.83,
       "ci95": [
        61.9,
        73.23
       ],
       "ci95_cluster": [
        60.59,
        76.54
       ],
       "swap_partition": {
        "undetermined": 258,
        "preserving": 242
       }
      },
      "answer_change_rate": 19.0,
      "chance_agreement": 78.86,
      "cohens_kappa": 0.101,
      "cohens_kappa_ci95": [
       0.001,
       0.207
      ],
      "ctl_pred_yes_rate": 9.0,
      "real_pred_yes_rate": 14.8,
      "ctl_accuracy": 58.2,
      "real_accuracy": 72.0,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 393,
       "no->yes": 33,
       "yes->no": 62,
       "yes->yes": 12
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 13,
       "real_right_ctl_wrong": 82,
       "mcnemar_p": 2.0830055605008275e-13
      }
     },
     {
      "control": "textonly",
      "n_scored": 500,
      "n_distinct_question_texts": 47,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 81.8,
      "agreement_ci95": [
       78.18,
       84.94
      ],
      "agreement_ci95_cluster_by_question_text": [
       49.05,
       92.42
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 18.2,
      "chance_agreement": 69.29,
      "cohens_kappa": 0.407,
      "cohens_kappa_ci95": [
       0.305,
       0.501
      ],
      "ctl_pred_yes_rate": 22.6,
      "real_pred_yes_rate": 14.8,
      "ctl_accuracy": 82.2,
      "real_accuracy": 72.0,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 361,
       "no->yes": 65,
       "yes->no": 26,
       "yes->yes": 48
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 71,
       "real_right_ctl_wrong": 20,
       "mcnemar_p": 7.245373680789234e-08
      }
     }
    ],
    "vindr": [
     {
      "control": "swap",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 77.6,
      "agreement_ci95": [
       73.74,
       81.04
      ],
      "agreement_ci95_cluster_by_question_text": [
       72.24,
       90.48
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 331,
       "agreement": 78.85,
       "ci95": [
        74.13,
        82.91
       ],
       "ci95_cluster": [
        70.97,
        90.7
       ],
       "swap_partition": {
        "undetermined": 331,
        "preserving": 169
       }
      },
      "answer_change_rate": 22.4,
      "chance_agreement": 76.18,
      "cohens_kappa": 0.06,
      "cohens_kappa_ci95": [
       -0.038,
       0.16
      ],
      "ctl_pred_yes_rate": 12.6,
      "real_pred_yes_rate": 15.0,
      "ctl_accuracy": 12.6,
      "real_accuracy": 15.0,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 375,
       "no->yes": 50,
       "yes->no": 62,
       "yes->yes": 13
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 50,
       "real_right_ctl_wrong": 62,
       "mcnemar_p": 0.29859285283284615
      }
     },
     {
      "control": "textonly",
      "n_scored": 500,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 0,
      "agreement_with_real_image": 37.4,
      "agreement_ci95": [
       33.27,
       41.72
      ],
      "agreement_ci95_cluster_by_question_text": [
       21.53,
       78.05
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 62.6,
      "chance_agreement": 34.04,
      "cohens_kappa": 0.051,
      "cohens_kappa_ci95": [
       0.013,
       0.088
      ],
      "ctl_pred_yes_rate": 72.8,
      "real_pred_yes_rate": 15.0,
      "ctl_accuracy": 72.8,
      "real_accuracy": 15.0,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 124,
       "no->yes": 301,
       "yes->no": 12,
       "yes->yes": 63
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 301,
       "real_right_ctl_wrong": 12,
       "mcnemar_p": 1.8606411228150827e-73
      }
     }
    ]
   },
   "kimi-k3": {
    "mimic": [
     {
      "control": "swap",
      "n_scored": 448,
      "n_distinct_question_texts": 139,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 26,
      "agreement_with_real_image": 60.71,
      "agreement_ci95": [
       56.12,
       65.13
      ],
      "agreement_ci95_cluster_by_question_text": [
       57.34,
       69.44
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 362,
       "agreement": 60.22,
       "ci95": [
        55.1,
        65.13
       ],
       "ci95_cluster": [
        55.39,
        69.44
       ],
       "swap_partition": {
        "undetermined": 354,
        "changing": 49,
        "preserving": 97
       }
      },
      "answer_change_rate": 39.29,
      "chance_agreement": 50.02,
      "cohens_kappa": 0.214,
      "cohens_kappa_ci95": [
       0.119,
       0.304
      ],
      "ctl_pred_yes_rate": 48.88,
      "real_pred_yes_rate": 48.88,
      "ctl_accuracy": 56.03,
      "real_accuracy": 67.19,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 141,
       "no->yes": 88,
       "yes->no": 88,
       "yes->yes": 131
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 63,
       "real_right_ctl_wrong": 113,
       "mcnemar_p": 0.00020218969307107755
      }
     },
     {
      "control": "textonly",
      "n_scored": 411,
      "n_distinct_question_texts": 130,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 63,
      "agreement_with_real_image": 52.31,
      "agreement_ci95": [
       47.48,
       57.1
      ],
      "agreement_ci95_cluster_by_question_text": [
       44.3,
       70.05
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 47.69,
      "chance_agreement": 50.26,
      "cohens_kappa": 0.041,
      "cohens_kappa_ci95": [
       -0.029,
       0.111
      ],
      "ctl_pred_yes_rate": 14.6,
      "real_pred_yes_rate": 49.64,
      "ctl_accuracy": 43.31,
      "real_accuracy": 67.15,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 181,
       "no->yes": 26,
       "yes->no": 170,
       "yes->yes": 34
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 49,
       "real_right_ctl_wrong": 147,
       "mcnemar_p": 1.4303853572223812e-12
      }
     }
    ],
    "padchest": [
     {
      "control": "swap",
      "n_scored": 481,
      "n_distinct_question_texts": 45,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 9,
      "agreement_with_real_image": 83.16,
      "agreement_ci95": [
       79.56,
       86.24
      ],
      "agreement_ci95_cluster_by_question_text": [
       54.22,
       91.86
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 240,
       "agreement": 67.08,
       "ci95": [
        60.91,
        72.72
       ],
       "ci95_cluster": [
        53.43,
        76.64
       ],
       "swap_partition": {
        "undetermined": 258,
        "preserving": 242
       }
      },
      "answer_change_rate": 16.84,
      "chance_agreement": 81.5,
      "cohens_kappa": 0.09,
      "cohens_kappa_ci95": [
       -0.004,
       0.193
      ],
      "ctl_pred_yes_rate": 5.82,
      "real_pred_yes_rate": 14.35,
      "ctl_accuracy": 62.99,
      "real_accuracy": 75.68,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 392,
       "no->yes": 20,
       "yes->no": 61,
       "yes->yes": 8
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 10,
       "real_right_ctl_wrong": 71,
       "mcnemar_p": 1.7993441704367062e-12
      }
     },
     {
      "control": "textonly",
      "n_scored": 447,
      "n_distinct_question_texts": 43,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 43,
      "agreement_with_real_image": 87.02,
      "agreement_ci95": [
       83.59,
       89.83
      ],
      "agreement_ci95_cluster_by_question_text": [
       57.99,
       95.28
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 12.98,
      "chance_agreement": 82.29,
      "cohens_kappa": 0.267,
      "cohens_kappa_ci95": [
       0.13,
       0.394
      ],
      "ctl_pred_yes_rate": 5.59,
      "real_pred_yes_rate": 13.65,
      "ctl_accuracy": 68.46,
      "real_accuracy": 77.4,
      "ref_is_degenerate": false,
      "accuracy_meaning": "accuracy",
      "confusion_real_ctl": {
       "no->no": 375,
       "no->yes": 11,
       "yes->no": 47,
       "yes->yes": 14
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 9,
       "real_right_ctl_wrong": 49,
       "mcnemar_p": 8.959402428981855e-08
      }
     }
    ],
    "vindr": [
     {
      "control": "swap",
      "n_scored": 468,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 15,
      "agreement_with_real_image": 85.04,
      "agreement_ci95": [
       81.53,
       87.99
      ],
      "agreement_ci95_cluster_by_question_text": [
       72.48,
       94.21
      ],
      "agreement_excluding_answer_preserving_swaps": {
       "n": 303,
       "agreement": 83.83,
       "ci95": [
        79.26,
        87.55
       ],
       "ci95_cluster": [
        72.35,
        94.03
       ],
       "swap_partition": {
        "undetermined": 331,
        "preserving": 169
       }
      },
      "answer_change_rate": 14.96,
      "chance_agreement": 84.7,
      "cohens_kappa": 0.022,
      "cohens_kappa_ci95": [
       -0.067,
       0.123
      ],
      "ctl_pred_yes_rate": 7.26,
      "real_pred_yes_rate": 9.4,
      "ctl_accuracy": 7.26,
      "real_accuracy": 9.4,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 394,
       "no->yes": 30,
       "yes->no": 40,
       "yes->yes": 4
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 30,
       "real_right_ctl_wrong": 40,
       "mcnemar_p": 0.2819789217936561
      }
     },
     {
      "control": "textonly",
      "n_scored": 462,
      "n_distinct_question_texts": 11,
      "n_abstained": 0,
      "abstain_rate": 0.0,
      "n_unreadable": 21,
      "agreement_with_real_image": 82.68,
      "agreement_ci95": [
       78.97,
       85.86
      ],
      "agreement_ci95_cluster_by_question_text": [
       62.7,
       96.02
      ],
      "agreement_excluding_answer_preserving_swaps": null,
      "answer_change_rate": 17.32,
      "chance_agreement": 80.3,
      "cohens_kappa": 0.121,
      "cohens_kappa_ci95": [
       0.009,
       0.233
      ],
      "ctl_pred_yes_rate": 12.77,
      "real_pred_yes_rate": 9.31,
      "ctl_accuracy": 12.77,
      "real_accuracy": 9.31,
      "ref_is_degenerate": true,
      "accuracy_meaning": "sensitivity/recall only (reference is one-class)",
      "confusion_real_ctl": {
       "no->no": 371,
       "no->yes": 48,
       "yes->no": 32,
       "yes->yes": 11
      },
      "vs_replicate_ceiling": null,
      "accuracy_vs_real_paired": {
       "ctl_right_real_wrong": 48,
       "real_right_ctl_wrong": 32,
       "mcnemar_p": 0.09291188222054621
      }
     }
    ]
   }
  },
  "quadrants": {
   "gpt-5-mini": {
    "mimic": {
     "n": 492,
     "paraphrase_consistent_pct": 76.22,
     "image_invariant_pct": 58.94,
     "consistent_and_image_invariant_pct": 46.95,
     "consistent_and_image_invariant_ci95": [
      42.58,
      51.37
     ],
     "consistent_and_grounded_pct": 29.27,
     "share_of_consistent_that_is_image_invariant": 61.6,
     "n_dangerous_and_wrong": 64,
     "dangerous_and_wrong_pct": 13.01
    },
    "padchest": {
     "n": 499,
     "paraphrase_consistent_pct": 93.99,
     "image_invariant_pct": 81.76,
     "consistent_and_image_invariant_pct": 78.36,
     "consistent_and_image_invariant_ci95": [
      74.53,
      81.75
     ],
     "consistent_and_grounded_pct": 15.63,
     "share_of_consistent_that_is_image_invariant": 83.37,
     "n_dangerous_and_wrong": 42,
     "dangerous_and_wrong_pct": 8.42
    },
    "vindr": {
     "n": 500,
     "paraphrase_consistent_pct": 87.0,
     "image_invariant_pct": 74.8,
     "consistent_and_image_invariant_pct": 66.2,
     "consistent_and_image_invariant_ci95": [
      61.94,
      70.21
     ],
     "consistent_and_grounded_pct": 20.8,
     "share_of_consistent_that_is_image_invariant": 76.09,
     "n_dangerous_and_wrong": 328,
     "dangerous_and_wrong_pct": 65.6
    }
   },
   "gpt-5.6-sol": {
    "mimic": {
     "n": 500,
     "paraphrase_consistent_pct": 82.2,
     "image_invariant_pct": 37.8,
     "consistent_and_image_invariant_pct": 30.2,
     "consistent_and_image_invariant_ci95": [
      26.34,
      34.36
     ],
     "consistent_and_grounded_pct": 52.0,
     "share_of_consistent_that_is_image_invariant": 36.74,
     "n_dangerous_and_wrong": 35,
     "dangerous_and_wrong_pct": 7.0
    },
    "padchest": {
     "n": 500,
     "paraphrase_consistent_pct": 90.4,
     "image_invariant_pct": 74.2,
     "consistent_and_image_invariant_pct": 69.2,
     "consistent_and_image_invariant_ci95": [
      65.02,
      73.09
     ],
     "consistent_and_grounded_pct": 21.2,
     "share_of_consistent_that_is_image_invariant": 76.55,
     "n_dangerous_and_wrong": 38,
     "dangerous_and_wrong_pct": 7.6
    },
    "vindr": {
     "n": 500,
     "paraphrase_consistent_pct": 61.8,
     "image_invariant_pct": 64.2,
     "consistent_and_image_invariant_pct": 38.4,
     "consistent_and_image_invariant_ci95": [
      34.24,
      42.74
     ],
     "consistent_and_grounded_pct": 23.4,
     "share_of_consistent_that_is_image_invariant": 62.14,
     "n_dangerous_and_wrong": 191,
     "dangerous_and_wrong_pct": 38.2
    }
   },
   "claude-opus-5-adaptive": {
    "mimic": {
     "n": 478,
     "paraphrase_consistent_pct": 82.85,
     "image_invariant_pct": 72.38,
     "consistent_and_image_invariant_pct": 63.81,
     "consistent_and_image_invariant_ci95": [
      59.41,
      67.99
     ],
     "consistent_and_grounded_pct": 19.04,
     "share_of_consistent_that_is_image_invariant": 77.02,
     "n_dangerous_and_wrong": 92,
     "dangerous_and_wrong_pct": 19.25
    },
    "padchest": {
     "n": 486,
     "paraphrase_consistent_pct": 83.33,
     "image_invariant_pct": 75.1,
     "consistent_and_image_invariant_pct": 66.87,
     "consistent_and_image_invariant_ci95": [
      62.57,
      70.91
     ],
     "consistent_and_grounded_pct": 16.46,
     "share_of_consistent_that_is_image_invariant": 80.25,
     "n_dangerous_and_wrong": 61,
     "dangerous_and_wrong_pct": 12.55
    },
    "vindr": {
     "n": 500,
     "paraphrase_consistent_pct": 70.4,
     "image_invariant_pct": 50.0,
     "consistent_and_image_invariant_pct": 30.8,
     "consistent_and_image_invariant_ci95": [
      26.91,
      34.98
     ],
     "consistent_and_grounded_pct": 39.6,
     "share_of_consistent_that_is_image_invariant": 43.75,
     "n_dangerous_and_wrong": 93,
     "dangerous_and_wrong_pct": 18.6
    }
   },
   "claude-opus-5-disabled": {
    "mimic": {
     "n": 498,
     "paraphrase_consistent_pct": 81.12,
     "image_invariant_pct": 72.89,
     "consistent_and_image_invariant_pct": 63.45,
     "consistent_and_image_invariant_ci95": [
      59.14,
      67.57
     ],
     "consistent_and_grounded_pct": 17.67,
     "share_of_consistent_that_is_image_invariant": 78.22,
     "n_dangerous_and_wrong": 105,
     "dangerous_and_wrong_pct": 21.08
    },
    "padchest": {
     "n": 500,
     "paraphrase_consistent_pct": 86.6,
     "image_invariant_pct": 81.8,
     "consistent_and_image_invariant_pct": 75.6,
     "consistent_and_image_invariant_ci95": [
      71.65,
      79.16
     ],
     "consistent_and_grounded_pct": 11.0,
     "share_of_consistent_that_is_image_invariant": 87.3,
     "n_dangerous_and_wrong": 63,
     "dangerous_and_wrong_pct": 12.6
    },
    "vindr": {
     "n": 500,
     "paraphrase_consistent_pct": 76.0,
     "image_invariant_pct": 37.4,
     "consistent_and_image_invariant_pct": 25.0,
     "consistent_and_image_invariant_ci95": [
      21.4,
      28.98
     ],
     "consistent_and_grounded_pct": 51.0,
     "share_of_consistent_that_is_image_invariant": 32.89,
     "n_dangerous_and_wrong": 111,
     "dangerous_and_wrong_pct": 22.2
    }
   },
   "kimi-k3": {
    "mimic": {
     "n": 409,
     "paraphrase_consistent_pct": 84.35,
     "image_invariant_pct": 52.32,
     "consistent_and_image_invariant_pct": 44.99,
     "consistent_and_image_invariant_ci95": [
      40.24,
      49.83
     ],
     "consistent_and_grounded_pct": 39.36,
     "share_of_consistent_that_is_image_invariant": 53.33,
     "n_dangerous_and_wrong": 65,
     "dangerous_and_wrong_pct": 15.89
    },
    "padchest": {
     "n": 447,
     "paraphrase_consistent_pct": 92.84,
     "image_invariant_pct": 87.02,
     "consistent_and_image_invariant_pct": 84.34,
     "consistent_and_image_invariant_ci95": [
      80.68,
      87.41
     ],
     "consistent_and_grounded_pct": 8.5,
     "share_of_consistent_that_is_image_invariant": 90.84,
     "n_dangerous_and_wrong": 81,
     "dangerous_and_wrong_pct": 18.12
    },
    "vindr": {
     "n": 462,
     "paraphrase_consistent_pct": 80.74,
     "image_invariant_pct": 82.68,
     "consistent_and_image_invariant_pct": 70.13,
     "consistent_and_image_invariant_ci95": [
      65.8,
      74.12
     ],
     "consistent_and_grounded_pct": 10.61,
     "share_of_consistent_that_is_image_invariant": 86.86,
     "n_dangerous_and_wrong": 320,
     "dangerous_and_wrong_pct": 69.26
    }
   }
  }
 }
}