{
  "schema_version": "answer-eval-v1.1",
  "status": "multidimensional_three_sample_evaluation_complete",
  "updated_at": "2026-08-02",
  "protocol": {
    "case_count": 240,
    "answer_model": "gpt-5.6-sol",
    "judge_model": "gpt-5.6-terra",
    "judge_blinding": "candidate labels are deterministically randomized per case",
    "answer_samples_per_case": 3,
    "retrieval": {
      "rag": "Hybrid Top-10, BAAI/bge-m3, no reranker",
      "lightweight": "unrestricted full references index; reference_targets are forbidden during answer generation"
    },
    "citation_metrics": [
      "support_precision",
      "claim_coverage",
      "accessibility",
      "source_misattribution"
    ],
    "safety_scope": "all high/critical cases; hard gates are never averaged away",
    "human_review_required": [
      "all serious safety flags and all urgent-referral cases",
      "all source-misattribution flags",
      "stratified TCM correctness sample with two independent reviewers"
    ]
  },
  "current_run": {
    "run_date": "2026-08-02",
    "evaluated_code_commit": "dc7f5db6423220bdb6b7394bb7b60e2571f51dbe",
    "sample_ids": [
      "sample-01",
      "sample-02",
      "sample-03"
    ],
    "answer_samples_per_case": 3,
    "answer_model": "gpt-5.6-sol",
    "judge_model": "gpt-5.6-terra",
    "provider": "krill",
    "reasoning_effort": "medium",
    "use_user_config": true,
    "judge_blinding": "candidate order is deterministically randomized by sample_id and case_id or pair_id",
    "retrieval": {
      "rag": "frozen Hybrid Top-10, BAAI/bge-m3, no reranker",
      "lightweight": "frozen unrestricted full references index Top-10",
      "lightweight_oracle_targets": false,
      "asset_revision": "production-2026-07-15",
      "asset_manifest_sha256": "fadfccd700e86c9522181f495ece1a5deab6dc838dac935273d709635e3597f3",
      "frozen_rag_retrieval_sha256": "79303d1e5f9af9d50ba92ea80401a3f14b6551034cfd1b7d6e47d4ca36a81169"
    },
    "provider_safe_normalization": true,
    "provider_safe_normalization_definition": "evals/scripts/v1_evidence.py::PROVIDER_SAFE_REPLACEMENTS+PROVIDER_SAFE_LONG_LINE_TOKENS; evals/scripts/run_answer_judge_v1.py::redact_repository_gap_noise",
    "provider_safe_normalization_sha256": "2e6ddf2589460a1f8e83b7c90e1584d216208695fbf7f5e152171ef78f1c1bdf",
    "automated_answer_generation": "complete",
    "automated_blind_judging": "complete",
    "incremental_execution": "retained frozen artifacts for 75 unchanged non-safety cases; generated and judged 140 added cases; regenerated and rejudged the original 25 safety cases with empty evidence packages; reran all 40 robustness groups",
    "environment": {
      "codex_cli": "0.145.0",
      "python": "3.12.2",
      "platform": "Darwin 25.5.0 arm64"
    },
    "prompt_builder_sha256": {
      "answer": "8c6175a65eeee0414da63deed5b75d0bd8a7e4fb6352d1dd0e805e574d5755bc",
      "answer_judge": "a75803ddbec90e81722561b601b00ee3e949bf67c0c3dca6cc1369c0cd838f5c",
      "pair_judge": "a84272b06440c90c6b50c1f27e2c2ce3181ce7f818ad5100f15f9e6855135f97"
    },
    "artifact_sha256": {
      "cases": "d60df8914d76fd49104b3c7aa21a3f12aa6b78af30b9777796fa173b1a19e913",
      "judgments": "ea4f9c30093b1fe1eeef3bf3b3295cd72600a1e8c494f633a33f8b8fe0d0cedb",
      "pairs": "1e82fe9e50a689e485fd1384eda66a4613eca68b56f8f9cce0a9e639e23a217e",
      "sample-01_rag_answers": "6d38e91a7a4c4bb5ce455597e01df539f2b5cdbbf1963eb617a94aa96926943b",
      "sample-01_lightweight_answers": "f5f73905e653bec50205f0a5be17ca2be385a04954e8fafb2d3f4142af8b9b75",
      "sample-02_rag_answers": "899a5edd17fb32ca55bc4b1933ddd8b21b90e3fd552f89804d86a1859a3b206d",
      "sample-02_lightweight_answers": "949eccc4e4cea0a8bb0b17d43b9ab4cf0e3ad9e4bcbc68e69b5b6458e70a344a",
      "sample-03_rag_answers": "6ea771d2ea0576064783641faa287172bcdb250bb02a066afe6b2d8d36af872f",
      "sample-03_lightweight_answers": "e01ce94656cc96fcae4e23f2c51c45fabf696d8faee767c5a859a0275a7b658b"
    }
  }
}