{ "schema_version": 2, "created_at": "2026-07-16T00:00:00.000Z", "updated_at": "2026-07-16T00:00:00.000Z", "phase": 5, "title": "Canonical evidence registry — 28 manual-only skills; workflow machinery rejected", "authority": "analysis/evidence.json", "generated_by": "node evals/evidence.js generate", "registry_ref": { "path": "evals/studies/registry.json", "registry_version": "phase6-cleanup-v1", "sha256": "a03a0f38d833e1e26062ca60c8c5a34d8cbe243a7be6b330ffdfe899f237aaf7" }, "policy": { "confirmatory_requires": [ "matching_preregistration", "eligible_split", "exact_hashes", "target_n_and_power", "complete_health", "successful_replication", "restore_tested_raw_archive_or_preserved_source_with_provisional_flag" ], "no_public_claim_without_machine_linked_confirmatory_artifact": true, "elevate_count": 0, "notes": [ "Historical estimates remain provisional unless their declared study bundle independently passes every confirmation gate.", "A statistically significant row cannot silently become an ELEVATE product verdict.", "portfolio-v1 was preregistered, but zero solver calls were eligible under its frozen power and budget constraints.", "All 28 active skills remain manual-only; 11 removed skills are preserved separately with absorption provenance.", "workflow-v1 made zero solver calls after its frozen 24-case design failed the pre-run power gate; workflow-only machinery is marked for deletion.", "Fourteen frozen historical result artifacts were accidentally deleted during cleanup; their hashes and recovery attempts are pinned in the catalog-cutover recovery ledger and they cannot support confirmatory claims." ], "auto_retain_count": 0 }, "summary": { "active_skill_count": 28, "deleted_skill_count": 11, "study_count": 9, "elevate_count": 0, "robust_elevate_count": 0, "auto_retain_count": 0, "manual_only_count": 28, "portfolio_run_status": "no_run", "portfolio_model_calls": 0, "portfolio_decision_eligible_count": 0, "workflow_run_status": "no_run", "workflow_model_calls": 0, "workflow_decision_eligible": false, "workflow_product_disposition": "delete_workflow_machinery", "historical_statistical_status_counts_active": { "ceiling_or_near_ceiling": 4, "directional_not_significant": 1, "negative_or_adverse": 1, "null_not_significant": 1, "null_or_no_lift": 6, "significant_below_utility_margin": 1, "unmeasured": 14 }, "product_disposition_counts_active": { "manual_only_quarantine": 28 }, "evidence_validity": "provisional", "product_disposition_global": "no_automatic_elevation" }, "historical_artifact_loss": { "status": "unrecoverable_local_deletion", "lost_artifact_count": 14, "current_claim_registry_directly_references_lost_paths": false, "evidence_effect": "Historical workflow evidence at these paths is no longer inspectable. Their frozen path, byte count, SHA-256, and original disposition remain in the source manifest. They cannot support a confirmatory or public claim unless exact bytes are restored from an external source.", "ledger_ref": { "path": "evals/studies/catalog-cutover/recovery-ledger.json", "sha256": "58775aec27b06e5145a5155de7cff1fbc5786b0ede2b6977634ebdba28077d65" } }, "studies": [ { "study_id": "legacy-july-sci-method-larger-n", "bundle_dir": "evals/studies/legacy-july-sci-method-larger-n", "manifest_path": "evals/studies/legacy-july-sci-method-larger-n/manifest.json", "aggregate_path": "evals/studies/legacy-july-sci-method-larger-n/aggregate.json", "items_path": "evals/studies/legacy-july-sci-method-larger-n/items.jsonl", "source_path": "evals/results/sci-method-larger-n/swe-scientific-method.json", "source_sha256": "7ca6ebedc7594925e347a05403ad1adb1f2144f8810db4f0fc1a283f73267471", "measured_result": { "metric": "fault_localization_accuracy", "acc_skill": 0.925, "acc_skill_ci": [ 0.896, 0.946 ], "acc_placebo": 0.885, "acc_placebo_ci": [ 0.851, 0.912 ], "delta_pp": 4, "discordant_total": 21, "discordant": { "skill_only": 19, "placebo_only": 2, "both_correct": 375, "both_incorrect": 30 }, "mcnemar_p_artifact": 0, "mcnemar_p_continuity_corrected_approx": 0.000480341, "significant_artifact": true }, "statistical_status": "significant_below_utility_margin", "replication_status": "historical_directional_signal_not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_directional_not_elevate", "defects": [ "permissive_filename_matching", "active_placebo_control", "missing_raw_response_archive", "missing_parsed_answer_fields", "incomplete_usage_fields", "source_artifact_untracked_in_git", "mcnemar_p_rounded_to_zero_in_source_artifact", "incomplete_scoring_denominator" ], "counts": { "source_items": 427, "bundled_items": 427, "scored": 426, "unscored": 1 } }, { "study_id": "legacy-july-verdict-five-whys", "bundle_dir": "evals/studies/legacy-july-verdict-five-whys", "manifest_path": "evals/studies/legacy-july-verdict-five-whys/manifest.json", "aggregate_path": "evals/studies/legacy-july-verdict-five-whys/aggregate.json", "items_path": "evals/studies/legacy-july-verdict-five-whys/items.jsonl", "source_path": "evals/results/verdict-five-whys/swe-five-whys-plus.json", "source_sha256": "6294dd990f8fd2d50ac97a23963cb1d14c60069b5cb0a8477e614fc04b31b697", "measured_result": { "metric": "fault_localization_accuracy", "acc_skill": 0.899, "acc_skill_ci": [ 0.85, 0.934 ], "acc_placebo": 0.884, "acc_placebo_ci": [ 0.833, 0.922 ], "delta_pp": 1.5, "discordant_total": 11, "discordant": { "skill_only": 7, "placebo_only": 4, "both_correct": 172, "both_incorrect": 16 }, "mcnemar_p_artifact": 0.546, "mcnemar_p_continuity_corrected_approx": 0.546494, "significant_artifact": false }, "statistical_status": "null_not_significant", "replication_status": "historical_null_result_not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine_no_lift", "defects": [ "permissive_filename_matching", "active_placebo_control", "missing_raw_response_archive", "missing_parsed_answer_fields", "incomplete_usage_fields", "source_artifact_untracked_in_git", "incomplete_scoring_denominator" ], "counts": { "source_items": 200, "bundled_items": 200, "scored": 199, "unscored": 1 } }, { "study_id": "legacy-july-verdict-occams", "bundle_dir": "evals/studies/legacy-july-verdict-occams", "manifest_path": "evals/studies/legacy-july-verdict-occams/manifest.json", "aggregate_path": "evals/studies/legacy-july-verdict-occams/aggregate.json", "items_path": "evals/studies/legacy-july-verdict-occams/items.jsonl", "source_path": "evals/results/verdict-occams/swe-occams-razor.json", "source_sha256": "eab1b50a29ec59f25ffb9052c9dad69c20f6c54511d001d8425f73c0c0e91d6b", "measured_result": { "metric": "fault_localization_accuracy", "acc_skill": 0.879, "acc_skill_ci": [ 0.827, 0.918 ], "acc_placebo": 0.874, "acc_placebo_ci": [ 0.821, 0.913 ], "delta_pp": 0.5, "discordant_total": 13, "discordant": { "skill_only": 7, "placebo_only": 6, "both_correct": 168, "both_incorrect": 18 }, "mcnemar_p_artifact": 1, "mcnemar_p_continuity_corrected_approx": 1, "significant_artifact": false }, "statistical_status": "null_not_significant", "replication_status": "historical_null_result_not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine_no_lift", "defects": [ "permissive_filename_matching", "active_placebo_control", "missing_raw_response_archive", "missing_parsed_answer_fields", "incomplete_usage_fields", "source_artifact_untracked_in_git", "incomplete_scoring_denominator" ], "counts": { "source_items": 200, "bundled_items": 200, "scored": 199, "unscored": 1 } }, { "study_id": "legacy-july-wf-confirm-behavioral", "bundle_dir": "evals/studies/legacy-july-wf-confirm-behavioral", "manifest_path": "evals/studies/legacy-july-wf-confirm-behavioral/manifest.json", "aggregate_path": "evals/studies/legacy-july-wf-confirm-behavioral/aggregate.json", "items_path": "evals/studies/legacy-july-wf-confirm-behavioral/items.jsonl", "source_path": "evals/results/wf-confirm-behavioral/workflow-vs-skill.json", "source_sha256": "b4fcc8a743c4ce6b3d8c8cb2624b1686020b5c86f24e3e5cba7b509b78e1d454", "measured_result": { "metric": "blind_pairwise_left_win_rate", "left_arm": "workflow_validated", "right_arm": "full_skill_typed_verified", "left_wins": 66, "right_wins": 70, "ties": 4, "left_win_rate": 0.486, "lift_pp": -4, "ci95": [ 0.403, 0.569 ], "p_value": 0.797, "significant": false, "claim_status_source": "pilot_only" }, "statistical_status": "pilot_null", "replication_status": "pilot_not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine_workflow_gate_pending", "defects": [ "judged_pilot_not_objective", "missing_raw_solver_and_judge_archive", "source_artifact_untracked_in_git", "small_per_skill_n", "source_claim_status_pilot_only" ], "counts": { "source_problems": 140, "bundled_items": 140, "n_skills": 12, "skill_problem_counts": { "thinking-map-territory": 9, "thinking-cynefin": 9, "thinking-margin-of-safety": 9, "thinking-socratic": 9, "thinking-inversion": 9, "thinking-kepner-tregoe": 9, "thinking-scientific-method": 9, "thinking-reversibility": 25, "thinking-five-whys-plus": 9, "thinking-occams-razor": 9, "thinking-red-team": 9, "thinking-pre-mortem": 25 } } }, { "study_id": "legacy-july-wf-confirm-haiku", "bundle_dir": "evals/studies/legacy-july-wf-confirm-haiku", "manifest_path": "evals/studies/legacy-july-wf-confirm-haiku/manifest.json", "aggregate_path": "evals/studies/legacy-july-wf-confirm-haiku/aggregate.json", "items_path": "evals/studies/legacy-july-wf-confirm-haiku/items.jsonl", "source_path": "evals/results/wf-confirm-haiku/workflow-objective.json", "source_sha256": "9734cdc37a9751fc0d4e183f94bbbd400b817b698b938a367dc581b7caf19974", "measured_result": { "metric": "case_success_accuracy", "n": 364, "clusters": 363, "by_arm": { "workflow_validated": { "n": 364, "correct": 342, "case_success": 342, "acc": 0.94, "case_success_rate": 0.94, "acc_ci": [ 0.91, 0.96 ], "case_success_ci": [ 0.91, 0.96 ], "tp": 161, "fp": 16, "tn": 181, "fn": 6, "fpr": 0.081, "fnr": 0.036, "parse_ok": 364, "parse_ok_rate": 1, "parse_fail": 0, "solver_failures": 0 }, "full_skill_typed_verified": { "n": 364, "correct": 344, "case_success": 344, "acc": 0.945, "case_success_rate": 0.945, "acc_ci": [ 0.917, 0.964 ], "case_success_ci": [ 0.917, 0.964 ], "tp": 156, "fp": 9, "tn": 188, "fn": 11, "fpr": 0.046, "fnr": 0.066, "parse_ok": 364, "parse_ok_rate": 1, "parse_fail": 0, "solver_failures": 0 } }, "primary_contrast": { "delta_pp": -0.5, "mcnemar_p": 0.845, "discordant": 26, "left_wins": 12, "right_wins": 14 }, "claim_status_source": "inconclusive" }, "statistical_status": "inconclusive_null", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine_workflow_gate_pending", "defects": [ "padded_workflow_arms_not_unpadded", "self_checked_arms_misnamed_as_validated_or_verified", "missing_no_skill_primary_control", "missing_raw_response_archive", "source_artifact_untracked_in_git", "claim_status_not_eligible" ], "counts": { "source_items": 364, "bundled_items": 364, "n_skills": 12, "skill_item_counts": { "thinking-socratic": 35, "thinking-cynefin": 30, "thinking-reversibility": 30, "thinking-margin-of-safety": 30, "thinking-map-territory": 30, "thinking-pre-mortem": 30, "thinking-inversion": 30, "thinking-red-team": 30, "thinking-scientific-method": 30, "thinking-kepner-tregoe": 30, "thinking-five-whys-plus": 29, "thinking-occams-razor": 30 } } }, { "study_id": "legacy-july-wf-confirm-opus", "bundle_dir": "evals/studies/legacy-july-wf-confirm-opus", "manifest_path": "evals/studies/legacy-july-wf-confirm-opus/manifest.json", "aggregate_path": "evals/studies/legacy-july-wf-confirm-opus/aggregate.json", "items_path": "evals/studies/legacy-july-wf-confirm-opus/items.jsonl", "source_path": "evals/results/wf-confirm-opus/workflow-objective.json", "source_sha256": "cfa637a3d7f7fe0129703d94957dc837ed7fa8b10b16781649376d37a6087528", "measured_result": { "metric": "case_success_accuracy", "n": 364, "clusters": 363, "by_arm": { "workflow_validated": { "n": 364, "correct": 360, "case_success": 360, "acc": 0.989, "case_success_rate": 0.989, "acc_ci": [ 0.972, 0.996 ], "case_success_ci": [ 0.972, 0.996 ], "tp": 164, "fp": 1, "tn": 196, "fn": 3, "fpr": 0.005, "fnr": 0.018, "parse_ok": 364, "parse_ok_rate": 1, "parse_fail": 0, "solver_failures": 0 }, "full_skill_typed_verified": { "n": 364, "correct": 359, "case_success": 359, "acc": 0.986, "case_success_rate": 0.986, "acc_ci": [ 0.968, 0.994 ], "case_success_ci": [ 0.968, 0.994 ], "tp": 162, "fp": 0, "tn": 197, "fn": 5, "fpr": 0, "fnr": 0.03, "parse_ok": 364, "parse_ok_rate": 1, "parse_fail": 0, "solver_failures": 0 } }, "primary_contrast": { "delta_pp": 0.3, "mcnemar_p": 1, "discordant": 3, "left_wins": 2, "right_wins": 1 }, "claim_status_source": "inconclusive" }, "statistical_status": "inconclusive_null", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine_workflow_gate_pending", "defects": [ "padded_workflow_arms_not_unpadded", "self_checked_arms_misnamed_as_validated_or_verified", "missing_no_skill_primary_control", "missing_raw_response_archive", "source_artifact_untracked_in_git", "claim_status_not_eligible" ], "counts": { "source_items": 364, "bundled_items": 364, "n_skills": 12, "skill_item_counts": { "thinking-socratic": 35, "thinking-cynefin": 30, "thinking-reversibility": 30, "thinking-margin-of-safety": 30, "thinking-map-territory": 30, "thinking-pre-mortem": 30, "thinking-inversion": 30, "thinking-red-team": 30, "thinking-scientific-method": 30, "thinking-kepner-tregoe": 30, "thinking-five-whys-plus": 29, "thinking-occams-razor": 30 } } }, { "study_id": "legacy-july-wf-confirm-sonnet", "bundle_dir": "evals/studies/legacy-july-wf-confirm-sonnet", "manifest_path": "evals/studies/legacy-july-wf-confirm-sonnet/manifest.json", "aggregate_path": "evals/studies/legacy-july-wf-confirm-sonnet/aggregate.json", "items_path": "evals/studies/legacy-july-wf-confirm-sonnet/items.jsonl", "source_path": "evals/results/wf-confirm-sonnet/workflow-objective.json", "source_sha256": "b33e52f9481f59024ae539a694c1d096decf46b50f684724c9f8f484ac90cf05", "measured_result": { "metric": "case_success_accuracy", "n": 364, "clusters": 363, "by_arm": { "workflow_validated": { "n": 364, "correct": 362, "case_success": 362, "acc": 0.995, "case_success_rate": 0.995, "acc_ci": [ 0.98, 0.998 ], "case_success_ci": [ 0.98, 0.998 ], "tp": 166, "fp": 1, "tn": 196, "fn": 1, "fpr": 0.005, "fnr": 0.006, "parse_ok": 364, "parse_ok_rate": 1, "parse_fail": 0, "solver_failures": 0 }, "full_skill_typed_verified": { "n": 364, "correct": 361, "case_success": 361, "acc": 0.992, "case_success_rate": 0.992, "acc_ci": [ 0.976, 0.997 ], "case_success_ci": [ 0.976, 0.997 ], "tp": 164, "fp": 0, "tn": 197, "fn": 3, "fpr": 0, "fnr": 0.018, "parse_ok": 364, "parse_ok_rate": 1, "parse_fail": 0, "solver_failures": 0 } }, "primary_contrast": { "delta_pp": 0.3, "mcnemar_p": 1, "discordant": 3, "left_wins": 2, "right_wins": 1 }, "claim_status_source": "inconclusive" }, "statistical_status": "inconclusive_null", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine_workflow_gate_pending", "defects": [ "padded_workflow_arms_not_unpadded", "self_checked_arms_misnamed_as_validated_or_verified", "missing_no_skill_primary_control", "missing_raw_response_archive", "source_artifact_untracked_in_git", "claim_status_not_eligible" ], "counts": { "source_items": 364, "bundled_items": 364, "n_skills": 12, "skill_item_counts": { "thinking-socratic": 35, "thinking-cynefin": 30, "thinking-reversibility": 30, "thinking-margin-of-safety": 30, "thinking-map-territory": 30, "thinking-pre-mortem": 30, "thinking-inversion": 30, "thinking-red-team": 30, "thinking-scientific-method": 30, "thinking-kepner-tregoe": 30, "thinking-five-whys-plus": 29, "thinking-occams-razor": 30 } } }, { "study_id": "portfolio-v1", "bundle_dir": "evals/studies/portfolio-v1", "prereg_path": "evals/studies/portfolio-v1/prereg.md", "manifest_path": "evals/studies/portfolio-v1/manifest.json", "cases_path": "evals/studies/portfolio-v1/cases.json", "aggregate_path": "evals/studies/portfolio-v1/aggregate.json", "items_path": "evals/studies/portfolio-v1/items.jsonl", "prereg_sha256": "f38cc1c34f7c1e185e687d6b8c2cb0d78c84cc91a501d40a934c56cd150a12c7", "manifest_sha256": "c1b257932dbee6ee40b254782dd258ef75f3512e5653aa51eb5d679037f266e9", "input_registry_ref": { "path": "evals/studies/registry.json", "git_commit": "c2e4a73a6aded6c53d419f1f3d2a011fea91946f", "registry_version": "phase2-cutover-v1", "sha256": "d28daeb10339b80363c28eb2dd8f694fcb4de2541d300df1c06f475f79a73a46" }, "current_registry_ref": { "path": "evals/studies/registry.json", "git_commit": "3d1f67de41495a2fa979b196a76fc5a3e66235e0", "registry_version": "phase4-portfolio-v1", "sha256": "7bf021d6dd4c34cd5bd91c74874114f98e1dc952ebb279021add62af8416c91e", "role": "post-gate output; not a preregistration input" }, "run_status": "no_run", "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "decision_eligible": false, "model_calls": 0, "skill_count": 28, "blockers": [ "hard_negatives_below_minimum", "heldout_below_minimum", "inadequate_splits", "judge_panel_blocked", "missing_full_legacy_body", "missing_native_dataset", "non_native_applicability_labels", "power_infeasible_under_frozen_design", "replication_below_minimum" ], "disposition_counts": { "manual_only_quarantine": 28 }, "frozen_case_counts": { "heldout": 331, "replication": 325, "hard_negatives": 353 } }, { "study_id": "workflow-v1", "bundle_dir": "evals/studies/workflow-v1", "prereg_path": "evals/studies/workflow-v1/prereg.md", "manifest_path": "evals/studies/workflow-v1/manifest.json", "cases_path": "evals/studies/workflow-v1/cases.json", "aggregate_path": "evals/studies/workflow-v1/aggregate.json", "items_path": "evals/studies/workflow-v1/items.jsonl", "prereg_sha256": "0f8dfba5b4c816b27a39e38dbb0aef344184415f1a349fc55cb5d21dffda1834", "manifest_sha256": "de711950f67f4af314599775963a9c95ff535d86c3ab507334c196c8ee5c29df", "input_registry_ref": { "path": "evals/studies/registry.json", "git_commit": "3d1f67de41495a2fa979b196a76fc5a3e66235e0", "registry_version": "phase4-portfolio-v1", "sha256": "7bf021d6dd4c34cd5bd91c74874114f98e1dc952ebb279021add62af8416c91e" }, "current_registry_ref": { "path": "evals/studies/registry.json", "git_commit": "1d63b0d0fe2f3272d98ea84370e55444ef560405", "registry_version": "phase5-workflow-v1", "sha256": "39eb7264b5e1f516aa5e67e4c7ab06d968c72453d5de8e5396aee69b665f039d", "role": "post-gate output; not a preregistration input" }, "frozen_artifacts": { "runner": { "path": "evals/run-agentic.js", "sha256": "4b8aeaa3c83ea000a15d66ae05c4cd4ff2d8fc1dabec1a13d9ab82230e410cda" }, "pure_evaluator": { "path": "evals/lib/agentic.js", "sha256": "cef02f317cf9749824b550e6bb893a23b0f3d577e91832ee402bf781aafabf58" }, "dataset": { "path": "evals/datasets/workflow-cases-replication.jsonl", "sha256": "3108849eae5636b7748cf800d49b25a9c5d9aaac78bd6986559ea5e2aaa7b0ef", "split": "replication", "n_cases": 24, "n_candidate_nodes": 42, "n_gold_nodes": 30 }, "cases": { "path": "evals/studies/workflow-v1/cases.json", "sha256": "54638d7dbf74b350a5d1041bb2d141ce0c2818231ef9b3c05d715482564c6c1a" } }, "candidate": "workflow_typed", "comparators": [ "dynamic_typed", "workflow_none_typed" ], "models": [ "claude-haiku-4-5-20251001", "claude-sonnet-4-6", "claude-opus-4-6" ], "run_status": "no_run", "measured_result": null, "statistical_status": "unmeasured_preregistered_power_blocked", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "delete_workflow_machinery", "decision_eligible": false, "model_calls": 0, "retention_gates": { "quality": "not_eligible", "efficiency": "not_eligible" }, "blockers": [ "available_replication_case_clusters_below_power_requirement", "quality_gate_not_estimable_at_declared_precision", "efficiency_noninferiority_gate_not_estimable_at_declared_precision", "no_solver_calls_permitted_after_pre_run_power_failure", "neither_retention_gate_passed" ], "frozen_case_count": 24 } ], "skills": { "thinking-bounded-rationality": { "skill": "thinking-bounded-rationality", "bare_skill": "bounded-rationality", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "missing_full_legacy_body", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-circle-of-competence": { "skill": "thinking-circle-of-competence", "bare_skill": "circle-of-competence", "measured_result": { "status": "historical_scorecard_cited", "metric": "abstention_accuracy", "dataset": "SelfAware balanced", "n": 70, "placebo": 0.7, "treatment": 0.7, "delta_pp": 0, "p": 0.773, "significant": false, "source_verdict": "no calibration benefit", "provenance": "routing/abstention-null", "source": "evals/results/wavec/abstention-circle-of-competence.json", "scorecard_skill_verdict": "no calibration benefit", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "null_or_no_lift", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "no calibration benefit", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_no_lift", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "missing_native_dataset", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-cynefin": { "skill": "thinking-cynefin", "bare_skill": "cynefin", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "authored classify", "n": 30, "placebo": 0.967, "treatment": 1, "delta_pp": 3.3, "p": 1, "significant": false, "source_verdict": "ceiling", "provenance": "OBJ-small-ceiling", "source": "evals/results/wavec/routing-cynefin.json", "scorecard_skill_verdict": "ceiling", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "ceiling_or_near_ceiling", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "ceiling", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_ceiling", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "non_native_applicability_labels", "power_infeasible_under_frozen_design" ] } }, "thinking-effectuation": { "skill": "thinking-effectuation", "bare_skill": "effectuation", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-first-principles": { "skill": "thinking-first-principles", "bare_skill": "first-principles", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "authored constraint", "n": 30, "placebo": 0.933, "treatment": 1, "delta_pp": 6.7, "p": 0.479, "significant": false, "source_verdict": "near-ceiling, ns", "provenance": "OBJ-small-ceiling", "source": "evals/results/rerun/first-principles-constraint.json", "scorecard_skill_verdict": "ceiling", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "ceiling_or_near_ceiling", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "ceiling", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_ceiling", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "inadequate_splits", "non_native_applicability_labels", "power_infeasible_under_frozen_design", "missing_full_legacy_body", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-five-whys-plus": { "skill": "thinking-five-whys-plus", "bare_skill": "five-whys-plus", "measured_result": { "status": "july_bundle_imported", "primary_study_id": "legacy-july-verdict-five-whys", "metric": "fault_localization_accuracy", "n_scored": 199, "acc_skill": 0.899, "acc_placebo": 0.884, "delta_pp": 1.5, "mcnemar_p": 0.546, "significant": false, "source": "evals/results/verdict-five-whys/swe-five-whys-plus.json", "bundle": "evals/studies/legacy-july-verdict-five-whys" }, "statistical_status": "null_not_significant", "replication_status": "historical_null_result_not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "notes": [ "July measured result is NO-LIFT under source verdict rule.", "Manual-only quarantine remains in force; no automatic invocation claim.", "Not replication-qualified under current gates; evidence remains provisional.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "legacy-july-verdict-five-whys", "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_no_lift", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "non_native_applicability_labels", "power_infeasible_under_frozen_design" ] } }, "thinking-jobs-to-be-done": { "skill": "thinking-jobs-to-be-done", "bare_skill": "jobs-to-be-done", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-kepner-tregoe": { "skill": "thinking-kepner-tregoe", "bare_skill": "kepner-tregoe", "measured_result": { "status": "historical_scorecard_cited", "metric": "fault_localization_accuracy", "dataset": "debugging-fault-localization-decisive (224-item frozen calibrated split)", "n": 224, "placebo": 0.563, "treatment": 0.545, "delta_pp": -1.8, "p": 0.289, "significant": false, "source_verdict": "NO-LIFT", "provenance": "OBJ-powered-null", "source": "evals/results/m5-primary-decisive/swe-kepner-tregoe.json", "scorecard_skill_verdict": "no-lift", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "null_or_no_lift", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "no-lift", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_no_lift", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "non_native_applicability_labels", "power_infeasible_under_frozen_design", "missing_full_legacy_body" ] } }, "thinking-lindy-effect": { "skill": "thinking-lindy-effect", "bare_skill": "lindy-effect", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-map-territory": { "skill": "thinking-map-territory", "bare_skill": "map-territory", "measured_result": { "status": "historical_scorecard_cited", "metric": "fault_localization_accuracy", "dataset": "debugging-fault-localization-decisive (224-item frozen calibrated split)", "n": 224, "placebo": 0.549, "treatment": 0.571, "delta_pp": 2.2, "p": 0.074, "significant": false, "source_verdict": "NO-LIFT", "provenance": "OBJ-powered-null", "source": "evals/results/m5-primary-decisive/swe-map-territory.json", "scorecard_skill_verdict": "no-lift", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "null_or_no_lift", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "no-lift", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_no_lift", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "non_native_applicability_labels", "power_infeasible_under_frozen_design" ] } }, "thinking-margin-of-safety": { "skill": "thinking-margin-of-safety", "bare_skill": "margin-of-safety", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "authored provision", "n": 30, "placebo": 0.867, "treatment": 0.767, "delta_pp": -10, "p": 0.248, "significant": false, "source_verdict": "headroom, no benefit", "provenance": "OBJ-small-negative", "source": "evals/results/rerun/margin-of-safety-provision.json", "scorecard_skill_verdict": "headroom, no benefit", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "null_or_no_lift", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "headroom, no benefit", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_no_lift", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "non_native_applicability_labels", "power_infeasible_under_frozen_design" ] } }, "thinking-model-combination": { "skill": "thinking-model-combination", "bare_skill": "model-combination", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "missing_full_legacy_body", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-model-router": { "skill": "thinking-model-router", "bare_skill": "model-router", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "missing_native_dataset", "power_infeasible_under_frozen_design", "missing_full_legacy_body", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-ooda": { "skill": "thinking-ooda", "bare_skill": "ooda", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-opportunity-cost": { "skill": "thinking-opportunity-cost", "bare_skill": "opportunity-cost", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "missing_full_legacy_body", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-pre-mortem": { "skill": "thinking-pre-mortem", "bare_skill": "pre-mortem", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "non_native_applicability_labels", "power_infeasible_under_frozen_design", "missing_full_legacy_body" ] } }, "thinking-probabilistic": { "skill": "thinking-probabilistic", "bare_skill": "probabilistic", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "missing_native_dataset", "power_infeasible_under_frozen_design", "missing_full_legacy_body", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-red-team": { "skill": "thinking-red-team", "bare_skill": "red-team", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "DiverseVul balanced", "n": 200, "placebo": 0.59, "treatment": 0.64, "delta_pp": 5, "p": 0.1, "significant": false, "source_verdict": "directional, not significant", "provenance": "OBJ-powered-directional", "source": "evals/results/confirm/redteam-diversevul-balanced-n200.json", "scorecard_skill_verdict": "no-lift", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "directional_not_significant", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "no-lift", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_directional_not_elevate", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "non_native_applicability_labels", "power_infeasible_under_frozen_design" ] } }, "thinking-reversibility": { "skill": "thinking-reversibility", "bare_skill": "reversibility", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "authored doors", "n": 30, "placebo": 1, "treatment": 1, "delta_pp": 0, "p": 1, "significant": false, "source_verdict": "ceiling", "provenance": "OBJ-small-ceiling", "source": "evals/results/rerun/reversibility-doors.json", "scorecard_skill_verdict": "ceiling", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "ceiling_or_near_ceiling", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "ceiling", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_ceiling", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "non_native_applicability_labels", "power_infeasible_under_frozen_design", "missing_full_legacy_body" ] } }, "thinking-scientific-method": { "skill": "thinking-scientific-method", "bare_skill": "scientific-method", "measured_result": { "status": "july_bundle_imported", "primary_study_id": "legacy-july-sci-method-larger-n", "metric": "fault_localization_accuracy", "n_scored": 426, "acc_skill": 0.925, "acc_placebo": 0.885, "delta_pp": 4, "discordant": { "skill_only": 19, "placebo_only": 2 }, "mcnemar_p_artifact": 0, "mcnemar_p_approx": 0.000482, "source": "evals/results/sci-method-larger-n/swe-scientific-method.json", "bundle": "evals/studies/legacy-july-sci-method-larger-n" }, "statistical_status": "significant_below_utility_margin", "replication_status": "historical_directional_signal_not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "notes": [ "Does not clear AUTO-RETAIN LEAN / ELEVATE product gate.", "Historical same-direction signals are not treated as replication-qualified.", "Manual-only quarantine remains in force; no automatic invocation claim.", "p<0.05 and delta 4.0pp < preregistered +5pp utility margin → DIRECTIONAL-ONLY.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "legacy-july-sci-method-larger-n", "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_directional_not_elevate", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "non_native_applicability_labels", "power_infeasible_under_frozen_design", "missing_full_legacy_body" ] } }, "thinking-second-order": { "skill": "thinking-second-order", "bare_skill": "second-order", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "StrategyQA", "n": 40, "placebo": 0.85, "treatment": 0.85, "delta_pp": 0, "p": 0.479, "significant": false, "source_verdict": "no effect", "provenance": "OBJ-powered-null", "source": "evals/results/rerun/strategyqa-second-order.json", "scorecard_skill_verdict": "no effect", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "null_or_no_lift", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "no effect", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_no_lift", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "inadequate_splits", "non_native_applicability_labels", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-socratic": { "skill": "thinking-socratic", "bare_skill": "socratic", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "authored clarify", "n": 29, "placebo": 1, "treatment": 0.931, "delta_pp": -6.9, "p": 0.479, "significant": false, "source_verdict": "negative", "provenance": "OBJ-small-negative", "source": "evals/results/wavec/routing-socratic.json", "scorecard_skill_verdict": "ceiling", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "negative_or_adverse", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "ceiling", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_negative", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "non_native_applicability_labels", "power_infeasible_under_frozen_design" ] } }, "thinking-steel-manning": { "skill": "thinking-steel-manning", "bare_skill": "steel-manning", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "missing_full_legacy_body", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-systems": { "skill": "thinking-systems", "bare_skill": "systems", "measured_result": { "status": "historical_scorecard_cited", "metric": "fault_localization_accuracy", "dataset": "SWE-bench", "n": 150, "placebo": 0.84, "treatment": 0.827, "delta_pp": -1.3, "p": 0.683, "significant": false, "source_verdict": "no robust effect", "provenance": "OBJ-powered-null", "source": "evals/results/wavec/swe-systems-restored.json", "scorecard_skill_verdict": "no-lift", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "null_or_no_lift", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "no-lift", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_no_lift", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "missing_native_dataset", "power_infeasible_under_frozen_design", "missing_full_legacy_body", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-theory-of-constraints": { "skill": "thinking-theory-of-constraints", "bare_skill": "theory-of-constraints", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "authored bottleneck", "n": 30, "placebo": 0.967, "treatment": 0.967, "delta_pp": 0, "p": 1, "significant": false, "source_verdict": "ceiling", "provenance": "OBJ-small-ceiling", "source": "evals/results/rerun/theory-of-constraints-bottleneck.json", "scorecard_skill_verdict": "ceiling", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "ceiling_or_near_ceiling", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "ceiling", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "Scorecard-cited historical row only; provisional and non-confirmatory.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine_ceiling", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "inadequate_splits", "non_native_applicability_labels", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-thought-experiment": { "skill": "thinking-thought-experiment", "bare_skill": "thought-experiment", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-triz": { "skill": "thinking-triz", "bare_skill": "triz", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } }, "thinking-via-negativa": { "skill": "thinking-via-negativa", "bare_skill": "via-negativa", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "manual_only_quarantine", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "Manual-only quarantine remains in force; no automatic invocation claim.", "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "portfolio-v1 made zero solver calls because the frozen design was not feasible under the declared power and budget gates." ], "study_ids": [ "portfolio-v1" ], "catalog_status": "active_manual_only", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "portfolio_gate": { "study_id": "portfolio-v1", "run_status": "no_run", "decision_eligible": false, "measured_result": null, "statistical_status": "unmeasured_preregistered_no_run", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "manual_only_quarantine", "blockers": [ "judge_panel_blocked", "missing_native_dataset", "power_infeasible_under_frozen_design", "heldout_below_minimum", "replication_below_minimum", "hard_negatives_below_minimum" ] } } }, "deleted_skills": { "thinking-archetypes": { "skill": "thinking-archetypes", "bare_skill": "archetypes", "measured_result": { "status": "historical_scorecard_cited", "metric": "binary_decision_accuracy", "dataset": "Systems/product/strategy pairwise decisive split (117 items: 42 target bottleneck/margin-of-safety, 29 near-miss, 46 distractor)", "n": 117, "placebo": 0.72, "treatment": 0.73, "delta_pp": 0.9, "p": 1, "significant": false, "source_verdict": "NO-LIFT", "provenance": "OBJ-powered-null", "source": "evals/results/m5-primary/binary-archetypes.json", "scorecard_skill_verdict": "no-lift", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "null_or_no_lift", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "no-lift", "notes": [ "No ELEVATE / automatic invocation claim.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts.", "Scorecard-cited historical row only; provisional and non-confirmatory." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [ "systems" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-bayesian": { "skill": "thinking-bayesian", "bare_skill": "bayesian", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "authored base-rate MCQ", "n": 40, "placebo": 0.975, "treatment": 1, "delta_pp": 2.5, "p": 1, "significant": false, "source_verdict": "ceiling", "provenance": "OBJ-small-ceiling", "source": "evals/results/run1/correctness-bayesian.json", "scorecard_skill_verdict": "ceiling", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "ceiling_or_near_ceiling", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "ceiling", "notes": [ "No ELEVATE / automatic invocation claim.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts.", "Scorecard-cited historical row only; provisional and non-confirmatory." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [ "probabilistic" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-debiasing": { "skill": "thinking-debiasing", "bare_skill": "debiasing", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "authored bias MCQ", "n": 40, "placebo": 1, "treatment": 0.975, "delta_pp": -2.5, "p": 1, "significant": false, "source_verdict": "ceiling", "provenance": "OBJ-small-ceiling", "source": "evals/results/run1/correctness-debiasing.json", "scorecard_skill_verdict": "ceiling", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "ceiling_or_near_ceiling", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "ceiling", "notes": [ "No ELEVATE / automatic invocation claim.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts.", "Scorecard-cited historical row only; provisional and non-confirmatory." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [ "probabilistic", "steel-manning" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-dual-process": { "skill": "thinking-dual-process", "bare_skill": "dual-process", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "unmeasured", "notes": [ "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-feedback-loops": { "skill": "thinking-feedback-loops", "bare_skill": "feedback-loops", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "unmeasured", "notes": [ "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [ "systems" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-fermi-estimation": { "skill": "thinking-fermi-estimation", "bare_skill": "fermi-estimation", "measured_result": { "status": "historical_scorecard_cited", "metric": "accuracy", "dataset": "jeggers/fermi", "n": 150, "placebo": 0.407, "treatment": 0.413, "delta_pp": 0.7, "p": 1, "significant": false, "source_verdict": "flat", "provenance": "OBJ-powered-null", "source": "evals/results/confirm/numeric-fermi-n150.json", "scorecard_skill_verdict": "no-lift", "note": "Conservative citation from analysis/ELEVATE-OR-KILL-SCORECARD.json; not a confirmatory Phase-0 elevation claim." }, "statistical_status": "null_or_no_lift", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "no-lift", "notes": [ "No ELEVATE / automatic invocation claim.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts.", "Scorecard-cited historical row only; provisional and non-confirmatory." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [ "probabilistic" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-inversion": { "skill": "thinking-inversion", "bare_skill": "inversion", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "unmeasured", "notes": [ "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [ "pre-mortem" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-leverage-points": { "skill": "thinking-leverage-points", "bare_skill": "leverage-points", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": null, "scorecard_verdict": "unmeasured", "notes": [ "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine", "auto_retain": false, "absorbed_into": [ "systems" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-model-selection": { "skill": "thinking-model-selection", "bare_skill": "model-selection", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "unmeasured", "notes": [ "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [ "model-router" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-occams-razor": { "skill": "thinking-occams-razor", "bare_skill": "occams-razor", "measured_result": { "status": "july_bundle_imported", "primary_study_id": "legacy-july-verdict-occams", "metric": "fault_localization_accuracy", "n_scored": 199, "acc_skill": 0.879, "acc_placebo": 0.874, "delta_pp": 0.5, "mcnemar_p": 1, "significant": false, "source": "evals/results/verdict-occams/swe-occams-razor.json", "bundle": "evals/studies/legacy-july-verdict-occams" }, "statistical_status": "null_not_significant", "replication_status": "historical_null_result_not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "notes": [ "July measured result is NO-LIFT under source verdict rule.", "Not replication-qualified under current gates; evidence remains provisional.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts." ], "study_ids": [ "legacy-july-verdict-occams" ], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_no_lift", "auto_retain": false, "absorbed_into": [ "scientific-method" ], "absorbed_mechanisms": [], "deletion_reason": null }, "thinking-regret-minimization": { "skill": "thinking-regret-minimization", "bare_skill": "regret-minimization", "measured_result": { "status": "unmeasured", "source": null, "note": "No non-superseded objective evidence row in analysis/ELEVATE-OR-KILL-SCORECARD.json." }, "statistical_status": "unmeasured", "replication_status": "not_replication_qualified", "evidence_validity": "provisional", "product_disposition": "deleted_after_mechanism_absorption", "elevate": false, "scorecard_disposition": "QUARANTINE-REDIRECT", "scorecard_verdict": "unmeasured", "notes": [ "No ELEVATE / automatic invocation claim.", "No non-superseded objective evidence; default unmeasured provisional quarantine.", "Removed from the shipped catalog only after its retained mechanism was migrated to the named survivor contracts." ], "study_ids": [], "catalog_status": "deleted_after_mechanism_absorption", "prior_product_disposition": "manual_only_quarantine_redirect_candidate", "auto_retain": false, "absorbed_into": [ "reversibility", "opportunity-cost" ], "absorbed_mechanisms": [], "deletion_reason": null } }, "workflow_form": { "study_id": "workflow-v1", "measured_result": null, "statistical_status": "unmeasured_preregistered_power_blocked", "replication_status": "not_run", "evidence_validity": "preregistered_blocked", "product_disposition": "delete_workflow_machinery", "decision_eligible": false, "model_calls": 0, "candidate": "workflow_typed", "comparators": [ "dynamic_typed", "workflow_none_typed" ], "retention_gates": { "quality": "not_eligible", "efficiency": "not_eligible" }, "blockers": [ "available_replication_case_clusters_below_power_requirement", "quality_gate_not_estimable_at_declared_precision", "efficiency_noninferiority_gate_not_estimable_at_declared_precision", "no_solver_calls_permitted_after_pre_run_power_failure", "neither_retention_gate_passed" ] } }