{ "generated_at": "2026-08-22T16:04:57.538734+00:00", "harnesses": [ { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9910650011618062, "cache_hit_rate_typical": 0.8796567078512081, "cache_hit_rate_typical_n": 20, "cache_hit_rate_typical_q1": 0.7711918285780834, "cache_hit_rate_typical_q3": 0.9681778231977365, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 1.778323635, "cost_per_success_normalized_lower_bound": 1.778323635, "duration_coverage": 1.0, "effective_cost_coverage": 1.0, "effective_cost_per_pass": 3.468296325, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 4700224.45, "mean_output_tokens": 16991.05, "mean_turns": 62.35, "median_cost_per_success_normalized": 0.12425444999999999, "median_duration_seconds": 403.3207145, "name": "codex", "output_tokens_coverage": 1.0, "pass_rate": 0.6666666666666666, "success_rate_expected": 0.6666666666666666, "successful": 20, "task_details": [ { "cache_hit_rate_normalized": 0.5210636998846754, "cost_first_cold_usd": 0.0561543, "duration_seconds": 269.257144, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 3 }, { "cache_hit_rate_normalized": 0.75464282204972, "cost_first_cold_usd": 0.058530000000000006, "duration_seconds": 238.366182, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 5 }, { "cache_hit_rate_normalized": 0.928555303041705, "cost_first_cold_usd": 0.2606442, "duration_seconds": 372.216841, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 22 }, { "cache_hit_rate_normalized": 0.8091166090147671, "cost_first_cold_usd": 0.125907, "duration_seconds": 289.388658, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.8793337126894625, "cost_first_cold_usd": 0.0940716, "duration_seconds": 289.15287, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 11 }, { "cache_hit_rate_normalized": 0.7704293376695872, "cost_first_cold_usd": 0.0547452, "duration_seconds": 457.588748, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 5 }, { "cache_hit_rate_normalized": 0.6114728204337944, "cost_first_cold_usd": 0.0711366, "duration_seconds": 516.156218, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 3 }, { "cache_hit_rate_normalized": 0.8576440385759134, "cost_first_cold_usd": 0.24642599999999998, "duration_seconds": 931.734494, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": false, "status": "failure", "success": false, "title": "G-code to text", "turns": 12 }, { "cache_hit_rate_normalized": 0.8542251067964725, "cost_first_cold_usd": 0.12831120000000001, "duration_seconds": 683.827087, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 10 }, { "cache_hit_rate_normalized": 0.7311144825842342, "cost_first_cold_usd": 0.3013353, "duration_seconds": 1593.750497, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 4 }, { "cache_hit_rate_normalized": 0.9310504285809632, "cost_first_cold_usd": 0.1490787, "duration_seconds": 395.903121, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 19 }, { "cache_hit_rate_normalized": 0.8730480069227777, "cost_first_cold_usd": 0.104964, "duration_seconds": 410.738308, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 11 }, { "cache_hit_rate_normalized": 0.8125311437051801, "cost_first_cold_usd": 0.1740654, "duration_seconds": 607.566402, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 8 }, { "cache_hit_rate_normalized": 0.965917086159647, "cost_first_cold_usd": 0.41905529999999996, "duration_seconds": 389.341468, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 43 }, { "cache_hit_rate_normalized": 0.9818559774518724, "cost_first_cold_usd": 1.4705378999999998, "duration_seconds": 1321.973638, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": 86 }, { "cache_hit_rate_normalized": 0.5334819365343458, "cost_first_cold_usd": 0.6023883000000001, "duration_seconds": 1236.802094, "id": "terminal-bench/chess-best-move", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Chess best move", "turns": 26 }, { "cache_hit_rate_normalized": 0.8799797030129537, "cost_first_cold_usd": 0.1226019, "duration_seconds": 373.859039, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 10 }, { "cache_hit_rate_normalized": 0.5713142428410654, "cost_first_cold_usd": 0.0756762, "duration_seconds": 296.342598, "id": "terminal-bench/code-from-image", "included_in_efficiency": true, "status": "success", "success": true, "title": "Code from image", "turns": 5 }, { "cache_hit_rate_normalized": 0.7757511215842661, "cost_first_cold_usd": 0.0619221, "duration_seconds": 429.313292, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 5 }, { "cache_hit_rate_normalized": 0.7714459922142488, "cost_first_cold_usd": 0.0662118, "duration_seconds": 340.924126, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 5 }, { "cache_hit_rate_normalized": 0.91410452735691, "cost_first_cold_usd": 0.21461580000000002, "duration_seconds": 496.75977, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 18 }, { "cache_hit_rate_normalized": 0.9876397798219312, "cost_first_cold_usd": 3.2826804, "duration_seconds": 1457.584988, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": true, "status": "success", "success": true, "title": "Anko typed variable bindings", "turns": 138 }, { "cache_hit_rate_normalized": 0.9944096353579661, "cost_first_cold_usd": 9.9867321, "duration_seconds": 3290.7652, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": true, "status": "success", "success": true, "title": "ArkType JSON Schema reference dependencies", "turns": 319 }, { "cache_hit_rate_normalized": 0.9922543332157787, "cost_first_cold_usd": 5.300454, "duration_seconds": 2587.068742, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": false, "status": "failure", "success": false, "title": "FastAPI deprecation response headers", "turns": 212 }, { "cache_hit_rate_normalized": 0.9749600343120053, "cost_first_cold_usd": 1.3304382, "duration_seconds": 1744.486507, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": true, "status": "success", "success": true, "title": "HTTPX multipart response parsing", "turns": 73 }, { "cache_hit_rate_normalized": 0.9950925402363955, "cost_first_cold_usd": 13.058473799999998, "duration_seconds": 3506.088246, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": true, "status": "success", "success": true, "title": "Expr try/catch errors", "turns": 358 }, { "cache_hit_rate_normalized": 0.9913535105576254, "cost_first_cold_usd": 5.9728335, "duration_seconds": 2206.858513, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python StateMachine state-data scoping", "turns": 187 }, { "cache_hit_rate_normalized": 0.9943933513971108, "cost_first_cold_usd": 11.628816, "duration_seconds": 2933.128495, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 322 }, { "cache_hit_rate_normalized": 0.989303985529252, "cost_first_cold_usd": 3.3220016999999995, "duration_seconds": 1469.408082, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 147 }, { "cache_hit_rate_normalized": 0.9944234247857372, "cost_first_cold_usd": 10.625118, "duration_seconds": 2731.327388, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 323 } ], "termination_anomalies": 3, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.983331853983117, "cache_hit_rate_typical": 0.8430020151050968, "cache_hit_rate_typical_n": 19, "cache_hit_rate_typical_q1": 0.7439275654554118, "cache_hit_rate_typical_q3": 0.9473186824410098, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 0.9358733052631578, "cost_per_success_normalized_lower_bound": 0.9358733052631578, "duration_coverage": 1.0, "effective_cost_coverage": 0.9, "effective_cost_per_pass": 3.284902642105263, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 2119232.947368421, "mean_output_tokens": 13648.631578947368, "mean_turns": 35.73684210526316, "median_cost_per_success_normalized": 0.1194408, "median_duration_seconds": 403.943982, "name": "dsh-creator", "output_tokens_coverage": 1.0, "pass_rate": 0.6333333333333333, "success_rate_expected": 0.6333333333333333, "successful": 19, "task_details": [ { "cache_hit_rate_normalized": 0.7961389436476106, "cost_first_cold_usd": 0.1647024, "duration_seconds": 300.212219, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 7 }, { "cache_hit_rate_normalized": 0.8628806118113581, "cost_first_cold_usd": 0.07487940000000001, "duration_seconds": 201.781255, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 9 }, { "cache_hit_rate_normalized": 0.6355332802473184, "cost_first_cold_usd": 0.0509184, "duration_seconds": 175.470124, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 3 }, { "cache_hit_rate_normalized": 0.7803091738469337, "cost_first_cold_usd": 0.1081893, "duration_seconds": 338.096856, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 5 }, { "cache_hit_rate_normalized": 0.8430020151050968, "cost_first_cold_usd": 0.0641958, "duration_seconds": 215.675326, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.70754595706389, "cost_first_cold_usd": 0.0509298, "duration_seconds": 238.593635, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 4 }, { "cache_hit_rate_normalized": 0.7057279780921953, "cost_first_cold_usd": 0.0731775, "duration_seconds": 285.208935, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 3 }, { "cache_hit_rate_normalized": 0.9606018189633492, "cost_first_cold_usd": 0.6998489999999999, "duration_seconds": 1065.827946, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": false, "status": "failure", "success": false, "title": "G-code to text", "turns": 53 }, { "cache_hit_rate_normalized": 0.8669986617019145, "cost_first_cold_usd": 0.1667766, "duration_seconds": 685.100725, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 11 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1177.521942, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": null }, { "cache_hit_rate_normalized": 0.8265273601507614, "cost_first_cold_usd": 0.08464139999999999, "duration_seconds": 399.992206, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 8 }, { "cache_hit_rate_normalized": 0.8850579443428565, "cost_first_cold_usd": 0.1354254, "duration_seconds": 585.810976, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 12 }, { "cache_hit_rate_normalized": 0.914385171790235, "cost_first_cold_usd": 0.2837139, "duration_seconds": 754.273971, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 22 }, { "cache_hit_rate_normalized": 0.9638331384987451, "cost_first_cold_usd": 0.549537, "duration_seconds": 768.176657, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 46 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1217.852537, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": null }, { "cache_hit_rate_normalized": 0.9308042263832745, "cost_first_cold_usd": 0.3796179, "duration_seconds": 743.120946, "id": "terminal-bench/chess-best-move", "included_in_efficiency": true, "status": "success", "success": true, "title": "Chess best move", "turns": 22 }, { "cache_hit_rate_normalized": 0.7881261344547289, "cost_first_cold_usd": 0.1194408, "duration_seconds": 378.442162, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 9 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1424.649013, "id": "terminal-bench/code-from-image", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Code from image", "turns": null }, { "cache_hit_rate_normalized": 0.6850232156354605, "cost_first_cold_usd": 0.09214019999999999, "duration_seconds": 544.562487, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 6 }, { "cache_hit_rate_normalized": 0.6654720380236882, "cost_first_cold_usd": 0.1070169, "duration_seconds": 452.85957, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 6 }, { "cache_hit_rate_normalized": 0.8686288672934864, "cost_first_cold_usd": 0.1697052, "duration_seconds": 403.943982, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 12 }, { "cache_hit_rate_normalized": 0.9772815335869226, "cost_first_cold_usd": 1.8242469, "duration_seconds": 1132.600716, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": true, "status": "success", "success": true, "title": "Anko typed variable bindings", "turns": 62 }, { "cache_hit_rate_normalized": 0.9930100587139196, "cost_first_cold_usd": 10.231463399999999, "duration_seconds": 3600.0, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 288 }, { "cache_hit_rate_normalized": 0.9857058378306893, "cost_first_cold_usd": 2.8352105999999995, "duration_seconds": 1147.007054, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": true, "status": "success", "success": true, "title": "FastAPI deprecation response headers", "turns": 97 }, { "cache_hit_rate_normalized": 0.9848678766921772, "cost_first_cold_usd": 3.0592152, "duration_seconds": 2232.195525, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": true, "status": "success", "success": true, "title": "HTTPX multipart response parsing", "turns": 131 }, { "cache_hit_rate_normalized": 0.9937819903357911, "cost_first_cold_usd": 11.2894746, "duration_seconds": 3379.557856, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": 267 }, { "cache_hit_rate_normalized": 0.9880508845336988, "cost_first_cold_usd": 3.883305, "duration_seconds": 3600.0, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Python StateMachine state-data scoping", "turns": 109 }, { "cache_hit_rate_normalized": 0.992476733928211, "cost_first_cold_usd": 7.8384027000000005, "duration_seconds": 2200.397788, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": true, "status": "success", "success": true, "title": "KaTeX multicolumn array spans", "turns": 229 }, { "cache_hit_rate_normalized": 0.9815199852448317, "cost_first_cold_usd": 2.4072639, "duration_seconds": 1826.522808, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 76 }, { "cache_hit_rate_normalized": 0.9951146585223677, "cost_first_cold_usd": 15.669711, "duration_seconds": 3096.129088, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 347 } ], "termination_anomalies": 5, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.2494923504998629, "cache_hit_rate_typical": 0.677949269665246, "cache_hit_rate_typical_n": 19, "cache_hit_rate_typical_q1": 0.544392038057746, "cache_hit_rate_typical_q3": 0.7677244248490436, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 4.5796623157894745, "cost_per_success_normalized_lower_bound": 4.5796623157894745, "duration_coverage": 1.0, "effective_cost_coverage": 0.9666666666666667, "effective_cost_per_pass": 18.33683390526316, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 1903406.105263158, "mean_output_tokens": 10108.947368421053, "mean_turns": 49.26315789473684, "median_cost_per_success_normalized": 0.2879886, "median_duration_seconds": 577.971591, "name": "claude-code", "output_tokens_coverage": 1.0, "pass_rate": 0.6333333333333333, "success_rate_expected": 0.6333333333333333, "successful": 19, "task_details": [ { "cache_hit_rate_normalized": 0.6455760240039076, "cost_first_cold_usd": 0.11493809999999999, "duration_seconds": 1110.142744, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 4 }, { "cache_hit_rate_normalized": 0.8191245204630359, "cost_first_cold_usd": 0.1706694, "duration_seconds": 278.785253, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 14 }, { "cache_hit_rate_normalized": 0.8407751946625941, "cost_first_cold_usd": 0.21184980000000003, "duration_seconds": 994.241353, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 16 }, { "cache_hit_rate_normalized": 0.7869970847812786, "cost_first_cold_usd": 0.30568560000000006, "duration_seconds": 1106.905284, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 15 }, { "cache_hit_rate_normalized": 0.8092848180677541, "cost_first_cold_usd": 0.205452, "duration_seconds": 303.050194, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 18 }, { "cache_hit_rate_normalized": 0.7364787111622555, "cost_first_cold_usd": 0.11888100000000001, "duration_seconds": 501.205725, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 5 }, { "cache_hit_rate_normalized": 0.6503661649292717, "cost_first_cold_usd": 0.1400004, "duration_seconds": 483.954407, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 8 }, { "cache_hit_rate_normalized": 0.7712183979712351, "cost_first_cold_usd": 0.39405, "duration_seconds": 1117.182348, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": true, "status": "success", "success": true, "title": "G-code to text", "turns": 19 }, { "cache_hit_rate_normalized": 0.716466899158925, "cost_first_cold_usd": 0.3103014, "duration_seconds": 659.388252, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 15 }, { "cache_hit_rate_normalized": 0.6974455793839054, "cost_first_cold_usd": 0.5421260999999999, "duration_seconds": 1538.001148, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 25 }, { "cache_hit_rate_normalized": 0.6786481896547757, "cost_first_cold_usd": 0.6003216, "duration_seconds": 577.971591, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 32 }, { "cache_hit_rate_normalized": 0.7642304517268521, "cost_first_cold_usd": 0.3378474, "duration_seconds": 497.414039, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 19 }, { "cache_hit_rate_normalized": 0.7720363415186303, "cost_first_cold_usd": 0.4282782, "duration_seconds": 1489.465068, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 23 }, { "cache_hit_rate_normalized": 0.3552603216026306, "cost_first_cold_usd": 5.1758172, "duration_seconds": 1287.389872, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 132 }, { "cache_hit_rate_normalized": 0.4799361271374122, "cost_first_cold_usd": 4.951542, "duration_seconds": 1553.14974, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": 118 }, { "cache_hit_rate_normalized": 0.6309681341869032, "cost_first_cold_usd": 0.1371315, "duration_seconds": 433.740405, "id": "terminal-bench/chess-best-move", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Chess best move", "turns": 4 }, { "cache_hit_rate_normalized": 0.677949269665246, "cost_first_cold_usd": 0.2879886, "duration_seconds": 385.502184, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 17 }, { "cache_hit_rate_normalized": 0.4361506721682417, "cost_first_cold_usd": 0.1465014, "duration_seconds": 416.536223, "id": "terminal-bench/code-from-image", "included_in_efficiency": true, "status": "success", "success": true, "title": "Code from image", "turns": 4 }, { "cache_hit_rate_normalized": 0.6446713490330324, "cost_first_cold_usd": 0.1551318, "duration_seconds": 343.753262, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 12 }, { "cache_hit_rate_normalized": 0.6948359462248548, "cost_first_cold_usd": 0.19567320000000002, "duration_seconds": 409.635183, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 10 }, { "cache_hit_rate_normalized": 0.5425591277846933, "cost_first_cold_usd": 0.648633, "duration_seconds": 856.452532, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 25 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 3600.0, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Anko typed variable bindings", "turns": null }, { "cache_hit_rate_normalized": 0.1848305347590537, "cost_first_cold_usd": 49.838078100000004, "duration_seconds": 3600.0, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 380 }, { "cache_hit_rate_normalized": 0.3080441871816448, "cost_first_cold_usd": 11.3539824, "duration_seconds": 2666.04378, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": true, "status": "success", "success": true, "title": "FastAPI deprecation response headers", "turns": 153 }, { "cache_hit_rate_normalized": 0.5462249483307989, "cost_first_cold_usd": 2.0863947, "duration_seconds": 3165.503342, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": true, "status": "success", "success": true, "title": "HTTPX multipart response parsing", "turns": 52 }, { "cache_hit_rate_normalized": 0.19696557961380587, "cost_first_cold_usd": 56.931856200000006, "duration_seconds": 3600.0, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": 376 }, { "cache_hit_rate_normalized": 0.15678926962443693, "cost_first_cold_usd": 64.3637664, "duration_seconds": 3595.917111, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python StateMachine state-data scoping", "turns": 381 }, { "cache_hit_rate_normalized": 0.18680356708506876, "cost_first_cold_usd": 44.203989299999996, "duration_seconds": 2778.236437, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 290 }, { "cache_hit_rate_normalized": 0.19464126065913315, "cost_first_cold_usd": 26.4038373, "duration_seconds": 3213.00458, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 176 }, { "cache_hit_rate_normalized": 0.1917205459242061, "cost_first_cold_usd": 77.6391201, "duration_seconds": 3600.0, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 306 } ], "termination_anomalies": 10, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9724315643288303, "cache_hit_rate_typical": 0.7937786348907315, "cache_hit_rate_typical_n": 18, "cache_hit_rate_typical_q1": 0.746594307975921, "cache_hit_rate_typical_q3": 0.8797504643976021, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 0.35792705, "cost_per_success_normalized_lower_bound": 0.35792705, "duration_coverage": 1.0, "effective_cost_coverage": 1.0, "effective_cost_per_pass": 2.4329655, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 638338.7222222222, "mean_output_tokens": 7927.388888888889, "mean_turns": 18.72222222222222, "median_cost_per_success_normalized": 0.07088475, "median_duration_seconds": 453.28672900000004, "name": "pi-responses", "output_tokens_coverage": 1.0, "pass_rate": 0.6, "success_rate_expected": 0.6, "successful": 18, "task_details": [ { "cache_hit_rate_normalized": 0.559798149705635, "cost_first_cold_usd": 0.07090439999999999, "duration_seconds": 240.860853, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 3 }, { "cache_hit_rate_normalized": 0.7484448867877581, "cost_first_cold_usd": 0.0356616, "duration_seconds": 222.437499, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 7 }, { "cache_hit_rate_normalized": 0.6353671147880041, "cost_first_cold_usd": 0.0618912, "duration_seconds": 205.96533, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 4 }, { "cache_hit_rate_normalized": 0.8198353724136206, "cost_first_cold_usd": 0.1371084, "duration_seconds": 302.872795, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 10 }, { "cache_hit_rate_normalized": 0.7726688928846648, "cost_first_cold_usd": 0.0287046, "duration_seconds": 202.12871, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.5399045399045399, "cost_first_cold_usd": 0.030349200000000003, "duration_seconds": 458.630664, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 3 }, { "cache_hit_rate_normalized": 0.6561218512009374, "cost_first_cold_usd": 0.046926, "duration_seconds": 559.449999, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 4 }, { "cache_hit_rate_normalized": 0.8837070254110613, "cost_first_cold_usd": 0.3038232, "duration_seconds": 1558.030075, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": false, "status": "failure", "success": false, "title": "G-code to text", "turns": 24 }, { "cache_hit_rate_normalized": 0.7459774483719752, "cost_first_cold_usd": 0.10955699999999999, "duration_seconds": 485.996955, "id": "terminal-bench/dna-insert", "included_in_efficiency": true, "status": "success", "success": true, "title": "DNA insertion", "turns": 7 }, { "cache_hit_rate_normalized": 0.9246322984313377, "cost_first_cold_usd": 0.3967293, "duration_seconds": 624.206517, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 24 }, { "cache_hit_rate_normalized": 0.8957723109222355, "cost_first_cold_usd": 0.0708651, "duration_seconds": 374.652839, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 15 }, { "cache_hit_rate_normalized": 0.8283041201173933, "cost_first_cold_usd": 0.08718719999999999, "duration_seconds": 501.455594, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 11 }, { "cache_hit_rate_normalized": 0.9063238925962903, "cost_first_cold_usd": 0.3197268, "duration_seconds": 653.283447, "id": "terminal-bench/extract-elf", "included_in_efficiency": true, "status": "success", "success": true, "title": "Extract ELF metadata", "turns": 17 }, { "cache_hit_rate_normalized": 0.9602261351941294, "cost_first_cold_usd": 0.5402478000000001, "duration_seconds": 1022.734195, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 46 }, { "cache_hit_rate_normalized": 0.9684645075196027, "cost_first_cold_usd": 0.5391171, "duration_seconds": 1397.6538, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": 72 }, { "cache_hit_rate_normalized": 0.8888928978625756, "cost_first_cold_usd": 0.1655253, "duration_seconds": 904.71296, "id": "terminal-bench/chess-best-move", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Chess best move", "turns": 17 }, { "cache_hit_rate_normalized": 0.8148883768967983, "cost_first_cold_usd": 0.0504192, "duration_seconds": 432.89674, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.9883414702222545, "cost_first_cold_usd": 2.263617, "duration_seconds": 1418.793841, "id": "terminal-bench/code-from-image", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Code from image", "turns": 156 }, { "cache_hit_rate_normalized": 0.7525874211525507, "cost_first_cold_usd": 0.035186699999999994, "duration_seconds": 330.120205, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 5 }, { "cache_hit_rate_normalized": 0.7676660663205848, "cost_first_cold_usd": 0.0426156, "duration_seconds": 530.525501, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 6 }, { "cache_hit_rate_normalized": 0.8316849248237016, "cost_first_cold_usd": 0.1237824, "duration_seconds": 447.942794, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 11 }, { "cache_hit_rate_normalized": 0.9818397561278911, "cost_first_cold_usd": 2.1483075, "duration_seconds": 1542.064479, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": true, "status": "success", "success": true, "title": "Anko typed variable bindings", "turns": 83 }, { "cache_hit_rate_normalized": 0.9893218363962255, "cost_first_cold_usd": 10.4290962, "duration_seconds": 3600.0, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 334 }, { "cache_hit_rate_normalized": 0.9852994566335483, "cost_first_cold_usd": 2.9887040999999996, "duration_seconds": 2944.090855, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": false, "status": "failure", "success": false, "title": "FastAPI deprecation response headers", "turns": 117 }, { "cache_hit_rate_normalized": 0.9502599096121899, "cost_first_cold_usd": 0.7210892999999999, "duration_seconds": 1345.115233, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": false, "status": "failure", "success": false, "title": "HTTPX multipart response parsing", "turns": 31 }, { "cache_hit_rate_normalized": 0.869753655499686, "cost_first_cold_usd": 0.5579430000000001, "duration_seconds": 2744.35367, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": 12 }, { "cache_hit_rate_normalized": 0.9819335434707966, "cost_first_cold_usd": 2.5032462, "duration_seconds": 2350.514284, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python StateMachine state-data scoping", "turns": 90 }, { "cache_hit_rate_normalized": 0.9916176761021973, "cost_first_cold_usd": 6.5588067, "duration_seconds": 2324.304995, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 217 }, { "cache_hit_rate_normalized": 0.9857295835752848, "cost_first_cold_usd": 3.2807807999999996, "duration_seconds": 1499.491454, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 100 }, { "cache_hit_rate_normalized": 0.9929738861576375, "cost_first_cold_usd": 9.1454601, "duration_seconds": 2734.38183, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 254 } ], "termination_anomalies": 2, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9687183076263836, "cache_hit_rate_typical": 0.8718780463171638, "cache_hit_rate_typical_n": 18, "cache_hit_rate_typical_q1": 0.7956912380612775, "cache_hit_rate_typical_q3": 0.9213288258837472, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 0.426188, "cost_per_success_normalized_lower_bound": 0.426188, "duration_coverage": 1.0, "effective_cost_coverage": 0.9, "effective_cost_per_pass": 4.5774473833333325, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 802528.3333333334, "mean_output_tokens": 7843.166666666667, "mean_turns": 22.22222222222222, "median_cost_per_success_normalized": 0.13697925, "median_duration_seconds": 463.7817995, "name": "dsh-ptc", "output_tokens_coverage": 1.0, "pass_rate": 0.6, "success_rate_expected": 0.6, "successful": 18, "task_details": [ { "cache_hit_rate_normalized": 0.7942089991417712, "cost_first_cold_usd": 0.1417131, "duration_seconds": 452.62179, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 6 }, { "cache_hit_rate_normalized": 0.8001379548197965, "cost_first_cold_usd": 0.06388200000000001, "duration_seconds": 229.627694, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 6 }, { "cache_hit_rate_normalized": 0.7538457914202926, "cost_first_cold_usd": 0.15878699999999998, "duration_seconds": 260.172065, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 5 }, { "cache_hit_rate_normalized": 0.8376785687233946, "cost_first_cold_usd": 0.1113138, "duration_seconds": 225.080213, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.8691922802001429, "cost_first_cold_usd": 0.090363, "duration_seconds": 241.832395, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 10 }, { "cache_hit_rate_normalized": 0.8100077530750371, "cost_first_cold_usd": 0.0603378, "duration_seconds": 249.927216, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 6 }, { "cache_hit_rate_normalized": 0.7049772036474165, "cost_first_cold_usd": 0.0546615, "duration_seconds": 263.698732, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 3 }, { "cache_hit_rate_normalized": 0.9190068614274599, "cost_first_cold_usd": 0.267267, "duration_seconds": 1043.703022, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": true, "status": "success", "success": true, "title": "G-code to text", "turns": 21 }, { "cache_hit_rate_normalized": 0.8353699177960453, "cost_first_cold_usd": 0.098256, "duration_seconds": 304.177601, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 8 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1236.71173, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": null }, { "cache_hit_rate_normalized": 0.9221028140358429, "cost_first_cold_usd": 0.1518537, "duration_seconds": 403.016272, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 17 }, { "cache_hit_rate_normalized": 0.8923578995130975, "cost_first_cold_usd": 0.1312467, "duration_seconds": 477.191215, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 12 }, { "cache_hit_rate_normalized": 0.9166516373380053, "cost_first_cold_usd": 0.364344, "duration_seconds": 480.640926, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 23 }, { "cache_hit_rate_normalized": 0.9440362328858685, "cost_first_cold_usd": 0.5371368000000001, "duration_seconds": 548.423624, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 49 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1220.353113, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": null }, { "cache_hit_rate_normalized": 0.8745638124341847, "cost_first_cold_usd": 0.13224539999999999, "duration_seconds": 474.941809, "id": "terminal-bench/chess-best-move", "included_in_efficiency": true, "status": "success", "success": true, "title": "Chess best move", "turns": 16 }, { "cache_hit_rate_normalized": 0.9333643878069996, "cost_first_cold_usd": 0.1648212, "duration_seconds": 520.011397, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 21 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1515.01004, "id": "terminal-bench/code-from-image", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Code from image", "turns": null }, { "cache_hit_rate_normalized": 0.7755028816181911, "cost_first_cold_usd": 0.07552260000000001, "duration_seconds": 690.537836, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 5 }, { "cache_hit_rate_normalized": 0.7426679549846539, "cost_first_cold_usd": 0.1187898, "duration_seconds": 222.209811, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 8 }, { "cache_hit_rate_normalized": 0.8840590539833684, "cost_first_cold_usd": 0.2972562, "duration_seconds": 508.533732, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 17 }, { "cache_hit_rate_normalized": 0.9759749496189809, "cost_first_cold_usd": 1.597662, "duration_seconds": 2866.46446, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": true, "status": "success", "success": true, "title": "Anko typed variable bindings", "turns": 60 }, { "cache_hit_rate_normalized": 0.9941404544864554, "cost_first_cold_usd": 11.713647, "duration_seconds": 3600.0, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 323 }, { "cache_hit_rate_normalized": 0.9907299672238197, "cost_first_cold_usd": 4.0568526, "duration_seconds": 1621.4972, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": false, "status": "failure", "success": false, "title": "FastAPI deprecation response headers", "turns": 175 }, { "cache_hit_rate_normalized": 0.988599880563787, "cost_first_cold_usd": 3.5165243999999998, "duration_seconds": 1468.280258, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": true, "status": "success", "success": true, "title": "HTTPX multipart response parsing", "turns": 130 }, { "cache_hit_rate_normalized": 0.9919067139103346, "cost_first_cold_usd": 19.679399399999998, "duration_seconds": 3600.0, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": 382 }, { "cache_hit_rate_normalized": 0.9950534905415047, "cost_first_cold_usd": 12.949990499999998, "duration_seconds": 3600.0, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Python StateMachine state-data scoping", "turns": 285 }, { "cache_hit_rate_normalized": 0.9896999147345363, "cost_first_cold_usd": 4.632226199999999, "duration_seconds": 1510.337537, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 173 }, { "cache_hit_rate_normalized": 0.9908971060372745, "cost_first_cold_usd": 8.9758725, "duration_seconds": 3020.700291, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 179 }, { "cache_hit_rate_normalized": 0.9940397669759767, "cost_first_cold_usd": 12.252080699999999, "duration_seconds": 3322.275611, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 277 } ], "termination_anomalies": 6, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9845967496707884, "cache_hit_rate_typical": 0.8646567782129603, "cache_hit_rate_typical_n": 18, "cache_hit_rate_typical_q1": 0.808430707854667, "cache_hit_rate_typical_q3": 0.9053307290744409, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 0.8669509333333333, "cost_per_success_normalized_lower_bound": 0.8669509333333333, "duration_coverage": 1.0, "effective_cost_coverage": 0.9, "effective_cost_per_pass": 3.4588863333333335, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 2109078.6666666665, "mean_output_tokens": 9767.555555555555, "mean_turns": 31.0, "median_cost_per_success_normalized": 0.12009539999999999, "median_duration_seconds": 376.5249955, "name": "dsh-standard", "output_tokens_coverage": 1.0, "pass_rate": 0.6, "success_rate_expected": 0.6, "successful": 18, "task_details": [ { "cache_hit_rate_normalized": 0.8732929441692329, "cost_first_cold_usd": 0.18425460000000002, "duration_seconds": 478.292231, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 12 }, { "cache_hit_rate_normalized": 0.8647235085807682, "cost_first_cold_usd": 0.0774324, "duration_seconds": 201.133217, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 9 }, { "cache_hit_rate_normalized": 0.8821332931014974, "cost_first_cold_usd": 0.1279242, "duration_seconds": 305.306002, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 10 }, { "cache_hit_rate_normalized": 0.8180935397870726, "cost_first_cold_usd": 0.10776480000000001, "duration_seconds": 231.953584, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.8052097638771984, "cost_first_cold_usd": 0.0544926, "duration_seconds": 207.369832, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 6 }, { "cache_hit_rate_normalized": 0.7993274888915576, "cost_first_cold_usd": 0.0707388, "duration_seconds": 281.204139, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 6 }, { "cache_hit_rate_normalized": 0.7136824091197659, "cost_first_cold_usd": 0.0701358, "duration_seconds": 251.503434, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 4 }, { "cache_hit_rate_normalized": 0.9239079414801895, "cost_first_cold_usd": 0.4075635, "duration_seconds": 939.75331, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": true, "status": "success", "success": true, "title": "G-code to text", "turns": 28 }, { "cache_hit_rate_normalized": 0.8645900478451524, "cost_first_cold_usd": 0.17592960000000002, "duration_seconds": 424.040142, "id": "terminal-bench/dna-insert", "included_in_efficiency": true, "status": "success", "success": true, "title": "DNA insertion", "turns": 11 }, { "cache_hit_rate_normalized": 0.979490393198769, "cost_first_cold_usd": 1.6989387, "duration_seconds": 1225.098511, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 83 }, { "cache_hit_rate_normalized": 0.9078215544101802, "cost_first_cold_usd": 0.11226660000000001, "duration_seconds": 331.334339, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 14 }, { "cache_hit_rate_normalized": 0.8561947614725554, "cost_first_cold_usd": 0.14232270000000002, "duration_seconds": 219.927734, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 14 }, { "cache_hit_rate_normalized": 0.9076250962764681, "cost_first_cold_usd": 0.3087537, "duration_seconds": 586.559476, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 15 }, { "cache_hit_rate_normalized": 0.968049216289849, "cost_first_cold_usd": 0.5836676999999999, "duration_seconds": 599.409531, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 52 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1295.579095, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": null }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1153.460359, "id": "terminal-bench/chess-best-move", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Chess best move", "turns": null }, { "cache_hit_rate_normalized": 0.7502871834144741, "cost_first_cold_usd": 0.1104042, "duration_seconds": 419.77477, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.9884860631976079, "cost_first_cold_usd": 3.1613801999999995, "duration_seconds": 1356.424876, "id": "terminal-bench/code-from-image", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Code from image", "turns": 146 }, { "cache_hit_rate_normalized": 0.8353998760074396, "cost_first_cold_usd": 0.1023465, "duration_seconds": 427.654299, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 7 }, { "cache_hit_rate_normalized": 0.7564356981423296, "cost_first_cold_usd": 0.108054, "duration_seconds": 333.275221, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 8 }, { "cache_hit_rate_normalized": 0.8978582530672229, "cost_first_cold_usd": 0.5413176, "duration_seconds": 461.408932, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 17 }, { "cache_hit_rate_normalized": 0.9773138242980748, "cost_first_cold_usd": 1.6680831, "duration_seconds": 1064.789992, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Anko typed variable bindings", "turns": 62 }, { "cache_hit_rate_normalized": 0.9939059583445391, "cost_first_cold_usd": 11.6499921, "duration_seconds": 3600.0, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 314 }, { "cache_hit_rate_normalized": 0.9866962049969631, "cost_first_cold_usd": 2.4873890999999997, "duration_seconds": 1708.402664, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": true, "status": "success", "success": true, "title": "FastAPI deprecation response headers", "turns": 118 }, { "cache_hit_rate_normalized": 0.9871389405074537, "cost_first_cold_usd": 3.4286256, "duration_seconds": 1180.561553, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": false, "status": "failure", "success": false, "title": "HTTPX multipart response parsing", "turns": 126 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 3600.0, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": null }, { "cache_hit_rate_normalized": 0.9933735800298378, "cost_first_cold_usd": 10.141112099999999, "duration_seconds": 2835.545963, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python StateMachine state-data scoping", "turns": 227 }, { "cache_hit_rate_normalized": 0.9895836094476065, "cost_first_cold_usd": 6.0233193, "duration_seconds": 2920.538414, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 205 }, { "cache_hit_rate_normalized": 0.9861118828806551, "cost_first_cold_usd": 3.7978254, "duration_seconds": 2104.67607, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 110 }, { "cache_hit_rate_normalized": 0.9945276745450541, "cost_first_cold_usd": 14.917919099999999, "duration_seconds": 3600.0, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 331 } ], "termination_anomalies": 5, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9819904494177557, "cache_hit_rate_typical": 0.8215358636533213, "cache_hit_rate_typical_n": 17, "cache_hit_rate_typical_q1": 0.7889588637277234, "cache_hit_rate_typical_q3": 0.9207865881771227, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 0.8183768647058823, "cost_per_success_normalized_lower_bound": 0.8183768647058823, "duration_coverage": 1.0, "effective_cost_coverage": 1.0, "effective_cost_per_pass": 4.745542235294117, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 2009005.9411764706, "mean_output_tokens": 7865.705882352941, "mean_turns": 26.58823529411765, "median_cost_per_success_normalized": 0.1353654, "median_duration_seconds": 406.40901, "name": "oh-my-pi", "output_tokens_coverage": 1.0, "pass_rate": 0.5666666666666667, "success_rate_expected": 0.5666666666666667, "successful": 17, "task_details": [ { "cache_hit_rate_normalized": 0.6555496373991068, "cost_first_cold_usd": 0.11500200000000001, "duration_seconds": 330.739926, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 3 }, { "cache_hit_rate_normalized": 0.8190644307149162, "cost_first_cold_usd": 0.09894420000000001, "duration_seconds": 196.842292, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 6 }, { "cache_hit_rate_normalized": 0.7889588637277234, "cost_first_cold_usd": 0.1353654, "duration_seconds": 302.130295, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 5 }, { "cache_hit_rate_normalized": 0.8411089263978618, "cost_first_cold_usd": 0.1448508, "duration_seconds": 226.058651, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.8621985523009413, "cost_first_cold_usd": 0.10875119999999999, "duration_seconds": 225.866616, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.7872791856390207, "cost_first_cold_usd": 0.08977740000000001, "duration_seconds": 444.418426, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 5 }, { "cache_hit_rate_normalized": 0.7813885263759612, "cost_first_cold_usd": 0.1321614, "duration_seconds": 603.241675, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 5 }, { "cache_hit_rate_normalized": 0.90542524923581, "cost_first_cold_usd": 0.3566412, "duration_seconds": 962.404377, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": true, "status": "success", "success": true, "title": "G-code to text", "turns": 17 }, { "cache_hit_rate_normalized": 0.8910027245381672, "cost_first_cold_usd": 0.15804240000000003, "duration_seconds": 457.31137, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 11 }, { "cache_hit_rate_normalized": 0.9715839233414972, "cost_first_cold_usd": 0.9679485, "duration_seconds": 1154.675865, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 54 }, { "cache_hit_rate_normalized": 0.9426535776548265, "cost_first_cold_usd": 0.2284299, "duration_seconds": 406.40901, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 21 }, { "cache_hit_rate_normalized": 0.9258159715607864, "cost_first_cold_usd": 0.2095908, "duration_seconds": 513.980984, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 17 }, { "cache_hit_rate_normalized": 0.9078999562639218, "cost_first_cold_usd": 0.25006079999999997, "duration_seconds": 676.373175, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 13 }, { "cache_hit_rate_normalized": 0.9700287757973463, "cost_first_cold_usd": 0.8500073999999999, "duration_seconds": 837.100414, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 51 }, { "cache_hit_rate_normalized": 0.9619214243182819, "cost_first_cold_usd": 0.5728812, "duration_seconds": 1007.720841, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": 43 }, { "cache_hit_rate_normalized": 0.8176227790298003, "cost_first_cold_usd": 0.212634, "duration_seconds": 401.91998, "id": "terminal-bench/chess-best-move", "included_in_efficiency": true, "status": "success", "success": true, "title": "Chess best move", "turns": 11 }, { "cache_hit_rate_normalized": 0.9207865881771227, "cost_first_cold_usd": 0.2159874, "duration_seconds": 438.568068, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 15 }, { "cache_hit_rate_normalized": 0.6087466022349743, "cost_first_cold_usd": 0.1273947, "duration_seconds": 187.524743, "id": "terminal-bench/code-from-image", "included_in_efficiency": true, "status": "success", "success": true, "title": "Code from image", "turns": 5 }, { "cache_hit_rate_normalized": 0.7894973743435859, "cost_first_cold_usd": 0.09800880000000001, "duration_seconds": 460.286037, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 5 }, { "cache_hit_rate_normalized": 0.8215358636533213, "cost_first_cold_usd": 0.113283, "duration_seconds": 370.942431, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 6 }, { "cache_hit_rate_normalized": 0.9283859391443259, "cost_first_cold_usd": 0.2570226, "duration_seconds": 588.594952, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Sanitize git repo", "turns": 19 }, { "cache_hit_rate_normalized": 0.983394938073762, "cost_first_cold_usd": 2.1883542, "duration_seconds": 1042.971676, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Anko typed variable bindings", "turns": 93 }, { "cache_hit_rate_normalized": 0.9926142326413046, "cost_first_cold_usd": 10.6755771, "duration_seconds": 2997.462833, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": true, "status": "success", "success": true, "title": "ArkType JSON Schema reference dependencies", "turns": 265 }, { "cache_hit_rate_normalized": 0.9933354195784234, "cost_first_cold_usd": 8.499726899999999, "duration_seconds": 3227.812803, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": false, "status": "failure", "success": false, "title": "FastAPI deprecation response headers", "turns": 219 }, { "cache_hit_rate_normalized": 0.9864589181184936, "cost_first_cold_usd": 3.4049085, "duration_seconds": 3200.175741, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": false, "status": "failure", "success": false, "title": "HTTPX multipart response parsing", "turns": 130 }, { "cache_hit_rate_normalized": 0.9934447815645797, "cost_first_cold_usd": 9.762501899999998, "duration_seconds": 2024.865805, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": 249 }, { "cache_hit_rate_normalized": 0.9831156188148463, "cost_first_cold_usd": 2.7089568, "duration_seconds": 2546.988718, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Python StateMachine state-data scoping", "turns": 86 }, { "cache_hit_rate_normalized": 0.9871793450590869, "cost_first_cold_usd": 3.4433733, "duration_seconds": 3337.082177, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 133 }, { "cache_hit_rate_normalized": 0.995197599490519, "cost_first_cold_usd": 13.081106400000001, "duration_seconds": 2884.536606, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 322 }, { "cache_hit_rate_normalized": 0.9961072378929074, "cost_first_cold_usd": 21.4669278, "duration_seconds": 3600.0, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 418 } ], "termination_anomalies": 1, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9831361826498795, "cache_hit_rate_typical": 0.8802864677647251, "cache_hit_rate_typical_n": 17, "cache_hit_rate_typical_q1": 0.8437232900503941, "cache_hit_rate_typical_q3": 0.9223928819207066, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 1.3381292823529412, "cost_per_success_normalized_lower_bound": 1.3381292823529412, "duration_coverage": 1.0, "effective_cost_coverage": 1.0, "effective_cost_per_pass": 3.6470870647058824, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 3395606.1764705884, "mean_output_tokens": 10989.176470588236, "mean_turns": 39.88235294117647, "median_cost_per_success_normalized": 0.18175319999999998, "median_duration_seconds": 475.880631, "name": "kimi-code", "output_tokens_coverage": 1.0, "pass_rate": 0.5666666666666667, "success_rate_expected": 0.5666666666666667, "successful": 17, "task_details": [ { "cache_hit_rate_normalized": 0.7299451204963016, "cost_first_cold_usd": 0.18175319999999998, "duration_seconds": 414.60563, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 4 }, { "cache_hit_rate_normalized": 0.8480476369734096, "cost_first_cold_usd": 0.1243242, "duration_seconds": 331.682332, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 7 }, { "cache_hit_rate_normalized": 0.8799307241557006, "cost_first_cold_usd": 0.1985076, "duration_seconds": 402.664333, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 9 }, { "cache_hit_rate_normalized": 0.8437232900503941, "cost_first_cold_usd": 0.15203760000000002, "duration_seconds": 289.976316, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.8681938931021812, "cost_first_cold_usd": 0.1316664, "duration_seconds": 306.719032, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.790340451725934, "cost_first_cold_usd": 0.10736880000000001, "duration_seconds": 336.234659, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 5 }, { "cache_hit_rate_normalized": 0.7371735257537899, "cost_first_cold_usd": 0.1205136, "duration_seconds": 362.632817, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 4 }, { "cache_hit_rate_normalized": 0.9108731567031757, "cost_first_cold_usd": 0.2439444, "duration_seconds": 1176.254916, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": false, "status": "failure", "success": false, "title": "G-code to text", "turns": 14 }, { "cache_hit_rate_normalized": 0.8595140552649572, "cost_first_cold_usd": 0.1719462, "duration_seconds": 375.61529, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 8 }, { "cache_hit_rate_normalized": 0.9722015213438747, "cost_first_cold_usd": 1.5818799, "duration_seconds": 1112.983033, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 58 }, { "cache_hit_rate_normalized": 0.8991681567286014, "cost_first_cold_usd": 0.1898352, "duration_seconds": 497.426455, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 11 }, { "cache_hit_rate_normalized": 0.8953452222818867, "cost_first_cold_usd": 0.19904400000000003, "duration_seconds": 366.125137, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 12 }, { "cache_hit_rate_normalized": 0.9172773630995957, "cost_first_cold_usd": 0.2838405, "duration_seconds": 445.424176, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 15 }, { "cache_hit_rate_normalized": 0.9724672822159776, "cost_first_cold_usd": 0.711273, "duration_seconds": 732.928995, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 50 }, { "cache_hit_rate_normalized": 0.9511572397967745, "cost_first_cold_usd": 0.420819, "duration_seconds": 1474.533994, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": 27 }, { "cache_hit_rate_normalized": 0.9309371279550995, "cost_first_cold_usd": 0.2176368, "duration_seconds": 581.982615, "id": "terminal-bench/chess-best-move", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Chess best move", "turns": 16 }, { "cache_hit_rate_normalized": 0.8911116826728163, "cost_first_cold_usd": 0.1737879, "duration_seconds": 538.422357, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 10 }, { "cache_hit_rate_normalized": 0.9887959801082877, "cost_first_cold_usd": 3.6536916, "duration_seconds": 1591.018641, "id": "terminal-bench/code-from-image", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Code from image", "turns": 143 }, { "cache_hit_rate_normalized": 0.7377011388261331, "cost_first_cold_usd": 0.10039799999999999, "duration_seconds": 529.180515, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 4 }, { "cache_hit_rate_normalized": 0.8802864677647251, "cost_first_cold_usd": 0.15870420000000002, "duration_seconds": 487.409898, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 9 }, { "cache_hit_rate_normalized": 0.9223928819207066, "cost_first_cold_usd": 0.2687964, "duration_seconds": 475.880631, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 16 }, { "cache_hit_rate_normalized": 0.9861015670135074, "cost_first_cold_usd": 2.3534477999999996, "duration_seconds": 1612.744473, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Anko typed variable bindings", "turns": 92 }, { "cache_hit_rate_normalized": 0.9912460149524793, "cost_first_cold_usd": 4.807935, "duration_seconds": 2929.515774, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 176 }, { "cache_hit_rate_normalized": 0.987987740147873, "cost_first_cold_usd": 2.8207434, "duration_seconds": 2144.688679, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": true, "status": "success", "success": true, "title": "FastAPI deprecation response headers", "turns": 122 }, { "cache_hit_rate_normalized": 0.9825290460029232, "cost_first_cold_usd": 1.7338854, "duration_seconds": 3468.329435, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": false, "status": "failure", "success": false, "title": "HTTPX multipart response parsing", "turns": 75 }, { "cache_hit_rate_normalized": 0.9921455606304462, "cost_first_cold_usd": 16.1651922, "duration_seconds": 2935.417421, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": 303 }, { "cache_hit_rate_normalized": 0.9933344876596961, "cost_first_cold_usd": 9.234883199999999, "duration_seconds": 2298.022375, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python StateMachine state-data scoping", "turns": 209 }, { "cache_hit_rate_normalized": 0.9915653515453358, "cost_first_cold_usd": 5.5781022, "duration_seconds": 3150.868921, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 184 }, { "cache_hit_rate_normalized": 0.9845239173042901, "cost_first_cold_usd": 2.0399613, "duration_seconds": 2668.571971, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 69 }, { "cache_hit_rate_normalized": 0.9825592941637771, "cost_first_cold_usd": 7.874561099999999, "duration_seconds": 2433.972661, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": true, "status": "success", "success": true, "title": "Meriyah explicit resource declarations", "turns": 191 } ], "termination_anomalies": 4, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.986112014082598, "cache_hit_rate_typical": 0.8463733761676777, "cache_hit_rate_typical_n": 17, "cache_hit_rate_typical_q1": 0.7652086475615887, "cache_hit_rate_typical_q3": 0.936116910229645, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 1.8177769235294117, "cost_per_success_normalized_lower_bound": 1.8177769235294117, "duration_coverage": 1.0, "effective_cost_coverage": 0.8666666666666667, "effective_cost_per_pass": 4.717955664705882, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 4790827.235294118, "mean_output_tokens": 13392.29411764706, "mean_turns": 47.64705882352941, "median_cost_per_success_normalized": 0.12140220000000002, "median_duration_seconds": 341.183775, "name": "dsh-minimal", "output_tokens_coverage": 1.0, "pass_rate": 0.5666666666666667, "success_rate_expected": 0.5666666666666667, "successful": 17, "task_details": [ { "cache_hit_rate_normalized": 0.7597954857214116, "cost_first_cold_usd": 0.12140220000000002, "duration_seconds": 269.141038, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 5 }, { "cache_hit_rate_normalized": 0.8463733761676777, "cost_first_cold_usd": 0.0710346, "duration_seconds": 206.252218, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 8 }, { "cache_hit_rate_normalized": 0.6439051655875914, "cost_first_cold_usd": 0.1110972, "duration_seconds": 246.150388, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 3 }, { "cache_hit_rate_normalized": 0.8521120444829751, "cost_first_cold_usd": 0.12517440000000002, "duration_seconds": 490.112418, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.8364254733744769, "cost_first_cold_usd": 0.055870800000000005, "duration_seconds": 185.732666, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.823590131782679, "cost_first_cold_usd": 0.0739686, "duration_seconds": 278.877852, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 7 }, { "cache_hit_rate_normalized": 0.7052183497396526, "cost_first_cold_usd": 0.057889499999999997, "duration_seconds": 255.964282, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 3 }, { "cache_hit_rate_normalized": 0.9247592865353432, "cost_first_cold_usd": 0.5255453999999999, "duration_seconds": 968.132519, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": false, "status": "failure", "success": false, "title": "G-code to text", "turns": 28 }, { "cache_hit_rate_normalized": 0.8545031010113917, "cost_first_cold_usd": 0.1498704, "duration_seconds": 363.550286, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 10 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1029.596032, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": null }, { "cache_hit_rate_normalized": 0.9107764306990682, "cost_first_cold_usd": 0.15140879999999998, "duration_seconds": 608.007116, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 17 }, { "cache_hit_rate_normalized": 0.8890689886135298, "cost_first_cold_usd": 0.12597239999999998, "duration_seconds": 272.290892, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 13 }, { "cache_hit_rate_normalized": 0.9660734327311624, "cost_first_cold_usd": 3.1168962000000002, "duration_seconds": 1090.329152, "id": "terminal-bench/extract-elf", "included_in_efficiency": true, "status": "success", "success": true, "title": "Extract ELF metadata", "turns": 51 }, { "cache_hit_rate_normalized": 0.9691902026307618, "cost_first_cold_usd": 0.7855827, "duration_seconds": 818.92359, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 54 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1221.753624, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": null }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1079.159044, "id": "terminal-bench/chess-best-move", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Chess best move", "turns": null }, { "cache_hit_rate_normalized": 0.7286679536679537, "cost_first_cold_usd": 0.0997839, "duration_seconds": 301.327965, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 8 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 1578.177277, "id": "terminal-bench/code-from-image", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Code from image", "turns": null }, { "cache_hit_rate_normalized": 0.7652086475615887, "cost_first_cold_usd": 0.1108614, "duration_seconds": 414.517787, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 8 }, { "cache_hit_rate_normalized": 0.8049392243874204, "cost_first_cold_usd": 0.081591, "duration_seconds": 341.183775, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 6 }, { "cache_hit_rate_normalized": 0.936116910229645, "cost_first_cold_usd": 0.5355144, "duration_seconds": 584.581518, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 25 }, { "cache_hit_rate_normalized": 0.9886979767354775, "cost_first_cold_usd": 3.838266, "duration_seconds": 1760.447038, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Anko typed variable bindings", "turns": 131 }, { "cache_hit_rate_normalized": 0.9947743715160204, "cost_first_cold_usd": 11.5114092, "duration_seconds": 3290.132075, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 342 }, { "cache_hit_rate_normalized": 0.9631315050877757, "cost_first_cold_usd": 2.4124182000000003, "duration_seconds": 1342.552375, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": false, "status": "failure", "success": false, "title": "FastAPI deprecation response headers", "turns": 82 }, { "cache_hit_rate_normalized": 0.9793643265902463, "cost_first_cold_usd": 1.5411852, "duration_seconds": 1075.04796, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": false, "status": "failure", "success": false, "title": "HTTPX multipart response parsing", "turns": 72 }, { "cache_hit_rate_normalized": 0.9910526062598207, "cost_first_cold_usd": 14.7911466, "duration_seconds": 2675.040662, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": true, "status": "success", "success": true, "title": "Expr try/catch errors", "turns": 328 }, { "cache_hit_rate_normalized": 0.9910411145404256, "cost_first_cold_usd": 10.487013, "duration_seconds": 2662.587099, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python StateMachine state-data scoping", "turns": 259 }, { "cache_hit_rate_normalized": 0.9923977058914512, "cost_first_cold_usd": 7.1623977000000005, "duration_seconds": 2485.273986, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 217 }, { "cache_hit_rate_normalized": 0.9910619379825494, "cost_first_cold_usd": 5.576249399999999, "duration_seconds": 2374.015802, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 171 }, { "cache_hit_rate_normalized": 0.9951068352328717, "cost_first_cold_usd": 16.5856971, "duration_seconds": 3600.0, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 359 } ], "termination_anomalies": 5, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9448246779872019, "cache_hit_rate_typical": 0.7032915021509234, "cache_hit_rate_typical_n": 16, "cache_hit_rate_typical_q1": 0.5687611114199898, "cache_hit_rate_typical_q3": 0.8640121062605165, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 0.25160306250000003, "cost_per_success_normalized_lower_bound": 0.25160306250000003, "duration_coverage": 1.0, "effective_cost_coverage": 1.0, "effective_cost_per_pass": 1.04517110625, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 375125.8125, "mean_output_tokens": 5545.4375, "mean_turns": 12.0, "median_cost_per_success_normalized": 0.07481280000000001, "median_duration_seconds": 377.205152, "name": "exo", "output_tokens_coverage": 1.0, "pass_rate": 0.5333333333333333, "success_rate_expected": 0.5333333333333333, "successful": 16, "task_details": [ { "cache_hit_rate_normalized": 0.16253968253968254, "cost_first_cold_usd": 0.0640638, "duration_seconds": 271.996297, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 2 }, { "cache_hit_rate_normalized": 0.5668593922991758, "cost_first_cold_usd": 0.0382104, "duration_seconds": 200.243704, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 6 }, { "cache_hit_rate_normalized": 0.421630524293165, "cost_first_cold_usd": 0.08556180000000001, "duration_seconds": 244.082444, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 3 }, { "cache_hit_rate_normalized": 0.5693950177935944, "cost_first_cold_usd": 0.12669, "duration_seconds": 375.985151, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.6952743074416078, "cost_first_cold_usd": 0.036357, "duration_seconds": 235.854449, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.711308696860239, "cost_first_cold_usd": 0.05619, "duration_seconds": 235.03227, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 7 }, { "cache_hit_rate_normalized": 0.4336310440406529, "cost_first_cold_usd": 0.0552138, "duration_seconds": 271.298409, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 3 }, { "cache_hit_rate_normalized": 0.8401435792249652, "cost_first_cold_usd": 0.16711620000000002, "duration_seconds": 378.425153, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": true, "status": "success", "success": true, "title": "G-code to text", "turns": 14 }, { "cache_hit_rate_normalized": 0.7661857300166587, "cost_first_cold_usd": 0.0940788, "duration_seconds": 291.393088, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 8 }, { "cache_hit_rate_normalized": 0.9609247717224553, "cost_first_cold_usd": 0.6953436, "duration_seconds": 856.606099, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 51 }, { "cache_hit_rate_normalized": 0.9040425849303991, "cost_first_cold_usd": 0.1779351, "duration_seconds": 443.064617, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 22 }, { "cache_hit_rate_normalized": 0.8776464718904929, "cost_first_cold_usd": 0.1669494, "duration_seconds": 411.090475, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 13 }, { "cache_hit_rate_normalized": 0.8825076144356553, "cost_first_cold_usd": 0.1815405, "duration_seconds": 458.259041, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 16 }, { "cache_hit_rate_normalized": 0.9734799622446241, "cost_first_cold_usd": 1.8345993, "duration_seconds": 736.118803, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 46 }, { "cache_hit_rate_normalized": 0.955423389208718, "cost_first_cold_usd": 0.7547739, "duration_seconds": 1087.813479, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": 50 }, { "cache_hit_rate_normalized": 0.9414928386295172, "cost_first_cold_usd": 0.4468065, "duration_seconds": 1136.36834, "id": "terminal-bench/chess-best-move", "included_in_efficiency": true, "status": "success", "success": true, "title": "Chess best move", "turns": 28 }, { "cache_hit_rate_normalized": 0.8190338095911728, "cost_first_cold_usd": 0.0556587, "duration_seconds": 296.665963, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 10 }, { "cache_hit_rate_normalized": 0.9603973956202123, "cost_first_cold_usd": 0.6736188000000001, "duration_seconds": 472.889313, "id": "terminal-bench/code-from-image", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Code from image", "turns": 51 }, { "cache_hit_rate_normalized": 0.6789850587923261, "cost_first_cold_usd": 0.039792, "duration_seconds": 748.410841, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 5 }, { "cache_hit_rate_normalized": 0.6102793935348527, "cost_first_cold_usd": 0.053781, "duration_seconds": 591.398103, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 5 }, { "cache_hit_rate_normalized": 0.8594673177171911, "cost_first_cold_usd": 0.620724, "duration_seconds": 668.324059, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 13 }, { "cache_hit_rate_normalized": 0.9169453626378099, "cost_first_cold_usd": 1.5780666, "duration_seconds": 954.5815, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Anko typed variable bindings", "turns": 51 }, { "cache_hit_rate_normalized": 0.9659783061323136, "cost_first_cold_usd": 0.7602618000000001, "duration_seconds": 767.904323, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 51 }, { "cache_hit_rate_normalized": 0.9657976226406185, "cost_first_cold_usd": 0.9321957, "duration_seconds": 903.298315, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": false, "status": "failure", "success": false, "title": "FastAPI deprecation response headers", "turns": 51 }, { "cache_hit_rate_normalized": 0.970457094572505, "cost_first_cold_usd": 0.9154883999999999, "duration_seconds": 615.677396, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": false, "status": "failure", "success": false, "title": "HTTPX multipart response parsing", "turns": 51 }, { "cache_hit_rate_normalized": 0.967546083165983, "cost_first_cold_usd": 1.6453358999999999, "duration_seconds": 1451.505758, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": 51 }, { "cache_hit_rate_normalized": 0.9604718099857591, "cost_first_cold_usd": 1.4576319, "duration_seconds": 2416.253724, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Python StateMachine state-data scoping", "turns": 51 }, { "cache_hit_rate_normalized": 0.968871939864911, "cost_first_cold_usd": 0.7664088, "duration_seconds": 768.425484, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 51 }, { "cache_hit_rate_normalized": 0.9723796422232297, "cost_first_cold_usd": 1.3749165, "duration_seconds": 2859.990994, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 51 }, { "cache_hit_rate_normalized": 0.9672830442898566, "cost_first_cold_usd": 0.8674275, "duration_seconds": 2311.092173, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 51 } ], "termination_anomalies": 1, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9157781520720139, "cache_hit_rate_typical": 0.7841402195184208, "cache_hit_rate_typical_n": 15, "cache_hit_rate_typical_q1": 0.7508016284649194, "cache_hit_rate_typical_q3": 0.8425748720342752, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 0.11694735999999999, "cost_per_success_normalized_lower_bound": 0.11694735999999999, "duration_coverage": 1.0, "effective_cost_coverage": 0.9666666666666667, "effective_cost_per_pass": 3.24434552, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 153533.4, "mean_output_tokens": 2398.266666666667, "mean_turns": 11.2, "median_cost_per_success_normalized": 0.0614517, "median_duration_seconds": 387.253527, "name": "opencode", "output_tokens_coverage": 1.0, "pass_rate": 0.5, "success_rate_expected": 0.5, "successful": 15, "task_details": [ { "cache_hit_rate_normalized": 0.7149173842215499, "cost_first_cold_usd": 0.06409619999999999, "duration_seconds": 661.527459, "id": "terminal-bench/regex-log", "included_in_efficiency": true, "status": "success", "success": true, "title": "Regex log", "turns": 4 }, { "cache_hit_rate_normalized": 0.7802031779109143, "cost_first_cold_usd": 0.0442746, "duration_seconds": 186.768145, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 5 }, { "cache_hit_rate_normalized": 0.7828389830508474, "cost_first_cold_usd": 0.055341299999999996, "duration_seconds": 290.151698, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 5 }, { "cache_hit_rate_normalized": 0.8247369273109924, "cost_first_cold_usd": 0.08698620000000001, "duration_seconds": 231.245223, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.773600042787613, "cost_first_cold_usd": 0.0449364, "duration_seconds": 216.49833, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 5 }, { "cache_hit_rate_normalized": 0.7139834583573764, "cost_first_cold_usd": 0.0438126, "duration_seconds": 463.111223, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 4 }, { "cache_hit_rate_normalized": 0.6279395732751908, "cost_first_cold_usd": 0.055331399999999996, "duration_seconds": 613.075636, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 3 }, { "cache_hit_rate_normalized": 0.9062220441585669, "cost_first_cold_usd": 0.369669, "duration_seconds": 1957.845858, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": false, "status": "failure", "success": false, "title": "G-code to text", "turns": 30 }, { "cache_hit_rate_normalized": 0.8164858370348795, "cost_first_cold_usd": 0.0768672, "duration_seconds": 560.833266, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 7 }, { "cache_hit_rate_normalized": 0.7845784431431162, "cost_first_cold_usd": 0.0548673, "duration_seconds": 1383.530415, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 5 }, { "cache_hit_rate_normalized": 0.8727057757544282, "cost_first_cold_usd": 0.0878337, "duration_seconds": 349.54894, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 10 }, { "cache_hit_rate_normalized": 0.887810698523598, "cost_first_cold_usd": 0.1085382, "duration_seconds": 436.319414, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 13 }, { "cache_hit_rate_normalized": 0.8903783872422416, "cost_first_cold_usd": 0.0854358, "duration_seconds": 216.81892, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 11 }, { "cache_hit_rate_normalized": 0.9766758584465892, "cost_first_cold_usd": 0.6692096999999999, "duration_seconds": 1081.03508, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 66 }, { "cache_hit_rate_normalized": 0.975736005039769, "cost_first_cold_usd": 1.1044407, "duration_seconds": 1081.375585, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": 81 }, { "cache_hit_rate_normalized": 0.8472635762835439, "cost_first_cold_usd": 0.2662152, "duration_seconds": 729.751536, "id": "terminal-bench/chess-best-move", "included_in_efficiency": true, "status": "success", "success": true, "title": "Chess best move", "turns": 22 }, { "cache_hit_rate_normalized": 0.8306784660766962, "cost_first_cold_usd": 0.0562206, "duration_seconds": 672.965218, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.7841402195184208, "cost_first_cold_usd": 0.0637032, "duration_seconds": 387.253527, "id": "terminal-bench/code-from-image", "included_in_efficiency": true, "status": "success", "success": true, "title": "Code from image", "turns": 6 }, { "cache_hit_rate_normalized": 0.7280032141422258, "cost_first_cold_usd": 0.0462594, "duration_seconds": 271.643418, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 4 }, { "cache_hit_rate_normalized": 0.8378861677850064, "cost_first_cold_usd": 0.0614517, "duration_seconds": 207.50588, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 7 }, { "cache_hit_rate_normalized": 0.642570281124498, "cost_first_cold_usd": 0.031575, "duration_seconds": 1746.344619, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Sanitize git repo", "turns": 3 }, { "cache_hit_rate_normalized": 0.9749787893114784, "cost_first_cold_usd": 0.9619736999999999, "duration_seconds": 1066.581182, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Anko typed variable bindings", "turns": 49 }, { "cache_hit_rate_normalized": 0.9950570218436406, "cost_first_cold_usd": 12.5675406, "duration_seconds": 3600.0, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": 351 }, { "cache_hit_rate_normalized": 0.9619978538296623, "cost_first_cold_usd": 0.6162837, "duration_seconds": 3600.0, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": false, "status": "failure", "success": false, "title": "FastAPI deprecation response headers", "turns": 37 }, { "cache_hit_rate_normalized": 0.9738576548039392, "cost_first_cold_usd": 0.9214398, "duration_seconds": 1125.910344, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": false, "status": "failure", "success": false, "title": "HTTPX multipart response parsing", "turns": 51 }, { "cache_hit_rate_normalized": 0.9941860802678416, "cost_first_cold_usd": 9.5371572, "duration_seconds": 2979.952022, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": 234 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 3600.0, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Python StateMachine state-data scoping", "turns": null }, { "cache_hit_rate_normalized": 0.9864113575783735, "cost_first_cold_usd": 2.0523428999999997, "duration_seconds": 1270.540777, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 112 }, { "cache_hit_rate_normalized": 0.9917102218366836, "cost_first_cold_usd": 5.4572895, "duration_seconds": 2098.990524, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 197 }, { "cache_hit_rate_normalized": 0.99514345854642, "cost_first_cold_usd": 13.07409, "duration_seconds": 2944.994884, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 350 } ], "termination_anomalies": 5, "turns_coverage": 1.0, "valid_coverage": 1.0 }, { "cache_coverage": 1.0, "cache_hit_rate_normalized": 0.9783811552006846, "cache_hit_rate_typical": 0.8591308757616211, "cache_hit_rate_typical_n": 15, "cache_hit_rate_typical_q1": 0.8246243365411987, "cache_hit_rate_typical_q3": 0.9292190955604376, "completed": 30, "cost_coverage": 1.0, "cost_per_success_normalized": 0.76843384, "cost_per_success_normalized_lower_bound": 0.76843384, "duration_coverage": 1.0, "effective_cost_coverage": 0.9333333333333333, "effective_cost_per_pass": 2.9043034800000003, "expected": 30, "infra_invalid": 0, "input_tokens_coverage": 1.0, "mean_input_tokens": 1805810.3333333333, "mean_output_tokens": 8085.6, "mean_turns": 25.933333333333334, "median_cost_per_success_normalized": 0.174564, "median_duration_seconds": 417.767917, "name": "hermes", "output_tokens_coverage": 1.0, "pass_rate": 0.5, "success_rate_expected": 0.5, "successful": 15, "task_details": [ { "cache_hit_rate_normalized": 0.7255014834167294, "cost_first_cold_usd": 0.1311966, "duration_seconds": 342.900472, "id": "terminal-bench/regex-log", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Regex log", "turns": 4 }, { "cache_hit_rate_normalized": 0.7875597856129978, "cost_first_cold_usd": 0.0927798, "duration_seconds": 199.77349, "id": "terminal-bench/openssl-selfsigned-cert", "included_in_efficiency": true, "status": "success", "success": true, "title": "OpenSSL self-signed certificate", "turns": 5 }, { "cache_hit_rate_normalized": 0.841607643714992, "cost_first_cold_usd": 0.174564, "duration_seconds": 547.60882, "id": "terminal-bench/polyglot-c-py", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python / C polyglot", "turns": 7 }, { "cache_hit_rate_normalized": 0.8934383738673949, "cost_first_cold_usd": 0.27574079999999995, "duration_seconds": 323.932332, "id": "terminal-bench/sqlite-db-truncate", "included_in_efficiency": true, "status": "success", "success": true, "title": "SQLite database recovery", "turns": 12 }, { "cache_hit_rate_normalized": 0.8591308757616211, "cost_first_cold_usd": 0.109437, "duration_seconds": 235.085136, "id": "terminal-bench/git-leak-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Git leak recovery", "turns": 8 }, { "cache_hit_rate_normalized": 0.7817408193279669, "cost_first_cold_usd": 0.09884760000000001, "duration_seconds": 495.179967, "id": "terminal-bench/log-summary-date-ranges", "included_in_efficiency": true, "status": "success", "success": true, "title": "Log summary date ranges", "turns": 5 }, { "cache_hit_rate_normalized": 0.8080357142857143, "cost_first_cold_usd": 0.1569096, "duration_seconds": 650.349146, "id": "terminal-bench/constraints-scheduling", "included_in_efficiency": true, "status": "success", "success": true, "title": "Constraints scheduling", "turns": 6 }, { "cache_hit_rate_normalized": 0.9599569107542542, "cost_first_cold_usd": 0.9200453999999999, "duration_seconds": 1341.767009, "id": "terminal-bench/gcode-to-text", "included_in_efficiency": false, "status": "failure", "success": false, "title": "G-code to text", "turns": 42 }, { "cache_hit_rate_normalized": 0.8771219436224887, "cost_first_cold_usd": 0.19970459999999998, "duration_seconds": 620.355562, "id": "terminal-bench/dna-insert", "included_in_efficiency": false, "status": "failure", "success": false, "title": "DNA insertion", "turns": 10 }, { "cache_hit_rate_normalized": 0.9388242308721555, "cost_first_cold_usd": 0.3137253, "duration_seconds": 586.837393, "id": "terminal-bench/largest-eigenval", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Largest eigenvalue", "turns": 20 }, { "cache_hit_rate_normalized": 0.8861811837281517, "cost_first_cold_usd": 0.1746147, "duration_seconds": 377.570819, "id": "terminal-bench/merge-diff-arc-agi-task", "included_in_efficiency": true, "status": "success", "success": true, "title": "Merge-diff ARC-AGI task", "turns": 10 }, { "cache_hit_rate_normalized": 0.894523722990626, "cost_first_cold_usd": 0.2052225, "duration_seconds": 340.019699, "id": "terminal-bench/vulnerable-secret", "included_in_efficiency": true, "status": "success", "success": true, "title": "Vulnerable secret", "turns": 12 }, { "cache_hit_rate_normalized": 0.9591445049091102, "cost_first_cold_usd": 0.6459762, "duration_seconds": 691.145963, "id": "terminal-bench/extract-elf", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Extract ELF metadata", "turns": 34 }, { "cache_hit_rate_normalized": 0.9690751469069813, "cost_first_cold_usd": 0.8629613999999999, "duration_seconds": 961.360162, "id": "terminal-bench/build-cython-ext", "included_in_efficiency": true, "status": "success", "success": true, "title": "Build Cython extension", "turns": 43 }, { "cache_hit_rate_normalized": 0.9718969349555452, "cost_first_cold_usd": 0.5524808999999999, "duration_seconds": 945.340685, "id": "terminal-bench/kv-store-grpc", "included_in_efficiency": false, "status": "failure", "success": false, "title": "gRPC key-value store", "turns": 52 }, { "cache_hit_rate_normalized": 0.9665617075013715, "cost_first_cold_usd": 1.1974271999999997, "duration_seconds": 1364.584125, "id": "terminal-bench/chess-best-move", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Chess best move", "turns": 55 }, { "cache_hit_rate_normalized": 0.841325023559318, "cost_first_cold_usd": 0.116793, "duration_seconds": 417.767917, "id": "terminal-bench/db-wal-recovery", "included_in_efficiency": true, "status": "success", "success": true, "title": "Database WAL recovery", "turns": 7 }, { "cache_hit_rate_normalized": 0.9866880373769646, "cost_first_cold_usd": 4.1954631, "duration_seconds": 1343.814004, "id": "terminal-bench/code-from-image", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Code from image", "turns": 134 }, { "cache_hit_rate_normalized": 0.7891442377928864, "cost_first_cold_usd": 0.0938856, "duration_seconds": 310.114343, "id": "terminal-bench/modernize-scientific-stack", "included_in_efficiency": true, "status": "success", "success": true, "title": "Modernize scientific stack", "turns": 5 }, { "cache_hit_rate_normalized": 0.8412129587966831, "cost_first_cold_usd": 0.1302492, "duration_seconds": 382.078408, "id": "terminal-bench/multi-source-data-merger", "included_in_efficiency": true, "status": "success", "success": true, "title": "Multi-source data merger", "turns": 7 }, { "cache_hit_rate_normalized": 0.9639144681302492, "cost_first_cold_usd": 0.6791280000000001, "duration_seconds": 880.176858, "id": "terminal-bench/sanitize-git-repo", "included_in_efficiency": true, "status": "success", "success": true, "title": "Sanitize git repo", "turns": 41 }, { "cache_hit_rate_normalized": 0.9662230372058471, "cost_first_cold_usd": 1.0657785, "duration_seconds": 1611.125838, "id": "datacurve/anko-typed-variable-bindings", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Anko typed variable bindings", "turns": 43 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 3600.0, "id": "datacurve/arktype-json-schema-refs-dependencies", "included_in_efficiency": false, "status": "failure", "success": false, "title": "ArkType JSON Schema reference dependencies", "turns": null }, { "cache_hit_rate_normalized": 0.952792874404266, "cost_first_cold_usd": 1.9794789, "duration_seconds": 1369.124073, "id": "datacurve/fastapi-deprecation-response-headers", "included_in_efficiency": false, "status": "failure", "success": false, "title": "FastAPI deprecation response headers", "turns": 67 }, { "cache_hit_rate_normalized": 0.9781849148042598, "cost_first_cold_usd": 1.6602534, "duration_seconds": 911.3716, "id": "datacurve/httpx-multipart-response-parsing", "included_in_efficiency": true, "status": "success", "success": true, "title": "HTTPX multipart response parsing", "turns": 65 }, { "cache_hit_rate_normalized": null, "cost_first_cold_usd": null, "duration_seconds": 3600.0, "id": "datacurve/expr-try-catch-errors", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Expr try/catch errors", "turns": null }, { "cache_hit_rate_normalized": 0.9911223704490625, "cost_first_cold_usd": 6.695121, "duration_seconds": 1728.459216, "id": "datacurve/python-statemachine-state-data-scoping", "included_in_efficiency": true, "status": "success", "success": true, "title": "Python StateMachine state-data scoping", "turns": 156 }, { "cache_hit_rate_normalized": 0.988767633726645, "cost_first_cold_usd": 4.3431783, "duration_seconds": 1332.619645, "id": "datacurve/katex-multicolumn-array-spans", "included_in_efficiency": false, "status": "failure", "success": false, "title": "KaTeX multicolumn array spans", "turns": 141 }, { "cache_hit_rate_normalized": 0.9601382345719525, "cost_first_cold_usd": 0.6454529999999999, "duration_seconds": 3517.80501, "id": "datacurve/scc-bounded-memory-spilling", "included_in_efficiency": false, "status": "failure", "success": false, "title": "SCC bounded-memory spilling", "turns": 32 }, { "cache_hit_rate_normalized": 0.9947323027702997, "cost_first_cold_usd": 15.8481366, "duration_seconds": 3290.534308, "id": "datacurve/meriyah-explicit-resource-declarations", "included_in_efficiency": false, "status": "failure", "success": false, "title": "Meriyah explicit resource declarations", "turns": 327 } ], "termination_anomalies": 5, "turns_coverage": 1.0, "valid_coverage": 1.0 } ], "model": "k3", "overview": { "cache_hit_rate_normalized": 0.9236274135100379, "checkpoint_tasks": 30, "completed_cells": 360, "cost_coverage": 1.0, "cost_per_success_normalized": 1.219856537799043, "cost_per_success_normalized_lower_bound": 1.219856537799043, "expected_cells": 360, "harnesses": 9, "harness_configurations": 12, "infra_invalid_cells": 0, "pass_rate": 0.5805555555555556, "success_rate_expected": 0.5805555555555556, "successful_cells": 209, "termination_anomalies": 52, "valid_coverage": 1.0 }, "tasks": [ { "checkpoint": "ce-regex-log-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/regex-log", "successful": 11, "title": "Regex log", "valid_coverage": 1.0 }, { "checkpoint": "ce-openssl-cert-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/openssl-selfsigned-cert", "successful": 12, "title": "OpenSSL self-signed certificate", "valid_coverage": 1.0 }, { "checkpoint": "ce-polyglot-c-py-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/polyglot-c-py", "successful": 12, "title": "Python / C polyglot", "valid_coverage": 1.0 }, { "checkpoint": "ce-sqlite-recovery-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/sqlite-db-truncate", "successful": 12, "title": "SQLite database recovery", "valid_coverage": 1.0 }, { "checkpoint": "ce-git-leak-recovery-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/git-leak-recovery", "successful": 12, "title": "Git leak recovery", "valid_coverage": 1.0 }, { "checkpoint": "ce-log-summary-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/log-summary-date-ranges", "successful": 12, "title": "Log summary date ranges", "valid_coverage": 1.0 }, { "checkpoint": "ce-constraints-scheduling-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/constraints-scheduling", "successful": 12, "title": "Constraints scheduling", "valid_coverage": 1.0 }, { "checkpoint": "ce-gcode-text-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/gcode-to-text", "successful": 5, "title": "G-code to text", "valid_coverage": 1.0 }, { "checkpoint": "ce-dna-insert-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/dna-insert", "successful": 2, "title": "DNA insertion", "valid_coverage": 1.0 }, { "checkpoint": "ce-largest-eigen-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/largest-eigenval", "successful": 0, "title": "Largest eigenvalue", "valid_coverage": 1.0 }, { "checkpoint": "ce-merge-arc-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/merge-diff-arc-agi-task", "successful": 12, "title": "Merge-diff ARC-AGI task", "valid_coverage": 1.0 }, { "checkpoint": "ce-vulnerable-secret-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/vulnerable-secret", "successful": 12, "title": "Vulnerable secret", "valid_coverage": 1.0 }, { "checkpoint": "ce-extract-elf-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/extract-elf", "successful": 2, "title": "Extract ELF metadata", "valid_coverage": 1.0 }, { "checkpoint": "ce-build-cython-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/build-cython-ext", "successful": 12, "title": "Build Cython extension", "valid_coverage": 1.0 }, { "checkpoint": "ce-kv-grpc-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/kv-store-grpc", "successful": 0, "title": "gRPC key-value store", "valid_coverage": 1.0 }, { "checkpoint": "ce-chess-move-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/chess-best-move", "successful": 5, "title": "Chess best move", "valid_coverage": 1.0 }, { "checkpoint": "ce-db-wal-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/db-wal-recovery", "successful": 12, "title": "Database WAL recovery", "valid_coverage": 1.0 }, { "checkpoint": "ce-code-image-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/code-from-image", "successful": 4, "title": "Code from image", "valid_coverage": 1.0 }, { "checkpoint": "ce-modernize-science-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/modernize-scientific-stack", "successful": 12, "title": "Modernize scientific stack", "valid_coverage": 1.0 }, { "checkpoint": "ce-data-merger-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/multi-source-data-merger", "successful": 12, "title": "Multi-source data merger", "valid_coverage": 1.0 }, { "checkpoint": "ce-sanitize-git-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "terminal-bench/sanitize-git-repo", "successful": 10, "title": "Sanitize git repo", "valid_coverage": 1.0 }, { "checkpoint": "ce-anko-bindings-docker-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/anko-typed-variable-bindings", "successful": 4, "title": "Anko typed variable bindings", "valid_coverage": 1.0 }, { "checkpoint": "ce-arktype-refs-docker-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/arktype-json-schema-refs-dependencies", "successful": 2, "title": "ArkType JSON Schema reference dependencies", "valid_coverage": 1.0 }, { "checkpoint": "ce-fastapi-deprecation-docker-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/fastapi-deprecation-response-headers", "successful": 4, "title": "FastAPI deprecation response headers", "valid_coverage": 1.0 }, { "checkpoint": "ce-httpx-multipart-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/httpx-multipart-response-parsing", "successful": 5, "title": "HTTPX multipart response parsing", "valid_coverage": 1.0 }, { "checkpoint": "ce-expr-errors-docker-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/expr-try-catch-errors", "successful": 2, "title": "Expr try/catch errors", "valid_coverage": 1.0 }, { "checkpoint": "ce-python-state-data-docker-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/python-statemachine-state-data-scoping", "successful": 7, "title": "Python StateMachine state-data scoping", "valid_coverage": 1.0 }, { "checkpoint": "ce-katex-spans-docker-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/katex-multicolumn-array-spans", "successful": 1, "title": "KaTeX multicolumn array spans", "valid_coverage": 1.0 }, { "checkpoint": "ce-scc-spilling-docker-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/scc-bounded-memory-spilling", "successful": 0, "title": "SCC bounded-memory spilling", "valid_coverage": 1.0 }, { "checkpoint": "ce-meriyah-resource-docker-v1", "checkpoint_state": "ready", "completed": 12, "expected": 12, "id": "datacurve/meriyah-explicit-resource-declarations", "successful": 1, "title": "Meriyah explicit resource declarations", "valid_coverage": 1.0 } ] }