{
  "baseline_thinking": {
    "accuracy_available": false,
    "status": "TIMEOUT_CENSORED",
    "timeout_seconds": 7200
  },
  "best": {
    "best_accuracy": "evolved_instruct",
    "best_overall": "evolved_instruct",
    "best_overall_eligible": [
      "manual_instruct",
      "evolved_instruct"
    ],
    "best_speed": "evolved_instruct",
    "best_speed_eligible": [
      "manual_instruct",
      "evolved_instruct"
    ],
    "best_speed_floor_correct": 74,
    "overall_scores": {
      "evolved_instruct": 1.0,
      "manual_instruct": 0.9535093822476024
    }
  },
  "campaign_id": "d3a319ae-cdb0-48d4-860c-70f44585e15c",
  "campaign_summary_sha256": "fdead42e781094afbe4f3800126839e2c896ec2dc1697cb38c1f7dea1022703d",
  "metrics": {
    "baseline_instruct": {
      "dev128": {
        "accuracy": 0.3984375,
        "correct_count": 51,
        "error_count": 0,
        "generated_token_total": 5208,
        "peak_memory_bytes": 4532016640,
        "sample_count": 128,
        "termination_counts": {
          "eos": 107,
          "length": 21,
          "other": 0,
          "stop_text": 0
        }
      },
      "source_sha256": "5d8e635e28ea5cc6c1458daecbc6c5aa80ab84ad1ce1ad021f9a2c4a2775a8d4",
      "speed32": {
        "generated_token_median_sum": 1342,
        "mean_seconds": 13.551426325750072,
        "measured_seconds": 433.6456424240023,
        "median_seconds": 14.541101270355284,
        "per_query_median_latency_seconds": [
          9.811798666603863,
          9.642910950817168,
          5.399396408349276,
          23.732810150831938,
          11.229596567340195,
          14.640659658238292,
          14.628536286763847,
          18.449916650541127,
          15.660303776152432,
          20.523028102703393,
          14.453666253946722,
          16.335607569664717,
          18.098674747161567,
          3.5791870718821883,
          12.418502423912287,
          9.64294655341655,
          8.936336943879724,
          14.78745496738702,
          17.755739107728004,
          13.57640447281301,
          9.624033161439002,
          6.087712108157575,
          15.896250516176224,
          11.822727962397039,
          15.313639609143138,
          14.729376120492816,
          17.686118968762457,
          9.458174258470535,
          13.656660032458603,
          16.674703435972333,
          10.315106925554574,
          19.077661994844675
        ],
        "query_order": [
          "239_3",
          "1125_0",
          "284_1",
          "677_0",
          "80_3",
          "1520_3",
          "1219_1",
          "2114_2",
          "1980_1",
          "172_2",
          "1917_1",
          "1297_1",
          "757_1",
          "2242_3",
          "2157_2",
          "132_1",
          "1916_3",
          "2068_3",
          "1284_1",
          "1791_1",
          "177_1",
          "198_2",
          "1969_0",
          "1568_3",
          "1431_0",
          "1948_2",
          "938_1",
          "1590_0",
          "87_3",
          "2347_3",
          "1603_3",
          "1258_3"
        ],
        "raw_repeat_latency_seconds": [
          [
            9.811798666603863,
            9.642910950817168,
            5.399396408349276,
            23.732810150831938,
            11.229596567340195,
            14.640659658238292,
            14.628536286763847,
            18.37813539430499,
            15.660303776152432,
            20.523028102703393,
            14.369377811439335,
            16.31413401849568,
            18.078556162305176,
            3.5632938109338284,
            12.418502423912287,
            9.64294655341655,
            8.936336943879724,
            14.78745496738702,
            17.755739107728004,
            13.57640447281301,
            9.601136713288724,
            6.077843279577792,
            15.896250516176224,
            10.94260501023382,
            15.112714109010994,
            14.707969612441957,
            17.686118968762457,
            9.458174258470535,
            13.656660032458603,
            16.674703435972333,
            10.272035975940526,
            19.170377059839666
          ],
          [
            11.072747349739075,
            10.85622662678361,
            6.195115533657372,
            27.368020536378026,
            12.980258188210428,
            16.872188417240977,
            16.85369807574898,
            21.408132875338197,
            19.01740226801485,
            24.021293916739523,
            16.884825739078224,
            19.00000993255526,
            21.039189682342112,
            4.207569061778486,
            14.439404242672026,
            11.325721935369074,
            10.484268729574978,
            17.37110636383295,
            20.777446530759335,
            15.885303881950676,
            11.27276073768735,
            7.1253483928740025,
            18.51826520357281,
            12.740548395551741,
            17.62681207060814,
            17.209459764882922,
            20.48988602682948,
            10.842055678367615,
            15.921122456900775,
            19.376215413212776,
            10.601993216201663,
            19.00543038547039
          ],
          [
            9.58250224776566,
            9.529770448803902,
            5.336649868637323,
            23.308916540816426,
            11.035382622852921,
            14.347471903078258,
            14.377965544350445,
            18.449916650541127,
            15.659753357991576,
            20.516560222022235,
            14.453666253946722,
            16.335607569664717,
            18.098674747161567,
            3.5791870718821883,
            12.387332094833255,
            9.577226167544723,
            8.919831371866167,
            14.721310530789196,
            17.691233068704605,
            13.534616119228303,
            9.624033161439002,
            6.087712108157575,
            15.855095093138516,
            11.822727962397039,
            15.313639609143138,
            14.729376120492816,
            17.500899174250662,
            9.440136919729412,
            13.626757287420332,
            16.590522079728544,
            10.315106925554574,
            19.077661994844675
          ]
        ],
        "repeat_count": 3,
        "sample_count": 32,
        "throughput_qps": 0.07379297027205363,
        "type7_p95_seconds": 19.728076743381095
      }
    },
    "evolved_instruct": {
      "dev128": {
        "accuracy": 0.609375,
        "correct_count": 78,
        "error_count": 0,
        "generated_token_total": 751,
        "peak_memory_bytes": 4536605696,
        "sample_count": 128,
        "termination_counts": {
          "eos": 112,
          "length": 0,
          "other": 0,
          "stop_text": 16
        }
      },
      "source_sha256": "1c58d489733b7031c7bed3d7a27a9840cf364a9c916cd6eab54febd4b43bb984",
      "speed32": {
        "generated_token_median_sum": 179,
        "mean_seconds": 12.977740366506623,
        "measured_seconds": 415.28769172821194,
        "median_seconds": 14.07175780273974,
        "per_query_median_latency_seconds": [
          9.62473756633699,
          7.614347465336323,
          4.929239496588707,
          21.925082660280168,
          10.997477670200169,
          14.535657216794789,
          14.418939762748778,
          16.407450147904456,
          13.716786287724972,
          20.427526294253767,
          14.500682804733515,
          14.554440731182694,
          16.518629810772836,
          3.704547240398824,
          12.251966219395399,
          9.830246601253748,
          9.15820427890867,
          14.869286972098053,
          17.88465595524758,
          11.787039310671389,
          9.685279536992311,
          5.697239729575813,
          15.952160290442407,
          11.530807896517217,
          15.106143085286021,
          14.452724640257657,
          16.34523467347026,
          7.66655723657459,
          13.724575842730701,
          15.860466600395739,
          10.400212439708412,
          19.209345263428986
        ],
        "query_order": [
          "239_3",
          "1125_0",
          "284_1",
          "677_0",
          "80_3",
          "1520_3",
          "1219_1",
          "2114_2",
          "1980_1",
          "172_2",
          "1917_1",
          "1297_1",
          "757_1",
          "2242_3",
          "2157_2",
          "132_1",
          "1916_3",
          "2068_3",
          "1284_1",
          "1791_1",
          "177_1",
          "198_2",
          "1969_0",
          "1568_3",
          "1431_0",
          "1948_2",
          "938_1",
          "1590_0",
          "87_3",
          "2347_3",
          "1603_3",
          "1258_3"
        ],
        "raw_repeat_latency_seconds": [
          [
            10.30484311003238,
            8.127470790408552,
            5.324782622978091,
            23.11075219977647,
            11.58316557109356,
            15.124506236054003,
            15.213710094802082,
            17.288092576898634,
            14.441375007852912,
            21.56999512668699,
            15.242161331698298,
            15.17552145011723,
            17.307603468187153,
            3.704547240398824,
            13.01076882891357,
            10.215342063456774,
            9.51527607254684,
            15.642574020661414,
            18.746319085359573,
            12.423084600828588,
            10.16782601363957,
            5.86988093983382,
            16.663616148754954,
            11.530807896517217,
            15.96524557378143,
            15.218855779618025,
            17.318009039387107,
            8.078675219789147,
            14.435826822184026,
            16.538497323170304,
            10.880924657918513,
            20.147350925952196
          ],
          [
            9.62473756633699,
            7.614347465336323,
            4.929239496588707,
            21.925082660280168,
            10.964275897480547,
            14.535657216794789,
            14.418939762748778,
            16.407450147904456,
            13.716786287724972,
            20.427526294253767,
            14.500682804733515,
            14.554440731182694,
            16.518629810772836,
            3.7512039560824633,
            12.251966219395399,
            9.55359436571598,
            9.15820427890867,
            14.869286972098053,
            17.88465595524758,
            11.752576389349997,
            9.657554681412876,
            5.697239729575813,
            15.952160290442407,
            12.79553752578795,
            15.106143085286021,
            14.415294707752764,
            16.272409172728658,
            7.635159961879253,
            13.724575842730701,
            15.860466600395739,
            10.400212439708412,
            19.074668526649475
          ],
          [
            9.563080594874918,
            7.58717483188957,
            4.86398474033922,
            21.838871216401458,
            10.997477670200169,
            14.306238180026412,
            14.323268520645797,
            16.30247941799462,
            13.553551989607513,
            20.391460185870528,
            14.419367972761393,
            14.333276161924005,
            16.514796689152718,
            3.491271219216287,
            12.2451709844172,
            9.830246601253748,
            9.028629048727453,
            14.823167041875422,
            17.773846226744354,
            11.787039310671389,
            9.685279536992311,
            5.565341852605343,
            15.832013672217727,
            10.950116964057088,
            15.100894493050873,
            14.452724640257657,
            16.34523467347026,
            7.66655723657459,
            13.68978865351528,
            15.630811460316181,
            10.251256939955056,
            19.209345263428986
          ]
        ],
        "repeat_count": 3,
        "sample_count": 32,
        "throughput_qps": 0.07705501664841691,
        "type7_p95_seconds": 19.757526727300135
      }
    },
    "evolved_thinking": {
      "dev128": {
        "accuracy": 0.34375,
        "correct_count": 44,
        "error_count": 0,
        "generated_token_total": 33193,
        "peak_memory_bytes": 4542966272,
        "sample_count": 128,
        "termination_counts": {
          "eos": 29,
          "length": 48,
          "other": 0,
          "stop_text": 51
        }
      },
      "source_sha256": "81bedc0e16a3954bc55aea73120ed47f661534a79f01ff9dfd66069c2af1efe6",
      "speed32": {
        "generated_token_median_sum": 9838,
        "mean_seconds": 17.793894787551835,
        "measured_seconds": 569.4046332016587,
        "median_seconds": 17.91290564322844,
        "per_query_median_latency_seconds": [
          15.635723488405347,
          13.466084221377969,
          10.874625414609909,
          28.11545221786946,
          17.01019410043955,
          20.494141128845513,
          17.90921923238784,
          19.89456288330257,
          17.916592054069042,
          25.822390402667224,
          17.48471179790795,
          20.497177880257368,
          21.60422188602388,
          9.607972905039787,
          14.228417340666056,
          10.897624723613262,
          14.942204627208412,
          20.43934498075396,
          24.01995027065277,
          17.85078425332904,
          10.614264174364507,
          6.796770879067481,
          20.561999822035432,
          17.06778014358133,
          21.153475332073867,
          20.269150022417307,
          19.429727006703615,
          13.63214449211955,
          19.647059437818825,
          17.99544072896242,
          16.200322133488953,
          27.32510321959853
        ],
        "query_order": [
          "239_3",
          "1125_0",
          "284_1",
          "677_0",
          "80_3",
          "1520_3",
          "1219_1",
          "2114_2",
          "1980_1",
          "172_2",
          "1917_1",
          "1297_1",
          "757_1",
          "2242_3",
          "2157_2",
          "132_1",
          "1916_3",
          "2068_3",
          "1284_1",
          "1791_1",
          "177_1",
          "198_2",
          "1969_0",
          "1568_3",
          "1431_0",
          "1948_2",
          "938_1",
          "1590_0",
          "87_3",
          "2347_3",
          "1603_3",
          "1258_3"
        ],
        "raw_repeat_latency_seconds": [
          [
            15.589340995065868,
            13.466084221377969,
            10.84695007931441,
            27.908937955275178,
            16.95971847511828,
            20.33989812526852,
            17.76901018153876,
            19.871940076351166,
            17.916592054069042,
            25.75347469933331,
            17.127262038178742,
            20.17494591884315,
            21.312706938944757,
            9.607972905039787,
            14.20374187733978,
            10.897624723613262,
            14.894049373455346,
            20.37271001841873,
            23.566809315234423,
            17.441502899862826,
            10.433977823704481,
            6.648517062887549,
            20.300575265660882,
            16.829422295093536,
            20.964644922874868,
            20.269150022417307,
            19.429727006703615,
            13.63214449211955,
            19.647059437818825,
            17.99544072896242,
            16.200322133488953,
            27.32510321959853
          ],
          [
            17.01776162162423,
            14.777252751402557,
            11.809890887700021,
            30.316146915778518,
            18.751734290271997,
            22.43234993238002,
            19.586908159777522,
            21.948041920550168,
            19.613497643731534,
            28.37006969191134,
            19.27584060654044,
            22.37518428172916,
            23.666155808605254,
            10.24969999678433,
            15.691981963813305,
            11.976870506070554,
            16.273931628093123,
            22.567399743013084,
            26.29355358146131,
            19.37162490002811,
            11.692175278440118,
            7.459338312968612,
            22.687949155457318,
            18.66301118861884,
            22.865966520272195,
            22.086720778606832,
            21.269862404093146,
            14.595215324312449,
            21.42370004672557,
            19.720671094954014,
            17.647994784638286,
            27.34389982651919
          ],
          [
            15.635723488405347,
            13.349520123563707,
            10.874625414609909,
            28.11545221786946,
            17.01019410043955,
            20.494141128845513,
            17.90921923238784,
            19.89456288330257,
            17.91639099922031,
            25.822390402667224,
            17.48471179790795,
            20.497177880257368,
            21.60422188602388,
            9.495414514094591,
            14.228417340666056,
            10.851077345199883,
            14.942204627208412,
            20.43934498075396,
            24.01995027065277,
            17.85078425332904,
            10.614264174364507,
            6.796770879067481,
            20.561999822035432,
            17.06778014358133,
            21.153475332073867,
            20.212607517838478,
            19.318582919426262,
            13.378388334996998,
            19.435208396986127,
            17.84760631341487,
            16.19474435877055,
            24.857253678143024
          ]
        ],
        "repeat_count": 3,
        "sample_count": 32,
        "throughput_qps": 0.056199050963231224,
        "type7_p95_seconds": 26.49861117028631
      }
    },
    "manual_instruct": {
      "dev128": {
        "accuracy": 0.609375,
        "correct_count": 78,
        "error_count": 0,
        "generated_token_total": 751,
        "peak_memory_bytes": 4536605696,
        "sample_count": 128,
        "termination_counts": {
          "eos": 112,
          "length": 0,
          "other": 0,
          "stop_text": 16
        }
      },
      "source_sha256": "397f3a92d21f2dac64e5bf012ccdeca6e2c48fcfed54c11985268df48cf516f4",
      "speed32": {
        "generated_token_median_sum": 179,
        "mean_seconds": 14.184930760849966,
        "measured_seconds": 453.9177843471989,
        "median_seconds": 15.443961842451245,
        "per_query_median_latency_seconds": [
          10.605580841191113,
          8.405649589374661,
          5.373762308619916,
          24.134168510325253,
          12.171308802440763,
          15.81790214125067,
          16.021335100755095,
          17.968898658640683,
          15.12461050786078,
          22.537259954959154,
          15.886884447187185,
          16.039522987790406,
          17.9351475359872,
          3.9404284628108144,
          13.522393506020308,
          10.62656707316637,
          9.826638229191303,
          16.40696391183883,
          19.508635103702545,
          12.871043575927615,
          10.654018803499639,
          6.182953059673309,
          17.4683417994529,
          11.96673272550106,
          16.61823496874422,
          15.76331317704171,
          17.97501477971673,
          8.383753524161875,
          14.88924801722169,
          17.2222471954301,
          11.246530828997493,
          20.822694218717515
        ],
        "query_order": [
          "239_3",
          "1125_0",
          "284_1",
          "677_0",
          "80_3",
          "1520_3",
          "1219_1",
          "2114_2",
          "1980_1",
          "172_2",
          "1917_1",
          "1297_1",
          "757_1",
          "2242_3",
          "2157_2",
          "132_1",
          "1916_3",
          "2068_3",
          "1284_1",
          "1791_1",
          "177_1",
          "198_2",
          "1969_0",
          "1568_3",
          "1431_0",
          "1948_2",
          "938_1",
          "1590_0",
          "87_3",
          "2347_3",
          "1603_3",
          "1258_3"
        ],
        "raw_repeat_latency_seconds": [
          [
            9.790541868656874,
            7.724491393193603,
            4.941454503685236,
            22.213915781117976,
            11.161740264855325,
            14.618881460279226,
            14.608545129187405,
            16.702226854860783,
            13.938558054156601,
            20.82125368900597,
            14.709829521365464,
            14.607883961871266,
            16.641846834681928,
            3.5928828604519367,
            12.549097226001322,
            9.760864738374949,
            9.157317200675607,
            15.031248186714947,
            18.08828063122928,
            11.871278449892998,
            9.831501476466656,
            5.639676366001368,
            16.155637663789093,
            11.166579500772059,
            15.326375089585781,
            14.653488026931882,
            16.6708904504776,
            7.743043093942106,
            13.897030101157725,
            15.949868047609925,
            10.494213237427175,
            19.127783357165754
          ],
          [
            10.876715525053442,
            8.64229141548276,
            5.4809742933139205,
            24.542173620313406,
            12.325026327744126,
            16.16027377638966,
            16.18284252844751,
            18.431468785740435,
            15.533418802544475,
            22.980133735574782,
            16.310640000738204,
            16.11651910841465,
            18.515810628421605,
            3.9404284628108144,
            13.833396084606647,
            10.821296099573374,
            10.249783464707434,
            16.672664631158113,
            19.86048313602805,
            13.183068628422916,
            11.010747840628028,
            6.235659845173359,
            17.79542497638613,
            12.26636263076216,
            17.065976103767753,
            16.223237859085202,
            18.36694052722305,
            8.64770536776632,
            15.521075623109937,
            17.66126629896462,
            11.532877957448363,
            21.535955352708697
          ],
          [
            10.605580841191113,
            8.405649589374661,
            5.373762308619916,
            24.134168510325253,
            12.171308802440763,
            15.81790214125067,
            16.021335100755095,
            17.968898658640683,
            15.12461050786078,
            22.537259954959154,
            15.886884447187185,
            16.039522987790406,
            17.9351475359872,
            3.942544324323535,
            13.522393506020308,
            10.62656707316637,
            9.826638229191303,
            16.40696391183883,
            19.508635103702545,
            12.871043575927615,
            10.654018803499639,
            6.182953059673309,
            17.4683417994529,
            11.96673272550106,
            16.61823496874422,
            15.76331317704171,
            17.97501477971673,
            8.383753524161875,
            14.88924801722169,
            17.2222471954301,
            11.246530828997493,
            20.822694218717515
          ]
        ],
        "repeat_count": 3,
        "sample_count": 32,
        "throughput_qps": 0.07049734798565063,
        "type7_p95_seconds": 21.59424880002625
      }
    },
    "manual_thinking": {
      "dev128": {
        "accuracy": 0.1796875,
        "correct_count": 23,
        "error_count": 0,
        "generated_token_total": 33541,
        "peak_memory_bytes": 4536763392,
        "sample_count": 128,
        "termination_counts": {
          "eos": 50,
          "length": 51,
          "other": 0,
          "stop_text": 27
        }
      },
      "source_sha256": "60db13925030b2e346f363ab4a96eb6317641e0e964e4dd7519f7b2f4073fb58",
      "speed32": {
        "generated_token_median_sum": 9710,
        "mean_seconds": 19.01968845783267,
        "measured_seconds": 608.6300306506455,
        "median_seconds": 20.043466536793858,
        "per_query_median_latency_seconds": [
          16.77445063740015,
          11.794437852688134,
          11.352561135776341,
          29.41381785646081,
          17.830957776866853,
          21.542682067491114,
          17.815662840381265,
          21.97480741608888,
          20.75370982196182,
          28.043769251555204,
          19.333223251625896,
          21.489438670687377,
          22.274653734639287,
          10.372444045729935,
          16.628006529062986,
          11.793475741520524,
          11.432049002498388,
          22.546114400029182,
          25.908816976472735,
          19.191308059729636,
          11.905072974972427,
          7.662332077510655,
          23.52682911232114,
          18.40597383119166,
          23.308025821112096,
          22.081589022651315,
          21.39368369989097,
          14.624703499488533,
          21.575447275303304,
          24.07742726802826,
          14.156458779238164,
          27.646100220270455
        ],
        "query_order": [
          "239_3",
          "1125_0",
          "284_1",
          "677_0",
          "80_3",
          "1520_3",
          "1219_1",
          "2114_2",
          "1980_1",
          "172_2",
          "1917_1",
          "1297_1",
          "757_1",
          "2242_3",
          "2157_2",
          "132_1",
          "1916_3",
          "2068_3",
          "1284_1",
          "1791_1",
          "177_1",
          "198_2",
          "1969_0",
          "1568_3",
          "1431_0",
          "1948_2",
          "938_1",
          "1590_0",
          "87_3",
          "2347_3",
          "1603_3",
          "1258_3"
        ],
        "raw_repeat_latency_seconds": [
          [
            17.599763449281454,
            11.794437852688134,
            11.352561135776341,
            29.41381785646081,
            17.830957776866853,
            21.542682067491114,
            17.815662840381265,
            21.97480741608888,
            20.75370982196182,
            28.043769251555204,
            19.333223251625896,
            21.489438670687377,
            22.274653734639287,
            10.014518701471388,
            16.001526064239442,
            11.376851649023592,
            10.983149662613869,
            21.723697301000357,
            25.397520792670548,
            18.793169604614377,
            11.653080253861845,
            7.321266079321504,
            23.10664953198284,
            17.9334805496037,
            22.5551772210747,
            21.712907092645764,
            20.78987394552678,
            14.265778915025294,
            20.809926484711468,
            23.095628869719803,
            13.564317282289267,
            26.70730825792998
          ],
          [
            16.77445063740015,
            12.02466796990484,
            11.675225312821567,
            30.119937692768872,
            18.363632074557245,
            23.04343843087554,
            18.42773634660989,
            22.569785149767995,
            21.383613160811365,
            28.776991517283022,
            20.111793098039925,
            22.58245179988444,
            23.19333293940872,
            10.372444045729935,
            16.628006529062986,
            11.793475741520524,
            11.432049002498388,
            22.546114400029182,
            25.908816976472735,
            19.191308059729636,
            11.905072974972427,
            7.662332077510655,
            23.52682911232114,
            18.40597383119166,
            23.308025821112096,
            22.081589022651315,
            21.39368369989097,
            14.624703499488533,
            21.575447275303304,
            24.07742726802826,
            14.156458779238164,
            27.646100220270455
          ],
          [
            15.739309794269502,
            11.179099679924548,
            10.85173340793699,
            28.13498918991536,
            17.01568598486483,
            20.439582336694002,
            16.827887408435345,
            20.465888602659106,
            19.542070508003235,
            26.298393272794783,
            18.041693474166095,
            20.035276119597256,
            21.687716651707888,
            10.570032180286944,
            17.010666685178876,
            12.190346976742148,
            11.744728482328355,
            23.077415952458978,
            27.182175231166184,
            19.98620695155114,
            12.437892743386328,
            7.824792293831706,
            24.809259111993015,
            19.199496652930975,
            23.95322227012366,
            23.078301336616278,
            22.21738401427865,
            15.229610540904105,
            22.511173410341144,
            24.771146366372705,
            14.623882615938783,
            28.651667513884604
          ]
        ],
        "repeat_count": 3,
        "sample_count": 32,
        "throughput_qps": 0.05257709673936225,
        "type7_p95_seconds": 27.825051284348593
      }
    }
  },
  "protocol_id": "phase3-recovery-v2",
  "trace_version": "phase3-recovery-report-number-trace-v2"
}