{
 "label": "ours",
 "model": "Qwen3.8-Flash-Next",
 "concurrency": 4,
 "effort": "xhigh",
 "n": 60,
 "wall_s": 513.023425579071,
 "latency_median_s": 10.809839248657227,
 "latency_mean_s": 27.985220499833424,
 "latency_p90_s": 50.487741470336914,
 "correct": 50,
 "completion_tokens_mean": 930.6,
 "length_stops": 0,
 "items": [
  {
   "id": "mmlupro-3593",
   "suite": "mmlupro",
   "latency_s": 9.357788324356079,
   "ttft_s": 0.3298375606536865,
   "completion_tokens": 262,
   "finish": "stop",
   "reasoning_tokens": 253,
   "correct": true
  },
  {
   "id": "mmlupro-6249",
   "suite": "mmlupro",
   "latency_s": 18.975754976272583,
   "ttft_s": 1.7056357860565186,
   "completion_tokens": 509,
   "finish": "stop",
   "reasoning_tokens": 502,
   "correct": true
  },
  {
   "id": "mmlupro-6182",
   "suite": "mmlupro",
   "latency_s": 13.75492548942566,
   "ttft_s": 1.7056519985198975,
   "completion_tokens": 314,
   "finish": "stop",
   "reasoning_tokens": 307,
   "correct": true
  },
  {
   "id": "mmlupro-4993",
   "suite": "mmlupro",
   "latency_s": 5.647428274154663,
   "ttft_s": 1.705463171005249,
   "completion_tokens": 112,
   "finish": "stop",
   "reasoning_tokens": 105,
   "correct": true
  },
  {
   "id": "mmlupro-12095",
   "suite": "mmlupro",
   "latency_s": 299.95667791366577,
   "ttft_s": 0.2790262699127197,
   "completion_tokens": 9873,
   "finish": "stop",
   "reasoning_tokens": 9866,
   "correct": true
  },
  {
   "id": "mmlupro-2253",
   "suite": "mmlupro",
   "latency_s": 2.247868537902832,
   "ttft_s": 0.24042940139770508,
   "completion_tokens": 88,
   "finish": "stop",
   "reasoning_tokens": 81,
   "correct": true
  },
  {
   "id": "mmlupro-2570",
   "suite": "mmlupro",
   "latency_s": 5.1933465003967285,
   "ttft_s": 0.24005627632141113,
   "completion_tokens": 162,
   "finish": "stop",
   "reasoning_tokens": 155,
   "correct": true
  },
  {
   "id": "mmlupro-2192",
   "suite": "mmlupro",
   "latency_s": 21.93352484703064,
   "ttft_s": 0.2587769031524658,
   "completion_tokens": 661,
   "finish": "stop",
   "reasoning_tokens": 654,
   "correct": true
  },
  {
   "id": "mmlupro-3354",
   "suite": "mmlupro",
   "latency_s": 11.099048852920532,
   "ttft_s": 0.37213587760925293,
   "completion_tokens": 365,
   "finish": "stop",
   "reasoning_tokens": 358,
   "correct": true
  },
  {
   "id": "mmlupro-4643",
   "suite": "mmlupro",
   "latency_s": 5.257594347000122,
   "ttft_s": 0.2896742820739746,
   "completion_tokens": 199,
   "finish": "stop",
   "reasoning_tokens": 192,
   "correct": true
  },
  {
   "id": "mmlupro-10930",
   "suite": "mmlupro",
   "latency_s": 114.83962559700012,
   "ttft_s": 0.2320094108581543,
   "completion_tokens": 3560,
   "finish": "stop",
   "reasoning_tokens": 3553,
   "correct": false
  },
  {
   "id": "mmlupro-10659",
   "suite": "mmlupro",
   "latency_s": 17.373138427734375,
   "ttft_s": 0.2879495620727539,
   "completion_tokens": 531,
   "finish": "stop",
   "reasoning_tokens": 524,
   "correct": true
  },
  {
   "id": "mmlupro-4834",
   "suite": "mmlupro",
   "latency_s": 6.723873138427734,
   "ttft_s": 0.3689579963684082,
   "completion_tokens": 195,
   "finish": "stop",
   "reasoning_tokens": 188,
   "correct": true
  },
  {
   "id": "mmlupro-2618",
   "suite": "mmlupro",
   "latency_s": 29.509873867034912,
   "ttft_s": 0.2722282409667969,
   "completion_tokens": 835,
   "finish": "stop",
   "reasoning_tokens": 828,
   "correct": true
  },
  {
   "id": "mmlupro-5034",
   "suite": "mmlupro",
   "latency_s": 10.719884395599365,
   "ttft_s": 0.24869155883789062,
   "completion_tokens": 373,
   "finish": "stop",
   "reasoning_tokens": 366,
   "correct": true
  },
  {
   "id": "mmlupro-9174",
   "suite": "mmlupro",
   "latency_s": 12.140578508377075,
   "ttft_s": 0.2917022705078125,
   "completion_tokens": 534,
   "finish": "stop",
   "reasoning_tokens": 527,
   "correct": true
  },
  {
   "id": "mmlupro-4212",
   "suite": "mmlupro",
   "latency_s": 4.685399293899536,
   "ttft_s": 0.27164316177368164,
   "completion_tokens": 208,
   "finish": "stop",
   "reasoning_tokens": 201,
   "correct": true
  },
  {
   "id": "mmlupro-2139",
   "suite": "mmlupro",
   "latency_s": 6.630961894989014,
   "ttft_s": 0.282912015914917,
   "completion_tokens": 171,
   "finish": "stop",
   "reasoning_tokens": 164,
   "correct": true
  },
  {
   "id": "mmlupro-7387",
   "suite": "mmlupro",
   "latency_s": 2.487053632736206,
   "ttft_s": 0.2946150302886963,
   "completion_tokens": 80,
   "finish": "stop",
   "reasoning_tokens": 73,
   "correct": true
  },
  {
   "id": "mmlupro-10998",
   "suite": "mmlupro",
   "latency_s": 9.312188386917114,
   "ttft_s": 0.27962279319763184,
   "completion_tokens": 270,
   "finish": "stop",
   "reasoning_tokens": 263,
   "correct": true
  },
  {
   "id": "mmlupro-11035",
   "suite": "mmlupro",
   "latency_s": 10.205484390258789,
   "ttft_s": 0.30455613136291504,
   "completion_tokens": 314,
   "finish": "stop",
   "reasoning_tokens": 307,
   "correct": true
  },
  {
   "id": "mmlupro-1452",
   "suite": "mmlupro",
   "latency_s": 118.27378273010254,
   "ttft_s": 0.35979604721069336,
   "completion_tokens": 3539,
   "finish": "stop",
   "reasoning_tokens": 3532,
   "correct": true
  },
  {
   "id": "mmlupro-9852",
   "suite": "mmlupro",
   "latency_s": 10.161287784576416,
   "ttft_s": 0.28719639778137207,
   "completion_tokens": 350,
   "finish": "stop",
   "reasoning_tokens": 343,
   "correct": true
  },
  {
   "id": "mmlupro-1840",
   "suite": "mmlupro",
   "latency_s": 64.44758200645447,
   "ttft_s": 0.28658318519592285,
   "completion_tokens": 2210,
   "finish": "stop",
   "reasoning_tokens": 2203,
   "correct": true
  },
  {
   "id": "mmlupro-9244",
   "suite": "mmlupro",
   "latency_s": 23.748263359069824,
   "ttft_s": 0.30838704109191895,
   "completion_tokens": 865,
   "finish": "stop",
   "reasoning_tokens": 858,
   "correct": true
  },
  {
   "id": "mmlupro-11200",
   "suite": "mmlupro",
   "latency_s": 50.487741470336914,
   "ttft_s": 0.26276111602783203,
   "completion_tokens": 1415,
   "finish": "stop",
   "reasoning_tokens": 1408,
   "correct": false
  },
  {
   "id": "mmlupro-804",
   "suite": "mmlupro",
   "latency_s": 38.68356275558472,
   "ttft_s": 0.3088235855102539,
   "completion_tokens": 1430,
   "finish": "stop",
   "reasoning_tokens": 1423,
   "correct": true
  },
  {
   "id": "mmlupro-11025",
   "suite": "mmlupro",
   "latency_s": 27.181222677230835,
   "ttft_s": 0.25023913383483887,
   "completion_tokens": 728,
   "finish": "stop",
   "reasoning_tokens": 721,
   "correct": false
  },
  {
   "id": "mmlupro-8937",
   "suite": "mmlupro",
   "latency_s": 22.482237815856934,
   "ttft_s": 0.24799108505249023,
   "completion_tokens": 705,
   "finish": "stop",
   "reasoning_tokens": 698,
   "correct": false
  },
  {
   "id": "mmlupro-8442",
   "suite": "mmlupro",
   "latency_s": 6.478299379348755,
   "ttft_s": 0.3144505023956299,
   "completion_tokens": 212,
   "finish": "stop",
   "reasoning_tokens": 205,
   "correct": true
  },
  {
   "id": "bbh-object_counting-28",
   "suite": "bbh",
   "latency_s": 2.9997751712799072,
   "ttft_s": 0.23450899124145508,
   "completion_tokens": 81,
   "finish": "stop",
   "reasoning_tokens": 74,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-75",
   "suite": "bbh",
   "latency_s": 17.98623490333557,
   "ttft_s": 0.294966459274292,
   "completion_tokens": 486,
   "finish": "stop",
   "reasoning_tokens": 479,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-136",
   "suite": "bbh",
   "latency_s": 10.899794101715088,
   "ttft_s": 0.2825899124145508,
   "completion_tokens": 388,
   "finish": "stop",
   "reasoning_tokens": 381,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-139",
   "suite": "bbh",
   "latency_s": 4.437288284301758,
   "ttft_s": 0.2838568687438965,
   "completion_tokens": 133,
   "finish": "stop",
   "reasoning_tokens": 126,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-207",
   "suite": "bbh",
   "latency_s": 208.79953575134277,
   "ttft_s": 0.21629762649536133,
   "completion_tokens": 6963,
   "finish": "stop",
   "reasoning_tokens": 6952,
   "correct": false
  },
  {
   "id": "bbh-movie_recommendation-26",
   "suite": "bbh",
   "latency_s": 28.171064138412476,
   "ttft_s": 0.2446150779724121,
   "completion_tokens": 809,
   "finish": "stop",
   "reasoning_tokens": 802,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_five_objects-223",
   "suite": "bbh",
   "latency_s": 7.685796737670898,
   "ttft_s": 0.2872893810272217,
   "completion_tokens": 321,
   "finish": "stop",
   "reasoning_tokens": 314,
   "correct": true
  },
  {
   "id": "bbh-reasoning_about_colored_objects-95",
   "suite": "bbh",
   "latency_s": 2.2530198097229004,
   "ttft_s": 0.2489008903503418,
   "completion_tokens": 73,
   "finish": "stop",
   "reasoning_tokens": 66,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-117",
   "suite": "bbh",
   "latency_s": 18.80911135673523,
   "ttft_s": 0.30389928817749023,
   "completion_tokens": 813,
   "finish": "stop",
   "reasoning_tokens": 806,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-67",
   "suite": "bbh",
   "latency_s": 15.800595760345459,
   "ttft_s": 0.29239511489868164,
   "completion_tokens": 600,
   "finish": "stop",
   "reasoning_tokens": 593,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-2",
   "suite": "bbh",
   "latency_s": 33.96745300292969,
   "ttft_s": 0.30127525329589844,
   "completion_tokens": 971,
   "finish": "stop",
   "reasoning_tokens": 964,
   "correct": false
  },
  {
   "id": "bbh-multistep_arithmetic_two-202",
   "suite": "bbh",
   "latency_s": 3.8809762001037598,
   "ttft_s": 0.21029067039489746,
   "completion_tokens": 169,
   "finish": "stop",
   "reasoning_tokens": 161,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-125",
   "suite": "bbh",
   "latency_s": 16.385467290878296,
   "ttft_s": 0.29839277267456055,
   "completion_tokens": 608,
   "finish": "stop",
   "reasoning_tokens": 601,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-9",
   "suite": "bbh",
   "latency_s": 8.146201848983765,
   "ttft_s": 0.284343957901001,
   "completion_tokens": 253,
   "finish": "stop",
   "reasoning_tokens": 246,
   "correct": true
  },
  {
   "id": "bbh-snarks-87",
   "suite": "bbh",
   "latency_s": 8.896967887878418,
   "ttft_s": 0.2690443992614746,
   "completion_tokens": 320,
   "finish": "stop",
   "reasoning_tokens": 313,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-161",
   "suite": "bbh",
   "latency_s": 15.271605014801025,
   "ttft_s": 0.26135873794555664,
   "completion_tokens": 589,
   "finish": "stop",
   "reasoning_tokens": 582,
   "correct": false
  },
  {
   "id": "bbh-ruin_names-76",
   "suite": "bbh",
   "latency_s": 4.101072072982788,
   "ttft_s": 0.24625587463378906,
   "completion_tokens": 116,
   "finish": "stop",
   "reasoning_tokens": 109,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-9",
   "suite": "bbh",
   "latency_s": 11.063869714736938,
   "ttft_s": 0.28249502182006836,
   "completion_tokens": 384,
   "finish": "stop",
   "reasoning_tokens": 377,
   "correct": true
  },
  {
   "id": "bbh-web_of_lies-160",
   "suite": "bbh",
   "latency_s": 4.079145431518555,
   "ttft_s": 0.23072171211242676,
   "completion_tokens": 124,
   "finish": "stop",
   "reasoning_tokens": 117,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-33",
   "suite": "bbh",
   "latency_s": 196.18096089363098,
   "ttft_s": 0.4244847297668457,
   "completion_tokens": 8049,
   "finish": "stop",
   "reasoning_tokens": 8042,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-21",
   "suite": "bbh",
   "latency_s": 7.716946125030518,
   "ttft_s": 0.29221153259277344,
   "completion_tokens": 193,
   "finish": "stop",
   "reasoning_tokens": 186,
   "correct": false
  },
  {
   "id": "bbh-reasoning_about_colored_objects-57",
   "suite": "bbh",
   "latency_s": 4.0277605056762695,
   "ttft_s": 0.27427220344543457,
   "completion_tokens": 166,
   "finish": "stop",
   "reasoning_tokens": 159,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-197",
   "suite": "bbh",
   "latency_s": 13.056562185287476,
   "ttft_s": 0.20557546615600586,
   "completion_tokens": 396,
   "finish": "stop",
   "reasoning_tokens": 387,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-203",
   "suite": "bbh",
   "latency_s": 2.7181153297424316,
   "ttft_s": 0.312119722366333,
   "completion_tokens": 125,
   "finish": "stop",
   "reasoning_tokens": 118,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_three_objects-227",
   "suite": "bbh",
   "latency_s": 2.344696283340454,
   "ttft_s": 0.2503962516784668,
   "completion_tokens": 87,
   "finish": "stop",
   "reasoning_tokens": 80,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-84",
   "suite": "bbh",
   "latency_s": 20.649802207946777,
   "ttft_s": 0.2808253765106201,
   "completion_tokens": 587,
   "finish": "stop",
   "reasoning_tokens": 580,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-201",
   "suite": "bbh",
   "latency_s": 4.8684868812561035,
   "ttft_s": 0.2622220516204834,
   "completion_tokens": 176,
   "finish": "stop",
   "reasoning_tokens": 169,
   "correct": true
  },
  {
   "id": "bbh-penguins_in_a_table-74",
   "suite": "bbh",
   "latency_s": 2.928929328918457,
   "ttft_s": 0.3354003429412842,
   "completion_tokens": 81,
   "finish": "stop",
   "reasoning_tokens": 74,
   "correct": true
  },
  {
   "id": "bbh-disambiguation_qa-168",
   "suite": "bbh",
   "latency_s": 8.422342777252197,
   "ttft_s": 0.24522900581359863,
   "completion_tokens": 255,
   "finish": "stop",
   "reasoning_tokens": 248,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-201",
   "suite": "bbh",
   "latency_s": 12.56765341758728,
   "ttft_s": 0.2870299816131592,
   "completion_tokens": 450,
   "finish": "stop",
   "reasoning_tokens": 443,
   "correct": true
  }
 ]
}