{
 "label": "stock",
 "model": "Qwen3.8-Flash-Next",
 "concurrency": 1,
 "effort": "xhigh",
 "n": 60,
 "wall_s": 3939.7396392822266,
 "latency_median_s": 8.315284848213196,
 "latency_mean_s": 65.66219737529755,
 "latency_p90_s": 157.46970963478088,
 "correct": 50,
 "completion_tokens_mean": 3227.4666666666667,
 "length_stops": 0,
 "items": [
  {
   "id": "mmlupro-3593",
   "suite": "mmlupro",
   "latency_s": 7.978620529174805,
   "ttft_s": 1.0622375011444092,
   "completion_tokens": 325,
   "finish": "stop",
   "reasoning_tokens": 316,
   "correct": true
  },
  {
   "id": "mmlupro-6249",
   "suite": "mmlupro",
   "latency_s": 69.9693751335144,
   "ttft_s": 1.433650016784668,
   "completion_tokens": 3064,
   "finish": "stop",
   "reasoning_tokens": 3057,
   "correct": true
  },
  {
   "id": "mmlupro-6182",
   "suite": "mmlupro",
   "latency_s": 23.836480140686035,
   "ttft_s": 1.7013301849365234,
   "completion_tokens": 920,
   "finish": "stop",
   "reasoning_tokens": 913,
   "correct": true
  },
  {
   "id": "mmlupro-4993",
   "suite": "mmlupro",
   "latency_s": 5.701629161834717,
   "ttft_s": 1.2715604305267334,
   "completion_tokens": 189,
   "finish": "stop",
   "reasoning_tokens": 182,
   "correct": true
  },
  {
   "id": "mmlupro-12095",
   "suite": "mmlupro",
   "latency_s": 720.8065445423126,
   "ttft_s": 0.7451081275939941,
   "completion_tokens": 36716,
   "finish": "stop",
   "reasoning_tokens": 36709,
   "correct": true
  },
  {
   "id": "mmlupro-2253",
   "suite": "mmlupro",
   "latency_s": 2.1417577266693115,
   "ttft_s": 0.33441615104675293,
   "completion_tokens": 98,
   "finish": "stop",
   "reasoning_tokens": 91,
   "correct": true
  },
  {
   "id": "mmlupro-2570",
   "suite": "mmlupro",
   "latency_s": 4.894963979721069,
   "ttft_s": 0.3914468288421631,
   "completion_tokens": 233,
   "finish": "stop",
   "reasoning_tokens": 226,
   "correct": true
  },
  {
   "id": "mmlupro-2192",
   "suite": "mmlupro",
   "latency_s": 136.19433975219727,
   "ttft_s": 0.4541795253753662,
   "completion_tokens": 6324,
   "finish": "stop",
   "reasoning_tokens": 6317,
   "correct": true
  },
  {
   "id": "mmlupro-3354",
   "suite": "mmlupro",
   "latency_s": 9.871742725372314,
   "ttft_s": 0.7337350845336914,
   "completion_tokens": 491,
   "finish": "stop",
   "reasoning_tokens": 484,
   "correct": true
  },
  {
   "id": "mmlupro-4643",
   "suite": "mmlupro",
   "latency_s": 3.879021406173706,
   "ttft_s": 0.37209010124206543,
   "completion_tokens": 197,
   "finish": "stop",
   "reasoning_tokens": 190,
   "correct": true
  },
  {
   "id": "mmlupro-10930",
   "suite": "mmlupro",
   "latency_s": 19.543538093566895,
   "ttft_s": 0.26308298110961914,
   "completion_tokens": 972,
   "finish": "stop",
   "reasoning_tokens": 965,
   "correct": false
  },
  {
   "id": "mmlupro-10659",
   "suite": "mmlupro",
   "latency_s": 11.739620685577393,
   "ttft_s": 0.5129568576812744,
   "completion_tokens": 588,
   "finish": "stop",
   "reasoning_tokens": 581,
   "correct": true
  },
  {
   "id": "mmlupro-4834",
   "suite": "mmlupro",
   "latency_s": 10.795685291290283,
   "ttft_s": 1.1014127731323242,
   "completion_tokens": 458,
   "finish": "stop",
   "reasoning_tokens": 451,
   "correct": true
  },
  {
   "id": "mmlupro-2618",
   "suite": "mmlupro",
   "latency_s": 104.16047787666321,
   "ttft_s": 0.5296354293823242,
   "completion_tokens": 5080,
   "finish": "stop",
   "reasoning_tokens": 5073,
   "correct": true
  },
  {
   "id": "mmlupro-5034",
   "suite": "mmlupro",
   "latency_s": 5.86430811882019,
   "ttft_s": 0.31945085525512695,
   "completion_tokens": 309,
   "finish": "stop",
   "reasoning_tokens": 302,
   "correct": true
  },
  {
   "id": "mmlupro-9174",
   "suite": "mmlupro",
   "latency_s": 8.421902418136597,
   "ttft_s": 0.3600895404815674,
   "completion_tokens": 543,
   "finish": "stop",
   "reasoning_tokens": 536,
   "correct": true
  },
  {
   "id": "mmlupro-4212",
   "suite": "mmlupro",
   "latency_s": 4.726139068603516,
   "ttft_s": 0.30332279205322266,
   "completion_tokens": 319,
   "finish": "stop",
   "reasoning_tokens": 312,
   "correct": true
  },
  {
   "id": "mmlupro-2139",
   "suite": "mmlupro",
   "latency_s": 5.076724529266357,
   "ttft_s": 0.44997429847717285,
   "completion_tokens": 182,
   "finish": "stop",
   "reasoning_tokens": 175,
   "correct": true
  },
  {
   "id": "mmlupro-7387",
   "suite": "mmlupro",
   "latency_s": 1.6677882671356201,
   "ttft_s": 0.4060664176940918,
   "completion_tokens": 69,
   "finish": "stop",
   "reasoning_tokens": 62,
   "correct": true
  },
  {
   "id": "mmlupro-10998",
   "suite": "mmlupro",
   "latency_s": 7.229863405227661,
   "ttft_s": 0.45441675186157227,
   "completion_tokens": 360,
   "finish": "stop",
   "reasoning_tokens": 353,
   "correct": true
  },
  {
   "id": "mmlupro-11035",
   "suite": "mmlupro",
   "latency_s": 8.208667278289795,
   "ttft_s": 0.5183944702148438,
   "completion_tokens": 353,
   "finish": "stop",
   "reasoning_tokens": 346,
   "correct": true
  },
  {
   "id": "mmlupro-1452",
   "suite": "mmlupro",
   "latency_s": 314.6976947784424,
   "ttft_s": 0.8671646118164062,
   "completion_tokens": 14796,
   "finish": "stop",
   "reasoning_tokens": 14789,
   "correct": true
  },
  {
   "id": "mmlupro-9852",
   "suite": "mmlupro",
   "latency_s": 8.807569742202759,
   "ttft_s": 0.32941412925720215,
   "completion_tokens": 467,
   "finish": "stop",
   "reasoning_tokens": 460,
   "correct": true
  },
  {
   "id": "mmlupro-1840",
   "suite": "mmlupro",
   "latency_s": 476.6919982433319,
   "ttft_s": 0.46718311309814453,
   "completion_tokens": 23118,
   "finish": "stop",
   "reasoning_tokens": 23111,
   "correct": true
  },
  {
   "id": "mmlupro-9244",
   "suite": "mmlupro",
   "latency_s": 11.809073448181152,
   "ttft_s": 0.3530139923095703,
   "completion_tokens": 640,
   "finish": "stop",
   "reasoning_tokens": 633,
   "correct": true
  },
  {
   "id": "mmlupro-11200",
   "suite": "mmlupro",
   "latency_s": 11.168717861175537,
   "ttft_s": 0.25267934799194336,
   "completion_tokens": 497,
   "finish": "stop",
   "reasoning_tokens": 490,
   "correct": false
  },
  {
   "id": "mmlupro-804",
   "suite": "mmlupro",
   "latency_s": 24.397027254104614,
   "ttft_s": 0.33543920516967773,
   "completion_tokens": 1323,
   "finish": "stop",
   "reasoning_tokens": 1316,
   "correct": true
  },
  {
   "id": "mmlupro-11025",
   "suite": "mmlupro",
   "latency_s": 38.20766592025757,
   "ttft_s": 0.25737524032592773,
   "completion_tokens": 1739,
   "finish": "stop",
   "reasoning_tokens": 1732,
   "correct": true
  },
  {
   "id": "mmlupro-8937",
   "suite": "mmlupro",
   "latency_s": 150.94404530525208,
   "ttft_s": 0.23380565643310547,
   "completion_tokens": 7886,
   "finish": "stop",
   "reasoning_tokens": 7879,
   "correct": false
  },
  {
   "id": "mmlupro-8442",
   "suite": "mmlupro",
   "latency_s": 4.512511730194092,
   "ttft_s": 0.45368027687072754,
   "completion_tokens": 213,
   "finish": "stop",
   "reasoning_tokens": 206,
   "correct": true
  },
  {
   "id": "bbh-object_counting-28",
   "suite": "bbh",
   "latency_s": 2.843022346496582,
   "ttft_s": 0.2155468463897705,
   "completion_tokens": 133,
   "finish": "stop",
   "reasoning_tokens": 126,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-75",
   "suite": "bbh",
   "latency_s": 16.850605487823486,
   "ttft_s": 0.5008041858673096,
   "completion_tokens": 823,
   "finish": "stop",
   "reasoning_tokens": 816,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-136",
   "suite": "bbh",
   "latency_s": 5.084399461746216,
   "ttft_s": 0.560823917388916,
   "completion_tokens": 267,
   "finish": "stop",
   "reasoning_tokens": 260,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-139",
   "suite": "bbh",
   "latency_s": 4.980353832244873,
   "ttft_s": 0.2774770259857178,
   "completion_tokens": 225,
   "finish": "stop",
   "reasoning_tokens": 218,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-207",
   "suite": "bbh",
   "latency_s": 197.5942645072937,
   "ttft_s": 0.2016913890838623,
   "completion_tokens": 9678,
   "finish": "stop",
   "reasoning_tokens": 9670,
   "correct": false
  },
  {
   "id": "bbh-movie_recommendation-26",
   "suite": "bbh",
   "latency_s": 25.902090311050415,
   "ttft_s": 0.22563409805297852,
   "completion_tokens": 1359,
   "finish": "stop",
   "reasoning_tokens": 1352,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_five_objects-223",
   "suite": "bbh",
   "latency_s": 4.862995624542236,
   "ttft_s": 0.28118133544921875,
   "completion_tokens": 295,
   "finish": "stop",
   "reasoning_tokens": 288,
   "correct": true
  },
  {
   "id": "bbh-reasoning_about_colored_objects-95",
   "suite": "bbh",
   "latency_s": 2.206169366836548,
   "ttft_s": 0.2512705326080322,
   "completion_tokens": 114,
   "finish": "stop",
   "reasoning_tokens": 107,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-117",
   "suite": "bbh",
   "latency_s": 157.46970963478088,
   "ttft_s": 0.321821928024292,
   "completion_tokens": 9629,
   "finish": "stop",
   "reasoning_tokens": 9622,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-67",
   "suite": "bbh",
   "latency_s": 12.692587852478027,
   "ttft_s": 0.5081820487976074,
   "completion_tokens": 671,
   "finish": "stop",
   "reasoning_tokens": 664,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-2",
   "suite": "bbh",
   "latency_s": 121.2611517906189,
   "ttft_s": 0.3921382427215576,
   "completion_tokens": 6098,
   "finish": "stop",
   "reasoning_tokens": 6091,
   "correct": false
  },
  {
   "id": "bbh-multistep_arithmetic_two-202",
   "suite": "bbh",
   "latency_s": 2.5523717403411865,
   "ttft_s": 0.18918275833129883,
   "completion_tokens": 169,
   "finish": "stop",
   "reasoning_tokens": 161,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-125",
   "suite": "bbh",
   "latency_s": 11.463846921920776,
   "ttft_s": 0.3391530513763428,
   "completion_tokens": 631,
   "finish": "stop",
   "reasoning_tokens": 624,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-9",
   "suite": "bbh",
   "latency_s": 4.301033973693848,
   "ttft_s": 0.33090734481811523,
   "completion_tokens": 226,
   "finish": "stop",
   "reasoning_tokens": 219,
   "correct": true
  },
  {
   "id": "bbh-snarks-87",
   "suite": "bbh",
   "latency_s": 7.339878797531128,
   "ttft_s": 0.23053383827209473,
   "completion_tokens": 406,
   "finish": "stop",
   "reasoning_tokens": 399,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-161",
   "suite": "bbh",
   "latency_s": 13.744938373565674,
   "ttft_s": 0.2626972198486328,
   "completion_tokens": 818,
   "finish": "stop",
   "reasoning_tokens": 811,
   "correct": false
  },
  {
   "id": "bbh-ruin_names-76",
   "suite": "bbh",
   "latency_s": 3.3858089447021484,
   "ttft_s": 0.23415040969848633,
   "completion_tokens": 172,
   "finish": "stop",
   "reasoning_tokens": 165,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-9",
   "suite": "bbh",
   "latency_s": 4.508142471313477,
   "ttft_s": 0.2759709358215332,
   "completion_tokens": 270,
   "finish": "stop",
   "reasoning_tokens": 263,
   "correct": true
  },
  {
   "id": "bbh-web_of_lies-160",
   "suite": "bbh",
   "latency_s": 7.153756380081177,
   "ttft_s": 0.20583343505859375,
   "completion_tokens": 436,
   "finish": "stop",
   "reasoning_tokens": 429,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-33",
   "suite": "bbh",
   "latency_s": 378.6096260547638,
   "ttft_s": 0.8347740173339844,
   "completion_tokens": 17646,
   "finish": "stop",
   "reasoning_tokens": 17639,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-21",
   "suite": "bbh",
   "latency_s": 4.7388222217559814,
   "ttft_s": 0.36198902130126953,
   "completion_tokens": 202,
   "finish": "stop",
   "reasoning_tokens": 195,
   "correct": false
  },
  {
   "id": "bbh-reasoning_about_colored_objects-57",
   "suite": "bbh",
   "latency_s": 2.8381528854370117,
   "ttft_s": 0.32496070861816406,
   "completion_tokens": 162,
   "finish": "stop",
   "reasoning_tokens": 155,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-197",
   "suite": "bbh",
   "latency_s": 39.30435276031494,
   "ttft_s": 0.17471933364868164,
   "completion_tokens": 1796,
   "finish": "stop",
   "reasoning_tokens": 1789,
   "correct": false
  },
  {
   "id": "bbh-salient_translation_error_detection-203",
   "suite": "bbh",
   "latency_s": 1.979987382888794,
   "ttft_s": 0.32410740852355957,
   "completion_tokens": 137,
   "finish": "stop",
   "reasoning_tokens": 130,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_three_objects-227",
   "suite": "bbh",
   "latency_s": 1.7053883075714111,
   "ttft_s": 0.2415308952331543,
   "completion_tokens": 87,
   "finish": "stop",
   "reasoning_tokens": 80,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-84",
   "suite": "bbh",
   "latency_s": 668.0304148197174,
   "ttft_s": 0.3035707473754883,
   "completion_tokens": 31588,
   "finish": "stop",
   "reasoning_tokens": 31581,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-201",
   "suite": "bbh",
   "latency_s": 1.8208801746368408,
   "ttft_s": 0.2580382823944092,
   "completion_tokens": 93,
   "finish": "stop",
   "reasoning_tokens": 86,
   "correct": true
  },
  {
   "id": "bbh-penguins_in_a_table-74",
   "suite": "bbh",
   "latency_s": 2.845646858215332,
   "ttft_s": 0.3865182399749756,
   "completion_tokens": 127,
   "finish": "stop",
   "reasoning_tokens": 120,
   "correct": true
  },
  {
   "id": "bbh-disambiguation_qa-168",
   "suite": "bbh",
   "latency_s": 6.782041311264038,
   "ttft_s": 0.2553892135620117,
   "completion_tokens": 315,
   "finish": "stop",
   "reasoning_tokens": 308,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-201",
   "suite": "bbh",
   "latency_s": 10.933907508850098,
   "ttft_s": 0.3279540538787842,
   "completion_tokens": 606,
   "finish": "stop",
   "reasoning_tokens": 599,
   "correct": true
  }
 ]
}