{
 "label": "tp2-ours-dflash2",
 "model": "GLM-5.3-Flash-EXL3",
 "concurrency": 1,
 "effort": "max",
 "n": 60,
 "wall_s": 907.4492361545563,
 "latency_median_s": 3.6780320405960083,
 "latency_mean_s": 15.123925244808197,
 "latency_p90_s": 24.814213037490845,
 "correct": 52,
 "completion_tokens_mean": 742.8333333333334,
 "length_stops": 0,
 "items": [
  {
   "id": "mmlupro-3593",
   "suite": "mmlupro",
   "latency_s": 3.548213243484497,
   "ttft_s": 0.4641916751861572,
   "completion_tokens": 168,
   "finish": "stop",
   "reasoning_tokens": 162,
   "correct": true
  },
  {
   "id": "mmlupro-6249",
   "suite": "mmlupro",
   "latency_s": 2.997450351715088,
   "ttft_s": 0.49426698684692383,
   "completion_tokens": 141,
   "finish": "stop",
   "reasoning_tokens": 135,
   "correct": true
  },
  {
   "id": "mmlupro-6182",
   "suite": "mmlupro",
   "latency_s": 4.142920970916748,
   "ttft_s": 0.6224861145019531,
   "completion_tokens": 195,
   "finish": "stop",
   "reasoning_tokens": 189,
   "correct": true
  },
  {
   "id": "mmlupro-4993",
   "suite": "mmlupro",
   "latency_s": 1.2596535682678223,
   "ttft_s": 0.5439422130584717,
   "completion_tokens": 41,
   "finish": "stop",
   "reasoning_tokens": 35,
   "correct": true
  },
  {
   "id": "mmlupro-12095",
   "suite": "mmlupro",
   "latency_s": 31.151567697525024,
   "ttft_s": 0.47200632095336914,
   "completion_tokens": 1642,
   "finish": "stop",
   "reasoning_tokens": 1636,
   "correct": true
  },
  {
   "id": "mmlupro-2253",
   "suite": "mmlupro",
   "latency_s": 1.6959209442138672,
   "ttft_s": 0.4100456237792969,
   "completion_tokens": 87,
   "finish": "stop",
   "reasoning_tokens": 81,
   "correct": true
  },
  {
   "id": "mmlupro-2570",
   "suite": "mmlupro",
   "latency_s": 3.534773111343384,
   "ttft_s": 0.4486520290374756,
   "completion_tokens": 213,
   "finish": "stop",
   "reasoning_tokens": 207,
   "correct": true
  },
  {
   "id": "mmlupro-2192",
   "suite": "mmlupro",
   "latency_s": 12.500617504119873,
   "ttft_s": 0.4309713840484619,
   "completion_tokens": 623,
   "finish": "stop",
   "reasoning_tokens": 617,
   "correct": true
  },
  {
   "id": "mmlupro-3354",
   "suite": "mmlupro",
   "latency_s": 9.100717544555664,
   "ttft_s": 0.6425480842590332,
   "completion_tokens": 466,
   "finish": "stop",
   "reasoning_tokens": 460,
   "correct": true
  },
  {
   "id": "mmlupro-4643",
   "suite": "mmlupro",
   "latency_s": 3.1273140907287598,
   "ttft_s": 0.5232985019683838,
   "completion_tokens": 164,
   "finish": "stop",
   "reasoning_tokens": 158,
   "correct": true
  },
  {
   "id": "mmlupro-10930",
   "suite": "mmlupro",
   "latency_s": 54.4726402759552,
   "ttft_s": 0.35054540634155273,
   "completion_tokens": 2720,
   "finish": "stop",
   "reasoning_tokens": 2714,
   "correct": false
  },
  {
   "id": "mmlupro-10659",
   "suite": "mmlupro",
   "latency_s": 9.034830808639526,
   "ttft_s": 0.5495176315307617,
   "completion_tokens": 447,
   "finish": "stop",
   "reasoning_tokens": 441,
   "correct": true
  },
  {
   "id": "mmlupro-4834",
   "suite": "mmlupro",
   "latency_s": 3.0455470085144043,
   "ttft_s": 0.6664857864379883,
   "completion_tokens": 136,
   "finish": "stop",
   "reasoning_tokens": 130,
   "correct": true
  },
  {
   "id": "mmlupro-2618",
   "suite": "mmlupro",
   "latency_s": 43.25056290626526,
   "ttft_s": 0.5422649383544922,
   "completion_tokens": 2149,
   "finish": "stop",
   "reasoning_tokens": 2143,
   "correct": false
  },
  {
   "id": "mmlupro-5034",
   "suite": "mmlupro",
   "latency_s": 1.439279556274414,
   "ttft_s": 0.45388269424438477,
   "completion_tokens": 68,
   "finish": "stop",
   "reasoning_tokens": 62,
   "correct": true
  },
  {
   "id": "mmlupro-9174",
   "suite": "mmlupro",
   "latency_s": 4.819552183151245,
   "ttft_s": 0.5159711837768555,
   "completion_tokens": 280,
   "finish": "stop",
   "reasoning_tokens": 274,
   "correct": true
  },
  {
   "id": "mmlupro-4212",
   "suite": "mmlupro",
   "latency_s": 2.5455541610717773,
   "ttft_s": 0.44820666313171387,
   "completion_tokens": 125,
   "finish": "stop",
   "reasoning_tokens": 119,
   "correct": true
  },
  {
   "id": "mmlupro-2139",
   "suite": "mmlupro",
   "latency_s": 2.502991199493408,
   "ttft_s": 0.5581855773925781,
   "completion_tokens": 106,
   "finish": "stop",
   "reasoning_tokens": 100,
   "correct": true
  },
  {
   "id": "mmlupro-7387",
   "suite": "mmlupro",
   "latency_s": 1.6722712516784668,
   "ttft_s": 0.5036416053771973,
   "completion_tokens": 74,
   "finish": "stop",
   "reasoning_tokens": 68,
   "correct": true
  },
  {
   "id": "mmlupro-10998",
   "suite": "mmlupro",
   "latency_s": 11.094930171966553,
   "ttft_s": 0.504570722579956,
   "completion_tokens": 512,
   "finish": "stop",
   "reasoning_tokens": 506,
   "correct": true
  },
  {
   "id": "mmlupro-11035",
   "suite": "mmlupro",
   "latency_s": 4.457267999649048,
   "ttft_s": 0.585320234298706,
   "completion_tokens": 229,
   "finish": "stop",
   "reasoning_tokens": 223,
   "correct": true
  },
  {
   "id": "mmlupro-1452",
   "suite": "mmlupro",
   "latency_s": 161.39625811576843,
   "ttft_s": 0.6385960578918457,
   "completion_tokens": 7617,
   "finish": "stop",
   "reasoning_tokens": 7611,
   "correct": true
  },
  {
   "id": "mmlupro-9852",
   "suite": "mmlupro",
   "latency_s": 2.8168489933013916,
   "ttft_s": 0.5054423809051514,
   "completion_tokens": 178,
   "finish": "stop",
   "reasoning_tokens": 172,
   "correct": true
  },
  {
   "id": "mmlupro-1840",
   "suite": "mmlupro",
   "latency_s": 43.45290565490723,
   "ttft_s": 0.5333297252655029,
   "completion_tokens": 2104,
   "finish": "stop",
   "reasoning_tokens": 2098,
   "correct": true
  },
  {
   "id": "mmlupro-9244",
   "suite": "mmlupro",
   "latency_s": 11.555127620697021,
   "ttft_s": 0.4774954319000244,
   "completion_tokens": 570,
   "finish": "stop",
   "reasoning_tokens": 564,
   "correct": true
  },
  {
   "id": "mmlupro-11200",
   "suite": "mmlupro",
   "latency_s": 285.7364182472229,
   "ttft_s": 0.4816756248474121,
   "completion_tokens": 13151,
   "finish": "stop",
   "reasoning_tokens": 13145,
   "correct": true
  },
  {
   "id": "mmlupro-804",
   "suite": "mmlupro",
   "latency_s": 16.109041690826416,
   "ttft_s": 0.5555052757263184,
   "completion_tokens": 806,
   "finish": "stop",
   "reasoning_tokens": 800,
   "correct": true
  },
  {
   "id": "mmlupro-11025",
   "suite": "mmlupro",
   "latency_s": 24.814213037490845,
   "ttft_s": 0.4513390064239502,
   "completion_tokens": 1181,
   "finish": "stop",
   "reasoning_tokens": 1175,
   "correct": true
  },
  {
   "id": "mmlupro-8937",
   "suite": "mmlupro",
   "latency_s": 2.9885339736938477,
   "ttft_s": 0.4510035514831543,
   "completion_tokens": 164,
   "finish": "stop",
   "reasoning_tokens": 158,
   "correct": false
  },
  {
   "id": "mmlupro-8442",
   "suite": "mmlupro",
   "latency_s": 2.5117886066436768,
   "ttft_s": 0.44095635414123535,
   "completion_tokens": 125,
   "finish": "stop",
   "reasoning_tokens": 119,
   "correct": true
  },
  {
   "id": "bbh-object_counting-28",
   "suite": "bbh",
   "latency_s": 1.6411402225494385,
   "ttft_s": 0.3317444324493408,
   "completion_tokens": 88,
   "finish": "stop",
   "reasoning_tokens": 63,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-75",
   "suite": "bbh",
   "latency_s": 5.724930286407471,
   "ttft_s": 0.45522260665893555,
   "completion_tokens": 314,
   "finish": "stop",
   "reasoning_tokens": 226,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-136",
   "suite": "bbh",
   "latency_s": 3.308251142501831,
   "ttft_s": 0.5426816940307617,
   "completion_tokens": 196,
   "finish": "stop",
   "reasoning_tokens": 190,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-139",
   "suite": "bbh",
   "latency_s": 2.8635129928588867,
   "ttft_s": 0.4682154655456543,
   "completion_tokens": 147,
   "finish": "stop",
   "reasoning_tokens": 141,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-207",
   "suite": "bbh",
   "latency_s": 17.550854206085205,
   "ttft_s": 0.37813901901245117,
   "completion_tokens": 833,
   "finish": "stop",
   "reasoning_tokens": 826,
   "correct": false
  },
  {
   "id": "bbh-movie_recommendation-26",
   "suite": "bbh",
   "latency_s": 5.072333335876465,
   "ttft_s": 0.41106748580932617,
   "completion_tokens": 262,
   "finish": "stop",
   "reasoning_tokens": 256,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_five_objects-223",
   "suite": "bbh",
   "latency_s": 3.4376673698425293,
   "ttft_s": 0.4338417053222656,
   "completion_tokens": 206,
   "finish": "stop",
   "reasoning_tokens": 200,
   "correct": true
  },
  {
   "id": "bbh-reasoning_about_colored_objects-95",
   "suite": "bbh",
   "latency_s": 1.9724960327148438,
   "ttft_s": 0.4566645622253418,
   "completion_tokens": 115,
   "finish": "stop",
   "reasoning_tokens": 45,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-117",
   "suite": "bbh",
   "latency_s": 7.732858419418335,
   "ttft_s": 0.5141463279724121,
   "completion_tokens": 532,
   "finish": "stop",
   "reasoning_tokens": 526,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-67",
   "suite": "bbh",
   "latency_s": 5.321647644042969,
   "ttft_s": 0.5443458557128906,
   "completion_tokens": 334,
   "finish": "stop",
   "reasoning_tokens": 328,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-2",
   "suite": "bbh",
   "latency_s": 5.065318822860718,
   "ttft_s": 0.5076093673706055,
   "completion_tokens": 275,
   "finish": "stop",
   "reasoning_tokens": 171,
   "correct": true
  },
  {
   "id": "bbh-multistep_arithmetic_two-202",
   "suite": "bbh",
   "latency_s": 2.7861223220825195,
   "ttft_s": 0.3401494026184082,
   "completion_tokens": 177,
   "finish": "stop",
   "reasoning_tokens": 171,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-125",
   "suite": "bbh",
   "latency_s": 9.462363958358765,
   "ttft_s": 0.4801783561706543,
   "completion_tokens": 538,
   "finish": "stop",
   "reasoning_tokens": 532,
   "correct": false
  },
  {
   "id": "bbh-logical_deduction_seven_objects-9",
   "suite": "bbh",
   "latency_s": 3.2893991470336914,
   "ttft_s": 0.5501110553741455,
   "completion_tokens": 198,
   "finish": "stop",
   "reasoning_tokens": 192,
   "correct": true
  },
  {
   "id": "bbh-snarks-87",
   "suite": "bbh",
   "latency_s": 3.270685911178589,
   "ttft_s": 0.4379112720489502,
   "completion_tokens": 170,
   "finish": "stop",
   "reasoning_tokens": 164,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-161",
   "suite": "bbh",
   "latency_s": 11.613611459732056,
   "ttft_s": 0.4992103576660156,
   "completion_tokens": 740,
   "finish": "stop",
   "reasoning_tokens": 525,
   "correct": false
  },
  {
   "id": "bbh-ruin_names-76",
   "suite": "bbh",
   "latency_s": 3.8078508377075195,
   "ttft_s": 0.422652006149292,
   "completion_tokens": 194,
   "finish": "stop",
   "reasoning_tokens": 188,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-9",
   "suite": "bbh",
   "latency_s": 4.855116367340088,
   "ttft_s": 0.47874975204467773,
   "completion_tokens": 282,
   "finish": "stop",
   "reasoning_tokens": 276,
   "correct": true
  },
  {
   "id": "bbh-web_of_lies-160",
   "suite": "bbh",
   "latency_s": 1.9318606853485107,
   "ttft_s": 0.3892042636871338,
   "completion_tokens": 112,
   "finish": "stop",
   "reasoning_tokens": 106,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-33",
   "suite": "bbh",
   "latency_s": 7.5562520027160645,
   "ttft_s": 0.6658382415771484,
   "completion_tokens": 339,
   "finish": "stop",
   "reasoning_tokens": 333,
   "correct": false
  },
  {
   "id": "bbh-causal_judgement-21",
   "suite": "bbh",
   "latency_s": 8.697814702987671,
   "ttft_s": 0.5583481788635254,
   "completion_tokens": 456,
   "finish": "stop",
   "reasoning_tokens": 292,
   "correct": false
  },
  {
   "id": "bbh-reasoning_about_colored_objects-57",
   "suite": "bbh",
   "latency_s": 1.9683341979980469,
   "ttft_s": 0.5233414173126221,
   "completion_tokens": 114,
   "finish": "stop",
   "reasoning_tokens": 108,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-197",
   "suite": "bbh",
   "latency_s": 3.3141136169433594,
   "ttft_s": 0.2726755142211914,
   "completion_tokens": 152,
   "finish": "stop",
   "reasoning_tokens": 144,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-203",
   "suite": "bbh",
   "latency_s": 2.430769920349121,
   "ttft_s": 0.5829124450683594,
   "completion_tokens": 117,
   "finish": "stop",
   "reasoning_tokens": 38,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_three_objects-227",
   "suite": "bbh",
   "latency_s": 1.5040488243103027,
   "ttft_s": 0.41304755210876465,
   "completion_tokens": 86,
   "finish": "stop",
   "reasoning_tokens": 62,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-84",
   "suite": "bbh",
   "latency_s": 3.4738657474517822,
   "ttft_s": 0.504023551940918,
   "completion_tokens": 161,
   "finish": "stop",
   "reasoning_tokens": 155,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-201",
   "suite": "bbh",
   "latency_s": 3.075610399246216,
   "ttft_s": 0.4535987377166748,
   "completion_tokens": 231,
   "finish": "stop",
   "reasoning_tokens": 108,
   "correct": true
  },
  {
   "id": "bbh-penguins_in_a_table-74",
   "suite": "bbh",
   "latency_s": 1.7062242031097412,
   "ttft_s": 0.5366020202636719,
   "completion_tokens": 93,
   "finish": "stop",
   "reasoning_tokens": 29,
   "correct": true
  },
  {
   "id": "bbh-disambiguation_qa-168",
   "suite": "bbh",
   "latency_s": 4.605801343917847,
   "ttft_s": 0.4740135669708252,
   "completion_tokens": 243,
   "finish": "stop",
   "reasoning_tokens": 199,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-201",
   "suite": "bbh",
   "latency_s": 5.618946075439453,
   "ttft_s": 0.528059720993042,
   "completion_tokens": 383,
   "finish": "stop",
   "reasoning_tokens": 233,
   "correct": true
  }
 ]
}