{
 "label": "tp2-stock-dflash2",
 "model": "GLM-5.3-Flash-EXL3",
 "concurrency": 1,
 "effort": "max",
 "n": 60,
 "wall_s": 1910.495135307312,
 "latency_median_s": 5.842950940132141,
 "latency_mean_s": 31.8414608279864,
 "latency_p90_s": 57.68627405166626,
 "correct": 50,
 "completion_tokens_mean": 1589.4833333333333,
 "length_stops": 0,
 "items": [
  {
   "id": "mmlupro-3593",
   "suite": "mmlupro",
   "latency_s": 3.127324342727661,
   "ttft_s": 0.4590320587158203,
   "completion_tokens": 159,
   "finish": "stop",
   "reasoning_tokens": 153,
   "correct": true
  },
  {
   "id": "mmlupro-6249",
   "suite": "mmlupro",
   "latency_s": 1.545644760131836,
   "ttft_s": 0.4953806400299072,
   "completion_tokens": 69,
   "finish": "stop",
   "reasoning_tokens": 63,
   "correct": true
  },
  {
   "id": "mmlupro-6182",
   "suite": "mmlupro",
   "latency_s": 16.802945852279663,
   "ttft_s": 0.6238148212432861,
   "completion_tokens": 822,
   "finish": "stop",
   "reasoning_tokens": 816,
   "correct": true
  },
  {
   "id": "mmlupro-4993",
   "suite": "mmlupro",
   "latency_s": 1.5131182670593262,
   "ttft_s": 0.5459232330322266,
   "completion_tokens": 58,
   "finish": "stop",
   "reasoning_tokens": 52,
   "correct": true
  },
  {
   "id": "mmlupro-12095",
   "suite": "mmlupro",
   "latency_s": 70.50711607933044,
   "ttft_s": 0.4706461429595947,
   "completion_tokens": 3753,
   "finish": "stop",
   "reasoning_tokens": 3747,
   "correct": true
  },
  {
   "id": "mmlupro-2253",
   "suite": "mmlupro",
   "latency_s": 2.0650298595428467,
   "ttft_s": 0.41229772567749023,
   "completion_tokens": 114,
   "finish": "stop",
   "reasoning_tokens": 108,
   "correct": true
  },
  {
   "id": "mmlupro-2570",
   "suite": "mmlupro",
   "latency_s": 3.524019956588745,
   "ttft_s": 0.4472348690032959,
   "completion_tokens": 180,
   "finish": "stop",
   "reasoning_tokens": 174,
   "correct": true
  },
  {
   "id": "mmlupro-2192",
   "suite": "mmlupro",
   "latency_s": 9.67050051689148,
   "ttft_s": 0.4294898509979248,
   "completion_tokens": 518,
   "finish": "stop",
   "reasoning_tokens": 512,
   "correct": true
  },
  {
   "id": "mmlupro-3354",
   "suite": "mmlupro",
   "latency_s": 8.342082262039185,
   "ttft_s": 0.6432352066040039,
   "completion_tokens": 431,
   "finish": "stop",
   "reasoning_tokens": 425,
   "correct": true
  },
  {
   "id": "mmlupro-4643",
   "suite": "mmlupro",
   "latency_s": 3.142749071121216,
   "ttft_s": 0.5245361328125,
   "completion_tokens": 155,
   "finish": "stop",
   "reasoning_tokens": 149,
   "correct": true
  },
  {
   "id": "mmlupro-10930",
   "suite": "mmlupro",
   "latency_s": 227.45690369606018,
   "ttft_s": 0.3502836227416992,
   "completion_tokens": 11224,
   "finish": "stop",
   "reasoning_tokens": 11218,
   "correct": true
  },
  {
   "id": "mmlupro-10659",
   "suite": "mmlupro",
   "latency_s": 8.020220756530762,
   "ttft_s": 0.5461876392364502,
   "completion_tokens": 514,
   "finish": "stop",
   "reasoning_tokens": 348,
   "correct": true
  },
  {
   "id": "mmlupro-4834",
   "suite": "mmlupro",
   "latency_s": 6.531796932220459,
   "ttft_s": 0.6626448631286621,
   "completion_tokens": 411,
   "finish": "stop",
   "reasoning_tokens": 405,
   "correct": true
  },
  {
   "id": "mmlupro-2618",
   "suite": "mmlupro",
   "latency_s": 143.33988118171692,
   "ttft_s": 0.5448932647705078,
   "completion_tokens": 6897,
   "finish": "stop",
   "reasoning_tokens": 6891,
   "correct": false
  },
  {
   "id": "mmlupro-5034",
   "suite": "mmlupro",
   "latency_s": 2.3143725395202637,
   "ttft_s": 0.45661377906799316,
   "completion_tokens": 126,
   "finish": "stop",
   "reasoning_tokens": 120,
   "correct": true
  },
  {
   "id": "mmlupro-9174",
   "suite": "mmlupro",
   "latency_s": 5.806189060211182,
   "ttft_s": 0.514153242111206,
   "completion_tokens": 319,
   "finish": "stop",
   "reasoning_tokens": 313,
   "correct": true
  },
  {
   "id": "mmlupro-4212",
   "suite": "mmlupro",
   "latency_s": 2.1987345218658447,
   "ttft_s": 0.44666004180908203,
   "completion_tokens": 129,
   "finish": "stop",
   "reasoning_tokens": 123,
   "correct": true
  },
  {
   "id": "mmlupro-2139",
   "suite": "mmlupro",
   "latency_s": 8.89849066734314,
   "ttft_s": 0.5592677593231201,
   "completion_tokens": 445,
   "finish": "stop",
   "reasoning_tokens": 439,
   "correct": true
  },
  {
   "id": "mmlupro-7387",
   "suite": "mmlupro",
   "latency_s": 2.599092960357666,
   "ttft_s": 0.5025584697723389,
   "completion_tokens": 109,
   "finish": "stop",
   "reasoning_tokens": 103,
   "correct": true
  },
  {
   "id": "mmlupro-10998",
   "suite": "mmlupro",
   "latency_s": 13.48658299446106,
   "ttft_s": 0.5018982887268066,
   "completion_tokens": 673,
   "finish": "stop",
   "reasoning_tokens": 667,
   "correct": true
  },
  {
   "id": "mmlupro-11035",
   "suite": "mmlupro",
   "latency_s": 6.813897132873535,
   "ttft_s": 0.5857579708099365,
   "completion_tokens": 351,
   "finish": "stop",
   "reasoning_tokens": 345,
   "correct": true
  },
  {
   "id": "mmlupro-1452",
   "suite": "mmlupro",
   "latency_s": 681.7279677391052,
   "ttft_s": 0.644479513168335,
   "completion_tokens": 32242,
   "finish": "stop",
   "reasoning_tokens": 32236,
   "correct": false
  },
  {
   "id": "mmlupro-9852",
   "suite": "mmlupro",
   "latency_s": 2.5429940223693848,
   "ttft_s": 0.5050168037414551,
   "completion_tokens": 170,
   "finish": "stop",
   "reasoning_tokens": 164,
   "correct": true
  },
  {
   "id": "mmlupro-1840",
   "suite": "mmlupro",
   "latency_s": 140.91893815994263,
   "ttft_s": 0.5332591533660889,
   "completion_tokens": 7036,
   "finish": "stop",
   "reasoning_tokens": 7030,
   "correct": true
  },
  {
   "id": "mmlupro-9244",
   "suite": "mmlupro",
   "latency_s": 24.927550315856934,
   "ttft_s": 0.4774959087371826,
   "completion_tokens": 1347,
   "finish": "stop",
   "reasoning_tokens": 1341,
   "correct": true
  },
  {
   "id": "mmlupro-11200",
   "suite": "mmlupro",
   "latency_s": 57.68627405166626,
   "ttft_s": 0.48154401779174805,
   "completion_tokens": 3112,
   "finish": "stop",
   "reasoning_tokens": 3106,
   "correct": false
  },
  {
   "id": "mmlupro-804",
   "suite": "mmlupro",
   "latency_s": 15.641289472579956,
   "ttft_s": 0.5583493709564209,
   "completion_tokens": 822,
   "finish": "stop",
   "reasoning_tokens": 816,
   "correct": true
  },
  {
   "id": "mmlupro-11025",
   "suite": "mmlupro",
   "latency_s": 137.79909944534302,
   "ttft_s": 0.45281434059143066,
   "completion_tokens": 6133,
   "finish": "stop",
   "reasoning_tokens": 6127,
   "correct": true
  },
  {
   "id": "mmlupro-8937",
   "suite": "mmlupro",
   "latency_s": 4.5331196784973145,
   "ttft_s": 0.4477424621582031,
   "completion_tokens": 232,
   "finish": "stop",
   "reasoning_tokens": 226,
   "correct": false
  },
  {
   "id": "mmlupro-8442",
   "suite": "mmlupro",
   "latency_s": 2.981215476989746,
   "ttft_s": 0.43999576568603516,
   "completion_tokens": 145,
   "finish": "stop",
   "reasoning_tokens": 139,
   "correct": true
  },
  {
   "id": "bbh-object_counting-28",
   "suite": "bbh",
   "latency_s": 1.6831085681915283,
   "ttft_s": 0.3330352306365967,
   "completion_tokens": 95,
   "finish": "stop",
   "reasoning_tokens": 89,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-75",
   "suite": "bbh",
   "latency_s": 39.13052177429199,
   "ttft_s": 0.4553954601287842,
   "completion_tokens": 2086,
   "finish": "stop",
   "reasoning_tokens": 1956,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-136",
   "suite": "bbh",
   "latency_s": 4.80443549156189,
   "ttft_s": 0.5447404384613037,
   "completion_tokens": 373,
   "finish": "stop",
   "reasoning_tokens": 200,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-139",
   "suite": "bbh",
   "latency_s": 3.329444169998169,
   "ttft_s": 0.46735072135925293,
   "completion_tokens": 182,
   "finish": "stop",
   "reasoning_tokens": 176,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-207",
   "suite": "bbh",
   "latency_s": 27.67297887802124,
   "ttft_s": 0.3793330192565918,
   "completion_tokens": 1316,
   "finish": "stop",
   "reasoning_tokens": 1308,
   "correct": false
  },
  {
   "id": "bbh-movie_recommendation-26",
   "suite": "bbh",
   "latency_s": 6.529251337051392,
   "ttft_s": 0.4115440845489502,
   "completion_tokens": 344,
   "finish": "stop",
   "reasoning_tokens": 338,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_five_objects-223",
   "suite": "bbh",
   "latency_s": 3.969346761703491,
   "ttft_s": 0.4368736743927002,
   "completion_tokens": 245,
   "finish": "stop",
   "reasoning_tokens": 239,
   "correct": true
  },
  {
   "id": "bbh-reasoning_about_colored_objects-95",
   "suite": "bbh",
   "latency_s": 1.9048147201538086,
   "ttft_s": 0.4591703414916992,
   "completion_tokens": 115,
   "finish": "stop",
   "reasoning_tokens": 46,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-117",
   "suite": "bbh",
   "latency_s": 13.304147005081177,
   "ttft_s": 0.5132613182067871,
   "completion_tokens": 1131,
   "finish": "stop",
   "reasoning_tokens": 963,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-67",
   "suite": "bbh",
   "latency_s": 6.439768552780151,
   "ttft_s": 0.5468952655792236,
   "completion_tokens": 362,
   "finish": "stop",
   "reasoning_tokens": 356,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-2",
   "suite": "bbh",
   "latency_s": 38.24448776245117,
   "ttft_s": 0.5136661529541016,
   "completion_tokens": 2197,
   "finish": "stop",
   "reasoning_tokens": 2056,
   "correct": true
  },
  {
   "id": "bbh-multistep_arithmetic_two-202",
   "suite": "bbh",
   "latency_s": 2.601210832595825,
   "ttft_s": 0.3423943519592285,
   "completion_tokens": 148,
   "finish": "stop",
   "reasoning_tokens": 142,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-125",
   "suite": "bbh",
   "latency_s": 5.936519384384155,
   "ttft_s": 0.48206424713134766,
   "completion_tokens": 412,
   "finish": "stop",
   "reasoning_tokens": 255,
   "correct": false
  },
  {
   "id": "bbh-logical_deduction_seven_objects-9",
   "suite": "bbh",
   "latency_s": 3.664289951324463,
   "ttft_s": 0.5521996021270752,
   "completion_tokens": 256,
   "finish": "stop",
   "reasoning_tokens": 142,
   "correct": true
  },
  {
   "id": "bbh-snarks-87",
   "suite": "bbh",
   "latency_s": 9.665472984313965,
   "ttft_s": 0.4371051788330078,
   "completion_tokens": 501,
   "finish": "stop",
   "reasoning_tokens": 495,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-161",
   "suite": "bbh",
   "latency_s": 31.124103546142578,
   "ttft_s": 0.5000190734863281,
   "completion_tokens": 1959,
   "finish": "stop",
   "reasoning_tokens": 1758,
   "correct": false
  },
  {
   "id": "bbh-ruin_names-76",
   "suite": "bbh",
   "latency_s": 3.8001410961151123,
   "ttft_s": 0.4234786033630371,
   "completion_tokens": 208,
   "finish": "stop",
   "reasoning_tokens": 202,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-9",
   "suite": "bbh",
   "latency_s": 5.746826410293579,
   "ttft_s": 0.48311281204223633,
   "completion_tokens": 298,
   "finish": "stop",
   "reasoning_tokens": 292,
   "correct": true
  },
  {
   "id": "bbh-web_of_lies-160",
   "suite": "bbh",
   "latency_s": 16.899516105651855,
   "ttft_s": 0.3888249397277832,
   "completion_tokens": 911,
   "finish": "stop",
   "reasoning_tokens": 903,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-33",
   "suite": "bbh",
   "latency_s": 32.40258765220642,
   "ttft_s": 0.6588959693908691,
   "completion_tokens": 1700,
   "finish": "stop",
   "reasoning_tokens": 1567,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-21",
   "suite": "bbh",
   "latency_s": 4.068762540817261,
   "ttft_s": 0.5587444305419922,
   "completion_tokens": 203,
   "finish": "stop",
   "reasoning_tokens": 113,
   "correct": false
  },
  {
   "id": "bbh-reasoning_about_colored_objects-57",
   "suite": "bbh",
   "latency_s": 1.895463466644287,
   "ttft_s": 0.5205967426300049,
   "completion_tokens": 114,
   "finish": "stop",
   "reasoning_tokens": 108,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-197",
   "suite": "bbh",
   "latency_s": 5.879712820053101,
   "ttft_s": 0.2712562084197998,
   "completion_tokens": 257,
   "finish": "stop",
   "reasoning_tokens": 251,
   "correct": false
  },
  {
   "id": "bbh-salient_translation_error_detection-203",
   "suite": "bbh",
   "latency_s": 2.63400936126709,
   "ttft_s": 0.5830304622650146,
   "completion_tokens": 161,
   "finish": "stop",
   "reasoning_tokens": 111,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_three_objects-227",
   "suite": "bbh",
   "latency_s": 1.6440682411193848,
   "ttft_s": 0.4146590232849121,
   "completion_tokens": 98,
   "finish": "stop",
   "reasoning_tokens": 61,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-84",
   "suite": "bbh",
   "latency_s": 4.004756689071655,
   "ttft_s": 0.501683235168457,
   "completion_tokens": 192,
   "finish": "stop",
   "reasoning_tokens": 186,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-201",
   "suite": "bbh",
   "latency_s": 2.41017484664917,
   "ttft_s": 0.45395946502685547,
   "completion_tokens": 120,
   "finish": "stop",
   "reasoning_tokens": 114,
   "correct": true
  },
  {
   "id": "bbh-penguins_in_a_table-74",
   "suite": "bbh",
   "latency_s": 1.4482805728912354,
   "ttft_s": 0.538529634475708,
   "completion_tokens": 59,
   "finish": "stop",
   "reasoning_tokens": 53,
   "correct": true
  },
  {
   "id": "bbh-disambiguation_qa-168",
   "suite": "bbh",
   "latency_s": 6.9439637660980225,
   "ttft_s": 0.47281455993652344,
   "completion_tokens": 375,
   "finish": "stop",
   "reasoning_tokens": 316,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-201",
   "suite": "bbh",
   "latency_s": 4.240342617034912,
   "ttft_s": 0.5277049541473389,
   "completion_tokens": 255,
   "finish": "stop",
   "reasoning_tokens": 249,
   "correct": true
  }
 ]
}