{ "concurrency": { "1": { "aggregate_end_to_end_tokens_per_second": { "maximum": 62.227, "median": 61.034, "minimum": 59.161, "p90": 62.227 }, "per_stream_decode_tokens_per_second": { "maximum": 65.325, "median": 63.961, "minimum": 61.904, "p90": 65.325 }, "per_stream_ttft_seconds": { "maximum": 0.209, "median": 0.207, "minimum": 0.206, "p90": 0.209 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 61.034, "completion_tokens": 256, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 3.986786, "decode_tokens_per_second": 63.961, "finish_reason": "length", "measured_sse_events": 97, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n", "output_characters": 87, "output_sha256": "85977a1b1ce971013871c7fe2a034407fb1b664b807cd58daa18407dad871657", "prompt_tokens": 169, "reasoning_characters": 916, "reasoning_sha256": "5d5b31016f6fe912647ff4dd714cd3a807fdc36115e83e62255e4a2a2a08f54a", "response_tail_seconds": 8.1e-05, "stream_done_marker": true, "stream_sha256": "90aae23a2ab1856b3cee28eb216bb7a101c0ba411769d1f9a4df3b3292d041a4", "time_to_first_visible_seconds": 3.87485, "time_to_last_output_seconds": 4.193244, "ttft_seconds": 0.206458, "wall_seconds": 4.193325 } ], "wall_seconds": 4.194382 }, { "aggregate_end_to_end_tokens_per_second": 62.227, "completion_tokens": 256, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 3.903558, "decode_tokens_per_second": 65.325, "finish_reason": "length", "measured_sse_events": 96, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n", "output_characters": 87, "output_sha256": "85977a1b1ce971013871c7fe2a034407fb1b664b807cd58daa18407dad871657", "prompt_tokens": 169, "reasoning_characters": 928, "reasoning_sha256": "25902398412c4da150bd274d18b044f9601333017679aa373ea50e3d49b1eb96", "response_tail_seconds": 0.000126, "stream_done_marker": true, "stream_sha256": "b403ce483fe8c2cc3fbed699eacf2dd0131d252d0b9c258a892892b4159d19f7", "time_to_first_visible_seconds": 3.8006, "time_to_last_output_seconds": 4.1127, "ttft_seconds": 0.209142, "wall_seconds": 4.112826 } ], "wall_seconds": 4.113952 }, { "aggregate_end_to_end_tokens_per_second": 59.161, "completion_tokens": 256, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.119297, "decode_tokens_per_second": 61.904, "finish_reason": "length", "measured_sse_events": 101, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for testability.\ntype Clock interface {\n\t", "output_characters": 148, "output_sha256": "9193f309e4906c0054f04caeb4da015dedb1941529f8696dd268bc3942b67cd1", "prompt_tokens": 168, "reasoning_characters": 858, "reasoning_sha256": "9aa90f25d24964da8f5244b3f550aa571e984e53aeced28fe15bd9bc0c8a3648", "response_tail_seconds": 0.000116, "stream_done_marker": true, "stream_sha256": "f83c86534694b93f386cf8aebb5f8c707f8f8e1688bce1ce0a314fdbdc854fb4", "time_to_first_visible_seconds": 3.741287, "time_to_last_output_seconds": 4.326114, "ttft_seconds": 0.206817, "wall_seconds": 4.32623 } ], "wall_seconds": 4.327196 } ] }, "2": { "aggregate_end_to_end_tokens_per_second": { "maximum": 96.15, "median": 94.687, "minimum": 94.074, "p90": 96.15 }, "per_stream_decode_tokens_per_second": { "maximum": 52.524, "median": 51.736, "minimum": 48.682, "p90": 51.919 }, "per_stream_ttft_seconds": { "maximum": 0.47, "median": 0.338, "minimum": 0.203, "p90": 0.47 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 96.15, "completion_tokens": 512, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.85492, "decode_tokens_per_second": 52.524, "finish_reason": "length", "measured_sse_events": 98, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time", "output_characters": 113, "output_sha256": "bf3dadbdd444e807deb23b506670ab8a7d3e7b79fe5876a5e4914aba99ddbcf3", "prompt_tokens": 169, "reasoning_characters": 892, "reasoning_sha256": "9b0fe301ecf775f6ffbbed0030afa58a4b594ac734d03c0f9f80dd47b692380f", "response_tail_seconds": 8.4e-05, "stream_done_marker": true, "stream_sha256": "adc5d7c2c55bc0419cd0e1fb1f6fd3be39cc1ed16150c9d8ab3b40fffb8116ef", "time_to_first_visible_seconds": 4.768691, "time_to_last_output_seconds": 5.323262, "ttft_seconds": 0.468343, "wall_seconds": 5.323347 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.911482, "decode_tokens_per_second": 51.919, "finish_reason": "length", "measured_sse_events": 95, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for testability.\n", "output_characters": 138, "output_sha256": "01862f646e6d94aefdfa99e15f115597d2560640b449015019ab95728f7aee7b", "prompt_tokens": 168, "reasoning_characters": 867, "reasoning_sha256": "adb1f8f4f63539cd34e7ef07be09df700f31452b3aa554d408d49be3cd01bd5b", "response_tail_seconds": 0.00018, "stream_done_marker": true, "stream_sha256": "5dc607dc71aaad2ca18bf4f60ac32f6eee62e34efda9d85ea5bb0b3337fad778", "time_to_first_visible_seconds": 4.347144, "time_to_last_output_seconds": 5.120109, "ttft_seconds": 0.208627, "wall_seconds": 5.120289 } ], "wall_seconds": 5.325033 }, { "aggregate_end_to_end_tokens_per_second": 94.687, "completion_tokens": 512, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.936242, "decode_tokens_per_second": 51.659, "finish_reason": "length", "measured_sse_events": 99, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n", "output_characters": 80, "output_sha256": "bc4834114aef06133250aec0f3dd5184a1c9ce9d7133c9218575e242978d4abc", "prompt_tokens": 168, "reasoning_characters": 942, "reasoning_sha256": "bdd9da9a3b891eae2b25062fbad358f5e6bd5d3ca182714781fa2c3587a685ba", "response_tail_seconds": 7.8e-05, "stream_done_marker": true, "stream_sha256": "9242c109f9a60f4b1f230116289cc90141cd0b4de206bd2bbcb235a6e48200cf", "time_to_first_visible_seconds": 4.962142, "time_to_last_output_seconds": 5.40604, "ttft_seconds": 0.469799, "wall_seconds": 5.406118 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.941232, "decode_tokens_per_second": 51.607, "finish_reason": "length", "measured_sse_events": 95, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n", "output_characters": 83, "output_sha256": "ee4741c84e7cd624b52d4e557d9b797966e5e62b84f3b3bf6e5c3b68e548e739", "prompt_tokens": 166, "reasoning_characters": 935, "reasoning_sha256": "1e21648b0934a6203df766623e5bdb2b1525740410eff5e7d1f8886e74ae3d06", "response_tail_seconds": 0.000139, "stream_done_marker": true, "stream_sha256": "05dae90bb335cac38faaa4db22c3d31201771078af93eeeeb4ad4f56b9e43d2f", "time_to_first_visible_seconds": 4.784383, "time_to_last_output_seconds": 5.147883, "ttft_seconds": 0.20665, "wall_seconds": 5.148021 } ], "wall_seconds": 5.407263 }, { "aggregate_end_to_end_tokens_per_second": 94.074, "completion_tokens": 512, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.92145, "decode_tokens_per_second": 51.814, "finish_reason": "length", "measured_sse_events": 102, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for testability.\ntype Clock interface {\n", "output_characters": 147, "output_sha256": "8ac566abc6fd844c5df0e1c4b2d4220107ee9fe0073cf6b9fac38bd97a7f989f", "prompt_tokens": 166, "reasoning_characters": 858, "reasoning_sha256": "9fffa6b758cefeed2cf9b409de38ba8655fb6bde8b78b498dd2f8c45f4d8227c", "response_tail_seconds": 0.000134, "stream_done_marker": true, "stream_sha256": "666b572733132b19d7e9ed236ecee9609f44725ffa69e7ae2c80bbb7310391cf", "time_to_first_visible_seconds": 4.633453, "time_to_last_output_seconds": 5.391717, "ttft_seconds": 0.470267, "wall_seconds": 5.391851 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.23812, "decode_tokens_per_second": 48.682, "finish_reason": "length", "measured_sse_events": 104, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time", "output_characters": 78, "output_sha256": "bce1b80a5668fd2173cbdf76a2ed5de3d4f612d6bb5f91ffca73c11748dc605d", "prompt_tokens": 170, "reasoning_characters": 948, "reasoning_sha256": "45aa86cbf2cea826f8bf41bfcd06912e5fd805d93c82b499cc61681cc9dc62b6", "response_tail_seconds": 0.000121, "stream_done_marker": true, "stream_sha256": "23faa854dafd989898aeffe6d1aa23c5e0ff47b3d927fb5db822c85938c489ba", "time_to_first_visible_seconds": 4.979365, "time_to_last_output_seconds": 5.441289, "ttft_seconds": 0.203169, "wall_seconds": 5.44141 } ], "wall_seconds": 5.442526 } ] }, "4": { "aggregate_end_to_end_tokens_per_second": { "maximum": 152.504, "median": 151.207, "minimum": 145.812, "p90": 152.504 }, "per_stream_decode_tokens_per_second": { "maximum": 47.95, "median": 41.442, "minimum": 38.627, "p90": 44.601 }, "per_stream_ttft_seconds": { "maximum": 0.591, "median": 0.571, "minimum": 0.208, "p90": 0.591 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 145.812, "completion_tokens": 1024, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.717412, "decode_tokens_per_second": 44.601, "finish_reason": "length", "measured_sse_events": 93, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for deterministic testing.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// Bucket implements a concurrency", "output_characters": 228, "output_sha256": "0287a858e5b09dffed49845b747be464a4596b3532a2fd6f8b8d5f02a67458a2", "prompt_tokens": 171, "reasoning_characters": 829, "reasoning_sha256": "fc457d5be8b90425e79bf511e4c4b270af312a458a853a50f1696d026797ec2f", "response_tail_seconds": 0.000282, "stream_done_marker": true, "stream_sha256": "eadba8b5edc8794360c30c7a9569a0515314406dec14049d8db42ddfa41b818e", "time_to_first_visible_seconds": 5.102416, "time_to_last_output_seconds": 6.28831, "ttft_seconds": 0.570898, "wall_seconds": 6.288593 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.449994, "decode_tokens_per_second": 39.535, "finish_reason": "length", "measured_sse_events": 106, "output": "```ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for test", "output_characters": 112, "output_sha256": "d07b614078df23109d1fd251c5156532e413b91ea4f17d840ff600a82c853dd4", "prompt_tokens": 167, "reasoning_characters": 928, "reasoning_sha256": "4fba91fac770a5bde2532972f31c38e324e6425436884f1cb907aff04858cb41", "response_tail_seconds": 7.8e-05, "stream_done_marker": true, "stream_sha256": "0b8bd2ded2b8a89282376aeac03fb2f22ced203295b7a2ff7c38409142761d55", "time_to_first_visible_seconds": 6.411296, "time_to_last_output_seconds": 7.020477, "ttft_seconds": 0.570482, "wall_seconds": 7.020555 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.601639, "decode_tokens_per_second": 38.627, "finish_reason": "length", "measured_sse_events": 104, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for testability", "output_characters": 125, "output_sha256": "97ce7ec8896e863814fd45418813aff6dee168e9b138097ba8a6751ddb12eb38", "prompt_tokens": 169, "reasoning_characters": 899, "reasoning_sha256": "1d13f52ddaa535889f292489fb46529baa3ebce6f74adbad390c488a09d1e004", "response_tail_seconds": 0.000126, "stream_done_marker": true, "stream_sha256": "0189948be10015e5c59c56c3fdb3197513508de2142b2a9b46771525c0443db7", "time_to_first_visible_seconds": 6.031364, "time_to_last_output_seconds": 6.855357, "ttft_seconds": 0.253718, "wall_seconds": 6.855483 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.416925, "decode_tokens_per_second": 39.739, "finish_reason": "length", "measured_sse_events": 101, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"", "output_characters": 82, "output_sha256": "b40619821a63fffb9dfe597790aec6214d24b20133a664331300bb774f511f7d", "prompt_tokens": 171, "reasoning_characters": 928, "reasoning_sha256": "c45a29de4c9cbb6b02794c4dd5e0de15d9e539bfd17cc40029f2a3c443cdd73a", "response_tail_seconds": 0.000139, "stream_done_marker": true, "stream_sha256": "ec39cd7e33c93fa8b04e5162da5ed70dd0ed0ed65ae88bd0dfea04ae81b3817b", "time_to_first_visible_seconds": 6.288025, "time_to_last_output_seconds": 6.670735, "ttft_seconds": 0.253809, "wall_seconds": 6.670874 } ], "wall_seconds": 7.022726 }, { "aggregate_end_to_end_tokens_per_second": 151.207, "completion_tokens": 1024, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.184567, "decode_tokens_per_second": 41.232, "finish_reason": "length", "measured_sse_events": 105, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)", "output_characters": 81, "output_sha256": "75395a2d4f395ba4396e1428f77531449bec0a064c465b74011ab8001975d913", "prompt_tokens": 169, "reasoning_characters": 933, "reasoning_sha256": "ee64c99e706afca980e6831f151a84fe64b43fe8495f46a74945e35b4e3d2178", "response_tail_seconds": 0.000391, "stream_done_marker": true, "stream_sha256": "d0b17a091b7f94fb2dbafc5e2f6ad7ab2fb8ad2872d5924a7fc2ce832c6bc6fa", "time_to_first_visible_seconds": 6.278076, "time_to_last_output_seconds": 6.769927, "ttft_seconds": 0.58536, "wall_seconds": 6.770318 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.318061, "decode_tokens_per_second": 47.95, "finish_reason": "length", "measured_sse_events": 91, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for testability.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// Bucket is", "output_characters": 196, "output_sha256": "002afffa5408d66a3570f0e289aa71290c99c9fbe73cdaea81cfb1adbe02298b", "prompt_tokens": 167, "reasoning_characters": 818, "reasoning_sha256": "f18b443416da048469c0437e77a2fa8bdc9fa3586f0547d5530f58b46eadea4a", "response_tail_seconds": 0.000219, "stream_done_marker": true, "stream_sha256": "17841c8a0c2e578fd28c9cfb1736cc9591e34c7a341bb39ccd00e64facc6e6b7", "time_to_first_visible_seconds": 4.864641, "time_to_last_output_seconds": 5.902784, "ttft_seconds": 0.584722, "wall_seconds": 5.903003 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.392215, "decode_tokens_per_second": 39.892, "finish_reason": "length", "measured_sse_events": 104, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock provides the current time.\ntype Clock interface {\n\t", "output_characters": 147, "output_sha256": "cb5677d33e462a9e95c29a2234fae0bb1ee59e90a5f4b4066dc9e4408a95cebc", "prompt_tokens": 169, "reasoning_characters": 841, "reasoning_sha256": "ba09eca2a816cbac5c49411a6651b0c5614f2770a0709a1ca8cc0ddb84f1cd42", "response_tail_seconds": 0.000141, "stream_done_marker": true, "stream_sha256": "52334d43dbc511736e69eb6ffd954e1f2178fda7e13f914a2781c160415ff68c", "time_to_first_visible_seconds": 5.822581, "time_to_last_output_seconds": 6.656888, "ttft_seconds": 0.264673, "wall_seconds": 6.657029 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.505384, "decode_tokens_per_second": 39.198, "finish_reason": "length", "measured_sse_events": 106, "output": "```ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts", "output_characters": 98, "output_sha256": "3c20c4ed60d77d5f0f29569c4b41c67cf438cbd22ccb0ac466e08f11011f5ce6", "prompt_tokens": 166, "reasoning_characters": 920, "reasoning_sha256": "7931b134de96488ad7b292edae5c8be8c543daac571b1f82234a27e8fc3bf9ab", "response_tail_seconds": 6.7e-05, "stream_done_marker": true, "stream_sha256": "b44de776d057d979fa58c3e83f0d4147369596f054e13307815d2a569347164f", "time_to_first_visible_seconds": 6.216761, "time_to_last_output_seconds": 6.770169, "ttft_seconds": 0.264785, "wall_seconds": 6.770235 } ], "wall_seconds": 6.772164 }, { "aggregate_end_to_end_tokens_per_second": 152.504, "completion_tokens": 1024, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.031569, "decode_tokens_per_second": 42.278, "finish_reason": "length", "measured_sse_events": 105, "output": "```ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for deterministic testing.\ntype Clock interface {\n\tNow() time", "output_characters": 179, "output_sha256": "39b7f41417c462446363a1f959d86729870fd9f04e742d2cb6e5d008589d85c8", "prompt_tokens": 166, "reasoning_characters": 867, "reasoning_sha256": "078d050de51f327e0639d125794cf5884c1fd10fdeed790f82a291372e622eef", "response_tail_seconds": 8e-05, "stream_done_marker": true, "stream_sha256": "a99a30d9f0b5ba1308e9ba8cd4714f26f7ec551b65a6daeefe30ff082b1367e3", "time_to_first_visible_seconds": 5.691305, "time_to_last_output_seconds": 6.622069, "ttft_seconds": 0.5905, "wall_seconds": 6.622149 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.978211, "decode_tokens_per_second": 42.655, "finish_reason": "length", "measured_sse_events": 104, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for", "output_characters": 114, "output_sha256": "576e947e0f3ee0cdcd57c4af3bd7721c4dd3face6bba4fcb18a15b20c16c16e3", "prompt_tokens": 169, "reasoning_characters": 894, "reasoning_sha256": "893e50f90c16e47a660c63bb1552deb45b948753186ba6be754e54a173bda34b", "response_tail_seconds": 0.000281, "stream_done_marker": true, "stream_sha256": "83d5f287ae93da56b6e8d703696c9bf527472d3c7f0088d37a371e287d355b2d", "time_to_first_visible_seconds": 5.877939, "time_to_last_output_seconds": 6.568697, "ttft_seconds": 0.590486, "wall_seconds": 6.568978 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.119765, "decode_tokens_per_second": 41.668, "finish_reason": "length", "measured_sse_events": 101, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock", "output_characters": 91, "output_sha256": "3050129a65469e55de48290a612d128ce08976410fae70caa9bc631489983f71", "prompt_tokens": 168, "reasoning_characters": 930, "reasoning_sha256": "85a847ccf6b89bba5263253f21ce37b5dfe5972bfb7366e9d71f7a22e3c07e96", "response_tail_seconds": 0.000392, "stream_done_marker": true, "stream_sha256": "259d5d79ca759a2b5bdfd8cc2ce7b11f83dc441a37add2ac9a677bbe86d518b1", "time_to_first_visible_seconds": 5.752652, "time_to_last_output_seconds": 6.327384, "ttft_seconds": 0.207618, "wall_seconds": 6.327776 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.121987, "decode_tokens_per_second": 41.653, "finish_reason": "length", "measured_sse_events": 107, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock provides the current time.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// Real", "output_characters": 172, "output_sha256": "e31da768ab33985f49acc237cc078bb452cd1df449d78bab63370c320ae661e3", "prompt_tokens": 168, "reasoning_characters": 891, "reasoning_sha256": "2f629405be51c49c4ae5f44a1abca780d52f74a7e0e3e8003b5bf42fbb76cacb", "response_tail_seconds": 0.000119, "stream_done_marker": true, "stream_sha256": "b065a49550dbeefb8cdd216f4c8ca9b7796ab28c60975621c9aefac486aba22b", "time_to_first_visible_seconds": 5.691795, "time_to_last_output_seconds": 6.71286, "ttft_seconds": 0.590874, "wall_seconds": 6.712979 } ], "wall_seconds": 6.714557 } ] } }, "decode": { "code": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 94.269, "median": 93.315, "minimum": 89.826, "p90": 94.269 }, "pooled_decode_tokens_per_second": 92.481, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 1839, "completion_validation": { "valid": true }, "decode_seconds": 20.461746, "decode_tokens_per_second": 89.826, "finish_reason": "stop", "measured_sse_events": 407, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// Bucket is a concurrency-safe token-bucket rate limiter.\ntype Bucket struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trefillRate float64 // tokens per second\n\ttokens float64\n\tlastRefill time.Time\n\tclock Clock\n}\n\n// New creates a Bucket with the given capacity (max burst) and refillRate (tokens/sec).\n// The clock parameter must not be nil.\nfunc New(capacity, refillRate float64, clock Clock) (*Bucket, error) {\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be positive\")\n\t}\n\tif refillRate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: refillRate must be positive\")\n\t}\n\treturn &Bucket{\n\t\tcapacity: capacity,\n\t\trefillRate: refillRate,\n\t\ttokens: capacity,\n\t\tlastRefill: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN attempts to consume n tokens. It returns true if tokens were available.\nfunc (b *Bucket) AllowN(n int) bool {\n\tif n < 0 {\n\t\treturn false\n\t}\n\tif n == 0 {\n\t\treturn true\n\t}\n\tb.mu.Lock()\n\tdefer b.mu.Unlock()\n\n\tnow := b.clock.Now()\n\telapsed := now.Sub(b.lastRefill).Seconds()\n\tif elapsed > 0 {\n\t\tb.tokens += elapsed * b.refillRate\n\t\tif b.tokens > b.capacity {\n\t\t\tb.tokens = b.capacity\n\t\t}\n\t\tb.lastRefill = now\n\t}\n\n\tif b.tokens >= float64(n) {\n\t\tb.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```go\n// ratelimit/bucket_test.go\npackage ratelimit\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tt time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time { return c.t }\nfunc (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) }\n\nfunc TestNew(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\ttests := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 1, fc, false},\n\t\t{\"nil clock\", 10, 1, nil, true},\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 10, 0, fc, true},\n\t\t{\"negative rate\", 10, -1, fc, true},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := New(tt.capacity, tt.refillRate, tt.clock)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"New() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tadvances []time.Duration\n\t\tn int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"burst then deny\",\n\t\t\tcapacity: 5,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 0, 0, 0, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"refill after wait\",\n\t\t\tcapacity: 2,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 1 * time.Second, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\tcapacity: 10,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 3 * time.Second, 0},\n\t\t\tn: 5,\n\t\t\twant: []bool{true, false, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"n equals zero\",\n\t\t\tcapacity: 1,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\tn: 0,\n\t\t\twant: []bool{true},\n\t\t},\n\t\t{\n\t\t\tname: \"negative n\",\n\t\t\tcapacity: 1,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\tn: -1,\n\t\t\twant: []bool{false},\n\t\t},\n\t\t{\n\t\t\tname: \"n exceeds capacity\",\n\t\t\tcapacity: 3,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\tn: 5,\n\t\t\twant: []bool{false},\n\t\t},\n\t\t{\n\t\t\tname: \"capped at capacity\",\n\t\t\tcapacity: 2,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0, 10 * time.Second, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, false},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: time.Unix(0, 0)}\n\t\t\tb, err := New(tt.capacity, tt.refillRate, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, adv := range tt.advances {\n\t\t\t\tfc.Advance(adv)\n\t\t\t\tgot := b.AllowN(tt.n)\n\t\t\t\tif got != tt.want[i] {\n\t\t\t\t\tt.Errorf(\"call %d: AllowN(%d) = %v, want %v\", i, tt.n, got, tt.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 4298, "output_sha256": "944ff6cd816d8df36b6582d69d9401a36a93ce7a4aebe6a3acfe2ac58d8dd722", "prompt_tokens": 167, "reasoning_characters": 896, "reasoning_sha256": "dc197f260de545b04b7256e113e3738b014528ecc6dd662c1f18af4c0d4e4a3b", "response_tail_seconds": 0.000371, "run": 1, "stream_done_marker": true, "stream_sha256": "1188185ea920584592993e172d1c2082cce6e8f0823219036ebbf72feb8c5885", "time_to_first_visible_seconds": 4.434854, "time_to_last_output_seconds": 20.738752, "ttft_seconds": 0.277006, "wall_seconds": 20.739124 }, { "cached_prompt_tokens": 0, "completion_tokens": 1878, "completion_validation": { "valid": true }, "decode_seconds": 20.600543, "decode_tokens_per_second": 91.114, "finish_reason": "stop", "measured_sse_events": 423, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for testability.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// RealClock uses the system wall clock.\ntype RealClock struct{}\n\nfunc (RealClock) Now() time.Time { return time.Now() }\n\n// Limiter is a concurrency-safe token-bucket rate limiter.\ntype Limiter struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trefillRate float64 // tokens per second\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\n// New creates a Limiter. capacity must be > 0, refillRate must be > 0,\n// and clock must not be nil.\nfunc New(capacity, refillRate float64, clock Clock) (*Limiter, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be > 0\")\n\t}\n\tif refillRate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: refillRate must be > 0\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &Limiter{\n\t\tcapacity: capacity,\n\t\trefillRate: refillRate,\n\t\ttokens: capacity,\n\t\tlast: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN attempts to consume n tokens. It returns true if the bucket\n// had enough tokens, false otherwise.\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n < 0 {\n\t\treturn false\n\t}\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\telapsed := now.Sub(l.last).Seconds()\n\tif elapsed > 0 {\n\t\tl.tokens += elapsed * l.refillRate\n\t\tif l.tokens > l.capacity {\n\t\t\tl.tokens = l.capacity\n\t\t}\n\t\tl.last = now\n\t}\n\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```ratelimit/ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tt time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time { return c.t }\nfunc (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) }\n\nfunc TestNew_Validation(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tcases := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 1, fc, false},\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 10, 0, fc, true},\n\t\t{\"negative rate\", 10, -1, fc, true},\n\t\t{\"nil clock\", 10, 1, nil, true},\n\t}\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\t_, err := New(tc.capacity, tc.refillRate, tc.clock)\n\t\t\tif (err != nil) != tc.wantErr {\n\t\t\t\tt.Fatalf(\"err=%v, wantErr=%v\", err, tc.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tadvances []time.Duration\n\t\tn int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"initial burst\",\n\t\t\tcapacity: 5,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 0, 0, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, true, true},\n\t\t},\n\t\t{\n\t\t\tname: \"exhaust then deny\",\n\t\t\tcapacity: 3,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 0, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"refill after wait\",\n\t\t\tcapacity: 2,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 1 * time.Second, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"AllowN larger than capacity always false\",\n\t\t\tcapacity: 5,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\tn: 10,\n\t\t\twant: []bool{false},\n\t\t},\n\t\t{\n\t\t\tname: \"AllowN zero always true\",\n\t\t\tcapacity: 1,\n\t\t\trefillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0},\n\t\t\tn: 0,\n\t\t\twant: []bool{true, true},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\tcapacity: 10,\n\t\t\trefillRate: 2,\n\t\t\tadvances: []time.Duration{0, 0, 500 * time.Millisecond, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, true},\n\t\t},\n\t}\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: time.Unix(0, 0)}\n\t\t\tl, err := New(tc.capacity, tc.refillRate, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, d := range tc.advances {\n\t\t\t\tfc.Advance(d)\n\t\t\t\tgot := l.AllowN(tc.n)\n\t\t\t\tif got != tc.want[i] {\n\t\t\t\t\tt.Fatalf(\"step %d: got %v, want %v\", i, got, tc.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN_Negative(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tl, _ := New(10, 1, fc)\n\tif l.AllowN(-1) {\n\t\tt.Fatal(\"negative n should return false\")\n\t}\n}\n```", "output_characters": 4407, "output_sha256": "690fa3c5718dbafb9d8bb51d5505b6d96ac5b0fe9a66959532f4e8b0ab5f554c", "prompt_tokens": 169, "reasoning_characters": 932, "reasoning_sha256": "1121fa2f74c1a670b00889aaa75bd4e468cfb4d3d8fb03f9a6f77ac21b732012", "response_tail_seconds": 8.1e-05, "run": 2, "stream_done_marker": true, "stream_sha256": "d70d34f21578fe212758b8b4e42fe20cfb1957d27b2c6959c3ac72a9b929557b", "time_to_first_visible_seconds": 4.0395, "time_to_last_output_seconds": 20.821911, "ttft_seconds": 0.221368, "wall_seconds": 20.821991 }, { "cached_prompt_tokens": 0, "completion_tokens": 1788, "completion_validation": { "valid": true }, "decode_seconds": 19.150091, "decode_tokens_per_second": 93.315, "finish_reason": "stop", "measured_sse_events": 395, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for testability.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// TokenBucket implements a concurrency-safe token-bucket rate limiter.\ntype TokenBucket struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trefillRate float64 // tokens per second\n\ttokens float64\n\tlastRefill time.Time\n\tclock Clock\n}\n\n// New creates a TokenBucket. capacity must be > 0, refillRate must be > 0,\n// and clock must not be nil.\nfunc New(capacity, refillRate float64, clock Clock) (*TokenBucket, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be > 0\")\n\t}\n\tif refillRate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: refillRate must be > 0\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &TokenBucket{\n\t\tcapacity: capacity,\n\t\trefillRate: refillRate,\n\t\ttokens: capacity,\n\t\tlastRefill: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN attempts to consume n tokens. It returns true if the bucket\n// contained at least n tokens after refilling based on elapsed time.\nfunc (tb *TokenBucket) AllowN(n int) bool {\n\tif n < 0 {\n\t\treturn false\n\t}\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\n\tnow := tb.clock.Now()\n\telapsed := now.Sub(tb.lastRefill).Seconds()\n\tif elapsed > 0 {\n\t\ttb.tokens += elapsed * tb.refillRate\n\t\tif tb.tokens > tb.capacity {\n\t\t\ttb.tokens = tb.capacity\n\t\t}\n\t\ttb.lastRefill = now\n\t}\n\n\tif tb.tokens >= float64(n) {\n\t\ttb.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```ratelimit/ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tt time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time { return c.t }\nfunc (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) }\n\nfunc TestNewValidation(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\ttests := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 1, fc, false},\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 10, 0, fc, true},\n\t\t{\"negative rate\", 10, -1, fc, true},\n\t\t{\"nil clock\", 10, 1, nil, true},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := New(tt.capacity, tt.refillRate, tt.clock)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"New() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tstart time.Time\n\t\tadvances []time.Duration\n\t\tn int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"initial burst\",\n\t\t\tcapacity: 5,\n\t\t\trefillRate: 1,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tadvances: []time.Duration{0, 0, 0, 0, 0, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"refill over time\",\n\t\t\tcapacity: 2,\n\t\t\trefillRate: 1,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tadvances: []time.Duration{0, 0, 1 * time.Second, 0, 1 * time.Second, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, false, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"allowN larger than capacity\",\n\t\t\tcapacity: 3,\n\t\t\trefillRate: 1,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tadvances: []time.Duration{0, 0},\n\t\t\tn: 4,\n\t\t\twant: []bool{false, false},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\tcapacity: 10,\n\t\t\trefillRate: 2,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tadvances: []time.Duration{0, 500 * time.Millisecond, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true},\n\t\t},\n\t\t{\n\t\t\tname: \"negative n returns false\",\n\t\t\tcapacity: 5,\n\t\t\trefillRate: 1,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tadvances: []time.Duration{0},\n\t\t\tn: -1,\n\t\t\twant: []bool{false},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: tt.start}\n\t\t\ttb, err := New(tt.capacity, tt.refillRate, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, adv := range tt.advances {\n\t\t\t\tfc.Advance(adv)\n\t\t\t\tgot := tb.AllowN(tt.n)\n\t\t\t\tif got != tt.want[i] {\n\t\t\t\t\tt.Errorf(\"step %d: AllowN(%d) = %v, want %v\", i, tt.n, got, tt.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 4240, "output_sha256": "31f91e1ecb32be93b34ab2b3282312f35a9f62d1dfb75826375899c33156db8c", "prompt_tokens": 167, "reasoning_characters": 930, "reasoning_sha256": "85a847ccf6b89bba5263253f21ce37b5dfe5972bfb7366e9d71f7a22e3c07e96", "response_tail_seconds": 8.3e-05, "run": 3, "stream_done_marker": true, "stream_sha256": "16ce4fd25ad655643ee48b909c4d69b0fba81a85b48a4d3337f0ede9a52f49d3", "time_to_first_visible_seconds": 4.044245, "time_to_last_output_seconds": 19.366083, "ttft_seconds": 0.215992, "wall_seconds": 19.366165 }, { "cached_prompt_tokens": 0, "completion_tokens": 1827, "completion_validation": { "valid": true }, "decode_seconds": 19.419103, "decode_tokens_per_second": 94.031, "finish_reason": "stop", "measured_sse_events": 398, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for testability.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// RealClock uses the system clock.\ntype RealClock struct{}\n\nfunc (RealClock) Now() time.Time { return time.Now() }\n\n// Limiter is a concurrency-safe token-bucket rate limiter.\ntype Limiter struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trefillRate float64 // tokens per second\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\n// New creates a Limiter. capacity must be > 0, refillRate must be > 0.\nfunc New(capacity, refillRate float64, clock Clock) (*Limiter, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be > 0\")\n\t}\n\tif refillRate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: refillRate must be > 0\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &Limiter{\n\t\tcapacity: capacity,\n\t\trefillRate: refillRate,\n\t\ttokens: capacity,\n\t\tlast: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN reports whether n tokens are available and consumes them if so.\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n < 0 {\n\t\treturn false\n\t}\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\telapsed := now.Sub(l.last).Seconds()\n\tif elapsed > 0 {\n\t\tl.tokens += elapsed * l.refillRate\n\t\tif l.tokens > l.capacity {\n\t\t\tl.tokens = l.capacity\n\t\t}\n\t\tl.last = now\n\t}\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```ratelimit/ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tt time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time { return c.t }\nfunc (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) }\n\nfunc TestNew_Validation(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\ttests := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 1, fc, false},\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 10, 0, fc, true},\n\t\t{\"negative rate\", 10, -1, fc, true},\n\t\t{\"nil clock\", 10, 1, nil, true},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := New(tt.capacity, tt.refillRate, tt.clock)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"New() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tadvances []time.Duration\n\t\trequests []int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"initial burst\",\n\t\t\tcapacity: 5, refillRate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\trequests: []int{5},\n\t\t\twant: []bool{true},\n\t\t},\n\t\t{\n\t\t\tname: \"exceed burst\",\n\t\t\tcapacity: 5, refillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0},\n\t\t\trequests: []int{5, 1},\n\t\t\twant: []bool{true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"refill after time\",\n\t\t\tcapacity: 5, refillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 3 * time.Second},\n\t\t\trequests: []int{5, 1, 3},\n\t\t\twant: []bool{true, false, true},\n\t\t},\n\t\t{\n\t\t\tname: \"refill capped at capacity\",\n\t\t\tcapacity: 5, refillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 100 * time.Second},\n\t\t\trequests: []int{5, 1, 5},\n\t\t\twant: []bool{true, false, true},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\tcapacity: 10, refillRate: 2,\n\t\t\tadvances: []time.Duration{0, 0, 1 * time.Second},\n\t\t\trequests: []int{10, 1, 2},\n\t\t\twant: []bool{true, false, true},\n\t\t},\n\t\t{\n\t\t\tname: \"zero request always allowed\",\n\t\t\tcapacity: 1, refillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0},\n\t\t\trequests: []int{1, 0},\n\t\t\twant: []bool{true, true},\n\t\t},\n\t\t{\n\t\t\tname: \"negative request rejected\",\n\t\t\tcapacity: 1, refillRate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\trequests: []int{-1},\n\t\t\twant: []bool{false},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: time.Unix(0, 0)}\n\t\t\tl, err := New(tt.capacity, tt.refillRate, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, adv := range tt.advances {\n\t\t\t\tif adv > 0 {\n\t\t\t\t\tfc.Advance(adv)\n\t\t\t\t}\n\t\t\t\tgot := l.AllowN(tt.requests[i])\n\t\t\t\tif got != tt.want[i] {\n\t\t\t\t\tt.Errorf(\"AllowN(%d) at step %d = %v, want %v\", tt.requests[i], i, got, tt.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 4294, "output_sha256": "cfc6566fb659d20ecfd7261ca66806bfc745f1dd87ec40add6e226cfbf3ac27f", "prompt_tokens": 166, "reasoning_characters": 855, "reasoning_sha256": "ee381c3496175a91957c26fc10295120958df1957b27cc2897f3a6e8847de91c", "response_tail_seconds": 0.000289, "run": 4, "stream_done_marker": true, "stream_sha256": "193bd4cf74fbcaa15c9fb15a0c79f08b8e8f4c622f88b0cc69d9740ef0a79a7f", "time_to_first_visible_seconds": 3.657992, "time_to_last_output_seconds": 19.639578, "ttft_seconds": 0.220476, "wall_seconds": 19.639867 }, { "cached_prompt_tokens": 0, "completion_tokens": 1922, "completion_validation": { "valid": true }, "decode_seconds": 20.377943, "decode_tokens_per_second": 94.269, "finish_reason": "stop", "measured_sse_events": 419, "output": "```go\n// ratelimit/tokenbucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for deterministic testing.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// TokenBucket is a concurrency-safe token-bucket rate limiter.\ntype TokenBucket struct {\n\tmu sync.Mutex\n\tclock Clock\n\tcapacity float64\n\ttokens float64\n\trate float64 // tokens per second\n\tlast time.Time\n}\n\n// NewTokenBucket creates a TokenBucket. rate is tokens/second, capacity is max burst.\nfunc NewTokenBucket(clock Clock, rate, capacity float64) (*TokenBucket, error) {\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\tif rate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: rate must be positive\")\n\t}\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be positive\")\n\t}\n\treturn &TokenBucket{\n\t\tclock: clock,\n\t\tcapacity: capacity,\n\t\trate: rate,\n\t\ttokens: capacity,\n\t\tlast: clock.Now(),\n\t}, nil\n}\n\n// AllowN attempts to consume n tokens. It returns whether the request is\n// allowed and the minimum duration to wait before it would be allowed.\nfunc (tb *TokenBucket) AllowN(n int) (bool, time.Duration) {\n\tif n <= 0 {\n\t\treturn true, 0\n\t}\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\n\tnow := tb.clock.Now()\n\telapsed := now.Sub(tb.last).Seconds()\n\ttb.last = now\n\n\ttb.tokens += elapsed * tb.rate\n\tif tb.tokens > tb.capacity {\n\t\ttb.tokens = tb.capacity\n\t}\n\n\tif tb.tokens >= float64(n) {\n\t\ttb.tokens -= float64(n)\n\t\treturn true, 0\n\t}\n\n\tdeficit := float64(n) - tb.tokens\n\twait := time.Duration(deficit / tb.rate * float64(time.Second))\n\treturn false, wait\n}\n```\n\n```go\n// ratelimit/tokenbucket_test.go\npackage ratelimit\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tnow time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.now\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.now = c.now.Add(d)\n}\n\nfunc TestConstructorValidation(t *testing.T) {\n\tfc := &fakeClock{now: time.Unix(0, 0)}\n\ttests := []struct {\n\t\tname string\n\t\tclock Clock\n\t\trate float64\n\t\tcapacity float64\n\t\twantErr bool\n\t}{\n\t\t{\"nil clock\", nil, 1, 1, true},\n\t\t{\"zero rate\", fc, 0, 1, true},\n\t\t{\"negative rate\", fc, -1, 1, true},\n\t\t{\"zero capacity\", fc, 1, 0, true},\n\t\t{\"negative capacity\", fc, 1, -1, true},\n\t\t{\"valid\", fc, 1, 1, false},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := NewTokenBucket(tt.clock, tt.rate, tt.capacity)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"err = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\trate float64\n\t\tcapacity float64\n\t\tstart time.Time\n\t\tsteps []struct {\n\t\t\tadvance time.Duration\n\t\t\tn int\n\t\t\twantOK bool\n\t\t\twantWait time.Duration\n\t\t}\n\t}{\n\t\t{\n\t\t\tname: \"initial burst\", rate: 1, capacity: 5,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twantOK bool\n\t\t\t\twantWait time.Duration\n\t\t\t}{\n\t\t\t\t{0, 5, true, 0},\n\t\t\t\t{0, 1, false, time.Second},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"refill after wait\", rate: 1, capacity: 5,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twantOK bool\n\t\t\t\twantWait time.Duration\n\t\t\t}{\n\t\t\t\t{0, 5, true, 0},\n\t\t\t\t{2 * time.Second, 2, true, 0},\n\t\t\t\t{0, 1, false, time.Second},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\", rate: 2, capacity: 10,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twantOK bool\n\t\t\t\twantWait time.Duration\n\t\t\t}{\n\t\t\t\t{0, 10, true, 0},\n\t\t\t\t{1 * time.Second, 2, true, 0},\n\t\t\t\t{0, 1, false, 500 * time.Millisecond},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"n zero or negative\", rate: 1, capacity: 5,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twantOK bool\n\t\t\t\twantWait time.Duration\n\t\t\t}{\n\t\t\t\t{0, 0, true, 0},\n\t\t\t\t{0, -1, true, 0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"capacity cap\", rate: 1, capacity: 3,\n\t\t\tstart: time.Unix(0, 0),\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twantOK bool\n\t\t\t\twantWait time.Duration\n\t\t\t}{\n\t\t\t\t{10 * time.Second, 3, true, 0},\n\t\t\t\t{0, 1, false, time.Second},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{now: tt.start}\n\t\t\ttb, err := NewTokenBucket(fc, tt.rate, tt.capacity)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, s := range tt.steps {\n\t\t\t\tfc.Advance(s.advance)\n\t\t\t\tok, wait := tb.AllowN(s.n)\n\t\t\t\tif ok != s.wantOK {\n\t\t\t\t\tt.Errorf(\"step %d: ok = %v, want %v\", i, ok, s.wantOK)\n\t\t\t\t}\n\t\t\t\tif wait != s.wantWait {\n\t\t\t\t\tt.Errorf(\"step %d: wait = %v, want %v\", i, wait, s.wantWait)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 4740, "output_sha256": "89cf05783db21e38f4c7ca200cab3e0b989595f6657183e9387a04513edc054d", "prompt_tokens": 168, "reasoning_characters": 867, "reasoning_sha256": "fca5eb023edb14aeb3c311a33612dcd7a21027c1866c90fc8535b904ff02354d", "response_tail_seconds": 9.7e-05, "run": 5, "stream_done_marker": true, "stream_sha256": "9985ba977b93c8eaa2d5ec98975d1914b79dd71f316d7a92ea2872feeb943c6d", "time_to_first_visible_seconds": 3.481308, "time_to_last_output_seconds": 20.599054, "ttft_seconds": 0.221111, "wall_seconds": 20.59915 } ], "time_to_last_output_seconds": { "maximum": 20.822, "median": 20.599, "minimum": 19.366, "p90": 20.822 }, "ttft_seconds": { "maximum": 0.277, "median": 0.221, "minimum": 0.216, "p90": 0.277 }, "wall_seconds": { "maximum": 20.822, "median": 20.599, "minimum": 19.366, "p90": 20.822 } }, "prose": { "completion_gate": { "passed": 4, "total": 5 }, "decode_tokens_per_second": { "maximum": 100.557, "median": 46.78, "minimum": 44.723, "p90": 100.557 }, "pooled_decode_tokens_per_second": 69.557, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 1244, "completion_validation": { "valid": true }, "decode_seconds": 27.793059, "decode_tokens_per_second": 44.723, "finish_reason": "stop", "measured_sse_events": 680, "output": "**MEMO**\n\n**From:** Engineering Lead, Platform & Inference\n**To:** All Team Members\n**Date:** June 2025\n**Re:** Local AI Hardware \u2014 Buy More or Squeeze What We Have\n\n---\n\nI want to take a few paragraphs to lay out the hardware question that has been circling our standups for the past three weeks, because I think we deserve a shared frame before the next budget cycle locks. The short version is that we have been running at roughly seventy-three percent aggregate GPU utilisation across our inference cluster, and the question is whether that number justifies a purchase of additional cards or whether we should spend the next quarter trying to push it higher.\n\nLet me start with the uncomfortable part. Buying hardware is the most expensive form of engineering laziness. A new set of cards arrives, gets racked, gets configured, and then sits in a state of partial readiness while we figure out what workloads actually need it. The capital outlay is real\u2014somewhere in the neighbourhood of forty to fifty thousand dollars for a meaningful expansion\u2014and that money is not fungible once it leaves the account. We cannot claw it back if the workloads shift, if the model landscape changes, or if a better cloud offering appears in eighteen months. Reversibility is the property we most often undervalue in these decisions, and hardware purchases are about as irreversible as engineering decisions get. You can sell used cards, sure, but the depreciation curve is steep and the transaction friction is real.\n\nNow let me be fair to the other side. Our latency profile matters. When we serve inference to the product team's real-time features, we need predictable tail latencies, and local hardware gives us that in a way that shared cloud instances do not. We have seen p99 spikes of two to three hundred milliseconds when a cloud provider's noisy neighbour decides to saturate a network interface, and those spikes propagate into user-facing timeouts that are genuinely hard to debug from the outside. If our utilisation climbs past eighty-five percent, we start to lose that headroom. The queueing theory is not kind to us at that point, and the latency cliff is steeper than most people expect. So there is a legitimate engineering argument that we are one bad quarter of demand growth away from needing more capacity, and that the cost of discovering that need through a production incident is higher than the cost of buying the cards now.\n\nReliability cuts both ways as well. More hardware means more redundancy, more ability to absorb a node failure without degrading service. But it also means more firmware to patch, more thermal management, more failure modes to monitor, and more of our own time spent being sysadmins rather than being engineers. We are a small team. Every hour we spend maintaining a larger cluster is an hour we do not spend building features or improving model quality. The opportunity cost here is not abstract; it is the specific projects we will not finish this year because we are busy babysitting GPUs.\n\nSo what do I actually recommend? I think we should spend the next eight to ten weeks aggressively improving utilisation of what we already own, and we should set a clear, quantitative trigger for revisiting the purchase. Concretely, that means finishing the dynamic batching work we started in March, implementing the model quantisation pass that Priya sketched out, and getting our autoscaling logic to actually respect the queue-depth signals we already emit. I believe we can push aggregate utilisation to eighty-five or eighty-eight percent without buying a single new card, and I believe that is enough headroom for the next two quarters of projected demand.\n\nThe trigger I want to set is this: if we exceed eighty-five percent sustained utilisation for more than two consecutive weeks, or if we see a p99 latency regression attributable to queueing rather than model complexity, we reconvene and buy. I want that trigger written into our capacity planning document so that the decision is not made by whoever is loudest in a meeting.\n\nI want to acknowledge the uncertainty honestly. I could be wrong about the demand curve. A new product feature could land in July that triples our inference volume overnight, and in that case my \"wait and optimise\" recommendation would have cost us a bad user experience and a scramble. I accept that risk, and I think it is the right risk to accept, because the alternative\u2014buying preemptively for a demand spike that may never come\u2014is the kind of decision that quietly drains a small team's budget for years.\n\nLet's reconvene in mid-August with real numbers from the optimisation work. Bring your best estimates, your worst-case scenarios, and your honest assessment of whether the queue-depth metrics are telling us the truth.\n\n\u2014 M.", "output_characters": 4812, "output_sha256": "6f8c221be9dbcee64b66c634eb54b3f1487662bd35ab7fb2779da8f067f5ef42", "prompt_tokens": 157, "reasoning_characters": 1347, "reasoning_sha256": "6b4889541f51fbd05b8d8c5127e465c51fbf961ceafe021b863e6f377d296fb5", "response_tail_seconds": 0.000297, "run": 1, "stream_done_marker": true, "stream_sha256": "eb95c2edd5b0d1d0e17dcbcc96d541433e2a1e44cfa0f424307b2c3f939d989a", "time_to_first_visible_seconds": 5.280609, "time_to_last_output_seconds": 28.006033, "ttft_seconds": 0.212974, "wall_seconds": 28.006329 }, { "cached_prompt_tokens": 0, "completion_tokens": 1346, "completion_validation": { "valid": true }, "decode_seconds": 28.751674, "decode_tokens_per_second": 46.78, "finish_reason": "stop", "measured_sse_events": 685, "output": "**MEMO**\n\n**To:** Engineering Team\n**From:** Platform & Infrastructure\n**Re:** AI Hardware: Buy More or Squeeze What We Have\n**Date:** June 2025\n\n---\n\nI want to lay out the hardware question honestly, because I think we have been circling it for two quarters without committing to a position. The short version: I believe we should spend the next six to eight weeks aggressively improving utilisation of the GPUs we already own, and only revisit purchasing if a specific, measurable threshold is crossed. I want to explain why, and I want to be transparent about where I might be wrong.\n\nLet me start with the uncomfortable truth about our current setup. We have four A100s sitting in the rack, and our monitoring tells us they are averaging somewhere between 35 and 50 percent utilisation across a typical week. That number is not a hardware problem. It is a scheduling problem, a batching problem, and a problem of us not having built the tooling to queue and prioritise workloads the way a well-run cluster would. When a junior engineer kicks off a fine-tuning job at 11 p.m. and it hogs a card for nine hours while three other people wait until morning, that is not a capacity constraint. That is a coordination failure dressed up as a hardware shortage.\n\nThe cost argument here is straightforward, and I think most of us feel it intuitively. A single A100 node runs roughly $15,000 to $18,000, and that is before we account for the power, cooling, networking, and rack space that make the total cost of ownership closer to $25,000 per node per year. If we buy two more nodes, we are committing to roughly $50,000 in capital and another $50,000 in operational overhead over the next three years. That money is not abstract. It is a contractor we could hire for six months, or a tooling project we keep deferring, or the salary bump we keep postponing. The opportunity cost of a hardware purchase is not the sticker price. It is everything else we could have done with that capital, and right now I think the highest-return use of that money is not more silicon. It is better software around the silicon we already have.\n\nLatency is where the argument gets more nuanced, and I want to acknowledge that honestly. If we were running inference for a product feature with real-time user-facing requirements, I would be having a very different conversation. Local hardware gives us deterministic latency that no amount of utilisation tuning can replicate if we are queuing behind other workloads. But we are not there yet. Our current workloads are training runs, batch evaluations, and exploratory experiments. These are latency-tolerant by nature. A training job that waits forty minutes in a queue instead of starting immediately is not a product failure. It is an inconvenience. And an inconvenience is a much cheaper problem to solve than a $50,000 capital commitment.\n\nReliability cuts both ways, and I do not want to pretend it is a one-sided argument. More hardware means more failure modes. More drivers to update, more thermal events to monitor, more network interfaces to troubleshoot. A team of our size does not have the bandwidth to maintain a larger cluster, and the failure surface grows faster than the capacity we gain. On the other hand, a single overloaded node that crashes under sustained 95 percent utilisation is also a reliability problem. The answer is not to buy redundancy. The answer is to build a scheduler that prevents any single node from reaching that state, and to accept that occasional job failures are a normal part of operating a shared cluster.\n\nThe reversibility argument is the one that tips my recommendation. A software fix to our job queue, a change to our batching strategy, a new priority scheme, these are all things we can try, measure, and undo within a week. A hardware purchase is a three-year commitment that locks us into a specific vendor, a specific generation of silicon, and a specific maintenance burden. In a field where the hardware landscape shifts every eighteen months, committing capital to today's GPU is a bet that the next generation will not make us obsolete. I am not confident enough in that bet to write the check.\n\nNow, the uncertainty. I could be wrong. If our utilisation numbers are not what I think they are, if there is a workload pattern I am not seeing, if the product roadmap accelerates in a way that demands real-time inference within the next quarter, then the calculus changes. I want to set a clear trigger: if, after eight weeks of improved scheduling and tooling, we are still consistently above 85 percent utilisation across all four nodes, we reconvene and buy. That is not a vague hope. That is a measurable condition, and I want us to build the monitoring to know when we hit it.\n\nI would rather we earn the right to buy more hardware by proving we have exhausted what we have.", "output_characters": 4867, "output_sha256": "64e8a56add3c715bb0b25ee066cb3d844cb9d49b80b76b1d91eb09780a1454f5", "prompt_tokens": 161, "reasoning_characters": 1484, "reasoning_sha256": "0c655c89c520c19b6bafd62a2224b1e7cf6b3efa4e48273192dda1f119b632df", "response_tail_seconds": 0.0001, "run": 2, "stream_done_marker": true, "stream_sha256": "00d4a5a8e4da6afa07dc3123106a75e20a889d936fe99c563fb7cc519770baf0", "time_to_first_visible_seconds": 5.626065, "time_to_last_output_seconds": 28.97134, "ttft_seconds": 0.219666, "wall_seconds": 28.971439 }, { "cached_prompt_tokens": 0, "completion_tokens": 8192, "completion_validation": { "error": "no visible answer", "valid": false }, "decode_seconds": 81.456012, "decode_tokens_per_second": 100.557, "finish_reason": "length", "measured_sse_events": 1689, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 158, "reasoning_characters": 32739, "reasoning_sha256": "d77bdd21c98801543244374bc87a96af2438137dd3005363cae5c359abe9a816", "response_tail_seconds": 0.000698, "run": 3, "stream_done_marker": true, "stream_sha256": "d77bdd21c98801543244374bc87a96af2438137dd3005363cae5c359abe9a816", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 81.658916, "ttft_seconds": 0.202904, "wall_seconds": 81.659615 }, { "cached_prompt_tokens": 0, "completion_tokens": 1276, "completion_validation": { "valid": true }, "decode_seconds": 27.693791, "decode_tokens_per_second": 46.039, "finish_reason": "stop", "measured_sse_events": 686, "output": "**MEMO: Hardware Strategy \u2014 Utilisation Before Acquisition**\n\n**From:** Engineering Lead\n**To:** ML Platform Team\n**Re:** Local inference hardware \u2014 next quarter decision\n\nI want to lay out my thinking on the hardware question before we discuss it in Thursday's meeting, because I think we have been circling the same conclusion from different angles and I would rather we arrive at it together than by accident.\n\nThe short version: I believe we should spend the next quarter aggressively improving utilisation of the GPUs we already own, and hold off on any new purchase until we have either exhausted that effort or hit a hard, measurable ceiling. I want to be clear that this is a recommendation, not a decree, and I expect to be argued out of it if the data supports a different conclusion.\n\nThe reason I keep coming back to utilisation is reversibility. A purchase of four A100s or a cluster of H100s is a decision that locks us into a particular architecture, a particular vendor relationship, and a particular depreciation schedule for three to five years. The secondary market for enterprise GPUs is not liquid, and the technology moves fast enough that hardware bought today may be mediocre value eighteen months from now. By contrast, the work of squeezing more throughput out of existing hardware \u2014 better batching, smarter scheduling, quantisation where quality tolerates it, eliminating the idle time between inference calls that our current serving stack leaves on the table \u2014 is almost entirely reversible. If we try a new scheduler and it does not work, we revert. If we find that a particular model can run at lower precision without degradation, we have gained capacity permanently. If we buy hardware and the demand pattern shifts, we are stuck with it.\n\nThat said, I do not want to romanticise the current situation. Our monitoring tells us that average GPU utilisation across the inference fleet sits around thirty-one percent, and that is not a rounding error. It is the result of a serving architecture that was built for a different traffic pattern, one that allocates a full GPU to a single request and then lets it sit idle while the response streams back. The latency numbers look fine in isolation \u2014 p99 is within our SLA \u2014 but that is partly because we are paying for latency with capacity. We have enough headroom that the inefficiency is invisible in the dashboard, and that is exactly the kind of thing that makes teams comfortable enough to defer the work.\n\nThe cost question is where I think the argument gets genuinely uncertain, and I want to acknowledge that. If we project forward and the traffic growth curve holds, we will hit a wall in roughly five months where no amount of scheduling cleverness will keep us under our latency budget. At that point, the question becomes whether we buy hardware or whether we move some workloads to cloud inference and accept the per-token cost and the loss of data locality. The cloud option is more reversible in the contractual sense \u2014 we can scale down in a week \u2014 but it introduces a reliability dependency we do not currently have, and the latency variance of a network round-trip is something we have not yet measured under our actual workload. I would like someone to run that measurement before Thursday.\n\nOpportunity cost is the quiet factor here. The budget for a hardware refresh is roughly equivalent to two quarters of a senior engineer's salary. If we spend it on GPUs, we are not spending it on the data pipeline work that would let us retrain more frequently, or on the evaluation tooling that would let us confidently deploy smaller models. I do not think the hardware purchase is wrong in the abstract, but I think it is wrong *this quarter* because the marginal value of the next GPU is lower than the marginal value of the next person or the next tool.\n\nMy recommendation, then, is this: we commit to a focused utilisation sprint over the next six weeks, targeting fifty percent fleet utilisation as a concrete milestone. We instrument the serving layer properly, we try the batching and scheduling changes we have been putting off, and we measure the cloud latency question in parallel. At the end of that sprint, we reconvene with real numbers and decide whether the ceiling is close enough to justify a purchase. If it is, we buy. If it is not, we have saved capital and gained operational knowledge.\n\nI am not certain this is right. The traffic projections could be wrong, the scheduling work could hit a wall I have not anticipated, or a competitor's move could change the competitive calculus in a way that makes speed more important than efficiency. But the asymmetry of reversibility favours the incremental path, and I would rather be slightly late to a hardware decision than early and stuck.\n\nLet me know if you want to push back before Thursday.", "output_characters": 4853, "output_sha256": "ad1a49e8ee17342cbfbc4821fff34e947ed5df0690774327daaff72ea56e92de", "prompt_tokens": 160, "reasoning_characters": 1501, "reasoning_sha256": "946e0b5000945a0531cc89d0b218a60d7169c22477dca76bd282625bdc042ca8", "response_tail_seconds": 0.054743, "run": 4, "stream_done_marker": true, "stream_sha256": "ea11c7cbe55881e4d6617a2d49c9061d12c7cd48ac059bbda4ad323ab663d1b5", "time_to_first_visible_seconds": 5.741738, "time_to_last_output_seconds": 27.902356, "ttft_seconds": 0.208565, "wall_seconds": 27.957099 }, { "cached_prompt_tokens": 0, "completion_tokens": 1091, "completion_validation": { "valid": true }, "decode_seconds": 23.272981, "decode_tokens_per_second": 46.835, "finish_reason": "stop", "measured_sse_events": 581, "output": "**MEMO**\n\n**To:** Engineering Team\n**From:** Infrastructure Lead\n**Date:** June 12, 2025\n**Re:** AI Hardware: Expansion vs. Utilisation\n\n---\n\nI want to write down my thinking on the question that has come up in three separate planning conversations this month: whether we should spend the remaining capital on additional local inference hardware or invest the same effort into squeezing more throughput out of the machines we already own. This is not a decision I take lightly, and I do not think the answer is obvious, but I have been sitting with it long enough to offer a recommendation.\n\nThe case for buying more hardware is seductive in its simplicity. We have watched our GPU queue times creep upward over the past two quarters, and the temptation is to treat that as a capacity problem with a capacity solution. A new box arrives, the queue shortens, and everyone feels a small relief. The cost is concrete and bounded, the latency improvement is immediate, and the reliability gains are real in the sense that we would have fewer single points of failure in our inference path. There is something honest about that reasoning, and I do not want to dismiss it.\n\nBut I keep returning to the question of what we are actually solving. Our current utilisation sits somewhere in the neighborhood of forty to fifty percent across the fleet, depending on the week and the model mix. That number is not a hardware problem. It is a scheduling problem, a batching problem, and in some cases a model-selection problem. We have workloads that could share a GPU more aggressively, and we have inference paths that are running at batch size one because nobody has yet built the orchestration layer that would let them run at batch size eight. The latency we are experiencing is not the latency of a saturated machine; it is the latency of a poorly coordinated one.\n\nThis is where opportunity cost enters the picture in a way that I think we have been underweighting. Every engineer-hour we spend evaluating, purchasing, racking, and integrating new hardware is an hour we are not spending building the scheduler that would make our existing fleet behave like a larger fleet. The hardware purchase is a one-time cost, yes, but the engineering attention it demands is a recurring tax on the team's focus. And the scheduler work, once built, compounds. It makes every future hardware purchase cheaper per unit of throughput, because we will actually use what we buy.\n\nThere is also the question of reversibility, which I think matters more than we typically give it credit for in infrastructure decisions. A hardware purchase is difficult to reverse. We cannot un-buy a GPU, and the secondary market for inference hardware is thin enough that we would take a meaningful haircut if our needs shifted. The scheduler work, by contrast, is entirely reversible. If we build it and it does not deliver the throughput gains we expect, we have lost engineering time but not capital, and we have not locked ourselves into a particular hardware topology. In a world where model architectures are still evolving rapidly and our product requirements are still being shaped by customer feedback, I find that asymmetry genuinely persuasive.\n\nI want to be honest about my uncertainty here. I am not certain that the scheduler work will get us to the throughput we need. I am not certain that our current utilisation figures are not being depressed by something structural in the workloads themselves that no amount of clever batching will fix. I am not certain that the latency we are seeing is not, in some cases, genuinely a function of the hardware being at its physical limit for the specific operations we are running. These are real possibilities, and if they turn out to be true, the recommendation below is wrong.\n\nMy recommendation is this: we commit to a focused six-week effort to improve utilisation of our existing fleet, with a clear and measurable throughput target. We build the batching and scheduling layer, we instrument the workloads properly, and we see what the numbers look like when the machines are actually being asked to do what they are capable of. If, at the end of that effort, we are still queueing and still missing latency targets, we buy hardware. But we buy it knowing exactly what we need and exactly why, rather than buying it to relieve a symptom we have not yet diagnosed.\n\nI would rather spend six weeks being wrong about the diagnosis than six months being wrong about the prescription.", "output_characters": 4501, "output_sha256": "df052f401e028269faa567cc85b85ca1d9b6ad16bec291a9fd19185d2453c8f1", "prompt_tokens": 157, "reasoning_characters": 946, "reasoning_sha256": "83a3d9caff94776d53fdb4960f78b8933034a3606ef76462023470a0c906ce10", "response_tail_seconds": 0.062914, "run": 5, "stream_done_marker": true, "stream_sha256": "c8020ae5f6e874abc0ce391c7e71190a8f9d1bd22820ee41c10f63a0c645c549", "time_to_first_visible_seconds": 3.353265, "time_to_last_output_seconds": 23.479838, "ttft_seconds": 0.206857, "wall_seconds": 23.542752 } ], "time_to_last_output_seconds": { "maximum": 81.659, "median": 28.006, "minimum": 23.48, "p90": 81.659 }, "ttft_seconds": { "maximum": 0.22, "median": 0.209, "minimum": 0.203, "p90": 0.22 }, "wall_seconds": { "maximum": 81.66, "median": 28.006, "minimum": 23.543, "p90": 81.66 } }, "structured": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 141.071, "median": 138.223, "minimum": 131.601, "p90": 141.071 }, "pooled_decode_tokens_per_second": 137.674, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 600, "completion_validation": { "valid": true }, "decode_seconds": 4.551626, "decode_tokens_per_second": 131.601, "finish_reason": "stop", "measured_sse_events": 70, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 143, "reasoning_characters": 174, "reasoning_sha256": "5300080e924f84eefba6784b86f3f397c95ad22915fb49d597ef9e1add49fe07", "response_tail_seconds": 0.000108, "run": 1, "stream_done_marker": true, "stream_sha256": "bd0e99de880e569393bbfd256cc388b7ca4d21ed4c7e5c42bacbffb53bdea7d8", "time_to_first_visible_seconds": 0.810118, "time_to_last_output_seconds": 4.74355, "ttft_seconds": 0.191924, "wall_seconds": 4.743658 }, { "cached_prompt_tokens": 0, "completion_tokens": 608, "completion_validation": { "valid": true }, "decode_seconds": 4.391466, "decode_tokens_per_second": 138.223, "finish_reason": "stop", "measured_sse_events": 69, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 147, "reasoning_characters": 212, "reasoning_sha256": "c273f99bcf06b315d7f047ed2511e4c2fdc705cd0e326f0f175f931d3e6c842e", "response_tail_seconds": 7.2e-05, "run": 2, "stream_done_marker": true, "stream_sha256": "3c52692c22e10b31ec595a43f2f49363604ba7d93b5130b114ab728a48f119b8", "time_to_first_visible_seconds": 0.963537, "time_to_last_output_seconds": 4.586835, "ttft_seconds": 0.195369, "wall_seconds": 4.586907 }, { "cached_prompt_tokens": 0, "completion_tokens": 598, "completion_validation": { "valid": true }, "decode_seconds": 4.231913, "decode_tokens_per_second": 141.071, "finish_reason": "stop", "measured_sse_events": 65, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 146, "reasoning_characters": 161, "reasoning_sha256": "94a5c7a5e1cb334d5cb480d8394fc46aa3a4a6c3d2e6f6a403657c33ea5a1c10", "response_tail_seconds": 8e-05, "run": 3, "stream_done_marker": true, "stream_sha256": "511ce7ba517a2ffa477c1f2f45db60aff5823691a0b72eb54101ade3b9936ca4", "time_to_first_visible_seconds": 0.800529, "time_to_last_output_seconds": 4.440057, "ttft_seconds": 0.208144, "wall_seconds": 4.440137 }, { "cached_prompt_tokens": 0, "completion_tokens": 598, "completion_validation": { "valid": true }, "decode_seconds": 4.274745, "decode_tokens_per_second": 139.657, "finish_reason": "stop", "measured_sse_events": 68, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 146, "reasoning_characters": 161, "reasoning_sha256": "94a5c7a5e1cb334d5cb480d8394fc46aa3a4a6c3d2e6f6a403657c33ea5a1c10", "response_tail_seconds": 7.6e-05, "run": 4, "stream_done_marker": true, "stream_sha256": "511ce7ba517a2ffa477c1f2f45db60aff5823691a0b72eb54101ade3b9936ca4", "time_to_first_visible_seconds": 0.768009, "time_to_last_output_seconds": 4.483004, "ttft_seconds": 0.208259, "wall_seconds": 4.48308 }, { "cached_prompt_tokens": 0, "completion_tokens": 608, "completion_validation": { "valid": true }, "decode_seconds": 4.391681, "decode_tokens_per_second": 138.216, "finish_reason": "stop", "measured_sse_events": 70, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 146, "reasoning_characters": 212, "reasoning_sha256": "c273f99bcf06b315d7f047ed2511e4c2fdc705cd0e326f0f175f931d3e6c842e", "response_tail_seconds": 6.8e-05, "run": 5, "stream_done_marker": true, "stream_sha256": "3c52692c22e10b31ec595a43f2f49363604ba7d93b5130b114ab728a48f119b8", "time_to_first_visible_seconds": 0.990448, "time_to_last_output_seconds": 4.608871, "ttft_seconds": 0.21719, "wall_seconds": 4.608939 } ], "time_to_last_output_seconds": { "maximum": 4.744, "median": 4.587, "minimum": 4.44, "p90": 4.744 }, "ttft_seconds": { "maximum": 0.217, "median": 0.208, "minimum": 0.192, "p90": 0.217 }, "wall_seconds": { "maximum": 4.744, "median": 4.587, "minimum": 4.44, "p90": 4.744 } } }, "prefill": { "32768": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2517.67, "median": 2514.838, "minimum": 2499.846, "p90": 2517.67 }, "runs": [ { "cache_salt_sha256": "5088acf4221f85c4360881592892d7539095966cd0c2fe448867aa819c91d408", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.084184, "decode_tokens_per_second": 83.151, "effective_prefill_tokens_per_second": 2499.846, "finish_reason": "length", "measured_sse_events": 2, "output": " evidence, constraints, and the next action", "output_characters": 43, "output_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000114, "run": 1, "stream_done_marker": true, "stream_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "time_to_first_visible_seconds": 13.108009, "time_to_last_output_seconds": 13.192193, "ttft_seconds": 13.108009, "wall_seconds": 13.192307 }, { "cache_salt_sha256": "a12c950f6b95ffb19f753b9aa8cbff10ffb10344e1f67172ebae3c15edcfbb46", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.084641, "decode_tokens_per_second": 82.703, "effective_prefill_tokens_per_second": 2517.67, "finish_reason": "length", "measured_sse_events": 2, "output": " and the next action. The repository", "output_characters": 36, "output_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 9.9e-05, "run": 2, "stream_done_marker": true, "stream_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "time_to_first_visible_seconds": 13.015209, "time_to_last_output_seconds": 13.09985, "ttft_seconds": 13.015209, "wall_seconds": 13.099948 }, { "cache_salt_sha256": "8583f5c85e99bd7d0cf0e49d5298d8fccdafbfec742d28085197dbe8911640ce", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.085362, "decode_tokens_per_second": 82.004, "effective_prefill_tokens_per_second": 2514.838, "finish_reason": "length", "measured_sse_events": 2, "output": ", constraints, and the next action.", "output_characters": 35, "output_sha256": "63fb378cf97093f800d90fa6fb06b86f90d2035af5f46c65a3a49798e320e5a2", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 8.3e-05, "run": 3, "stream_done_marker": true, "stream_sha256": "63fb378cf97093f800d90fa6fb06b86f90d2035af5f46c65a3a49798e320e5a2", "time_to_first_visible_seconds": 13.029866, "time_to_last_output_seconds": 13.115228, "ttft_seconds": 13.029866, "wall_seconds": 13.115311 } ], "ttft_seconds": { "maximum": 13.108, "median": 13.03, "minimum": 13.015, "p90": 13.108 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 2526.917, "median": 2524.717, "minimum": 2519.341, "p90": 2526.917 }, "runs": [ { "cache_salt_sha256": "5088acf4221f85c4360881592892d7539095966cd0c2fe448867aa819c91d408", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.084612, "decode_tokens_per_second": 82.731, "effective_prefill_tokens_per_second": 2526.917, "finish_reason": "length", "measured_sse_events": 2, "output": " evidence, constraints, and the next action", "output_characters": 43, "output_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000112, "run": 1, "stream_done_marker": true, "stream_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "time_to_first_visible_seconds": 12.967582, "time_to_last_output_seconds": 13.052194, "ttft_seconds": 12.967582, "wall_seconds": 13.052306 }, { "cache_salt_sha256": "a12c950f6b95ffb19f753b9aa8cbff10ffb10344e1f67172ebae3c15edcfbb46", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.083533, "decode_tokens_per_second": 83.799, "effective_prefill_tokens_per_second": 2519.341, "finish_reason": "length", "measured_sse_events": 2, "output": " and the next action. The repository", "output_characters": 36, "output_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 8.8e-05, "run": 2, "stream_done_marker": true, "stream_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "time_to_first_visible_seconds": 13.006578, "time_to_last_output_seconds": 13.090111, "ttft_seconds": 13.006578, "wall_seconds": 13.090199 }, { "cache_salt_sha256": "8583f5c85e99bd7d0cf0e49d5298d8fccdafbfec742d28085197dbe8911640ce", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.084987, "decode_tokens_per_second": 82.365, "effective_prefill_tokens_per_second": 2524.717, "finish_reason": "length", "measured_sse_events": 2, "output": ", constraints, and the next action.", "output_characters": 35, "output_sha256": "63fb378cf97093f800d90fa6fb06b86f90d2035af5f46c65a3a49798e320e5a2", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 9.5e-05, "run": 3, "stream_done_marker": true, "stream_sha256": "63fb378cf97093f800d90fa6fb06b86f90d2035af5f46c65a3a49798e320e5a2", "time_to_first_visible_seconds": 12.978879, "time_to_last_output_seconds": 13.063866, "ttft_seconds": 12.978879, "wall_seconds": 13.063961 } ], "ttft_seconds": { "maximum": 13.007, "median": 12.979, "minimum": 12.968, "p90": 13.007 } } }, "65536": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2430.18, "median": 2429.38, "minimum": 2427.01, "p90": 2430.18 }, "runs": [ { "cache_salt_sha256": "b29ab144facd1a11d2d11c26a761db203c4b4bced105367f815856731896922b", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.102082, "decode_tokens_per_second": 68.572, "effective_prefill_tokens_per_second": 2430.18, "finish_reason": "length", "measured_sse_events": 2, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000127, "run": 1, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 26.967547, "time_to_last_output_seconds": 27.069629, "ttft_seconds": 26.967547, "wall_seconds": 27.069755 }, { "cache_salt_sha256": "2063f1d2a698b442df24552a625af80852ce77a9c8f5d05c7a4f143422a45fc5", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.101932, "decode_tokens_per_second": 68.673, "effective_prefill_tokens_per_second": 2429.38, "finish_reason": "length", "measured_sse_events": 2, "output": " records behaviour, evidence, constraints, and", "output_characters": 46, "output_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 8e-05, "run": 2, "stream_done_marker": true, "stream_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "time_to_first_visible_seconds": 26.976427, "time_to_last_output_seconds": 27.078359, "ttft_seconds": 26.976427, "wall_seconds": 27.078439 }, { "cache_salt_sha256": "756c78147baecbfa188a599c7ef26207f424df06f22047f471ee681edfba477e", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.104506, "decode_tokens_per_second": 66.982, "effective_prefill_tokens_per_second": 2427.01, "finish_reason": "length", "measured_sse_events": 2, "output": " records behaviour, evidence, constraints, and", "output_characters": 46, "output_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000116, "run": 3, "stream_done_marker": true, "stream_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "time_to_first_visible_seconds": 27.00277, "time_to_last_output_seconds": 27.107276, "ttft_seconds": 27.00277, "wall_seconds": 27.107391 } ], "ttft_seconds": { "maximum": 27.003, "median": 26.976, "minimum": 26.968, "p90": 27.003 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 2432.872, "median": 2431.714, "minimum": 2428.909, "p90": 2432.872 }, "runs": [ { "cache_salt_sha256": "b29ab144facd1a11d2d11c26a761db203c4b4bced105367f815856731896922b", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.102128, "decode_tokens_per_second": 68.541, "effective_prefill_tokens_per_second": 2432.872, "finish_reason": "length", "measured_sse_events": 2, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000112, "run": 1, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 26.937707, "time_to_last_output_seconds": 27.039835, "ttft_seconds": 26.937707, "wall_seconds": 27.039947 }, { "cache_salt_sha256": "2063f1d2a698b442df24552a625af80852ce77a9c8f5d05c7a4f143422a45fc5", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.102317, "decode_tokens_per_second": 68.415, "effective_prefill_tokens_per_second": 2428.909, "finish_reason": "length", "measured_sse_events": 2, "output": " records behaviour, evidence, constraints, and", "output_characters": 46, "output_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000118, "run": 2, "stream_done_marker": true, "stream_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "time_to_first_visible_seconds": 26.981665, "time_to_last_output_seconds": 27.083981, "ttft_seconds": 26.981665, "wall_seconds": 27.0841 }, { "cache_salt_sha256": "756c78147baecbfa188a599c7ef26207f424df06f22047f471ee681edfba477e", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.101068, "decode_tokens_per_second": 69.26, "effective_prefill_tokens_per_second": 2431.714, "finish_reason": "length", "measured_sse_events": 2, "output": " records behaviour, evidence, constraints, and", "output_characters": 46, "output_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000102, "run": 3, "stream_done_marker": true, "stream_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "time_to_first_visible_seconds": 26.950539, "time_to_last_output_seconds": 27.051607, "ttft_seconds": 26.950539, "wall_seconds": 27.051709 } ], "ttft_seconds": { "maximum": 26.982, "median": 26.951, "minimum": 26.938, "p90": 26.982 } } }, "8192": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2597.486, "median": 2585.806, "minimum": 2584.91, "p90": 2597.486 }, "runs": [ { "cache_salt_sha256": "839e9ae8bf8eac90108c286b34ac400088d4e31e03acc2f36d560afb87a476ed", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.076847, "decode_tokens_per_second": 91.09, "effective_prefill_tokens_per_second": 2597.486, "finish_reason": "length", "measured_sse_events": 2, "output": " contains a service, tests, documentation,", "output_characters": 42, "output_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 8.4e-05, "run": 1, "stream_done_marker": true, "stream_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "time_to_first_visible_seconds": 3.153819, "time_to_last_output_seconds": 3.230666, "ttft_seconds": 3.153819, "wall_seconds": 3.23075 }, { "cache_salt_sha256": "44669fd72d7cecf71c2772aaf9db3983772a49d2c8315ec356275f022936fb94", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.071421, "decode_tokens_per_second": 98.01, "effective_prefill_tokens_per_second": 2585.806, "finish_reason": "length", "measured_sse_events": 2, "output": " service, tests, documentation, and release", "output_characters": 43, "output_sha256": "932ced224e08d6d3d62621f7f4231ec6916b3b172b0389fdf7398eeda88b7538", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 7.9e-05, "run": 2, "stream_done_marker": true, "stream_sha256": "932ced224e08d6d3d62621f7f4231ec6916b3b172b0389fdf7398eeda88b7538", "time_to_first_visible_seconds": 3.168065, "time_to_last_output_seconds": 3.239486, "ttft_seconds": 3.168065, "wall_seconds": 3.239566 }, { "cache_salt_sha256": "c535b2b8926610482cc571660c4f3f264f4c06ea29a59284f76edfd671f09047", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.07199, "decode_tokens_per_second": 97.236, "effective_prefill_tokens_per_second": 2584.91, "finish_reason": "length", "measured_sse_events": 2, "output": " tests, documentation, and release notes.", "output_characters": 41, "output_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000119, "run": 3, "stream_done_marker": true, "stream_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "time_to_first_visible_seconds": 3.169162, "time_to_last_output_seconds": 3.241152, "ttft_seconds": 3.169162, "wall_seconds": 3.241271 } ], "ttft_seconds": { "maximum": 3.169, "median": 3.168, "minimum": 3.154, "p90": 3.169 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 2607.352, "median": 2600.703, "minimum": 2592.558, "p90": 2607.352 }, "runs": [ { "cache_salt_sha256": "839e9ae8bf8eac90108c286b34ac400088d4e31e03acc2f36d560afb87a476ed", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.07366, "decode_tokens_per_second": 95.031, "effective_prefill_tokens_per_second": 2607.352, "finish_reason": "length", "measured_sse_events": 2, "output": " contains a service, tests, documentation,", "output_characters": 42, "output_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 9.6e-05, "run": 1, "stream_done_marker": true, "stream_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "time_to_first_visible_seconds": 3.141885, "time_to_last_output_seconds": 3.215546, "ttft_seconds": 3.141885, "wall_seconds": 3.215642 }, { "cache_salt_sha256": "44669fd72d7cecf71c2772aaf9db3983772a49d2c8315ec356275f022936fb94", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.073552, "decode_tokens_per_second": 95.171, "effective_prefill_tokens_per_second": 2592.558, "finish_reason": "length", "measured_sse_events": 2, "output": " service, tests, documentation, and release", "output_characters": 43, "output_sha256": "932ced224e08d6d3d62621f7f4231ec6916b3b172b0389fdf7398eeda88b7538", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.00013, "run": 2, "stream_done_marker": true, "stream_sha256": "932ced224e08d6d3d62621f7f4231ec6916b3b172b0389fdf7398eeda88b7538", "time_to_first_visible_seconds": 3.159814, "time_to_last_output_seconds": 3.233366, "ttft_seconds": 3.159814, "wall_seconds": 3.233496 }, { "cache_salt_sha256": "c535b2b8926610482cc571660c4f3f264f4c06ea29a59284f76edfd671f09047", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.072558, "decode_tokens_per_second": 96.475, "effective_prefill_tokens_per_second": 2600.703, "finish_reason": "length", "measured_sse_events": 2, "output": " tests, documentation, and release notes.", "output_characters": 41, "output_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000152, "run": 3, "stream_done_marker": true, "stream_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "time_to_first_visible_seconds": 3.149917, "time_to_last_output_seconds": 3.222474, "ttft_seconds": 3.149917, "wall_seconds": 3.222626 } ], "ttft_seconds": { "maximum": 3.16, "median": 3.15, "minimum": 3.142, "p90": 3.16 } } } }, "protocol": { "prompts_sha256": "0c3ac4015813af6a1b967659bd2289990fde895a5e43ae065b894d1e1f909ecb", "prompts_version": "1.0.0", "repository_dirty": false, "repository_revision": "770c3106729d5b73543afa2d9b498cebda6456f3", "repository_source": "git-worktree", "repository_source_sha256": "f5503bd3fe66d96d42992fedb6cf0b47f1e7ba560359d455a6e8fc1c471261d7", "version": "1.3.0" }, "run": { "appliance": { "competing_traffic": "none", "context_limit": 262144, "drafter": "checkpoint MTP head (Mia recipe DRAFTS=1 default)", "hardware": "1x NVIDIA DGX Spark (GB10, 128 GB unified memory)", "kv_cache_dtype": "fp8 (Mia recipe default)", "model": "AxonZeta/Qwen3.8-Flash-Next-AZ-Focus-INT4", "model_revision": "5d2f89941ae201fa3458515a6ab06dd51c25e60b", "negotiated_link_speed": "not applicable", "quantisation": "INT4 AutoRound (GPTQ int4 routed experts), FP8 n-gram table", "scheduler": "Mia recipe defaults (--parallel 8, shared KV pool) with VISION=0 and MAX_TOKENS=131072 in .env", "serving_engine": "TensorFold Zig engine (tensorfold-native) db281878 + Mia Qwen3.8-Flash-Next single-Spark kit (commit 364f21a)", "serving_image": "ghcr.io/miaai-lab/qwen3.8-flash-next-single-dgx-spark-tensorfold:zig-db28187-4144696f21e3 (image id sha256:12f9c5e0ce63f94aa7111736cedfcf4336195473ce0f917b2ad412880414005e)", "topology": "single Spark, TP1" }, "comparison_id": "q38-int4-1spark-low", "finished_at": "2026-10-14T17:18:43.093603+00:00", "label": "q38-int4-ours-low", "model": "Qwen3.8-Flash-Next", "platform": "Linux-6.17.0-1026-nvidia-aarch64-with-glibc2.39", "python": "3.12.3", "rigmark_release_check": { "ahead_by": 5, "behind_by": 0, "dirty": false, "prerelease": true, "release_url": "https://github.com/alexellis/rigmark/releases/tag/v0.0.4", "revision": "770c3106729d5b73543afa2d9b498cebda6456f3", "status": "ahead", "tag": "v0.0.4" }, "started_at": "2026-10-14T17:08:19.116127+00:00" }, "schema": 1, "settings": { "concurrency": [ 1, 2, 4 ], "concurrency_runs": 3, "concurrency_tokens": 256, "concurrency_workload": "code", "decode_tokens": 8192, "extra_body": { "chat_template_kwargs": { "reasoning_effort": "low" } }, "prefill_cache_isolation": "random-salt-per-pair", "prefill_depths": [ 8192, 32768, 65536 ], "prefill_runs": 3, "runs": 5, "seed": 20260905, "temperature": 0.0, "top_p": 1.0 } }