{ "concurrency": { "1": { "aggregate_end_to_end_tokens_per_second": { "maximum": 60.923, "median": 60.329, "minimum": 58.84, "p90": 60.923 }, "per_stream_decode_tokens_per_second": { "maximum": 63.77, "median": 63.207, "minimum": 61.513, "p90": 63.77 }, "per_stream_ttft_seconds": { "maximum": 0.208, "median": 0.205, "minimum": 0.203, "p90": 0.208 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 60.329, "completion_tokens": 256, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.034358, "decode_tokens_per_second": 63.207, "finish_reason": "length", "measured_sse_events": 98, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts", "output_characters": 105, "output_sha256": "7049c0348ae4422757cb7277c4cf41cc3244b22c611480b0b8606c2ea8a084f8", "prompt_tokens": 169, "reasoning_characters": 909, "reasoning_sha256": "3b779b46c73b41f13f398ea45ab6a78bc71b8e98145f07396a6dfc0d63901b1d", "response_tail_seconds": 0.000172, "stream_done_marker": true, "stream_sha256": "405fd9086aff649c9361cdf0ea6ca74e8141637aac023e1c5dbc485458457432", "time_to_first_visible_seconds": 3.837271, "time_to_last_output_seconds": 4.242515, "ttft_seconds": 0.208157, "wall_seconds": 4.242688 } ], "wall_seconds": 4.243377 }, { "aggregate_end_to_end_tokens_per_second": 60.923, "completion_tokens": 256, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 3.998749, "decode_tokens_per_second": 63.77, "finish_reason": "length", "measured_sse_events": 97, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n//", "output_characters": 88, "output_sha256": "baa8653f87efd02cc90685b182a922e952b0f7a9b63b3c9873fe45b06cdd5846", "prompt_tokens": 169, "reasoning_characters": 918, "reasoning_sha256": "d0c3b55c500b1a6c79f3efcb568a4054ba0156e5e2edae0bf94d81bcb94d5c75", "response_tail_seconds": 0.000151, "stream_done_marker": true, "stream_sha256": "8cd8aa415538a8c0cb58b397e742e5885dbd88ca44e518820bc3a7268c3359da", "time_to_first_visible_seconds": 3.888785, "time_to_last_output_seconds": 4.201324, "ttft_seconds": 0.202575, "wall_seconds": 4.201475 } ], "wall_seconds": 4.202048 }, { "aggregate_end_to_end_tokens_per_second": 58.84, "completion_tokens": 256, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.145449, "decode_tokens_per_second": 61.513, "finish_reason": "length", "measured_sse_events": 102, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n", "output_characters": 80, "output_sha256": "bc4834114aef06133250aec0f3dd5184a1c9ce9d7133c9218575e242978d4abc", "prompt_tokens": 168, "reasoning_characters": 932, "reasoning_sha256": "a1eafaa99deb042129020fe679e547477c1ab5376028a4cda58d45d893376e6b", "response_tail_seconds": 0.000159, "stream_done_marker": true, "stream_sha256": "ca4752e0de95d37de336644f48ccc2140c7f93384f9f1fbb27b564405a732f7a", "time_to_first_visible_seconds": 3.98401, "time_to_last_output_seconds": 4.350084, "ttft_seconds": 0.204635, "wall_seconds": 4.350243 } ], "wall_seconds": 4.350785 } ] }, "2": { "aggregate_end_to_end_tokens_per_second": { "maximum": 95.073, "median": 93.544, "minimum": 89.827, "p90": 95.073 }, "per_stream_decode_tokens_per_second": { "maximum": 53.711, "median": 49.514, "minimum": 48.722, "p90": 50.956 }, "per_stream_ttft_seconds": { "maximum": 0.476, "median": 0.337, "minimum": 0.203, "p90": 0.468 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 89.827, "completion_tokens": 512, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.233782, "decode_tokens_per_second": 48.722, "finish_reason": "length", "measured_sse_events": 102, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n", "output_characters": 76, "output_sha256": "a72356efd2b5fe11b5a1dcf4b06b7f8f49005b822d0bab39389426085e227c29", "prompt_tokens": 169, "reasoning_characters": 916, "reasoning_sha256": "d14b5ae8fbb03f4ed561132a855b3a9792218ba4415771440059f448ea543e1d", "response_tail_seconds": 6.8e-05, "stream_done_marker": true, "stream_sha256": "1a3d8a719d84f277cca736d26580035f8f6925fd9186060f99f4ed621f410cfc", "time_to_first_visible_seconds": 5.391243, "time_to_last_output_seconds": 5.698445, "ttft_seconds": 0.464663, "wall_seconds": 5.698513 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.187349, "decode_tokens_per_second": 49.158, "finish_reason": "length", "measured_sse_events": 97, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for deterministic testing.\ntype Clock", "output_characters": 158, "output_sha256": "b6e9cdd03c43cf5fc1c42ab3d6802ee65ba0322a090d3f0b8cd29f1660c7c468", "prompt_tokens": 168, "reasoning_characters": 849, "reasoning_sha256": "e44b5494d654b6fde548c82494cf747ab4dc0fb13c6bbe29efe5cdbc595949f1", "response_tail_seconds": 0.000149, "stream_done_marker": true, "stream_sha256": "1ef5afb2847d85c990caa236963ef0430f5d244c2edadfe3b5447ebd3639d775", "time_to_first_visible_seconds": 4.588567, "time_to_last_output_seconds": 5.39068, "ttft_seconds": 0.203331, "wall_seconds": 5.390829 } ], "wall_seconds": 5.699837 }, { "aggregate_end_to_end_tokens_per_second": 95.073, "completion_tokens": 512, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.747663, "decode_tokens_per_second": 53.711, "finish_reason": "length", "measured_sse_events": 97, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source", "output_characters": 124, "output_sha256": "2bd2fcbaf4e737b51237af09d1eca3a3ccc2c5e2f66cbd14a4a89b4bdf202cf0", "prompt_tokens": 168, "reasoning_characters": 866, "reasoning_sha256": "c0c4aab6fb3f8ca041f2f126a8cdd7e9b9cdebf5410dc68693cccb9d9e50fb31", "response_tail_seconds": 0.0002, "stream_done_marker": true, "stream_sha256": "628fb8590ba895d87a984da999635227c195b4a3a49af2d79438e7a6e855d67a", "time_to_first_visible_seconds": 4.674116, "time_to_last_output_seconds": 5.224139, "ttft_seconds": 0.476476, "wall_seconds": 5.22434 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.175505, "decode_tokens_per_second": 49.271, "finish_reason": "length", "measured_sse_events": 101, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n", "output_characters": 83, "output_sha256": "914e40317914fcb8d2942dcc7127fc815ea3e562b40373c725968d518383e402", "prompt_tokens": 166, "reasoning_characters": 929, "reasoning_sha256": "0b65487f777f0f963a53d8bf2be05c3463ec7c2ce0f3ddc7e5dd64f3adc86e95", "response_tail_seconds": 0.000152, "stream_done_marker": true, "stream_sha256": "de95ac457bd93be934700ab204e2e34b19a37c4591e0264c156c63f1450ae12a", "time_to_first_visible_seconds": 4.913892, "time_to_last_output_seconds": 5.384419, "ttft_seconds": 0.208914, "wall_seconds": 5.384572 } ], "wall_seconds": 5.385327 }, { "aggregate_end_to_end_tokens_per_second": 93.544, "completion_tokens": 512, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.12479, "decode_tokens_per_second": 49.758, "finish_reason": "length", "measured_sse_events": 101, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n//", "output_characters": 89, "output_sha256": "eb11efbf482d70343d379b4b442ccfcdd22111b105fd150bc4cfe4e8c984c60d", "prompt_tokens": 166, "reasoning_characters": 894, "reasoning_sha256": "488805af15670e9d3d830f96dcbd2af1b5967494af2c64da75ba02c48104c09e", "response_tail_seconds": 0.000226, "stream_done_marker": true, "stream_sha256": "95be9ed7b32bd092396c6f8c226946bae7f824b587c29dff22f7bac423275845", "time_to_first_visible_seconds": 4.834632, "time_to_last_output_seconds": 5.330647, "ttft_seconds": 0.205857, "wall_seconds": 5.330873 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.004294, "decode_tokens_per_second": 50.956, "finish_reason": "length", "measured_sse_events": 103, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts", "output_characters": 101, "output_sha256": "5eb73dfb21c60d4970601bc043b32739645ada3c7229d08ab577e951bc225b66", "prompt_tokens": 170, "reasoning_characters": 906, "reasoning_sha256": "7f6bc87e561e4f29154eb2dff56d36c44b03ab3e7cef09bca17610dd4f0fb124", "response_tail_seconds": 0.000121, "stream_done_marker": true, "stream_sha256": "53eb7e45c21e65496c0c6900f4676a75e81072e1e1e474dfa03fad8b8742e38c", "time_to_first_visible_seconds": 4.898834, "time_to_last_output_seconds": 5.472272, "ttft_seconds": 0.467978, "wall_seconds": 5.472393 } ], "wall_seconds": 5.473349 } ] }, "4": { "aggregate_end_to_end_tokens_per_second": { "maximum": 154.693, "median": 150.229, "minimum": 148.631, "p90": 154.693 }, "per_stream_decode_tokens_per_second": { "maximum": 43.832, "median": 41.594, "minimum": 39.653, "p90": 43.558 }, "per_stream_ttft_seconds": { "maximum": 0.606, "median": 0.584, "minimum": 0.205, "p90": 0.605 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 150.229, "completion_tokens": 1024, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.074676, "decode_tokens_per_second": 41.978, "finish_reason": "length", "measured_sse_events": 104, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts", "output_characters": 105, "output_sha256": "7049c0348ae4422757cb7277c4cf41cc3244b22c611480b0b8606c2ea8a084f8", "prompt_tokens": 171, "reasoning_characters": 883, "reasoning_sha256": "545e3beef54593abf726bf9d5e065327f0fa8725268a0fdd88022c1f44246209", "response_tail_seconds": 0.000127, "stream_done_marker": true, "stream_sha256": "1f9d4de4ac1f4fa4ceb83cfab814b5f712927caa93078b1013648083903cbb48", "time_to_first_visible_seconds": 5.96367, "time_to_last_output_seconds": 6.655123, "ttft_seconds": 0.580447, "wall_seconds": 6.65525 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.187728, "decode_tokens_per_second": 41.211, "finish_reason": "length", "measured_sse_events": 105, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock provides the current time", "output_characters": 120, "output_sha256": "5e1f24f399d7009ed70fcea546071495a67ee5dada7b028a83e485a1986fba37", "prompt_tokens": 167, "reasoning_characters": 877, "reasoning_sha256": "cf0c8fd86ba3e255b86627c85e909a630069351989a5d162dfa7efb5222ded45", "response_tail_seconds": 0.000156, "stream_done_marker": true, "stream_sha256": "0bfe2fdb4afb82cad2fce872a44689bba3c996888fb03c3396e01ad2120139b8", "time_to_first_visible_seconds": 6.179213, "time_to_last_output_seconds": 6.768695, "ttft_seconds": 0.580968, "wall_seconds": 6.768852 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.233506, "decode_tokens_per_second": 40.908, "finish_reason": "length", "measured_sse_events": 106, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n//", "output_characters": 85, "output_sha256": "3263ba9335640f8a93d919821a765ebd70ac9657c67459199c266d01592a5e41", "prompt_tokens": 169, "reasoning_characters": 935, "reasoning_sha256": "412dbbf810a94ca674381b5e0aac87e356a13cd0a5fafcf3145d855abea9bab4", "response_tail_seconds": 0.000134, "stream_done_marker": true, "stream_sha256": "d8438326edc677fbc1d70a4ec2a0728b0bfb33e24c550072bd59c9b8da1820fe", "time_to_first_visible_seconds": 6.244431, "time_to_last_output_seconds": 6.814509, "ttft_seconds": 0.581003, "wall_seconds": 6.814643 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.259972, "decode_tokens_per_second": 40.735, "finish_reason": "length", "measured_sse_events": 102, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for test", "output_characters": 119, "output_sha256": "aa05514addfa14a02ffcec1b68194ad0c6f7752ef32e2e54d41e041d3384ee5f", "prompt_tokens": 171, "reasoning_characters": 887, "reasoning_sha256": "144d7051d18b832f9894713a0e6a6dd6d32c483d2d02a7e515cbe5262048b831", "response_tail_seconds": 0.000168, "stream_done_marker": true, "stream_sha256": "dbf9508af0fe666fef55a64f892891f684b00227113e53db410660781518028b", "time_to_first_visible_seconds": 5.631868, "time_to_last_output_seconds": 6.464745, "ttft_seconds": 0.204772, "wall_seconds": 6.464913 } ], "wall_seconds": 6.816282 }, { "aggregate_end_to_end_tokens_per_second": 148.631, "completion_tokens": 1024, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.854304, "decode_tokens_per_second": 43.558, "finish_reason": "length", "measured_sse_events": 101, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for testability.\ntype Clock interface {\n", "output_characters": 151, "output_sha256": "e5cbebb02772ec8ce4c3fca0b50cb2a8d8326584a49f42db6271f804c8ed4439", "prompt_tokens": 169, "reasoning_characters": 852, "reasoning_sha256": "6149c98aa05df2717cb0d006f34e9b52a7dde837eaaf9b3a1cb9708d61293c31", "response_tail_seconds": 0.000281, "stream_done_marker": true, "stream_sha256": "4e1cf5e138224227760f0403137444fa4f41b71e90f0e3d80af98e6177984052", "time_to_first_visible_seconds": 5.473092, "time_to_last_output_seconds": 6.440732, "ttft_seconds": 0.586427, "wall_seconds": 6.441013 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.299668, "decode_tokens_per_second": 40.478, "finish_reason": "length", "measured_sse_events": 108, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time", "output_characters": 78, "output_sha256": "bce1b80a5668fd2173cbdf76a2ed5de3d4f612d6bb5f91ffca73c11748dc605d", "prompt_tokens": 167, "reasoning_characters": 947, "reasoning_sha256": "6395f7a1d11e9b9be08ba7d1883ea6a159341908451bd691e17f06fdfee097aa", "response_tail_seconds": 0.000113, "stream_done_marker": true, "stream_sha256": "7d3386cf6a1b792d9fa67f0acf891a9dc75e03b2f44d3dde0f64a0b676edbd10", "time_to_first_visible_seconds": 6.440955, "time_to_last_output_seconds": 6.886846, "ttft_seconds": 0.587178, "wall_seconds": 6.886959 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.05429, "decode_tokens_per_second": 42.119, "finish_reason": "length", "measured_sse_events": 103, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time", "output_characters": 117, "output_sha256": "f155585e16cc1ff3d4f576d21fb8032a2e2816293d8b010d6de4141cd2cc7c2b", "prompt_tokens": 169, "reasoning_characters": 887, "reasoning_sha256": "d6354582cde72d99da5de4c84e69b958be27f8f6ec510ad16f1ca2230c60cdcf", "response_tail_seconds": 0.000118, "stream_done_marker": true, "stream_sha256": "e9cdc9ad9daf314789cd5c4bb10d0d201c49d551c128ebc57c50aefc5b91e194", "time_to_first_visible_seconds": 5.88486, "time_to_last_output_seconds": 6.641884, "ttft_seconds": 0.587594, "wall_seconds": 6.642002 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.430793, "decode_tokens_per_second": 39.653, "finish_reason": "length", "measured_sse_events": 105, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)", "output_characters": 85, "output_sha256": "462c23d10fb0eb7b8ab9e8d5851ef03095c3f158144795f53bc25755ebd3f71f", "prompt_tokens": 166, "reasoning_characters": 916, "reasoning_sha256": "abddc5454d68e0de13d41780bf679bdbf8fc7a3da62c3ea72880461ad7c4a501", "response_tail_seconds": 0.000138, "stream_done_marker": true, "stream_sha256": "9d212013d4066a77b2cdc86b1faab7dc75036f6b310409c1a72420295e09332d", "time_to_first_visible_seconds": 6.151219, "time_to_last_output_seconds": 6.641654, "ttft_seconds": 0.210861, "wall_seconds": 6.641792 } ], "wall_seconds": 6.88955 }, { "aggregate_end_to_end_tokens_per_second": 154.693, "completion_tokens": 1024, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.211224, "decode_tokens_per_second": 41.055, "finish_reason": "length", "measured_sse_events": 104, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"", "output_characters": 82, "output_sha256": "b40619821a63fffb9dfe597790aec6214d24b20133a664331300bb774f511f7d", "prompt_tokens": 166, "reasoning_characters": 929, "reasoning_sha256": "834df41ce3b14085c5641791fc13446a4dcd35e5b43a9303e1392622a59d9987", "response_tail_seconds": 0.00019, "stream_done_marker": true, "stream_sha256": "f766882f5f51a9d4312f774d210576676c4edc88a31ea91fd82a08362457dcdd", "time_to_first_visible_seconds": 5.954329, "time_to_last_output_seconds": 6.422453, "ttft_seconds": 0.211229, "wall_seconds": 6.422643 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.012383, "decode_tokens_per_second": 42.412, "finish_reason": "length", "measured_sse_events": 106, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time", "output_characters": 106, "output_sha256": "386d183317de96e769ab96be7cde79c4b5310bc36b1fcbbf77f091d3a382220d", "prompt_tokens": 169, "reasoning_characters": 895, "reasoning_sha256": "0ee0a5a23920b6db47601b97517bb42eb682d1502ba801330c529b099e78702c", "response_tail_seconds": 0.000212, "stream_done_marker": true, "stream_sha256": "2ed2acf002135a5badb58f0b081f91bd18f89e2dead9d3c3f29a2437c108a22c", "time_to_first_visible_seconds": 6.009631, "time_to_last_output_seconds": 6.617678, "ttft_seconds": 0.605295, "wall_seconds": 6.61789 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.817666, "decode_tokens_per_second": 43.832, "finish_reason": "length", "measured_sse_events": 103, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for testability", "output_characters": 126, "output_sha256": "b649adfbd99c7a597bce8279aeaef70b68890b7ed4bc2e6014651136838aef64", "prompt_tokens": 168, "reasoning_characters": 886, "reasoning_sha256": "e43b32e0a8420d91264e16dce72e1c61d7a15d05ba2c665efd70c84224b87d15", "response_tail_seconds": 0.000193, "stream_done_marker": true, "stream_sha256": "1d3b6dad9e2a6c45797cc65769e575ce9b58e99da7a8bee341ef183588ee6fba", "time_to_first_visible_seconds": 5.65661, "time_to_last_output_seconds": 6.422616, "ttft_seconds": 0.604949, "wall_seconds": 6.422808 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.923491, "decode_tokens_per_second": 43.049, "finish_reason": "length", "measured_sse_events": 104, "output": "```go\n// ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n", "output_characters": 86, "output_sha256": "58a068241e9dbcb29fa98e16b7ae9531693bc881ade86fa1fccd2c7e03bc0b9b", "prompt_tokens": 168, "reasoning_characters": 917, "reasoning_sha256": "b4d8111d898cb9dcf9ed9f8adf4029113b857059c159d4f685504198be902a9c", "response_tail_seconds": 0.00012, "stream_done_marker": true, "stream_sha256": "96689aaf613a13f3cee08ddfba80de05d27cfad61712c24aadb23eb65ebb3179", "time_to_first_visible_seconds": 6.0102, "time_to_last_output_seconds": 6.529321, "ttft_seconds": 0.60583, "wall_seconds": 6.529442 } ], "wall_seconds": 6.619545 } ] } }, "decode": { "code": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 92.507, "median": 91.073, "minimum": 87.227, "p90": 92.507 }, "pooled_decode_tokens_per_second": 90.626, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 1801, "completion_validation": { "valid": true }, "decode_seconds": 20.63593, "decode_tokens_per_second": 87.227, "finish_reason": "stop", "measured_sse_events": 408, "output": "```go\n// ratelimit/tokenbucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for deterministic testing.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// TokenBucket is a concurrency-safe token-bucket rate limiter.\ntype TokenBucket struct {\n\tmu sync.Mutex\n\tcapacity float64\n\ttokens float64\n\trefillRate float64 // tokens per second\n\tlastRefill time.Time\n\tclock Clock\n}\n\n// NewTokenBucket creates a TokenBucket. capacity must be > 0,\n// refillRate (tokens/sec) must be > 0, and clock must be non-nil.\nfunc NewTokenBucket(capacity, refillRate float64, clock Clock) (*TokenBucket, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be > 0\")\n\t}\n\tif refillRate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: refillRate must be > 0\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &TokenBucket{\n\t\tcapacity: capacity,\n\t\ttokens: capacity,\n\t\trefillRate: refillRate,\n\t\tlastRefill: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN reports whether n tokens are available, consuming them if so.\nfunc (tb *TokenBucket) AllowN(n int) bool {\n\tif n <= 0 {\n\t\treturn true\n\t}\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\n\ttb.refill()\n\tif tb.tokens >= float64(n) {\n\t\ttb.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (tb *TokenBucket) refill() {\n\tnow := tb.clock.Now()\n\telapsed := now.Sub(tb.lastRefill).Seconds()\n\tif elapsed <= 0 {\n\t\treturn\n\t}\n\ttb.tokens += elapsed * tb.refillRate\n\tif tb.tokens > tb.capacity {\n\t\ttb.tokens = tb.capacity\n\t}\n\ttb.lastRefill = now\n}\n```\n\n```go\n// ratelimit/tokenbucket_test.go\npackage ratelimit\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct{ t time.Time }\n\nfunc (c *fakeClock) Now() time.Time { return c.t }\nfunc (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) }\n\nfunc TestNewTokenBucket_Validation(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tcases := []struct {\n\t\tname string\n\t\tcap, rate float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 1, 0, fc, true},\n\t\t{\"negative rate\", 1, -1, fc, true},\n\t\t{\"nil clock\", 1, 1, nil, true},\n\t\t{\"valid\", 10, 1, fc, false},\n\t}\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\t_, err := NewTokenBucket(tc.cap, tc.rate, tc.clock)\n\t\t\tif (err != nil) != tc.wantErr {\n\t\t\t\tt.Fatalf(\"err=%v, wantErr=%v\", err, tc.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tcap, rate float64\n\t\tadvances []time.Duration\n\t\trequests []int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"burst then deny\",\n\t\t\tcap: 5, rate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 0},\n\t\t\trequests: []int{3, 3, 1},\n\t\t\twant: []bool{true, false, true},\n\t\t},\n\t\t{\n\t\t\tname: \"refill over time\",\n\t\t\tcap: 5, rate: 1,\n\t\t\tadvances: []time.Duration{0, 3 * time.Second, 0},\n\t\t\trequests: []int{5, 3, 1},\n\t\t\twant: []bool{true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"cap at capacity\",\n\t\t\tcap: 3, rate: 1,\n\t\t\tadvances: []time.Duration{0, 10 * time.Second, 0},\n\t\t\trequests: []int{3, 1},\n\t\t\twant: []bool{true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"zero n always allowed\",\n\t\t\tcap: 1, rate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\trequests: []int{0},\n\t\t\twant: []bool{true},\n\t\t},\n\t\t{\n\t\t\tname: \"fractional refill\",\n\t\t\tcap: 1, rate: 1,\n\t\t\tadvances: []time.Duration{0, 500 * time.Millisecond, 0},\n\t\t\trequests: []int{1, 1},\n\t\t\twant: []bool{true, false},\n\t\t},\n\t}\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: time.Unix(0, 0)}\n\t\t\ttb, err := NewTokenBucket(tc.cap, tc.rate, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i := range tc.requests {\n\t\t\t\tif i > 0 {\n\t\t\t\t\tfc.Advance(tc.advances[i])\n\t\t\t\t}\n\t\t\t\tgot := tb.AllowN(tc.requests[i])\n\t\t\t\tif got != tc.want[i] {\n\t\t\t\t\tt.Errorf(\"AllowN(%d)=%v, want %v\", tc.requests[i], got, tc.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN_Concurrent(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\ttb, _ := NewTokenBucket(100, 1, fc)\n\tdone := make(chan bool, 1)\n\tgo func() {\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\ttb.AllowN(1)\n\t\t}\n\t\tdone <- true\n\t}()\n\tfor i := 0; i < 1000; i++ {\n\t\ttb.AllowN(1)\n\t}\n\t<-done\n}\n```", "output_characters": 4229, "output_sha256": "09fafe175ce3150cbf3a7dba9c45fe38e9d3e7c6bdb2a3206698de7a2ffb106b", "prompt_tokens": 167, "reasoning_characters": 829, "reasoning_sha256": "391fd0cdce31d5a74c910a41676e7d6e3adb00c74e95fe039b0db88e3eaa9ba2", "response_tail_seconds": 8.4e-05, "run": 1, "stream_done_marker": true, "stream_sha256": "d5a1931098e6c3bdb3ede2467d10368b472d30818045e8631b45281e348ffb3b", "time_to_first_visible_seconds": 3.59829, "time_to_last_output_seconds": 20.887747, "ttft_seconds": 0.251817, "wall_seconds": 20.887831 }, { "cached_prompt_tokens": 0, "completion_tokens": 1925, "completion_validation": { "valid": true }, "decode_seconds": 21.125905, "decode_tokens_per_second": 91.073, "finish_reason": "stop", "measured_sse_events": 424, "output": "```go\n// ratelimit/limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock provides the current time.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// RealClock uses the system clock.\ntype RealClock struct{}\n\nfunc (RealClock) Now() time.Time { return time.Now() }\n\n// Limiter is a concurrency-safe token-bucket rate limiter.\ntype Limiter struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trefillRate float64 // tokens per second\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\n// New creates a token-bucket limiter.\n// capacity must be > 0, refillRate must be > 0, clock must be non-nil.\nfunc New(capacity, refillRate float64, clock Clock) (*Limiter, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be positive\")\n\t}\n\tif refillRate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: refillRate must be positive\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &Limiter{\n\t\tcapacity: capacity,\n\t\trefillRate: refillRate,\n\t\ttokens: capacity,\n\t\tlast: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN reports whether n tokens are available and consumes them if so.\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n <= 0 {\n\t\treturn true\n\t}\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\telapsed := now.Sub(l.last).Seconds()\n\tl.tokens += elapsed * l.refillRate\n\tif l.tokens > l.capacity {\n\t\tl.tokens = l.capacity\n\t}\n\tl.last = now\n\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```go\n// ratelimit/limiter_test.go\npackage ratelimit\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tt time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.t\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.t = c.t.Add(d)\n}\n\nfunc TestNewValidation(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tcases := []struct {\n\t\tname string\n\t\tcap float64\n\t\trate float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 1, fc, false},\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 10, 0, fc, true},\n\t\t{\"negative rate\", 10, -1, fc, true},\n\t\t{\"nil clock\", 10, 1, nil, true},\n\t}\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\t_, err := New(tc.cap, tc.rate, tc.clock)\n\t\t\tif (err != nil) != tc.wantErr {\n\t\t\t\tt.Fatalf(\"got err=%v, wantErr=%v\", err, tc.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tcap float64\n\t\trate float64\n\t\tadvances []time.Duration\n\t\trequests []int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"initial burst\",\n\t\t\tcap: 5, rate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\trequests: []int{5},\n\t\t\twant: []bool{true},\n\t\t},\n\t\t{\n\t\t\tname: \"exceed burst\",\n\t\t\tcap: 5, rate: 1,\n\t\t\tadvances: []time.Duration{0, 0},\n\t\t\trequests: []int{5, 1},\n\t\t\twant: []bool{true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"refill after wait\",\n\t\t\tcap: 5, rate: 1,\n\t\t\tadvances: []time.Duration{0, 3 * time.Second},\n\t\t\trequests: []int{5, 3},\n\t\t\twant: []bool{true, true},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\tcap: 10, rate: 2,\n\t\t\tadvances: []time.Duration{0, 1 * time.Second},\n\t\t\trequests: []int{10, 3},\n\t\t\twant: []bool{true, true},\n\t\t},\n\t\t{\n\t\t\tname: \"refill capped at capacity\",\n\t\t\tcap: 5, rate: 10,\n\t\t\tadvances: []time.Duration{0, 10 * time.Second},\n\t\t\trequests: []int{5, 6},\n\t\t\twant: []bool{true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"zero request always allowed\",\n\t\t\tcap: 1, rate: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\trequests: []int{0},\n\t\t\twant: []bool{true},\n\t\t},\n\t}\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: time.Unix(0, 0)}\n\t\t\tl, err := New(tc.cap, tc.rate, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i := range tc.requests {\n\t\t\t\tif i > 0 {\n\t\t\t\t\tfc.Advance(tc.advances[i])\n\t\t\t\t}\n\t\t\t\tgot := l.AllowN(tc.requests[i])\n\t\t\t\tif got != tc.want[i] {\n\t\t\t\t\tt.Errorf(\"step %d: got %v, want %v\", i, got, tc.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConcurrentAllowN(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tl, _ := New(100, 1, fc)\n\tvar wg sync.WaitGroup\n\tvar mu sync.Mutex\n\tallowed := 0\n\tfor i := 0; i < 200; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tif l.AllowN(1) {\n\t\t\t\tmu.Lock()\n\t\t\t\tallowed++\n\t\t\t\tmu.Unlock()\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\tif allowed != 100 {\n\t\tt.Errorf(\"expected 100 allowed, got %d\", allowed)\n\t}\n}\n```", "output_characters": 4514, "output_sha256": "a61b0e8a617c7e390ea57c0c55246c4e36411779fbd564925b1bb7253f071c33", "prompt_tokens": 169, "reasoning_characters": 887, "reasoning_sha256": "144d7051d18b832f9894713a0e6a6dd6d32c483d2d02a7e515cbe5262048b831", "response_tail_seconds": 0.00049, "run": 2, "stream_done_marker": true, "stream_sha256": "6dd02d254eb6f047dbb2f80a80aeea5de4a20374fdb90ce9ca007e98fc168795", "time_to_first_visible_seconds": 3.951772, "time_to_last_output_seconds": 21.331654, "ttft_seconds": 0.20575, "wall_seconds": 21.332144 }, { "cached_prompt_tokens": 0, "completion_tokens": 1877, "completion_validation": { "valid": true }, "decode_seconds": 20.279638, "decode_tokens_per_second": 92.507, "finish_reason": "stop", "measured_sse_events": 413, "output": "```ratelimit/bucket.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for testability.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// Bucket is a concurrency-safe token-bucket rate limiter.\ntype Bucket struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trate float64 // tokens per second\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\n// New creates a token-bucket limiter.\n// capacity must be > 0, rate must be > 0, clock must be non-nil.\nfunc New(capacity, rate float64, clock Clock) (*Bucket, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be > 0\")\n\t}\n\tif rate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: rate must be > 0\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &Bucket{\n\t\tcapacity: capacity,\n\t\trate: rate,\n\t\ttokens: capacity,\n\t\tlast: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN reports whether n tokens can be consumed.\n// n must be > 0.\nfunc (b *Bucket) AllowN(n int) bool {\n\tif n <= 0 {\n\t\treturn false\n\t}\n\tb.mu.Lock()\n\tdefer b.mu.Unlock()\n\n\tnow := b.clock.Now()\n\telapsed := now.Sub(b.last).Seconds()\n\tb.tokens += elapsed * b.rate\n\tif b.tokens > b.capacity {\n\t\tb.tokens = b.capacity\n\t}\n\tb.last = now\n\n\tif b.tokens >= float64(n) {\n\t\tb.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```ratelimit/bucket_test.go\npackage ratelimit\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tt time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.t\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.t = c.t.Add(d)\n}\n\nfunc TestNew_Validation(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\ttests := []struct {\n\t\tname string\n\t\tcap, rat float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"ok\", 10, 1, fc, false},\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 10, 0, fc, true},\n\t\t{\"negative rate\", 10, -1, fc, true},\n\t\t{\"nil clock\", 10, 1, nil, true},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := New(tt.cap, tt.rat, tt.clock)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"New(%v,%v) err=%v wantErr=%v\", tt.cap, tt.rat, err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tcap, rat float64\n\t\tadvances []time.Duration\n\t\tn int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"full bucket allows\",\n\t\t\tcap: 5, rat: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\tn: 5,\n\t\t\twant: []bool{true},\n\t\t},\n\t\t{\n\t\t\tname: \"exceeds capacity\",\n\t\t\tcap: 5, rat: 1,\n\t\t\tadvances: []time.Duration{0, 0},\n\t\t\tn: 5,\n\t\t\twant: []bool{true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"refill after time\",\n\t\t\tcap: 5, rat: 1,\n\t\t\tadvances: []time.Duration{0, 3 * time.Second},\n\t\t\tn: 3,\n\t\t\twant: []bool{true, true},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill insufficient\",\n\t\t\tcap: 5, rat: 1,\n\t\t\tadvances: []time.Duration{0, 2 * time.Second},\n\t\t\tn: 3,\n\t\t\twant: []bool{true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"capped at capacity\",\n\t\t\tcap: 5, rat: 1,\n\t\t\tadvances: []time.Duration{0, 100 * time.Second},\n\t\t\tn: 6,\n\t\t\twant: []bool{true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"n zero rejected\",\n\t\t\tcap: 5, rat: 1,\n\t\t\tadvances: []time.Duration{0},\n\t\t\tn: 0,\n\t\t\twant: []bool{false},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: time.Unix(0, 0)}\n\t\t\tb, err := New(tt.cap, tt.rat, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, d := range tt.advances {\n\t\t\t\tfc.Advance(d)\n\t\t\t\tgot := b.AllowN(tt.n)\n\t\t\t\tif got != tt.want[i] {\n\t\t\t\t\tt.Fatalf(\"step %d: AllowN(%d)=%v want %v\", i, tt.n, got, tt.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN_Concurrent(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tb, err := New(100, 1, fc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar wg sync.WaitGroup\n\tvar mu sync.Mutex\n\tallowed := 0\n\tfor i := 0; i < 200; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tif b.AllowN(1) {\n\t\t\t\tmu.Lock()\n\t\t\t\tallowed++\n\t\t\t\tmu.Unlock()\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\tif allowed != 100 {\n\t\tt.Fatalf(\"expected 100 allowed, got %d\", allowed)\n\t}\n}\n```", "output_characters": 4198, "output_sha256": "c7e8f0fd200b9fda286281f2b7a3a1a5254190b35a0a7e73e0ad1c10970f5e89", "prompt_tokens": 167, "reasoning_characters": 978, "reasoning_sha256": "3519b020d602dbe92b5e6497213b1bb7e639868672618094ea82c6e9f2c83cab", "response_tail_seconds": 0.000363, "run": 3, "stream_done_marker": true, "stream_sha256": "e612e429122f8b4782ec14564d75689a8194811d3402eb408ab7e57e6b9033ae", "time_to_first_visible_seconds": 4.1982, "time_to_last_output_seconds": 20.496062, "ttft_seconds": 0.216425, "wall_seconds": 20.496425 }, { "cached_prompt_tokens": 0, "completion_tokens": 1841, "completion_validation": { "valid": true }, "decode_seconds": 20.066743, "decode_tokens_per_second": 91.694, "finish_reason": "stop", "measured_sse_events": 418, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock abstracts time for testability.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// RealClock uses the system clock.\ntype RealClock struct{}\n\nfunc (RealClock) Now() time.Time { return time.Now() }\n\n// Limiter is a concurrency-safe token-bucket rate limiter.\ntype Limiter struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trefillRate float64 // tokens per second\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\n// New creates a Limiter. capacity must be > 0, refillRate must be > 0,\n// and clock must be non-nil.\nfunc New(capacity, refillRate float64, clock Clock) (*Limiter, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be > 0\")\n\t}\n\tif refillRate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: refillRate must be > 0\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &Limiter{\n\t\tcapacity: capacity,\n\t\trefillRate: refillRate,\n\t\ttokens: capacity,\n\t\tlast: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN reports whether n tokens are available, consuming them if so.\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n <= 0 {\n\t\treturn true\n\t}\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\tl.refill()\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (l *Limiter) refill() {\n\tnow := l.clock.Now()\n\telapsed := now.Sub(l.last).Seconds()\n\tif elapsed > 0 {\n\t\tl.tokens += elapsed * l.refillRate\n\t\tif l.tokens > l.capacity {\n\t\t\tl.tokens = l.capacity\n\t\t}\n\t\tl.last = now\n\t}\n}\n```\n\n```ratelimit/ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tt time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.t\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.t = c.t.Add(d)\n}\n\nfunc TestNewValidation(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\ttests := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 1, fc, false},\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 10, 0, fc, true},\n\t\t{\"negative rate\", 10, -1, fc, true},\n\t\t{\"nil clock\", 10, 1, nil, true},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := New(tt.capacity, tt.refillRate, tt.clock)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"New() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tcapacity float64\n\t\trefillRate float64\n\t\tadvances []time.Duration\n\t\trequests []int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"burst then deny\",\n\t\t\tcapacity: 5, refillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0, 0},\n\t\t\trequests: []int{3, 3, 1},\n\t\t\twant: []bool{true, false, true},\n\t\t},\n\t\t{\n\t\t\tname: \"refill over time\",\n\t\t\tcapacity: 5, refillRate: 1,\n\t\t\tadvances: []time.Duration{0, 3 * time.Second, 0},\n\t\t\trequests: []int{5, 3, 1},\n\t\t\twant: []bool{true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"n zero always allowed\",\n\t\t\tcapacity: 1, refillRate: 1,\n\t\t\tadvances: []time.Duration{0, 0},\n\t\t\trequests: []int{0, 1},\n\t\t\twant: []bool{true, true},\n\t\t},\n\t\t{\n\t\t\tname: \"capped at capacity\",\n\t\t\tcapacity: 3, refillRate: 10,\n\t\t\tadvances: []time.Duration{0, 10 * time.Second, 0},\n\t\t\trequests: []int{3, 1},\n\t\t\twant: []bool{true, false},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: time.Unix(0, 0)}\n\t\t\tl, err := New(tt.capacity, tt.refillRate, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i := range tt.requests {\n\t\t\t\tfc.Advance(tt.advances[i])\n\t\t\t\tgot := l.AllowN(tt.requests[i])\n\t\t\t\tif got != tt.want[i] {\n\t\t\t\t\tt.Errorf(\"AllowN(%d) = %v, want %v\", tt.requests[i], got, tt.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConcurrent(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tl, _ := New(100, 100, fc)\n\tvar wg sync.WaitGroup\n\tvar mu sync.Mutex\n\tallowed := 0\n\tfor i := 0; i < 200; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tif l.AllowN(1) {\n\t\t\t\tmu.Lock()\n\t\t\t\tallowed++\n\t\t\t\tmu.Unlock()\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\tif allowed != 100 {\n\t\tt.Errorf(\"allowed = %d, want 100\", allowed)\n\t}\n}\n```", "output_characters": 4308, "output_sha256": "f57b809238ea29c9ad32e834df221eb7417dcfca59fbad2577646c505b7215ef", "prompt_tokens": 166, "reasoning_characters": 858, "reasoning_sha256": "6a73d1f8deda9cdb6da2e30190826e9bd18642364f973175a6e1ce4873ac966f", "response_tail_seconds": 0.000457, "run": 4, "stream_done_marker": true, "stream_sha256": "5d032e37ecfb4e5904a895f0485b53f2e4870d3abe8bfc6bf2d6e43fb216dbcd", "time_to_first_visible_seconds": 3.754581, "time_to_last_output_seconds": 20.306848, "ttft_seconds": 0.240104, "wall_seconds": 20.307305 }, { "cached_prompt_tokens": 0, "completion_tokens": 1840, "completion_validation": { "valid": true }, "decode_seconds": 20.280179, "decode_tokens_per_second": 90.68, "finish_reason": "stop", "measured_sse_events": 422, "output": "```ratelimit/ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock is an injectable time source for testability.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// RealClock uses the system wall clock.\ntype RealClock struct{}\n\nfunc (RealClock) Now() time.Time { return time.Now() }\n\n// Limiter is a concurrency-safe token-bucket rate limiter.\ntype Limiter struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trate float64 // tokens per second\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\n// New creates a Limiter. capacity must be > 0, rate must be > 0, clock must be non-nil.\nfunc New(capacity, rate float64, clock Clock) (*Limiter, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be > 0\")\n\t}\n\tif rate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: rate must be > 0\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &Limiter{\n\t\tcapacity: capacity,\n\t\trate: rate,\n\t\ttokens: capacity,\n\t\tlast: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN attempts to consume n tokens. Returns true if allowed.\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n <= 0 {\n\t\treturn true\n\t}\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\telapsed := now.Sub(l.last).Seconds()\n\tl.tokens += elapsed * l.rate\n\tif l.tokens > l.capacity {\n\t\tl.tokens = l.capacity\n\t}\n\tl.last = now\n\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```ratelimit/ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tt time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.t\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.t = c.t.Add(d)\n}\n\nfunc TestNewValidation(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\ttests := []struct {\n\t\tname string\n\t\tcap, rat float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 1, fc, false},\n\t\t{\"zero capacity\", 0, 1, fc, true},\n\t\t{\"negative capacity\", -1, 1, fc, true},\n\t\t{\"zero rate\", 10, 0, fc, true},\n\t\t{\"negative rate\", 10, -1, fc, true},\n\t\t{\"nil clock\", 10, 1, nil, true},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := New(tt.cap, tt.rat, tt.clock)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"New() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tcap, rat float64\n\t\tadvances []time.Duration\n\t\tn int\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tname: \"burst then deny\",\n\t\t\tcap: 3, rat: 1,\n\t\t\tadvances: []time.Duration{0, 0, 0, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"refill after wait\",\n\t\t\tcap: 2, rat: 1,\n\t\t\tadvances: []time.Duration{0, 0, 1 * time.Second, 0},\n\t\t\tn: 1,\n\t\t\twant: []bool{true, true, true, false},\n\t\t},\n\t\t{\n\t\t\tname: \"allowN larger than capacity\",\n\t\t\tcap: 5, rat: 1,\n\t\t\tadvances: []time.Duration{0, 0},\n\t\t\tn: 6,\n\t\t\twant: []bool{false, false},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\tcap: 10, rat: 10,\n\t\t\tadvances: []time.Duration{0, 500 * time.Millisecond, 0},\n\t\t\tn: 8,\n\t\t\twant: []bool{true, true, false},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfc := &fakeClock{t: time.Unix(0, 0)}\n\t\t\tl, err := New(tt.cap, tt.rat, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, adv := range tt.advances {\n\t\t\t\tfc.Advance(adv)\n\t\t\t\tgot := l.AllowN(tt.n)\n\t\t\t\tif got != tt.want[i] {\n\t\t\t\t\tt.Errorf(\"call %d: AllowN(%d) = %v, want %v\", i, tt.n, got, tt.want[i])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowNZero(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tl, _ := New(1, 1, fc)\n\tif !l.AllowN(0) {\n\t\tt.Error(\"AllowN(0) should return true\")\n\t}\n}\n\nfunc TestConcurrentAccess(t *testing.T) {\n\tfc := &fakeClock{t: time.Unix(0, 0)}\n\tl, _ := New(100, 100, fc)\n\tvar wg sync.WaitGroup\n\tvar mu sync.Mutex\n\tallowed := 0\n\tfor i := 0; i < 200; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tif l.AllowN(1) {\n\t\t\t\tmu.Lock()\n\t\t\t\tallowed++\n\t\t\t\tmu.Unlock()\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\tif allowed > 100 {\n\t\tt.Errorf(\"allowed %d, expected <= 100\", allowed)\n\t}\n}\n```", "output_characters": 4207, "output_sha256": "6163daf61dd46bffe2c10903acb40c50207aac482639725fefc551a403488a36", "prompt_tokens": 168, "reasoning_characters": 853, "reasoning_sha256": "f7e91a3ea18b11847ee7465c5565be7c924095c1c65334971e401957dfc79500", "response_tail_seconds": 0.000343, "run": 5, "stream_done_marker": true, "stream_sha256": "2bf5fbca6bdd99d87a285b6c6fbc7a1c96628e2c09ec9b92616a040e238598b1", "time_to_first_visible_seconds": 3.737183, "time_to_last_output_seconds": 20.520041, "ttft_seconds": 0.239862, "wall_seconds": 20.520383 } ], "time_to_last_output_seconds": { "maximum": 21.332, "median": 20.52, "minimum": 20.307, "p90": 21.332 }, "ttft_seconds": { "maximum": 0.252, "median": 0.24, "minimum": 0.206, "p90": 0.252 }, "wall_seconds": { "maximum": 21.332, "median": 20.52, "minimum": 20.307, "p90": 21.332 } }, "prose": { "completion_gate": { "passed": 2, "total": 5 }, "decode_tokens_per_second": { "maximum": 98.515, "median": 82.692, "minimum": 41.278, "p90": 98.515 }, "pooled_decode_tokens_per_second": 81.979, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 1212, "completion_validation": { "valid": true }, "decode_seconds": 27.942782, "decode_tokens_per_second": 43.339, "finish_reason": "stop", "measured_sse_events": 636, "output": "**MEMO**\n\n**To:** Engineering Team\n**From:** Platform & Infrastructure\n**Re:** AI Hardware: Purchase vs. Utilisation\n**Date:** June 2025\n\n---\n\nI want to take a few minutes to lay out the thinking behind the question that has come up in three separate planning sessions: do we buy more local AI hardware, or do we squeeze more out of what we already have? This is not a trivial decision, and I do not think the answer is obvious, so I want to walk through the tradeoffs honestly before we commit to a path.\n\nLet us start with the uncomfortable truth about our current situation. We own four A100 nodes and two consumer-grade inference boxes. Utilisation across the fleet, measured over the last quarter, hovers around thirty-one percent. That number is genuinely embarrassing, and it is the single strongest argument against buying more hardware. If we cannot keep what we have warm, we have no business asking for more. The capital cost of a new node is roughly forty thousand dollars, and that is money that could fund a quarter of a contract engineer, a year of better observability tooling, or a meaningful raise for two of us. The opportunity cost of that purchase is not abstract. It is the thing we will not do because we chose to do this instead.\n\nBut I want to be fair to the other side of the argument, because there are real reasons people keep raising this question. Our latency profile is not what it should be. When the queue fills during peak inference windows, we see p99 response times drift past two seconds, and that is unacceptable for the interactive features we are shipping. A new node would flatten that queue and give us predictable latency without any software work. There is also a reliability angle. Right now, if one of our A100s drops, we lose twenty-five percent of capacity and the remaining nodes absorb the load with a degradation that is noticeable to users. A fifth node gives us a buffer that is not just about throughput but about graceful failure. These are legitimate concerns, and dismissing them because of a utilisation metric would be lazy.\n\nWhere I think the decision tilts is on reversibility. A software improvement that raises utilisation from thirty-one to fifty-five percent is something we can iterate on, roll back, or refine. We can tune batch sizes, add a smarter scheduler, implement model quantisation for the workloads that tolerate it, or introduce a tiered caching layer. Each of those changes is a small experiment. If one does not work, we revert the commit and try another. A hardware purchase is not like that. Forty thousand dollars spent on a node is spent. It depreciates. It sits in a rack whether we need it or not. It adds another machine to monitor, patch, and eventually replace. The reversibility asymmetry is real, and I think it should carry significant weight in a small team that cannot afford to lock capital into idle assets.\n\nThat said, I want to acknowledge the uncertainty honestly. We do not fully understand why utilisation is low. It could be a scheduling problem, a workload mismatch, or a sign that our current capacity is genuinely sufficient and the latency issues are a software problem masquerading as a hardware problem. If we spend three months optimising and discover that the bottleneck was never capacity, we have lost time we could have spent building product. That is a real risk, and I do not want to pretend otherwise.\n\nMy recommendation is this: we commit to a six-week utilisation improvement sprint with a hard checkpoint. We define the target\u2014fifty percent fleet utilisation, p99 latency under eight hundred milliseconds\u2014and we work the problem with the tools we have. If, at the checkpoint, we have hit the latency target through software work, we do not buy hardware. If we have improved utilisation but latency remains stubbornly high, we buy one node, not two, and we revisit the question in a quarter. This gives us the best of both paths: we avoid irreversible capital expenditure unless the evidence demands it, and we do not let a software problem hide behind a hardware purchase.\n\nI know this is not the answer everyone wants. Some of you want the hardware now, and your frustration is justified. Others think we should not spend a single dollar until we have exhausted every optimisation. Both of you are partially right. The six-week checkpoint is our compromise, and I would rather we be wrong together in a way that is easy to correct than right in a way that is expensive to undo.\n\nLet us reconvene in six weeks with data, not opinions.", "output_characters": 4552, "output_sha256": "8ff1905861c8fa44bda384c4ad26974b54e768ded6289bdb53f130e28625b985", "prompt_tokens": 157, "reasoning_characters": 1279, "reasoning_sha256": "6b78caf22f76bf57bbd106cf438c5e6eb17b6f8fc2455608038ca5db129a6715", "response_tail_seconds": 6.6e-05, "run": 1, "stream_done_marker": true, "stream_sha256": "b2067e3208dda5a8a9afe8b1316c966db2f9f943af633677505fcb1595929664", "time_to_first_visible_seconds": 5.674854, "time_to_last_output_seconds": 28.184219, "ttft_seconds": 0.241437, "wall_seconds": 28.184284 }, { "cached_prompt_tokens": 0, "completion_tokens": 1072, "completion_validation": { "valid": true }, "decode_seconds": 25.946219, "decode_tokens_per_second": 41.278, "finish_reason": "stop", "measured_sse_events": 610, "output": "**MEMO: Hardware Strategy Review**\n\n**From:** Engineering Lead, ML Platform\n**To:** Engineering Team\n**Date:** June 2024\n**Re:** Local AI Hardware \u2014 Expansion vs. Utilisation\n\n---\n\nI want to take a few minutes to lay out my thinking on the hardware question that has come up repeatedly in our last three sprint retrospectives. We have been running at roughly 87% GPU utilisation across our four A100 nodes, and the question on the table is whether we should acquire additional hardware or invest in squeezing more out of what we already own. I have been sitting with this for a couple of weeks, and I want to share where I have landed, along with the genuine uncertainty that remains.\n\nThe case for buying more hardware is not unreasonable. Our inference latency on the production summarisation service has crept above our p99 target by about forty milliseconds, and the fine-tuning queue has a backlog that stretches into the second week of every cycle. A new node would relieve both pressures immediately, and the capital cost of a single A100 node is something we could absorb without a board conversation. The temptation is real, and I understand the instinct. When you are in the middle of a sprint and the queue is growing, the fastest path to relief is to add capacity.\n\nBut I want to push back gently on the assumption that capacity is our actual constraint. When I look at the profiling data from the last month, I see something that troubles me. Our training jobs are running at roughly 62% effective compute utilisation because of data loading bottlenecks and suboptimal mixed-precision settings. Our inference serving layer is holding model weights in memory that could be shared across concurrent requests, and we are paying for redundant copies across nodes that could be consolidated. The latency regression I mentioned is not a hardware problem. It is a software problem that we have been treating as a hardware problem because the hardware solution is faster to implement than the software one.\n\nThis is where opportunity cost enters the picture in a way that I think we have been underweighting. Every engineer-hour we spend provisioning, cooling, networking, and monitoring a new node is an hour we are not spending on the data pipeline optimisations that would give us a twenty to thirty percent throughput improvement across every workload we run. That improvement compounds. It applies to the summarisation service, to the fine-tuning queue, to the experimental work that is currently deferred because we do not have headroom. A new node solves one problem. Fixing the pipeline solves all of them, and it solves them for the next three years of hardware we will own, not just the next three months.\n\nReliability deserves a mention here too. More hardware means more failure domains. We have seen that already. The node we added in March has thrown three ECC errors and required two unplanned reboots. Our incident response time on hardware issues is not great, and I do not think we have the operational maturity to absorb another node's failure modes without a meaningful impact on our on-call load. Utilisation improvements, by contrast, tend to reduce the number of moving parts we have to keep alive.\n\nNow, the honest uncertainty. I could be wrong about the software fixes. The data loading bottleneck might be harder to resolve than my profiling suggests, and the mixed-precision tuning might require more experimentation than I am estimating. There is a real possibility that we spend two sprints chasing optimisations and end up with a marginal improvement that does not move the needle. In that scenario, we will have lost time and the latency problem will have worsened. I want to be clear that I am not dismissing this risk.\n\nWhat I am confident about is reversibility. If we try the software path for two sprints and it does not work, we can buy the node then. The capital cost will be roughly the same, and the market is not going to shift dramatically in the next quarter. But if we buy the node now and then discover that the real problem was always in our code, we will have spent money that could have funded a contractor or a conference trip or a tooling investment, and we will have introduced operational complexity that is much harder to reverse than a failed optimisation branch.\n\nMy recommendation is this. We commit two sprints to a focused utilisation improvement effort, with a clear checkpoint at the end of sprint two. If we have not recovered our latency target and cleared the fine-tuning backlog by then, we buy the node without further debate. I would rather spend the money late and sure than spend it early and uncertain.\n\nI welcome disagreement on this. The people closest to the serving layer in particular may have a different read on the latency data, and I want to hear it.", "output_characters": 4833, "output_sha256": "884ca3b06c2afd677f639b25f8344e2046066a6f7d46cbfe80230c4cc2e95d38", "prompt_tokens": 161, "reasoning_characters": 463, "reasoning_sha256": "f7772fe29f01f8602edf94c5a5b71d61f2bc17c5dc93f27c857f0665f5de2b55", "response_tail_seconds": 0.000345, "run": 2, "stream_done_marker": true, "stream_sha256": "2126b63bec0e58b06c6d893d4e766a6a513738dfbc78bb65dec5d8f2b81c333e", "time_to_first_visible_seconds": 1.831202, "time_to_last_output_seconds": 26.170616, "ttft_seconds": 0.224396, "wall_seconds": 26.170961 }, { "cached_prompt_tokens": 0, "completion_tokens": 8192, "completion_validation": { "error": "no visible answer", "valid": false }, "decode_seconds": 91.494742, "decode_tokens_per_second": 89.524, "finish_reason": "length", "measured_sse_events": 1801, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 158, "reasoning_characters": 25082, "reasoning_sha256": "8ac851f8d96ed298494a9a14096ae82011b9cd73595016856dba5c11aeff97bb", "response_tail_seconds": 0.000832, "run": 3, "stream_done_marker": true, "stream_sha256": "8ac851f8d96ed298494a9a14096ae82011b9cd73595016856dba5c11aeff97bb", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 91.706441, "ttft_seconds": 0.211698, "wall_seconds": 91.707273 }, { "cached_prompt_tokens": 0, "completion_tokens": 8192, "completion_validation": { "error": "no visible answer", "valid": false }, "decode_seconds": 99.054173, "decode_tokens_per_second": 82.692, "finish_reason": "length", "measured_sse_events": 2133, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 160, "reasoning_characters": 28140, "reasoning_sha256": "8c3ae60fc39ba097d4ee83652804d2eeba9a63f5120f67e908b1a1c67f81927e", "response_tail_seconds": 0.000825, "run": 4, "stream_done_marker": true, "stream_sha256": "8c3ae60fc39ba097d4ee83652804d2eeba9a63f5120f67e908b1a1c67f81927e", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 99.249904, "ttft_seconds": 0.195731, "wall_seconds": 99.250729 }, { "cached_prompt_tokens": 0, "completion_tokens": 8192, "completion_validation": { "error": "no visible answer", "valid": false }, "decode_seconds": 83.145111, "decode_tokens_per_second": 98.515, "finish_reason": "length", "measured_sse_events": 1775, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 157, "reasoning_characters": 33434, "reasoning_sha256": "ec95056fcba20765e8fe412fb80a6d11abda744309e1a578cf95bdb0e14d45c4", "response_tail_seconds": 0.000828, "run": 5, "stream_done_marker": true, "stream_sha256": "ec95056fcba20765e8fe412fb80a6d11abda744309e1a578cf95bdb0e14d45c4", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 83.361168, "ttft_seconds": 0.216057, "wall_seconds": 83.361996 } ], "time_to_last_output_seconds": { "maximum": 99.25, "median": 83.361, "minimum": 26.171, "p90": 99.25 }, "ttft_seconds": { "maximum": 0.241, "median": 0.216, "minimum": 0.196, "p90": 0.241 }, "wall_seconds": { "maximum": 99.251, "median": 83.362, "minimum": 26.171, "p90": 99.251 } }, "structured": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 142.934, "median": 136.779, "minimum": 134.706, "p90": 142.934 }, "pooled_decode_tokens_per_second": 138.011, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 600, "completion_validation": { "valid": true }, "decode_seconds": 4.446711, "decode_tokens_per_second": 134.706, "finish_reason": "stop", "measured_sse_events": 69, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 143, "reasoning_characters": 174, "reasoning_sha256": "5300080e924f84eefba6784b86f3f397c95ad22915fb49d597ef9e1add49fe07", "response_tail_seconds": 6.5e-05, "run": 1, "stream_done_marker": true, "stream_sha256": "bd0e99de880e569393bbfd256cc388b7ca4d21ed4c7e5c42bacbffb53bdea7d8", "time_to_first_visible_seconds": 0.813669, "time_to_last_output_seconds": 4.651344, "ttft_seconds": 0.204634, "wall_seconds": 4.651409 }, { "cached_prompt_tokens": 0, "completion_tokens": 608, "completion_validation": { "valid": true }, "decode_seconds": 4.437826, "decode_tokens_per_second": 136.779, "finish_reason": "stop", "measured_sse_events": 70, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 147, "reasoning_characters": 212, "reasoning_sha256": "c273f99bcf06b315d7f047ed2511e4c2fdc705cd0e326f0f175f931d3e6c842e", "response_tail_seconds": 7.8e-05, "run": 2, "stream_done_marker": true, "stream_sha256": "3c52692c22e10b31ec595a43f2f49363604ba7d93b5130b114ab728a48f119b8", "time_to_first_visible_seconds": 0.962625, "time_to_last_output_seconds": 4.626496, "ttft_seconds": 0.18867, "wall_seconds": 4.626575 }, { "cached_prompt_tokens": 0, "completion_tokens": 598, "completion_validation": { "valid": true }, "decode_seconds": 4.176746, "decode_tokens_per_second": 142.934, "finish_reason": "stop", "measured_sse_events": 64, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 146, "reasoning_characters": 161, "reasoning_sha256": "94a5c7a5e1cb334d5cb480d8394fc46aa3a4a6c3d2e6f6a403657c33ea5a1c10", "response_tail_seconds": 6.7e-05, "run": 3, "stream_done_marker": true, "stream_sha256": "511ce7ba517a2ffa477c1f2f45db60aff5823691a0b72eb54101ade3b9936ca4", "time_to_first_visible_seconds": 0.786053, "time_to_last_output_seconds": 4.382967, "ttft_seconds": 0.206221, "wall_seconds": 4.383034 }, { "cached_prompt_tokens": 0, "completion_tokens": 600, "completion_validation": { "valid": true }, "decode_seconds": 4.301938, "decode_tokens_per_second": 139.24, "finish_reason": "stop", "measured_sse_events": 69, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 146, "reasoning_characters": 174, "reasoning_sha256": "5300080e924f84eefba6784b86f3f397c95ad22915fb49d597ef9e1add49fe07", "response_tail_seconds": 6.9e-05, "run": 4, "stream_done_marker": true, "stream_sha256": "bd0e99de880e569393bbfd256cc388b7ca4d21ed4c7e5c42bacbffb53bdea7d8", "time_to_first_visible_seconds": 0.773237, "time_to_last_output_seconds": 4.49998, "ttft_seconds": 0.198041, "wall_seconds": 4.500048 }, { "cached_prompt_tokens": 0, "completion_tokens": 608, "completion_validation": { "valid": true }, "decode_seconds": 4.4394, "decode_tokens_per_second": 136.73, "finish_reason": "stop", "measured_sse_events": 71, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 146, "reasoning_characters": 212, "reasoning_sha256": "c273f99bcf06b315d7f047ed2511e4c2fdc705cd0e326f0f175f931d3e6c842e", "response_tail_seconds": 0.000152, "run": 5, "stream_done_marker": true, "stream_sha256": "3c52692c22e10b31ec595a43f2f49363604ba7d93b5130b114ab728a48f119b8", "time_to_first_visible_seconds": 0.926018, "time_to_last_output_seconds": 4.659893, "ttft_seconds": 0.220493, "wall_seconds": 4.660045 } ], "time_to_last_output_seconds": { "maximum": 4.66, "median": 4.626, "minimum": 4.383, "p90": 4.66 }, "ttft_seconds": { "maximum": 0.22, "median": 0.205, "minimum": 0.189, "p90": 0.22 }, "wall_seconds": { "maximum": 4.66, "median": 4.627, "minimum": 4.383, "p90": 4.66 } } }, "prefill": { "32768": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2528.219, "median": 2524.59, "minimum": 2448.255, "p90": 2528.219 }, "runs": [ { "cache_salt_sha256": "274a9958708a9418fafb0e6dc045e3468dd3d01b1633e05679f1be7b33bb0e68", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.08434, "decode_tokens_per_second": 82.997, "effective_prefill_tokens_per_second": 2448.255, "finish_reason": "length", "measured_sse_events": 2, "output": " evidence, constraints, and the next action", "output_characters": 43, "output_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000125, "run": 1, "stream_done_marker": true, "stream_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "time_to_first_visible_seconds": 13.384229, "time_to_last_output_seconds": 13.46857, "ttft_seconds": 13.384229, "wall_seconds": 13.468695 }, { "cache_salt_sha256": "3fdff4b6dcb44ce2dfbe2a7f9a3a2600d89a599f603255df52a27788a74c5d31", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.086043, "decode_tokens_per_second": 81.355, "effective_prefill_tokens_per_second": 2528.219, "finish_reason": "length", "measured_sse_events": 2, "output": " and the next action. The repository", "output_characters": 36, "output_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000108, "run": 2, "stream_done_marker": true, "stream_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "time_to_first_visible_seconds": 12.960904, "time_to_last_output_seconds": 13.046947, "ttft_seconds": 12.960904, "wall_seconds": 13.047055 }, { "cache_salt_sha256": "567986c247417906d2df781094473fba1f40b599547115f9f09746c6ecec5af3", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.085952, "decode_tokens_per_second": 81.441, "effective_prefill_tokens_per_second": 2524.59, "finish_reason": "length", "measured_sse_events": 2, "output": ", constraints, and the next action.", "output_characters": 35, "output_sha256": "63fb378cf97093f800d90fa6fb06b86f90d2035af5f46c65a3a49798e320e5a2", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 6.8e-05, "run": 3, "stream_done_marker": true, "stream_sha256": "63fb378cf97093f800d90fa6fb06b86f90d2035af5f46c65a3a49798e320e5a2", "time_to_first_visible_seconds": 12.979533, "time_to_last_output_seconds": 13.065485, "ttft_seconds": 12.979533, "wall_seconds": 13.065552 } ], "ttft_seconds": { "maximum": 13.384, "median": 12.98, "minimum": 12.961, "p90": 13.384 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 2528.358, "median": 2523.261, "minimum": 2491.408, "p90": 2528.358 }, "runs": [ { "cache_salt_sha256": "274a9958708a9418fafb0e6dc045e3468dd3d01b1633e05679f1be7b33bb0e68", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.086102, "decode_tokens_per_second": 81.299, "effective_prefill_tokens_per_second": 2491.408, "finish_reason": "length", "measured_sse_events": 2, "output": " evidence, constraints, and the next action", "output_characters": 43, "output_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 9.5e-05, "run": 1, "stream_done_marker": true, "stream_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "time_to_first_visible_seconds": 13.152402, "time_to_last_output_seconds": 13.238504, "ttft_seconds": 13.152402, "wall_seconds": 13.2386 }, { "cache_salt_sha256": "3fdff4b6dcb44ce2dfbe2a7f9a3a2600d89a599f603255df52a27788a74c5d31", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.085325, "decode_tokens_per_second": 82.04, "effective_prefill_tokens_per_second": 2528.358, "finish_reason": "length", "measured_sse_events": 2, "output": " and the next action. The repository", "output_characters": 36, "output_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000115, "run": 2, "stream_done_marker": true, "stream_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "time_to_first_visible_seconds": 12.960188, "time_to_last_output_seconds": 13.045512, "ttft_seconds": 12.960188, "wall_seconds": 13.045627 }, { "cache_salt_sha256": "567986c247417906d2df781094473fba1f40b599547115f9f09746c6ecec5af3", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.086395, "decode_tokens_per_second": 81.023, "effective_prefill_tokens_per_second": 2523.261, "finish_reason": "length", "measured_sse_events": 2, "output": ", constraints, and the next action.", "output_characters": 35, "output_sha256": "63fb378cf97093f800d90fa6fb06b86f90d2035af5f46c65a3a49798e320e5a2", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 7.8e-05, "run": 3, "stream_done_marker": true, "stream_sha256": "63fb378cf97093f800d90fa6fb06b86f90d2035af5f46c65a3a49798e320e5a2", "time_to_first_visible_seconds": 12.986371, "time_to_last_output_seconds": 13.072766, "ttft_seconds": 12.986371, "wall_seconds": 13.072844 } ], "ttft_seconds": { "maximum": 13.152, "median": 12.986, "minimum": 12.96, "p90": 13.152 } } }, "65536": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2435.553, "median": 2434.752, "minimum": 2428.012, "p90": 2435.553 }, "runs": [ { "cache_salt_sha256": "4051b3532ce429e0c2737c2676b2734fce4df6fc7e2ce3b28781f60773b02261", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.103211, "decode_tokens_per_second": 67.822, "effective_prefill_tokens_per_second": 2428.012, "finish_reason": "length", "measured_sse_events": 2, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000117, "run": 1, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 26.991634, "time_to_last_output_seconds": 27.094845, "ttft_seconds": 26.991634, "wall_seconds": 27.094962 }, { "cache_salt_sha256": "c9fcf61ba589c9fa91c7333ad85624b4e2983c1870ef1b5c74d073a97b9d81b8", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.102784, "decode_tokens_per_second": 68.104, "effective_prefill_tokens_per_second": 2434.752, "finish_reason": "length", "measured_sse_events": 2, "output": " records behaviour, evidence, constraints, and", "output_characters": 46, "output_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000129, "run": 2, "stream_done_marker": true, "stream_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "time_to_first_visible_seconds": 26.916914, "time_to_last_output_seconds": 27.019698, "ttft_seconds": 26.916914, "wall_seconds": 27.019827 }, { "cache_salt_sha256": "81902a09808ffdba79540c6a306c0e7d16ef6637e51762fefe8e0f055a48bc45", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.102529, "decode_tokens_per_second": 68.273, "effective_prefill_tokens_per_second": 2435.553, "finish_reason": "length", "measured_sse_events": 2, "output": " records behaviour, evidence, constraints, and", "output_characters": 46, "output_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000143, "run": 3, "stream_done_marker": true, "stream_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "time_to_first_visible_seconds": 26.908052, "time_to_last_output_seconds": 27.010581, "ttft_seconds": 26.908052, "wall_seconds": 27.010724 } ], "ttft_seconds": { "maximum": 26.992, "median": 26.917, "minimum": 26.908, "p90": 26.992 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 2439.446, "median": 2436.159, "minimum": 2435.431, "p90": 2439.446 }, "runs": [ { "cache_salt_sha256": "4051b3532ce429e0c2737c2676b2734fce4df6fc7e2ce3b28781f60773b02261", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.102177, "decode_tokens_per_second": 68.508, "effective_prefill_tokens_per_second": 2439.446, "finish_reason": "length", "measured_sse_events": 2, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000116, "run": 1, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 26.865115, "time_to_last_output_seconds": 26.967292, "ttft_seconds": 26.865115, "wall_seconds": 26.967409 }, { "cache_salt_sha256": "c9fcf61ba589c9fa91c7333ad85624b4e2983c1870ef1b5c74d073a97b9d81b8", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.102646, "decode_tokens_per_second": 68.195, "effective_prefill_tokens_per_second": 2435.431, "finish_reason": "length", "measured_sse_events": 2, "output": " records behaviour, evidence, constraints, and", "output_characters": 46, "output_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000113, "run": 2, "stream_done_marker": true, "stream_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "time_to_first_visible_seconds": 26.909408, "time_to_last_output_seconds": 27.012054, "ttft_seconds": 26.909408, "wall_seconds": 27.012166 }, { "cache_salt_sha256": "81902a09808ffdba79540c6a306c0e7d16ef6637e51762fefe8e0f055a48bc45", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.101876, "decode_tokens_per_second": 68.711, "effective_prefill_tokens_per_second": 2436.159, "finish_reason": "length", "measured_sse_events": 2, "output": " records behaviour, evidence, constraints, and", "output_characters": 46, "output_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000143, "run": 3, "stream_done_marker": true, "stream_sha256": "f23b3bec4868cd4bb2a9e53fb2c9efd4e5e04b814a4ed7cf5e709584669551b9", "time_to_first_visible_seconds": 26.901368, "time_to_last_output_seconds": 27.003244, "ttft_seconds": 26.901368, "wall_seconds": 27.003386 } ], "ttft_seconds": { "maximum": 26.909, "median": 26.901, "minimum": 26.865, "p90": 26.909 } } }, "8192": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2595.151, "median": 2581.331, "minimum": 2580.861, "p90": 2595.151 }, "runs": [ { "cache_salt_sha256": "86e70aa36bff30ccc813737bc596706d4211936c72a9e330abc8a13d4b8ae3b4", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.074086, "decode_tokens_per_second": 94.485, "effective_prefill_tokens_per_second": 2595.151, "finish_reason": "length", "measured_sse_events": 2, "output": " contains a service, tests, documentation,", "output_characters": 42, "output_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000157, "run": 1, "stream_done_marker": true, "stream_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "time_to_first_visible_seconds": 3.156656, "time_to_last_output_seconds": 3.230742, "ttft_seconds": 3.156656, "wall_seconds": 3.2309 }, { "cache_salt_sha256": "a4b48c2784f97465e8e5a8a710451a781136d8b416cad4c029a13b4ae02e9b4b", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.072903, "decode_tokens_per_second": 96.018, "effective_prefill_tokens_per_second": 2581.331, "finish_reason": "length", "measured_sse_events": 2, "output": " service, tests, documentation, and release", "output_characters": 43, "output_sha256": "932ced224e08d6d3d62621f7f4231ec6916b3b172b0389fdf7398eeda88b7538", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000153, "run": 2, "stream_done_marker": true, "stream_sha256": "932ced224e08d6d3d62621f7f4231ec6916b3b172b0389fdf7398eeda88b7538", "time_to_first_visible_seconds": 3.173556, "time_to_last_output_seconds": 3.246459, "ttft_seconds": 3.173556, "wall_seconds": 3.246611 }, { "cache_salt_sha256": "399ab9bb74eaf38824bcc22d7c8f4e0db7b041b2ef462159c49e2768850e1366", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.073193, "decode_tokens_per_second": 95.637, "effective_prefill_tokens_per_second": 2580.861, "finish_reason": "length", "measured_sse_events": 2, "output": " tests, documentation, and release notes.", "output_characters": 41, "output_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000111, "run": 3, "stream_done_marker": true, "stream_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "time_to_first_visible_seconds": 3.174135, "time_to_last_output_seconds": 3.247328, "ttft_seconds": 3.174135, "wall_seconds": 3.247438 } ], "ttft_seconds": { "maximum": 3.174, "median": 3.174, "minimum": 3.157, "p90": 3.174 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 2614.36, "median": 2610.641, "minimum": 2602.99, "p90": 2614.36 }, "runs": [ { "cache_salt_sha256": "86e70aa36bff30ccc813737bc596706d4211936c72a9e330abc8a13d4b8ae3b4", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.071632, "decode_tokens_per_second": 97.722, "effective_prefill_tokens_per_second": 2614.36, "finish_reason": "length", "measured_sse_events": 2, "output": " contains a service, tests, documentation,", "output_characters": 42, "output_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000154, "run": 1, "stream_done_marker": true, "stream_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "time_to_first_visible_seconds": 3.133463, "time_to_last_output_seconds": 3.205094, "ttft_seconds": 3.133463, "wall_seconds": 3.205249 }, { "cache_salt_sha256": "a4b48c2784f97465e8e5a8a710451a781136d8b416cad4c029a13b4ae02e9b4b", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.071333, "decode_tokens_per_second": 98.131, "effective_prefill_tokens_per_second": 2610.641, "finish_reason": "length", "measured_sse_events": 2, "output": " service, tests, documentation, and release", "output_characters": 43, "output_sha256": "932ced224e08d6d3d62621f7f4231ec6916b3b172b0389fdf7398eeda88b7538", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000143, "run": 2, "stream_done_marker": true, "stream_sha256": "932ced224e08d6d3d62621f7f4231ec6916b3b172b0389fdf7398eeda88b7538", "time_to_first_visible_seconds": 3.137927, "time_to_last_output_seconds": 3.20926, "ttft_seconds": 3.137927, "wall_seconds": 3.209403 }, { "cache_salt_sha256": "399ab9bb74eaf38824bcc22d7c8f4e0db7b041b2ef462159c49e2768850e1366", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.071929, "decode_tokens_per_second": 97.318, "effective_prefill_tokens_per_second": 2602.99, "finish_reason": "length", "measured_sse_events": 2, "output": " tests, documentation, and release notes.", "output_characters": 41, "output_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 7.3e-05, "run": 3, "stream_done_marker": true, "stream_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "time_to_first_visible_seconds": 3.14715, "time_to_last_output_seconds": 3.219079, "ttft_seconds": 3.14715, "wall_seconds": 3.219152 } ], "ttft_seconds": { "maximum": 3.147, "median": 3.138, "minimum": 3.133, "p90": 3.147 } } } }, "protocol": { "prompts_sha256": "0c3ac4015813af6a1b967659bd2289990fde895a5e43ae065b894d1e1f909ecb", "prompts_version": "1.0.0", "repository_dirty": false, "repository_revision": "770c3106729d5b73543afa2d9b498cebda6456f3", "repository_source": "git-worktree", "repository_source_sha256": "f5503bd3fe66d96d42992fedb6cf0b47f1e7ba560359d455a6e8fc1c471261d7", "version": "1.3.0" }, "run": { "appliance": { "competing_traffic": "none", "context_limit": 262144, "drafter": "checkpoint MTP head (Mia recipe DRAFTS=1 default)", "hardware": "1x NVIDIA DGX Spark (GB10, 128 GB unified memory)", "kv_cache_dtype": "fp8 (Mia recipe default)", "model": "azampatti/Qwen3.8-Flash-Next-125B-A5B-INT4-AutoRound", "model_revision": "1464274120d36a4d8fcaa934552334a7d83ce0fd", "negotiated_link_speed": "not applicable", "quantisation": "INT4 AutoRound (GPTQ int4 routed experts), FP8 n-gram table", "scheduler": "Mia recipe defaults (--parallel 8, shared KV pool) with VISION=0 and MAX_TOKENS=131072 in .env", "serving_engine": "TensorFold Zig engine (tensorfold-native) db281878 + Mia Qwen3.8-Flash-Next single-Spark kit (commit 364f21a)", "serving_image": "ghcr.io/miaai-lab/qwen3.8-flash-next-single-dgx-spark-tensorfold:zig-db28187-4144696f21e3 (image id sha256:12f9c5e0ce63f94aa7111736cedfcf4336195473ce0f917b2ad412880414005e)", "topology": "single Spark, TP1" }, "comparison_id": "q38-int4-1spark-low", "finished_at": "2026-10-14T16:40:39.943968+00:00", "label": "q38-int4-base-low", "model": "Qwen3.8-Flash-Next", "platform": "Linux-6.17.0-1026-nvidia-aarch64-with-glibc2.39", "python": "3.12.3", "rigmark_release_check": { "ahead_by": 5, "behind_by": 0, "dirty": false, "prerelease": true, "release_url": "https://github.com/alexellis/rigmark/releases/tag/v0.0.4", "revision": "770c3106729d5b73543afa2d9b498cebda6456f3", "status": "ahead", "tag": "v0.0.4" }, "started_at": "2026-10-14T16:27:54.755471+00:00" }, "schema": 1, "settings": { "concurrency": [ 1, 2, 4 ], "concurrency_runs": 3, "concurrency_tokens": 256, "concurrency_workload": "code", "decode_tokens": 8192, "extra_body": { "chat_template_kwargs": { "reasoning_effort": "low" } }, "prefill_cache_isolation": "random-salt-per-pair", "prefill_depths": [ 8192, 32768, 65536 ], "prefill_runs": 3, "runs": 5, "seed": 20260905, "temperature": 0.0, "top_p": 1.0 } }