sustained_load.py

#!/usr/bin/env python3
"""Sustained-load throughput test.

Measures STEADY-STATE aggregate throughput, unlike scripts/benchmark.py which
measures short bursts. Uses long generations so the load outlives vLLM's ~10s
logging window -- otherwise prefill ramp-up and the ragged completion tail
dominate the wall clock and understate throughput (~270 vs ~350 tok/s here).

Cross-check the result against the server's own accounting:
    docker logs <container> 2>&1 | grep "generation throughput"
Look for windows reporting "Running: N reqs" at your full concurrency.

Usage: python3 sustained_load.py <host> [port] [concurrency]
"""

import sys
HOST = sys.argv[1] if len(sys.argv) > 1 else "localhost"
PORT = int(sys.argv[2]) if len(sys.argv) > 2 else 8000
import json, time, urllib.request, concurrent.futures

URL=f"http://{HOST}:{PORT}/v1/chat/completions"
MODEL="Intel/Qwen3-Coder-Next-int4-AutoRound"
N = int(sys.argv[3]) if len(sys.argv) > 3 else 16
# Prompts engineered to force LONG generation so load sustains past vLLM's 10s log window
P=("Write a complete Python implementation of a red-black tree with insert, delete, "
   "search, and in-order traversal. Include full docstrings for every method, inline "
   "comments explaining each rotation case, and a comprehensive unittest suite at the "
   "end covering edge cases. Be exhaustive and verbose. Variant {}.")

def one(i):
    body=json.dumps({"model":MODEL,"messages":[{"role":"user","content":P.format(i)}],
                     "max_tokens":1500,"temperature":0.4}).encode()
    t0=time.time()
    r=urllib.request.urlopen(urllib.request.Request(URL,body,{"Content-Type":"application/json"}),timeout=600)
    d=json.loads(r.read())
    return d["usage"]["completion_tokens"], time.time()-t0

t0=time.time()
with concurrent.futures.ThreadPoolExecutor(N) as ex:
    res=list(ex.map(one, range(N)))
wall=time.time()-t0
tot=sum(c for c,_ in res)
print(f"requests: {N}")
print(f"total completion tokens: {tot}")
print(f"wall clock: {wall:.1f}s")
print(f"CLIENT-SIDE AGGREGATE: {tot/wall:.1f} tok/s")
print(f"per-request avg: {sum(c/t for c,t in res)/len(res):.1f} tok/s")