No GPU? Simulate One: vLLM, llm-d, and 16 Fake H100s on a Laptop

colima start --cpu 6 --memory 12 --disk 60
brew install kind

# kind-config.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: gpu-sim
nodes:
- role: control-plane
- role: worker
  labels:
    run.ai/simulated-gpu-node-pool: h100
- role: worker
  labels:
    run.ai/simulated-gpu-node-pool: h100

kind create cluster --config kind-config.yaml

# fake-gpu-values.yaml
topology:
  nodePools:
    h100:            # matches the node label value
      gpu:
        backend: fake
        profile: h100

helm upgrade -i gpu-operator \
  oci://ghcr.io/run-ai/fake-gpu-operator/fake-gpu-operator \
  --version 0.2.1 -n gpu-operator --create-namespace \
  -f fake-gpu-values.yaml

# gpu-test-pod.yaml
apiVersion: v1
kind: Pod
metadata:
  name: gpu-test
spec:
  restartPolicy: Never
  containers:
  - name: cuda
    image: ubuntu:24.04
    command: ["sh", "-c", "nvidia-smi && sleep 3600"]
    resources:
      limits:
        nvidia.com/gpu: 2

# vllm-sim.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: qwen3-32b-sim
  namespace: llm-d-sim
spec:
  replicas: 8
  strategy:
    rollingUpdate:
      maxSurge: 0          # no spare GPUs for a surge pod
      maxUnavailable: 1    # replace one replica at a time
  selector:
    matchLabels:
      app: qwen3-32b-sim
  template:
    metadata:
      labels:
        app: qwen3-32b-sim
        llm-d.ai/guide: sim-lab          # the router selects pods by this label
    spec:
      containers:
      - name: vllm
        image: ghcr.io/llm-d/llm-d-inference-sim:v0.11.4
        args:
        - --model=Qwen/Qwen3-32B          # name only; no weights are loaded
        - --port=8000
        - --max-num-seqs=4                # concurrent "running" requests per pod
        - --max-model-len=16384
        - --enable-kvcache                # track prefix-cache blocks like vLLM
        - --kv-cache-size=1024            # 1024 blocks x 16 = 16k tokens per pod
        - --block-size=16
        - --latency-calculator=per-token  # prefill cost scales with uncached tokens
        - --prefill-overhead=20ms
        - --prefill-time-per-token=200us  # 4,000 uncached tokens ≈ 0.8 s TTFT
        - --inter-token-latency=15ms
        - --time-factor-under-load=2.0
        env:
        - name: POD_NAME                  # sim echoes this in x-inference-pod
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: POD_IP                    # required when --enable-kvcache is set
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        ports:
        - name: http
          containerPort: 8000
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
        resources:
          requests:
            cpu: 50m
            memory: 64Mi
          limits:
            nvidia.com/gpu: 2             # fake H100s from fake-gpu-operator
---
apiVersion: v1
kind: Service
metadata:
  name: qwen3-32b
  namespace: llm-d-sim
spec:
  selector:
    app: qwen3-32b-sim
  ports:
  - name: http
    port: 8000
    targetPort: 8000

kubectl create namespace llm-d-sim
kubectl apply -f vllm-sim.yaml

kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.6.2/v1-manifests.yaml

# router-values.yaml
router:
  extraServicePorts:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8081
  modelServers:
    matchLabels:
      llm-d.ai/guide: sim-lab       # pick up the simulator pods
  inferencePool:
    failureMode: FailOpen
  epp:
    image:
      tag: v0.10.0
      pullPolicy: IfNotPresent
    flags:
      v: 3                          # log every scheduling decision
    resources:
      requests: {cpu: 200m, memory: 256Mi}
      limits: {memory: 1Gi}
    pluginsConfigFile: sim-lab-plugins.yaml
    pluginsCustomConfig:
      sim-lab-plugins.yaml: |
        apiVersion: llm-d.ai/v1alpha1
        kind: EndpointPickerConfig
        plugins:
        - type: approx-prefix-cache-producer
        - type: inflight-load-producer
        - type: prefix-cache-affinity-filter
        - type: token-load-scorer
        schedulingProfiles:
        - name: default
          plugins:
          - pluginRef: prefix-cache-affinity-filter
          - pluginRef: token-load-scorer
  proxy:
    args: ["--service-node", "envoy-sidecar", "--log-level", "warn",
           "--concurrency", "2", "-c", "/etc/envoy/envoy.yaml"]
    resources:
      requests: {cpu: 100m, memory: 128Mi}
      limits: {memory: 512Mi}

helm upgrade -i sim-lab oci://ghcr.io/llm-d/charts/llm-d-router-standalone \
  --version v0 -n llm-d-sim -f router-values.yaml

kubectl -n llm-d-sim port-forward svc/sim-lab-epp 8080:80 &

{
  "model": "Qwen/Qwen3-32B",
  "messages": [{"role": "user", "content": "How do I drain a Kubernetes node?"}],
  "max_tokens": 24
}

kubectl -n llm-d-sim run client --image=python:3.13-slim --command -- sleep infinity
kubectl -n llm-d-sim exec client -- sh -c 'apt-get -qq update && apt-get -qq install -y curl'
kubectl cp loadtest.py llm-d-sim/client:/loadtest.py

kubectl -n llm-d-sim delete pod -l app=qwen3-32b-sim --wait
kubectl -n llm-d-sim rollout status deploy/qwen3-32b-sim

git clone --depth 1 --branch v0.10.0 https://github.com/llm-d/llm-d.git
GUIDE_NAME=sim-lab NAMESPACE=llm-d-sim CHUNK_SIZE=4096 \
  ./llm-d/guides/recipes/router/calibration/calibrate.sh

         - type: prefix-cache-affinity-filter
+          parameters:
+            peakPrefillThroughput: 4838   # from calibrate.sh
+            maxTTFTPenaltyMs: 1000        # default 18000
-        - type: token-load-scorer
+        - type: active-request-scorer
         schedulingProfiles:
         - name: default
           plugins:
           - pluginRef: prefix-cache-affinity-filter
-          - pluginRef: token-load-scorer
+          - pluginRef: active-request-scorer

helm upgrade sim-lab oci://ghcr.io/llm-d/charts/llm-d-router-standalone \
  --version v0 -n llm-d-sim -f router-values-decode.yaml
kubectl -n llm-d-sim rollout restart deploy/sim-lab-epp

"""Tiny load generator: N tenants, each with a long shared system prompt.

Sends the same request sequence to a target URL and reports which pod served
each request, prefix-cache hits, and time to first token (TTFT).
Stdlib only, so it runs in a plain python:3.13-slim pod.
"""
import argparse, collections, json, random, statistics, threading, time, urllib.request

WORDS = ("node drain cordon pod deployment rollout restart replica service ingress "
         "secret configmap volume claim taint toleration affinity quota limit probe "
         "readiness liveness gpu driver kernel upgrade backup restore alert page "
         "oncall escalate rollback canary metrics logs trace dashboard").split()


def system_prompt(tenant, words=3800):
    rnd = random.Random(tenant)
    body = " ".join(rnd.choice(WORDS) for _ in range(words))
    return f"[tenant-{tenant:02d}] You are the runbook assistant for team {tenant}. Runbook: {body}"


QUESTIONS = ["How do I drain a node?", "What are the pre-upgrade checks?",
             "How do I roll back a deploy?", "Who do I page for GPU alerts?",
             "How do I restore from backup?", "Why is the pod not ready?"]


def one_request(url, model, tenant, q, max_tokens):
    body = json.dumps({
        "model": model, "stream": True, "max_tokens": max_tokens,
        "stream_options": {"include_usage": True},
        "messages": [{"role": "system", "content": system_prompt(tenant)},
                     {"role": "user", "content": q}],
    }).encode()
    req = urllib.request.Request(url, body, {"Content-Type": "application/json"})
    t0 = time.perf_counter(); ttft = None; usage = {}
    with urllib.request.urlopen(req, timeout=120) as r:
        pod = r.headers.get("x-inference-pod", "?")
        for line in r:
            line = line.strip()
            if not line.startswith(b"data:") or line == b"data: [DONE]":
                continue
            chunk = json.loads(line[5:])
            if ttft is None and chunk.get("choices"):
                ttft = time.perf_counter() - t0
            if chunk.get("usage"):
                usage = chunk["usage"]
    prompt = usage.get("prompt_tokens", 0)
    cached = (usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0)
    return dict(tenant=tenant, pod=pod, ttft=ttft, e2e=time.perf_counter() - t0,
                prompt=prompt, cached=cached)


def pct(xs, p):
    xs = sorted(xs); return xs[min(len(xs) - 1, int(round(p / 100 * (len(xs) - 1))))]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--url", required=True)
    ap.add_argument("--label", default="run")
    ap.add_argument("--model", default="Qwen/Qwen3-32B")
    ap.add_argument("--tenants", type=int, default=16)
    ap.add_argument("--requests", type=int, default=320)
    ap.add_argument("--concurrency", type=int, default=8)
    ap.add_argument("--max-tokens", type=int, default=30)
    ap.add_argument("--seed", type=int, default=7)
    ap.add_argument("--trace", type=int, default=0, help="print the first N requests")
    a = ap.parse_args()

    rnd = random.Random(a.seed)
    jobs = [(rnd.randrange(a.tenants), rnd.choice(QUESTIONS)) for _ in range(a.requests)]
    results, lock, it = [], threading.Lock(), iter(enumerate(jobs))

    def worker():
        while True:
            with lock:
                nxt = next(it, None)
            if nxt is None:
                return
            i, (t, q) = nxt
            r = one_request(a.url, a.model, t, q, a.max_tokens)
            r["i"] = i
            with lock:
                results.append(r)

    t0 = time.perf_counter()
    threads = [threading.Thread(target=worker) for _ in range(a.concurrency)]
    [t.start() for t in threads]; [t.join() for t in threads]
    wall = time.perf_counter() - t0
    results.sort(key=lambda r: r["i"])

    for r in results[:a.trace]:
        hit = "HIT " if r["cached"] > r["prompt"] / 2 else "miss"
        print(f"#{r['i']:<3} tenant-{r['tenant']:02d} -> {r['pod'][-5:]}  {hit}  "
              f"cached {r['cached']:>4}/{r['prompt']:<4}  ttft {r['ttft']*1000:6.0f} ms")
    if a.trace:
        print()

    ttfts = [r["ttft"] for r in results]
    prompt = sum(r["prompt"] for r in results); cached = sum(r["cached"] for r in results)
    per_pod = collections.Counter(r["pod"][-5:] for r in results)
    tenants_per_pod = collections.defaultdict(set)
    for r in results:
        tenants_per_pod[r["pod"][-5:]].add(r["tenant"])

    print(f"== {a.label}: {len(results)} requests, {a.tenants} tenants, concurrency {a.concurrency}")
    print(f"prefix cache hit rate : {cached / prompt:6.1%}  ({cached:,} of {prompt:,} prompt tokens)")
    print(f"TTFT p50 / p90 / p99  : {pct(ttfts,50)*1000:5.0f} / {pct(ttfts,90)*1000:5.0f} / {pct(ttfts,99)*1000:5.0f} ms")
    print(f"mean TTFT             : {statistics.mean(ttfts)*1000:5.0f} ms")
    print(f"throughput            : {len(results)/wall:5.1f} req/s  (wall {wall:.1f}s)")
    print("pod    requests  tenants-seen")
    for pod in sorted(per_pod):
        print(f"{pod}  {per_pod[pod]:>8}  {len(tenants_per_pod[pod]):>12}")


if __name__ == "__main__":
    main()

# 16 tenants (default): the plain Service, then the router
kubectl -n llm-d-sim exec client -- python /loadtest.py \
  --url http://qwen3-32b:8000/v1/chat/completions --label "plain Service"
kubectl -n llm-d-sim exec client -- python /loadtest.py \
  --url http://sim-lab-epp/v1/chat/completions --label "llm-d router"

# one hot tenant
kubectl -n llm-d-sim exec client -- python /loadtest.py \
  --url http://sim-lab-epp/v1/chat/completions \
  --tenants 1 --requests 200 --concurrency 24 --max-tokens 60

# the 12-request trace
kubectl -n llm-d-sim exec client -- python /loadtest.py \
  --url http://sim-lab-epp/v1/chat/completions \
  --tenants 3 --requests 12 --concurrency 1 --trace 12

kind delete cluster --name gpu-sim
colima stop

kind                  = Kubernetes nodes as Docker containers
fake-gpu-operator     = nvidia.com/gpu, nvidia-smi, DCGM metrics, no GPU
llm-d-inference-sim   = vLLM's API + metrics + prefix cache, no model
EPP                   = Endpoint Picker: llm-d's router brain, picks the pod
Envoy                 = the proxy that asks the EPP, then forwards
ext_proc              = the Envoy hook Envoy uses to ask the EPP
x-inference-pod       = which pod served the request
cached_tokens         = did it hit the prefix cache
calibrate.sh          = measures peakPrefillThroughput, works on the sim
prefill-bound traffic = token-load-scorer, stay sticky
decode-bound traffic  = active-request-scorer, lower maxTTFTPenaltyMs
the lab can't tell you = real speed, memory, CUDA, quality