Benchmark local LLMs reproducibly: TTFT, throughput, memory, quality, context, and tools

mkdir -p local-llm-bench/{config,fixtures,src,reports} && cd local-llm-bench && chmod 0700 reports && cat > package.json <<'JSON'
{
  "name": "reproducible-local-llm-benchmark",
  "version": "1.0.0",
  "private": true,
  "type": "module",
  "engines": { "node": "22.22.2" },
  "scripts": {
    "check": "node --check src/benchmark.mjs && node --check src/score.mjs",
    "benchmark": "node src/benchmark.mjs config/manifest.json config/workloads.json reports",
    "score": "node src/score.mjs reports/results.ndjson config/workloads.json reports/quality.json"
  }
}
JSON

{
  "suiteRevision": "local-llm-benchmark-2026-07-29.1",
  "verifiedAt": "2026-07-29",
  "measurementPolicy": {
    "warmups": 3,
    "measuredRuns": 15,
    "requestTimeoutMs": 120000,
    "maxResponseBytes": 2097152,
    "concurrencyLevels": [1, 4, 8],
    "cooldownSeconds": 15,
    "persistRawPrompts": false,
    "persistRawOutputs": false
  },
  "host": {
    "os": "Ubuntu Server 24.04.2 LTS",
    "kernel": "6.8.0-64-generic",
    "cpu": "AMD EPYC 9454P",
    "logicalCpuCount": 96,
    "ramBytes": 274877906944,
    "gpu": ["NVIDIA L40S 48 GB"],
    "driver": "570.172.08",
    "cudaRuntime": "12.8",
    "powerLimitWatts": [350],
    "persistenceMode": true,
    "clocksPolicy": "documented fixed lab profile",
    "backgroundServices": "operator-must-replace-with-captured-list",
    "thermalState": "operator-must-replace"
  },
  "deployment": {
    "provider": "operator-must-replace",
    "engineVersion": "operator-must-replace",
    "baseUrl": "http://127.0.0.1:8000/v1",
    "tokenEnvironment": "BENCHMARK_API_TOKEN",
    "modelAlias": "benchmark-model",
    "modelArtifactSha256": "operator-must-replace",
    "tokenizerRevision": "operator-must-replace",
    "chatTemplateSha256": "operator-must-replace",
    "dtypeOrQuantization": "operator-must-replace",
    "contextLimitTokens": 32768,
    "startupArguments": ["operator-must-replace"],
    "tensorParallelSize": 1,
    "gpuMemoryUtilization": "operator-must-replace"
  }
}

{
  "revision": "workloads-2026-07-29.1",
  "defaults": {
    "temperature": 0,
    "seed": 424242,
    "stream": true
  },
  "workloads": [
    {
      "id": "exact-transform",
      "purpose": "short deterministic instruction following",
      "prompt": "Return exactly PORTABLE-LLM-OK and nothing else.",
      "maxTokens": 32,
      "score": { "kind": "exact", "expected": "PORTABLE-LLM-OK" }
    },
    {
      "id": "bounded-summary",
      "purpose": "medium generation throughput and instruction adherence",
      "prompt": "Summarize this synthetic incident in exactly three bullets: service alpha returned 503 for 90 seconds; the upstream pool had zero healthy members; a configuration rollback restored health. Include duration, cause, and recovery.",
      "maxTokens": 128,
      "score": { "kind": "required-substrings", "values": ["90 seconds", "zero healthy", "rollback"] }
    },
    {
      "id": "structured-ticket",
      "purpose": "strict structured output",
      "prompt": "Return a JSON object with status equal to resolved, severity equal to medium, and retry equal to false.",
      "maxTokens": 96,
      "responseFormat": {
        "type": "json_schema",
        "json_schema": {
          "name": "ticket",
          "strict": true,
          "schema": {
            "type": "object",
            "additionalProperties": false,
            "required": ["status", "severity", "retry"],
            "properties": {
              "status": { "const": "resolved" },
              "severity": { "const": "medium" },
              "retry": { "const": false }
            }
          }
        }
      },
      "score": { "kind": "json-ticket" }
    },
    {
      "id": "forced-tool",
      "purpose": "tool name and argument correctness",
      "prompt": "Use get_service_status for service synthetic-api in region lab-1.",
      "maxTokens": 96,
      "tool": {
        "type": "function",
        "function": {
          "name": "get_service_status",
          "description": "Returns synthetic service status; never executed by this benchmark.",
          "parameters": {
            "type": "object",
            "additionalProperties": false,
            "required": ["service", "region"],
            "properties": {
              "service": { "type": "string" },
              "region": { "type": "string" }
            }
          }
        }
      },
      "score": { "kind": "tool", "name": "get_service_status", "arguments": { "service": "synthetic-api", "region": "lab-1" } }
    },
    {
      "id": "context-sentinel",
      "purpose": "effective long-context retrieval",
      "promptFile": "fixtures/context-sentinel.txt",
      "maxTokens": 48,
      "score": { "kind": "exact", "expected": "SENTINEL-7H3K-29QP" }
    }
  ]
}

import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile, appendFile } from "node:fs/promises";
import path from "node:path";
import { performance } from "node:perf_hooks";
import { scoreOutput } from "./score.mjs";

const [manifestPath, workloadsPath, outputDirectory] = process.argv.slice(2);
if (!manifestPath || !workloadsPath || !outputDirectory) {
  throw new Error("Usage: node benchmark.mjs <manifest.json> <workloads.json> <output-directory>");
}
const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
const workloadConfig = JSON.parse(await readFile(workloadsPath, "utf8"));
await mkdir(outputDirectory, { recursive: true });
const outputPath = path.join(outputDirectory, "results.ndjson");
await writeFile(outputPath, "");

function hash(value) {
  return createHash("sha256").update(value).digest("hex");
}

function assertLoopback(raw) {
  const url = new URL(raw);
  if (url.protocol !== "http:" || url.username || url.password) throw new Error("Benchmark URL must be credential-free loopback HTTP.");
  if (!["127.0.0.1", "[::1]", "localhost"].includes(url.hostname)) throw new Error("Remote benchmark targets require a separately reviewed profile.");
  return url;
}

function percentile(values, p) {
  if (!values.length) return null;
  const ordered = [...values].sort((a, b) => a - b);
  const index = Math.min(ordered.length - 1, Math.max(0, Math.ceil(p * ordered.length) - 1));
  return ordered[index];
}

function deterministicOrder(items, seed) {
  let state = seed >>> 0;
  const copy = [...items];
  for (let i = copy.length - 1; i > 0; i -= 1) {
    state = (1664525 * state + 1013904223) >>> 0;
    const j = state % (i + 1);
    [copy[i], copy[j]] = [copy[j], copy[i]];
  }
  return copy;
}

async function loadPrompt(item) {
  if (item.prompt) return item.prompt;
  const text = await readFile(item.promptFile, "utf8");
  if (Buffer.byteLength(text) > 524288) throw new Error("Prompt fixture exceeds 512 KiB.");
  return text;
}

async function runOne(item, phase, runIndex, requestIndex, concurrency) {
  const prompt = await loadPrompt(item);
  const url = new URL("chat/completions", assertLoopback(manifest.deployment.baseUrl.endsWith("/") ? manifest.deployment.baseUrl : manifest.deployment.baseUrl + "/"));
  const token = process.env[manifest.deployment.tokenEnvironment];
  if (!token) throw new Error("Missing benchmark token environment variable.");
  const body = {
    model: manifest.deployment.modelAlias,
    messages: [{ role: "user", content: prompt }],
    temperature: workloadConfig.defaults.temperature,
    seed: workloadConfig.defaults.seed,
    max_tokens: item.maxTokens,
    stream: true,
    stream_options: { include_usage: true }
  };
  if (item.responseFormat) body.response_format = item.responseFormat;
  if (item.tool) {
    body.tools = [item.tool];
    body.tool_choice = { type: "function", function: { name: item.tool.function.name } };
    body.parallel_tool_calls = false;
  }
  const started = performance.now();
  const response = await fetch(url, {
    method: "POST",
    redirect: "error",
    headers: { authorization: "Bearer " + token, "content-type": "application/json", accept: "text/event-stream" },
    body: JSON.stringify(body),
    signal: AbortSignal.timeout(manifest.measurementPolicy.requestTimeoutMs)
  });
  if (!response.ok || !response.body) throw new Error("HTTP " + response.status);
  if (!(response.headers.get("content-type") || "").toLowerCase().includes("text/event-stream")) {
    throw new Error("Expected text/event-stream.");
  }
  const reader = response.body.getReader();
  const decoder = new TextDecoder();
  let buffered = "";
  let bytes = 0;
  let firstContentAt = null;
  let text = "";
  let finishReason = null;
  let usage = null;
  const toolFragments = new Map();
  while (true) {
    const part = await reader.read();
    if (part.done) break;
    bytes += part.value.byteLength;
    if (bytes > manifest.measurementPolicy.maxResponseBytes) {
      await reader.cancel();
      throw new Error("Response exceeded byte cap.");
    }
    buffered += decoder.decode(part.value, { stream: true });
    let boundary;
    while ((boundary = buffered.indexOf("\n\n")) >= 0) {
      const event = buffered.slice(0, boundary);
      buffered = buffered.slice(boundary + 2);
      for (const line of event.split("\n")) {
        if (!line.startsWith("data:")) continue;
        const data = line.slice(5).trim();
        if (!data || data === "[DONE]") continue;
        const parsed = JSON.parse(data);
        usage = parsed.usage || usage;
        for (const choice of parsed.choices || []) {
          finishReason = choice.finish_reason || finishReason;
          const delta = choice.delta || {};
          if (typeof delta.content === "string" && delta.content.length) {
            if (firstContentAt === null) firstContentAt = performance.now();
            text += delta.content;
          }
          for (const call of delta.tool_calls || []) {
            if (firstContentAt === null) firstContentAt = performance.now();
            const current = toolFragments.get(call.index) || { name: "", arguments: "" };
            if (call.function?.name) current.name += call.function.name;
            if (call.function?.arguments) current.arguments += call.function.arguments;
            toolFragments.set(call.index, current);
          }
        }
      }
    }
  }
  const ended = performance.now();
  const completionTokens = Number.isInteger(usage?.completion_tokens) ? usage.completion_tokens : null;
  const ttftMs = firstContentAt === null ? null : firstContentAt - started;
  const generationMs = firstContentAt === null ? null : ended - firstContentAt;
  const outputTokensPerSecond =
    completionTokens !== null && generationMs !== null && generationMs > 0
      ? completionTokens / (generationMs / 1000)
      : null;
  const quality = scoreOutput(item, { text, toolCalls: [...toolFragments.values()] });
  return {
    schemaVersion: 1,
    suiteRevision: manifest.suiteRevision,
    deploymentDigest: hash(JSON.stringify(manifest.deployment)),
    workloadRevision: workloadConfig.revision,
    workloadId: item.id,
    phase,
    runIndex,
    requestIndex,
    concurrency,
    promptSha256: hash(prompt),
    outputSha256: hash(text + JSON.stringify([...toolFragments.values()])),
    rawContentStored: false,
    ttftMs,
    endToEndMs: ended - started,
    generationMs,
    completionTokens,
    outputTokensPerSecond,
    throughputAvailable: completionTokens !== null,
    finishReason,
    failed: false,
    error: null,
    quality
  };
}

async function runGroup(item, phase, runIndex, concurrency) {
  const prompt = await loadPrompt(item);
  const results = await Promise.all(Array.from({ length: concurrency }, async (_unused, requestIndex) => {
    try {
      return await runOne(item, phase, runIndex, requestIndex, concurrency);
    } catch (error) {
      const message = error instanceof Error ? error.message.replace(/\s+/g, " ").slice(0, 240) : "Unknown benchmark failure";
      return {
        schemaVersion: 1,
        suiteRevision: manifest.suiteRevision,
        deploymentDigest: hash(JSON.stringify(manifest.deployment)),
        workloadRevision: workloadConfig.revision,
        workloadId: item.id,
        phase,
        runIndex,
        requestIndex,
        concurrency,
        promptSha256: hash(prompt),
        outputSha256: null,
        rawContentStored: false,
        ttftMs: null,
        endToEndMs: null,
        generationMs: null,
        completionTokens: null,
        outputTokensPerSecond: null,
        throughputAvailable: false,
        finishReason: null,
        failed: true,
        error: { name: error instanceof Error ? error.name : "Error", message },
        quality: { pass: false, reason: "transport or protocol failure" }
      };
    }
  }));
  for (const result of results) await appendFile(outputPath, JSON.stringify(result) + "\n");
}

for (const concurrency of manifest.measurementPolicy.concurrencyLevels) {
  for (let index = 0; index < manifest.measurementPolicy.warmups; index += 1) {
    for (const item of deterministicOrder(workloadConfig.workloads, 1000 + index)) {
      await runGroup(item, "warmup", index, concurrency);
    }
  }
  for (let index = 0; index < manifest.measurementPolicy.measuredRuns; index += 1) {
    for (const item of deterministicOrder(workloadConfig.workloads, 424242 + index)) {
      await runGroup(item, "measured", index, concurrency);
    }
  }
}

const measured = (await readFile(outputPath, "utf8")).trim().split("\n").filter(Boolean).map(JSON.parse).filter((item) => item.phase === "measured");
const summary = {};
for (const item of measured) {
  const key = item.workloadId + "@c" + item.concurrency;
  const bucket = summary[key] ||= { total: 0, errors: 0, ttft: [], e2e: [], tps: [], unavailableThroughput: 0 };
  bucket.total += 1;
  if (item.failed) bucket.errors += 1;
  if (item.ttftMs !== null) bucket.ttft.push(item.ttftMs);
  if (item.endToEndMs !== null) bucket.e2e.push(item.endToEndMs);
  if (item.outputTokensPerSecond !== null) bucket.tps.push(item.outputTokensPerSecond);
  else bucket.unavailableThroughput += 1;
}
const report = Object.fromEntries(Object.entries(summary).map(([key, value]) => [key, {
  runs: value.total,
  successfulLatencyRuns: value.e2e.length,
  errorCount: value.errors,
  ttftMs: { p50: percentile(value.ttft, 0.5), p95: percentile(value.ttft, 0.95) },
  endToEndMs: { p50: percentile(value.e2e, 0.5), p95: percentile(value.e2e, 0.95) },
  outputTokensPerSecond: value.tps.length ? { p50: percentile(value.tps, 0.5), p95: percentile(value.tps, 0.95) } : null,
  unavailableThroughput: value.unavailableThroughput
}]));
await writeFile(path.join(outputDirectory, "latency-summary.json"), JSON.stringify({
  measuredFactsOnly: true,
  generatedAt: new Date().toISOString(),
  manifestSha256: hash(JSON.stringify(manifest)),
  workloadSha256: hash(JSON.stringify(workloadConfig)),
  warmupsExcluded: true,
  summary
}, null, 2) + "\n");
console.log("Wrote " + outputPath + " and latency-summary.json; raw content is present only in-memory for immediate scoring.");

import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";

function canonical(value) {
  if (Array.isArray(value)) return value.map(canonical);
  if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]));
  return value;
}
export function scoreOutput(workload, output) {
  const rule = workload.score;
  if (!rule) return { pass: null, reason: "no deterministic scorer" };
  if (rule.kind === "exact") return { pass: output.text.trim() === rule.expected, reason: "exact normalized text" };
  if (rule.kind === "required-substrings") return { pass: rule.values.every((value) => output.text.includes(value)), reason: "all required substrings" };
  if (rule.kind === "json-ticket") {
    try {
      const value = JSON.parse(output.text);
      const pass = value && value.status === "resolved" && value.severity === "medium" && value.retry === false && Object.keys(value).length === 3;
      return { pass, reason: "independent ticket object validation" };
    } catch { return { pass: false, reason: "invalid JSON" }; }
  }
  if (rule.kind === "tool") {
    const call = output.toolCalls[0];
    if (!call || call.name !== rule.name) return { pass: false, reason: "wrong or missing tool name" };
    try {
      return { pass: JSON.stringify(canonical(JSON.parse(call.arguments))) === JSON.stringify(canonical(rule.arguments)), reason: "tool name and canonical arguments" };
    } catch { return { pass: false, reason: "invalid tool arguments JSON" }; }
  }
  return { pass: null, reason: "unknown scorer" };
}

if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
  const [resultsPath, workloadsPath, outputPath] = process.argv.slice(2);
  if (!resultsPath || !workloadsPath || !outputPath) throw new Error("Usage: node score.mjs <results.ndjson> <workloads.json> <quality.json>");
  const workloadConfig = JSON.parse(await readFile(workloadsPath, "utf8"));
  const knownIds = new Set(workloadConfig.workloads.map((item) => item.id));
  const rows = (await readFile(resultsPath, "utf8")).trim().split("\n").filter(Boolean).map(JSON.parse);
  const scored = rows.filter((row) => row.phase === "measured").map((row) => {
    if (!knownIds.has(row.workloadId)) throw new Error("Unknown workload " + row.workloadId);
    if (!row.quality || !["boolean", "object"].includes(typeof row.quality.pass)) {
      throw new Error("Missing immediate deterministic score for " + row.workloadId);
    }
    return {
      workloadId: row.workloadId,
      concurrency: row.concurrency,
      runIndex: row.runIndex,
      pass: row.quality.pass,
      reason: row.quality.reason,
      outputSha256: row.outputSha256
    };
  });
  const summary = {};
  for (const row of scored) {
    const key = row.workloadId + "@c" + row.concurrency;
    const bucket = summary[key] ||= { pass: 0, fail: 0, unscored: 0 };
    if (row.pass === true) bucket.pass += 1;
    else if (row.pass === false) bucket.fail += 1;
    else bucket.unscored += 1;
  }
  await writeFile(outputPath, JSON.stringify({
    measuredFactsOnly: true,
    rawContentStored: false,
    scorerSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"),
    workloadSha256: createHash("sha256").update(JSON.stringify(workloadConfig)).digest("hex"),
    summary,
    rows: scored
  }, null, 2) + "\n");
  console.log("Wrote deterministic quality scores to " + outputPath);
}

#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 3 ]]; then
  echo "Usage: collect-memory.sh <pid> <cgroup-path-or-none> <output.csv>" >&2
  exit 64
fi
pid="$1"
cgroup="$2"
output="$3"
[[ "$pid" =~ ^[0-9]+$ ]] || { echo "PID must be numeric" >&2; exit 64; }
kill -0 "$pid" 2>/dev/null || { echo "PID is not running" >&2; exit 1; }
start_time="$(awk '{print $22}' /proc/"$pid"/stat)"
printf 'captured_at,pid,rss_bytes,cgroup_current_bytes,gpu_memory_bytes\n' > "$output"
while kill -0 "$pid" 2>/dev/null; do
  current_start="$(awk '{print $22}' /proc/"$pid"/stat 2>/dev/null || true)"
  [[ "$current_start" == "$start_time" ]] || { echo "PID reused; stopping" >&2; exit 1; }
  rss_pages="$(awk '/VmRSS:/ { print int($2 * 1024) }' /proc/"$pid"/status)"
  cgroup_bytes=""
  if [[ "$cgroup" != "none" && -r "$cgroup/memory.current" ]]; then
    cgroup_bytes="$(cat "$cgroup/memory.current")"
  fi
  gpu_mib="$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits 2>/dev/null | awk -F, -v p="$pid" '$1+0 == p { sum += $2+0 } END { print sum+0 }')"
  gpu_bytes="$((gpu_mib * 1024 * 1024))"
  printf '%s,%s,%s,%s,%s\n' "$(date --iso-8601=ns)" "$pid" "$rss_pages" "$cgroup_bytes" "$gpu_bytes" >> "$output"
  sleep 1
done