Compare Ollama, LM Studio, and vLLM with a portable OpenAI API contract suite

mkdir -p local-api-compat/{config,src,test,reports} && cd local-api-compat && cat > package.json <<'JSON'
{
  "name": "local-openai-compatibility-suite",
  "version": "1.0.0",
  "private": true,
  "type": "module",
  "engines": { "node": "22.22.2" },
  "scripts": {
    "check": "node --check src/compat.mjs && node --check src/portable-client.mjs",
    "test": "node --test test/*.test.mjs",
    "compat": "node src/compat.mjs config/release-manifest.json config/providers.json config/contract.json reports/compatibility.json"
  }
}
JSON

{
  "suiteRevision": "compat-suite-2026-07-29.1",
  "verifiedAt": "2026-07-29",
  "host": {
    "os": "Ubuntu 24.04.2 LTS",
    "kernel": "6.8.0-64-generic",
    "cpu": "AMD EPYC 9454P",
    "ramBytes": 274877906944,
    "gpu": "NVIDIA L40S 48 GB",
    "driver": "570.172.08",
    "powerMode": "fixed-reviewed-profile"
  },
  "engines": {
    "ollama": "0.32.0",
    "lmStudio": "0.4.1",
    "vllm": "0.23.0"
  },
  "modelFamily": "openai/gpt-oss-20b",
  "artifacts": {
    "ollama": {
      "servedName": "compat-gpt-oss-20b",
      "format": "Ollama packaged local artifact",
      "digest": "operator-must-replace-ollama-show-digest",
      "quantization": "operator-must-replace-observed-value",
      "chatTemplateHash": "operator-must-replace-reviewed-sha256"
    },
    "lmStudio": {
      "servedName": "compat-gpt-oss-20b",
      "format": "operator-must-replace-with-GGUF-or-MLX-artifact",
      "digest": "operator-must-replace-downloaded-file-sha256",
      "quantization": "operator-must-replace-observed-value",
      "chatTemplateHash": "operator-must-replace-reviewed-sha256"
    },
    "vllm": {
      "servedName": "compat-gpt-oss-20b",
      "format": "Hugging Face safetensors",
      "digest": "operator-must-replace-snapshot-tree-sha256",
      "dtype": "operator-must-replace-observed-value",
      "tokenizerHash": "operator-must-replace-reviewed-sha256",
      "chatTemplateHash": "operator-must-replace-reviewed-sha256"
    }
  },
  "comparisonBoundary": {
    "apiCompatibilityOnly": true,
    "performanceComparable": false,
    "qualityComparable": false,
    "reason": "Artifacts, quantization, kernels, templates, schedulers, and hardware placement are not proven identical."
  }
}

{
  "requestTimeoutMs": 45000,
  "maximumResponseBytes": 2097152,
  "providers": [
    {
      "id": "ollama",
      "product": "Ollama",
      "engineVersion": "0.32.0",
      "baseUrl": "http://127.0.0.1:11434/v1",
      "apiKeyEnvironment": "OLLAMA_COMPAT_API_KEY",
      "model": "compat-gpt-oss-20b",
      "artifactDigest": "operator-must-replace-ollama-show-digest",
      "expectedEndpoints": ["models", "chat", "responses"],
      "notes": "Local Ollama ignores the compatibility API key; loopback or an authenticated proxy is the security boundary."
    },
    {
      "id": "lm-studio",
      "product": "LM Studio",
      "engineVersion": "0.4.1",
      "baseUrl": "http://127.0.0.1:1234/v1",
      "apiKeyEnvironment": "LM_STUDIO_API_TOKEN",
      "model": "compat-gpt-oss-20b",
      "artifactDigest": "operator-must-replace-downloaded-file-sha256",
      "expectedEndpoints": ["models", "chat", "responses"],
      "notes": "Run the server with API-token authentication enabled and keep the port on loopback."
    },
    {
      "id": "vllm",
      "product": "vLLM",
      "engineVersion": "0.23.0",
      "baseUrl": "http://127.0.0.1:8000/v1",
      "apiKeyEnvironment": "VLLM_API_KEY",
      "model": "compat-gpt-oss-20b",
      "artifactDigest": "operator-must-replace-snapshot-tree-sha256",
      "expectedEndpoints": ["models", "chat", "responses"],
      "notes": "Start vLLM with an explicit API key, model alias, generation defaults, tool parser, and loopback listener."
    }
  ]
}

{
  "contractRevision": "openai-compat-local-v1",
  "syntheticInput": "Return the single word READY.",
  "chat": {
    "path": "/chat/completions",
    "requiredRequestFields": ["model", "messages", "temperature", "max_tokens"],
    "requiredResponsePaths": ["id", "object", "choices", "choices.0.message.content", "choices.0.finish_reason"]
  },
  "stream": {
    "path": "/chat/completions",
    "requireDataDone": true,
    "requireAtLeastOneTextDelta": true,
    "maximumEvents": 2048
  },
  "structured": {
    "schema": {
      "type": "object",
      "properties": {
        "status": { "type": "string", "enum": ["ready"] },
        "count": { "type": "integer", "minimum": 1, "maximum": 1 }
      },
      "required": ["status", "count"],
      "additionalProperties": false
    }
  },
  "tool": {
    "name": "lookup_status",
    "parameters": {
      "type": "object",
      "properties": {
        "service": { "type": "string", "enum": ["api"] }
      },
      "required": ["service"],
      "additionalProperties": false
    }
  },
  "responses": {
    "path": "/responses",
    "requiredResponsePaths": ["id", "object", "output"]
  }
}

import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname } from "node:path";

function sha256(value) {
  return createHash("sha256").update(value).digest("hex");
}

function assert(condition, message) {
  if (!condition) throw new Error(message);
}

function readPath(value, path) {
  return path.split(".").reduce((current, key) => {
    if (current === null || current === undefined) return undefined;
    if (Array.isArray(current) && /^\d+$/.test(key)) return current[Number(key)];
    return current[key];
  }, value);
}

function requireLoopbackBaseUrl(raw) {
  const url = new URL(raw);
  const loopback = new Set(["127.0.0.1", "localhost", "[::1]"]);
  assert(url.protocol === "http:" || url.protocol === "https:", "unsupported_protocol");
  assert(loopback.has(url.hostname), "provider_must_be_loopback");
  assert(!url.username && !url.password, "credentials_in_url");
  assert(url.pathname === "/v1" || url.pathname === "/v1/", "base_url_must_end_in_v1");
  return url.toString().replace(/\/$/, "");
}

async function readBoundedBody(response, maximumBytes) {
  if (!response.body) return "";
  const reader = response.body.getReader();
  const decoder = new TextDecoder();
  let bytes = 0;
  let text = "";
  for (;;) {
    const part = await reader.read();
    if (part.done) break;
    bytes += part.value.byteLength;
    if (bytes > maximumBytes) {
      await reader.cancel();
      throw new Error("response_too_large");
    }
    text += decoder.decode(part.value, { stream: true });
  }
  return text + decoder.decode();
}

async function request(provider, config, path, options = {}) {
  const key = process.env[provider.apiKeyEnvironment];
  assert(key, "missing_api_key_environment:" + provider.apiKeyEnvironment);
  const baseUrl = requireLoopbackBaseUrl(provider.baseUrl);
  const started = performance.now();
  const response = await fetch(baseUrl + path, {
    method: options.method || "POST",
    redirect: "error",
    headers: {
      authorization: "Bearer " + key,
      "content-type": "application/json",
      accept: options.accept || "application/json"
    },
    body: options.body === undefined ? undefined : JSON.stringify(options.body),
    signal: AbortSignal.timeout(config.requestTimeoutMs)
  });
  const body = await readBoundedBody(response, config.maximumResponseBytes);
  return {
    ok: response.ok,
    status: response.status,
    contentType: response.headers.get("content-type") || "",
    durationMs: Number((performance.now() - started).toFixed(3)),
    body
  };
}

function validateStructured(value) {
  assert(value && typeof value === "object" && !Array.isArray(value), "structured_not_object");
  assert(value.status === "ready", "structured_status");
  assert(value.count === 1, "structured_count");
  assert(Object.keys(value).every((key) => ["status", "count"].includes(key)), "structured_extra_property");
}

function validateToolCall(message, contract) {
  const calls = message?.tool_calls;
  assert(Array.isArray(calls) && calls.length === 1, "tool_call_count");
  const call = calls[0];
  assert(call?.type === "function", "tool_call_type");
  assert(call?.function?.name === contract.tool.name, "tool_call_name");
  const args = JSON.parse(call.function.arguments);
  assert(args.service === "api", "tool_call_argument");
  assert(Object.keys(args).length === 1, "tool_call_extra_argument");
}

function parseSse(body, maximumEvents) {
  const events = [];
  let done = false;
  for (const block of body.split(/\r?\n\r?\n/)) {
    for (const line of block.split(/\r?\n/)) {
      if (!line.startsWith("data:")) continue;
      const data = line.slice(5).trim();
      if (data === "[DONE]") {
        done = true;
        continue;
      }
      if (!data) continue;
      events.push(JSON.parse(data));
      if (events.length > maximumEvents) throw new Error("too_many_stream_events");
    }
  }
  return { events, done };
}

async function runCase(provider, name, operation) {
  const startedAt = new Date().toISOString();
  try {
    const evidence = await operation();
    return { name, status: "pass", startedAt, ...evidence };
  } catch (error) {
    return {
      name,
      status: "fail",
      startedAt,
      errorClass: String(error?.message || "unknown").slice(0, 160)
    };
  }
}

async function evaluateProvider(provider, config, contract) {
  const cases = [];
  cases.push(await runCase(provider, "models", async () => {
    const response = await request(provider, config, "/models", { method: "GET" });
    assert(response.ok, "models_http_" + response.status);
    const body = JSON.parse(response.body);
    assert(Array.isArray(body.data), "models_data");
    assert(body.data.some((item) => item.id === provider.model), "served_model_missing");
    return { httpStatus: response.status, durationMs: response.durationMs, modelCount: body.data.length };
  }));

  cases.push(await runCase(provider, "chat", async () => {
    const response = await request(provider, config, contract.chat.path, {
      body: {
        model: provider.model,
        messages: [{ role: "user", content: contract.syntheticInput }],
        temperature: 0,
        max_tokens: 16
      }
    });
    assert(response.ok, "chat_http_" + response.status);
    const body = JSON.parse(response.body);
    for (const path of contract.chat.requiredResponsePaths) {
      assert(readPath(body, path) !== undefined, "chat_missing:" + path);
    }
    const text = String(body.choices[0].message.content || "");
    return {
      httpStatus: response.status,
      durationMs: response.durationMs,
      outputHash: sha256(text),
      outputChars: text.length
    };
  }));

  cases.push(await runCase(provider, "stream", async () => {
    const response = await request(provider, config, contract.stream.path, {
      accept: "text/event-stream",
      body: {
        model: provider.model,
        messages: [{ role: "user", content: contract.syntheticInput }],
        temperature: 0,
        max_tokens: 16,
        stream: true,
        stream_options: { include_usage: true }
      }
    });
    assert(response.ok, "stream_http_" + response.status);
    assert(response.contentType.includes("text/event-stream"), "stream_content_type");
    const parsed = parseSse(response.body, contract.stream.maximumEvents);
    const deltas = parsed.events
      .map((event) => event.choices?.[0]?.delta?.content)
      .filter((value) => typeof value === "string" && value.length > 0);
    assert(!contract.stream.requireDataDone || parsed.done, "stream_missing_done");
    assert(!contract.stream.requireAtLeastOneTextDelta || deltas.length > 0, "stream_missing_text_delta");
    return {
      httpStatus: response.status,
      durationMs: response.durationMs,
      eventCount: parsed.events.length,
      textDeltaCount: deltas.length,
      outputHash: sha256(deltas.join(""))
    };
  }));

  cases.push(await runCase(provider, "structured-output", async () => {
    const response = await request(provider, config, contract.chat.path, {
      body: {
        model: provider.model,
        messages: [{
          role: "user",
          content: "Return status ready and count 1 using only the requested JSON schema."
        }],
        temperature: 0,
        max_tokens: 80,
        response_format: {
          type: "json_schema",
          json_schema: {
            name: "compat_record",
            strict: true,
            schema: contract.structured.schema
          }
        }
      }
    });
    assert(response.ok, "structured_http_" + response.status);
    const body = JSON.parse(response.body);
    const content = body.choices?.[0]?.message?.content;
    validateStructured(JSON.parse(content));
    return { httpStatus: response.status, durationMs: response.durationMs, outputHash: sha256(content) };
  }));

  cases.push(await runCase(provider, "tool-call", async () => {
    const response = await request(provider, config, contract.chat.path, {
      body: {
        model: provider.model,
        messages: [{ role: "user", content: "Check the status of service api." }],
        temperature: 0,
        max_tokens: 100,
        tools: [{
          type: "function",
          function: {
            name: contract.tool.name,
            description: "Return the synthetic status of an allowlisted service.",
            parameters: contract.tool.parameters
          }
        }],
        tool_choice: {
          type: "function",
          function: { name: contract.tool.name }
        },
        parallel_tool_calls: false
      }
    });
    assert(response.ok, "tool_http_" + response.status);
    const body = JSON.parse(response.body);
    validateToolCall(body.choices?.[0]?.message, contract);
    return { httpStatus: response.status, durationMs: response.durationMs, requestedTool: contract.tool.name };
  }));

  cases.push(await runCase(provider, "responses", async () => {
    const response = await request(provider, config, contract.responses.path, {
      body: {
        model: provider.model,
        input: contract.syntheticInput,
        temperature: 0,
        max_output_tokens: 16
      }
    });
    assert(response.ok, "responses_http_" + response.status);
    const body = JSON.parse(response.body);
    for (const path of contract.responses.requiredResponsePaths) {
      assert(readPath(body, path) !== undefined, "responses_missing:" + path);
    }
    return {
      httpStatus: response.status,
      durationMs: response.durationMs,
      responseObject: String(body.object || ""),
      outputItemCount: Array.isArray(body.output) ? body.output.length : 0
    };
  }));

  return {
    providerId: provider.id,
    product: provider.product,
    declaredEngineVersion: provider.engineVersion,
    model: provider.model,
    artifactDigest: provider.artifactDigest,
    testedAt: new Date().toISOString(),
    cases,
    pass: cases.every((item) => item.status === "pass")
  };
}

const [manifestPath, providersPath, contractPath, outputPath] = process.argv.slice(2);
assert(manifestPath && providersPath && contractPath && outputPath, "usage: compat release-manifest.json providers.json contract.json output.json");
const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
const providersConfig = JSON.parse(await readFile(providersPath, "utf8"));
const contract = JSON.parse(await readFile(contractPath, "utf8"));
assert(Number.isInteger(providersConfig.requestTimeoutMs), "invalid_timeout");
assert(Number.isInteger(providersConfig.maximumResponseBytes), "invalid_response_cap");
assert(Array.isArray(providersConfig.providers) && providersConfig.providers.length > 0, "providers_required");
const results = [];
for (const provider of providersConfig.providers) {
  results.push(await evaluateProvider(provider, providersConfig, contract));
}
const report = {
  schemaVersion: 1,
  contractRevision: contract.contractRevision,
  manifestSha256: sha256(JSON.stringify(manifest)),
  generatedAt: new Date().toISOString(),
  measuredFactsOnly: true,
  products: results
};
await mkdir(dirname(outputPath), { recursive: true });
await writeFile(outputPath, JSON.stringify(report, null, 2) + "\n", { flag: "wx" });
console.log(JSON.stringify({
  report: outputPath,
  providers: results.length,
  passing: results.filter((item) => item.pass).length
}));

export class PortableChatClient {
  constructor({ baseUrl, apiKey, model, timeoutMs = 45000, maxResponseBytes = 2097152 }) {
    const parsed = new URL(baseUrl);
    if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) {
      throw new Error("portable_client_invalid_url");
    }
    if (!["127.0.0.1", "localhost", "[::1]"].includes(parsed.hostname)) {
      throw new Error("portable_client_requires_loopback");
    }
    if (parsed.pathname !== "/v1" && parsed.pathname !== "/v1/") {
      throw new Error("portable_client_requires_v1_base");
    }
    this.baseUrl = parsed.toString().replace(/\/$/, "");
    this.apiKey = apiKey;
    this.model = model;
    this.timeoutMs = timeoutMs;
    this.maxResponseBytes = maxResponseBytes;
  }

  async chat(messages, options = {}) {
    const supported = new Set([
      "temperature",
      "max_tokens",
      "stop",
      "seed",
      "response_format",
      "tools",
      "tool_choice",
      "parallel_tool_calls"
    ]);
    for (const key of Object.keys(options)) {
      if (!supported.has(key)) throw new Error("unsupported_portable_option:" + key);
    }
    const response = await fetch(this.baseUrl + "/chat/completions", {
      method: "POST",
      redirect: "error",
      headers: {
        authorization: "Bearer " + this.apiKey,
        "content-type": "application/json"
      },
      body: JSON.stringify({ model: this.model, messages, ...options }),
      signal: AbortSignal.timeout(this.timeoutMs)
    });
    if (!response.ok) {
      throw new Error("portable_chat_http_" + response.status);
    }
    const declaredBytes = Number(response.headers.get("content-length") || "0");
    if (declaredBytes > this.maxResponseBytes) throw new Error("portable_chat_response_too_large");
    const bytes = new Uint8Array(await response.arrayBuffer());
    if (bytes.byteLength > this.maxResponseBytes) throw new Error("portable_chat_response_too_large");
    const body = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
    const choice = body?.choices?.[0];
    if (!choice?.message || typeof choice.finish_reason !== "string") {
      throw new Error("portable_chat_shape");
    }
    return {
      id: String(body.id || ""),
      model: String(body.model || this.model),
      finishReason: choice.finish_reason,
      content: typeof choice.message.content === "string" ? choice.message.content : null,
      toolCalls: Array.isArray(choice.message.tool_calls) ? choice.message.tool_calls : [],
      usage: body.usage ?? null
    };
  }
}

import assert from "node:assert/strict";
import test from "node:test";
import { PortableChatClient } from "../src/portable-client.mjs";

async function withFetch(replacement, operation) {
  const original = globalThis.fetch;
  globalThis.fetch = replacement;
  try {
    return await operation();
  } finally {
    globalThis.fetch = original;
  }
}

test("portable client rejects a non-loopback endpoint", () => {
  assert.throws(
    () => new PortableChatClient({
      baseUrl: "https://example.com/v1",
      apiKey: "synthetic",
      model: "synthetic"
    }),
    /requires_loopback/
  );
});

test("portable client rejects credentials embedded in the URL", () => {
  assert.throws(
    () => new PortableChatClient({
      baseUrl: "http://user:password@127.0.0.1:8000/v1",
      apiKey: "synthetic",
      model: "synthetic"
    }),
    /invalid_url/
  );
});

test("portable client rejects a non-HTTP protocol", () => {
  assert.throws(
    () => new PortableChatClient({
      baseUrl: "ftp://127.0.0.1/v1",
      apiKey: "synthetic",
      model: "synthetic"
    }),
    /invalid_url/
  );
});

test("portable client requires a v1 base path", () => {
  assert.throws(
    () => new PortableChatClient({
      baseUrl: "http://127.0.0.1:8000/native",
      apiKey: "synthetic",
      model: "synthetic"
    }),
    /requires_v1_base/
  );
});

test("portable client accepts reviewed loopback address forms", () => {
  for (const baseUrl of [
    "http://127.0.0.1:8000/v1",
    "http://localhost:8000/v1",
    "http://[::1]:8000/v1"
  ]) {
    assert.doesNotThrow(() => new PortableChatClient({
      baseUrl,
      apiKey: "synthetic",
      model: "synthetic"
    }));
  }
});

test("portable client rejects an untested option before network access", async () => {
  const client = new PortableChatClient({
    baseUrl: "http://127.0.0.1:65534/v1",
    apiKey: "synthetic",
    model: "synthetic"
  });
  await assert.rejects(
    () => client.chat([{ role: "user", content: "synthetic" }], { vendor_magic: true }),
    /unsupported_portable_option/
  );
});

test("portable client maps a valid response to the narrow result", async () => {
  await withFetch(
    async () => new Response(JSON.stringify({
      id: "chatcmpl-synthetic",
      model: "synthetic",
      choices: [{
        finish_reason: "stop",
        message: { role: "assistant", content: "READY", tool_calls: [] }
      }],
      usage: { prompt_tokens: 4, completion_tokens: 1 }
    }), { status: 200, headers: { "content-type": "application/json" } }),
    async () => {
      const client = new PortableChatClient({
        baseUrl: "http://127.0.0.1:8000/v1",
        apiKey: "synthetic",
        model: "synthetic"
      });
      const result = await client.chat([{ role: "user", content: "synthetic" }]);
      assert.equal(result.content, "READY");
      assert.equal(result.finishReason, "stop");
      assert.equal(result.usage.completion_tokens, 1);
    }
  );
});

test("portable client rejects an HTTP failure without parsing the body", async () => {
  await withFetch(
    async () => new Response("synthetic error", { status: 503 }),
    async () => {
      const client = new PortableChatClient({
        baseUrl: "http://127.0.0.1:8000/v1",
        apiKey: "synthetic",
        model: "synthetic"
      });
      await assert.rejects(() => client.chat([]), /portable_chat_http_503/);
    }
  );
});

test("portable client rejects an unknown success response shape", async () => {
  await withFetch(
    async () => new Response(JSON.stringify({ result: "READY" }), {
      status: 200,
      headers: { "content-type": "application/json" }
    }),
    async () => {
      const client = new PortableChatClient({
        baseUrl: "http://127.0.0.1:8000/v1",
        apiKey: "synthetic",
        model: "synthetic"
      });
      await assert.rejects(() => client.chat([]), /portable_chat_shape/);
    }
  );
});

test("portable client rejects a declared oversized response", async () => {
  await withFetch(
    async () => new Response("{}", {
      status: 200,
      headers: { "content-length": "1000", "content-type": "application/json" }
    }),
    async () => {
      const client = new PortableChatClient({
        baseUrl: "http://127.0.0.1:8000/v1",
        apiKey: "synthetic",
        model: "synthetic",
        maxResponseBytes: 16
      });
      await assert.rejects(() => client.chat([]), /response_too_large/);
    }
  );
});

test("portable client rejects an actually oversized response", async () => {
  await withFetch(
    async () => ({
      ok: true,
      status: 200,
      headers: { get: () => null },
      arrayBuffer: async () => new TextEncoder().encode("x".repeat(128)).buffer
    }),
    async () => {
      const client = new PortableChatClient({
        baseUrl: "http://127.0.0.1:8000/v1",
        apiKey: "synthetic",
        model: "synthetic",
        maxResponseBytes: 16
      });
      await assert.rejects(() => client.chat([]), /response_too_large/);
    }
  );
});

test("portable client sends authentication and redirect safeguards", async () => {
  let observed;
  await withFetch(
    async (url, init) => {
      observed = { url: String(url), init };
      return new Response(JSON.stringify({
        id: "chatcmpl-synthetic",
        choices: [{ finish_reason: "stop", message: { content: "READY" } }]
      }), { status: 200, headers: { "content-type": "application/json" } });
    },
    async () => {
      const client = new PortableChatClient({
        baseUrl: "http://127.0.0.1:8000/v1",
        apiKey: "secret-not-logged",
        model: "synthetic"
      });
      await client.chat([{ role: "user", content: "synthetic" }], { temperature: 0 });
    }
  );
  assert.equal(observed.url, "http://127.0.0.1:8000/v1/chat/completions");
  assert.equal(observed.init.redirect, "error");
  assert.equal(observed.init.headers.authorization, "Bearer secret-not-logged");
});