Build a hybrid bot that routes between local and cloud LLMs

import { createHmac, randomUUID } from "node:crypto";
import { appendFile } from "node:fs/promises";
import OpenAI from "openai";
import { z } from "zod";

const env = z.object({
  OPENAI_API_KEY: z.string().min(20),
  OPENAI_MODEL: z.string().default("gpt-5.4-nano"),
  OLLAMA_MODEL: z.string().min(1),
  AUDIT_HMAC_KEY: z.string().min(32),
  AUDIT_FILE: z.string().default("/var/lib/hybrid-router/audit.ndjson")
}).parse(process.env);

const requestSchema = z.object({
  input: z.string().min(1).max(32_768),
  dataClass: z.enum(["public", "internal", "confidential", "restricted"]),
  capability: z.enum(["chat", "json"]),
  allowCloud: z.boolean(),
  confirmCloud: z.boolean(),
  minimumValidationScore: z.number().min(0.5).max(1).default(0.8)
}).strict();

type Request = z.infer<typeof requestSchema>;
type Route = "local" | "cloud";
type Decision = {
  route: Route;
  reason: string;
  cloudEligible: boolean;
  requiresApproval: boolean;
};

const local = new OpenAI({
  baseURL: "http://127.0.0.1:11434/v1",
  apiKey: "ollama"
});
const cloud = new OpenAI({ apiKey: env.OPENAI_API_KEY });

export function decide(input: Request, localAvailable: boolean): Decision {
  const cloudEligible =
    input.allowCloud &&
    input.confirmCloud &&
    (input.dataClass === "public" || input.dataClass === "internal");
  if (localAvailable) {
    return {
      route: "local",
      reason: input.dataClass === "restricted" ? "restricted_local_only" : "local_first",
      cloudEligible,
      requiresApproval: false
    };
  }
  if (cloudEligible) {
    return {
      route: "cloud",
      reason: "local_unavailable_cloud_explicitly_approved",
      cloudEligible,
      requiresApproval: true
    };
  }
  throw new Error("no_policy_compliant_route");
}

async function localAvailable(): Promise<boolean> {
  try {
    const result = await local.models.list({ timeout: 2_000 });
    return result.data.some((item) => item.id === env.OLLAMA_MODEL);
  } catch {
    return false;
  }
}

function validateAnswer(answer: string, capability: Request["capability"]): number {
  if (!answer.trim() || answer.length > 32_768) return 0;
  if (capability === "json") {
    try {
      const parsed = JSON.parse(answer) as unknown;
      return parsed && typeof parsed === "object" ? 1 : 0;
    } catch {
      return 0;
    }
  }
  const hasUncertainty = /uncertain|cannot verify|verify|evidence/i.test(answer);
  const hasUnsafeClaim = /I (ran|executed|changed|deleted)/i.test(answer);
  return Math.max(0, 0.75 + (hasUncertainty ? 0.15 : 0) - (hasUnsafeClaim ? 0.75 : 0));
}

async function callLocal(input: Request): Promise<string> {
  const response = await local.chat.completions.create({
    model: env.OLLAMA_MODEL,
    messages: [
      { role: "system", content: "Return an unverified draft. Never claim to execute actions. State uncertainty and verification needs." },
      { role: "user", content: input.input }
    ],
    temperature: 0.2
  });
  return response.choices[0]?.message.content ?? "";
}

async function callCloud(input: Request): Promise<string> {
  const response = await cloud.responses.create({
    model: env.OPENAI_MODEL,
    input: input.input,
    instructions: "Return an unverified draft. Never claim to execute actions. State uncertainty and verification needs.",
    store: false,
    max_output_tokens: 1200
  });
  return response.output_text;
}

async function audit(event: Record<string, unknown>, input: Request): Promise<void> {
  const digest = createHmac("sha256", env.AUDIT_HMAC_KEY)
    .update(input.input)
    .digest("hex");
  await appendFile(env.AUDIT_FILE, JSON.stringify({
    timestamp: new Date().toISOString(),
    promptDigest: digest,
    dataClass: input.dataClass,
    capability: input.capability,
    ...event
  }) + "\n", { encoding: "utf8", mode: 0o600 });
}

export async function route(raw: unknown): Promise<Record<string, unknown>> {
  const requestId = randomUUID();
  const input = requestSchema.parse(raw);
  const decision = decide(input, await localAvailable());
  const answer = decision.route === "local"
    ? await callLocal(input)
    : await callCloud(input);
  const score = validateAnswer(answer, input.capability);
  const accepted = score >= input.minimumValidationScore;
  await audit({
    requestId,
    route: decision.route,
    reason: decision.reason,
    policyRevision: "2026-07-29.1",
    model: decision.route === "local" ? env.OLLAMA_MODEL : env.OPENAI_MODEL,
    validationScore: score,
    accepted
  }, input);
  return {
    requestId,
    route: decision.route,
    answer: accepted ? answer : null,
    validationScore: score,
    notice: accepted
      ? "Unverified draft; review before use."
      : "Answer withheld because deterministic validation did not pass.",
    retry: accepted ? null : {
      localOnly: input.dataClass === "confidential" || input.dataClass === "restricted",
      cloudPossibleOnlyWithNewExplicitApproval:
        input.dataClass === "public" || input.dataClass === "internal"
    }
  };
}

import assert from "node:assert/strict";
import test from "node:test";
import { decide } from "../src/router.js";

const base = {
  input: "synthetic",
  capability: "chat" as const,
  minimumValidationScore: 0.8
};

test("restricted data never routes to cloud", () => {
  assert.throws(() => decide({
    ...base,
    dataClass: "restricted",
    allowCloud: true,
    confirmCloud: true
  }, false), /no_policy_compliant_route/);
});

test("cloud fallback needs both policy and explicit confirmation", () => {
  assert.throws(() => decide({
    ...base,
    dataClass: "internal",
    allowCloud: true,
    confirmCloud: false
  }, false));
  assert.equal(decide({
    ...base,
    dataClass: "internal",
    allowCloud: true,
    confirmCloud: true
  }, false).route, "cloud");
});

test("available local model remains the default", () => {
  assert.equal(decide({
    ...base,
    dataClass: "public",
    allowCloud: true,
    confirmCloud: true
  }, true).route, "local");
});

OLLAMA_MODEL={{ollamaModel}}
OPENAI_MODEL=gpt-5.4-nano
OPENAI_API_KEY=provision-from-secret-manager
AUDIT_HMAC_KEY=provision-an-independent-random-secret
AUDIT_FILE=/var/lib/hybrid-router/audit.ndjson