engine.js

/**
 * Countersign Engine — standalone RAG server (Node 18+, zero dependencies)
 * -------------------------------------------------------------------------
 * A self-contained answer engine: holds your API key, holds your knowledge
 * base, and turns questions into documentation-grounded answers over HTTP.
 * Anything can call it — the Countersign web app, a CRM, a Slack bot, curl.
 *
 * Knowledge base sources (all optional, combined):
 *   1. countersign-data.json  — a workspace exported from the web app
 *                               (the app parses PDF/DOCX/XLSX in-browser,
 *                               so this is how binary formats get in)
 *   2. kb/ directory          — plain .txt / .md / .csv files
 *   3. POST /ingest           — push documents at runtime (persisted to kb/)
 *
 * Run:
 *   ANTHROPIC_API_KEY=sk-ant-... COUNTERSIGN_TOKEN=secret node engine.js
 *
 * Env:
 *   PORT=8790                 listen port
 *   HOST=127.0.0.1            listen address; any other address requires COUNTERSIGN_TOKEN
 *                             unless ALLOW_NO_TOKEN=1
 *   KB_DIR=./kb               directory of plain-text sources
 *   DATA_FILE=./countersign-data.json
 *   MODEL=claude-sonnet-5-5   generation model
 *   ALLOW_ORIGIN=*            CORS origin for browser callers; with neither this nor a token
 *                             set, requests from other origins are refused
 *   COUNTERSIGN_TOKEN=...     shared secret; required on every POST when set
 *   CONCURRENCY=2             parallel upstream calls for /batch (1 to 4)
 *
 * Endpoints (JSON in, JSON out; X-Team-Token header when token is set):
 *   GET  /health   -> { ok, docs, chunks, model, token_required }
 *   POST /search   -> { query, k? }            => top chunks, no AI call
 *   POST /answer   -> { question, tone?, type?, max_words? }
 *                                              => grounded answer + sources
 *   POST /ingest   -> { name, text }           => add/replace a document
 *   POST /forget   -> { name }                 => remove a document
 *   POST /proxy    -> raw Anthropic /v1/messages pass-through (web-app compat)
 */

const http = require("http");
const net = require("net");
const crypto = require("crypto");
const fs = require("fs");
const path = require("path");

/* ---------------- config ---------------- */
const API_KEY      = process.env.ANTHROPIC_API_KEY || "";
const TEAM_TOKEN   = process.env.COUNTERSIGN_TOKEN || "";
const PORT         = parseInt(process.env.PORT || "8790", 10);
const HOST         = process.env.HOST || "127.0.0.1";
const ALLOW_NO_TOKEN = process.env.ALLOW_NO_TOKEN === "1";
const KB_DIR       = process.env.KB_DIR || path.join(__dirname, "kb");
const DATA_FILE    = process.env.DATA_FILE || path.join(__dirname, "countersign-data.json");
const MODEL        = process.env.MODEL || "claude-sonnet-5-5";
const ALLOW_ORIGIN = process.env.ALLOW_ORIGIN || "*";
const CHUNK_SIZE     = 1100;   // chars per retrieval chunk
const CONTEXT_BUDGET = 11000;  // chars of retrieved context per answer
const MAX_BODY_BYTES = 1_500_000; // /ingest can carry a whole document
const MAX_DOC_CHARS  = 220000;
// Retrieval cost grows with (query terms x chunks); an unbounded question could pin the
// event loop for tens of seconds (a 20k-term question took ~22s on a 60k-chunk KB).
const MAX_QUESTION_CHARS = 4000;
const MAX_QUERY_TERMS    = 80;
const MAX_TONE_CHARS     = 80;
const CONCURRENCY    = Math.min(Math.max(parseInt(process.env.CONCURRENCY || "2", 10) || 2, 1), 4);
const MAX_BATCH      = 100;
// Current Claude models think before answering and thinking tokens count against
// max_tokens, so an answer-sized cap can be spent before any text is written.
const ANSWER_MAX_TOKENS = 8192;
const PROXY_MAX_TOKENS = 16384; // ceiling for the raw /proxy pass-through

/* ================================================================
   Knowledge base: load, chunk, TF-IDF index
================================================================ */
/* Stop words. RFP questions are mostly instruction verbiage ("confirm whether your
   organisation holds…"), and every word left in here ends up in `missing_terms`,
   which the gap report presents to the user as documentation they should go and write.
   Only genuinely topic-free words belong here. Words like "level" (service level),
   "type" (SOC 2 Type II) and "number" (certificate number) must stay searchable. */
const STOP = new Set(("the a an and or of to in for with on is are does do your our you we can how what which " +
  "will be by as at from that this it its any all provide describe please detail such other " +
  "state outline confirm explain list specify identify give including included include " +
  "whether have has had been being was were would should must shall may might able " +
  "used make makes made take taken hold holds held ensure kindly " +
  "each every both either also more most much many few less least own same than then " +
  "there their they them these those when where who whom why while about above below " +
  "into onto over under between during after before within without across per via " +
  "if but not only just still even ever never always currently " +
  "relevant applicable appropriate additional further brief often").split(/\s+/));
/* Light stemmer so query/doc inflections match ("supports"→"support", "policies"→"policy").
   Must be applied identically at index time (termFreqs) and query time — tf/df keys are stems. */
const stem = (w) => {
  // joined numbers, percentages and CJK pairs are matched exactly as written
  if (!/^\p{L}+$/u.test(w) || CJK.test(w)) return w;
  const base = stripSuffix(w);
  // Fold a trailing "e" so "service"/"services", "type"/"types", "license"/"licensing"
  // and "price"/"pricing" meet at one stem (stripping "es"/"ing" alone left "servic" vs "service").
  return base.length > 3 && base.endsWith("e") ? base.slice(0, -1) : base;
};
function stripSuffix(w) {
  if (w.length > 4 && w.endsWith("ies")) return w.slice(0, -3) + "y";
  if (w.length > 5 && w.endsWith("ing")) return w.slice(0, -3);
  // "es" is only a plural marker after a sibilant (processes, boxes, matches); elsewhere
  // it is just "s" after a stem ending in "e" (services, types), so drop one letter only.
  if (w.length > 4 && w.endsWith("es")) return /(s|x|z|ch|sh)es$/.test(w) ? w.slice(0, -2) : w.slice(0, -1);
  // "agreed" must meet "agree": drop only the d, or the trailing-e fold strips a second letter
  if (w.length > 4 && w.endsWith("eed")) return w.slice(0, -1);
  if (w.length > 4 && w.endsWith("ed")) return w.slice(0, -2);
  if (w.length > 3 && w.endsWith("s") && !w.endsWith("ss")) return w.slice(0, -1);
  return w;
}
/* Tokens, the same for documents and questions. Letters and digits of any script are word
   characters, so accented words stay whole ("zürich", "données"). Han and kana are written
   without spaces, so a run of them becomes overlapping character pairs. A short number
   after a word ("SOC 2", "TLS 1.3", "OAuth 2.0", "Type II") also yields a joined token
   ("soc2", "tls1.3", "oauth2", "type2"): the bare number is too short to index, and
   "SOC 1" must not look like "SOC 2". A percentage ("99.9%") is kept whole. */
const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;
const CJK_SPLIT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]+|[^\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]+/gu;
// "Section 3", "Question 7": numbered references, not topics
const REF_WORDS = new Set(("section sections question questions requirement requirements req item items part " +
  "clause appendix annex page step table figure exhibit attachment chapter paragraph point no number q").split(" "));
const ROMAN = { i: "1", ii: "2", iii: "3", iv: "4" };
function textTokens(s) {
  const text = String(s).normalize("NFKC").toLowerCase();
  const out = [];
  for (const run of text.split(/[^\p{L}\p{N}]+/u)) {
    if (!run) continue;
    if (!CJK.test(run)) { out.push(run); continue; }
    for (const part of run.match(CJK_SPLIT)) {
      if (!CJK.test(part)) { out.push(part); continue; }
      if (part.length === 1) out.push(part);
      for (let i = 0; i + 1 < part.length; i++) out.push(part.slice(i, i + 2));
    }
  }
  const joined = (word, num, separated) => {
    if (STOP.has(word) || REF_WORDS.has(word) || CJK.test(word)) return;
    // "soc2" written without a space is already a token of its own
    if (separated || num.includes(".")) out.push(word + num.replace(/\.0+$/, ""));
  };
  for (const m of text.matchAll(/(?<![\p{L}\p{N}])(\p{L}+)([ -]?)(\d{1,2}(?:\.\d{1,2})?)(?![\p{L}\p{N}]|\.\d|\s?%)/gu))
    joined(m[1], m[3], !!m[2]);
  for (const m of text.matchAll(/(?<![\p{L}\p{N}])(type|tier|level|phase|class|stage)[ -](iv|i{1,3})(?![\p{L}\p{N}])/gu))
    joined(m[1], ROMAN[m[2]], true);
  for (const m of text.matchAll(/(?<![\d.])(\d+(?:\.\d+)?)\s?%/g)) out.push(m[1] + "%");
  return out;
}
/* Two-letter words are dropped as noise EXCEPT, in questions, the ones the alias table
   knows about ("dr", "sla"…), otherwise those aliases could never fire. */
const keepToken = (w, query) => CJK.test(w) || (/\d/.test(w) && /[\p{L}%]/u.test(w)) ||
  ((w.length > 2 || (query && ALIASES[w])) && !STOP.has(w));
const tokenize = (s) => [...new Set(textTokens(s).filter((w) => keepToken(w, true)))];
const termFreqs = (s) => {
  const tf = Object.create(null);
  for (const w of textTokens(s)) if (keepToken(w, false)) { const t = stem(w); tf[t] = (tf[t] || 0) + 1; }
  return tf;
};

// files: document key -> the kb/ file that backs it (documents from DATA_FILE have none)
const KB = { docs: new Map(), files: new Map(), chunks: [], df: Object.create(null) };

/* The index is updated per document, so pushing a bucket of n documents costs n
   document-sized updates rather than n rebuilds of everything (which blocked every
   other request for minutes on a large bucket). idf is derived from df at query time. */
function indexDoc(name, text) {
  for (let i = 0; i < text.length; i += CHUNK_SIZE) {
    const t = text.slice(i, i + CHUNK_SIZE + 200);
    const tf = termFreqs(t);
    KB.chunks.push({ doc: name, text: t, tf });
    for (const term in tf) KB.df[term] = (KB.df[term] || 0) + 1;
  }
}
function unindexDoc(name) {
  KB.chunks = KB.chunks.filter((ch) => {
    if (ch.doc !== name) return true;
    for (const term in ch.tf) if (--KB.df[term] === 0) delete KB.df[term];
    return false;
  });
}
function reindex() {
  KB.chunks = [];
  KB.df = Object.create(null);
  for (const [name, text] of KB.docs) indexDoc(name, text);
  console.log(`[kb] indexed ${KB.docs.size} docs -> ${KB.chunks.length} chunks`);
}

/* Names the boot scanner deliberately skips: persisting a document under one of them
   would clobber a folder doc and the document itself would vanish on the next restart. */
const RESERVED_NAMES = /^(README\.txt|countersign-data\.json)$/i;
/* The file name a pushed document is stored under. Replacing characters or cutting a long
   name loses information, so two different names ("Policy (2024).pdf", "Policy [2024].pdf")
   could land on one file and silently overwrite each other: such names get a short hash
   of the original. Applying it to its own output changes nothing. */
const canonicalName = (name) => {
  const raw = String(name);
  const cleaned = raw.replace(/[^a-zA-Z0-9 ._~-]/g, "_");
  const ext = (/\.(txt|md|csv|json)$/i.exec(cleaned) || [""])[0];
  let stem = cleaned.slice(0, cleaned.length - ext.length);
  if (cleaned !== raw || stem.length > 120 || !stem) {
    const tag = crypto.createHash("sha256").update(raw, "utf8").digest("hex").slice(0, 8);
    stem = (stem || "doc").slice(0, 111) + "~" + tag;
  }
  const named = stem + (ext || ".txt");
  return RESERVED_NAMES.test(named) ? "_" + named : named;
};
// The name the scheme above replaced, so files stored by an older engine are still found.
const legacyName = (name) => {
  const safe = String(name).replace(/[^a-zA-Z0-9 ._-]/g, "_").slice(0, 120) || "doc";
  const named = /\.(txt|md|csv|json)$/i.test(safe) ? safe : safe + ".txt";
  return RESERVED_NAMES.test(named) ? "_" + named : named;
};
function addDoc(name, text, persist) {
  // CRLF and LF copies of one file must chunk identically: chunks are cut by length
  text = String(text).replace(/\r\n?/g, "\n");
  // measured after stripping NULs, or any document containing one reports truncated
  const rawChars = text.split(String.fromCharCode(0)).join("").trim().length;
  text = text.replace(/\u0000/g, "").trim().slice(0, MAX_DOC_CHARS);
  if (!text) return false;
  // persisted docs are stored under their canonical filename, so use the same
  // key in memory — /forget and re-ingest then behave identically across restarts
  const key = persist ? canonicalName(name) : name;
  // A pushed document replaces any copy loaded under its original name (from DATA_FILE,
  // or a kb/ file whose name is not canonical) or stored under the older naming scheme;
  // keeping both indexes the text twice.
  const superseded = [];
  if (persist) {
    for (const prior of new Set([String(name), legacyName(name)])) {
      if (prior === key || !KB.docs.has(prior)) continue;
      KB.docs.delete(prior);
      superseded.push(prior);
      const old = KB.files.get(prior);
      if (old && old !== key) { try { fs.unlinkSync(path.join(KB_DIR, old)); } catch (e) {} }
      KB.files.delete(prior);
    }
  }
  KB.docs.set(key, text);
  let persisted = null;
  if (persist) {
    try {
      fs.mkdirSync(KB_DIR, { recursive: true });
      fs.writeFileSync(path.join(KB_DIR, key), text);
      KB.files.set(key, key);
      persisted = true;
    } catch (e) { console.warn("[kb] persist failed:", e.message); persisted = false; }
  }
  return { key, superseded, persisted, chars: text.length, truncated: rawChars > text.length };
}

function loadKnowledgeBase() {
  // 1. workspace export from the web app
  const fromData = new Map();   // canonical name -> DATA_FILE document name
  const loaded = (name, stored) => {
    if (stored && stored.truncated) console.warn(`[kb] ${name}: only the first ${MAX_DOC_CHARS} characters are indexed`);
    return stored;
  };
  let raw = null;
  try { raw = fs.readFileSync(DATA_FILE, "utf8"); } catch (e) { /* file absent is fine */ }
  if (raw !== null) {
    let ws = null;
    try { ws = JSON.parse(raw); } catch (e) { console.warn(`[kb] ${path.basename(DATA_FILE)} is not valid JSON and was ignored: ${e.message}`); }
    if (ws && Array.isArray(ws.docs)) {
      for (const d of ws.docs) if (d.name && d.text && loaded(d.name, addDoc(d.name, d.text, false))) {
        fromData.set(canonicalName(d.name), String(d.name));
        fromData.set(legacyName(d.name), String(d.name));
      }
      console.log(`[kb] loaded ${ws.docs.length} docs from ${path.basename(DATA_FILE)}`);
    } else if (ws) console.warn(`[kb] ${path.basename(DATA_FILE)} has no docs array and was ignored`);
  }
  // 2. plain-text directory
  let names = [];
  try { names = fs.readdirSync(KB_DIR); } catch (e) { /* dir absent is fine */ }
  for (const f of names) {
    if (!/\.(txt|md|csv|json)$/i.test(f)) continue;
    if (/^countersign-data\.json$/i.test(f) || /^README\.txt$/i.test(f)) continue; // workspace exports & folder docs aren't KB sources
    try {
      if (!loaded(f, addDoc(f, fs.readFileSync(path.join(KB_DIR, f), "utf8"), false))) continue;
    } catch (e) { console.warn(`[kb] skipped ${f}: ${e.message}`); continue; }   // one unreadable entry must not end the scan
    KB.files.set(f, f);
    // a pushed copy of a DATA_FILE document supersedes it
    const dup = fromData.get(f) ?? fromData.get(canonicalName(f));
    if (dup !== undefined && dup !== f) KB.docs.delete(dup);
  }
  reindex();
}

/* Small alias table so acronym-heavy RFP questions still hit spelled-out docs. No
   prototype: a question containing "constructor" must not find Object.prototype's. */
const ALIASES = Object.assign(Object.create(null), {
  sso: ["single", "sign"], mfa: ["multi", "factor", "authentication"],
  dr: ["disaster", "recovery"], sla: ["service", "level", "uptime"],
  bcp: ["business", "continuity"], iam: ["identity", "access"],
  pii: ["personal", "data"], dpa: ["data", "processing"],
  uptime: ["availability"], pricing: ["price", "cost", "licensing"],
});
const expandTerms = (terms) => {
  const out = new Set(terms);
  for (const t of terms) if (ALIASES[t]) for (const a of ALIASES[t]) out.add(a);
  return [...out];
};

function retrieve(question, budget = CONTEXT_BUDGET, maxChunks = 12) {
  // Two term sets. `qTerms` is expanded with acronym aliases (ALIASES is keyed on
  // unstemmed words) and drives chunk scoring, so "SSO" can find "single sign-on".
  // `askedTerms` is what the user actually wrote, and is what coverage and
  // missing_terms are measured over: reporting the expansions would tell someone whose
  // docs say "SSO" that they are missing documentation on "single" and "sign".
  const rawTerms = tokenize(question);
  // only the first MAX_QUERY_TERMS terms drive chunk scoring (cost); coverage and
  // missing_terms still look at every term, so the tail of a long question is not ignored
  const scoreTerms = rawTerms.slice(0, MAX_QUERY_TERMS);
  const askedTerms = [...new Set(rawTerms.map(stem))];
  const qTerms = [...new Set(expandTerms(scoreTerms).map(stem))];
  const N = Math.max(1, KB.chunks.length);
  const idf = Object.create(null);
  for (const t of qTerms) idf[t] = KB.df[t] ? Math.log(1 + N / KB.df[t]) : 1;
  const scored = [];
  for (const ch of KB.chunks) {
    let score = 0;
    for (const t of qTerms) if (ch.tf[t]) score += ch.tf[t] * idf[t];
    if (score > 0) scored.push({ score, ch });
  }
  scored.sort((a, b) => b.score - a.score);
  const picked = [], sources = new Set();
  let used = 0;
  for (const s of scored) {
    if (used + s.ch.text.length > budget) continue;
    picked.push(s); sources.add(s.ch.doc); used += s.ch.text.length;
    if (picked.length >= maxChunks) break;
  }
  // ALIASES is keyed on unstemmed words, so look an alias up by the words the user wrote
  // (a stemmed key like "uptim" or "pric" can never match "uptime"/"pricing"). Several words
  // can share a stem ("pricing", "prices"), and any of them may carry the alias.
  const rawsOf = new Map();
  for (const r of rawTerms) {
    const s = stem(r);
    if (!rawsOf.has(s)) rawsOf.set(s, []);
    rawsOf.get(s).push(r);
  }
  const inKB = (t) => !!KB.df[t] || rawsOf.get(t).some(r => (ALIASES[r] || []).some(a => KB.df[stem(a)]));
  /* Coverage is how well the KB substantiates the QUESTION, so it is driven by which of
     the question's terms appear at all; retrieval depth only modulates it. (Adding a flat
     0.4 for "we found 6 chunks" put a floor under every score, so on any KB with a handful
     of chunks nothing could land under the gap report's threshold.)

     Terms are weighted by rarity, because a plain term count hides the gaps that matter:
     "Do you hold ISO 14001 certification?" shares "iso" and "certification" with an
     ISO 27001 policy, so it scored as half-covered while the one term that decides the
     answer ("14001") was absent. A term in many chunks is generic and counts for little;
     a term in none is maximally distinctive and counts for a lot. */
  const weightOf = (t) => 1 / (1 + (KB.df[t] || 0));
  let covWeight = 0, totWeight = 0;
  for (const t of askedTerms) {
    const w = weightOf(t);
    totWeight += w;
    if (inKB(t)) covWeight += w;
  }
  const termRatio = totWeight ? covWeight / totWeight : 0;
  const depth = Math.min(picked.length, 6) / 6;
  const coverage = askedTerms.length ? Math.min(1, termRatio * (0.75 + 0.25 * depth)) : 0;
  const blocks = picked.map(s => `[Source: ${s.ch.doc}]\n${s.ch.text}`);
  return {
    context: blocks.join("\n\n---\n\n"),
    // callers that want the individual chunks must use this — re-splitting `context`
    // on the separator breaks whenever a source contains a Markdown horizontal rule
    blocks,
    // the same chunks in rank order with their scores, for evaluation (eval/run.js)
    ranked: picked.map(s => ({ doc: s.ch.doc, score: s.score, text: s.ch.text })),
    sources: [...sources],
    coverage,
    chunks_used: picked.length,
    // the words the user wrote: a stem such as "mobil" or "sourc" is no use in a gap report
    missing_terms: askedTerms.filter(t => !inKB(t)).map(t => rawsOf.get(t)[0]),
  };
}

/* ================================================================
   Generation
================================================================ */
const TYPE_RULES = Object.assign(Object.create(null), {
  compliance: 'This is a COMPLIANCE question: open with a one-word verdict ("Yes.", "No." or "Partially."), then substantiate briefly. Only answer "Yes" if the reference material supports it.',
  technical:  "This is a TECHNICAL question: be concrete about architecture, standards and mechanisms; evaluators are engineers. Avoid marketing language.",
  commercial: "This is a COMMERCIAL question: describe the pricing/licensing model and flexibility without quoting exact figures unless they appear in the reference material.",
  narrative:  "This is a NARRATIVE question: tell a credible, client-focused story about capability and approach.",
});

function buildPrompt(question, retrieval, opts) {
  // tone is caller-supplied text that lands inside the system-style prompt: keep it to one
  // short line of plain text (no newlines to smuggle in extra instructions, no objects).
  const tone = (typeof opts.tone === "string" ? opts.tone.replace(/\s+/g, " ").trim().slice(0, MAX_TONE_CHARS) : "") || "professional and precise";
  const words = Math.min(Math.max(parseInt(opts.max_words, 10) || 220, 20), 500);
  const typeRule = (typeof opts.type === "string" && TYPE_RULES[opts.type]) || "";
  return `You are a senior proposal writer for a SaaS vendor, drafting an answer to one RFP question.

Write in first-person plural ("we", "our platform"), tone: ${tone}. Target about ${words} words. ${typeRule} Be specific and confident; never invent certifications, customers, SLAs or numbers that are not in the reference material. If the reference material doesn't cover something, answer at the level it supports and note what can be confirmed on request. Respond with the answer text only: no headings, no preamble.

REFERENCE MATERIAL FROM OUR DOCUMENTATION:
${retrieval.context || "(No matching documentation found. Write a competent generic SaaS response and clearly hedge specifics.)"}

RFP QUESTION: ${question}`;
}

async function callClaude(body) {
  try {
    const r = await fetch("https://api.anthropic.com/v1/messages", {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        "x-api-key": API_KEY,
        "anthropic-version": "2023-06-01",
      },
      body: JSON.stringify(body),
      signal: AbortSignal.timeout(120_000),
    });
    const text = await r.text();
    return { status: r.status, text };
  } catch (err) {
    return { status: 504, text: JSON.stringify({ error: { type: "timeout", message: "Upstream timed out or aborted: " + err.message } }) };
  }
}

/* Text of a Messages API response. A refusal, or a response that spent its whole
   budget before writing text, is an error: returning "" as an answer hides it. */
function answerText(data) {
  const text = (data.content || []).map(b => (b.type === "text" ? b.text : "")).join("").trim();
  if (data.stop_reason === "refusal") {
    const category = data.stop_details && data.stop_details.category;
    throw new Error("Model declined to answer" + (category ? " (" + category + ")" : ""));
  }
  if (!text) throw new Error(data.stop_reason === "max_tokens"
    ? "Model ran out of output tokens before answering" : "Model returned an empty answer");
  return text;
}

/* ================================================================
   HTTP server
================================================================ */
function send(res, code, obj) {
  res.writeHead(code, {
    "Content-Type": "application/json",
    "Access-Control-Allow-Origin": ALLOW_ORIGIN,
    "X-Content-Type-Options": "nosniff",
  });
  res.end(typeof obj === "string" ? obj : JSON.stringify(obj));
}
const fail = (res, code, message) => send(res, code, { error: { type: "engine_error", message } });

// Hash both sides so the compare is constant-time in the token's length as well.
const sha = (s) => crypto.createHash("sha256").update(String(s || ""), "utf8").digest();
/* With no team token and no ALLOW_ORIGIN, the only protection is that the port is local, and a
   web page open in the operator's own browser can still reach a local port. So refuse browser
   requests from other origins, and Host names that are not loopback (a DNS name rebound to
   127.0.0.1). curl, scripts and same-machine tools send no Origin and are unaffected. */
const ORIGIN_CONFIGURED = !!process.env.ALLOW_ORIGIN;
const isLoopback = (h) => h === "localhost" || h === "::1" || (net.isIPv4(h) && h.startsWith("127."));
function browserRefusal(req) {
  if (TEAM_TOKEN || ORIGIN_CONFIGURED) return null;
  const rawHost = String(req.headers.host || "").toLowerCase();
  const host = rawHost.startsWith("[") ? rawHost.slice(1, rawHost.indexOf("]")) : rawHost.split(":")[0];
  if (isLoopback(HOST) && host && !isLoopback(host))
    return "Host " + host + " is not allowed. Set COUNTERSIGN_TOKEN to serve other host names";
  const origin = req.headers.origin;
  if (origin && origin !== "http://" + req.headers.host)
    return "Requests from web pages need COUNTERSIGN_TOKEN or ALLOW_ORIGIN on the engine";
  return null;
}

function tokenOk(given) {
  return crypto.timingSafeEqual(sha(given), sha(TEAM_TOKEN));
}
const questionOk = (q) => typeof q === "string" && q.trim().length >= 3 && q.length <= MAX_QUESTION_CHARS;

const messagesOk = (m) => Array.isArray(m) && m.length > 0 && m.every((x) => x && typeof x === "object" && !Array.isArray(x));

const routes = {
  "/answer": async (res, body) => {
    if (!questionOk(body.question))
      return fail(res, 400, `Provide { question: string } of at most ${MAX_QUESTION_CHARS} characters`);
    const retrieval = retrieve(body.question);
    const up = await callClaude({
      model: MODEL,
      max_tokens: ANSWER_MAX_TOKENS,
      messages: [{ role: "user", content: buildPrompt(body.question, retrieval, body) }],
    });
    if (up.status !== 200) return send(res, up.status, up.text);
    let data;
    try { data = JSON.parse(up.text); } catch (e) { return fail(res, 502, "Bad upstream response"); }
    let answer;
    try { answer = answerText(data); } catch (e) { return fail(res, 502, e.message); }
    send(res, 200, {
      answer,
      sources: retrieval.sources,
      coverage: Math.round(retrieval.coverage * 100) / 100,
      chunks_used: retrieval.chunks_used,
      missing_terms: retrieval.missing_terms,
      model: MODEL,
    });
  },

  "/search": async (res, body) => {
    if (typeof body.query !== "string" || body.query.length > MAX_QUESTION_CHARS)
      return fail(res, 400, `Provide { query: string } of at most ${MAX_QUESTION_CHARS} characters`);
    const k = Math.min(Math.max(parseInt(body.k, 10) || 5, 1), 20);
    // chunks are up to CHUNK_SIZE+200 long and retrieve() stops at 12 by default, so both
    // the budget and the count must follow k or k>12 silently returned only 12
    const r = retrieve(body.query, (CHUNK_SIZE + 200) * k, k);
    send(res, 200, {
      query: body.query,
      coverage: Math.round(r.coverage * 100) / 100,
      sources: r.sources,
      missing_terms: r.missing_terms,
      chunks: r.blocks.slice(0, k),
    });
  },

  "/ingest": async (res, body) => {
    if (typeof body.name !== "string" || !body.name.trim() || typeof body.text !== "string")
      return fail(res, 400, "Provide { name: string, text: string } (parse binary formats client-side or via the web app)");
    const stored = addDoc(body.name, body.text, true);
    if (!stored) return fail(res, 400, "Document text is empty");
    for (const name of [stored.key, ...stored.superseded]) unindexDoc(name);
    indexDoc(stored.key, KB.docs.get(stored.key));
    // report truncation and persistence honestly — a silent ok:true here means the
    // caller believes the whole document is indexed and will survive a restart
    send(res, 200, {
      ok: true, name: stored.key, chars: stored.chars,
      truncated: stored.truncated, max_doc_chars: MAX_DOC_CHARS,
      persisted: stored.persisted,
      docs: KB.docs.size, chunks: KB.chunks.length,
    });
  },

  "/forget": async (res, body) => {
    // A missing name must not fall through to canonicalName("") -> "doc.txt" and
    // delete an unrelated document.
    if (typeof body.name !== "string" || !body.name.trim())
      return fail(res, 400, "Provide { name: string }");
    // Resolve the key we actually removed, and only unlink the file backing THAT key.
    // A document from the workspace export has no file, so forgetting it never removes
    // a same-named file that belongs to a different document.
    const canon = canonicalName(body.name);
    let key = null;
    if (KB.docs.delete(body.name)) key = body.name;
    else if (KB.docs.delete(canon)) key = canon;
    else if (KB.docs.delete(legacyName(body.name))) key = legacyName(body.name);
    if (!key) return fail(res, 404, "No such document");
    let removedFile = false;
    const file = KB.files.get(key);
    if (file) {
      try { fs.unlinkSync(path.join(KB_DIR, file)); removedFile = true; } catch (e) {}
      KB.files.delete(key);
    }
    unindexDoc(key);
    send(res, 200, { ok: true, name: key, docs: KB.docs.size, file_removed: removedFile });
  },

  "/batch": async (res, body) => {
    const items = Array.isArray(body.questions) ? body.questions : null;
    if (!items || !items.length)
      return fail(res, 400, "Provide { questions: [{ question, ref?, id?, type?, tone?, max_words? }] }");
    if (items.length > MAX_BATCH) return fail(res, 400, `Max ${MAX_BATCH} questions per batch`);
    for (const [i, it] of items.entries())
      if (!questionOk(it && it.question))
        return fail(res, 400, `questions[${i}].question must be a string of 3-${MAX_QUESTION_CHARS} characters`);

    const streaming = body.stream !== false;   // default: stream NDJSON lines as answers complete
    const t0 = Date.now();
    let closed = false;
    res.on("close", () => { closed = true; });

    const results = [];
    const emit = (obj) => {
      if (streaming) { if (!closed) res.write(JSON.stringify(obj) + "\n"); }
      else if (obj.kind === "answer" || obj.kind === "error") results.push(obj);
    };

    if (streaming) {
      res.writeHead(200, {
        "Content-Type": "application/x-ndjson; charset=utf-8",
        "Access-Control-Allow-Origin": ALLOW_ORIGIN,
        "Cache-Control": "no-cache",
        "X-Content-Type-Options": "nosniff",
      });
      emit({ kind: "start", total: items.length, model: MODEL, concurrency: CONCURRENCY });
    }

    let cursor = 0, ok = 0, failed = 0;

    async function processItem(i) {
      const it = items[i];
      const t = Date.now();
      const retrieval = retrieve(it.question);
      // per-item values override the batch-level defaults (documented in ENGINE.md)
      const opts = { type: it.type ?? body.type, tone: it.tone ?? body.tone, max_words: it.max_words ?? body.max_words };
      const payload = () => ({
        model: MODEL, max_tokens: ANSWER_MAX_TOKENS,
        messages: [{ role: "user", content: buildPrompt(it.question, retrieval, opts) }],
      });
      let up = await callClaude(payload());
      if ((up.status === 429 || up.status >= 500) && !closed) {   // one retry on rate-limit/overload
        await new Promise(r => setTimeout(r, 2000));
        up = await callClaude(payload());
      }
      if (up.status !== 200) {
        failed++;
        let message = "Upstream " + up.status;
        try { message += ": " + JSON.parse(up.text).error.message; } catch (e) {}
        return emit({ kind: "error", index: i, id: it.id, ref: it.ref, question: it.question, status: up.status, message });
      }
      let answer;
      try { answer = answerText(JSON.parse(up.text)); }
      catch (e) {
        failed++;
        const message = e instanceof SyntaxError ? "Bad upstream response" : e.message;
        return emit({ kind: "error", index: i, id: it.id, ref: it.ref, question: it.question, status: 502, message });
      }
      ok++;
      emit({
        kind: "answer", index: i, id: it.id, ref: it.ref, question: it.question, answer,
        sources: retrieval.sources,
        coverage: Math.round(retrieval.coverage * 100) / 100,
        chunks_used: retrieval.chunks_used,
        missing_terms: retrieval.missing_terms,
        ms: Date.now() - t,
      });
    }

    async function worker() {
      while (!closed) {
        const i = cursor++;
        if (i >= items.length) return;
        await processItem(i);
      }
    }
    await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, worker));

    const done = { kind: "done", total: items.length, succeeded: ok, failed, elapsed_ms: Date.now() - t0 };
    if (streaming) { if (!closed) { res.write(JSON.stringify(done) + "\n"); res.end(); } }
    else send(res, 200, { ...done, results: results.sort((a, b) => a.index - b.index) });
  },

  "/proxy": async (res, body) => {  // web-app compatibility: raw pass-through
    if (!messagesOk(body.messages)) return fail(res, 400, "Body must be Anthropic /v1/messages JSON");
    const up = await callClaude({
      model: MODEL,
      // Cap must stay above the app's largest request ("Extract questions" returns a
      // JSON array of up to 40 items); 1024 truncated that JSON mid-array.
      max_tokens: Math.min(parseInt(body.max_tokens, 10) || PROXY_MAX_TOKENS, PROXY_MAX_TOKENS),
      messages: body.messages,
      ...(typeof body.system === "string" ? { system: body.system } : {}),
    });
    send(res, up.status, up.text);
  },
};

const server = http.createServer((req, res) => {
  const url = req.url.split("?")[0];
  const refusal = browserRefusal(req);
  if (refusal) return fail(res, 403, refusal);
  if (req.method === "OPTIONS") {
    res.writeHead(204, {
      "Access-Control-Allow-Origin": ALLOW_ORIGIN,
      "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
      "Access-Control-Allow-Headers": "Content-Type, X-Team-Token",
      "Access-Control-Max-Age": "86400",
    });
    return res.end();
  }
  if (req.method === "GET" && url === "/health")
    return send(res, 200, { ok: true, docs: KB.docs.size, chunks: KB.chunks.length, model: MODEL, token_required: !!TEAM_TOKEN });
  if (req.method !== "POST" || !routes[url]) return fail(res, 404, "Unknown endpoint");
  if (!API_KEY && url !== "/ingest" && url !== "/forget" && url !== "/search")
    return fail(res, 500, "Server not configured: set ANTHROPIC_API_KEY");
  if (TEAM_TOKEN && !tokenOk(req.headers["x-team-token"]))
    return fail(res, 401, "Missing or wrong team token");
  // A cross-site page can POST text/plain without a CORS preflight. Requiring JSON forces
  // the preflight, so ALLOW_ORIGIN actually decides who can change the knowledge base.
  if (!/^application\/json\b/i.test(req.headers["content-type"] || ""))
    return fail(res, 415, "Send the body as Content-Type: application/json");

  let size = 0;
  const chunks = [];
  req.on("data", (c) => {
    size += c.length;
    if (res.writableEnded) {                     // 413 already sent — drain so the client can read it,
      if (size > MAX_BODY_BYTES * 8) req.destroy();  // but hard-stop a runaway upload
      return;
    }
    if (size > MAX_BODY_BYTES) {
      chunks.length = 0;
      fail(res, 413, "Request too large");       // destroying here would RST before the 413 arrives
      return;
    }
    chunks.push(c);
  });
  req.on("end", async () => {
    if (res.writableEnded) return;
    let body;
    try { body = JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}"); }
    catch { return fail(res, 400, "Invalid JSON"); }
    // `null`, `[]` and bare scalars all parse fine but would blow up on property access
    if (body === null || typeof body !== "object" || Array.isArray(body))
      return fail(res, 400, "Body must be a JSON object");
    try { await routes[url](res, body); }
    catch (err) {
      if (res.headersSent) { try { res.end(); } catch (e) {} }
      else if (!res.writableEnded) fail(res, 502, "Engine error: " + err.message);
    }
  });
});

if (require.main === module) {
  if (!isLoopback(HOST) && !TEAM_TOKEN && !ALLOW_NO_TOKEN) {
    console.error(`Refusing to listen on ${HOST} without COUNTERSIGN_TOKEN: anyone who can reach the port ` +
      "could spend your API key and rewrite the knowledge base. Set COUNTERSIGN_TOKEN, or ALLOW_NO_TOKEN=1 to accept that.");
    process.exit(1);
  }
  loadKnowledgeBase();
  server.listen(PORT, HOST, () =>
    console.log(
      `Countersign Engine on http://${HOST}:${PORT}  ·  ${KB.docs.size} docs / ${KB.chunks.length} chunks` +
      (TEAM_TOKEN ? "  ·  token required" : "  ·  WARNING: no team token set" +
        (ORIGIN_CONFIGURED ? "" : " (browser requests from other origins are refused)"))
    )
  );
} else {
  // Required as a module (the eval harness does this): expose the pipeline without
  // touching the filesystem or opening a port. Callers load their own knowledge base.
  module.exports = { KB, addDoc, reindex, indexDoc, unindexDoc, retrieve, buildPrompt, callClaude, answerText, tokenize, stem, STOP, MODEL, CHUNK_SIZE, ANSWER_MAX_TOKENS };
}