diff --git a/services/api-rs/crates/centaur-api-server/src/lib.rs b/services/api-rs/crates/centaur-api-server/src/lib.rs index 6162375c2..36945b41c 100644 --- a/services/api-rs/crates/centaur-api-server/src/lib.rs +++ b/services/api-rs/crates/centaur-api-server/src/lib.rs @@ -4,6 +4,7 @@ mod error; mod mcp; mod routes; mod slack_proxy; +mod status; mod tool_discovery; pub mod types; diff --git a/services/api-rs/crates/centaur-api-server/src/routes.rs b/services/api-rs/crates/centaur-api-server/src/routes.rs index 83a0e23e9..4c1a981ad 100644 --- a/services/api-rs/crates/centaur-api-server/src/routes.rs +++ b/services/api-rs/crates/centaur-api-server/src/routes.rs @@ -160,7 +160,7 @@ impl AppState { .ok_or_else(|| ApiError::BadRequest("workflow runtime is not enabled".to_owned())) } - fn pool(&self) -> Result { + pub(crate) fn pool(&self) -> Result { let initialized = self .initialized() .ok_or_else(|| ApiError::ServiceUnavailable("api-rs is still starting".to_owned()))?; @@ -209,6 +209,7 @@ pub fn build_router_with_app_state(state: AppState) -> Router { .route("/readyz", get(readyz)) .route("/metrics", get(metrics)) .route("/api/personas", get(list_personas)) + .route("/api/status", get(crate::status::status_report)) .route("/mcp", post(mcp_post).get(mcp_get)) .route( "/.well-known/oauth-protected-resource", diff --git a/services/api-rs/crates/centaur-api-server/src/status.rs b/services/api-rs/crates/centaur-api-server/src/status.rs new file mode 100644 index 000000000..ce1f726c0 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/src/status.rs @@ -0,0 +1,183 @@ +use std::{ + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, +}; + +use axum::{Json, extract::State}; +use serde::Serialize; +use serde_json::{Value, json}; +use sqlx::PgPool; + +use crate::{error::ApiError, routes::AppState}; + +// Read-only operational snapshot for status surfaces (chat-bot "status" +// commands, dashboards): recent/in-flight executions, a 24h tally, active +// session sandboxes, warm-pool state, and a 7-calendar-day (UTC) run +// histogram. api-rs owns the session schema, so the SQL lives here rather +// than in every ingress service that wants a status view. +// +// The report is cached briefly in-process: the history queries scan +// session_executions (append-only, never pruned), and status commands are +// human-triggered but scriptable — the cache caps the database cost at one +// scan set per TTL regardless of how often callers ask. + +const CACHE_TTL: Duration = Duration::from_secs(10); +const ERROR_SNIPPET_CHARS: i32 = 200; +const RECENT_LIMIT: i64 = 20; + +static CACHE: OnceLock>> = OnceLock::new(); + +pub(crate) async fn status_report(State(state): State) -> Result, ApiError> { + let cache = CACHE.get_or_init(|| Mutex::new(None)); + if let Some((stored_at, report)) = cache + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() + && stored_at.elapsed() < CACHE_TTL + { + return Ok(Json(report)); + } + + let pool = state.pool()?; + let report = build_report(&pool).await?; + *cache + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((Instant::now(), report.clone())); + Ok(Json(report)) +} + +#[derive(Serialize, sqlx::FromRow)] +struct ExecutionRow { + /// Seconds since the execution was created. + age_seconds: f64, + duration_seconds: Option, + error: Option, + status: String, + thread_key: String, + title: Option, + user_name: Option, +} + +#[derive(Serialize, sqlx::FromRow)] +struct CountRow { + count: i64, + status: String, +} + +#[derive(Serialize, sqlx::FromRow)] +struct SandboxRow { + idle_seconds: f64, + sandbox_id: String, + thread_key: String, +} + +#[derive(Serialize, sqlx::FromRow)] +struct DailyRow { + /// UTC calendar date, `YYYY-MM-DD`. + day: String, + failed: i64, + runs: i64, +} + +async fn build_report(pool: &PgPool) -> Result { + let recent = sqlx::query_as::<_, ExecutionRow>( + "SELECT e.thread_key, e.status, \ + left(e.error, $1) AS error, \ + extract(epoch FROM (now() - e.created_at))::float8 AS age_seconds, \ + extract(epoch FROM (e.completed_at - e.started_at))::float8 AS duration_seconds, \ + e.metadata ->> 'user_name' AS user_name, \ + coalesce(s.title, s.metadata ->> 'discord_conversation_name', \ + s.metadata ->> 'linear_conversation_name', \ + s.metadata ->> 'slack_conversation_name') AS title \ + FROM session_executions e \ + LEFT JOIN sessions s ON s.thread_key = e.thread_key \ + ORDER BY e.created_at DESC \ + LIMIT $2", + ) + .bind(ERROR_SNIPPET_CHARS) + .bind(RECENT_LIMIT) + .fetch_all(pool) + .await?; + + let in_flight = sqlx::query_as::<_, ExecutionRow>( + "SELECT e.thread_key, e.status, \ + NULL::text AS error, \ + extract(epoch FROM (now() - e.created_at))::float8 AS age_seconds, \ + NULL::float8 AS duration_seconds, \ + e.metadata ->> 'user_name' AS user_name, \ + coalesce(s.title, s.metadata ->> 'discord_conversation_name', \ + s.metadata ->> 'linear_conversation_name', \ + s.metadata ->> 'slack_conversation_name') AS title \ + FROM session_executions e \ + LEFT JOIN sessions s ON s.thread_key = e.thread_key \ + WHERE e.status IN ('queued', 'running') \ + ORDER BY e.created_at ASC \ + LIMIT $1", + ) + .bind(RECENT_LIMIT) + .fetch_all(pool) + .await?; + + let tally_24h = sqlx::query_as::<_, CountRow>( + "SELECT status, count(*) AS count \ + FROM session_executions \ + WHERE created_at > now() - interval '24 hours' \ + GROUP BY status", + ) + .fetch_all(pool) + .await?; + + let active_sandboxes = sqlx::query_as::<_, SandboxRow>( + "SELECT thread_key, sandbox_id, \ + extract(epoch FROM (now() - sandbox_last_active_at))::float8 AS idle_seconds \ + FROM sessions \ + WHERE sandbox_id IS NOT NULL \ + AND sandbox_last_active_at > now() - interval '2 hours' \ + ORDER BY sandbox_last_active_at DESC \ + LIMIT $1", + ) + .bind(RECENT_LIMIT) + .fetch_all(pool) + .await?; + + // ready/evicting are the pool's current state. claimed/failed rows are + // lifetime history (claiming flips status in place, rows are never + // deleted), so an unfiltered count reads like a leak — window them to 24h + // churn instead. + let warm_pool = sqlx::query_as::<_, CountRow>( + "SELECT status, count(*) AS count \ + FROM session_warm_sandboxes \ + WHERE status IN ('ready', 'evicting') \ + OR updated_at > now() - interval '24 hours' \ + GROUP BY status", + ) + .fetch_all(pool) + .await?; + + // Calendar-day (UTC) buckets over the last 7 days INCLUDING today, so the + // per-day rows and any total computed from them describe the same window + // (a rolling now()-7d fetch would include a partial 8th calendar day). + let daily = sqlx::query_as::<_, DailyRow>( + "SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS day, \ + count(*) AS runs, \ + count(*) FILTER (WHERE status = 'failed') AS failed \ + FROM session_executions \ + WHERE created_at >= \ + (date_trunc('day', now() AT TIME ZONE 'UTC') - interval '6 days') \ + AT TIME ZONE 'UTC' \ + GROUP BY 1 \ + ORDER BY 1", + ) + .fetch_all(pool) + .await?; + + Ok(json!({ + "ok": true, + "active_sandboxes": active_sandboxes, + "daily": daily, + "in_flight": in_flight, + "recent_executions": recent, + "tally_24h": tally_24h, + "warm_pool": warm_pool, + })) +} diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0049_session_executions_created_at_idx.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0049_session_executions_created_at_idx.sql new file mode 100644 index 000000000..4938c65fa --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0049_session_executions_created_at_idx.sql @@ -0,0 +1,7 @@ +-- The /api/status report orders and windows session_executions by created_at +-- globally (ORDER BY created_at DESC LIMIT n; created_at > now() - '24 hours'; +-- the 7-day histogram). The existing (thread_key, created_at, execution_id) +-- index cannot serve a global recency scan over this append-only, never-pruned +-- table, so give created_at its own index. +create index if not exists session_executions_created_at_idx + on session_executions (created_at desc); diff --git a/services/discordbot/src/index.ts b/services/discordbot/src/index.ts index 4491405c8..18af11b52 100644 --- a/services/discordbot/src/index.ts +++ b/services/discordbot/src/index.ts @@ -33,6 +33,12 @@ import { renameThreadFromMessage, } from "./discord-threading"; import { setGatewayConnected } from "./gateway"; +import { + STATUS_FAILURE_REPLY, + collectStatus, + formatStatus, + isStatusCommand, +} from "./status"; import { collectInitialContext, executeSessionTurn, @@ -259,9 +265,58 @@ export function createDiscordbot(options: DiscordbotOptions): Discordbot { DEFAULT_MAX_CONCURRENT_EXECUTIONS_PER_GUILD, ); + // A bare "status"/"health" mention answers straight from the control plane + // (api-rs /healthz + /api/status over HTTP — no SQL, no sandbox turn) so it + // still works when the agent pipeline is what's broken. The feature is + // OPT-IN per channel: statusChannelAllowlist empty/unset disables it + // entirely (the report exposes cross-platform activity — session titles, + // requester names, error snippets — so which channels may see it is a + // deployment decision, not a default). The exchange deliberately stays out + // of the session transcript: it's operational metadata, not conversation + // the agent should later see. Returns true when handled. + const statusChannels = new Set(options.statusChannelAllowlist ?? []); + const isStatusChannel = (threadKey: string): boolean => { + if (statusChannels.size === 0) return false; + const { channelId, threadId } = parseDiscordThreadKey(threadKey); + return ( + (channelId !== undefined && statusChannels.has(channelId)) || + (threadId !== undefined && statusChannels.has(threadId)) + ); + }; + const maybeReplyStatus = async ( + thread: Thread, + message: ChatMessage, + ): Promise => { + if (!isStatusCommand(message.text ?? "")) return false; + if (!isStatusChannel(thread.id)) return false; + try { + const report = await collectStatus({ + apiUrl: options.apiUrl, + fetchFn: options.fetch, + }); + await thread.post(formatStatus(report, userName)); + } catch (error) { + // Internals (hostnames, auth errors) stay in the logs; the channel gets + // a generic line. + logger.warn("discordbot_status_reply_failed", { + error: errorMessage(error), + }); + try { + await thread.post(STATUS_FAILURE_REPLY); + } catch { + // best-effort; nothing left to signal with. + } + } + return true; + }; + chat.onNewMention(async (thread, message) => { if (!isAllowedDiscordMessage(message, options, logger)) return; + // Subscribe before the status fast-path: the adapter has already created + // a thread for the mention, and an unsubscribed thread would silently + // ignore follow-ups. await thread.subscribe(); + if (await maybeReplyStatus(thread, message)) return; await syncThreadMessageToSession(thread, message, { executionLimiter, mode: "execute", @@ -272,6 +327,8 @@ export function createDiscordbot(options: DiscordbotOptions): Discordbot { chat.onSubscribedMessage(async (thread, message) => { if (!isAllowedDiscordMessage(message, options, logger)) return; + if (message.isMention === true && (await maybeReplyStatus(thread, message))) + return; await syncThreadMessageToSession(thread, message, { executionLimiter, mode: message.isMention === true ? "execute" : "append", diff --git a/services/discordbot/src/server.ts b/services/discordbot/src/server.ts index 44860e526..01a5b4bda 100644 --- a/services/discordbot/src/server.ts +++ b/services/discordbot/src/server.ts @@ -40,6 +40,9 @@ const options: DiscordbotOptions = { publicKey, discordApiUrl: optionalEnv("DISCORD_API_URL"), guildAllowlist: optionalList("DISCORDBOT_GUILD_ALLOWLIST"), + // Channels where "@bot status" answers with the control-plane report; + // unset = the status fast-path is disabled. + statusChannelAllowlist: optionalList("DISCORDBOT_STATUS_CHANNEL_IDS"), idleTimeoutMs: optionalNumberEnv("SESSION_IDLE_TIMEOUT_MS"), isGatewayActive: () => gateway.isActive(), maxConcurrentExecutionsPerGuild: optionalNumberEnv( diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts new file mode 100644 index 000000000..789d86715 --- /dev/null +++ b/services/discordbot/src/status.ts @@ -0,0 +1,415 @@ +import type { DiscordbotFetch } from "./types"; +import { sliceSurrogateSafe } from "./utils"; + +// A "status" mention answers directly from the control plane — no sandbox, no +// session turn — so it still works when the agent pipeline is what's broken. +// All data comes from api-rs over HTTP: /healthz + /readyz for liveness, and +// the read-only /api/status report (api-rs owns the session schema; this +// service deliberately runs no SQL). Every fetch carries a timeout and is +// best-effort, so one dead dependency never blanks the rest of the report or +// hangs the per-thread handler lock. + +const KEYWORD = /^(status|health)[?!.]*$/i; +// Raw Discord mention markup (<@123>, <@!123>, <@&role>, <#channel>) plus the +// adapter's rewritten form (`@name`): none of it counts as words. +const MENTION_TOKEN = /^(<[@#][!&]?\w+>|@[\w.-]+)$/; + +/** + * True when the message is ONLY a status request ("@bot status", + * "<@&123> health?"). Anything with more words ("status of the deploy") falls + * through to a normal agent turn so real questions are never hijacked. + */ +export function isStatusCommand(text: string): boolean { + const words = text + .split(/\s+/) + .filter((word) => word.length > 0 && !MENTION_TOKEN.test(word)); + return words.length === 1 && KEYWORD.test(words[0] ?? ""); +} + +export type ExecutionRow = { + ageSeconds: number | null; + durationSeconds: number | null; + error: string; + status: string; + threadKey: string; + /** Session title (the conversation name the bots set), when present. */ + title: string; + /** Display name of whoever triggered the turn, when recorded. */ + who: string; +}; + +export type DailyRow = { + /** UTC calendar date, `YYYY-MM-DD`. */ + day: string; + failed: number; + runs: number; +}; + +export type StatusReport = { + /** null = unreachable, false = responded unhealthy, true = healthy. */ + apiHealthy: boolean | null; + apiReady: boolean | null; + daily: DailyRow[]; + inFlight: ExecutionRow[]; + recent: ExecutionRow[]; + /** Whether the /api/status report fetch succeeded. */ + reportOk: boolean; + sandboxes: { idleSeconds: number | null; sandboxId: string; threadKey: string }[]; + tally: Record; + warmPool: Record; +}; + +const HEALTH_TIMEOUT_MS = 2_000; +const REPORT_TIMEOUT_MS = 5_000; + +export async function collectStatus(input: { + apiUrl: string; + fetchFn?: DiscordbotFetch; + nowMs?: number; +}): Promise { + const fetchFn = input.fetchFn ?? fetch; + const now = input.nowMs ?? Date.now(); + + // null = unreachable (nothing answered), false = answered non-2xx. + const probe = async (path: string): Promise => { + try { + const response = await fetchFn(`${input.apiUrl}${path}`, { + signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS), + }); + return response.ok; + } catch { + return null; + } + }; + + const fetchReport = async (): Promise | null> => { + try { + const response = await fetchFn(`${input.apiUrl}/api/status`, { + signal: AbortSignal.timeout(REPORT_TIMEOUT_MS), + }); + if (!response.ok) return null; + const body: unknown = await response.json(); + return typeof body === "object" && body !== null + ? (body as Record) + : null; + } catch { + return null; + } + }; + + const [apiHealthy, apiReady, report] = await Promise.all([ + probe("/healthz"), + probe("/readyz"), + fetchReport(), + ]); + + const rows = (key: string): Record[] => { + const value = report?.[key]; + return Array.isArray(value) + ? value.filter( + (row): row is Record => + typeof row === "object" && row !== null, + ) + : []; + }; + + const toExecutionRow = (row: Record): ExecutionRow => ({ + ageSeconds: numberOrNull(row.age_seconds), + durationSeconds: numberOrNull(row.duration_seconds), + error: String(row.error ?? ""), + status: String(row.status ?? "unknown"), + threadKey: String(row.thread_key ?? "?"), + title: String(row.title ?? ""), + who: String(row.user_name ?? ""), + }); + + return { + apiHealthy, + apiReady, + daily: zeroFilledWeek(rows("daily"), now), + inFlight: rows("in_flight").map(toExecutionRow), + recent: rows("recent_executions").map(toExecutionRow), + reportOk: report !== null, + sandboxes: rows("active_sandboxes").map((row) => ({ + idleSeconds: numberOrNull(row.idle_seconds), + sandboxId: String(row.sandbox_id ?? "?"), + threadKey: String(row.thread_key ?? "?"), + })), + tally: countsByStatus(rows("tally_24h")), + warmPool: countsByStatus(rows("warm_pool")), + }; +} + +// Discord caps messages at 2000 chars; stay under it with honest truncation. +const STATUS_MAX_CHARS = 1_900; + +// Short ASCII tags: emoji are double-width in Discord's code blocks and wreck +// column alignment, which is the whole point of the tabular layout. +const STATUS_TAG: Record = { + cancelled: "cxl", + completed: "ok", + failed: "FAIL", + queued: "que", + running: "run", +}; + +const TAG_WIDTH = 5; +const THREAD_WIDTH = 24; +const WHO_WIDTH = 10; +const AGE_WIDTH = 4; +const DUR_WIDTH = 5; +const ERROR_LINE_CHARS = 60; +const BAR_WIDTH = 16; +const RECENT_ROWS_SHOWN = 8; + +const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; + +function weekdayLabel(dayIso: string): string { + const parsed = new Date(`${dayIso}T00:00:00Z`); + const label = WEEKDAYS[parsed.getUTCDay()]; + return label ?? "???"; +} + +/** + * Discord has no table markup; the closest thing is a monospace code block + * with hand-padded columns. Header line stays OUTSIDE the block (bold + emoji + * work there); rows stay ~50 chars wide to limit wrapping on mobile. The + * histogram gets its own second block. `botName` labels the header — this is + * generic service code, so the deployment's bot name is a parameter. + */ +export function formatStatus(report: StatusReport, botName: string): string { + const mark = (value: boolean | null): string => + value === null ? "❓" : value ? "✅" : "❌"; + const header = + `**${botName} status** · api-rs ${mark(report.apiHealthy)} ` + + `ready ${mark(report.apiReady)} · data ${report.reportOk ? "✅" : "❌"}`; + + const lines: string[] = []; + + const tallyEntries = Object.entries(report.tally).sort(); + if (tallyEntries.length > 0) { + lines.push( + `24h: ${tallyEntries + .map(([status, count]) => `${count} ${STATUS_TAG[status] ?? status}`) + .join(" · ")}`, + ); + lines.push(""); + } + + const tableRow = ( + tag: string, + thread: string, + who: string, + age: string, + took: string, + ): string => + `${tag.padEnd(TAG_WIDTH)} ${fit(thread, THREAD_WIDTH)} ` + + `${fit(who, WHO_WIDTH, "head")} ${age.padStart(AGE_WIDTH)} ` + + `${took.padStart(DUR_WIDTH)}`; + + // One table: in-flight turns first (no duration yet), then settled recent + // turns. The recent list also carries queued/running rows — skip those so + // an in-flight turn isn't listed twice. AGE = when the turn was requested, + // TOOK = how long it ran. + const turnRow = (row: ExecutionRow): string => + tableRow( + STATUS_TAG[row.status] ?? row.status, + inline(threadLabel(row)), + inline(row.who), + formatAge(row.ageSeconds), + row.durationSeconds !== null ? formatDuration(row.durationSeconds) : "-", + ).trimEnd(); + const settled = report.recent + .filter((row) => row.status !== "queued" && row.status !== "running") + .slice(0, RECENT_ROWS_SHOWN); + const turns = [...report.inFlight, ...settled]; + if (turns.length > 0) { + lines.push(tableRow("", "THREAD", "WHO", "AGE", "TOOK").trimEnd()); + for (const row of turns) { + lines.push(turnRow(row)); + if (row.error) { + lines.push(` └ ${inline(row.error).slice(0, ERROR_LINE_CHARS)}`); + } + } + } + + const sandboxBits: string[] = []; + if (report.sandboxes.length > 0) { + sandboxBits.push(`${report.sandboxes.length} active`); + } + // ready/evicting are the pool's current state; claimed/failed rows are + // historical (the report windows them to 24h). "failed" here is a warm + // SPAWN failure (a standby sandbox that didn't provision — the next session + // cold-starts instead), NOT a failed turn; label it so it can't be confused + // with the histogram's FAIL column. + const WARM_LABEL: Record = { failed: "spawn-failed" }; + const warmLine = (statuses: string[]): string => + statuses + .filter((status) => (report.warmPool[status] ?? 0) > 0) + .map( + (status) => `${report.warmPool[status]} ${WARM_LABEL[status] ?? status}`, + ) + .join(", "); + const warmNow = warmLine(["ready", "evicting"]); + const warmChurn = warmLine(["claimed", "failed"]); + if (warmNow) sandboxBits.push(`warm: ${warmNow}`); + if (warmChurn) sandboxBits.push(`warm 24h: ${warmChurn}`); + if (sandboxBits.length > 0) { + lines.push(""); + lines.push(`sandboxes: ${sandboxBits.join(" · ")}`); + } + + if (!report.reportOk) { + lines.push("! status report unavailable — turn history not shown"); + } + + // 7-day histogram, in its OWN code block below the live view: the bar + // encodes ONE measure (runs); failures get their own labeled column rather + // than a second scale or color-alone marking; the failure rate is a plain + // stat line. + const histogramLines: string[] = []; + const week = report.daily; + const totalRuns = week.reduce((sum, day) => sum + day.runs, 0); + if (totalRuns > 0) { + const totalFailed = week.reduce((sum, day) => sum + day.failed, 0); + const maxRuns = Math.max(...week.map((day) => day.runs)); + histogramLines.push(` ${"LAST 7 DAYS".padEnd(BAR_WIDTH + 1)}RUNS FAIL`); + for (const day of week) { + const bar = "█".repeat( + day.runs === 0 + ? 0 + : Math.max(1, Math.round((day.runs / maxRuns) * BAR_WIDTH)), + ); + const fail = day.failed > 0 ? String(day.failed) : "-"; + histogramLines.push( + `${weekdayLabel(day.day)} ${bar.padEnd(BAR_WIDTH + 1)}` + + `${String(day.runs).padStart(4)} ${fail.padStart(4)}`, + ); + } + const rate = (totalFailed / totalRuns) * 100; + histogramLines.push( + `7d: ${totalRuns} runs · ${totalFailed} failed (${rate.toFixed(1)}%)`, + ); + } + const histogram = histogramLines.join("\n"); + const histogramBlock = histogram ? `\n\`\`\`\n${histogram}\n\`\`\`` : ""; + + if (lines.length === 0 && !histogramBlock) return header; + const body = lines.join("\n"); + // The histogram block is small and fixed-size; give the live view whatever + // budget remains under Discord's cap. + const budget = STATUS_MAX_CHARS - header.length - histogramBlock.length - 20; + const bounded = + body.length <= budget + ? body + : `${sliceSurrogateSafe(body, budget - 12).trimEnd()}\n[truncated]`; + const liveBlock = lines.length > 0 ? `\n\`\`\`\n${bounded}\n\`\`\`` : ""; + return `${header}${liveBlock}${histogramBlock}`; +} + +/** Generic failure reply — internals go to logs, not the channel. */ +export const STATUS_FAILURE_REPLY = "⚠️ status check failed — see service logs."; + +/** + * Human label for a turn: the session title when the bots set one, otherwise + * a friendlier rendering of the thread key ("GH PR splits-teams#1799" beats + * "github-manage:0xSplits/splits-teams:1799"; raw Discord ids stay raw). + */ +function threadLabel(row: { threadKey: string; title: string }): string { + if (row.title.trim()) return row.title.trim(); + const parts = row.threadKey.split(":"); + const platform = parts[0] ?? row.threadKey; + const rest = parts.slice(1).join(":"); + if (platform === "github-manage" && parts.length >= 3) { + const repo = (parts[1] ?? "").split("/").pop() ?? parts[1]; + return `GH PR ${repo}#${parts[2]}`; + } + if (platform.startsWith("github")) return `GH ${rest}`; + if (platform === "linear") return `Linear ${rest}`; + if (platform === "slack") return `Slack ${rest}`; + if (platform === "discord") return `Discord ${rest}`; + return row.threadKey; +} + +/** + * Neutralize markdown/code-fence breakouts in interpolated values (titles and + * error strings are user/agent-influenced): backticks become apostrophes and + * whitespace collapses to single spaces so a value can never close the + * surrounding fence or smuggle its own line. + */ +function inline(value: string): string { + return value.replace(/`/g, "'").replace(/\s+/g, " ").trim(); +} + +/** + * Truncate + pad to the column. Middle ellipsis by default so both ends stay + * readable ("GH PR splits-con…eams#1799", "Discord 90294…:1391220231" — the + * head names the thing, the tail discriminates); plain head-cut for names. + */ +function fit( + value: string, + width: number, + keep: "edges" | "head" = "edges", +): string { + if (value.length <= width) return value.padEnd(width); + if (keep === "head" || width < 12) { + return `${value.slice(0, width - 1)}…`; + } + const tail = 7; + return `${value.slice(0, width - tail - 1)}…${value.slice(-tail)}`; +} + +function formatAge(seconds: number | null): string { + if (seconds === null) return "?"; + return formatDuration(seconds); +} + +function formatDuration(seconds: number): string { + const s = Math.max(0, Math.round(seconds)); + if (s < 60) return `${s}s`; + if (s < 3600) return `${Math.round(s / 60)}m`; + if (s < 86400) return `${Math.round(s / 3600)}h`; + return `${Math.round(s / 86400)}d`; +} + +/** The last 7 UTC calendar days (oldest→today), zero-filling days with no runs. */ +function zeroFilledWeek( + rows: Record[], + nowMs: number, +): DailyRow[] { + const byDay = new Map(); + for (const row of rows) { + byDay.set(String(row.day ?? ""), { + failed: numberOrNull(row.failed) ?? 0, + runs: numberOrNull(row.runs) ?? 0, + }); + } + const days: DailyRow[] = []; + for (let offset = 6; offset >= 0; offset -= 1) { + const day = new Date(nowMs - offset * 86_400_000) + .toISOString() + .slice(0, 10); + days.push({ day, failed: 0, runs: 0, ...byDay.get(day) }); + } + return days; +} + +function countsByStatus( + rows: Record[], +): Record { + const counts: Record = {}; + for (const row of rows) { + const count = numberOrNull(row.count); + if (count !== null) counts[String(row.status ?? "unknown")] = count; + } + return counts; +} + +function numberOrNull(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim() !== "") { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + return null; +} diff --git a/services/discordbot/src/types.ts b/services/discordbot/src/types.ts index 9e5df03ee..574fc9d65 100644 --- a/services/discordbot/src/types.ts +++ b/services/discordbot/src/types.ts @@ -114,6 +114,13 @@ export type DiscordbotOptions = { recoverRenderObligationsOnStart?: boolean; state?: StateAdapter; stateKeyPrefix?: string; + /** + * Channel (or thread) ids where a bare "status"/"health" mention gets the + * control-plane status reply. Empty/unset disables the fast-path entirely — + * the report surfaces cross-platform activity (session titles, requester + * names, error snippets), so exposure is an explicit per-channel opt-in. + */ + statusChannelAllowlist?: readonly string[]; /** * Discord delta (mirrors slackbotv2's `triggerBotAllowlist`): bot user ids * whose messages may trigger/append despite being bot-authored. diff --git a/services/discordbot/test/status.test.ts b/services/discordbot/test/status.test.ts new file mode 100644 index 000000000..1754965c8 --- /dev/null +++ b/services/discordbot/test/status.test.ts @@ -0,0 +1,307 @@ +import { describe, expect, it } from "bun:test"; +import { + collectStatus, + formatStatus, + isStatusCommand, + type StatusReport, +} from "../src/status"; +import type { DiscordbotFetch } from "../src/types"; + +describe("isStatusCommand", () => { + it("matches bare status/health requests with mention markup", () => { + expect(isStatusCommand("status")).toBe(true); + expect(isStatusCommand("Status?")).toBe(true); + expect(isStatusCommand("health!")).toBe(true); + expect(isStatusCommand("<@123456> status")).toBe(true); + expect(isStatusCommand("<@!123456> health")).toBe(true); + expect(isStatusCommand("<@&987> status")).toBe(true); + expect(isStatusCommand("@gerard status")).toBe(true); + expect(isStatusCommand(" @gerard STATUS ")).toBe(true); + }); + + it("rejects real questions and ordinary messages", () => { + expect(isStatusCommand("status of the deploy")).toBe(false); + expect(isStatusCommand("@gerard what's the status?")).toBe(false); + expect(isStatusCommand("can you check the health of api-rs")).toBe(false); + expect(isStatusCommand("hello")).toBe(false); + expect(isStatusCommand("")).toBe(false); + expect(isStatusCommand("<@123456>")).toBe(false); + }); +}); + +const NOW = Date.parse("2026-08-12T12:00:00Z"); + +const FULL_REPORT = { + ok: true, + recent_executions: [ + { + age_seconds: 300, + duration_seconds: 63, + error: null, + status: "completed", + thread_key: "github-manage:0xSplits/splits-teams:1799", + title: null, + user_name: "0xdiid", + }, + { + age_seconds: 1900, + duration_seconds: 12, + error: "sandbox spawn timeout after 120s", + status: "failed", + thread_key: "discord:1:2:9", + title: null, + user_name: "jaan", + }, + ], + in_flight: [ + { + age_seconds: 120, + duration_seconds: null, + error: null, + status: "running", + thread_key: "discord:1:2:3", + title: "fix the deploy pipeline", + user_name: "oliver", + }, + ], + tally_24h: [ + { count: 41, status: "completed" }, + { count: 2, status: "failed" }, + ], + active_sandboxes: [ + { + idle_seconds: 60, + sandbox_id: "asbx-1755000000-1", + thread_key: "discord:1:2:3", + }, + ], + warm_pool: [ + { count: 2, status: "ready" }, + { count: 41, status: "claimed" }, + ], + daily: [ + { day: "2026-08-10", failed: 0, runs: 12 }, + { day: "2026-08-11", failed: 3, runs: 40 }, + { day: "2026-08-12", failed: 0, runs: 20 }, + ], +}; + +function apiFetch(input: { + health?: number; + ready?: number; + report?: unknown; + reportStatus?: number; +}): DiscordbotFetch { + return async (url) => { + const path = String(url); + if (path.endsWith("/healthz")) { + return new Response("{}", { status: input.health ?? 200 }); + } + if (path.endsWith("/readyz")) { + return new Response("{}", { status: input.ready ?? 200 }); + } + if (path.endsWith("/api/status")) { + return new Response(JSON.stringify(input.report ?? FULL_REPORT), { + status: input.reportStatus ?? 200, + }); + } + throw new Error(`unexpected fetch: ${path}`); + }; +} + +describe("collectStatus", () => { + it("assembles a full report when everything is up", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + fetchFn: apiFetch({}), + nowMs: NOW, + }); + expect(report.apiHealthy).toBe(true); + expect(report.apiReady).toBe(true); + expect(report.reportOk).toBe(true); + expect(report.tally).toEqual({ completed: 41, failed: 2 }); + expect(report.recent).toHaveLength(2); + expect(report.recent[1]?.error).toContain("sandbox spawn timeout"); + expect(report.inFlight).toHaveLength(1); + expect(report.sandboxes[0]?.sandboxId).toBe("asbx-1755000000-1"); + expect(report.warmPool).toEqual({ claimed: 41, ready: 2 }); + // Zero-filled to exactly 7 UTC days, oldest first, today last. + expect(report.daily).toHaveLength(7); + expect(report.daily[0]).toEqual({ day: "2026-08-06", failed: 0, runs: 0 }); + expect(report.daily[5]).toEqual({ day: "2026-08-11", failed: 3, runs: 40 }); + expect(report.daily[6]).toEqual({ day: "2026-08-12", failed: 0, runs: 20 }); + }); + + it("marks api-rs unreachable (null) but still parses the report", async () => { + const fetchFn: DiscordbotFetch = async (url) => { + const path = String(url); + if (path.endsWith("/healthz") || path.endsWith("/readyz")) { + throw new Error("connect ECONNREFUSED"); + } + return new Response(JSON.stringify(FULL_REPORT), { status: 200 }); + }; + const report = await collectStatus({ + apiUrl: "http://api", + fetchFn, + nowMs: NOW, + }); + expect(report.apiHealthy).toBeNull(); + expect(report.apiReady).toBeNull(); + expect(report.reportOk).toBe(true); + }); + + it("distinguishes unhealthy (false) from unreachable (null)", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + fetchFn: apiFetch({ ready: 503 }), + nowMs: NOW, + }); + expect(report.apiHealthy).toBe(true); + expect(report.apiReady).toBe(false); + }); + + it("still reports health when the status report fails", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + fetchFn: apiFetch({ reportStatus: 500 }), + nowMs: NOW, + }); + expect(report.apiHealthy).toBe(true); + expect(report.reportOk).toBe(false); + expect(report.recent).toEqual([]); + expect(report.daily.every((day) => day.runs === 0)).toBe(true); + }); + + it("tolerates a malformed report body", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + fetchFn: apiFetch({ report: "not an object" }), + nowMs: NOW, + }); + expect(report.reportOk).toBe(false); + expect(report.recent).toEqual([]); + }); +}); + +describe("formatStatus", () => { + const baseReport = (): StatusReport => ({ + apiHealthy: true, + apiReady: true, + daily: [], + inFlight: [], + recent: [], + reportOk: true, + sandboxes: [], + tally: {}, + warmPool: {}, + }); + + it("renders a code-block table with tags, tallies, and sandboxes", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + fetchFn: apiFetch({}), + nowMs: NOW, + }); + const text = formatStatus(report, "centaur"); + // Header names the bot and stays outside the block. + expect(text.startsWith("**centaur status** · api-rs ✅")).toBe(true); + expect(text).toContain("24h: 41 ok · 2 FAIL"); + // Column headings above the turn table. + expect(text).toMatch(/THREAD\s+WHO\s+AGE\s+TOOK/); + // In-flight row first: session title, requester, no duration yet. + const lines = text.split("\n"); + const runLine = lines.find((line) => line.startsWith("run")); + expect(runLine).toContain("fix the deploy pipeline"); + expect(runLine).toContain("oliver"); + expect(runLine?.trimEnd().endsWith("-")).toBe(true); + // Untitled management turn falls back to the friendly PR label. + expect(text).toContain("GH PR splits-teams#1799"); + expect(text).toMatch(/ok\s+GH PR splits-teams#1799\s+0xdiid\s+5m\s+1m/); + // Errors land on their own indented line. + expect(text).toContain("└ sandbox spawn timeout"); + expect(text).toContain( + "sandboxes: 1 active · warm: 2 ready · warm 24h: 41 claimed", + ); + // Histogram in its OWN code block, after the live view. + expect(text.split("```")).toHaveLength(5); + expect(text.indexOf("LAST 7 DAYS")).toBeGreaterThan( + text.indexOf("sandboxes:"), + ); + expect(text).toMatch(/Tue {2}█{16}\s+40\s+3/); + expect(text).toMatch(/Wed {2}█+\s+20\s+-/); + expect(text).toMatch(/Thu {2}\s+0\s+-/); + expect(text).toContain("7d: 72 runs · 3 failed (4.2%)"); + expect(text.length).toBeLessThanOrEqual(2000); + }); + + it("neutralizes backticks and newlines in titles and errors", () => { + const report = baseReport(); + report.recent = [ + { + ageSeconds: 60, + durationSeconds: 5, + error: "boom ``` **bold**\nnext line", + status: "failed", + threadKey: "discord:1:2", + title: "evil ``` title", + who: "someone", + }, + ]; + const text = formatStatus(report, "centaur"); + // Exactly the wrapper's own fence pair — no fences leaked from values. + expect(text.split("```")).toHaveLength(3); + expect(text).toContain("evil ''' title"); + expect(text).toContain("boom ''' **bold** next line"); + }); + + it("keeps thread rows within the column budget", () => { + const report = baseReport(); + report.recent = [ + { + ageSeconds: 60, + durationSeconds: 30, + error: "", + status: "completed", + threadKey: `discord:${"9".repeat(60)}`, + title: "", + who: "someone-with-a-long-name", + }, + ]; + const text = formatStatus(report, "centaur"); + const row = text.split("\n").find((line) => line.startsWith("ok")); + expect(row).toBeDefined(); + // Middle ellipsis keeps the platform head and the id tail. + expect(row).toContain("Discord 9"); + expect(row).toContain("…"); + expect(row).toContain("someone-w…"); + expect(row?.length ?? 0).toBeLessThanOrEqual(52); + }); + + it("marks a down api-rs and missing report honestly", () => { + const report = baseReport(); + report.apiHealthy = null; + report.apiReady = false; + report.reportOk = false; + const text = formatStatus(report, "centaur"); + expect(text).toContain("api-rs ❓"); + expect(text).toContain("ready ❌"); + expect(text).toContain("data ❌"); + expect(text).toContain("status report unavailable"); + }); + + it("stays under the Discord cap with oversized errors", () => { + const report = baseReport(); + report.recent = Array.from({ length: 30 }, (_, index) => ({ + ageSeconds: 60 * index, + durationSeconds: 5, + error: "x".repeat(150), + status: "failed", + threadKey: `discord:${"y".repeat(80)}:${index}`, + title: "", + who: "someone", + })); + const text = formatStatus(report, "centaur"); + expect(text.length).toBeLessThanOrEqual(2000); + expect(text.endsWith("```")).toBe(true); + }); +});