Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions services/api-rs/crates/centaur-api-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ mod error;
mod mcp;
mod routes;
mod slack_proxy;
mod status;
mod tool_discovery;
pub mod types;

Expand Down
3 changes: 2 additions & 1 deletion services/api-rs/crates/centaur-api-server/src/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ impl AppState {
.ok_or_else(|| ApiError::BadRequest("workflow runtime is not enabled".to_owned()))
}

fn pool(&self) -> Result<PgPool, ApiError> {
pub(crate) fn pool(&self) -> Result<PgPool, ApiError> {
let initialized = self
.initialized()
.ok_or_else(|| ApiError::ServiceUnavailable("api-rs is still starting".to_owned()))?;
Expand Down Expand Up @@ -209,6 +209,7 @@ pub fn build_router_with_app_state(state: AppState) -> Router {
.route("/readyz", get(readyz))
.route("/metrics", get(metrics))
.route("/api/personas", get(list_personas))
.route("/api/status", get(crate::status::status_report))
.route("/mcp", post(mcp_post).get(mcp_get))
.route(
"/.well-known/oauth-protected-resource",
Expand Down
183 changes: 183 additions & 0 deletions services/api-rs/crates/centaur-api-server/src/status.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
use std::{
sync::{Mutex, OnceLock},
time::{Duration, Instant},
};

use axum::{Json, extract::State};
use serde::Serialize;
use serde_json::{Value, json};
use sqlx::PgPool;

use crate::{error::ApiError, routes::AppState};

// Read-only operational snapshot for status surfaces (chat-bot "status"
// commands, dashboards): recent/in-flight executions, a 24h tally, active
// session sandboxes, warm-pool state, and a 7-calendar-day (UTC) run
// histogram. api-rs owns the session schema, so the SQL lives here rather
// than in every ingress service that wants a status view.
//
// The report is cached briefly in-process: the history queries scan
// session_executions (append-only, never pruned), and status commands are
// human-triggered but scriptable — the cache caps the database cost at one
// scan set per TTL regardless of how often callers ask.

const CACHE_TTL: Duration = Duration::from_secs(10);
const ERROR_SNIPPET_CHARS: i32 = 200;
const RECENT_LIMIT: i64 = 20;

static CACHE: OnceLock<Mutex<Option<(Instant, Value)>>> = OnceLock::new();

pub(crate) async fn status_report(State(state): State<AppState>) -> Result<Json<Value>, ApiError> {
let cache = CACHE.get_or_init(|| Mutex::new(None));
if let Some((stored_at, report)) = cache
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone()
&& stored_at.elapsed() < CACHE_TTL
{
return Ok(Json(report));
}

let pool = state.pool()?;
let report = build_report(&pool).await?;
*cache
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((Instant::now(), report.clone()));
Ok(Json(report))
}

#[derive(Serialize, sqlx::FromRow)]
struct ExecutionRow {
/// Seconds since the execution was created.
age_seconds: f64,
duration_seconds: Option<f64>,
error: Option<String>,
status: String,
thread_key: String,
title: Option<String>,
user_name: Option<String>,
}

#[derive(Serialize, sqlx::FromRow)]
struct CountRow {
count: i64,
status: String,
}

#[derive(Serialize, sqlx::FromRow)]
struct SandboxRow {
idle_seconds: f64,
sandbox_id: String,
thread_key: String,
}

#[derive(Serialize, sqlx::FromRow)]
struct DailyRow {
/// UTC calendar date, `YYYY-MM-DD`.
day: String,
failed: i64,
runs: i64,
}

async fn build_report(pool: &PgPool) -> Result<Value, ApiError> {
let recent = sqlx::query_as::<_, ExecutionRow>(
"SELECT e.thread_key, e.status, \
left(e.error, $1) AS error, \
extract(epoch FROM (now() - e.created_at))::float8 AS age_seconds, \
extract(epoch FROM (e.completed_at - e.started_at))::float8 AS duration_seconds, \
e.metadata ->> 'user_name' AS user_name, \
coalesce(s.title, s.metadata ->> 'discord_conversation_name', \
s.metadata ->> 'linear_conversation_name', \
s.metadata ->> 'slack_conversation_name') AS title \
FROM session_executions e \
LEFT JOIN sessions s ON s.thread_key = e.thread_key \
ORDER BY e.created_at DESC \
LIMIT $2",
)
.bind(ERROR_SNIPPET_CHARS)
.bind(RECENT_LIMIT)
.fetch_all(pool)
.await?;

let in_flight = sqlx::query_as::<_, ExecutionRow>(
"SELECT e.thread_key, e.status, \
NULL::text AS error, \
extract(epoch FROM (now() - e.created_at))::float8 AS age_seconds, \
NULL::float8 AS duration_seconds, \
e.metadata ->> 'user_name' AS user_name, \
coalesce(s.title, s.metadata ->> 'discord_conversation_name', \
s.metadata ->> 'linear_conversation_name', \
s.metadata ->> 'slack_conversation_name') AS title \
FROM session_executions e \
LEFT JOIN sessions s ON s.thread_key = e.thread_key \
WHERE e.status IN ('queued', 'running') \
ORDER BY e.created_at ASC \
LIMIT $1",
)
.bind(RECENT_LIMIT)
.fetch_all(pool)
.await?;

let tally_24h = sqlx::query_as::<_, CountRow>(
"SELECT status, count(*) AS count \
FROM session_executions \
WHERE created_at > now() - interval '24 hours' \
GROUP BY status",
)
.fetch_all(pool)
.await?;

let active_sandboxes = sqlx::query_as::<_, SandboxRow>(
"SELECT thread_key, sandbox_id, \
extract(epoch FROM (now() - sandbox_last_active_at))::float8 AS idle_seconds \
FROM sessions \
WHERE sandbox_id IS NOT NULL \
AND sandbox_last_active_at > now() - interval '2 hours' \
ORDER BY sandbox_last_active_at DESC \
LIMIT $1",
)
.bind(RECENT_LIMIT)
.fetch_all(pool)
.await?;

// ready/evicting are the pool's current state. claimed/failed rows are
// lifetime history (claiming flips status in place, rows are never
// deleted), so an unfiltered count reads like a leak — window them to 24h
// churn instead.
let warm_pool = sqlx::query_as::<_, CountRow>(
"SELECT status, count(*) AS count \
FROM session_warm_sandboxes \
WHERE status IN ('ready', 'evicting') \
OR updated_at > now() - interval '24 hours' \
GROUP BY status",
)
.fetch_all(pool)
.await?;

// Calendar-day (UTC) buckets over the last 7 days INCLUDING today, so the
// per-day rows and any total computed from them describe the same window
// (a rolling now()-7d fetch would include a partial 8th calendar day).
let daily = sqlx::query_as::<_, DailyRow>(
"SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS day, \
count(*) AS runs, \
count(*) FILTER (WHERE status = 'failed') AS failed \
FROM session_executions \
WHERE created_at >= \
(date_trunc('day', now() AT TIME ZONE 'UTC') - interval '6 days') \
AT TIME ZONE 'UTC' \
GROUP BY 1 \
ORDER BY 1",
)
.fetch_all(pool)
.await?;

Ok(json!({
"ok": true,
"active_sandboxes": active_sandboxes,
"daily": daily,
"in_flight": in_flight,
"recent_executions": recent,
"tally_24h": tally_24h,
"warm_pool": warm_pool,
}))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-- The /api/status report orders and windows session_executions by created_at
-- globally (ORDER BY created_at DESC LIMIT n; created_at > now() - '24 hours';
-- the 7-day histogram). The existing (thread_key, created_at, execution_id)
-- index cannot serve a global recency scan over this append-only, never-pruned
-- table, so give created_at its own index.
create index if not exists session_executions_created_at_idx
on session_executions (created_at desc);
57 changes: 57 additions & 0 deletions services/discordbot/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ import {
renameThreadFromMessage,
} from "./discord-threading";
import { setGatewayConnected } from "./gateway";
import {
STATUS_FAILURE_REPLY,
collectStatus,
formatStatus,
isStatusCommand,
} from "./status";
import {
collectInitialContext,
executeSessionTurn,
Expand Down Expand Up @@ -259,9 +265,58 @@ export function createDiscordbot(options: DiscordbotOptions): Discordbot {
DEFAULT_MAX_CONCURRENT_EXECUTIONS_PER_GUILD,
);

// A bare "status"/"health" mention answers straight from the control plane
// (api-rs /healthz + /api/status over HTTP — no SQL, no sandbox turn) so it
// still works when the agent pipeline is what's broken. The feature is
// OPT-IN per channel: statusChannelAllowlist empty/unset disables it
// entirely (the report exposes cross-platform activity — session titles,
// requester names, error snippets — so which channels may see it is a
// deployment decision, not a default). The exchange deliberately stays out
// of the session transcript: it's operational metadata, not conversation
// the agent should later see. Returns true when handled.
const statusChannels = new Set(options.statusChannelAllowlist ?? []);
const isStatusChannel = (threadKey: string): boolean => {
if (statusChannels.size === 0) return false;
const { channelId, threadId } = parseDiscordThreadKey(threadKey);
return (
(channelId !== undefined && statusChannels.has(channelId)) ||
(threadId !== undefined && statusChannels.has(threadId))
);
};
const maybeReplyStatus = async (
thread: Thread<DiscordbotThreadState>,
message: ChatMessage,
): Promise<boolean> => {
if (!isStatusCommand(message.text ?? "")) return false;
if (!isStatusChannel(thread.id)) return false;
try {
const report = await collectStatus({
apiUrl: options.apiUrl,
fetchFn: options.fetch,
});
await thread.post(formatStatus(report, userName));
} catch (error) {
// Internals (hostnames, auth errors) stay in the logs; the channel gets
// a generic line.
logger.warn("discordbot_status_reply_failed", {
error: errorMessage(error),
});
try {
await thread.post(STATUS_FAILURE_REPLY);
} catch {
// best-effort; nothing left to signal with.
}
}
return true;
};

chat.onNewMention(async (thread, message) => {
if (!isAllowedDiscordMessage(message, options, logger)) return;
// Subscribe before the status fast-path: the adapter has already created
// a thread for the mention, and an unsubscribed thread would silently
// ignore follow-ups.
await thread.subscribe();
if (await maybeReplyStatus(thread, message)) return;
await syncThreadMessageToSession(thread, message, {
executionLimiter,
mode: "execute",
Expand All @@ -272,6 +327,8 @@ export function createDiscordbot(options: DiscordbotOptions): Discordbot {

chat.onSubscribedMessage(async (thread, message) => {
if (!isAllowedDiscordMessage(message, options, logger)) return;
if (message.isMention === true && (await maybeReplyStatus(thread, message)))
return;
await syncThreadMessageToSession(thread, message, {
executionLimiter,
mode: message.isMention === true ? "execute" : "append",
Expand Down
3 changes: 3 additions & 0 deletions services/discordbot/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ const options: DiscordbotOptions = {
publicKey,
discordApiUrl: optionalEnv("DISCORD_API_URL"),
guildAllowlist: optionalList("DISCORDBOT_GUILD_ALLOWLIST"),
// Channels where "@bot status" answers with the control-plane report;
// unset = the status fast-path is disabled.
statusChannelAllowlist: optionalList("DISCORDBOT_STATUS_CHANNEL_IDS"),
idleTimeoutMs: optionalNumberEnv("SESSION_IDLE_TIMEOUT_MS"),
isGatewayActive: () => gateway.isActive(),
maxConcurrentExecutionsPerGuild: optionalNumberEnv(
Expand Down
Loading