Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -1126,6 +1126,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added

- `ErrorResponse` adds an optional typed `code: ErrorCode` (`"create_timeout"`,
`"exec_timeout"`, `"vm_not_found"`), `vm_id`, and `timeout_secs` on the HTTP
error wire contract, with service-side exec timeouts returning HTTP 504
(`StatusCode::GATEWAY_TIMEOUT`). The Python SDK adds host gateway discovery
(`discover_gateway`, `capsem_run_dir`, `GatewayEndpoint`,
`Hypervisor.connect`, and optional `url`/`token`/`run_dir` on `Hypervisor`
and `VM` resolving `CAPSEM_GATEWAY_URL`, `CAPSEM_GATEWAY_TOKEN`, and
`<run_dir>/gateway.{port,token}`) and structured exceptions
(`CapsemError`, `CapsemTimeoutError`, `CreateTimeoutError`,
`ExecTimeoutError`, and `VmNotFoundError`) dispatched from
`HttpError.code` (`models.ErrorCode`). The Python and TypeScript SDKs also
export `MAX_REQUEST_BODY_BYTES` and `CREATE_READY_SECS` guarded by Citadel
against the service and API constants.
- `POST /vms/create` (`ProvisionRequest`) and `POST /vms/{id}/fork` (`ForkRequest`)
accept optional advisory `labels` (`<= 64` entries, ASCII keys `1..=64`
matching the VM-name rule `[A-Za-z0-9][A-Za-z0-9_-]{0,63}`, values `<= 255`
Expand Down
16 changes: 15 additions & 1 deletion crates/capsem-api/src/files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,21 @@ pub struct LogsResponse {
pub process_logs: Option<String>,
}

#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum ErrorCode {
VmNotFound,
CreateTimeout,
ExecTimeout,
}

#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default, ToSchema)]
pub struct ErrorResponse {
pub error: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub code: Option<ErrorCode>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub vm_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub timeout_secs: Option<u64>,
}
2 changes: 1 addition & 1 deletion crates/capsem-service/src/active_policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ pub(crate) async fn push_policy_to_running_instances(
if failures.is_empty() {
Ok(total)
} else {
Err(AppError(
Err(AppError::new(
StatusCode::INTERNAL_SERVER_ERROR,
format!(
"policy applied to {} of {total} running VMs; not applied: {}",
Expand Down
6 changes: 6 additions & 0 deletions crates/capsem-service/src/api/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,16 @@ fn logs_response_roundtrip() {
fn error_response_roundtrip() {
let r = ErrorResponse {
error: "sandbox not found".into(),
code: Some(ErrorCode::VmNotFound),
vm_id: Some("vm-1".into()),
timeout_secs: None,
};
let json = serde_json::to_string(&r).unwrap();
assert!(!json.contains("timeout_secs"));
let r2: ErrorResponse = serde_json::from_str(&json).unwrap();
assert!(r2.error.contains("not found"));
assert_eq!(r2.code, Some(ErrorCode::VmNotFound));
assert_eq!(r2.vm_id.as_deref(), Some("vm-1"));
}

#[test]
Expand Down
2 changes: 1 addition & 1 deletion crates/capsem-service/src/asset_routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ pub(super) fn asset_manifest_status(state: &ServiceState) -> Result<Arc<api::Ass
let inputs = [manifest_file_identity(&path), manifest_file_identity(&metadata_path)];
let cache = || {
state.asset_manifest_cache.lock().map_err(|error| {
AppError(
AppError::new(
StatusCode::INTERNAL_SERVER_ERROR,
format!("asset manifest cache lock poisoned: {error}"),
)
Expand Down
2 changes: 1 addition & 1 deletion crates/capsem-service/src/blocking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ impl ServiceState {
{
let state = Arc::clone(self);
tokio::task::spawn_blocking(move || f(state)).await.map_err(|error| {
AppError(
AppError::new(
StatusCode::INTERNAL_SERVER_ERROR,
format!("blocking task failed: {error}"),
)
Expand Down
41 changes: 21 additions & 20 deletions crates/capsem-service/src/container_setup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -450,7 +450,7 @@ async fn run(state: &Arc<ServiceState>, id: &str, generation: u64, spec: Contain
{
return Ok(());
}
let uds_path = running_uds_path(state, id).map_err(|e| e.1)?;
let uds_path = running_uds_path(state, id).map_err(AppError::into_message)?;
let reply = send_ipc_command(
&uds_path,
ServiceToProcess::Exec {
Expand Down Expand Up @@ -532,7 +532,10 @@ pub(crate) async fn grant_surface(state: &Arc<ServiceState>, id: &str, generatio
}
let exposure = match crate::router_runtime::exposures::create_exposure(state, id, request).await {
Ok(exposure) => exposure,
Err(AppError(status, error)) => {
Err(AppError {
status,
body: ErrorResponse { error, .. },
}) => {
warn!(vm_id = id, %status, %error, "container surface exposure refused");
return;
}
Expand All @@ -550,7 +553,11 @@ pub(crate) async fn grant_surface(state: &Arc<ServiceState>, id: &str, generatio
Path((id.to_owned(), exposure.id.clone())),
)
.await;
if let Err(AppError(status, error)) = revoked {
if let Err(AppError {
status,
body: ErrorResponse { error, .. },
}) = revoked
{
if status != StatusCode::NOT_FOUND {
warn!(vm_id = id, exposure_id = exposure.id.as_str(), %status, %error, "late container exposure could not be revoked");
}
Expand Down Expand Up @@ -650,18 +657,12 @@ pub(crate) async fn wait_for_create(
/// Create, resume and carried-image clones share the same workload barrier.
/// A timeout leaves setup with its service owner; terminal failures are errors.
pub(crate) async fn require_ready(state: &Arc<ServiceState>, id: &str) -> Result<(), AppError> {
let status = wait_for_create(state, id).await.map_err(|timed_out| {
warn!(vm_id = id, attempts = timed_out.attempts, "container workload readiness timed out");
AppError(
StatusCode::GATEWAY_TIMEOUT,
format!(
"container workload for VM {id} did not become ready before the HTTP deadline; setup continues under service ownership"
),
)
})?;
let status = wait_for_create(state, id)
.await
.map_err(|t| AppError::create_timeout(id, t.attempts))?;
match status.state {
ContainerState::Running | ContainerState::Staged => Ok(()),
ContainerState::Exited | ContainerState::Failed => Err(AppError(
ContainerState::Exited | ContainerState::Failed => Err(AppError::new(
StatusCode::INTERNAL_SERVER_ERROR,
format!(
"container workload for VM {id} reached {:?}: {}",
Expand Down Expand Up @@ -700,7 +701,7 @@ async fn wait_observed(
}

pub(crate) fn record_launched(state: &ServiceState, id: &str) -> Result<(), String> {
let session_dir = resolve_session_dir(state, id).map_err(|e| e.1)?;
let session_dir = resolve_session_dir(state, id).map_err(AppError::into_message)?;
let Some(record) = state.containers.launch_record(id) else {
return Ok(());
};
Expand Down Expand Up @@ -744,7 +745,7 @@ pub(crate) async fn handle_container_status(
.off_worker(move |state| observe(&state, &id, live))
.await??
.map(Json)
.ok_or_else(|| AppError(StatusCode::NOT_FOUND, "VM has no container workload".into()))
.ok_or_else(|| AppError::new(StatusCode::NOT_FOUND, "VM has no container workload".into()))
}

fn observe(
Expand Down Expand Up @@ -820,7 +821,7 @@ async fn share_image(
.containers
.work_lease(id, generation)
.ok_or("container share cancelled")?;
let session_dir = resolve_session_dir(state, id).map_err(|e| e.1)?;
let session_dir = resolve_session_dir(state, id).map_err(AppError::into_message)?;
let (root, files) = (image.root.clone(), image.files.clone());
tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
let _lease = lease;
Expand Down Expand Up @@ -878,14 +879,14 @@ async fn stage_file(
Ok::<_, AppError>((parent, name, lease))
})
.await
.map_err(|e| e.1)?
.map_err(|e| e.1)?;
.map_err(AppError::into_message)?
.map_err(AppError::into_message)?;
let preview = file_security_preview_bytes(&file.bytes);
let size = file.bytes.len() as u64;
let uds_path = running_uds_path(state, id).map_err(|e| e.1)?;
let uds_path = running_uds_path(state, id).map_err(AppError::into_message)?;
if log_file_boundary_on_owner(state, &uds_path, FileBoundaryAction::Import, path, preview, size, None)
.await
.map_err(|e| e.1)?
.map_err(AppError::into_message)?
.is_some()
{
return Err(format!("file import policy rewrote staged image file {}", file.name));
Expand Down
6 changes: 3 additions & 3 deletions crates/capsem-service/src/container_setup/images.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ impl From<ImageError> for AppError {
ImageError::Refused(_) => StatusCode::FORBIDDEN,
ImageError::Failed(_) => StatusCode::BAD_GATEWAY,
};
AppError(status, error.to_string())
AppError::new(status, error.to_string())
}
}

Expand Down Expand Up @@ -360,8 +360,8 @@ pub(crate) async fn handle_list_images(
images: Vec::new(),
}));
};
let architecture =
stage::catalog_architecture().map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, format!("{e:#}")))?;
let architecture = stage::catalog_architecture()
.map_err(|e| AppError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("{e:#}")))?;
let mut images: Vec<ImageInfo> = loaded
.catalog
.entries()
Expand Down
4 changes: 2 additions & 2 deletions crates/capsem-service/src/container_setup/relaunch/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,8 @@ async fn restored_workload_failure_is_not_lifecycle_success() {
);
insert_fake_instance_with_session_dir(&state, "box", 1, session);
let error = restore_ready(&state, "box").await.unwrap_err();
assert_eq!(error.0, StatusCode::INTERNAL_SERVER_ERROR);
assert!(error.1.contains("Failed") && error.1.contains("did not start"));
assert_eq!(error.status, StatusCode::INTERNAL_SERVER_ERROR);
assert!(error.body.error.contains("Failed") && error.body.error.contains("did not start"));
}

#[tokio::test]
Expand Down
50 changes: 50 additions & 0 deletions crates/capsem-service/src/container_setup/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -941,3 +941,53 @@ fn admission_takes_either_content_address_and_nothing_else() {
let refused = admit(&policy, &requested, &pulled(&b, &c)).unwrap_err();
assert!(refused.contains("not admitted"), "{refused}");
}

#[tokio::test]
async fn container_helpers_propagate_missing_vm_and_boundary_errors() {
let fx = fixture(images());
assert!(record_launched(&fx.state, "missing").is_err());
let generation = fx.state.containers.begin("missing", "registry.example/app:1");
let pulled = PulledImage {
root: PathBuf::new(),
files: Vec::new(),
digest: "sha256:a".into(),
image_digest: "sha256:a".into(),
cache_key: None,
_hold: Box::new(()),
};
assert!(share_image(&fx.state, "missing", generation, &pulled, None)
.await
.is_err());
let mk_file = || stage::StagedFile {
name: "spec.json".to_string(),
bytes: vec![b'{', b'}'],
};
assert!(stage_file(&fx.state, "missing", generation, mk_file()).await.is_err());

let stopped_dir = fx._dir.path().join("stopped-session");
std::fs::create_dir_all(stopped_dir.join("guest/workspace")).unwrap();
let entry = crate::tests::test_persistent_entry("stopped-box", stopped_dir);
let stopped_id = entry.id.clone();
fx.state
.persistent_registry
.lock()
.unwrap()
.data
.vms
.insert("stopped-box".into(), entry);
let gen2 = fx.state.containers.begin(&stopped_id, "registry.example/app:1");
assert!(stage_file(&fx.state, &stopped_id, gen2, mk_file()).await.is_err());

let bad_uds = std::path::Path::new("/nonexistent/uds.sock");
assert!(log_file_boundary_on_owner(
&fx.state,
bad_uds,
FileBoundaryAction::Import,
"a".into(),
vec![],
0,
None
)
.await
.is_err());
}
6 changes: 3 additions & 3 deletions crates/capsem-service/src/credential_routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ pub(crate) async fn inject(
payload: Result<Json<api::CredentialInjectRequest>, axum::extract::rejection::JsonRejection>,
) -> Result<Json<api::CredentialInjectResponse>, AppError> {
let Json(request) =
payload.map_err(|_| AppError(StatusCode::BAD_REQUEST, "invalid credential injection request".into()))?;
payload.map_err(|_| AppError::new(StatusCode::BAD_REQUEST, "invalid credential injection request".into()))?;
let storage = request.storage;
let reference = state
.off_worker(move |_| {
Expand All @@ -25,7 +25,7 @@ pub(crate) async fn inject(
CredentialStore::global().inject(provider, &request.value, persistence)
})
.await?
.map_err(|error| AppError(StatusCode::BAD_REQUEST, error))?;
.map_err(|error| AppError::new(StatusCode::BAD_REQUEST, error))?;
if storage == api::CredentialStorage::Memory {
let paths: Vec<_> = state
.instances
Expand All @@ -36,7 +36,7 @@ pub(crate) async fn inject(
.collect();
for path in paths {
sync_memory(&state, &path).await.map_err(|_| {
AppError(
AppError::new(
StatusCode::SERVICE_UNAVAILABLE,
"credential stored in service memory; a VM owner has not acknowledged injection; retry explicitly"
.into(),
Expand Down
Loading
Loading