diff --git a/Cargo.lock b/Cargo.lock index 73cf622..78318ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1502,7 +1502,7 @@ dependencies = [ [[package]] name = "rawtree-cli" -version = "0.7.5" +version = "0.7.6" dependencies = [ "anyhow", "axoupdater", diff --git a/Cargo.toml b/Cargo.toml index 0f1b67a..e209dfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rawtree-cli" -version = "0.7.5" +version = "0.7.6" edition = "2021" repository = "https://github.com/rawtreedb/rawtree-cli" diff --git a/src/client.rs b/src/client.rs index 7422419..3466615 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,4 +1,5 @@ mod discovery; +pub(crate) use discovery::UnavailableRegion; mod regions; mod routing; #[cfg(test)] diff --git a/src/client/discovery.rs b/src/client/discovery.rs index 2bcde8f..cbf6afa 100644 --- a/src/client/discovery.rs +++ b/src/client/discovery.rs @@ -1,14 +1,99 @@ use super::{ handle_response, - regions::Regions, + regions::{Region, Regions}, routing::{before_deadline, valid_routing}, ApiClient, }; use anyhow::{Context, Result}; -use reqwest::Method; +use reqwest::{blocking::RequestBuilder, Method, StatusCode}; use serde::Deserialize; use serde_json::{json, Value}; -use std::{collections::HashMap, time::Instant}; +use std::{ + collections::HashMap, + sync::Mutex, + thread, + time::{Duration, Instant}, +}; + +const MAX_CATALOG_WORKERS: usize = 8; +const REGIONAL_LOOKUP_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Clone, Deserialize, serde::Serialize)] +pub(crate) struct UnavailableRegion { + pub id: String, + pub name: String, +} + +struct DeploymentTarget { + routing: Option<(String, String)>, + regions: Vec, +} + +struct OrganizationCatalog { + name: String, + regions: HashMap, + deployments: Vec, +} + +// Build requests on the caller thread: the routing cache is intentionally not shared. +// Bound native threads, and include queueing and response bodies in the lookup deadline. +fn load_catalogs(requests: Vec, deadline: Instant) -> Result>> { + let worker_count = requests.len().min(MAX_CATALOG_WORKERS); + let requests = Mutex::new(requests.into_iter().enumerate()); + thread::scope(|scope| { + let workers: Vec<_> = (0..worker_count) + .map(|_| { + thread::Builder::new() + .spawn_scoped(scope, || { + let mut responses = Vec::new(); + loop { + let next = requests + .lock() + .map_err(|_| anyhow::anyhow!("Regional discovery queue failed"))? + .next(); + let Some((index, request)) = next else { + break; + }; + let response = before_deadline(request, Some(deadline)) + .and_then(|request| Ok(request.send()?)); + let result = match response { + Ok(response) => { + let status = response.status(); + let result = handle_response(response); + if matches!( + status, + StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN + ) { + return Err(result.err().unwrap_or_else(|| { + anyhow::anyhow!("Access denied ({status})") + })); + } + result + } + Err(error) => Err(error), + }; + responses.push((index, result)); + } + Ok(responses) + }) + .context("Failed to start regional discovery worker") + }) + .collect::>>()?; + let mut responses = Vec::new(); + for worker in workers { + responses.extend( + worker + .join() + .map_err(|_| anyhow::anyhow!("Regional discovery worker failed"))??, + ); + } + responses.sort_by_key(|(index, _)| *index); + Ok(responses + .into_iter() + .map(|(_, response)| response) + .collect()) + }) +} // Metadata identifies placement; only the owning catalog supplies authoritative fields. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -36,7 +121,7 @@ impl ApiClient { .is_some_and(|token| !token.starts_with("rt_")) } - /// Discover each enabled deployment before publishing or caching a complete list. + /// Discover enabled deployments concurrently, retaining explicit partial-catalog metadata. pub(super) fn discover_clusters( &self, organization: Option<&str>, @@ -58,9 +143,7 @@ impl ApiClient { organization.is_none() || !selected.is_empty(), "Organization not found." ); - let mut result = json!({"clusters": []}); - let mut clusters = Vec::new(); - let mut positions = HashMap::new(); + let mut catalogs = Vec::new(); for owner in selected { let regions: Regions = handle_response( before_deadline( @@ -75,89 +158,175 @@ impl ApiClient { )? .send()?, )?; - let mut deployments = Vec::new(); + let mut deployments: Vec = Vec::new(); for region in regions.regions.iter().filter(|region| region.enabled) { let pair = region .deployment_id .as_ref() .filter(|id| valid_routing(id, ®ion.cloud_region)) .map(|id| (id.clone(), region.cloud_region.clone())); - if !deployments.contains(&pair) { - deployments.push(pair); + let unavailable = UnavailableRegion { + id: region.id.clone(), + name: region.name.clone(), + }; + if let Some(target) = deployments.iter_mut().find(|target| target.routing == pair) { + target.regions.push(unavailable); + } else { + deployments.push(DeploymentTarget { + routing: pair, + regions: vec![unavailable], + }); } } - let regions: HashMap<_, _> = regions - .regions - .iter() - .map(|region| (region.id.as_str(), region)) - .collect(); - for deployment in deployments { - let response: Value = handle_response( - before_deadline( - self.deployment_request( - Method::GET, - &format!( - "/v1/clusters?organization={}", - urlencoding::encode(&owner.name) - ), - deployment.as_ref(), - ), - deadline, - )? - .send()?, - )?; - let items = response - .get("clusters") - .and_then(Value::as_array) - .context("Invalid cluster catalog response")?; - if organization.is_some() { - result = response.clone(); + catalogs.push(OrganizationCatalog { + name: owner.name, + regions: regions + .regions + .into_iter() + .map(|region| (region.id.clone(), region)) + .collect(), + deployments, + }); + } + // All organizations share one concurrent regional lookup window. + let targets: Vec<_> = catalogs + .iter() + .flat_map(|catalog| { + catalog + .deployments + .iter() + .map(move |target| (catalog, target)) + }) + .collect(); + let mut result = json!({"clusters": []}); + let mut clusters = Vec::new(); + let mut positions = HashMap::new(); + let mut unavailable_regions = Vec::new(); + let lookup_deadline = Instant::now() + REGIONAL_LOOKUP_TIMEOUT; + let lookup_deadline = + deadline.map_or(lookup_deadline, |deadline| deadline.min(lookup_deadline)); + let requests = targets + .iter() + .map(|(catalog, target)| { + self.deployment_request( + Method::GET, + &format!( + "/v1/clusters?organization={}", + urlencoding::encode(&catalog.name) + ), + target.routing.as_ref(), + ) + }) + .collect(); + let responses = load_catalogs(requests, lookup_deadline)?; + if !responses.is_empty() && responses.iter().all(Result::is_err) { + return Err(responses + .into_iter() + .find_map(Result::err) + .context("Missing regional failure")?); + } + for ((catalog, target), response) in targets.into_iter().zip(responses) { + let response = match response { + Ok(response) => response, + Err(_) => { + unavailable_regions.extend(target.regions.iter().cloned()); + continue; } - for item in items { - let mut cluster = item.clone(); - let id = item - .get("id") - .and_then(Value::as_str) - .context("Cluster is missing its ID")?; - let region_id = item - .get("region_id") - .and_then(Value::as_str) - .filter(|id| !id.is_empty()); - let mut authority = CatalogAuthority::Fallback; - if let Some(region_id) = region_id { - let region = regions.get(region_id).with_context(|| { - format!("Region '{region_id}' for cluster '{id}' was not found.") - })?; - let owns_cluster = match deployment.as_ref() { - Some((id, cloud_region)) => { - region.deployment_id.as_ref() == Some(id) - && region.cloud_region == *cloud_region - } - None => region.deployment_id.is_none(), - }; - authority = if owns_cluster { - CatalogAuthority::OwningDeployment - } else { - CatalogAuthority::RegionMetadata - }; - cluster["deployment_id"] = json!(region.deployment_id); - cluster["cloud_region"] = json!(region.cloud_region); - } - match positions.get_mut(id) { - Some((index, previous)) if authority > *previous => { - clusters[*index] = cluster; - *previous = authority; - } - Some(_) => {} - None => { - positions.insert(id.to_owned(), (clusters.len(), authority)); - clusters.push(cluster); + }; + let deployment = &target.routing; + let items = response + .get("clusters") + .and_then(Value::as_array) + .context("Invalid cluster catalog response")?; + if organization.is_some() { + result = response.clone(); + } + for item in items { + let mut cluster = item.clone(); + let id = item + .get("id") + .and_then(Value::as_str) + .context("Cluster is missing its ID")?; + let region_id = item + .get("region_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()); + let mut authority = CatalogAuthority::Fallback; + if let Some(region_id) = region_id { + let region = catalog.regions.get(region_id).with_context(|| { + format!("Region '{region_id}' for cluster '{id}' was not found.") + })?; + let owns_cluster = match deployment.as_ref() { + Some((id, cloud_region)) => { + region.deployment_id.as_ref() == Some(id) + && region.cloud_region == *cloud_region } + None => region.deployment_id.is_none(), + }; + authority = if owns_cluster { + CatalogAuthority::OwningDeployment + } else { + CatalogAuthority::RegionMetadata + }; + cluster["deployment_id"] = json!(region.deployment_id); + cluster["cloud_region"] = json!(region.cloud_region); + } + match positions.get_mut(id) { + Some((index, previous)) if authority > *previous => { + clusters[*index] = cluster; + *previous = authority; + } + Some(_) => {} + None => { + positions.insert(id.to_owned(), (clusters.len(), authority)); + clusters.push(cluster); } } } } result["clusters"] = json!(clusters); + if !unavailable_regions.is_empty() { + result["unavailable_regions"] = json!(unavailable_regions); + } Ok(result) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + + #[test] + fn queued_catalogs_share_the_deadline_and_bound_active_connections() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let (stop, stopped) = std::sync::mpsc::channel(); + let server = thread::spawn(move || { + let mut sockets = Vec::new(); + while stopped.try_recv().is_err() { + match listener.accept() { + Ok((socket, _)) => sockets.push(socket), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(2)) + } + Err(error) => panic!("accept: {error}"), + } + } + sockets.len() + }); + let client = reqwest::blocking::Client::new(); + let requests = (0..24).map(|_| client.get(&url)).collect(); + let started = Instant::now(); + let responses = load_catalogs(requests, started + Duration::from_millis(300)).unwrap(); + stop.send(()).unwrap(); + assert_eq!(server.join().unwrap(), 8); + assert_eq!(responses.len(), 24); + assert!(responses.iter().all(Result::is_err)); + assert!( + started.elapsed() < Duration::from_millis(750), + "queued requests must not receive a fresh timeout" + ); + } +} diff --git a/src/client/routing.rs b/src/client/routing.rs index 3db375a..1f9f1a4 100644 --- a/src/client/routing.rs +++ b/src/client/routing.rs @@ -1,4 +1,4 @@ -use super::{handle_response, ApiClient}; +use super::{discovery::UnavailableRegion, handle_response, ApiClient}; use anyhow::{Context, Result}; use reqwest::{blocking::RequestBuilder, Method, Url}; use serde::Deserialize; @@ -26,15 +26,17 @@ struct Cluster { #[serde(default)] cloud_region: Option, } -#[derive(Deserialize)] +#[derive(Clone, Deserialize)] struct Clusters { clusters: Vec, + #[serde(default)] + unavailable_regions: Vec, } #[derive(Default)] pub(super) struct RoutingCache { organizations: RefCell)>>, - clusters: RefCell)>>, + clusters: RefCell>, } impl ApiClient { @@ -52,10 +54,10 @@ impl ApiClient { url.query_pairs().find(|(key, _)| key == "organization") { if let Ok(response) = serde_json::from_value::(value.clone()) { - self.routing.clusters.borrow_mut().insert( - clusters_path(&organization), - (Instant::now(), response.clusters), - ); + self.routing + .clusters + .borrow_mut() + .insert(clusters_path(&organization), (Instant::now(), response)); } } } @@ -167,14 +169,22 @@ impl ApiClient { self.routing .clusters .borrow_mut() - .insert(collection, (Instant::now(), response.clusters.clone())); - response.clusters + .insert(collection, (Instant::now(), response.clone())); + response } }; - let target = clusters.iter().find(|item| match &cluster { + let target = clusters.clusters.iter().find(|item| match &cluster { ClusterSelector::Id(id) => item.id == *id, ClusterSelector::Name(name) => item.name == *name, }); + if target.is_none() && !clusters.unavailable_regions.is_empty() { + // Keep known routes usable, but retry missing routes after a region recovers. + self.routing + .clusters + .borrow_mut() + .remove(&clusters_path(&organization)); + anyhow::bail!("Some regions are unavailable. Cannot resolve the selected cluster; retry when regional discovery recovers."); + } let pair = target.and_then(|item| { item.deployment_id .as_deref() diff --git a/src/client/routing_tests.rs b/src/client/routing_tests.rs index e4751a3..885984d 100644 --- a/src/client/routing_tests.rs +++ b/src/client/routing_tests.rs @@ -19,7 +19,9 @@ fn server(responses: Vec<(&'static str, Value)>) -> (String, thread::JoinHandle< let base = format!("http://{}", listener.local_addr().unwrap()); let handle = thread::spawn(move || { let mut requests = Vec::new(); - for (status, payload) in responses { + let mut responses: Vec<_> = responses.into_iter().enumerate().collect(); + let mut regional_targets = Vec::new(); + while !responses.is_empty() { let deadline = Instant::now() + Duration::from_secs(5); let mut socket = loop { match listener.accept() { @@ -54,15 +56,49 @@ fn server(responses: Vec<(&'static str, Value)>) -> (String, thread::JoinHandle< .unwrap_or(0); let mut body = vec![0; length]; reader.read_exact(&mut body).unwrap(); + let index = if first.starts_with("GET /v1/clusters?") && !regional_targets.is_empty() { + let pair = headers + .get("x-rawtree-deployment-id") + .zip(headers.get("x-rawtree-region")) + .map(|(id, region)| (id.clone(), region.clone())); + regional_targets + .iter() + .position(|target| *target == pair) + .expect("unexpected regional request") + } else { + 0 + }; + if first.starts_with("GET /v1/clusters?") && !regional_targets.is_empty() { + regional_targets.remove(index); + } + let (order, (status, payload)) = responses.remove(index); + if let Some(regions) = payload["regions"].as_array() { + regional_targets.clear(); + for region in regions.iter().filter(|region| region["enabled"] == true) { + let pair = region["deployment_id"] + .as_str() + .zip(region["cloud_region"].as_str()) + .filter(|(id, region)| super::routing::valid_routing(id, region)) + .map(|(id, region)| (id.to_owned(), region.to_owned())); + if !regional_targets.contains(&pair) { + regional_targets.push(pair); + } + } + } let response = payload.to_string(); write!(socket, "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response}", response.len()).unwrap(); - requests.push(Request { - path: first.trim().to_owned(), - headers, - body, - }); + requests.push(( + order, + Request { + path: first.trim().to_owned(), + headers, + body, + }, + )); } - requests + // Return requests in fixture order so assertions do not depend on scheduling. + requests.sort_by_key(|(order, _)| *order); + requests.into_iter().map(|(_, request)| request).collect() }); (base, handle) } @@ -455,21 +491,28 @@ fn discovery_runs_before_a_direct_cluster_request_and_retries_after_regional_fai "503 Service Unavailable", json!({"message":"Europe unavailable"}), ); + responses.push(("200 OK", json!({"tables":[]}))); responses.extend(multi_region_catalog("1234abcd")); responses.push(("200 OK", json!({"data":[]}))); let (base, server) = server(responses); let client = ApiClient::new(base, Some("session.fixture".into())); + let partial: Value = client.get("/v1/clusters?organization=acme").unwrap(); + assert_eq!(partial["clusters"].as_array().unwrap().len(), 1); + assert_eq!(partial["unavailable_regions"].as_array().unwrap().len(), 2); + let _: Value = client + .get("/v1/tables?organization=acme&cluster=america") + .unwrap(); assert!(client - .get::("/v1/clusters?organization=acme") + .get::("/v1/databases?organization=acme&cluster=europe") .unwrap_err() .to_string() - .contains("Europe unavailable")); + .contains("Some regions are unavailable")); let _: Value = client .get("/v1/databases?organization=acme&cluster=europe") .unwrap(); let requests = server.join().unwrap(); - assert_eq!(requests.len(), 9); - assert_eq!(requests[8].headers["x-rawtree-region"], "eu-west-1"); + assert_eq!(requests.len(), 10); + assert_eq!(requests[9].headers["x-rawtree-region"], "eu-west-1"); } #[test] @@ -581,3 +624,121 @@ fn api_keys_preserve_the_public_gateway_url() { assert_eq!(request.url().host_str(), Some("api.rawtree.com")); assert!(!request.headers().contains_key("x-rawtree-region")); } + +#[test] +fn empty_partial_catalog_is_marked_and_explicit_refresh_replaces_it() { + let mut responses = multi_region_catalog("1234abcd"); + responses[2].1["clusters"] = json!([]); + responses[3] = ( + "503 Service Unavailable", + json!({"message":"Europe unavailable"}), + ); + responses.extend(multi_region_catalog("1234abcd")); + responses.push(("200 OK", json!({"tables":[]}))); + let (base, server) = server(responses); + let client = ApiClient::new(base, Some("session.fixture".into())); + let partial: Value = client.get("/v1/clusters?organization=acme").unwrap(); + assert_eq!(partial["clusters"], json!([])); + assert_eq!(partial["unavailable_regions"].as_array().unwrap().len(), 2); + let refreshed: Value = client.get("/v1/clusters?organization=acme").unwrap(); + assert!(refreshed.get("unavailable_regions").is_none()); + let _: Value = client + .get("/v1/tables?organization=acme&cluster=europe") + .unwrap(); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 9); + assert_eq!(requests[8].headers["x-rawtree-deployment-id"], "1234abcd"); +} + +#[test] +fn all_organization_discovery_is_concurrent_and_retains_healthy_catalogs() { + for reverse in [false, true] { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let server = thread::spawn(move || { + let read = || { + let deadline = Instant::now() + Duration::from_secs(2); + let socket = loop { + match listener.accept() { + Ok((socket, _)) => break socket, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "catalog requests must run concurrently across organizations" + ); + thread::sleep(Duration::from_millis(2)); + } + Err(error) => panic!("accept: {error}"), + } + }; + socket.set_nonblocking(false).unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut reader = BufReader::new(socket.try_clone().unwrap()); + let mut first = String::new(); + reader.read_line(&mut first).unwrap(); + loop { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + if line == "\r\n" { + break; + } + } + (socket, first) + }; + let reply = |socket: &mut std::net::TcpStream, status: u16, body: Value| { + let body = body.to_string(); + write!(socket, "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap(); + }; + let (mut socket, first) = read(); + assert!(first.starts_with("GET /v1/organizations ")); + let mut organizations = vec![ + json!({"organization_id":"acme-id","name":"acme"}), + json!({"organization_id":"other-id","name":"other"}), + ]; + if reverse { + organizations.reverse(); + } + reply(&mut socket, 200, json!({"organizations":organizations})); + for _ in 0..2 { + let (mut socket, first) = read(); + assert!(first.contains("/regions ")); + reply( + &mut socket, + 200, + json!({"regions":[{"id":"legacy", "name":"Virginia", "cloud_region":"us-east-1", "deployment_id":null, "enabled":true}]}), + ); + } + // Accept both catalog requests before replying to either. A serial + // implementation cannot satisfy this barrier, even with a short timeout. + let catalogs = [read(), read()]; + for (mut socket, first) in catalogs { + if first.contains("organization=acme ") { + reply( + &mut socket, + 200, + json!({"clusters":[{"id":"healthy", "name":"production", "region_id":"legacy"}]}), + ); + } else { + assert!(first.contains("organization=other ")); + reply( + &mut socket, + 503, + json!({"message":"Other organization unavailable"}), + ); + } + } + }); + let client = ApiClient::new(base, Some("session.fixture".into())); + let catalog: Value = client.get("/v1/clusters").unwrap(); + assert_eq!(catalog["clusters"].as_array().unwrap().len(), 1); + assert_eq!(catalog["clusters"][0]["id"], "healthy"); + assert_eq!( + catalog["unavailable_regions"], + json!([{"id":"legacy", "name":"Virginia"}]) + ); + server.join().unwrap(); + } +} diff --git a/src/commands/cluster.rs b/src/commands/cluster.rs index e0ba7da..718b616 100644 --- a/src/commands/cluster.rs +++ b/src/commands/cluster.rs @@ -6,7 +6,7 @@ use serde_json::{json, Value}; use super::table_output::new_cli_table; use crate::cli::ClusterSizeArg; -use crate::client::ApiClient; +use crate::client::{ApiClient, UnavailableRegion}; use crate::cluster_identity::ClusterIdentityArgs; use crate::config; use crate::output; @@ -16,6 +16,8 @@ use crate::s3_storage::{format_storage, S3StorageArgs, S3StorageMetadata}; struct ListClustersResponse { organization: Option, clusters: Vec, + #[serde(default)] + unavailable_regions: Vec, } #[derive(Deserialize)] @@ -215,7 +217,7 @@ pub fn update( } let (_, response) = load_dedicated_clusters(client, organization)?; - let cluster = resolve_cluster(&response.clusters, name_or_id)?; + let cluster = resolve_cluster(&response, name_or_id)?; let mut body = serde_json::Map::new(); if let Some(name) = name { body.insert("name".to_string(), json!(name)); @@ -256,8 +258,18 @@ pub fn update( pub fn list(client: &ApiClient, organization: Option<&str>, json_mode: bool) -> Result<()> { let (value, resp) = load_dedicated_clusters(client, organization)?; - output::print_result(&value, json_mode, |_| { + if !resp.unavailable_regions.is_empty() { + let names = resp + .unavailable_regions + .iter() + .map(|region| region.name.as_str()) + .collect::>() + .join(", "); + eprintln!( + "Warning: Some regions are unavailable ({names}). Cluster discovery is incomplete." + ); + } if resp.clusters.is_empty() { println!("{}", empty_list_message(&resp, organization)); return; @@ -377,7 +389,7 @@ pub fn status( json_mode: bool, ) -> Result<()> { let (_, resp) = load_dedicated_clusters(client, organization)?; - let cluster = resolve_cluster(&resp.clusters, name_or_id)?; + let cluster = resolve_cluster(&resp, name_or_id)?; output::print_result(cluster, json_mode, |cluster| { println!("Cluster: {}", cluster.name); @@ -452,7 +464,7 @@ pub fn delete( json_mode: bool, ) -> Result<()> { let (_, resp) = load_dedicated_clusters(client, organization)?; - let cluster = resolve_cluster(&resp.clusters, name_or_id)?.clone(); + let cluster = resolve_cluster(&resp, name_or_id)?.clone(); let path = cluster_path(&cluster.id, None, organization); let result: DeleteClusterResponse = client.delete(&path)?; let value = delete_output(&cluster, result.deleted); @@ -588,11 +600,17 @@ fn create_request_body( body } -fn resolve_cluster<'a>(clusters: &'a [ClusterItem], name_or_id: &str) -> Result<&'a ClusterItem> { - clusters +fn resolve_cluster<'a>( + catalog: &'a ListClustersResponse, + name_or_id: &str, +) -> Result<&'a ClusterItem> { + catalog.clusters .iter() .find(|cluster| cluster.name == name_or_id || cluster.id == name_or_id) .ok_or_else(|| { + if !catalog.unavailable_regions.is_empty() { + return anyhow::anyhow!("Some regions are unavailable. Cannot resolve cluster '{name_or_id}'; retry when regional discovery recovers."); + } output::coded_error( "cluster_not_found", format!("Dedicated cluster '{name_or_id}' not found."), @@ -609,7 +627,7 @@ fn request_lifecycle( json_mode: bool, ) -> Result<()> { let (_, resp) = load_dedicated_clusters(client, organization)?; - let cluster = resolve_cluster(&resp.clusters, name_or_id)?; + let cluster = resolve_cluster(&resp, name_or_id)?; let path = cluster_path(&cluster.id, Some(action.path_segment()), organization); let value: Value = client.post_empty(&path)?; let updated: ClusterItem = @@ -844,7 +862,11 @@ mod tests { #[test] fn resolves_dedicated_cluster_by_name_or_id() { - let clusters = vec![cluster()]; + let clusters = ListClustersResponse { + organization: None, + clusters: vec![cluster()], + unavailable_regions: vec![], + }; assert_eq!( resolve_cluster(&clusters, "production") .expect("cluster by name") @@ -868,6 +890,22 @@ mod tests { assert_eq!(cli_error.exit_code(), 4); } + #[test] + fn partial_catalog_resolves_known_clusters_and_reports_outages_for_missing_targets() { + let catalog: ListClustersResponse = serde_json::from_value(json!({ + "clusters": [cluster_json()], + "unavailable_regions": [{"id":"oregon", "name":"Oregon"}] + })) + .unwrap(); + for selector in ["dev", "11111111-1111-1111-1111-111111111111"] { + assert_eq!(resolve_cluster(&catalog, selector).unwrap().name, "dev"); + } + for selector in ["missing", "22222222-2222-4222-8222-222222222222"] { + let error = resolve_cluster(&catalog, selector).err().unwrap(); + assert!(error.to_string().contains("Some regions are unavailable")); + } + } + fn cluster_json() -> serde_json::Value { json!({ "id": "11111111-1111-1111-1111-111111111111", @@ -888,7 +926,7 @@ mod tests { .expect("response without organization"); assert!(resp.organization.is_none()); assert_eq!( - resolve_cluster(&resp.clusters, "dev").expect("cluster").id, + resolve_cluster(&resp, "dev").expect("cluster").id, "11111111-1111-1111-1111-111111111111" ); } diff --git a/tests/regional_discovery.rs b/tests/regional_discovery.rs new file mode 100644 index 0000000..cbc24d1 --- /dev/null +++ b/tests/regional_discovery.rs @@ -0,0 +1,253 @@ +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::process::{Command, Output}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +#[derive(Clone, Copy)] +enum Failure { + Http(u16), + Disconnect, + Stall, + SlowBody, +} + +fn reply(socket: &mut TcpStream, status: u16, payload: Value) { + let body = payload.to_string(); + write!(socket, "HTTP/1.1 {status} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap(); +} + +// Exercise the actual CLI, using routing headers to distinguish deployment catalogs. +fn run(args: &[&str], failure: Failure, all_fail: bool) -> (Output, Vec, Duration) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = requests.clone(); + let (stop, stopped) = std::sync::mpsc::channel(); + let server = thread::spawn(move || { + let mut workers = Vec::new(); + while stopped.try_recv().is_err() { + let (mut socket, _) = match listener.accept() { + Ok(connection) => connection, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(2)); + continue; + } + Err(error) => panic!("accept: {error}"), + }; + let requests = recorded.clone(); + workers.push(thread::spawn(move || { + socket.set_nonblocking(false).unwrap(); + socket.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut reader = BufReader::new(socket.try_clone().unwrap()); + let mut first = String::new(); + reader.read_line(&mut first).unwrap(); + let mut headers = HashMap::new(); + loop { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + if line == "\r\n" { break; } + let (name, value) = line.split_once(':').unwrap(); + headers.insert(name.to_ascii_lowercase(), value.trim().to_owned()); + } + assert_eq!(headers["authorization"], "Bearer session.fixture"); + requests.lock().unwrap().push(first.clone()); + if first.starts_with("GET /v1/organizations ") { + reply(&mut socket, 200, json!({"organizations":[{"organization_id":"org-id","name":"acme"}]})); + } else if first.starts_with("GET /v1/organizations/org-id/regions ") { + reply(&mut socket, 200, json!({"regions":[ + {"id":"healthy","name":"Virginia","cloud_region":"us-east-1","deployment_id":"11111111","enabled":true}, + {"id":"failed","name":"Oregon","cloud_region":"us-west-2","deployment_id":"22222222","enabled":true}, + {"id":"duplicate","name":"Oregon duplicate","cloud_region":"us-west-2","deployment_id":"22222222","enabled":true}, + {"id":"failed2","name":"Ireland","cloud_region":"eu-west-1","deployment_id":"33333333","enabled":true} + ]})); + } else if first.starts_with("GET /v1/clusters?") { + if !all_fail && headers["x-rawtree-deployment-id"] == "11111111" { + reply(&mut socket, 200, json!({"clusters":[{ + "id":"healthy-id", "name":"production", "region_id":"healthy", "created_at":"2026-10-28T00:00:00Z", "shared":false, + "status":{"phase":"running", "ready":true}, "idle_timeout_minutes":30, "can_pause":true, "can_resume":false + }]})); + } else { + match failure { + Failure::Http(status) => reply(&mut socket, status, json!({"message":"Regional failure"})), + Failure::Disconnect => (), + Failure::Stall => thread::sleep(Duration::from_secs(6)), + Failure::SlowBody => { + write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 200\r\n\r\n{{").unwrap(); + socket.flush().unwrap(); + thread::sleep(Duration::from_secs(6)); + } + } + } + } else { + assert!(first.starts_with("GET /v1/tables?"), "{first}"); + assert_eq!(headers["x-rawtree-deployment-id"], "11111111"); + assert_eq!(headers["x-rawtree-region"], "us-east-1"); + reply(&mut socket, 200, json!({"tables":[{"name":"events","total_rows":12,"total_bytes":34}]})); + } + })); + } + for worker in workers { + worker.join().unwrap(); + } + }); + let started = Instant::now(); + let output = Command::new(env!("CARGO_BIN_EXE_rtree")) + .env_remove("RAWTREE_API_KEY") + .env_remove("RAWTREE_API_URL") + .env("RAWTREE_DATABASE", "default") + .env("RAWTREE_CLUSTER", "production") + .args([ + "--api-url", + &url, + "--api-key", + "session.fixture", + "--org", + "acme", + ]) + .args(args) + .output() + .unwrap(); + let elapsed = started.elapsed(); + stop.send(()).unwrap(); + server.join().unwrap(); + let requests = requests.lock().unwrap().clone(); + (output, requests, elapsed) +} + +#[test] +fn table_list_survives_http_and_connection_failures_in_other_regions() { + for failure in [Failure::Http(503), Failure::Disconnect] { + let (output, requests, _) = run(&["--json", "table", "list"], failure, false); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + serde_json::from_slice::(&output.stdout).unwrap()["tables"][0]["name"], + "events" + ); + assert_eq!( + requests + .iter() + .filter(|path| path.starts_with("GET /v1/clusters?")) + .count(), + 3 + ); + assert_eq!(requests.len(), 6); + } +} + +#[test] +fn stalled_regions_and_bodies_share_one_five_second_discovery_window() { + for failure in [Failure::Stall, Failure::SlowBody] { + let (output, requests, elapsed) = run(&["--json", "table", "list"], failure, false); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(elapsed >= Duration::from_secs(4), "{elapsed:?}"); + assert!( + elapsed < Duration::from_secs(7), + "regional waits must overlap: {elapsed:?}" + ); + assert_eq!(requests.len(), 6); + } +} + +#[test] +fn cluster_list_exposes_partial_catalog_in_json_and_warns_in_text() { + for json_mode in [true, false] { + let args = if json_mode { + vec!["--json", "cluster", "list"] + } else { + vec!["cluster", "list"] + }; + let (output, _, _) = run(&args, Failure::Http(503), false); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + if json_mode { + let result: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(result["clusters"].as_array().unwrap().len(), 1); + assert_eq!(result["unavailable_regions"].as_array().unwrap().len(), 3); + assert!(output.stderr.is_empty()); + } else { + assert!(String::from_utf8_lossy(&output.stderr) + .contains("Oregon, Oregon duplicate, Ireland")); + } + } +} + +#[test] +fn authorization_and_total_outages_fail_without_executing_table_request() { + for (status, all_fail) in [(401, false), (403, false), (503, true)] { + let (output, requests, _) = run( + &["--json", "table", "list"], + Failure::Http(status), + all_fail, + ); + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + let error: Value = serde_json::from_slice(&output.stderr).unwrap(); + assert!(error["error"]["message"] + .as_str() + .unwrap() + .contains("Regional failure")); + assert!(!requests + .iter() + .any(|path| path.starts_with("GET /v1/tables?"))); + } +} + +#[test] +fn missing_cluster_in_partial_catalog_never_falls_back_to_gateway() { + let (output, requests, _) = run( + &["--json", "--cluster", "missing", "table", "list"], + Failure::Http(503), + false, + ); + assert!(!output.status.success()); + let error: Value = serde_json::from_slice(&output.stderr).unwrap(); + assert!(error["error"]["message"] + .as_str() + .unwrap() + .contains("Some regions are unavailable")); + assert!(!requests + .iter() + .any(|path| path.starts_with("GET /v1/tables?"))); +} + +#[test] +fn management_commands_report_outages_for_missing_targets_in_partial_catalogs() { + for target in ["missing-cluster", "44444444-4444-4444-8444-444444444444"] { + for command in ["status", "update", "stop", "resume", "delete"] { + let mut args = vec!["--json", "cluster", command, target]; + if command == "update" { + args.extend(["--idle-timeout-minutes", "60"]); + } + let (output, requests, _) = run(&args, Failure::Http(503), false); + assert!(!output.status.success(), "{command} {target}"); + assert!(output.stdout.is_empty()); + let error: Value = serde_json::from_slice(&output.stderr).unwrap(); + assert!( + error["error"]["message"] + .as_str() + .unwrap() + .contains("Some regions are unavailable"), + "{command} {target}: {error}" + ); + assert_ne!(error["error"]["code"], "cluster_not_found"); + assert_eq!(requests.len(), 5, "only discovery should run: {requests:?}"); + assert!(requests.iter().all(|request| request.starts_with("GET "))); + } + } +}