Compare commits
20 Commits
c2556ec236
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 083974c27f | |||
| bbb9394911 | |||
| 2759fde81f | |||
| 527efb510f | |||
| 66b9bfd875 | |||
| 6c5d0c62d0 | |||
| dd5dbba1f6 | |||
| 5818255cf5 | |||
| 343bb99bff | |||
| dc8171d052 | |||
| d92f817e73 | |||
| ad8390e297 | |||
| 466659c506 | |||
| 8375c7bbf0 | |||
| 2ccaba897d | |||
| 619394f509 | |||
| 501e0931c5 | |||
| c3188637b3 | |||
| 32cde544da | |||
| 10641faf43 |
+31
-4
@@ -232,7 +232,7 @@ jobs:
|
|||||||
--required-test 'operations page imports OpenAPI methods as drafts' \
|
--required-test 'operations page imports OpenAPI methods as drafts' \
|
||||||
--required-test 'OpenAPI upload rejects invalid files locally and restores focus after Escape' \
|
--required-test 'OpenAPI upload rejects invalid files locally and restores focus after Escape' \
|
||||||
--required-test 'OpenAPI upload recovers from pagehide and a preview server error' \
|
--required-test 'OpenAPI upload recovers from pagehide and a preview server error' \
|
||||||
--required-test 'OpenAPI upload invalidates active draft creation after language or workspace changes' \
|
--required-test 'OpenAPI apply preserves job authority after language or workspace changes' \
|
||||||
--required-test 'OpenAPI upload only renders the latest selected file and clears reset or close races' \
|
--required-test 'OpenAPI upload only renders the latest selected file and clears reset or close races' \
|
||||||
--required-test 'OpenAPI upload ignores a stale failure and renders only correlation identifiers'
|
--required-test 'OpenAPI upload ignores a stale failure and renders only correlation identifiers'
|
||||||
python3 ../../scripts/validate-capability-run.py \
|
python3 ../../scripts/validate-capability-run.py \
|
||||||
@@ -286,17 +286,44 @@ jobs:
|
|||||||
CRANK_SESSION_SECRET=ci-session-secret
|
CRANK_SESSION_SECRET=ci-session-secret
|
||||||
CRANK_PASSWORD_PEPPER=ci-password-pepper
|
CRANK_PASSWORD_PEPPER=ci-password-pepper
|
||||||
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
|
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
|
||||||
CRANK_BOOTSTRAP_ADMIN_PASSWORD=ci-admin-password
|
|
||||||
CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME=CI Owner
|
CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME=CI Owner
|
||||||
CRANK_BASE_URL=http://127.0.0.1
|
CRANK_BASE_URL=http://127.0.0.1
|
||||||
|
CRANK_ENVIRONMENT=ci
|
||||||
|
CRANK_OUTBOUND_ALLOWED_HOSTS=admin-api
|
||||||
CRANK_PUBLISH_BIND=127.0.0.1
|
CRANK_PUBLISH_BIND=127.0.0.1
|
||||||
CRANK_ADMIN_PUBLISH_PORT=0
|
CRANK_ADMIN_PUBLISH_PORT=0
|
||||||
CRANK_MCP_PUBLISH_PORT=0
|
CRANK_MCP_PUBLISH_PORT=0
|
||||||
CRANK_UI_PUBLISH_PORT=0
|
CRANK_UI_PUBLISH_PORT=0
|
||||||
CRANK_DEMO_SEED=true
|
CRANK_DEMO_SEED=true
|
||||||
EOF
|
EOF
|
||||||
docker compose -f deploy/community/docker-compose.images.yml \
|
compose=(docker compose -f deploy/community/docker-compose.images.yml \
|
||||||
--env-file .tmp/community-smoke.env --profile local-db up -d --wait
|
--env-file .tmp/community-smoke.env --profile local-db)
|
||||||
|
"${compose[@]}" up -d --wait postgres
|
||||||
|
"${compose[@]}" run --rm migrate
|
||||||
|
bootstrap_json="$("${compose[@]}" run --rm --no-deps \
|
||||||
|
--entrypoint crank-migrate migrate admin-auth bootstrap-create \
|
||||||
|
--email owner@crank.test --display-name 'CI Owner')"
|
||||||
|
bootstrap_token="$(python3 -c \
|
||||||
|
'import json,sys; print(json.loads(sys.stdin.read())["bootstrap_token"])' \
|
||||||
|
<<<"$bootstrap_json")"
|
||||||
|
install -d -m 700 .tmp/community-bootstrap
|
||||||
|
printf '%s' "$bootstrap_token" > .tmp/community-bootstrap/token
|
||||||
|
printf '%s' 'ci-admin-password' > .tmp/community-bootstrap/password
|
||||||
|
printf '%s' 'ci-password-pepper' > .tmp/community-bootstrap/password-pepper
|
||||||
|
chmod 600 .tmp/community-bootstrap/token \
|
||||||
|
.tmp/community-bootstrap/password \
|
||||||
|
.tmp/community-bootstrap/password-pepper
|
||||||
|
"${compose[@]}" run --rm --no-deps \
|
||||||
|
-v "$PWD/.tmp/community-bootstrap:/run/bootstrap:ro" \
|
||||||
|
--entrypoint crank-migrate migrate admin-auth bootstrap-complete \
|
||||||
|
--token-file /run/bootstrap/token \
|
||||||
|
--password-file /run/bootstrap/password \
|
||||||
|
--password-pepper-file /run/bootstrap/password-pepper
|
||||||
|
rm -f .tmp/community-bootstrap/token \
|
||||||
|
.tmp/community-bootstrap/password \
|
||||||
|
.tmp/community-bootstrap/password-pepper
|
||||||
|
rmdir .tmp/community-bootstrap
|
||||||
|
"${compose[@]}" up -d --wait
|
||||||
|
|
||||||
- name: Run authenticated Community image smoke
|
- name: Run authenticated Community image smoke
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -139,7 +139,7 @@ jobs:
|
|||||||
--required-test 'operations page imports OpenAPI methods as drafts' \
|
--required-test 'operations page imports OpenAPI methods as drafts' \
|
||||||
--required-test 'OpenAPI upload rejects invalid files locally and restores focus after Escape' \
|
--required-test 'OpenAPI upload rejects invalid files locally and restores focus after Escape' \
|
||||||
--required-test 'OpenAPI upload recovers from pagehide and a preview server error' \
|
--required-test 'OpenAPI upload recovers from pagehide and a preview server error' \
|
||||||
--required-test 'OpenAPI upload invalidates active draft creation after language or workspace changes' \
|
--required-test 'OpenAPI apply preserves job authority after language or workspace changes' \
|
||||||
--required-test 'OpenAPI upload only renders the latest selected file and clears reset or close races' \
|
--required-test 'OpenAPI upload only renders the latest selected file and clears reset or close races' \
|
||||||
--required-test 'OpenAPI upload ignores a stale failure and renders only correlation identifiers'
|
--required-test 'OpenAPI upload ignores a stale failure and renders only correlation identifiers'
|
||||||
python3 ../../scripts/validate-capability-run.py \
|
python3 ../../scripts/validate-capability-run.py \
|
||||||
@@ -217,8 +217,8 @@ jobs:
|
|||||||
CRANK_SESSION_SECRET=release-smoke-session
|
CRANK_SESSION_SECRET=release-smoke-session
|
||||||
CRANK_PASSWORD_PEPPER=release-smoke-pepper
|
CRANK_PASSWORD_PEPPER=release-smoke-pepper
|
||||||
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
|
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
|
||||||
CRANK_BOOTSTRAP_ADMIN_PASSWORD=release-smoke-password
|
|
||||||
CRANK_BASE_URL=http://127.0.0.1
|
CRANK_BASE_URL=http://127.0.0.1
|
||||||
|
CRANK_ENVIRONMENT=release-smoke
|
||||||
CRANK_PUBLISH_BIND=127.0.0.1
|
CRANK_PUBLISH_BIND=127.0.0.1
|
||||||
CRANK_ADMIN_PUBLISH_PORT=0
|
CRANK_ADMIN_PUBLISH_PORT=0
|
||||||
CRANK_MCP_PUBLISH_PORT=0
|
CRANK_MCP_PUBLISH_PORT=0
|
||||||
|
|||||||
@@ -18,21 +18,15 @@ FROM debian:bookworm-slim
|
|||||||
|
|
||||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||||
|
|
||||||
RUN test -s /etc/ssl/certs/ca-certificates.crt \
|
|
||||||
&& sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list.d/debian.sources \
|
|
||||||
&& grep -q '^URIs: https://deb.debian.org/' /etc/apt/sources.list.d/debian.sources \
|
|
||||||
&& ! grep -Eq '^[[:space:]]*URIs:[[:space:]]+http://' /etc/apt/sources.list.d/debian.sources \
|
|
||||||
&& apt-get -o Acquire::Retries=3 update \
|
|
||||||
&& apt-get -o Acquire::Retries=3 install -y --no-install-recommends ca-certificates curl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY --from=builder /tmp/admin-api /usr/local/bin/admin-api
|
COPY --from=builder /tmp/admin-api /usr/local/bin/admin-api
|
||||||
COPY --from=builder /tmp/crank-migrate /usr/local/bin/crank-migrate
|
COPY --from=builder /tmp/crank-migrate /usr/local/bin/crank-migrate
|
||||||
COPY apps/admin-api/docker-entrypoint.sh /usr/local/bin/crank-admin-entrypoint
|
COPY apps/admin-api/docker-entrypoint.sh /usr/local/bin/crank-admin-entrypoint
|
||||||
|
COPY scripts/docker-http-healthcheck.sh /usr/local/bin/crank-http-healthcheck
|
||||||
|
|
||||||
RUN chmod 0755 /usr/local/bin/crank-admin-entrypoint
|
RUN test -s /etc/ssl/certs/ca-certificates.crt \
|
||||||
|
&& chmod 0755 /usr/local/bin/crank-admin-entrypoint /usr/local/bin/crank-http-healthcheck
|
||||||
|
|
||||||
ENV CRANK_ADMIN_BIND=0.0.0.0:3001
|
ENV CRANK_ADMIN_BIND=0.0.0.0:3001
|
||||||
ENV CRANK_STORAGE_ROOT=/var/lib/crank/storage
|
ENV CRANK_STORAGE_ROOT=/var/lib/crank/storage
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use axum::{
|
|||||||
};
|
};
|
||||||
use crank_core::{CorrelationContext, RequestId, TraceContext};
|
use crank_core::{CorrelationContext, RequestId, TraceContext};
|
||||||
use crank_metrics::ExemplarTraceId;
|
use crank_metrics::ExemplarTraceId;
|
||||||
use crank_observability::{set_remote_trace_parent, with_request_correlation};
|
use crank_observability::with_request_correlation;
|
||||||
use tracing::{Instrument, info, info_span};
|
use tracing::{Instrument, info, info_span};
|
||||||
|
|
||||||
pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
|
pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
|
||||||
@@ -124,11 +124,7 @@ fn one_auxiliary_header_within_budget(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn set_canonical_parent(span: &tracing::Span, context: &TraceContext) {
|
fn set_canonical_parent(span: &tracing::Span, context: &TraceContext) {
|
||||||
let mut headers = axum::http::HeaderMap::new();
|
crank_trace::set_parent_from_trace_context(span, context);
|
||||||
if let Ok(value) = HeaderValue::from_str(context.traceparent()) {
|
|
||||||
headers.insert("traceparent", value);
|
|
||||||
set_remote_trace_parent(span, &headers);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -403,3 +403,14 @@ fn materialization_failure(stage: &'static str, error_code: &'static str, count:
|
|||||||
"external OpenAPI materialization failed"
|
"external OpenAPI materialization failed"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::external_fetch_error_code;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unexpected_status_has_a_stable_safe_error_code() {
|
||||||
|
let error = crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { status: 500 };
|
||||||
|
assert_eq!(external_fetch_error_code(&error), "unexpected_status");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -114,10 +114,14 @@ pub(super) fn build_test_app_with_external_references(
|
|||||||
allowed_url_prefixes: Vec<String>,
|
allowed_url_prefixes: Vec<String>,
|
||||||
) -> Router {
|
) -> Router {
|
||||||
build_app(AppState {
|
build_app(AppState {
|
||||||
service: test_service_with_external_references(
|
service: test_service_with_external_references_timeout(
|
||||||
registry,
|
registry,
|
||||||
storage_root,
|
storage_root,
|
||||||
allowed_url_prefixes,
|
allowed_url_prefixes,
|
||||||
|
// This HTTP fixture asserts classification of an immediate
|
||||||
|
// upstream status. Leave headroom for CI scheduling jitter so the
|
||||||
|
// unrelated chain-timeout branch cannot win first.
|
||||||
|
10_000,
|
||||||
),
|
),
|
||||||
api_rate_limiter: crank_runtime::RequestRateLimiter::new(
|
api_rate_limiter: crank_runtime::RequestRateLimiter::new(
|
||||||
crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(),
|
crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(),
|
||||||
@@ -174,6 +178,20 @@ pub(super) fn test_service_with_external_references(
|
|||||||
registry: PostgresRegistry,
|
registry: PostgresRegistry,
|
||||||
storage_root: std::path::PathBuf,
|
storage_root: std::path::PathBuf,
|
||||||
allowed_url_prefixes: Vec<String>,
|
allowed_url_prefixes: Vec<String>,
|
||||||
|
) -> AdminService {
|
||||||
|
test_service_with_external_references_timeout(
|
||||||
|
registry,
|
||||||
|
storage_root,
|
||||||
|
allowed_url_prefixes,
|
||||||
|
2_000,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_service_with_external_references_timeout(
|
||||||
|
registry: PostgresRegistry,
|
||||||
|
storage_root: std::path::PathBuf,
|
||||||
|
allowed_url_prefixes: Vec<String>,
|
||||||
|
fetch_timeout_ms: u64,
|
||||||
) -> AdminService {
|
) -> AdminService {
|
||||||
let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
|
let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
|
||||||
let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
|
let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
|
||||||
@@ -189,7 +207,7 @@ pub(super) fn test_service_with_external_references(
|
|||||||
max_depth: 8,
|
max_depth: 8,
|
||||||
max_documents: 32,
|
max_documents: 32,
|
||||||
max_fetch_bytes: 64 * 1024,
|
max_fetch_bytes: 64 * 1024,
|
||||||
fetch_timeout_ms: 2_000,
|
fetch_timeout_ms,
|
||||||
max_expanded_nodes: 10_000,
|
max_expanded_nodes: 10_000,
|
||||||
})
|
})
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|||||||
@@ -31,7 +31,11 @@ async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() {
|
|||||||
.build();
|
.build();
|
||||||
let tracer = provider.tracer("admin-openapi-source-test");
|
let tracer = provider.tracer("admin-openapi-source-test");
|
||||||
let subscriber = tracing_subscriber::registry()
|
let subscriber = tracing_subscriber::registry()
|
||||||
.with(tracing_subscriber::fmt::layer().with_writer(writer.clone()))
|
.with(
|
||||||
|
tracing_subscriber::fmt::layer()
|
||||||
|
.with_ansi(false)
|
||||||
|
.with_writer(writer.clone()),
|
||||||
|
)
|
||||||
.with(tracing_opentelemetry::layer().with_tracer(tracer));
|
.with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||||
let dispatch = tracing::Dispatch::new(subscriber);
|
let dispatch = tracing::Dispatch::new(subscriber);
|
||||||
tracing::dispatcher::set_global_default(dispatch)
|
tracing::dispatcher::set_global_default(dispatch)
|
||||||
@@ -118,10 +122,11 @@ paths:
|
|||||||
let logs = writer.output();
|
let logs = writer.output();
|
||||||
assert!(!logs.contains(CANARY));
|
assert!(!logs.contains(CANARY));
|
||||||
assert!(logs.contains(&trace_id));
|
assert!(logs.contains(&trace_id));
|
||||||
assert!(logs.contains("external OpenAPI materialization failed"));
|
let safe_materialization_failure = logs.lines().any(is_safe_materialization_failure);
|
||||||
assert!(logs.contains("stage=\"fetch\""));
|
assert!(
|
||||||
assert!(logs.contains("error_code=\"unexpected_status\""));
|
safe_materialization_failure,
|
||||||
assert!(logs.contains("count=0"));
|
"expected one bounded, sanitized materialization failure"
|
||||||
|
);
|
||||||
let spans = exported.lock().unwrap();
|
let spans = exported.lock().unwrap();
|
||||||
let rendered_spans = format!("{spans:?}");
|
let rendered_spans = format!("{spans:?}");
|
||||||
assert!(!rendered_spans.contains(CANARY));
|
assert!(!rendered_spans.contains(CANARY));
|
||||||
@@ -137,6 +142,34 @@ paths:
|
|||||||
provider.shutdown().unwrap();
|
provider.shutdown().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_safe_materialization_failure(line: &str) -> bool {
|
||||||
|
if !line.contains("external OpenAPI materialization failed") || !line.contains("count=0") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let immediate_status = contains_log_field(line, "stage", "fetch")
|
||||||
|
&& contains_log_field(line, "error_code", "unexpected_status");
|
||||||
|
let bounded_timeout = contains_log_field(line, "stage", "chain")
|
||||||
|
&& contains_log_field(line, "error_code", "timeout");
|
||||||
|
immediate_status || bounded_timeout
|
||||||
|
}
|
||||||
|
|
||||||
|
fn contains_log_field(line: &str, name: &str, value: &str) -> bool {
|
||||||
|
line.contains(&format!("{name}=\"{value}\"")) || line.contains(&format!("{name}={value}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn materialization_log_accepts_only_coherent_safe_outcomes() {
|
||||||
|
for line in [
|
||||||
|
r#"external OpenAPI materialization failed stage="fetch" error_code="unexpected_status" count=0"#,
|
||||||
|
"external OpenAPI materialization failed stage=chain error_code=timeout count=0",
|
||||||
|
] {
|
||||||
|
assert!(is_safe_materialization_failure(line));
|
||||||
|
}
|
||||||
|
assert!(!is_safe_materialization_failure(
|
||||||
|
r#"external OpenAPI materialization failed stage="fetch" error_code="timeout" count=0"#
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Default)]
|
#[derive(Clone, Default)]
|
||||||
struct SharedLogWriter {
|
struct SharedLogWriter {
|
||||||
buffer: Arc<Mutex<Vec<u8>>>,
|
buffer: Arc<Mutex<Vec<u8>>>,
|
||||||
@@ -154,17 +187,19 @@ impl<'a> MakeWriter<'a> for SharedLogWriter {
|
|||||||
fn make_writer(&'a self) -> Self::Writer {
|
fn make_writer(&'a self) -> Self::Writer {
|
||||||
SharedLogGuard {
|
SharedLogGuard {
|
||||||
buffer: Arc::clone(&self.buffer),
|
buffer: Arc::clone(&self.buffer),
|
||||||
|
pending: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct SharedLogGuard {
|
struct SharedLogGuard {
|
||||||
buffer: Arc<Mutex<Vec<u8>>>,
|
buffer: Arc<Mutex<Vec<u8>>>,
|
||||||
|
pending: Vec<u8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl io::Write for SharedLogGuard {
|
impl io::Write for SharedLogGuard {
|
||||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
self.pending.extend_from_slice(bytes);
|
||||||
Ok(bytes.len())
|
Ok(bytes.len())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -173,6 +208,24 @@ impl io::Write for SharedLogGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Drop for SharedLogGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.buffer.lock().unwrap().extend_from_slice(&self.pending);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn shared_log_writer_publishes_complete_records_only() {
|
||||||
|
let writer = SharedLogWriter::default();
|
||||||
|
let mut guard = writer.make_writer();
|
||||||
|
io::Write::write_all(&mut guard, b"message").unwrap();
|
||||||
|
assert!(writer.output().is_empty());
|
||||||
|
|
||||||
|
io::Write::write_all(&mut guard, b" stage=\"fetch\"\n").unwrap();
|
||||||
|
drop(guard);
|
||||||
|
assert_eq!(writer.output(), "message stage=\"fetch\"\n");
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
||||||
|
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ fn plan_is_deterministic_and_committed_contract_is_current() {
|
|||||||
);
|
);
|
||||||
assert_eq!(first.stdout, second.stdout);
|
assert_eq!(first.stdout, second.stdout);
|
||||||
let plan: serde_json::Value = serde_json::from_slice(&first.stdout).unwrap();
|
let plan: serde_json::Value = serde_json::from_slice(&first.stdout).unwrap();
|
||||||
assert_eq!(plan["sequence"].as_array().unwrap().len(), 12);
|
assert_eq!(plan["sequence"].as_array().unwrap().len(), 13);
|
||||||
|
|
||||||
let checked = command(&["plan", "--check"], None);
|
let checked = command(&["plan", "--check"], None);
|
||||||
assert!(
|
assert!(
|
||||||
@@ -82,7 +82,7 @@ async fn database_only_config_can_apply_and_preflight_a_fresh_schema() {
|
|||||||
);
|
);
|
||||||
let result: serde_json::Value = serde_json::from_slice(&preflight.stdout).unwrap();
|
let result: serde_json::Value = serde_json::from_slice(&preflight.stdout).unwrap();
|
||||||
assert_eq!(result["status"], "current");
|
assert_eq!(result["status"], "current");
|
||||||
assert_eq!(result["version"], 12);
|
assert_eq!(result["version"], 13);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
use admin_api::request_context::{TRACE_ID_HEADER, apply_request_context};
|
||||||
|
use axum::{Router, body::Body, http::Request, routing::get};
|
||||||
|
use opentelemetry::trace::TracerProvider as _;
|
||||||
|
use opentelemetry_sdk::trace::SdkTracerProvider;
|
||||||
|
use tower::ServiceExt;
|
||||||
|
use tracing_subscriber::layer::SubscriberExt;
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn preserves_remote_trace_id_with_an_active_tracer_and_no_global_propagator() {
|
||||||
|
let provider = SdkTracerProvider::builder().build();
|
||||||
|
let tracer = provider.tracer("admin-request-context-parent-test");
|
||||||
|
let subscriber =
|
||||||
|
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||||
|
let dispatch = tracing::Dispatch::new(subscriber);
|
||||||
|
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
|
||||||
|
let app = Router::new()
|
||||||
|
.route("/probe", get(|| async { "ok" }))
|
||||||
|
.layer(axum::middleware::from_fn(apply_request_context));
|
||||||
|
|
||||||
|
let response = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.uri("/probe")
|
||||||
|
.header(
|
||||||
|
"traceparent",
|
||||||
|
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
|
||||||
|
)
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()[TRACE_ID_HEADER],
|
||||||
|
"0af7651916cd43dd8448eb211c80319c"
|
||||||
|
);
|
||||||
|
provider.shutdown().unwrap();
|
||||||
|
}
|
||||||
@@ -55,17 +55,13 @@ FROM debian:bookworm-slim
|
|||||||
|
|
||||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||||
|
|
||||||
RUN test -s /etc/ssl/certs/ca-certificates.crt \
|
|
||||||
&& sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list.d/debian.sources \
|
|
||||||
&& grep -q '^URIs: https://deb.debian.org/' /etc/apt/sources.list.d/debian.sources \
|
|
||||||
&& ! grep -Eq '^[[:space:]]*URIs:[[:space:]]+http://' /etc/apt/sources.list.d/debian.sources \
|
|
||||||
&& apt-get -o Acquire::Retries=3 update \
|
|
||||||
&& apt-get -o Acquire::Retries=3 install -y --no-install-recommends ca-certificates curl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY --from=builder /tmp/mcp-server /usr/local/bin/mcp-server
|
COPY --from=builder /tmp/mcp-server /usr/local/bin/mcp-server
|
||||||
|
COPY scripts/docker-http-healthcheck.sh /usr/local/bin/crank-http-healthcheck
|
||||||
|
|
||||||
|
RUN test -s /etc/ssl/certs/ca-certificates.crt \
|
||||||
|
&& chmod 0755 /usr/local/bin/crank-http-healthcheck
|
||||||
|
|
||||||
ENV CRANK_MCP_BIND=0.0.0.0:3002
|
ENV CRANK_MCP_BIND=0.0.0.0:3002
|
||||||
|
|
||||||
|
|||||||
+5
-1
@@ -10,7 +10,11 @@ COPY crank-community.png ./crank-community.png
|
|||||||
|
|
||||||
RUN npm run build
|
RUN npm run build
|
||||||
|
|
||||||
FROM nginx:1.27-alpine
|
FROM nginx:1.30.4-alpine3.24-slim
|
||||||
|
|
||||||
|
RUN apk add --no-cache --upgrade \
|
||||||
|
'libcrypto3>=3.5.8-r0' \
|
||||||
|
'libssl3>=3.5.8-r0'
|
||||||
|
|
||||||
COPY apps/ui/nginx.conf /etc/nginx/conf.d/default.conf
|
COPY apps/ui/nginx.conf /etc/nginx/conf.d/default.conf
|
||||||
COPY --from=build /app/dist /usr/share/nginx/html
|
COPY --from=build /app/dist /usr/share/nginx/html
|
||||||
|
|||||||
@@ -72,6 +72,17 @@
|
|||||||
body:has(.drawer.open) .onboarding-trigger,
|
body:has(.drawer.open) .onboarding-trigger,
|
||||||
body:has(.drawer.open) .onboarding-panel { z-index: 149; }
|
body:has(.drawer.open) .onboarding-panel { z-index: 149; }
|
||||||
|
|
||||||
|
/* The wizard owns a fixed bottom action bar. Keep this optional helper above
|
||||||
|
it so the primary Continue action remains both visible and clickable. */
|
||||||
|
body.wizard-page .onboarding-trigger {
|
||||||
|
bottom: calc(96px + env(safe-area-inset-bottom, 0px));
|
||||||
|
}
|
||||||
|
|
||||||
|
body.wizard-page .onboarding-panel {
|
||||||
|
bottom: calc(150px + env(safe-area-inset-bottom, 0px));
|
||||||
|
max-height: min(700px, calc(100vh - 178px));
|
||||||
|
}
|
||||||
|
|
||||||
@media (max-width: 640px) {
|
@media (max-width: 640px) {
|
||||||
.onboarding-trigger { right: 14px; bottom: 14px; }
|
.onboarding-trigger { right: 14px; bottom: 14px; }
|
||||||
.onboarding-panel { right: 14px; bottom: 68px; }
|
.onboarding-panel { right: 14px; bottom: 68px; }
|
||||||
|
|||||||
@@ -17,6 +17,30 @@ POSTGRES_HOST="${CRANK_E2E_POSTGRES_HOST:-127.0.0.1}"
|
|||||||
USE_EXTERNAL_POSTGRES="${CRANK_E2E_USE_EXTERNAL_POSTGRES:-0}"
|
USE_EXTERNAL_POSTGRES="${CRANK_E2E_USE_EXTERNAL_POSTGRES:-0}"
|
||||||
ADMIN_EMAIL="${CRANK_E2E_ADMIN_EMAIL:-owner@crank.local}"
|
ADMIN_EMAIL="${CRANK_E2E_ADMIN_EMAIL:-owner@crank.local}"
|
||||||
ADMIN_PASSWORD="${CRANK_E2E_ADMIN_PASSWORD:-change-me-admin-password}"
|
ADMIN_PASSWORD="${CRANK_E2E_ADMIN_PASSWORD:-change-me-admin-password}"
|
||||||
|
STARTUP_TIMEOUT_SECONDS="${CRANK_E2E_STARTUP_TIMEOUT_SECONDS:-120}"
|
||||||
|
|
||||||
|
if [[ ! "$STARTUP_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
|
||||||
|
echo "CRANK_E2E_STARTUP_TIMEOUT_SECONDS must be a positive integer" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# These variables configure only this orchestration script. The application
|
||||||
|
# rejects unknown variables in the owned CRANK_* namespace, so never leak the
|
||||||
|
# E2E control plane into production process configuration.
|
||||||
|
unset \
|
||||||
|
CRANK_E2E_ADMIN_EMAIL \
|
||||||
|
CRANK_E2E_ADMIN_PASSWORD \
|
||||||
|
CRANK_E2E_ADMIN_PORT \
|
||||||
|
CRANK_E2E_MCP_PORT \
|
||||||
|
CRANK_E2E_POSTGRES_DB \
|
||||||
|
CRANK_E2E_POSTGRES_HOST \
|
||||||
|
CRANK_E2E_POSTGRES_PASSWORD \
|
||||||
|
CRANK_E2E_POSTGRES_PORT \
|
||||||
|
CRANK_E2E_POSTGRES_USER \
|
||||||
|
CRANK_E2E_STARTUP_TIMEOUT_SECONDS \
|
||||||
|
CRANK_E2E_STREAM_FIXTURE_PORT \
|
||||||
|
CRANK_E2E_UI_PORT \
|
||||||
|
CRANK_E2E_USE_EXTERNAL_POSTGRES
|
||||||
|
|
||||||
if [[ -f "$HOME/.cargo/env" ]]; then
|
if [[ -f "$HOME/.cargo/env" ]]; then
|
||||||
. "$HOME/.cargo/env"
|
. "$HOME/.cargo/env"
|
||||||
@@ -32,6 +56,7 @@ if [[ -z "$PYTHON_BIN" ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
mkdir -p "$LOG_DIR"
|
mkdir -p "$LOG_DIR"
|
||||||
|
find "$LOG_DIR" -maxdepth 1 -type f -name '*.log' -delete
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR/apps/ui"
|
cd "$ROOT_DIR/apps/ui"
|
||||||
@@ -77,10 +102,23 @@ trap 'exit 130' INT TERM
|
|||||||
|
|
||||||
cleanup
|
cleanup
|
||||||
|
|
||||||
|
show_startup_log() {
|
||||||
|
local log_file="$1"
|
||||||
|
if [[ -f "$log_file" ]]; then
|
||||||
|
echo "---- $log_file ----" >&2
|
||||||
|
tail -n 120 "$log_file" >&2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
wait_for_port() {
|
wait_for_port() {
|
||||||
local host="$1"
|
local label="$1"
|
||||||
local port="$2"
|
local host="$2"
|
||||||
until "$PYTHON_BIN" - "$host" "$port" <<'PY'
|
local port="$3"
|
||||||
|
local deadline=$((SECONDS + STARTUP_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
echo "Waiting for $label on $host:$port"
|
||||||
|
while true; do
|
||||||
|
if "$PYTHON_BIN" - "$host" "$port" <<'PY'
|
||||||
import socket, sys
|
import socket, sys
|
||||||
sock = socket.socket()
|
sock = socket.socket()
|
||||||
sock.settimeout(0.5)
|
sock.settimeout(0.5)
|
||||||
@@ -91,13 +129,75 @@ except OSError:
|
|||||||
finally:
|
finally:
|
||||||
sock.close()
|
sock.close()
|
||||||
PY
|
PY
|
||||||
do
|
then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "Timed out waiting for $label on $host:$port after ${STARTUP_TIMEOUT_SECONDS}s" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_http() {
|
||||||
|
local label="$1"
|
||||||
|
local url="$2"
|
||||||
|
local pid_file="$3"
|
||||||
|
local log_file="$4"
|
||||||
|
local deadline=$((SECONDS + STARTUP_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
echo "Waiting for $label at $url"
|
||||||
|
while true; do
|
||||||
|
if "$PYTHON_BIN" - "$url" <<'PY'
|
||||||
|
import sys, urllib.request
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(sys.argv[1], timeout=0.5) as response:
|
||||||
|
sys.exit(0 if 200 <= response.status < 400 else 1)
|
||||||
|
except Exception:
|
||||||
|
sys.exit(1)
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local pid=""
|
||||||
|
if [[ -f "$pid_file" ]]; then
|
||||||
|
pid="$(cat "$pid_file")"
|
||||||
|
fi
|
||||||
|
if [[ -z "$pid" ]] || ! kill -0 "$pid" >/dev/null 2>&1; then
|
||||||
|
echo "$label exited before becoming ready at $url" >&2
|
||||||
|
show_startup_log "$log_file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "Timed out waiting for $label at $url after ${STARTUP_TIMEOUT_SECONDS}s" >&2
|
||||||
|
show_startup_log "$log_file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_postgres_container() {
|
||||||
|
local deadline=$((SECONDS + STARTUP_TIMEOUT_SECONDS))
|
||||||
|
echo "Waiting for PostgreSQL container $POSTGRES_CONTAINER"
|
||||||
|
while ! docker exec "$POSTGRES_CONTAINER" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do
|
||||||
|
if ! docker inspect "$POSTGRES_CONTAINER" >/dev/null 2>&1; then
|
||||||
|
echo "PostgreSQL container $POSTGRES_CONTAINER exited before becoming ready" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "Timed out waiting for PostgreSQL container after ${STARTUP_TIMEOUT_SECONDS}s" >&2
|
||||||
|
docker logs "$POSTGRES_CONTAINER" >&2 || true
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
if [[ "$USE_EXTERNAL_POSTGRES" = "1" ]]; then
|
if [[ "$USE_EXTERNAL_POSTGRES" = "1" ]]; then
|
||||||
wait_for_port "$POSTGRES_HOST" "$POSTGRES_PORT"
|
wait_for_port "PostgreSQL" "$POSTGRES_HOST" "$POSTGRES_PORT" || exit 1
|
||||||
else
|
else
|
||||||
docker run -d --rm \
|
docker run -d --rm \
|
||||||
--name "$POSTGRES_CONTAINER" \
|
--name "$POSTGRES_CONTAINER" \
|
||||||
@@ -107,9 +207,7 @@ else
|
|||||||
-p "$POSTGRES_PORT:5432" \
|
-p "$POSTGRES_PORT:5432" \
|
||||||
postgres:16-alpine >/dev/null
|
postgres:16-alpine >/dev/null
|
||||||
|
|
||||||
until docker exec "$POSTGRES_CONTAINER" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do
|
wait_for_postgres_container || exit 1
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
export POSTGRES_HOST
|
export POSTGRES_HOST
|
||||||
@@ -141,40 +239,58 @@ chmod 700 "$CRANK_STORAGE_ROOT"
|
|||||||
) &
|
) &
|
||||||
echo $! > "$TMP_DIR/http-fixture.pid"
|
echo $! > "$TMP_DIR/http-fixture.pid"
|
||||||
|
|
||||||
until curl -fsS "http://127.0.0.1:$STREAM_FIXTURE_PORT/health" >/dev/null 2>&1; do
|
wait_for_http \
|
||||||
sleep 1
|
"Playwright HTTP fixture" \
|
||||||
done
|
"http://127.0.0.1:$STREAM_FIXTURE_PORT/health" \
|
||||||
|
"$TMP_DIR/http-fixture.pid" \
|
||||||
|
"$LOG_DIR/http-fixture.log" || exit 1
|
||||||
|
|
||||||
(
|
if ! (
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
cargo run -p admin-api --bin crank-migrate -- apply >"$LOG_DIR/migrate.log" 2>&1
|
cargo run -p admin-api --bin crank-migrate -- apply >"$LOG_DIR/migrate.log" 2>&1
|
||||||
)
|
); then
|
||||||
|
echo "Failed to apply E2E database migrations" >&2
|
||||||
|
show_startup_log "$LOG_DIR/migrate.log"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
BOOTSTRAP_JSON="$(
|
if ! (
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
cargo run -p admin-api --bin crank-migrate -- admin-auth bootstrap-create \
|
cargo run -p admin-api --bin crank-migrate -- admin-auth bootstrap-create \
|
||||||
--email "$ADMIN_EMAIL" \
|
--email "$ADMIN_EMAIL" \
|
||||||
--display-name "Crank E2E" \
|
--display-name "Crank E2E" \
|
||||||
>"$LOG_DIR/bootstrap-create.log" 2>&1
|
>"$LOG_DIR/bootstrap-create.log" 2>&1
|
||||||
tail -n 1 "$LOG_DIR/bootstrap-create.log"
|
); then
|
||||||
)"
|
echo "Failed to create the E2E bootstrap contract" >&2
|
||||||
BOOTSTRAP_TOKEN="$("$PYTHON_BIN" - <<'PY' "$BOOTSTRAP_JSON"
|
show_startup_log "$LOG_DIR/bootstrap-create.log"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
BOOTSTRAP_JSON="$(tail -n 1 "$LOG_DIR/bootstrap-create.log")"
|
||||||
|
if ! BOOTSTRAP_TOKEN="$("$PYTHON_BIN" - <<'PY' "$BOOTSTRAP_JSON"
|
||||||
import json, sys
|
import json, sys
|
||||||
print(json.loads(sys.argv[1])["bootstrap_token"])
|
print(json.loads(sys.argv[1])["bootstrap_token"])
|
||||||
PY
|
PY
|
||||||
)"
|
)"; then
|
||||||
|
echo "Failed to parse the E2E bootstrap token" >&2
|
||||||
|
show_startup_log "$LOG_DIR/bootstrap-create.log"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
printf '%s' "$BOOTSTRAP_TOKEN" >"$TMP_DIR/bootstrap-token.txt"
|
printf '%s' "$BOOTSTRAP_TOKEN" >"$TMP_DIR/bootstrap-token.txt"
|
||||||
printf '%s' "$ADMIN_PASSWORD" >"$TMP_DIR/admin-password.txt"
|
printf '%s' "$ADMIN_PASSWORD" >"$TMP_DIR/admin-password.txt"
|
||||||
printf '%s' "$CRANK_PASSWORD_PEPPER" >"$TMP_DIR/password-pepper.txt"
|
printf '%s' "$CRANK_PASSWORD_PEPPER" >"$TMP_DIR/password-pepper.txt"
|
||||||
|
|
||||||
(
|
if ! (
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
cargo run -p admin-api --bin crank-migrate -- admin-auth bootstrap-complete \
|
cargo run -p admin-api --bin crank-migrate -- admin-auth bootstrap-complete \
|
||||||
--token-file "$TMP_DIR/bootstrap-token.txt" \
|
--token-file "$TMP_DIR/bootstrap-token.txt" \
|
||||||
--password-file "$TMP_DIR/admin-password.txt" \
|
--password-file "$TMP_DIR/admin-password.txt" \
|
||||||
--password-pepper-file "$TMP_DIR/password-pepper.txt" \
|
--password-pepper-file "$TMP_DIR/password-pepper.txt" \
|
||||||
>"$LOG_DIR/bootstrap-complete.log" 2>&1
|
>"$LOG_DIR/bootstrap-complete.log" 2>&1
|
||||||
)
|
); then
|
||||||
|
echo "Failed to complete the E2E admin bootstrap" >&2
|
||||||
|
show_startup_log "$LOG_DIR/bootstrap-complete.log"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
@@ -183,9 +299,11 @@ printf '%s' "$CRANK_PASSWORD_PEPPER" >"$TMP_DIR/password-pepper.txt"
|
|||||||
) &
|
) &
|
||||||
echo $! > "$TMP_DIR/admin-api.pid"
|
echo $! > "$TMP_DIR/admin-api.pid"
|
||||||
|
|
||||||
until curl -fsS "http://127.0.0.1:$ADMIN_PORT/health" >/dev/null 2>&1; do
|
wait_for_http \
|
||||||
sleep 1
|
"admin-api" \
|
||||||
done
|
"http://127.0.0.1:$ADMIN_PORT/health" \
|
||||||
|
"$TMP_DIR/admin-api.pid" \
|
||||||
|
"$LOG_DIR/admin-api.log" || exit 1
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
@@ -196,9 +314,11 @@ done
|
|||||||
) &
|
) &
|
||||||
echo $! > "$TMP_DIR/mcp-server.pid"
|
echo $! > "$TMP_DIR/mcp-server.pid"
|
||||||
|
|
||||||
until curl -fsS "http://127.0.0.1:$MCP_PORT/health" >/dev/null 2>&1; do
|
wait_for_http \
|
||||||
sleep 1
|
"mcp-server" \
|
||||||
done
|
"http://127.0.0.1:$MCP_PORT/health" \
|
||||||
|
"$TMP_DIR/mcp-server.pid" \
|
||||||
|
"$LOG_DIR/mcp-server.log" || exit 1
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR/apps/ui"
|
cd "$ROOT_DIR/apps/ui"
|
||||||
@@ -206,9 +326,11 @@ done
|
|||||||
) &
|
) &
|
||||||
echo $! > "$TMP_DIR/ui-server.pid"
|
echo $! > "$TMP_DIR/ui-server.pid"
|
||||||
|
|
||||||
until curl -fsS "http://127.0.0.1:$UI_PORT/login" >/dev/null 2>&1; do
|
wait_for_http \
|
||||||
sleep 1
|
"Playwright UI server" \
|
||||||
done
|
"http://127.0.0.1:$UI_PORT/login" \
|
||||||
|
"$TMP_DIR/ui-server.pid" \
|
||||||
|
"$LOG_DIR/ui-server.log" || exit 1
|
||||||
|
|
||||||
echo "Crank UI e2e stack is ready on http://127.0.0.1:$UI_PORT"
|
echo "Crank UI e2e stack is ready on http://127.0.0.1:$UI_PORT"
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,25 @@
|
|||||||
const { test, expect } = require('@playwright/test');
|
const { test, expect } = require('@playwright/test');
|
||||||
const { getCurrentWorkspace, login, localized } = require('./helpers');
|
const { getCurrentWorkspace, login, localized } = require('./helpers');
|
||||||
|
|
||||||
|
async function expectOnboardingOutsideWizardNavigation(page) {
|
||||||
|
await expect(page.getByTestId('onboarding-trigger')).toBeVisible();
|
||||||
|
|
||||||
|
const overlapsContinue = await page.locator('#btn-continue').evaluate((continueButton) => {
|
||||||
|
const trigger = document.querySelector('[data-testid="onboarding-trigger"]');
|
||||||
|
if (!trigger) return false;
|
||||||
|
const continueRect = continueButton.getBoundingClientRect();
|
||||||
|
const triggerRect = trigger.getBoundingClientRect();
|
||||||
|
return !(
|
||||||
|
triggerRect.right <= continueRect.left
|
||||||
|
|| triggerRect.left >= continueRect.right
|
||||||
|
|| triggerRect.bottom <= continueRect.top
|
||||||
|
|| triggerRect.top >= continueRect.bottom
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(overlapsContinue).toBe(false);
|
||||||
|
}
|
||||||
|
|
||||||
test('mobile wizard progress connector crosses the indicator centers', async ({ page }) => {
|
test('mobile wizard progress connector crosses the indicator centers', async ({ page }) => {
|
||||||
await page.setViewportSize({ width: 720, height: 900 });
|
await page.setViewportSize({ width: 720, height: 900 });
|
||||||
await login(page);
|
await login(page);
|
||||||
@@ -24,6 +43,19 @@ test('mobile wizard progress connector crosses the indicator centers', async ({
|
|||||||
expect(geometry.topDelta).toBeLessThan(1);
|
expect(geometry.topDelta).toBeLessThan(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('onboarding helper does not cover wizard navigation', async ({ page }) => {
|
||||||
|
await login(page);
|
||||||
|
await page.goto('/wizard/');
|
||||||
|
await expectOnboardingOutsideWizardNavigation(page);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('onboarding helper does not cover wizard navigation on mobile', async ({ page }) => {
|
||||||
|
await page.setViewportSize({ width: 390, height: 844 });
|
||||||
|
await login(page);
|
||||||
|
await page.goto('/wizard/');
|
||||||
|
await expectOnboardingOutsideWizardNavigation(page);
|
||||||
|
});
|
||||||
|
|
||||||
test('wizard loads and protocol selection updates flow', async ({ page }) => {
|
test('wizard loads and protocol selection updates flow', async ({ page }) => {
|
||||||
await login(page);
|
await login(page);
|
||||||
await page.goto('/wizard/');
|
await page.goto('/wizard/');
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
use std::{
|
use std::{
|
||||||
fmt,
|
fmt,
|
||||||
os::fd::{AsRawFd, FromRawFd, OwnedFd},
|
os::fd::{AsRawFd, FromRawFd, OwnedFd},
|
||||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::temp_scan::{list_names_after, valid_temp_name};
|
use crate::temp_scan::valid_temp_name;
|
||||||
use crate::{
|
use crate::{
|
||||||
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
|
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
|
||||||
ReconciliationScan, ReconciliationScanStop,
|
ReconciliationScan, ReconciliationScanStop,
|
||||||
@@ -15,6 +14,10 @@ use crate::{
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod stale_temp;
|
||||||
|
|
||||||
|
pub use stale_temp::{StaleTemp, TempScan, TempScanCursor};
|
||||||
|
|
||||||
/// Opaque bounded-scan continuation, valid only for an unchanged namespace.
|
/// Opaque bounded-scan continuation, valid only for an unchanged namespace.
|
||||||
/// Discard it after any put, quarantine, delete, or external mutation.
|
/// Discard it after any put, quarantine, delete, or external mutation.
|
||||||
pub struct ReconciliationCursor {
|
pub struct ReconciliationCursor {
|
||||||
@@ -80,49 +83,6 @@ pub struct ReconciliationCandidate {
|
|||||||
pub(crate) modified_nanoseconds: i64,
|
pub(crate) modified_nanoseconds: i64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Opaque single-use evidence for a stale temporary inode under the pinned root.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct StaleTemp {
|
|
||||||
name: String,
|
|
||||||
shard: String,
|
|
||||||
root_dev: u64,
|
|
||||||
root_ino: u64,
|
|
||||||
dev: u64,
|
|
||||||
ino: u64,
|
|
||||||
modified_seconds: i64,
|
|
||||||
modified_nanoseconds: i64,
|
|
||||||
grace: Duration,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Opaque, inode-bound progress marker for the bounded stale-temp sweeper.
|
|
||||||
///
|
|
||||||
/// It deliberately advances by shard instead of retaining a directory offset:
|
|
||||||
/// directory offsets are invalidated by a concurrent writer, while round-robin
|
|
||||||
/// shard progress prevents a busy low-numbered shard from starving all others.
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
pub struct TempScanCursor {
|
|
||||||
root_dev: u64,
|
|
||||||
root_ino: u64,
|
|
||||||
next_shard: u8,
|
|
||||||
shard_continuation: Option<TempShardContinuation>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
struct TempShardContinuation {
|
|
||||||
shard: u8,
|
|
||||||
dev: u64,
|
|
||||||
ino: u64,
|
|
||||||
cookie: i64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
|
||||||
pub struct TempScan {
|
|
||||||
pub scanned: usize,
|
|
||||||
pub omitted: usize,
|
|
||||||
/// True when this page completed a full round-robin traversal.
|
|
||||||
pub complete: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ArtifactStore {
|
impl ArtifactStore {
|
||||||
pub(crate) fn scan_reconciliation_bounded(
|
pub(crate) fn scan_reconciliation_bounded(
|
||||||
&self,
|
&self,
|
||||||
@@ -340,229 +300,6 @@ impl ArtifactStore {
|
|||||||
Err(_) => Ok(ReconciliationMutation::Retryable),
|
Err(_) => Ok(ReconciliationMutation::Retryable),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// Scans at most `scan_budget` entries and returns `result_limit` stale capabilities.
|
|
||||||
pub fn scan_stale_temps(
|
|
||||||
&self,
|
|
||||||
grace: Duration,
|
|
||||||
scan_budget: usize,
|
|
||||||
result_limit: usize,
|
|
||||||
) -> Result<(TempScan, Vec<StaleTemp>), ArtifactError> {
|
|
||||||
self.scan_stale_temps_after(grace, None, scan_budget, result_limit)
|
|
||||||
.map(|(report, candidates, _)| (report, candidates))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Resumes stale-temp cleanup from an opaque round-robin marker. A
|
|
||||||
/// continuation is valid only for this exact pinned root.
|
|
||||||
pub fn scan_stale_temps_after(
|
|
||||||
&self,
|
|
||||||
grace: Duration,
|
|
||||||
continuation: Option<TempScanCursor>,
|
|
||||||
scan_budget: usize,
|
|
||||||
result_limit: usize,
|
|
||||||
) -> Result<(TempScan, Vec<StaleTemp>, TempScanCursor), ArtifactError> {
|
|
||||||
let root = self.root()?;
|
|
||||||
let _lock = RootLock::shared(root)?;
|
|
||||||
ensure_root_unchanged(root)?;
|
|
||||||
let sha = match open_existing_dir(root.fd.as_raw_fd(), b"sha256") {
|
|
||||||
Ok(fd) => fd,
|
|
||||||
Err(ArtifactError::NotFound) => {
|
|
||||||
return Ok((
|
|
||||||
TempScan {
|
|
||||||
complete: true,
|
|
||||||
..TempScan::default()
|
|
||||||
},
|
|
||||||
Vec::new(),
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard: 0,
|
|
||||||
shard_continuation: None,
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Err(error) => return Err(error),
|
|
||||||
};
|
|
||||||
let (start_shard, mut shard_continuation) = match continuation {
|
|
||||||
Some(cursor) if cursor.root_dev == root.dev && cursor.root_ino == root.ino => {
|
|
||||||
(cursor.next_shard, cursor.shard_continuation)
|
|
||||||
}
|
|
||||||
Some(_) => return Err(ArtifactError::UnsafeRoot),
|
|
||||||
None => (0, None),
|
|
||||||
};
|
|
||||||
let mut report = TempScan::default();
|
|
||||||
let mut remaining = scan_budget;
|
|
||||||
let mut result = Vec::new();
|
|
||||||
if remaining == 0 {
|
|
||||||
return Ok((
|
|
||||||
report,
|
|
||||||
result,
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard: start_shard,
|
|
||||||
shard_continuation,
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let mut next_shard = start_shard;
|
|
||||||
for offset in 0_u16..=255 {
|
|
||||||
if remaining == 0 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
let shard_number = start_shard.wrapping_add(offset as u8);
|
|
||||||
next_shard = shard_number.wrapping_add(1);
|
|
||||||
let shard = format!("{shard_number:02x}");
|
|
||||||
let shard_fd = match open_existing_dir(sha.as_raw_fd(), shard.as_bytes()) {
|
|
||||||
Ok(fd) => fd,
|
|
||||||
Err(ArtifactError::NotFound) => {
|
|
||||||
shard_continuation = None;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Err(error) => return Err(error),
|
|
||||||
};
|
|
||||||
let stat = stat_fd(shard_fd.as_raw_fd())?;
|
|
||||||
let resume = shard_continuation.take().and_then(|continuation| {
|
|
||||||
if continuation.shard == shard_number
|
|
||||||
&& continuation.dev == stat.st_dev
|
|
||||||
&& continuation.ino == stat.st_ino
|
|
||||||
{
|
|
||||||
Some(continuation.cookie)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let page = list_names_after(shard_fd.as_raw_fd(), &mut remaining, &mut report, resume)?;
|
|
||||||
for entry in page.names {
|
|
||||||
let name = entry.name;
|
|
||||||
if !valid_temp_name(&name) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
#[cfg(debug_assertions)]
|
|
||||||
let _ = crate::test_support::checkpoint("housekeeping_before_stat");
|
|
||||||
let stat = match nofollow_stat(shard_fd.as_raw_fd(), &name) {
|
|
||||||
Ok(stat) => stat,
|
|
||||||
Err(ArtifactError::NotFound) => continue,
|
|
||||||
Err(error) => return Err(error),
|
|
||||||
};
|
|
||||||
if (stat.st_mode & libc::S_IFMT) != libc::S_IFREG || stat.st_nlink != 1 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let modified = UNIX_EPOCH
|
|
||||||
.checked_add(Duration::new(
|
|
||||||
stat.st_mtime.max(0) as u64,
|
|
||||||
stat.st_mtime_nsec.max(0) as u32,
|
|
||||||
))
|
|
||||||
.unwrap_or(SystemTime::UNIX_EPOCH);
|
|
||||||
if SystemTime::now()
|
|
||||||
.duration_since(modified)
|
|
||||||
.is_ok_and(|age| age >= grace)
|
|
||||||
{
|
|
||||||
if result.len() == result_limit {
|
|
||||||
report.omitted += 1;
|
|
||||||
if result_limit > 0 {
|
|
||||||
return Ok((
|
|
||||||
report,
|
|
||||||
result,
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard: shard_number,
|
|
||||||
shard_continuation: Some(TempShardContinuation {
|
|
||||||
shard: shard_number,
|
|
||||||
dev: stat.st_dev,
|
|
||||||
ino: stat.st_ino,
|
|
||||||
cookie: entry.cookie_before,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
result.push(StaleTemp {
|
|
||||||
name,
|
|
||||||
shard: shard.clone(),
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
dev: stat.st_dev,
|
|
||||||
ino: stat.st_ino,
|
|
||||||
modified_seconds: stat.st_mtime,
|
|
||||||
modified_nanoseconds: stat.st_mtime_nsec,
|
|
||||||
grace,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(cookie) = page.continuation {
|
|
||||||
return Ok((
|
|
||||||
report,
|
|
||||||
result,
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard: shard_number,
|
|
||||||
shard_continuation: Some(TempShardContinuation {
|
|
||||||
shard: shard_number,
|
|
||||||
dev: stat.st_dev,
|
|
||||||
ino: stat.st_ino,
|
|
||||||
cookie,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
report.complete = true;
|
|
||||||
Ok((
|
|
||||||
report,
|
|
||||||
result,
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard,
|
|
||||||
shard_continuation: None,
|
|
||||||
},
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Deletes a revalidated stale inode under an exclusive lock, then fsyncs.
|
|
||||||
pub fn delete_stale_temp(&self, candidate: StaleTemp) -> Result<(), ArtifactError> {
|
|
||||||
let root = self.root()?;
|
|
||||||
let _lock = RootLock::exclusive(root)?;
|
|
||||||
ensure_root_unchanged(root)?;
|
|
||||||
if candidate.root_dev != root.dev || candidate.root_ino != root.ino {
|
|
||||||
return Err(ArtifactError::UnsafeRoot);
|
|
||||||
}
|
|
||||||
if !valid_temp_name(&candidate.name)
|
|
||||||
|| candidate.shard.len() != 2
|
|
||||||
|| !candidate.shard.bytes().all(is_lower_hex)
|
|
||||||
{
|
|
||||||
return Err(ArtifactError::UnsafeRoot);
|
|
||||||
}
|
|
||||||
let sha = open_existing_dir(root.fd.as_raw_fd(), b"sha256")?;
|
|
||||||
let shard = open_existing_dir(sha.as_raw_fd(), candidate.shard.as_bytes())?;
|
|
||||||
let stat = nofollow_stat(shard.as_raw_fd(), &candidate.name)?;
|
|
||||||
if stat.st_dev != candidate.dev
|
|
||||||
|| stat.st_ino != candidate.ino
|
|
||||||
|| stat.st_mtime != candidate.modified_seconds
|
|
||||||
|| stat.st_mtime_nsec != candidate.modified_nanoseconds
|
|
||||||
|| (stat.st_mode & libc::S_IFMT) != libc::S_IFREG
|
|
||||||
|| stat.st_nlink != 1
|
|
||||||
{
|
|
||||||
return Err(ArtifactError::UnsafeRoot);
|
|
||||||
}
|
|
||||||
let modified = UNIX_EPOCH
|
|
||||||
.checked_add(Duration::new(
|
|
||||||
stat.st_mtime.max(0) as u64,
|
|
||||||
stat.st_mtime_nsec.max(0) as u32,
|
|
||||||
))
|
|
||||||
.unwrap_or(SystemTime::UNIX_EPOCH);
|
|
||||||
if !SystemTime::now()
|
|
||||||
.duration_since(modified)
|
|
||||||
.is_ok_and(|age| age >= candidate.grace)
|
|
||||||
{
|
|
||||||
return Err(ArtifactError::UnsafeRoot);
|
|
||||||
}
|
|
||||||
unlinkat(shard.as_raw_fd(), candidate.name.as_bytes())?;
|
|
||||||
fsync_fd(shard.as_raw_fd())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reconciliation_cursor(
|
fn reconciliation_cursor(
|
||||||
|
|||||||
@@ -0,0 +1,280 @@
|
|||||||
|
use std::{
|
||||||
|
os::fd::AsRawFd,
|
||||||
|
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::{is_lower_hex, nofollow_stat};
|
||||||
|
use crate::{
|
||||||
|
ArtifactError, ArtifactStore,
|
||||||
|
store::{RootLock, ensure_root_unchanged, fsync_fd, open_existing_dir, stat_fd, unlinkat},
|
||||||
|
temp_scan::{list_names_after, valid_temp_name},
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Opaque single-use evidence for a stale temporary inode under the pinned root.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct StaleTemp {
|
||||||
|
name: String,
|
||||||
|
shard: String,
|
||||||
|
root_dev: u64,
|
||||||
|
root_ino: u64,
|
||||||
|
dev: u64,
|
||||||
|
ino: u64,
|
||||||
|
modified_seconds: i64,
|
||||||
|
modified_nanoseconds: i64,
|
||||||
|
grace: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opaque, inode-bound progress marker for the bounded stale-temp sweeper.
|
||||||
|
///
|
||||||
|
/// It deliberately advances by shard instead of retaining a directory offset:
|
||||||
|
/// directory offsets are invalidated by a concurrent writer, while round-robin
|
||||||
|
/// shard progress prevents a busy low-numbered shard from starving all others.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct TempScanCursor {
|
||||||
|
root_dev: u64,
|
||||||
|
root_ino: u64,
|
||||||
|
next_shard: u8,
|
||||||
|
shard_continuation: Option<TempShardContinuation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct TempShardContinuation {
|
||||||
|
shard: u8,
|
||||||
|
dev: u64,
|
||||||
|
ino: u64,
|
||||||
|
cookie: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||||
|
pub struct TempScan {
|
||||||
|
pub scanned: usize,
|
||||||
|
pub omitted: usize,
|
||||||
|
/// True when this page completed a full round-robin traversal.
|
||||||
|
pub complete: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ArtifactStore {
|
||||||
|
/// Scans at most `scan_budget` entries and returns `result_limit` stale capabilities.
|
||||||
|
pub fn scan_stale_temps(
|
||||||
|
&self,
|
||||||
|
grace: Duration,
|
||||||
|
scan_budget: usize,
|
||||||
|
result_limit: usize,
|
||||||
|
) -> Result<(TempScan, Vec<StaleTemp>), ArtifactError> {
|
||||||
|
self.scan_stale_temps_after(grace, None, scan_budget, result_limit)
|
||||||
|
.map(|(report, candidates, _)| (report, candidates))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resumes stale-temp cleanup from an opaque round-robin marker. A
|
||||||
|
/// continuation is valid only for this exact pinned root.
|
||||||
|
pub fn scan_stale_temps_after(
|
||||||
|
&self,
|
||||||
|
grace: Duration,
|
||||||
|
continuation: Option<TempScanCursor>,
|
||||||
|
scan_budget: usize,
|
||||||
|
result_limit: usize,
|
||||||
|
) -> Result<(TempScan, Vec<StaleTemp>, TempScanCursor), ArtifactError> {
|
||||||
|
let root = self.root()?;
|
||||||
|
let _lock = RootLock::shared(root)?;
|
||||||
|
ensure_root_unchanged(root)?;
|
||||||
|
let sha = match open_existing_dir(root.fd.as_raw_fd(), b"sha256") {
|
||||||
|
Ok(fd) => fd,
|
||||||
|
Err(ArtifactError::NotFound) => {
|
||||||
|
return Ok((
|
||||||
|
TempScan {
|
||||||
|
complete: true,
|
||||||
|
..TempScan::default()
|
||||||
|
},
|
||||||
|
Vec::new(),
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard: 0,
|
||||||
|
shard_continuation: None,
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
};
|
||||||
|
let (start_shard, mut shard_continuation) = match continuation {
|
||||||
|
Some(cursor) if cursor.root_dev == root.dev && cursor.root_ino == root.ino => {
|
||||||
|
(cursor.next_shard, cursor.shard_continuation)
|
||||||
|
}
|
||||||
|
Some(_) => return Err(ArtifactError::UnsafeRoot),
|
||||||
|
None => (0, None),
|
||||||
|
};
|
||||||
|
let mut report = TempScan::default();
|
||||||
|
let mut remaining = scan_budget;
|
||||||
|
let mut result = Vec::new();
|
||||||
|
if remaining == 0 {
|
||||||
|
return Ok((
|
||||||
|
report,
|
||||||
|
result,
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard: start_shard,
|
||||||
|
shard_continuation,
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut next_shard = start_shard;
|
||||||
|
for offset in 0_u16..=255 {
|
||||||
|
if remaining == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let shard_number = start_shard.wrapping_add(offset as u8);
|
||||||
|
next_shard = shard_number.wrapping_add(1);
|
||||||
|
let shard = format!("{shard_number:02x}");
|
||||||
|
let shard_fd = match open_existing_dir(sha.as_raw_fd(), shard.as_bytes()) {
|
||||||
|
Ok(fd) => fd,
|
||||||
|
Err(ArtifactError::NotFound) => {
|
||||||
|
shard_continuation = None;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
};
|
||||||
|
let stat = stat_fd(shard_fd.as_raw_fd())?;
|
||||||
|
let resume = shard_continuation.take().and_then(|continuation| {
|
||||||
|
if continuation.shard == shard_number
|
||||||
|
&& continuation.dev == stat.st_dev
|
||||||
|
&& continuation.ino == stat.st_ino
|
||||||
|
{
|
||||||
|
Some(continuation.cookie)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let page = list_names_after(shard_fd.as_raw_fd(), &mut remaining, &mut report, resume)?;
|
||||||
|
for entry in page.names {
|
||||||
|
let name = entry.name;
|
||||||
|
if !valid_temp_name(&name) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
let _ = crate::test_support::checkpoint("housekeeping_before_stat");
|
||||||
|
let stat = match nofollow_stat(shard_fd.as_raw_fd(), &name) {
|
||||||
|
Ok(stat) => stat,
|
||||||
|
Err(ArtifactError::NotFound) => continue,
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
};
|
||||||
|
if (stat.st_mode & libc::S_IFMT) != libc::S_IFREG || stat.st_nlink != 1 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let modified = UNIX_EPOCH
|
||||||
|
.checked_add(Duration::new(
|
||||||
|
stat.st_mtime.max(0) as u64,
|
||||||
|
stat.st_mtime_nsec.max(0) as u32,
|
||||||
|
))
|
||||||
|
.unwrap_or(SystemTime::UNIX_EPOCH);
|
||||||
|
if SystemTime::now()
|
||||||
|
.duration_since(modified)
|
||||||
|
.is_ok_and(|age| age >= grace)
|
||||||
|
{
|
||||||
|
if result.len() == result_limit {
|
||||||
|
report.omitted += 1;
|
||||||
|
if result_limit > 0 {
|
||||||
|
return Ok((
|
||||||
|
report,
|
||||||
|
result,
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard: shard_number,
|
||||||
|
shard_continuation: Some(TempShardContinuation {
|
||||||
|
shard: shard_number,
|
||||||
|
dev: stat.st_dev,
|
||||||
|
ino: stat.st_ino,
|
||||||
|
cookie: entry.cookie_before,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
result.push(StaleTemp {
|
||||||
|
name,
|
||||||
|
shard: shard.clone(),
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
dev: stat.st_dev,
|
||||||
|
ino: stat.st_ino,
|
||||||
|
modified_seconds: stat.st_mtime,
|
||||||
|
modified_nanoseconds: stat.st_mtime_nsec,
|
||||||
|
grace,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(cookie) = page.continuation {
|
||||||
|
return Ok((
|
||||||
|
report,
|
||||||
|
result,
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard: shard_number,
|
||||||
|
shard_continuation: Some(TempShardContinuation {
|
||||||
|
shard: shard_number,
|
||||||
|
dev: stat.st_dev,
|
||||||
|
ino: stat.st_ino,
|
||||||
|
cookie,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.complete = true;
|
||||||
|
Ok((
|
||||||
|
report,
|
||||||
|
result,
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard,
|
||||||
|
shard_continuation: None,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deletes a revalidated stale inode under an exclusive lock, then fsyncs.
|
||||||
|
pub fn delete_stale_temp(&self, candidate: StaleTemp) -> Result<(), ArtifactError> {
|
||||||
|
let root = self.root()?;
|
||||||
|
let _lock = RootLock::exclusive(root)?;
|
||||||
|
ensure_root_unchanged(root)?;
|
||||||
|
if candidate.root_dev != root.dev || candidate.root_ino != root.ino {
|
||||||
|
return Err(ArtifactError::UnsafeRoot);
|
||||||
|
}
|
||||||
|
if !valid_temp_name(&candidate.name)
|
||||||
|
|| candidate.shard.len() != 2
|
||||||
|
|| !candidate.shard.bytes().all(is_lower_hex)
|
||||||
|
{
|
||||||
|
return Err(ArtifactError::UnsafeRoot);
|
||||||
|
}
|
||||||
|
let sha = open_existing_dir(root.fd.as_raw_fd(), b"sha256")?;
|
||||||
|
let shard = open_existing_dir(sha.as_raw_fd(), candidate.shard.as_bytes())?;
|
||||||
|
let stat = nofollow_stat(shard.as_raw_fd(), &candidate.name)?;
|
||||||
|
if stat.st_dev != candidate.dev
|
||||||
|
|| stat.st_ino != candidate.ino
|
||||||
|
|| stat.st_mtime != candidate.modified_seconds
|
||||||
|
|| stat.st_mtime_nsec != candidate.modified_nanoseconds
|
||||||
|
|| (stat.st_mode & libc::S_IFMT) != libc::S_IFREG
|
||||||
|
|| stat.st_nlink != 1
|
||||||
|
{
|
||||||
|
return Err(ArtifactError::UnsafeRoot);
|
||||||
|
}
|
||||||
|
let modified = UNIX_EPOCH
|
||||||
|
.checked_add(Duration::new(
|
||||||
|
stat.st_mtime.max(0) as u64,
|
||||||
|
stat.st_mtime_nsec.max(0) as u32,
|
||||||
|
))
|
||||||
|
.unwrap_or(SystemTime::UNIX_EPOCH);
|
||||||
|
if !SystemTime::now()
|
||||||
|
.duration_since(modified)
|
||||||
|
.is_ok_and(|age| age >= candidate.grace)
|
||||||
|
{
|
||||||
|
return Err(ArtifactError::UnsafeRoot);
|
||||||
|
}
|
||||||
|
unlinkat(shard.as_raw_fd(), candidate.name.as_bytes())?;
|
||||||
|
fsync_fd(shard.as_raw_fd())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
use std::{
|
use std::{
|
||||||
fs,
|
fs,
|
||||||
io::{self, Read},
|
|
||||||
os::unix::fs::PermissionsExt,
|
os::unix::fs::PermissionsExt,
|
||||||
path::PathBuf,
|
path::PathBuf,
|
||||||
process::{Child, Command, ExitStatus},
|
process::{Child, Command, ExitStatus},
|
||||||
@@ -13,7 +12,7 @@ use std::{
|
|||||||
use crank_artifacts::test_support::{
|
use crank_artifacts::test_support::{
|
||||||
FaultAction, clear_checkpoint, set_checkpoint, wait_until_held,
|
FaultAction, clear_checkpoint, set_checkpoint, wait_until_held,
|
||||||
};
|
};
|
||||||
use crank_artifacts::{ArtifactError, ArtifactRef, ArtifactStore, MAX_ARTIFACT_BYTES, StaleTemp};
|
use crank_artifacts::{ArtifactError, ArtifactRef, ArtifactStore, MAX_ARTIFACT_BYTES};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
#[cfg(debug_assertions)]
|
#[cfg(debug_assertions)]
|
||||||
use std::sync::{Mutex, OnceLock};
|
use std::sync::{Mutex, OnceLock};
|
||||||
@@ -786,318 +785,3 @@ fn legacy_oversize_is_rejected() {
|
|||||||
ArtifactError::Integrity
|
ArtifactError::Integrity
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fifo_entries_fail_without_blocking_reads() {
|
|
||||||
let root = TestRoot::new("fifo");
|
|
||||||
let expected =
|
|
||||||
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"fifo bytes"))).unwrap();
|
|
||||||
let sha = root.0.join("sha256");
|
|
||||||
let shard = sha.join(&expected.digest_hex()[..2]);
|
|
||||||
fs::create_dir(&sha).unwrap();
|
|
||||||
fs::create_dir(&shard).unwrap();
|
|
||||||
fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap();
|
|
||||||
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
|
|
||||||
let blob_fifo = shard.join(expected.digest_hex());
|
|
||||||
let legacy = root.0.join("legacy");
|
|
||||||
fs::create_dir(&legacy).unwrap();
|
|
||||||
fs::set_permissions(&legacy, fs::Permissions::from_mode(0o700)).unwrap();
|
|
||||||
let legacy_fifo = legacy.join("source");
|
|
||||||
for path in [&blob_fifo, &legacy_fifo] {
|
|
||||||
let path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap();
|
|
||||||
assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o400) }, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
let executable = std::env::current_exe().unwrap();
|
|
||||||
for kind in ["blob", "legacy"] {
|
|
||||||
let child = Command::new(&executable)
|
|
||||||
.args(["--exact", "fifo_read_child", "--nocapture"])
|
|
||||||
.env("CRANK_ARTIFACTS_FIFO_ROOT", &root.0)
|
|
||||||
.env("CRANK_ARTIFACTS_FIFO_KIND", kind)
|
|
||||||
.env("CRANK_ARTIFACTS_FIFO_REF", expected.as_str())
|
|
||||||
.spawn()
|
|
||||||
.unwrap();
|
|
||||||
assert!(wait_for_child(child, Duration::from_secs(2)).success());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fifo_read_child() {
|
|
||||||
let Some(root) = std::env::var_os("CRANK_ARTIFACTS_FIFO_ROOT") else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
let root = PathBuf::from(root);
|
|
||||||
let kind = std::env::var("CRANK_ARTIFACTS_FIFO_KIND").unwrap();
|
|
||||||
let expected = ArtifactRef::parse(&std::env::var("CRANK_ARTIFACTS_FIFO_REF").unwrap()).unwrap();
|
|
||||||
let store = ArtifactStore::open(&root).unwrap();
|
|
||||||
let result = if kind == "blob" {
|
|
||||||
store.read(&expected)
|
|
||||||
} else {
|
|
||||||
store
|
|
||||||
.with_legacy_root(root.join("legacy"))
|
|
||||||
.read_legacy_file_url(
|
|
||||||
&format!("file://{}/legacy/source", root.display()),
|
|
||||||
&expected,
|
|
||||||
)
|
|
||||||
};
|
|
||||||
assert_eq!(result.unwrap_err(), ArtifactError::Integrity);
|
|
||||||
}
|
|
||||||
|
|
||||||
struct ShortReader {
|
|
||||||
bytes: Vec<u8>,
|
|
||||||
offset: usize,
|
|
||||||
interrupted: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct FailingReader;
|
|
||||||
|
|
||||||
struct HousekeepingReader {
|
|
||||||
store: ArtifactStore,
|
|
||||||
candidate: Option<StaleTemp>,
|
|
||||||
bytes: io::Cursor<Vec<u8>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Read for FailingReader {
|
|
||||||
fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
|
|
||||||
Err(io::Error::other("private reader detail"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
impl Read for HousekeepingReader {
|
|
||||||
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
|
|
||||||
if let Some(candidate) = self.candidate.take() {
|
|
||||||
self.store.delete_stale_temp(candidate).unwrap();
|
|
||||||
}
|
|
||||||
self.bytes.read(buffer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
impl Read for ShortReader {
|
|
||||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
|
||||||
if !self.interrupted {
|
|
||||||
self.interrupted = true;
|
|
||||||
return Err(io::Error::from(io::ErrorKind::Interrupted));
|
|
||||||
}
|
|
||||||
if self.offset == self.bytes.len() {
|
|
||||||
return Ok(0);
|
|
||||||
}
|
|
||||||
let amount = 1.min(buf.len()).min(self.bytes.len() - self.offset);
|
|
||||||
buf[..amount].copy_from_slice(&self.bytes[self.offset..self.offset + amount]);
|
|
||||||
self.offset += amount;
|
|
||||||
Ok(amount)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn streaming_input_handles_short_reads_and_interruption() {
|
|
||||||
let root = TestRoot::new("stream");
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let mut reader = ShortReader {
|
|
||||||
bytes: b"short chunk source".to_vec(),
|
|
||||||
offset: 0,
|
|
||||||
interrupted: false,
|
|
||||||
};
|
|
||||||
let stored = store.put_reader(&mut reader).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
store.read(&stored.artifact_ref).unwrap(),
|
|
||||||
b"short chunk source"
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
store.put_reader(&mut FailingReader).unwrap_err(),
|
|
||||||
ArtifactError::Storage
|
|
||||||
);
|
|
||||||
for error in [
|
|
||||||
ArtifactError::InvalidReference,
|
|
||||||
ArtifactError::Integrity,
|
|
||||||
ArtifactError::Storage,
|
|
||||||
ArtifactError::UnsafeRoot,
|
|
||||||
] {
|
|
||||||
let diagnostic = format!("{error} {error:?}");
|
|
||||||
assert!(!diagnostic.contains("private reader detail"));
|
|
||||||
assert!(!diagnostic.contains(std::env::temp_dir().to_string_lossy().as_ref()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn user_reader_can_run_housekeeping_without_self_deadlock() {
|
|
||||||
use std::sync::mpsc;
|
|
||||||
|
|
||||||
let root = TestRoot::new("reader-housekeeping");
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let stored = store.put(b"reader housekeeping shard").unwrap();
|
|
||||||
let temp = root
|
|
||||||
.0
|
|
||||||
.join("sha256")
|
|
||||||
.join(&stored.artifact_ref.digest_hex()[..2])
|
|
||||||
.join(".crank-artifact-tmp-v1-00000000000000000000000000000000-1-10");
|
|
||||||
fs::write(&temp, b"partial").unwrap();
|
|
||||||
let (_, candidates) = store.scan_stale_temps(Duration::ZERO, 128, 1).unwrap();
|
|
||||||
let mut reader = HousekeepingReader {
|
|
||||||
store: store.clone(),
|
|
||||||
candidate: candidates.into_iter().next(),
|
|
||||||
bytes: io::Cursor::new(b"reader bytes".to_vec()),
|
|
||||||
};
|
|
||||||
let writer_store = store.clone();
|
|
||||||
let (sender, receiver) = mpsc::channel();
|
|
||||||
thread::spawn(move || {
|
|
||||||
sender.send(writer_store.put_reader(&mut reader)).unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let stored = receiver
|
|
||||||
.recv_timeout(Duration::from_secs(2))
|
|
||||||
.expect("put_reader deadlocked while its reader ran housekeeping")
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(store.read(&stored.artifact_ref).unwrap(), b"reader bytes");
|
|
||||||
assert!(!temp.exists());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
#[cfg(debug_assertions)]
|
|
||||||
fn forked_crash_checkpoints_publish_only_complete_or_absent_blobs() {
|
|
||||||
let _guard = fault_guard();
|
|
||||||
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
|
||||||
set_checkpoint(
|
|
||||||
std::env::var("CRANK_ARTIFACTS_TEST_STAGE").unwrap(),
|
|
||||||
FaultAction::Exit,
|
|
||||||
);
|
|
||||||
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
|
||||||
let _ = store.put(b"crash-consistent bytes");
|
|
||||||
panic!("checkpoint did not terminate the child");
|
|
||||||
}
|
|
||||||
|
|
||||||
for stage in ["write", "file_fsync", "publish", "directory_fsync"] {
|
|
||||||
let root = TestRoot::new(stage);
|
|
||||||
let status = Command::new(std::env::current_exe().unwrap())
|
|
||||||
.args([
|
|
||||||
"--exact",
|
|
||||||
"forked_crash_checkpoints_publish_only_complete_or_absent_blobs",
|
|
||||||
"--nocapture",
|
|
||||||
])
|
|
||||||
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
|
||||||
.env("CRANK_ARTIFACTS_TEST_STAGE", stage)
|
|
||||||
.status()
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(status.code(), Some(86), "stage={stage}");
|
|
||||||
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let expected = ArtifactRef::from_digest_hex(&format!(
|
|
||||||
"{:x}",
|
|
||||||
Sha256::digest(b"crash-consistent bytes")
|
|
||||||
))
|
|
||||||
.unwrap();
|
|
||||||
match store.read(&expected) {
|
|
||||||
Ok(bytes) => assert_eq!(bytes, b"crash-consistent bytes", "stage={stage}"),
|
|
||||||
Err(ArtifactError::NotFound) => {}
|
|
||||||
Err(error) => panic!("stage={stage} exposed an invalid final blob: {error}"),
|
|
||||||
}
|
|
||||||
let stored = store.put(b"crash-consistent bytes").unwrap();
|
|
||||||
assert_eq!(stored.artifact_ref, expected, "stage={stage}");
|
|
||||||
assert_eq!(
|
|
||||||
store.read(&expected).unwrap(),
|
|
||||||
b"crash-consistent bytes",
|
|
||||||
"stage={stage}"
|
|
||||||
);
|
|
||||||
|
|
||||||
let (_, candidates) = store.scan_stale_temps(Duration::ZERO, 128, 16).unwrap();
|
|
||||||
for candidate in candidates {
|
|
||||||
store.delete_stale_temp(candidate).unwrap();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
#[cfg(debug_assertions)]
|
|
||||||
fn disk_full_fault_seams_leave_a_retryable_store() {
|
|
||||||
let _guard = fault_guard();
|
|
||||||
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
|
||||||
set_checkpoint(
|
|
||||||
std::env::var("CRANK_ARTIFACTS_TEST_STAGE").unwrap(),
|
|
||||||
FaultAction::Fail,
|
|
||||||
);
|
|
||||||
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
store.put(b"disk-full seam bytes").unwrap_err(),
|
|
||||||
ArtifactError::Storage
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for stage in ["write", "file_fsync", "publish", "directory_fsync"] {
|
|
||||||
let root = TestRoot::new(&format!("full-{stage}"));
|
|
||||||
let status = Command::new(std::env::current_exe().unwrap())
|
|
||||||
.args([
|
|
||||||
"--exact",
|
|
||||||
"disk_full_fault_seams_leave_a_retryable_store",
|
|
||||||
"--nocapture",
|
|
||||||
])
|
|
||||||
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
|
||||||
.env("CRANK_ARTIFACTS_TEST_STAGE", stage)
|
|
||||||
.status()
|
|
||||||
.unwrap();
|
|
||||||
assert!(status.success(), "stage={stage}");
|
|
||||||
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let expected =
|
|
||||||
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"disk-full seam bytes")))
|
|
||||||
.unwrap();
|
|
||||||
match store.read(&expected) {
|
|
||||||
Ok(bytes) => assert_eq!(bytes, b"disk-full seam bytes", "stage={stage}"),
|
|
||||||
Err(ArtifactError::NotFound) => {}
|
|
||||||
Err(error) => panic!("stage={stage} exposed an invalid final blob: {error}"),
|
|
||||||
}
|
|
||||||
assert_eq!(
|
|
||||||
store.put(b"disk-full seam bytes").unwrap().artifact_ref,
|
|
||||||
expected,
|
|
||||||
"stage={stage}"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
store.read(&expected).unwrap(),
|
|
||||||
b"disk-full seam bytes",
|
|
||||||
"stage={stage}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn real_process_write_limit_leaves_no_partial_final_blob() {
|
|
||||||
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
|
||||||
// Keep the test's host filesystem untouched while making the actual
|
|
||||||
// write syscall fail as it would for a quota/disk-full condition.
|
|
||||||
unsafe {
|
|
||||||
assert_ne!(libc::signal(libc::SIGXFSZ, libc::SIG_IGN), libc::SIG_ERR);
|
|
||||||
let limit = libc::rlimit {
|
|
||||||
rlim_cur: 1,
|
|
||||||
rlim_max: 1,
|
|
||||||
};
|
|
||||||
assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0);
|
|
||||||
}
|
|
||||||
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
store.put(b"real write limit bytes").unwrap_err(),
|
|
||||||
ArtifactError::Storage
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let root = TestRoot::new("real-write-limit");
|
|
||||||
let status = Command::new(std::env::current_exe().unwrap())
|
|
||||||
.args([
|
|
||||||
"--exact",
|
|
||||||
"real_process_write_limit_leaves_no_partial_final_blob",
|
|
||||||
"--nocapture",
|
|
||||||
])
|
|
||||||
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
|
||||||
.status()
|
|
||||||
.unwrap();
|
|
||||||
assert!(status.success());
|
|
||||||
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let expected =
|
|
||||||
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"real write limit bytes")))
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(store.read(&expected).unwrap_err(), ArtifactError::NotFound);
|
|
||||||
assert_eq!(
|
|
||||||
store.put(b"real write limit bytes").unwrap().artifact_ref,
|
|
||||||
expected
|
|
||||||
);
|
|
||||||
assert_eq!(store.read(&expected).unwrap(), b"real write limit bytes");
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,379 @@
|
|||||||
|
use std::{
|
||||||
|
fs,
|
||||||
|
io::{self, Read},
|
||||||
|
os::unix::fs::PermissionsExt,
|
||||||
|
path::PathBuf,
|
||||||
|
process::{Child, Command, ExitStatus},
|
||||||
|
sync::atomic::{AtomicU64, Ordering},
|
||||||
|
thread,
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
use crank_artifacts::test_support::{FaultAction, set_checkpoint};
|
||||||
|
use crank_artifacts::{ArtifactError, ArtifactRef, ArtifactStore, StaleTemp};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
|
||||||
|
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
|
||||||
|
static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
|
||||||
|
GUARD.get_or_init(|| Mutex::new(())).lock().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
struct TestRoot(PathBuf);
|
||||||
|
|
||||||
|
impl TestRoot {
|
||||||
|
fn new(name: &str) -> Self {
|
||||||
|
let path = std::env::temp_dir().join(format!(
|
||||||
|
"crank-artifacts-{name}-{}-{}",
|
||||||
|
std::process::id(),
|
||||||
|
NEXT_ROOT.fetch_add(1, Ordering::Relaxed)
|
||||||
|
));
|
||||||
|
fs::create_dir(&path).unwrap();
|
||||||
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
Self(path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for TestRoot {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700));
|
||||||
|
let _ = fs::remove_dir_all(&self.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_for_child(mut child: Child, timeout: Duration) -> ExitStatus {
|
||||||
|
let deadline = std::time::Instant::now() + timeout;
|
||||||
|
loop {
|
||||||
|
if let Some(status) = child.try_wait().unwrap() {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
if std::time::Instant::now() >= deadline {
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
panic!("child process did not finish within {timeout:?}");
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fifo_entries_fail_without_blocking_reads() {
|
||||||
|
let root = TestRoot::new("fifo");
|
||||||
|
let expected =
|
||||||
|
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"fifo bytes"))).unwrap();
|
||||||
|
let sha = root.0.join("sha256");
|
||||||
|
let shard = sha.join(&expected.digest_hex()[..2]);
|
||||||
|
fs::create_dir(&sha).unwrap();
|
||||||
|
fs::create_dir(&shard).unwrap();
|
||||||
|
fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
let blob_fifo = shard.join(expected.digest_hex());
|
||||||
|
let legacy = root.0.join("legacy");
|
||||||
|
fs::create_dir(&legacy).unwrap();
|
||||||
|
fs::set_permissions(&legacy, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
let legacy_fifo = legacy.join("source");
|
||||||
|
for path in [&blob_fifo, &legacy_fifo] {
|
||||||
|
let path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap();
|
||||||
|
assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o400) }, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let executable = std::env::current_exe().unwrap();
|
||||||
|
for kind in ["blob", "legacy"] {
|
||||||
|
let child = Command::new(&executable)
|
||||||
|
.args(["--exact", "fifo_read_child", "--nocapture"])
|
||||||
|
.env("CRANK_ARTIFACTS_FIFO_ROOT", &root.0)
|
||||||
|
.env("CRANK_ARTIFACTS_FIFO_KIND", kind)
|
||||||
|
.env("CRANK_ARTIFACTS_FIFO_REF", expected.as_str())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
assert!(wait_for_child(child, Duration::from_secs(2)).success());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fifo_read_child() {
|
||||||
|
let Some(root) = std::env::var_os("CRANK_ARTIFACTS_FIFO_ROOT") else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let root = PathBuf::from(root);
|
||||||
|
let kind = std::env::var("CRANK_ARTIFACTS_FIFO_KIND").unwrap();
|
||||||
|
let expected = ArtifactRef::parse(&std::env::var("CRANK_ARTIFACTS_FIFO_REF").unwrap()).unwrap();
|
||||||
|
let store = ArtifactStore::open(&root).unwrap();
|
||||||
|
let result = if kind == "blob" {
|
||||||
|
store.read(&expected)
|
||||||
|
} else {
|
||||||
|
store
|
||||||
|
.with_legacy_root(root.join("legacy"))
|
||||||
|
.read_legacy_file_url(
|
||||||
|
&format!("file://{}/legacy/source", root.display()),
|
||||||
|
&expected,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert_eq!(result.unwrap_err(), ArtifactError::Integrity);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ShortReader {
|
||||||
|
bytes: Vec<u8>,
|
||||||
|
offset: usize,
|
||||||
|
interrupted: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FailingReader;
|
||||||
|
|
||||||
|
struct HousekeepingReader {
|
||||||
|
store: ArtifactStore,
|
||||||
|
candidate: Option<StaleTemp>,
|
||||||
|
bytes: io::Cursor<Vec<u8>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Read for FailingReader {
|
||||||
|
fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
|
||||||
|
Err(io::Error::other("private reader detail"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Read for HousekeepingReader {
|
||||||
|
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
|
||||||
|
if let Some(candidate) = self.candidate.take() {
|
||||||
|
self.store.delete_stale_temp(candidate).unwrap();
|
||||||
|
}
|
||||||
|
self.bytes.read(buffer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Read for ShortReader {
|
||||||
|
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||||
|
if !self.interrupted {
|
||||||
|
self.interrupted = true;
|
||||||
|
return Err(io::Error::from(io::ErrorKind::Interrupted));
|
||||||
|
}
|
||||||
|
if self.offset == self.bytes.len() {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
let amount = 1.min(buf.len()).min(self.bytes.len() - self.offset);
|
||||||
|
buf[..amount].copy_from_slice(&self.bytes[self.offset..self.offset + amount]);
|
||||||
|
self.offset += amount;
|
||||||
|
Ok(amount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn streaming_input_handles_short_reads_and_interruption() {
|
||||||
|
let root = TestRoot::new("stream");
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let mut reader = ShortReader {
|
||||||
|
bytes: b"short chunk source".to_vec(),
|
||||||
|
offset: 0,
|
||||||
|
interrupted: false,
|
||||||
|
};
|
||||||
|
let stored = store.put_reader(&mut reader).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store.read(&stored.artifact_ref).unwrap(),
|
||||||
|
b"short chunk source"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
store.put_reader(&mut FailingReader).unwrap_err(),
|
||||||
|
ArtifactError::Storage
|
||||||
|
);
|
||||||
|
for error in [
|
||||||
|
ArtifactError::InvalidReference,
|
||||||
|
ArtifactError::Integrity,
|
||||||
|
ArtifactError::Storage,
|
||||||
|
ArtifactError::UnsafeRoot,
|
||||||
|
] {
|
||||||
|
let diagnostic = format!("{error} {error:?}");
|
||||||
|
assert!(!diagnostic.contains("private reader detail"));
|
||||||
|
assert!(!diagnostic.contains(std::env::temp_dir().to_string_lossy().as_ref()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn user_reader_can_run_housekeeping_without_self_deadlock() {
|
||||||
|
use std::sync::mpsc;
|
||||||
|
|
||||||
|
let root = TestRoot::new("reader-housekeeping");
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let stored = store.put(b"reader housekeeping shard").unwrap();
|
||||||
|
let temp = root
|
||||||
|
.0
|
||||||
|
.join("sha256")
|
||||||
|
.join(&stored.artifact_ref.digest_hex()[..2])
|
||||||
|
.join(".crank-artifact-tmp-v1-00000000000000000000000000000000-1-10");
|
||||||
|
fs::write(&temp, b"partial").unwrap();
|
||||||
|
let (_, candidates) = store.scan_stale_temps(Duration::ZERO, 128, 1).unwrap();
|
||||||
|
let mut reader = HousekeepingReader {
|
||||||
|
store: store.clone(),
|
||||||
|
candidate: candidates.into_iter().next(),
|
||||||
|
bytes: io::Cursor::new(b"reader bytes".to_vec()),
|
||||||
|
};
|
||||||
|
let writer_store = store.clone();
|
||||||
|
let (sender, receiver) = mpsc::channel();
|
||||||
|
thread::spawn(move || {
|
||||||
|
sender.send(writer_store.put_reader(&mut reader)).unwrap();
|
||||||
|
});
|
||||||
|
|
||||||
|
let stored = receiver
|
||||||
|
.recv_timeout(Duration::from_secs(2))
|
||||||
|
.expect("put_reader deadlocked while its reader ran housekeeping")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(store.read(&stored.artifact_ref).unwrap(), b"reader bytes");
|
||||||
|
assert!(!temp.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
fn forked_crash_checkpoints_publish_only_complete_or_absent_blobs() {
|
||||||
|
let _guard = fault_guard();
|
||||||
|
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
||||||
|
set_checkpoint(
|
||||||
|
std::env::var("CRANK_ARTIFACTS_TEST_STAGE").unwrap(),
|
||||||
|
FaultAction::Exit,
|
||||||
|
);
|
||||||
|
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
||||||
|
let _ = store.put(b"crash-consistent bytes");
|
||||||
|
panic!("checkpoint did not terminate the child");
|
||||||
|
}
|
||||||
|
|
||||||
|
for stage in ["write", "file_fsync", "publish", "directory_fsync"] {
|
||||||
|
let root = TestRoot::new(stage);
|
||||||
|
let status = Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"forked_crash_checkpoints_publish_only_complete_or_absent_blobs",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
||||||
|
.env("CRANK_ARTIFACTS_TEST_STAGE", stage)
|
||||||
|
.status()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(status.code(), Some(86), "stage={stage}");
|
||||||
|
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let expected = ArtifactRef::from_digest_hex(&format!(
|
||||||
|
"{:x}",
|
||||||
|
Sha256::digest(b"crash-consistent bytes")
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
match store.read(&expected) {
|
||||||
|
Ok(bytes) => assert_eq!(bytes, b"crash-consistent bytes", "stage={stage}"),
|
||||||
|
Err(ArtifactError::NotFound) => {}
|
||||||
|
Err(error) => panic!("stage={stage} exposed an invalid final blob: {error}"),
|
||||||
|
}
|
||||||
|
let stored = store.put(b"crash-consistent bytes").unwrap();
|
||||||
|
assert_eq!(stored.artifact_ref, expected, "stage={stage}");
|
||||||
|
assert_eq!(
|
||||||
|
store.read(&expected).unwrap(),
|
||||||
|
b"crash-consistent bytes",
|
||||||
|
"stage={stage}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let (_, candidates) = store.scan_stale_temps(Duration::ZERO, 128, 16).unwrap();
|
||||||
|
for candidate in candidates {
|
||||||
|
store.delete_stale_temp(candidate).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
fn disk_full_fault_seams_leave_a_retryable_store() {
|
||||||
|
let _guard = fault_guard();
|
||||||
|
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
||||||
|
set_checkpoint(
|
||||||
|
std::env::var("CRANK_ARTIFACTS_TEST_STAGE").unwrap(),
|
||||||
|
FaultAction::Fail,
|
||||||
|
);
|
||||||
|
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store.put(b"disk-full seam bytes").unwrap_err(),
|
||||||
|
ArtifactError::Storage
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for stage in ["write", "file_fsync", "publish", "directory_fsync"] {
|
||||||
|
let root = TestRoot::new(&format!("full-{stage}"));
|
||||||
|
let status = Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"disk_full_fault_seams_leave_a_retryable_store",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
||||||
|
.env("CRANK_ARTIFACTS_TEST_STAGE", stage)
|
||||||
|
.status()
|
||||||
|
.unwrap();
|
||||||
|
assert!(status.success(), "stage={stage}");
|
||||||
|
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let expected =
|
||||||
|
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"disk-full seam bytes")))
|
||||||
|
.unwrap();
|
||||||
|
match store.read(&expected) {
|
||||||
|
Ok(bytes) => assert_eq!(bytes, b"disk-full seam bytes", "stage={stage}"),
|
||||||
|
Err(ArtifactError::NotFound) => {}
|
||||||
|
Err(error) => panic!("stage={stage} exposed an invalid final blob: {error}"),
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
store.put(b"disk-full seam bytes").unwrap().artifact_ref,
|
||||||
|
expected,
|
||||||
|
"stage={stage}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.read(&expected).unwrap(),
|
||||||
|
b"disk-full seam bytes",
|
||||||
|
"stage={stage}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn real_process_write_limit_leaves_no_partial_final_blob() {
|
||||||
|
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
||||||
|
// Keep the test's host filesystem untouched while making the actual
|
||||||
|
// write syscall fail as it would for a quota/disk-full condition.
|
||||||
|
unsafe {
|
||||||
|
assert_ne!(libc::signal(libc::SIGXFSZ, libc::SIG_IGN), libc::SIG_ERR);
|
||||||
|
let limit = libc::rlimit {
|
||||||
|
rlim_cur: 1,
|
||||||
|
rlim_max: 1,
|
||||||
|
};
|
||||||
|
assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0);
|
||||||
|
}
|
||||||
|
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store.put(b"real write limit bytes").unwrap_err(),
|
||||||
|
ArtifactError::Storage
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let root = TestRoot::new("real-write-limit");
|
||||||
|
let status = Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"real_process_write_limit_leaves_no_partial_final_blob",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
||||||
|
.status()
|
||||||
|
.unwrap();
|
||||||
|
assert!(status.success());
|
||||||
|
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let expected =
|
||||||
|
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"real write limit bytes")))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(store.read(&expected).unwrap_err(), ArtifactError::NotFound);
|
||||||
|
assert_eq!(
|
||||||
|
store.put(b"real write limit bytes").unwrap().artifact_ref,
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
assert_eq!(store.read(&expected).unwrap(), b"real write limit bytes");
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@ mod onboarding_product_events_v11;
|
|||||||
mod owned_relations;
|
mod owned_relations;
|
||||||
mod platform_key_name_reuse_v6;
|
mod platform_key_name_reuse_v6;
|
||||||
mod schema_guard;
|
mod schema_guard;
|
||||||
|
mod schema_guard_legacy_v1;
|
||||||
mod schema_guard_v10;
|
mod schema_guard_v10;
|
||||||
mod schema_guard_v11;
|
mod schema_guard_v11;
|
||||||
mod schema_guard_v12;
|
mod schema_guard_v12;
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ use super::platform_key_name_reuse_v6;
|
|||||||
use super::schema_guard::{
|
use super::schema_guard::{
|
||||||
OWNED_RELATIONS, relation_exists, validate_required_relations, validate_schema_fingerprint,
|
OWNED_RELATIONS, relation_exists, validate_required_relations, validate_schema_fingerprint,
|
||||||
};
|
};
|
||||||
|
use super::schema_guard_legacy_v1::{
|
||||||
|
validate_ledgerless_baseline_fingerprint, validate_ledgerless_optional_fingerprints,
|
||||||
|
};
|
||||||
use super::{BASELINE_CHECKSUM, BASELINE_VERSION, apply_baseline};
|
use super::{BASELINE_CHECKSUM, BASELINE_VERSION, apply_baseline};
|
||||||
use crate::ext::ExtensionMigration;
|
use crate::ext::ExtensionMigration;
|
||||||
use sqlx::{PgConnection, PgPool, Row, Transaction, query};
|
use sqlx::{PgConnection, PgPool, Row, Transaction, query};
|
||||||
@@ -389,6 +392,19 @@ impl MigrationAuthority {
|
|||||||
if from < 13 {
|
if from < 13 {
|
||||||
artifact_cleanup_indexes_v13::apply(&mut transaction, &Self::sequence()[12]).await?;
|
artifact_cleanup_indexes_v13::apply(&mut transaction, &Self::sequence()[12]).await?;
|
||||||
}
|
}
|
||||||
|
match inspect(&mut transaction).await? {
|
||||||
|
MigrationPreflight::Current {
|
||||||
|
version: CURRENT_VERSION,
|
||||||
|
} => {}
|
||||||
|
_ => {
|
||||||
|
return Err(MigrationError::new(
|
||||||
|
"apply_failed",
|
||||||
|
"apply.postflight",
|
||||||
|
Some(CURRENT_VERSION),
|
||||||
|
"restore_known_good_backup",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
transaction
|
transaction
|
||||||
.commit()
|
.commit()
|
||||||
.await
|
.await
|
||||||
@@ -408,9 +424,15 @@ async fn inspect(connection: &mut PgConnection) -> Result<MigrationPreflight, Mi
|
|||||||
owned_exists |= relation_exists(connection, relation).await?;
|
owned_exists |= relation_exists(connection, relation).await?;
|
||||||
}
|
}
|
||||||
if owned_exists {
|
if owned_exists {
|
||||||
|
if is_ledgerless_legacy_baseline(connection, canonical_exists).await? {
|
||||||
|
return Ok(MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: CURRENT_VERSION,
|
||||||
|
});
|
||||||
|
}
|
||||||
return Err(MigrationError::new(
|
return Err(MigrationError::new(
|
||||||
"partial_sequence",
|
"partial_sequence",
|
||||||
"preflight.core",
|
"preflight.core_missing",
|
||||||
None,
|
None,
|
||||||
"restore_known_good_backup",
|
"restore_known_good_backup",
|
||||||
));
|
));
|
||||||
@@ -541,6 +563,47 @@ async fn inspect(connection: &mut PgConnection) -> Result<MigrationPreflight, Mi
|
|||||||
Ok(MigrationPreflight::Current { version: current })
|
Ok(MigrationPreflight::Current { version: current })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
async fn is_ledgerless_legacy_baseline(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
canonical_exists: bool,
|
||||||
|
) -> Result<bool, MigrationError> {
|
||||||
|
if canonical_exists {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
for relation in [
|
||||||
|
"__crank_migration_legacy_audit",
|
||||||
|
"master_key_identities",
|
||||||
|
"master_key_rotations",
|
||||||
|
"admin_bootstrap_contracts",
|
||||||
|
"admin_login_backoff",
|
||||||
|
"admin_security_audit_events",
|
||||||
|
"product_events",
|
||||||
|
"product_event_daily_rollups",
|
||||||
|
"onboarding_selections",
|
||||||
|
"artifact_blobs",
|
||||||
|
"artifact_sources",
|
||||||
|
] {
|
||||||
|
if relation_exists(connection, relation).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for relation in owned_relations::BASELINE {
|
||||||
|
if !relation_exists(connection, relation).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validate_required_relations(connection, owned_relations::BASELINE, 1).await?;
|
||||||
|
validate_ledgerless_baseline_fingerprint(connection).await?;
|
||||||
|
let mcp_ledger = relation_exists(connection, "__crank_mcp_migrations").await?;
|
||||||
|
if mcp_ledger {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
let mcp_sessions = relation_exists(connection, "mcp_transport_sessions").await?;
|
||||||
|
let extension_ledger = relation_exists(connection, "__crank_ext_migrations").await?;
|
||||||
|
validate_ledgerless_optional_fingerprints(connection, mcp_sessions, extension_ledger).await?;
|
||||||
|
inspect_optional_legacy_for_ledgerless_baseline(connection).await?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
async fn validate_core_ledger(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
async fn validate_core_ledger(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
||||||
let rows = query("select version, description, checksum from __crank_core_migrations order by version limit 2")
|
let rows = query("select version, description, checksum from __crank_core_migrations order by version limit 2")
|
||||||
.fetch_all(connection)
|
.fetch_all(connection)
|
||||||
@@ -549,7 +612,7 @@ async fn validate_core_ledger(connection: &mut PgConnection) -> Result<(), Migra
|
|||||||
if rows.len() != 1 {
|
if rows.len() != 1 {
|
||||||
return Err(MigrationError::new(
|
return Err(MigrationError::new(
|
||||||
"partial_sequence",
|
"partial_sequence",
|
||||||
"preflight.core",
|
"preflight.core_cardinality",
|
||||||
None,
|
None,
|
||||||
"restore_known_good_backup",
|
"restore_known_good_backup",
|
||||||
));
|
));
|
||||||
@@ -577,9 +640,21 @@ async fn validate_core_ledger(connection: &mut PgConnection) -> Result<(), Migra
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
async fn inspect_optional_legacy(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
async fn inspect_optional_legacy(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
||||||
|
inspect_optional_legacy_with_policy(connection, false).await
|
||||||
|
}
|
||||||
|
async fn inspect_optional_legacy_for_ledgerless_baseline(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
) -> Result<(), MigrationError> {
|
||||||
|
inspect_optional_legacy_with_policy(connection, true).await
|
||||||
|
}
|
||||||
|
async fn inspect_optional_legacy_with_policy(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
allow_sessions_without_ledger: bool,
|
||||||
|
) -> Result<(), MigrationError> {
|
||||||
let mcp_ledger = relation_exists(connection, "__crank_mcp_migrations").await?;
|
let mcp_ledger = relation_exists(connection, "__crank_mcp_migrations").await?;
|
||||||
let mcp_sessions = relation_exists(connection, "mcp_transport_sessions").await?;
|
let mcp_sessions = relation_exists(connection, "mcp_transport_sessions").await?;
|
||||||
if mcp_ledger != mcp_sessions {
|
if mcp_ledger != mcp_sessions && !(allow_sessions_without_ledger && !mcp_ledger && mcp_sessions)
|
||||||
|
{
|
||||||
return Err(MigrationError::new(
|
return Err(MigrationError::new(
|
||||||
"legacy_conflict",
|
"legacy_conflict",
|
||||||
"preflight.legacy_mcp",
|
"preflight.legacy_mcp",
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ fn sequence_is_deterministic_and_append_only() {
|
|||||||
MigrationAuthority::validate_sequence().unwrap();
|
MigrationAuthority::validate_sequence().unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
first.iter().map(|item| item.version).collect::<Vec<_>>(),
|
first.iter().map(|item| item.version).collect::<Vec<_>>(),
|
||||||
vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
|
vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
|
||||||
);
|
);
|
||||||
assert_eq!(first[0].checksum, "crank-community-baseline-v1");
|
assert_eq!(first[0].checksum, "crank-community-baseline-v1");
|
||||||
assert_eq!(first[0].source_digest, BASELINE_SOURCE_SHA256);
|
assert_eq!(first[0].source_digest, BASELINE_SOURCE_SHA256);
|
||||||
|
|||||||
@@ -325,6 +325,15 @@ pub(super) async fn validate_schema_fingerprint(
|
|||||||
.iter()
|
.iter()
|
||||||
.filter_map(|row| row.try_get::<String, _>("column_name").ok())
|
.filter_map(|row| row.try_get::<String, _>("column_name").ok())
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
let required = required
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|column| {
|
||||||
|
!(current_version == 1
|
||||||
|
&& *table == "__crank_ext_migrations"
|
||||||
|
&& *column == "checksum")
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
if actual.len() != required.len()
|
if actual.len() != required.len()
|
||||||
|| required
|
|| required
|
||||||
.iter()
|
.iter()
|
||||||
@@ -337,6 +346,9 @@ pub(super) async fn validate_schema_fingerprint(
|
|||||||
if !relation_exists(connection, table).await? {
|
if !relation_exists(connection, table).await? {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if current_version == 1 && *table == "__crank_ext_migrations" && *column == "checksum" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let row = query(
|
let row = query(
|
||||||
"select data_type, is_nullable from information_schema.columns
|
"select data_type, is_nullable from information_schema.columns
|
||||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||||
@@ -424,23 +436,25 @@ pub(super) async fn validate_schema_fingerprint(
|
|||||||
return Err(schema_error(current_version));
|
return Err(schema_error(current_version));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let required_indexes = [
|
if relation_exists(connection, "mcp_transport_sessions").await? {
|
||||||
"mcp_transport_sessions_workspace_agent_idx",
|
let required_indexes = [
|
||||||
"mcp_transport_sessions_expires_at_idx",
|
"mcp_transport_sessions_workspace_agent_idx",
|
||||||
];
|
"mcp_transport_sessions_expires_at_idx",
|
||||||
for index in required_indexes {
|
];
|
||||||
let present = query(
|
for index in required_indexes {
|
||||||
"select exists (select 1 from pg_catalog.pg_indexes
|
let present = query(
|
||||||
where schemaname = current_schema() and indexname = $1) as present",
|
"select exists (select 1 from pg_catalog.pg_indexes
|
||||||
)
|
where schemaname = current_schema() and indexname = $1) as present",
|
||||||
.bind(index)
|
)
|
||||||
.fetch_one(&mut *connection)
|
.bind(index)
|
||||||
.await
|
.fetch_one(&mut *connection)
|
||||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
.await
|
||||||
.try_get::<bool, _>("present")
|
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
.try_get::<bool, _>("present")
|
||||||
if !present {
|
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||||
return Err(schema_error(current_version));
|
if !present {
|
||||||
|
return Err(schema_error(current_version));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if current_version >= 3 {
|
if current_version >= 3 {
|
||||||
|
|||||||
@@ -0,0 +1,196 @@
|
|||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use sqlx::PgConnection;
|
||||||
|
|
||||||
|
use super::authority::MigrationError;
|
||||||
|
|
||||||
|
// Exact PostgreSQL 16 catalog contract produced by the last published
|
||||||
|
// pre-ledger Community schema (commit 8318e4b).
|
||||||
|
const LEDGERLESS_BASELINE_FINGERPRINT_SHA256: &str =
|
||||||
|
"36624ca42a28c1388f9c50e6d6c489a8e5ca1f73441a894d9af89e5e115b542c";
|
||||||
|
const LEDGERLESS_MCP_FINGERPRINTS_SHA256: &[&str] = &[
|
||||||
|
// Initial published session table.
|
||||||
|
"9240c3d85dbc9eeddb1cd99661ec8f7d8d6ece0e62ec486151dcc951a7f3c81c",
|
||||||
|
// Published session table after supports_elicitation was added.
|
||||||
|
"ac9f99a7e667552a07480d5d45379dd0a8b529b00ba98855ce6725535198249f",
|
||||||
|
];
|
||||||
|
const LEDGERLESS_EXTENSION_FINGERPRINT_SHA256: &str =
|
||||||
|
"0809a80c0bb6e80f68d0557006c4b62f2e63556f61f6fdedc5abc324950e2587";
|
||||||
|
|
||||||
|
const BASELINE_RELATIONS: &[&str] = &[
|
||||||
|
"workspaces",
|
||||||
|
"users",
|
||||||
|
"memberships",
|
||||||
|
"user_sessions",
|
||||||
|
"invitation_tokens",
|
||||||
|
"platform_api_keys",
|
||||||
|
"operations",
|
||||||
|
"operation_versions",
|
||||||
|
"published_operations",
|
||||||
|
"operation_samples",
|
||||||
|
"descriptors",
|
||||||
|
"agents",
|
||||||
|
"agent_versions",
|
||||||
|
"published_agents",
|
||||||
|
"agent_operation_bindings",
|
||||||
|
"secrets",
|
||||||
|
"secret_versions",
|
||||||
|
"auth_profiles",
|
||||||
|
"workspace_upstreams",
|
||||||
|
"yaml_import_jobs",
|
||||||
|
"import_jobs",
|
||||||
|
"approval_requests",
|
||||||
|
"invocation_logs",
|
||||||
|
"usage_rollups",
|
||||||
|
];
|
||||||
|
|
||||||
|
pub(super) async fn validate_ledgerless_baseline_fingerprint(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
) -> Result<(), MigrationError> {
|
||||||
|
let actual = catalog_fingerprint(connection, BASELINE_RELATIONS).await?;
|
||||||
|
if actual == LEDGERLESS_BASELINE_FINGERPRINT_SHA256 {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(fingerprint_error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn validate_ledgerless_optional_fingerprints(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
has_mcp_sessions: bool,
|
||||||
|
has_extension_ledger: bool,
|
||||||
|
) -> Result<(), MigrationError> {
|
||||||
|
if has_mcp_sessions {
|
||||||
|
let actual = catalog_fingerprint(connection, &["mcp_transport_sessions"]).await?;
|
||||||
|
if !LEDGERLESS_MCP_FINGERPRINTS_SHA256.contains(&actual.as_str()) {
|
||||||
|
return Err(fingerprint_error());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if has_extension_ledger {
|
||||||
|
let actual = catalog_fingerprint(connection, &["__crank_ext_migrations"]).await?;
|
||||||
|
if actual != LEDGERLESS_EXTENSION_FINGERPRINT_SHA256 {
|
||||||
|
return Err(fingerprint_error());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn catalog_fingerprint(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
relations: &[&str],
|
||||||
|
) -> Result<String, MigrationError> {
|
||||||
|
let relations = relations
|
||||||
|
.iter()
|
||||||
|
.map(|value| (*value).to_owned())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let unsafe_catalog_state = sqlx::query_scalar::<_, bool>(
|
||||||
|
"with selected(table_name) as (select unnest($1::text[]))
|
||||||
|
select
|
||||||
|
exists (
|
||||||
|
select 1 from pg_catalog.pg_class c
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = c.relnamespace
|
||||||
|
join selected s on s.table_name = c.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
and (c.relpersistence <> 'p' or c.relrowsecurity
|
||||||
|
or c.relforcerowsecurity or c.relreplident <> 'd')
|
||||||
|
)
|
||||||
|
or exists (
|
||||||
|
select 1 from pg_catalog.pg_index i
|
||||||
|
join pg_catalog.pg_class t on t.oid = i.indrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join selected s on s.table_name = t.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
and (not i.indisvalid or not i.indisready or not i.indislive)
|
||||||
|
)
|
||||||
|
or exists (
|
||||||
|
select 1 from pg_catalog.pg_policy p
|
||||||
|
join pg_catalog.pg_class t on t.oid = p.polrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join selected s on s.table_name = t.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
)
|
||||||
|
or exists (
|
||||||
|
select 1 from pg_catalog.pg_trigger tg
|
||||||
|
join pg_catalog.pg_class t on t.oid = tg.tgrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join selected s on s.table_name = t.relname
|
||||||
|
where n.nspname = current_schema() and tg.tgenabled <> 'O'
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.bind(relations.clone())
|
||||||
|
.fetch_one(&mut *connection)
|
||||||
|
.await
|
||||||
|
.map_err(|_| MigrationError::storage("preflight.legacy_fingerprint"))?;
|
||||||
|
if unsafe_catalog_state {
|
||||||
|
return Err(fingerprint_error());
|
||||||
|
}
|
||||||
|
let fingerprint = sqlx::query_scalar::<_, String>(
|
||||||
|
"with baseline(table_name) as (select unnest($1::text[])), relation_rows as (
|
||||||
|
select jsonb_build_array('relation', c.relname, c.relkind::text) item
|
||||||
|
from pg_catalog.pg_class c
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = c.relnamespace
|
||||||
|
join baseline b on b.table_name = c.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
), column_rows as (
|
||||||
|
select jsonb_build_array(
|
||||||
|
'column', c.table_name, c.column_name,
|
||||||
|
c.data_type, c.udt_name, c.is_nullable, coalesce(c.column_default, '')
|
||||||
|
) item
|
||||||
|
from information_schema.columns c
|
||||||
|
join baseline b using (table_name)
|
||||||
|
where c.table_schema = current_schema()
|
||||||
|
), constraint_rows as (
|
||||||
|
select jsonb_build_array(
|
||||||
|
'constraint', t.relname, c.conname, c.contype::text,
|
||||||
|
c.convalidated, pg_get_constraintdef(c.oid, true)
|
||||||
|
) item
|
||||||
|
from pg_catalog.pg_constraint c
|
||||||
|
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join baseline b on b.table_name = t.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
), index_rows as (
|
||||||
|
select jsonb_build_array(
|
||||||
|
'index', t.relname, idx.relname,
|
||||||
|
replace(pg_get_indexdef(i.indexrelid), format('%I.', current_schema()), '')
|
||||||
|
) item
|
||||||
|
from pg_catalog.pg_index i
|
||||||
|
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
|
||||||
|
join pg_catalog.pg_class t on t.oid = i.indrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join baseline b on b.table_name = t.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
), trigger_rows as (
|
||||||
|
select jsonb_build_array(
|
||||||
|
'trigger', t.relname, tg.tgname,
|
||||||
|
replace(pg_get_triggerdef(tg.oid, true), format('%I.', current_schema()), '')
|
||||||
|
) item
|
||||||
|
from pg_catalog.pg_trigger tg
|
||||||
|
join pg_catalog.pg_class t on t.oid = tg.tgrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join baseline b on b.table_name = t.relname
|
||||||
|
where n.nspname = current_schema() and not tg.tgisinternal
|
||||||
|
), all_rows as (
|
||||||
|
select item from relation_rows
|
||||||
|
union all select item from column_rows
|
||||||
|
union all select item from constraint_rows
|
||||||
|
union all select item from index_rows
|
||||||
|
union all select item from trigger_rows
|
||||||
|
)
|
||||||
|
select coalesce(jsonb_agg(item order by item::text), '[]'::jsonb)::text
|
||||||
|
from all_rows",
|
||||||
|
)
|
||||||
|
.bind(relations)
|
||||||
|
.fetch_one(connection)
|
||||||
|
.await
|
||||||
|
.map_err(|_| MigrationError::storage("preflight.legacy_fingerprint"))?;
|
||||||
|
Ok(format!("{:x}", Sha256::digest(fingerprint.as_bytes())))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fingerprint_error() -> MigrationError {
|
||||||
|
MigrationError::new(
|
||||||
|
"partial_sequence",
|
||||||
|
"preflight.legacy_fingerprint",
|
||||||
|
Some(1),
|
||||||
|
"restore_known_good_backup",
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -211,7 +211,7 @@ impl PostgresRegistry {
|
|||||||
let payload = row.try_get::<Value, _>("preview_payload")?;
|
let payload = row.try_get::<Value, _>("preview_payload")?;
|
||||||
// The job itself is expired regardless of whether a legacy or
|
// The job itself is expired regardless of whether a legacy or
|
||||||
// corrupt payload can be decoded. Do not let one bad row roll
|
// corrupt payload can be decoded. Do not let one bad row roll
|
||||||
// back cleanup for every tenant.
|
// back cleanup for every workspace.
|
||||||
for source_id in cleanup_source_ids(&payload) {
|
for source_id in cleanup_source_ids(&payload) {
|
||||||
if detach_source_in_transaction(&mut transaction, &workspace_id, &source_id, now)
|
if detach_source_in_transaction(&mut transaction, &workspace_id, &source_id, now)
|
||||||
.await?
|
.await?
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use crank_registry::{MigrationAuthority, MigrationPreflight, PostgresRegistry};
|
|||||||
use sqlx::Row;
|
use sqlx::Row;
|
||||||
|
|
||||||
mod artifact_metadata;
|
mod artifact_metadata;
|
||||||
|
mod legacy_adoption;
|
||||||
mod rollback;
|
mod rollback;
|
||||||
|
|
||||||
static EVENT_TRIGGER_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
static EVENT_TRIGGER_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||||
@@ -153,7 +154,7 @@ async fn changed_checksum_fails_closed_without_repair() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn legacy_core_baseline_is_consolidated_without_data_loss() {
|
async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
|
||||||
let database_url = crank_test_support::postgres_schema_url("test_legacy_core").await;
|
let database_url = crank_test_support::postgres_schema_url("test_legacy_core").await;
|
||||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
MigrationAuthority::apply(&pool).await.unwrap();
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
@@ -221,7 +222,8 @@ async fn legacy_core_baseline_is_consolidated_without_data_loss() {
|
|||||||
}
|
}
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"drop table __crank_migrations, __crank_migration_legacy_audit,
|
"drop table __crank_migrations, __crank_migration_legacy_audit,
|
||||||
__crank_mcp_migrations, mcp_transport_sessions, __crank_ext_migrations",
|
__crank_mcp_migrations, mcp_transport_sessions, __crank_ext_migrations,
|
||||||
|
__crank_core_migrations",
|
||||||
)
|
)
|
||||||
.execute(&pool)
|
.execute(&pool)
|
||||||
.await
|
.await
|
||||||
@@ -229,7 +231,7 @@ async fn legacy_core_baseline_is_consolidated_without_data_loss() {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
MigrationPreflight::MigrationRequired {
|
MigrationPreflight::MigrationRequired {
|
||||||
current: 1,
|
current: 0,
|
||||||
target: 13,
|
target: 13,
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@@ -937,6 +939,20 @@ async fn any_owned_relation_without_core_ledger_is_partial() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||||
assert_eq!(error.code(), "partial_sequence");
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.core_missing");
|
||||||
|
}
|
||||||
|
#[tokio::test]
|
||||||
|
async fn empty_core_ledger_is_reported_separately() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_empty_core_ledger").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
sqlx::query("delete from __crank_core_migrations")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.core_cardinality");
|
||||||
}
|
}
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn current_ledger_with_structural_drift_fails_closed() {
|
async fn current_ledger_with_structural_drift_fails_closed() {
|
||||||
|
|||||||
@@ -0,0 +1,439 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
async fn ledgerless_v1(pool: &sqlx::PgPool) {
|
||||||
|
MigrationAuthority::apply(pool).await.unwrap();
|
||||||
|
remove_v4_schema(pool).await;
|
||||||
|
remove_v3_schema(pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"drop table __crank_migrations, __crank_migration_legacy_audit;
|
||||||
|
drop table __crank_mcp_migrations;
|
||||||
|
drop index mcp_transport_sessions_expires_at_idx;
|
||||||
|
drop table __crank_ext_migrations;
|
||||||
|
create table __crank_ext_migrations (
|
||||||
|
extension_name text not null,
|
||||||
|
version integer not null,
|
||||||
|
applied_at timestamptz not null default now(),
|
||||||
|
primary key (extension_name, version)
|
||||||
|
);
|
||||||
|
drop table __crank_core_migrations;",
|
||||||
|
)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_optional_index_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_index").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"drop index mcp_transport_sessions_workspace_agent_idx;
|
||||||
|
create index mcp_transport_sessions_workspace_agent_idx
|
||||||
|
on mcp_transport_sessions(id);",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_disabled_integrity_triggers_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_triggers").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table memberships disable trigger all")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn published_ledgerless_baseline_upgrades_without_data_loss() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_baseline").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query(
|
||||||
|
"insert into operations
|
||||||
|
(id, workspace_id, name, display_name, protocol, status, created_at, updated_at)
|
||||||
|
values ('op_ledgerless', 'ws_default', 'ledgerless', 'Ledgerless', 'rest', 'draft', now(), now())",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: 13,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
|
||||||
|
let operation_count: i64 =
|
||||||
|
sqlx::query_scalar("select count(*) from operations where id = 'op_ledgerless'")
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(operation_count, 1);
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::Current { version: 13 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn published_ledgerless_historical_column_layout_upgrades_without_data_loss() {
|
||||||
|
let database_url =
|
||||||
|
crank_test_support::postgres_schema_url("test_ledgerless_historical_layout").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"alter table platform_api_keys
|
||||||
|
drop column key_kind,
|
||||||
|
drop column expires_at,
|
||||||
|
drop column allowed_origins_json;
|
||||||
|
alter table platform_api_keys
|
||||||
|
add column key_kind text not null default 'mcp_client',
|
||||||
|
add column expires_at timestamptz null,
|
||||||
|
add column allowed_origins_json jsonb not null default '[]'::jsonb;
|
||||||
|
|
||||||
|
drop index approval_requests_pending_fingerprint_idx;
|
||||||
|
alter table approval_requests
|
||||||
|
drop column execution_started_at,
|
||||||
|
drop column execution_attempts,
|
||||||
|
drop column request_fingerprint,
|
||||||
|
add column confirmation_title text not null default '',
|
||||||
|
add column confirmation_body text not null default '';
|
||||||
|
alter table approval_requests
|
||||||
|
alter column confirmation_title drop default,
|
||||||
|
alter column confirmation_body drop default,
|
||||||
|
drop column confirmation_title,
|
||||||
|
drop column confirmation_body,
|
||||||
|
add column execution_started_at timestamptz null,
|
||||||
|
add column execution_attempts integer not null default 0,
|
||||||
|
add column request_fingerprint text null;
|
||||||
|
create unique index approval_requests_pending_fingerprint_idx
|
||||||
|
on approval_requests(agent_id, operation_id, operation_version, request_fingerprint)
|
||||||
|
where status = 'pending' and request_fingerprint is not null;
|
||||||
|
|
||||||
|
insert into operations
|
||||||
|
(id, workspace_id, name, display_name, protocol, status, created_at, updated_at)
|
||||||
|
values ('op_historical_layout', 'ws_default', 'historical-layout', 'Historical layout', 'rest', 'draft', now(), now());
|
||||||
|
insert into agents
|
||||||
|
(id, workspace_id, slug, display_name, description, status, created_at, updated_at)
|
||||||
|
values ('agent_historical_layout', 'ws_default', 'historical-layout', 'Historical layout', '', 'draft', now(), now());
|
||||||
|
insert into platform_api_keys
|
||||||
|
(id, workspace_id, agent_id, name, prefix, secret_hash, key_kind, scopes_json, status,
|
||||||
|
created_at, expires_at, allowed_origins_json)
|
||||||
|
values ('key_historical_layout', 'ws_default', 'agent_historical_layout', 'Historical layout',
|
||||||
|
'cp_', 'hash', 'admin', '[]'::jsonb, 'active', now(),
|
||||||
|
'2030-01-02 03:04:05+00'::timestamptz, '[\"https://example.test\"]'::jsonb);
|
||||||
|
insert into approval_requests
|
||||||
|
(id, workspace_id, agent_id, operation_id, operation_version, status, risk_level,
|
||||||
|
request_payload_json, created_at, expires_at, execution_started_at,
|
||||||
|
execution_attempts, request_fingerprint)
|
||||||
|
values ('approval_historical_layout', 'ws_default', 'agent_historical_layout', 'op_historical_layout', 1,
|
||||||
|
'pending', 'high', '{\"layout\":\"historical\"}'::jsonb, now(), now() + interval '1 hour',
|
||||||
|
'2029-02-03 04:05:06+00'::timestamptz, 3, 'historical-fingerprint');",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let historical_ordinals: Vec<(String, i32)> = sqlx::query_as(
|
||||||
|
"select table_name || '.' || column_name, ordinal_position
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = current_schema()
|
||||||
|
and ((table_name = 'platform_api_keys'
|
||||||
|
and column_name in ('key_kind', 'expires_at', 'allowed_origins_json'))
|
||||||
|
or (table_name = 'approval_requests'
|
||||||
|
and column_name in ('execution_started_at', 'execution_attempts', 'request_fingerprint')))
|
||||||
|
order by table_name, ordinal_position",
|
||||||
|
)
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
historical_ordinals,
|
||||||
|
vec![
|
||||||
|
("approval_requests.execution_started_at".to_owned(), 22),
|
||||||
|
("approval_requests.execution_attempts".to_owned(), 23),
|
||||||
|
("approval_requests.request_fingerprint".to_owned(), 24),
|
||||||
|
("platform_api_keys.key_kind".to_owned(), 15),
|
||||||
|
("platform_api_keys.expires_at".to_owned(), 16),
|
||||||
|
("platform_api_keys.allowed_origins_json".to_owned(), 17),
|
||||||
|
],
|
||||||
|
"fixture must reproduce the published in-place column layout",
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: 13,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
|
||||||
|
let key_count: i64 = sqlx::query_scalar(
|
||||||
|
"select count(*) from platform_api_keys
|
||||||
|
where id = 'key_historical_layout'
|
||||||
|
and key_kind = 'admin'
|
||||||
|
and expires_at = '2030-01-02 03:04:05+00'::timestamptz
|
||||||
|
and allowed_origins_json = '[\"https://example.test\"]'::jsonb",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
key_count, 1,
|
||||||
|
"platform key row must survive the layout upgrade"
|
||||||
|
);
|
||||||
|
let approval_count: i64 = sqlx::query_scalar(
|
||||||
|
"select count(*) from approval_requests
|
||||||
|
where id = 'approval_historical_layout'
|
||||||
|
and request_payload_json = '{\"layout\":\"historical\"}'::jsonb
|
||||||
|
and execution_started_at = '2029-02-03 04:05:06+00'::timestamptz
|
||||||
|
and execution_attempts = 3
|
||||||
|
and request_fingerprint = 'historical-fingerprint'",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
approval_count, 1,
|
||||||
|
"approval row must survive the layout upgrade"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::Current { version: 13 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn published_initial_mcp_layout_is_accepted() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_initial_mcp").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"drop table mcp_transport_sessions;
|
||||||
|
create table mcp_transport_sessions (
|
||||||
|
id text primary key,
|
||||||
|
protocol_version text not null,
|
||||||
|
initialized boolean not null default false,
|
||||||
|
workspace_slug text not null,
|
||||||
|
agent_slug text not null,
|
||||||
|
created_at timestamptz not null,
|
||||||
|
updated_at timestamptz not null,
|
||||||
|
expires_at timestamptz null
|
||||||
|
);
|
||||||
|
create index mcp_transport_sessions_workspace_agent_idx
|
||||||
|
on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc);",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: 13,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
let supports_elicitation: bool = sqlx::query_scalar(
|
||||||
|
"select exists (
|
||||||
|
select 1 from information_schema.columns
|
||||||
|
where table_schema = current_schema()
|
||||||
|
and table_name = 'mcp_transport_sessions'
|
||||||
|
and column_name = 'supports_elicitation'
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(supports_elicitation);
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::Current { version: 13 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn published_in_place_mcp_upgrade_layout_is_accepted() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_mcp_upgrade").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"drop table mcp_transport_sessions;
|
||||||
|
create table mcp_transport_sessions (
|
||||||
|
id text primary key,
|
||||||
|
protocol_version text not null,
|
||||||
|
initialized boolean not null default false,
|
||||||
|
workspace_slug text not null,
|
||||||
|
agent_slug text not null,
|
||||||
|
created_at timestamptz not null,
|
||||||
|
updated_at timestamptz not null,
|
||||||
|
expires_at timestamptz null
|
||||||
|
);
|
||||||
|
create index mcp_transport_sessions_workspace_agent_idx
|
||||||
|
on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc);
|
||||||
|
alter table mcp_transport_sessions
|
||||||
|
add column supports_elicitation boolean not null default false;",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: 13,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::Current { version: 13 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_future_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_drift").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table invocation_logs add column trace_id text")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_default_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_default").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table workspaces alter column status set default 'active'")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_constraint_drift_is_rejected_without_writes() {
|
||||||
|
let database_url =
|
||||||
|
crank_test_support::postgres_schema_url("test_ledgerless_constraint_definition").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query(
|
||||||
|
"alter table workspaces
|
||||||
|
add constraint workspaces_status_nonempty check (status <> '')",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_rls_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_rls").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table workspaces enable row level security")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_nullability_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_constraint").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table workspaces alter column status drop not null")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
@@ -138,7 +138,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_ADMIN_PUBLISH_PORT:-3001}:3001"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_ADMIN_PUBLISH_PORT:-3001}:3001"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3001", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
@@ -199,7 +199,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_MCP_PUBLISH_PORT:-3002}:3002"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_MCP_PUBLISH_PORT:-3002}:3002"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3002", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
required: false
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
|
POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
|
||||||
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
|
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
|
||||||
@@ -141,7 +142,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_ADMIN_PUBLISH_PORT:-3001}:3001"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_ADMIN_PUBLISH_PORT:-3001}:3001"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3001", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
@@ -202,7 +203,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_MCP_PUBLISH_PORT:-3002}:3002"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_MCP_PUBLISH_PORT:-3002}:3002"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3002", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|||||||
+2
-2
@@ -127,7 +127,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:3001:3001"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:3001:3001"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3001", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
@@ -190,7 +190,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:3002:3002"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:3002:3002"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/health"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3002", "/health"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|||||||
+4
-1
@@ -16,7 +16,7 @@ docker compose -f deploy/community/docker-compose.yml --env-file deploy/communit
|
|||||||
2. Создайте и проверьте согласованный backup PostgreSQL и artifact storage. Для первой пустой установки зафиксируйте, что восстанавливать нечего.
|
2. Создайте и проверьте согласованный backup PostgreSQL и artifact storage. Для первой пустой установки зафиксируйте, что восстанавливать нечего.
|
||||||
3. Проверьте immutable plan: `cargo run -p admin-api --bin crank-migrate -- plan --check` в source checkout либо `<compose> run --rm migrate crank-migrate plan` для образа.
|
3. Проверьте immutable plan: `cargo run -p admin-api --bin crank-migrate -- plan --check` в source checkout либо `<compose> run --rm migrate crank-migrate plan` для образа.
|
||||||
4. Примените sequence: `<compose> run --rm migrate crank-migrate apply`.
|
4. Примените sequence: `<compose> run --rm migrate crank-migrate apply`.
|
||||||
5. Повторите preflight и убедитесь в `{"status":"current","version":12}`.
|
5. Повторите preflight и убедитесь в `{"status":"current","version":13}`.
|
||||||
6. Только теперь запускайте long-running services: `<compose> up -d`.
|
6. Только теперь запускайте long-running services: `<compose> up -d`.
|
||||||
|
|
||||||
Обычный `up` также содержит обязательный migration job, но при upgrade он не заменяет предварительные preflight и backup. Migrator делает до десяти bounded попыток подключения с секундной паузой и затем безопасно завершается ошибкой.
|
Обычный `up` также содержит обязательный migration job, но при upgrade он не заменяет предварительные preflight и backup. Migrator делает до десяти bounded попыток подключения с секундной паузой и затем безопасно завершается ошибкой.
|
||||||
@@ -90,6 +90,9 @@ cargo run -p admin-api --bin crank-migrate -- plan --check
|
|||||||
sensitivity и source lifecycle принадлежат scoped relation. Claim token/expiry
|
sensitivity и source lifecycle принадлежат scoped relation. Claim token/expiry
|
||||||
зарезервированы bounded all-or-none contract для последующего reconciliation,
|
зарезервированы bounded all-or-none contract для последующего reconciliation,
|
||||||
но эта версия не запускает cleanup и не удаляет physical blobs.
|
но эта версия не запускает cleanup и не удаляет physical blobs.
|
||||||
|
- V13 — expand-only indexes, поддерживающие bounded artifact/import cleanup и
|
||||||
|
reconciliation ранее зарезервированных claim token/expiry; эта версия сама
|
||||||
|
не выполняет cleanup, не переписывает данные и не удаляет physical blobs.
|
||||||
- Каждая версия имеет contiguous `i64` version, стабильное имя, lowercase SHA-256, owner, phase, explicit readable schema min/max и backfill policy.
|
- Каждая версия имеет contiguous `i64` version, стабильное имя, lowercase SHA-256, owner, phase, explicit readable schema min/max и backfill policy.
|
||||||
- `migrate` требует bounded cursor/batch policy; `contract` дополнительно требует tracked compatibility evidence и закрытого окна.
|
- `migrate` требует bounded cursor/batch policy; `contract` дополнительно требует tracked compatibility evidence и закрытого окна.
|
||||||
- Добавление descriptor без executable implementation блокируется `invalid_contract` до DB I/O.
|
- Добавление descriptor без executable implementation блокируется `invalid_contract` до DB I/O.
|
||||||
|
|||||||
@@ -192,6 +192,22 @@
|
|||||||
"source_digest": "052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe",
|
"source_digest": "052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe",
|
||||||
"transactional": true,
|
"transactional": true,
|
||||||
"version": 12
|
"version": 12
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"backfill": {
|
||||||
|
"kind": "none"
|
||||||
|
},
|
||||||
|
"checksum": "abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05",
|
||||||
|
"compatibility": "n-minus-one-readable",
|
||||||
|
"contract_evidence": null,
|
||||||
|
"name": "artifact-cleanup-indexes-v13",
|
||||||
|
"owner": "crank-registry",
|
||||||
|
"phase": "expand",
|
||||||
|
"readable_schema_max": 13,
|
||||||
|
"readable_schema_min": 12,
|
||||||
|
"source_digest": "abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05",
|
||||||
|
"transactional": true,
|
||||||
|
"version": 13
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,10 +21,17 @@ class SmokeError(RuntimeError):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def safe_error(stage: str, code: str, status: int | None = None) -> SmokeError:
|
def safe_error(
|
||||||
|
stage: str,
|
||||||
|
code: str,
|
||||||
|
status: int | None = None,
|
||||||
|
trace_id: str | None = None,
|
||||||
|
) -> SmokeError:
|
||||||
message = f"stage={stage[:64]} code={code[:96]}"
|
message = f"stage={stage[:64]} code={code[:96]}"
|
||||||
if status is not None:
|
if status is not None:
|
||||||
message += f" status={status}"
|
message += f" status={status}"
|
||||||
|
if trace_id is not None:
|
||||||
|
message += f" trace_id={trace_id}"
|
||||||
return SmokeError(message[:256])
|
return SmokeError(message[:256])
|
||||||
|
|
||||||
|
|
||||||
@@ -134,6 +141,7 @@ class Client:
|
|||||||
def __init__(self, base_url: str, timeout_seconds: int) -> None:
|
def __init__(self, base_url: str, timeout_seconds: int) -> None:
|
||||||
self.base_url = base_url.rstrip("/")
|
self.base_url = base_url.rstrip("/")
|
||||||
self.timeout_seconds = timeout_seconds
|
self.timeout_seconds = timeout_seconds
|
||||||
|
self.csrf_token: str | None = None
|
||||||
cookie_jar = http.cookiejar.CookieJar()
|
cookie_jar = http.cookiejar.CookieJar()
|
||||||
self.opener = urllib.request.build_opener(
|
self.opener = urllib.request.build_opener(
|
||||||
urllib.request.HTTPCookieProcessor(cookie_jar)
|
urllib.request.HTTPCookieProcessor(cookie_jar)
|
||||||
@@ -156,6 +164,21 @@ class Client:
|
|||||||
request_headers = {"Accept": "application/json"}
|
request_headers = {"Accept": "application/json"}
|
||||||
if headers:
|
if headers:
|
||||||
request_headers.update(headers)
|
request_headers.update(headers)
|
||||||
|
if (
|
||||||
|
self.csrf_token
|
||||||
|
and method not in ("GET", "HEAD", "OPTIONS")
|
||||||
|
and (
|
||||||
|
path_or_url.startswith("/api/admin/")
|
||||||
|
or path_or_url.startswith("/api/auth/")
|
||||||
|
)
|
||||||
|
and path_or_url
|
||||||
|
not in (
|
||||||
|
"/api/auth/login",
|
||||||
|
"/api/auth/bootstrap/complete",
|
||||||
|
"/api/auth/session/csrf",
|
||||||
|
)
|
||||||
|
):
|
||||||
|
request_headers.setdefault("x-csrf-token", self.csrf_token)
|
||||||
if payload is not None:
|
if payload is not None:
|
||||||
data = json.dumps(payload).encode("utf-8")
|
data = json.dumps(payload).encode("utf-8")
|
||||||
request_headers["Content-Type"] = "application/json"
|
request_headers["Content-Type"] = "application/json"
|
||||||
@@ -203,12 +226,21 @@ def admin_path(workspace_id: str, suffix: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def login(client: Client, email: str, password: str) -> None:
|
def login(client: Client, email: str, password: str) -> None:
|
||||||
client.request_json(
|
response = client.request_json(
|
||||||
"POST",
|
"POST",
|
||||||
"/api/auth/login",
|
"/api/auth/login",
|
||||||
{"email": email, "password": password},
|
{"email": email, "password": password},
|
||||||
expected=(200,),
|
expected=(200,),
|
||||||
)
|
)
|
||||||
|
session = require_object(response.body, "login")
|
||||||
|
csrf_token = session.get("csrf_token")
|
||||||
|
if (
|
||||||
|
not isinstance(csrf_token, str)
|
||||||
|
or not 32 <= len(csrf_token) <= 256
|
||||||
|
or not all(character.isalnum() or character in "-_." for character in csrf_token)
|
||||||
|
):
|
||||||
|
raise safe_error("login", "invalid_csrf_token")
|
||||||
|
client.csrf_token = csrf_token
|
||||||
|
|
||||||
|
|
||||||
def resolve_workspace(
|
def resolve_workspace(
|
||||||
@@ -261,6 +293,29 @@ def create_operation(
|
|||||||
raise safe_error("operation_create", "invalid_response") from error
|
raise safe_error("operation_create", "invalid_response") from error
|
||||||
|
|
||||||
|
|
||||||
|
def is_safe_diagnostic_identifier(value: Any) -> bool:
|
||||||
|
return (
|
||||||
|
isinstance(value, str)
|
||||||
|
and 1 <= len(value) <= 64
|
||||||
|
and value[0].islower()
|
||||||
|
and value[0].isascii()
|
||||||
|
and all(
|
||||||
|
character.isascii()
|
||||||
|
and (character.islower() or character.isdigit() or character == "_")
|
||||||
|
for character in value
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_safe_trace_id(value: Any) -> bool:
|
||||||
|
return (
|
||||||
|
isinstance(value, str)
|
||||||
|
and len(value) == 32
|
||||||
|
and value != "0" * 32
|
||||||
|
and all(character in "0123456789abcdef" for character in value)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def run_operation_test(
|
def run_operation_test(
|
||||||
client: Client,
|
client: Client,
|
||||||
workspace_id: str,
|
workspace_id: str,
|
||||||
@@ -272,9 +327,24 @@ def run_operation_test(
|
|||||||
admin_path(workspace_id, f"/operations/{operation_id}/test-runs"),
|
admin_path(workspace_id, f"/operations/{operation_id}/test-runs"),
|
||||||
{"version": operation_version, "input": {"probe": "ok"}},
|
{"version": operation_version, "input": {"probe": "ok"}},
|
||||||
).body
|
).body
|
||||||
if not isinstance(result, dict) or result.get("ok") is not True:
|
if isinstance(result, dict) and result.get("ok") is True:
|
||||||
|
return
|
||||||
|
if not isinstance(result, dict) or result.get("ok") is not False:
|
||||||
raise safe_error("operation_test", "outcome_not_ok")
|
raise safe_error("operation_test", "outcome_not_ok")
|
||||||
|
|
||||||
|
errors = result.get("errors")
|
||||||
|
failure = errors[0] if isinstance(errors, list) and errors else None
|
||||||
|
code = failure.get("code") if isinstance(failure, dict) else None
|
||||||
|
stage = failure.get("stage") if isinstance(failure, dict) else None
|
||||||
|
trace_id = result.get("trace_id")
|
||||||
|
if (
|
||||||
|
is_safe_diagnostic_identifier(code)
|
||||||
|
and is_safe_diagnostic_identifier(stage)
|
||||||
|
and is_safe_trace_id(trace_id)
|
||||||
|
):
|
||||||
|
raise safe_error(stage, code, trace_id=trace_id)
|
||||||
|
raise safe_error("operation_test", "outcome_not_ok")
|
||||||
|
|
||||||
|
|
||||||
def operation_etag(client: Client, workspace_id: str, operation_id: str) -> str:
|
def operation_etag(client: Client, workspace_id: str, operation_id: str) -> str:
|
||||||
response = client.request_json(
|
response = client.request_json(
|
||||||
@@ -326,6 +396,17 @@ def create_agent(client: Client, workspace_id: str, agent_slug: str) -> tuple[st
|
|||||||
raise safe_error("agent_create", "invalid_response") from error
|
raise safe_error("agent_create", "invalid_response") from error
|
||||||
|
|
||||||
|
|
||||||
|
def agent_etag(client: Client, workspace_id: str, agent_id: str) -> str:
|
||||||
|
response = client.request_json(
|
||||||
|
"GET",
|
||||||
|
admin_path(workspace_id, f"/agents/{agent_id}"),
|
||||||
|
)
|
||||||
|
etag = response.headers.get("ETag") if response.headers is not None else None
|
||||||
|
if not isinstance(etag, str) or len(etag) > 128 or not etag.startswith('"') or not etag.endswith('"'):
|
||||||
|
raise safe_error("agent_precondition", "invalid_response")
|
||||||
|
return etag
|
||||||
|
|
||||||
|
|
||||||
def edit_and_archive_operation(
|
def edit_and_archive_operation(
|
||||||
client: Client,
|
client: Client,
|
||||||
workspace_id: str,
|
workspace_id: str,
|
||||||
@@ -377,11 +458,13 @@ def bind_and_publish_agent(
|
|||||||
"enabled": True,
|
"enabled": True,
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
headers={"If-Match": agent_etag(client, workspace_id, agent_id)},
|
||||||
)
|
)
|
||||||
published = client.request_json(
|
published = client.request_json(
|
||||||
"POST",
|
"POST",
|
||||||
admin_path(workspace_id, f"/agents/{agent_id}/publish"),
|
admin_path(workspace_id, f"/agents/{agent_id}/publish"),
|
||||||
{"version": agent_version},
|
{"version": agent_version},
|
||||||
|
headers={"If-Match": agent_etag(client, workspace_id, agent_id)},
|
||||||
).body
|
).body
|
||||||
try:
|
try:
|
||||||
published_version = int(published["published_version"])
|
published_version = int(published["published_version"])
|
||||||
@@ -436,8 +519,15 @@ def cleanup_smoke_assets(
|
|||||||
if agent_id:
|
if agent_id:
|
||||||
try:
|
try:
|
||||||
client.request_json(
|
client.request_json(
|
||||||
"DELETE",
|
"POST",
|
||||||
admin_path(workspace_id, f"/agents/{agent_id}"),
|
admin_path(workspace_id, f"/agents/{agent_id}/unpublish"),
|
||||||
|
headers={"If-Match": agent_etag(client, workspace_id, agent_id)},
|
||||||
|
expected=(200, 404),
|
||||||
|
)
|
||||||
|
client.request_json(
|
||||||
|
"POST",
|
||||||
|
admin_path(workspace_id, f"/agents/{agent_id}/archive"),
|
||||||
|
headers={"If-Match": agent_etag(client, workspace_id, agent_id)},
|
||||||
expected=(200, 404),
|
expected=(200, 404),
|
||||||
)
|
)
|
||||||
except SmokeError as error:
|
except SmokeError as error:
|
||||||
|
|||||||
@@ -90,26 +90,52 @@ def playwright_verdict(report: dict[str, Any], required_titles: list[str]) -> tu
|
|||||||
return "fail", counts
|
return "fail", counts
|
||||||
tests = list(iter_tests(report))
|
tests = list(iter_tests(report))
|
||||||
if not tests:
|
if not tests:
|
||||||
|
if required_titles:
|
||||||
|
counts["failed"] = len(set(required_titles))
|
||||||
|
return "fail", counts
|
||||||
counts["not_run"] = 1
|
counts["not_run"] = 1
|
||||||
return "not_run", counts
|
return "not_run", counts
|
||||||
required = {title: False for title in required_titles}
|
required = {title: False for title in required_titles}
|
||||||
for test, title in tests:
|
for test, title in tests:
|
||||||
|
if required and title not in required:
|
||||||
|
continue
|
||||||
status = test.get("status")
|
status = test.get("status")
|
||||||
results = test.get("results") if isinstance(test.get("results"), list) else []
|
results = test.get("results")
|
||||||
result_statuses = [result.get("status") for result in results if isinstance(result, dict)]
|
if not isinstance(results, list):
|
||||||
retries = [result.get("retry", 0) for result in results if isinstance(result, dict)]
|
|
||||||
if len(result_statuses) != len(results):
|
|
||||||
counts["not_run"] += 1
|
counts["not_run"] += 1
|
||||||
continue
|
continue
|
||||||
if status == "flaky" or any(isinstance(retry, int) and retry > 0 for retry in retries):
|
result_statuses: list[Any] = []
|
||||||
counts["flaky"] += 1
|
retries: list[int] = []
|
||||||
|
malformed = False
|
||||||
|
for result in results:
|
||||||
|
if not isinstance(result, dict):
|
||||||
|
malformed = True
|
||||||
|
break
|
||||||
|
result_status = result.get("status")
|
||||||
|
retry = result.get("retry", 0)
|
||||||
|
if result_status not in ("passed", "failed", "timedOut", "skipped", "interrupted") \
|
||||||
|
or type(retry) is not int or retry < 0:
|
||||||
|
malformed = True
|
||||||
|
break
|
||||||
|
result_statuses.append(result_status)
|
||||||
|
retries.append(retry)
|
||||||
|
if malformed:
|
||||||
|
counts["not_run"] += 1
|
||||||
|
continue
|
||||||
|
final_status = result_statuses[-1] if result_statuses else None
|
||||||
|
if status in ("unexpected", "failed", "timedOut", "interrupted") \
|
||||||
|
or final_status in ("failed", "timedOut", "interrupted"):
|
||||||
|
counts["failed"] += 1
|
||||||
|
elif status == "flaky":
|
||||||
|
if final_status == "passed" and any(retry > 0 for retry in retries):
|
||||||
|
counts["flaky"] += 1
|
||||||
|
else:
|
||||||
|
counts["not_run"] += 1
|
||||||
elif status == "skipped" or (not results and status in ("skipped", "expected")):
|
elif status == "skipped" or (not results and status in ("skipped", "expected")):
|
||||||
counts["skipped"] += 1
|
counts["skipped"] += 1
|
||||||
elif status in ("unexpected", "failed", "timedOut", "interrupted") or any(
|
elif any(retry > 0 for retry in retries):
|
||||||
result_status in ("failed", "timedOut", "interrupted") for result_status in result_statuses
|
counts["flaky"] += 1
|
||||||
):
|
elif status == "expected" and results and all(result_status == "passed" for result_status in result_statuses):
|
||||||
counts["failed"] += 1
|
|
||||||
elif results and all(result_status == "passed" for result_status in result_statuses):
|
|
||||||
counts["passed"] += 1
|
counts["passed"] += 1
|
||||||
if title in required:
|
if title in required:
|
||||||
required[title] = True
|
required[title] = True
|
||||||
|
|||||||
+121
-8
@@ -26,13 +26,28 @@ compose_profiles=""
|
|||||||
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
||||||
compose_profiles="--profile cache"
|
compose_profiles="--profile cache"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
compose() {
|
compose() {
|
||||||
# Intentional word splitting: compose_profiles is either empty or two arguments.
|
# Intentional word splitting: compose_profiles is either empty or two arguments.
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
docker compose $compose_profiles "$@"
|
docker compose $compose_profiles "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
compose_up() {
|
||||||
|
postgres_scale=""
|
||||||
|
if [ "$(env_value POSTGRES_HOST postgres)" != "postgres" ] \
|
||||||
|
&& compose config --services | grep -Fxq postgres; then
|
||||||
|
postgres_scale="--scale postgres=0"
|
||||||
|
fi
|
||||||
|
# Intentional word splitting: postgres_scale is either empty or two arguments.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
compose up -d --remove-orphans $postgres_scale
|
||||||
|
}
|
||||||
|
|
||||||
|
show_failure_diagnostics() {
|
||||||
|
compose ps >&2 || true
|
||||||
|
compose logs --no-color migrate >&2 || true
|
||||||
|
}
|
||||||
|
|
||||||
wait_for_stack() {
|
wait_for_stack() {
|
||||||
readiness_path="$1"
|
readiness_path="$1"
|
||||||
attempt=1
|
attempt=1
|
||||||
@@ -48,6 +63,34 @@ wait_for_stack() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
find_artifact_container() {
|
||||||
|
expected_root="$1"
|
||||||
|
expected_mount="$(printf 'volume\t%s' "$expected_root")"
|
||||||
|
|
||||||
|
for service in admin-api artifact-storage-init; do
|
||||||
|
candidates="$(compose ps -aq "$service" 2>/dev/null || true)"
|
||||||
|
# Intentional word splitting: Docker container IDs cannot contain whitespace.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
set -- $candidates
|
||||||
|
if [ "$#" -gt 1 ]; then
|
||||||
|
echo "Artifact backup found multiple $service containers; refusing an ambiguous volume source" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [ "$#" -eq 1 ]; then
|
||||||
|
candidate="$1"
|
||||||
|
oneoff="$(docker inspect --format '{{index .Config.Labels "com.docker.compose.oneoff"}}' "$candidate" 2>/dev/null || true)"
|
||||||
|
mounts="$(docker inspect --format '{{range .Mounts}}{{printf "%s\t%s\n" .Type .Destination}}{{end}}' "$candidate" 2>/dev/null || true)"
|
||||||
|
if [ "$oneoff" != "True" ] && [ "$oneoff" != "true" ] \
|
||||||
|
&& printf '%s\n' "$mounts" | grep -Fxq "$expected_mount"; then
|
||||||
|
printf '%s' "$candidate"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
create_backup() {
|
create_backup() {
|
||||||
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
backup_dir="$(pwd)/backups/${timestamp}"
|
backup_dir="$(pwd)/backups/${timestamp}"
|
||||||
@@ -86,10 +129,9 @@ create_backup() {
|
|||||||
--username "$postgres_user" --dbname "$postgres_db" \
|
--username "$postgres_user" --dbname "$postgres_db" \
|
||||||
--format custom --file /backup/postgres.dump
|
--format custom --file /backup/postgres.dump
|
||||||
|
|
||||||
admin_container="$(compose ps -q admin-api 2>/dev/null || true)"
|
storage_root="$(env_value_from "$backup_env_file" CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
||||||
if [ -n "$admin_container" ]; then
|
if artifact_container="$(find_artifact_container "$storage_root")"; then
|
||||||
storage_root="$(env_value_from "$backup_env_file" CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
docker run --rm --volumes-from "$artifact_container:ro" \
|
||||||
docker run --rm --volumes-from "$admin_container" \
|
|
||||||
-v "$backup_dir:/backup" alpine:3.21 \
|
-v "$backup_dir:/backup" alpine:3.21 \
|
||||||
tar -C "$storage_root" -czf /backup/artifacts.tar.gz .
|
tar -C "$storage_root" -czf /backup/artifacts.tar.gz .
|
||||||
elif [ "$previous_deployment" = true ]; then
|
elif [ "$previous_deployment" = true ]; then
|
||||||
@@ -104,6 +146,76 @@ create_backup() {
|
|||||||
| sort -nr | awk 'NR > 5 { print $2 }' | xargs -r rm -rf
|
| sort -nr | awk 'NR > 5 { print $2 }' | xargs -r rm -rf
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validate_backup_migration() {
|
||||||
|
admin_image="$(env_value CRANK_ADMIN_API_IMAGE)"
|
||||||
|
if [ -z "$admin_image" ]; then
|
||||||
|
echo "CRANK_ADMIN_API_IMAGE is required for shadow migration validation" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
shadow_id="crank-migration-check-$$"
|
||||||
|
shadow_network="${shadow_id}-network"
|
||||||
|
shadow_postgres="${shadow_id}-postgres"
|
||||||
|
shadow_password="crank-shadow-migration-password"
|
||||||
|
shadow_admin="crank_shadow_admin"
|
||||||
|
shadow_user="crank_shadow_owner"
|
||||||
|
shadow_cleanup() {
|
||||||
|
docker rm -fv "$shadow_postgres" >/dev/null 2>&1 || true
|
||||||
|
docker network rm "$shadow_network" >/dev/null 2>&1 || true
|
||||||
|
}
|
||||||
|
trap shadow_cleanup EXIT
|
||||||
|
trap 'exit 130' HUP INT TERM
|
||||||
|
|
||||||
|
docker network create "$shadow_network" >/dev/null
|
||||||
|
docker run -d --name "$shadow_postgres" --network "$shadow_network" \
|
||||||
|
-e POSTGRES_USER="$shadow_admin" \
|
||||||
|
-e POSTGRES_PASSWORD="$shadow_password" \
|
||||||
|
-e POSTGRES_DB=crank \
|
||||||
|
postgres:16-alpine >/dev/null
|
||||||
|
|
||||||
|
shadow_ready=false
|
||||||
|
attempt=1
|
||||||
|
while [ "$attempt" -le 30 ]; do
|
||||||
|
if docker logs "$shadow_postgres" 2>&1 | grep -q 'PostgreSQL init process complete' \
|
||||||
|
&& docker exec "$shadow_postgres" pg_isready --username "$shadow_admin" --dbname crank >/dev/null 2>&1; then
|
||||||
|
shadow_ready=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
attempt=$((attempt + 1))
|
||||||
|
done
|
||||||
|
if [ "$shadow_ready" != true ]; then
|
||||||
|
echo "Shadow PostgreSQL did not become ready" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker exec "$shadow_postgres" psql --username "$shadow_admin" --dbname crank \
|
||||||
|
--set ON_ERROR_STOP=1 \
|
||||||
|
--command "create role ${shadow_user} login password '${shadow_password}' nosuperuser nocreatedb nocreaterole noreplication" \
|
||||||
|
--command "alter database crank owner to ${shadow_user}"
|
||||||
|
|
||||||
|
timeout 10m docker run --rm --network "$shadow_network" \
|
||||||
|
-e PGPASSWORD="$shadow_password" \
|
||||||
|
-v "$backup_dir:/backup:ro" \
|
||||||
|
postgres:16-alpine \
|
||||||
|
pg_restore --host "$shadow_postgres" --username "$shadow_user" --dbname crank \
|
||||||
|
--no-owner --no-privileges --single-transaction --exit-on-error \
|
||||||
|
/backup/postgres.dump
|
||||||
|
|
||||||
|
shadow_database_url="postgres://${shadow_user}:${shadow_password}@${shadow_postgres}:5432/crank"
|
||||||
|
timeout 10m docker run --rm --network "$shadow_network" \
|
||||||
|
-e CRANK_DATABASE_URL="$shadow_database_url" \
|
||||||
|
"$admin_image" crank-migrate apply
|
||||||
|
shadow_preflight="$(timeout 2m docker run --rm --network "$shadow_network" \
|
||||||
|
-e CRANK_DATABASE_URL="$shadow_database_url" \
|
||||||
|
"$admin_image" crank-migrate preflight)"
|
||||||
|
printf '%s\n' "$shadow_preflight" | grep -Fxq '{"status":"current","version":13}'
|
||||||
|
|
||||||
|
shadow_cleanup
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
echo "Shadow migration validation passed"
|
||||||
|
}
|
||||||
|
|
||||||
rollback() {
|
rollback() {
|
||||||
echo "New release failed readiness; restoring previous deployment" >&2
|
echo "New release failed readiness; restoring previous deployment" >&2
|
||||||
if [ ! -f .env.previous ] || [ ! -f docker-compose.previous.yml ]; then
|
if [ ! -f .env.previous ] || [ ! -f docker-compose.previous.yml ]; then
|
||||||
@@ -119,16 +231,17 @@ rollback() {
|
|||||||
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
||||||
compose_profiles="--profile cache"
|
compose_profiles="--profile cache"
|
||||||
fi
|
fi
|
||||||
compose up -d --remove-orphans
|
compose_up
|
||||||
wait_for_stack health
|
wait_for_stack health
|
||||||
}
|
}
|
||||||
|
|
||||||
compose config -q
|
compose config -q
|
||||||
create_backup
|
create_backup
|
||||||
compose pull
|
compose pull
|
||||||
|
validate_backup_migration
|
||||||
|
|
||||||
if ! compose up -d --remove-orphans || ! wait_for_stack ready; then
|
if ! compose_up || ! wait_for_stack ready; then
|
||||||
compose ps >&2 || true
|
show_failure_diagnostics
|
||||||
rollback
|
rollback
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
port="${1:?port is required}"
|
||||||
|
path="${2:?path is required}"
|
||||||
|
|
||||||
|
if [[ ! "$port" =~ ^[0-9]{1,5}$ ]] || (( port < 1 || port > 65535 )); then
|
||||||
|
exit 64
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$path" != /* || "$path" == *$'\r'* || "$path" == *$'\n'* ]]; then
|
||||||
|
exit 64
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Bash provides /dev/tcp without adding a network client package to the runtime
|
||||||
|
# image. Compose still applies its own five-second timeout to the whole probe.
|
||||||
|
exec 3<>"/dev/tcp/127.0.0.1/${port}"
|
||||||
|
printf 'GET %s HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n' "$path" >&3
|
||||||
|
|
||||||
|
IFS=$'\r' read -r -t 2 status <&3
|
||||||
|
case "$status" in
|
||||||
|
'HTTP/1.0 200 '*|'HTTP/1.1 200 '*) ;;
|
||||||
|
*) exit 1 ;;
|
||||||
|
esac
|
||||||
@@ -32,6 +32,15 @@ compose() {
|
|||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
docker compose $compose_profiles "$@"
|
docker compose $compose_profiles "$@"
|
||||||
}
|
}
|
||||||
|
compose_up() {
|
||||||
|
postgres_scale=""
|
||||||
|
if [ "$(env_value POSTGRES_HOST postgres)" != "postgres" ] \
|
||||||
|
&& compose config --services | grep -Fxq postgres; then
|
||||||
|
postgres_scale="--scale postgres=0"
|
||||||
|
fi
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
compose up -d --remove-orphans $postgres_scale
|
||||||
|
}
|
||||||
|
|
||||||
compose stop admin-api mcp-server ui
|
compose stop admin-api mcp-server ui
|
||||||
|
|
||||||
@@ -40,13 +49,18 @@ postgres_port="$(env_value POSTGRES_PORT 5432)"
|
|||||||
postgres_db="$(env_value POSTGRES_DB crank)"
|
postgres_db="$(env_value POSTGRES_DB crank)"
|
||||||
postgres_user="$(env_value POSTGRES_USER crank)"
|
postgres_user="$(env_value POSTGRES_USER crank)"
|
||||||
postgres_password="$(env_value POSTGRES_PASSWORD)"
|
postgres_password="$(env_value POSTGRES_PASSWORD)"
|
||||||
|
docker run --rm \
|
||||||
|
-v "$backup_dir:/backup:ro" \
|
||||||
|
postgres:16-alpine \
|
||||||
|
pg_restore --list /backup/postgres.dump >/dev/null
|
||||||
docker run --rm --network host \
|
docker run --rm --network host \
|
||||||
-e PGPASSWORD="$postgres_password" \
|
-e PGPASSWORD="$postgres_password" \
|
||||||
-v "$backup_dir:/backup:ro" \
|
-v "$backup_dir:/backup:ro" \
|
||||||
postgres:16-alpine \
|
postgres:16-alpine \
|
||||||
pg_restore --host "$postgres_host" --port "$postgres_port" \
|
pg_restore --host "$postgres_host" --port "$postgres_port" \
|
||||||
--username "$postgres_user" --dbname "$postgres_db" \
|
--username "$postgres_user" --dbname "$postgres_db" \
|
||||||
--clean --if-exists --no-owner --no-privileges /backup/postgres.dump
|
--clean --if-exists --no-owner --no-privileges \
|
||||||
|
--single-transaction --exit-on-error /backup/postgres.dump
|
||||||
|
|
||||||
admin_container="$(compose ps -aq admin-api)"
|
admin_container="$(compose ps -aq admin-api)"
|
||||||
storage_root="$(env_value CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
storage_root="$(env_value CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
||||||
@@ -54,7 +68,7 @@ docker run --rm --volumes-from "$admin_container" \
|
|||||||
-v "$backup_dir:/backup:ro" alpine:3.21 sh -eu -c \
|
-v "$backup_dir:/backup:ro" alpine:3.21 sh -eu -c \
|
||||||
"find '$storage_root' -mindepth 1 -delete; tar -C '$storage_root' -xzf /backup/artifacts.tar.gz"
|
"find '$storage_root' -mindepth 1 -delete; tar -C '$storage_root' -xzf /backup/artifacts.tar.gz"
|
||||||
|
|
||||||
compose up -d --remove-orphans
|
compose_up
|
||||||
attempt=1
|
attempt=1
|
||||||
while [ "$attempt" -le 45 ]; do
|
while [ "$attempt" -le 45 ]; do
|
||||||
if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \
|
if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \
|
||||||
|
|||||||
@@ -17,6 +17,48 @@ def load_smoke_module():
|
|||||||
|
|
||||||
|
|
||||||
class AuthenticatedProductSmokeTests(unittest.TestCase):
|
class AuthenticatedProductSmokeTests(unittest.TestCase):
|
||||||
|
def test_client_attaches_csrf_only_to_browser_api_mutations(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
|
||||||
|
class Response:
|
||||||
|
status = 200
|
||||||
|
headers = {}
|
||||||
|
|
||||||
|
def read(self, _limit):
|
||||||
|
return b"{}"
|
||||||
|
|
||||||
|
class Opener:
|
||||||
|
def __init__(self):
|
||||||
|
self.requests = []
|
||||||
|
|
||||||
|
def open(self, request, timeout):
|
||||||
|
self.requests.append((request, timeout))
|
||||||
|
return Response()
|
||||||
|
|
||||||
|
client = smoke.Client("http://crank.test", 5)
|
||||||
|
opener = Opener()
|
||||||
|
client.opener = opener
|
||||||
|
client.csrf_token = "a" * 32
|
||||||
|
|
||||||
|
client.request_json("POST", "/api/admin/workspaces/ws/operations", {})
|
||||||
|
client.request_json("POST", "http://mcp.test/v1/ws/agent", {})
|
||||||
|
|
||||||
|
self.assertEqual(opener.requests[0][0].get_header("X-csrf-token"), "a" * 32)
|
||||||
|
self.assertIsNone(opener.requests[1][0].get_header("X-csrf-token"))
|
||||||
|
|
||||||
|
def test_login_keeps_server_issued_csrf_token(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
csrf_token = None
|
||||||
|
|
||||||
|
def request_json(self, *args, **kwargs):
|
||||||
|
return smoke.JsonResponse(200, {}, {"csrf_token": "b" * 32})
|
||||||
|
|
||||||
|
client = FakeClient()
|
||||||
|
smoke.login(client, "owner@crank.test", "safe-password")
|
||||||
|
self.assertEqual(client.csrf_token, "b" * 32)
|
||||||
|
|
||||||
def test_operation_payload_uses_internal_upstream(self) -> None:
|
def test_operation_payload_uses_internal_upstream(self) -> None:
|
||||||
smoke = load_smoke_module()
|
smoke = load_smoke_module()
|
||||||
|
|
||||||
@@ -136,6 +178,8 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
|||||||
return smoke.JsonResponse(200, {}, {"operation_id": "op_safe", "version": 7})
|
return smoke.JsonResponse(200, {}, {"operation_id": "op_safe", "version": 7})
|
||||||
if path.endswith("/operations/op_safe"):
|
if path.endswith("/operations/op_safe"):
|
||||||
return smoke.JsonResponse(200, {"ETag": '"safe-etag"'}, {"id": "op_safe"})
|
return smoke.JsonResponse(200, {"ETag": '"safe-etag"'}, {"id": "op_safe"})
|
||||||
|
if path.endswith("/agents/agent_safe"):
|
||||||
|
return smoke.JsonResponse(200, {"ETag": '"safe-agent-etag"'}, {"id": "agent_safe"})
|
||||||
if path.endswith("/publish") and "/operations/" in path:
|
if path.endswith("/publish") and "/operations/" in path:
|
||||||
return smoke.JsonResponse(200, {}, {"published_version": 7})
|
return smoke.JsonResponse(200, {}, {"published_version": 7})
|
||||||
if path.endswith("/agents"):
|
if path.endswith("/agents"):
|
||||||
@@ -158,11 +202,14 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
|||||||
self.assertEqual((agent_id, agent_version, published_agent_version), ("agent_safe", 3, 3))
|
self.assertEqual((agent_id, agent_version, published_agent_version), ("agent_safe", 3, 3))
|
||||||
binding = next(payload for _, path, payload, _ in client.requests if path.endswith("/bindings"))[0]
|
binding = next(payload for _, path, payload, _ in client.requests if path.endswith("/bindings"))[0]
|
||||||
self.assertEqual(binding["operation_version"], 7)
|
self.assertEqual(binding["operation_version"], 7)
|
||||||
|
binding_request = next(request for request in client.requests if request[1].endswith("/bindings"))
|
||||||
|
self.assertEqual(binding_request[3]["headers"], {"If-Match": '"safe-agent-etag"'})
|
||||||
publish = next(request for request in client.requests if request[1].endswith("/operations/op_safe/publish"))
|
publish = next(request for request in client.requests if request[1].endswith("/operations/op_safe/publish"))
|
||||||
self.assertEqual(publish[3]["headers"], {"If-Match": '"safe-etag"'})
|
self.assertEqual(publish[3]["headers"], {"If-Match": '"safe-etag"'})
|
||||||
|
|
||||||
def test_admin_test_run_uses_created_version_and_rejects_failed_outcome(self) -> None:
|
def test_admin_test_run_uses_created_version_and_reports_safe_typed_failure(self) -> None:
|
||||||
smoke = load_smoke_module()
|
smoke = load_smoke_module()
|
||||||
|
trace_id = "0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
class FakeClient:
|
class FakeClient:
|
||||||
def __init__(self, ok):
|
def __init__(self, ok):
|
||||||
@@ -171,7 +218,22 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
|||||||
|
|
||||||
def request_json(self, method, path, payload=None, **kwargs):
|
def request_json(self, method, path, payload=None, **kwargs):
|
||||||
self.payload = payload
|
self.payload = payload
|
||||||
return smoke.JsonResponse(200, {}, {"ok": self.ok, "errors": [{"message": "secret-canary"}]})
|
return smoke.JsonResponse(
|
||||||
|
200,
|
||||||
|
{},
|
||||||
|
{
|
||||||
|
"ok": self.ok,
|
||||||
|
"trace_id": trace_id,
|
||||||
|
"errors": [
|
||||||
|
{
|
||||||
|
"code": "outbound_target_rejected",
|
||||||
|
"stage": "adapter",
|
||||||
|
"message": "secret-canary",
|
||||||
|
"context": {"url": "https://private.invalid/token-canary"},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
passing = FakeClient(True)
|
passing = FakeClient(True)
|
||||||
smoke.run_operation_test(passing, "ws", "op", 9)
|
smoke.run_operation_test(passing, "ws", "op", 9)
|
||||||
@@ -179,7 +241,188 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
|||||||
|
|
||||||
with self.assertRaises(smoke.SmokeError) as raised:
|
with self.assertRaises(smoke.SmokeError) as raised:
|
||||||
smoke.run_operation_test(FakeClient(False), "ws", "op", 9)
|
smoke.run_operation_test(FakeClient(False), "ws", "op", 9)
|
||||||
self.assertNotIn("secret-canary", str(raised.exception))
|
self.assertEqual(
|
||||||
|
str(raised.exception),
|
||||||
|
"stage=adapter code=outbound_target_rejected trace_id=0123456789abcdef0123456789abcdef",
|
||||||
|
)
|
||||||
|
for forbidden in ("secret-canary", "private.invalid", "token-canary", "context"):
|
||||||
|
self.assertNotIn(forbidden, str(raised.exception))
|
||||||
|
|
||||||
|
def test_admin_test_run_fails_closed_for_malformed_diagnostics(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
valid_trace_id = "0123456789abcdef0123456789abcdef"
|
||||||
|
valid_failure = {"code": "upstream_timeout", "stage": "upstream"}
|
||||||
|
malformed_results = (
|
||||||
|
None,
|
||||||
|
[],
|
||||||
|
"secret-canary",
|
||||||
|
{"trace_id": valid_trace_id, "errors": [valid_failure]},
|
||||||
|
{"ok": None, "trace_id": valid_trace_id, "errors": [valid_failure]},
|
||||||
|
{"ok": 0, "trace_id": valid_trace_id, "errors": [valid_failure]},
|
||||||
|
{"ok": "false", "trace_id": valid_trace_id, "errors": [valid_failure]},
|
||||||
|
{"ok": False, "errors": []},
|
||||||
|
{"ok": False, "errors": {}},
|
||||||
|
{"ok": False, "trace_id": valid_trace_id, "errors": [None]},
|
||||||
|
{"ok": False, "errors": ["secret-canary"]},
|
||||||
|
{"ok": False, "errors": [valid_failure]},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id[:-1],
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id + "0",
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": "0" * 32,
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id.upper(),
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": "a" * 31 + "١",
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": "a" * 31 + "\n",
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": 42,
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "a" * 65, "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "Upstream_timeout", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_таймаут", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": 42, "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": ""}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "a" * 65}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "Upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "вверх"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": 42}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "unsafe-value://secret-canary", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "unsafe stage secret-canary"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": "not-a-trace-id-secret-canary",
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def __init__(self, result):
|
||||||
|
self.result = result
|
||||||
|
|
||||||
|
def request_json(self, *args, **kwargs):
|
||||||
|
return smoke.JsonResponse(200, {}, self.result)
|
||||||
|
|
||||||
|
for result in malformed_results:
|
||||||
|
with self.subTest(result=result), self.assertRaises(smoke.SmokeError) as raised:
|
||||||
|
smoke.run_operation_test(FakeClient(result), "ws", "op", 9)
|
||||||
|
self.assertEqual(str(raised.exception), "stage=operation_test code=outcome_not_ok")
|
||||||
|
self.assertNotIn("secret-canary", str(raised.exception))
|
||||||
|
|
||||||
|
def test_admin_test_run_accepts_maximum_length_safe_diagnostics(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
identifier = "a" * 64
|
||||||
|
trace_id = "0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def request_json(self, *args, **kwargs):
|
||||||
|
return smoke.JsonResponse(
|
||||||
|
200,
|
||||||
|
{},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": trace_id,
|
||||||
|
"request_preview": {"credential": "secret-canary"},
|
||||||
|
"response_preview": {"token": "secret-canary"},
|
||||||
|
"errors": [{"code": identifier, "stage": identifier}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaises(smoke.SmokeError) as raised:
|
||||||
|
smoke.run_operation_test(FakeClient(), "ws", "op", 9)
|
||||||
|
rendered = str(raised.exception)
|
||||||
|
self.assertEqual(rendered, f"stage={identifier} code={identifier} trace_id={trace_id}")
|
||||||
|
self.assertLessEqual(len(rendered.encode("ascii")), 256)
|
||||||
|
self.assertNotIn("secret-canary", rendered)
|
||||||
|
|
||||||
|
def test_admin_test_run_success_ignores_malformed_diagnostics(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def request_json(self, *args, **kwargs):
|
||||||
|
return smoke.JsonResponse(
|
||||||
|
200,
|
||||||
|
{},
|
||||||
|
{
|
||||||
|
"ok": True,
|
||||||
|
"trace_id": "secret-canary",
|
||||||
|
"errors": "secret-canary",
|
||||||
|
"request_preview": {"token": "secret-canary"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
smoke.run_operation_test(FakeClient(), "ws", "op", 9)
|
||||||
|
|
||||||
def test_edit_and_archive_use_fresh_operation_preconditions(self) -> None:
|
def test_edit_and_archive_use_fresh_operation_preconditions(self) -> None:
|
||||||
smoke = load_smoke_module()
|
smoke = load_smoke_module()
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
|
|||||||
def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None:
|
def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None:
|
||||||
required = ["OpenAPI required scenario"]
|
required = ["OpenAPI required scenario"]
|
||||||
reports = [
|
reports = [
|
||||||
|
{"suites": []},
|
||||||
{"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]},
|
{"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]},
|
||||||
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]},
|
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]},
|
||||||
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]},
|
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]},
|
||||||
@@ -70,6 +71,73 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
|
|||||||
self.assertEqual(candidate["execution_verdict"], "fail")
|
self.assertEqual(candidate["execution_verdict"], "fail")
|
||||||
self.assertFalse(candidate["accepted"])
|
self.assertFalse(candidate["accepted"])
|
||||||
|
|
||||||
|
def test_required_scope_ignores_unrelated_flaky_tests(self) -> None:
|
||||||
|
required = "OpenAPI required scenario"
|
||||||
|
report = {
|
||||||
|
"suites": [
|
||||||
|
{
|
||||||
|
"specs": [
|
||||||
|
{
|
||||||
|
"title": required,
|
||||||
|
"tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Unrelated wizard scenario",
|
||||||
|
"tests": [
|
||||||
|
{
|
||||||
|
"status": "flaky",
|
||||||
|
"results": [
|
||||||
|
{"status": "failed", "retry": 0},
|
||||||
|
{"status": "passed", "retry": 1},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{"title": "Unrelated failure", "tests": [{"status": "unexpected", "results": [{"status": "failed", "retry": 0}]}]},
|
||||||
|
{"title": "Unrelated skip", "tests": [{"status": "skipped", "results": []}]},
|
||||||
|
{"title": "Unrelated malformed", "tests": [{"status": "expected", "results": None}]},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
result, output, temporary = self.run_playwright(report, [required])
|
||||||
|
self.addCleanup(temporary.cleanup)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
||||||
|
self.assertEqual(candidate["execution_verdict"], "pass")
|
||||||
|
self.assertTrue(candidate["accepted"])
|
||||||
|
self.assertEqual(candidate["summary"], {"passed": 1, "failed": 0, "flaky": 0, "skipped": 0, "not_run": 0})
|
||||||
|
|
||||||
|
def test_final_failure_after_retry_is_failed_not_flaky(self) -> None:
|
||||||
|
for status in ("unexpected", "flaky"):
|
||||||
|
with self.subTest(status=status):
|
||||||
|
report = {
|
||||||
|
"suites": [
|
||||||
|
{
|
||||||
|
"specs": [
|
||||||
|
{
|
||||||
|
"tests": [
|
||||||
|
{
|
||||||
|
"status": status,
|
||||||
|
"results": [
|
||||||
|
{"status": "failed", "retry": 0},
|
||||||
|
{"status": "failed", "retry": 1},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
result, output, temporary = self.run_playwright(report)
|
||||||
|
self.addCleanup(temporary.cleanup)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
||||||
|
self.assertEqual(candidate["execution_verdict"], "fail")
|
||||||
|
self.assertEqual(candidate["summary"]["failed"], 1)
|
||||||
|
self.assertEqual(candidate["summary"]["flaky"], 0)
|
||||||
|
|
||||||
def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
|
def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
|
||||||
canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
|
canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
|
||||||
report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
|
report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
|
||||||
@@ -84,6 +152,8 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
|
|||||||
reports = [
|
reports = [
|
||||||
{"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
|
{"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
|
||||||
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
|
||||||
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": None}]}]}]},
|
||||||
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": True}]}]}]}]},
|
||||||
]
|
]
|
||||||
for report in reports:
|
for report in reports:
|
||||||
with self.subTest(report=report):
|
with self.subTest(report=report):
|
||||||
|
|||||||
Reference in New Issue
Block a user