Owner-authorized recovery of accumulated source changes.
Includes accepted work and in-progress changes; NOT a release or acceptance.
Parent baseline 1b6db09bd1. Runtime data, build caches and local credentials excluded.
348 lines
12 KiB
Rust
348 lines
12 KiB
Rust
//! Story 3.8 / AT-3.8-04: the live PostgreSQL gate is authoritative. A warm catalog cache must
|
|
//! never serve a Resource when the gate itself fails, a corrupt page must never become a partial
|
|
//! success, and a commit-time audit failure must never look like a successful revoke.
|
|
//!
|
|
//! All three failure modes are injected into the real test schema and the real HTTP/MCP stack; no
|
|
//! product seam is stubbed and no assertion is relaxed.
|
|
|
|
use crank_artifacts::test_support::{artifact_reads, reset_artifact_reads};
|
|
use reqwest::StatusCode;
|
|
use serde_json::Value;
|
|
|
|
use super::helpers::read_response;
|
|
use super::revocation::{
|
|
KEPT_URI, LEGACY_REVISION, MODERN_REVISION, NOT_AVAILABLE_CODE, REVOKED_CANARY,
|
|
REVOKED_CANARY_V2, REVOKED_NAME, REVOKED_URI, listed_uris, mcp_list, mcp_read, read_text,
|
|
spawn_revocation_stand, spawn_revocation_stand_with_refresh, supported_revisions,
|
|
};
|
|
use super::revocation_security::revoke;
|
|
use std::time::Duration;
|
|
|
|
fn assert_internal_unavailable(body: &Value, what: &str) {
|
|
assert!(
|
|
body.get("result").is_none(),
|
|
"{what} must fail closed instead of returning data: {body}"
|
|
);
|
|
assert_eq!(
|
|
body["error"]["code"].as_i64(),
|
|
Some(-32603),
|
|
"{what} must answer the fixed internal error, not the safe NotAvailable shape: {body}"
|
|
);
|
|
let text = body.to_string();
|
|
for leak in [
|
|
REVOKED_CANARY.trim(),
|
|
REVOKED_NAME,
|
|
REVOKED_URI,
|
|
"text/plain",
|
|
] {
|
|
assert!(
|
|
!text.contains(leak),
|
|
"{what} must not disclose Resource data ({leak}): {body}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A warmed catalog cache plus a failing tombstone/authority read: list and read fail closed, the
|
|
/// artifact store is not touched, and the positive control still works once the gate is restored.
|
|
#[tokio::test]
|
|
async fn story38_gate_storage_failure_after_a_warm_cache_never_serves_cached_content() {
|
|
// A non-zero refresh interval is what makes "warmed cache" meaningful.
|
|
let stand = spawn_revocation_stand_with_refresh("gate_failure", Duration::from_secs(60)).await;
|
|
|
|
for revision in supported_revisions() {
|
|
let first = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url,
|
|
&stand.mcp_key,
|
|
&revision,
|
|
REVOKED_URI,
|
|
)
|
|
.await;
|
|
assert_eq!(read_text(&first).as_deref(), Some(REVOKED_CANARY));
|
|
let second = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url_second,
|
|
&stand.mcp_key_second,
|
|
&revision,
|
|
REVOKED_URI,
|
|
)
|
|
.await;
|
|
assert_eq!(read_text(&second).as_deref(), Some(REVOKED_CANARY_V2));
|
|
for (url, key) in [
|
|
(&stand.mcp_url, &stand.mcp_key),
|
|
(&stand.mcp_url_second, &stand.mcp_key_second),
|
|
] {
|
|
let listed = listed_uris(&mcp_list(&stand.mcp, url, key, &revision).await);
|
|
assert!(listed.contains(&REVOKED_URI.to_owned()));
|
|
}
|
|
}
|
|
|
|
sqlx::raw_sql(
|
|
"alter table resource_security_revocations rename to resource_security_revocations_offline",
|
|
)
|
|
.execute(stand.registry.pool())
|
|
.await
|
|
.expect("the tombstone read must be breakable in the test schema");
|
|
|
|
reset_artifact_reads();
|
|
for revision in supported_revisions() {
|
|
for (agent, url, key) in [
|
|
("first", &stand.mcp_url, &stand.mcp_key),
|
|
("second", &stand.mcp_url_second, &stand.mcp_key_second),
|
|
] {
|
|
let read = mcp_read(&stand.mcp, url, key, &revision, REVOKED_URI).await;
|
|
assert_internal_unavailable(&read, &format!("{agent} read on {revision}"));
|
|
assert_ne!(
|
|
read["error"]["code"].as_i64(),
|
|
Some(NOT_AVAILABLE_CODE),
|
|
"{agent} on {revision}: a storage failure must not be disguised as 'not available'"
|
|
);
|
|
let listed = mcp_list(&stand.mcp, url, key, &revision).await;
|
|
assert_internal_unavailable(&listed, &format!("{agent} list on {revision}"));
|
|
}
|
|
}
|
|
assert_eq!(
|
|
artifact_reads(),
|
|
0,
|
|
"a failing gate must not fall back to cached content or artifact bytes"
|
|
);
|
|
|
|
sqlx::raw_sql(
|
|
"alter table resource_security_revocations_offline rename to resource_security_revocations",
|
|
)
|
|
.execute(stand.registry.pool())
|
|
.await
|
|
.expect("the tombstone table must be restorable");
|
|
|
|
// Positive control: the same endpoints serve the Resource again; nothing was revoked falsely.
|
|
let restored = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url,
|
|
&stand.mcp_key,
|
|
MODERN_REVISION,
|
|
REVOKED_URI,
|
|
)
|
|
.await;
|
|
assert_eq!(read_text(&restored).as_deref(), Some(REVOKED_CANARY));
|
|
assert!(
|
|
listed_uris(&mcp_list(&stand.mcp, &stand.mcp_url, &stand.mcp_key, LEGACY_REVISION).await)
|
|
.contains(&REVOKED_URI.to_owned())
|
|
);
|
|
}
|
|
|
|
/// R1 regression: a corrupt bound publication must not be silently reported as "the URI does not
|
|
/// exist". The typed integrity error is preserved and answered with the safe unavailable outcome
|
|
/// before any artifact byte, while another bound Resource stays readable.
|
|
#[tokio::test]
|
|
async fn story38_resolver_integrity_failure_is_not_disguised_as_a_missing_uri() {
|
|
let stand = spawn_revocation_stand("resolver_integrity").await;
|
|
let pool = stand.registry.pool();
|
|
sqlx::query(
|
|
"update resource_publications
|
|
set size_bytes = size_bytes + 1
|
|
where workspace_id = 'ws_default' and resource_id = $1 and version = 1",
|
|
)
|
|
.bind(&stand.revoked_id)
|
|
.execute(pool)
|
|
.await
|
|
.expect("the bound publication row must be corruptible in the test schema");
|
|
|
|
reset_artifact_reads();
|
|
for revision in supported_revisions() {
|
|
let denied = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url,
|
|
&stand.mcp_key,
|
|
&revision,
|
|
REVOKED_URI,
|
|
)
|
|
.await;
|
|
assert_internal_unavailable(&denied, &format!("corrupt resolver read on {revision}"));
|
|
assert_ne!(
|
|
denied["error"]["code"].as_i64(),
|
|
Some(NOT_AVAILABLE_CODE),
|
|
"{revision}: a corrupt publication is not a missing URI"
|
|
);
|
|
}
|
|
assert_eq!(
|
|
artifact_reads(),
|
|
0,
|
|
"an integrity failure must not open artifact bytes"
|
|
);
|
|
|
|
// The whole bound snapshot is unverifiable, so every read from this Agent fails closed
|
|
// (the same rule the list page uses) instead of serving the remaining publications.
|
|
let other = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url,
|
|
&stand.mcp_key,
|
|
MODERN_REVISION,
|
|
KEPT_URI,
|
|
)
|
|
.await;
|
|
assert_internal_unavailable(&other, "a read while one bound publication is corrupt");
|
|
assert_eq!(
|
|
artifact_reads(),
|
|
0,
|
|
"an unverifiable snapshot must not open any artifact bytes"
|
|
);
|
|
|
|
sqlx::query(
|
|
"update resource_publications
|
|
set size_bytes = size_bytes - 1
|
|
where workspace_id = 'ws_default' and resource_id = $1 and version = 1",
|
|
)
|
|
.bind(&stand.revoked_id)
|
|
.execute(pool)
|
|
.await
|
|
.expect("the publication row must be restorable");
|
|
let restored = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url,
|
|
&stand.mcp_key,
|
|
MODERN_REVISION,
|
|
REVOKED_URI,
|
|
)
|
|
.await;
|
|
assert_eq!(read_text(&restored).as_deref(), Some(REVOKED_CANARY));
|
|
let restored_control = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url,
|
|
&stand.mcp_key,
|
|
MODERN_REVISION,
|
|
KEPT_URI,
|
|
)
|
|
.await;
|
|
assert_eq!(
|
|
read_text(&restored_control).as_deref(),
|
|
Some(super::revocation::KEPT_CANARY)
|
|
);
|
|
assert!(
|
|
artifact_reads() >= 2,
|
|
"the restored reads must be observable as artifact reads"
|
|
);
|
|
}
|
|
|
|
/// One corrupt publication on the requested page must fail the whole page, not silently drop an
|
|
/// element (a partial page would look like a successful, complete answer).
|
|
#[tokio::test]
|
|
async fn story38_corrupt_list_publication_never_returns_a_partial_page() {
|
|
let stand = spawn_revocation_stand("list_corruption").await;
|
|
let pool = stand.registry.pool();
|
|
|
|
let before =
|
|
listed_uris(&mcp_list(&stand.mcp, &stand.mcp_url, &stand.mcp_key, MODERN_REVISION).await);
|
|
assert!(before.contains(&REVOKED_URI.to_owned()) && before.contains(&KEPT_URI.to_owned()));
|
|
|
|
sqlx::query(
|
|
"update resource_publications
|
|
set size_bytes = size_bytes + 1
|
|
where workspace_id = 'ws_default' and resource_id = $1",
|
|
)
|
|
.bind(&stand.kept_id)
|
|
.execute(pool)
|
|
.await
|
|
.expect("the publication row must be corruptible in the test schema");
|
|
|
|
for revision in supported_revisions() {
|
|
let listed = mcp_list(&stand.mcp, &stand.mcp_url, &stand.mcp_key, &revision).await;
|
|
assert_internal_unavailable(&listed, &format!("corrupt list on {revision}"));
|
|
assert!(
|
|
listed.get("result").is_none(),
|
|
"{revision}: a corrupt page must not be reported as a (partial) success: {listed}"
|
|
);
|
|
}
|
|
// A read of the still-consistent Resource is unaffected by the page failure.
|
|
let readable = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url,
|
|
&stand.mcp_key,
|
|
MODERN_REVISION,
|
|
KEPT_URI,
|
|
)
|
|
.await;
|
|
assert_internal_unavailable(&readable, "a read of the corrupt publication");
|
|
|
|
sqlx::query(
|
|
"update resource_publications
|
|
set size_bytes = size_bytes - 1
|
|
where workspace_id = 'ws_default' and resource_id = $1",
|
|
)
|
|
.bind(&stand.kept_id)
|
|
.execute(pool)
|
|
.await
|
|
.expect("the publication row must be restorable");
|
|
let after =
|
|
listed_uris(&mcp_list(&stand.mcp, &stand.mcp_url, &stand.mcp_key, MODERN_REVISION).await);
|
|
assert_eq!(
|
|
after, before,
|
|
"after restoring the row the page must serve exactly the same elements again"
|
|
);
|
|
}
|
|
|
|
/// A failing COMMIT of the audited revoke is a sanitized 500 — never a 204 and never an
|
|
/// unaudited state change; the bounded retry then succeeds with exactly one event.
|
|
#[tokio::test]
|
|
async fn story38_commit_time_failure_never_answers_a_false_204() {
|
|
let stand = spawn_revocation_stand("commit_failure").await;
|
|
let pool = stand.registry.pool();
|
|
sqlx::raw_sql(
|
|
"create function reject_story38_revocation_commit() returns trigger language plpgsql as $$
|
|
begin
|
|
raise exception 'injected revocation commit failure' using errcode = '23514';
|
|
end $$;
|
|
create constraint trigger reject_story38_revocation_commit_trigger
|
|
after insert on resource_security_revocations
|
|
deferrable initially deferred
|
|
for each row execute function reject_story38_revocation_commit();",
|
|
)
|
|
.execute(pool)
|
|
.await
|
|
.expect("the deferred commit-time failure trigger must install");
|
|
|
|
let (status, text, body) =
|
|
read_response(revoke(&stand.admin, &stand.server, &stand.revoked_id).await).await;
|
|
assert_eq!(
|
|
status,
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
"a failed commit must be a sanitized storage error, not a false 204: {text}"
|
|
);
|
|
assert!(
|
|
!text.contains("injected") && !text.contains("resource_security_revocations"),
|
|
"the storage failure must not disclose SQL details: {body}"
|
|
);
|
|
assert_eq!(
|
|
super::helpers::table_row_count(pool, "resource_security_revocations").await,
|
|
0,
|
|
"the failed transaction must leave no audit event"
|
|
);
|
|
// The Resource is still readable: the failed revoke must not have partially applied.
|
|
let readable = mcp_read(
|
|
&stand.mcp,
|
|
&stand.mcp_url,
|
|
&stand.mcp_key,
|
|
MODERN_REVISION,
|
|
REVOKED_URI,
|
|
)
|
|
.await;
|
|
assert_eq!(read_text(&readable).as_deref(), Some(REVOKED_CANARY));
|
|
|
|
sqlx::raw_sql(
|
|
"drop trigger reject_story38_revocation_commit_trigger on resource_security_revocations;
|
|
drop function reject_story38_revocation_commit();",
|
|
)
|
|
.execute(pool)
|
|
.await
|
|
.expect("the failure trigger must be removable");
|
|
let retried = revoke(&stand.admin, &stand.server, &stand.revoked_id).await;
|
|
assert_eq!(
|
|
retried.status(),
|
|
StatusCode::NO_CONTENT,
|
|
"a bounded retry after a failed commit must succeed"
|
|
);
|
|
assert_eq!(
|
|
super::helpers::table_row_count(pool, "resource_security_revocations").await,
|
|
1
|
|
);
|
|
}
|