Files
crank/apps/admin-api/tests/integration/resources/revocation_failures.rs
Hermes DevOps 75fe5a123c [skip ci] wip: preserve Hermes crank development checkpoint
Owner-authorized recovery of accumulated source changes.
Includes accepted work and in-progress changes; NOT a release or acceptance.
Parent baseline 1b6db09bd1. Runtime data, build caches and local credentials excluded.
2026-10-25 10:00:22 +00:00

348 lines
12 KiB
Rust

//! Story 3.8 / AT-3.8-04: the live PostgreSQL gate is authoritative. A warm catalog cache must
//! never serve a Resource when the gate itself fails, a corrupt page must never become a partial
//! success, and a commit-time audit failure must never look like a successful revoke.
//!
//! All three failure modes are injected into the real test schema and the real HTTP/MCP stack; no
//! product seam is stubbed and no assertion is relaxed.
use crank_artifacts::test_support::{artifact_reads, reset_artifact_reads};
use reqwest::StatusCode;
use serde_json::Value;
use super::helpers::read_response;
use super::revocation::{
KEPT_URI, LEGACY_REVISION, MODERN_REVISION, NOT_AVAILABLE_CODE, REVOKED_CANARY,
REVOKED_CANARY_V2, REVOKED_NAME, REVOKED_URI, listed_uris, mcp_list, mcp_read, read_text,
spawn_revocation_stand, spawn_revocation_stand_with_refresh, supported_revisions,
};
use super::revocation_security::revoke;
use std::time::Duration;
fn assert_internal_unavailable(body: &Value, what: &str) {
assert!(
body.get("result").is_none(),
"{what} must fail closed instead of returning data: {body}"
);
assert_eq!(
body["error"]["code"].as_i64(),
Some(-32603),
"{what} must answer the fixed internal error, not the safe NotAvailable shape: {body}"
);
let text = body.to_string();
for leak in [
REVOKED_CANARY.trim(),
REVOKED_NAME,
REVOKED_URI,
"text/plain",
] {
assert!(
!text.contains(leak),
"{what} must not disclose Resource data ({leak}): {body}"
);
}
}
/// A warmed catalog cache plus a failing tombstone/authority read: list and read fail closed, the
/// artifact store is not touched, and the positive control still works once the gate is restored.
#[tokio::test]
async fn story38_gate_storage_failure_after_a_warm_cache_never_serves_cached_content() {
// A non-zero refresh interval is what makes "warmed cache" meaningful.
let stand = spawn_revocation_stand_with_refresh("gate_failure", Duration::from_secs(60)).await;
for revision in supported_revisions() {
let first = mcp_read(
&stand.mcp,
&stand.mcp_url,
&stand.mcp_key,
&revision,
REVOKED_URI,
)
.await;
assert_eq!(read_text(&first).as_deref(), Some(REVOKED_CANARY));
let second = mcp_read(
&stand.mcp,
&stand.mcp_url_second,
&stand.mcp_key_second,
&revision,
REVOKED_URI,
)
.await;
assert_eq!(read_text(&second).as_deref(), Some(REVOKED_CANARY_V2));
for (url, key) in [
(&stand.mcp_url, &stand.mcp_key),
(&stand.mcp_url_second, &stand.mcp_key_second),
] {
let listed = listed_uris(&mcp_list(&stand.mcp, url, key, &revision).await);
assert!(listed.contains(&REVOKED_URI.to_owned()));
}
}
sqlx::raw_sql(
"alter table resource_security_revocations rename to resource_security_revocations_offline",
)
.execute(stand.registry.pool())
.await
.expect("the tombstone read must be breakable in the test schema");
reset_artifact_reads();
for revision in supported_revisions() {
for (agent, url, key) in [
("first", &stand.mcp_url, &stand.mcp_key),
("second", &stand.mcp_url_second, &stand.mcp_key_second),
] {
let read = mcp_read(&stand.mcp, url, key, &revision, REVOKED_URI).await;
assert_internal_unavailable(&read, &format!("{agent} read on {revision}"));
assert_ne!(
read["error"]["code"].as_i64(),
Some(NOT_AVAILABLE_CODE),
"{agent} on {revision}: a storage failure must not be disguised as 'not available'"
);
let listed = mcp_list(&stand.mcp, url, key, &revision).await;
assert_internal_unavailable(&listed, &format!("{agent} list on {revision}"));
}
}
assert_eq!(
artifact_reads(),
0,
"a failing gate must not fall back to cached content or artifact bytes"
);
sqlx::raw_sql(
"alter table resource_security_revocations_offline rename to resource_security_revocations",
)
.execute(stand.registry.pool())
.await
.expect("the tombstone table must be restorable");
// Positive control: the same endpoints serve the Resource again; nothing was revoked falsely.
let restored = mcp_read(
&stand.mcp,
&stand.mcp_url,
&stand.mcp_key,
MODERN_REVISION,
REVOKED_URI,
)
.await;
assert_eq!(read_text(&restored).as_deref(), Some(REVOKED_CANARY));
assert!(
listed_uris(&mcp_list(&stand.mcp, &stand.mcp_url, &stand.mcp_key, LEGACY_REVISION).await)
.contains(&REVOKED_URI.to_owned())
);
}
/// R1 regression: a corrupt bound publication must not be silently reported as "the URI does not
/// exist". The typed integrity error is preserved and answered with the safe unavailable outcome
/// before any artifact byte, while another bound Resource stays readable.
#[tokio::test]
async fn story38_resolver_integrity_failure_is_not_disguised_as_a_missing_uri() {
let stand = spawn_revocation_stand("resolver_integrity").await;
let pool = stand.registry.pool();
sqlx::query(
"update resource_publications
set size_bytes = size_bytes + 1
where workspace_id = 'ws_default' and resource_id = $1 and version = 1",
)
.bind(&stand.revoked_id)
.execute(pool)
.await
.expect("the bound publication row must be corruptible in the test schema");
reset_artifact_reads();
for revision in supported_revisions() {
let denied = mcp_read(
&stand.mcp,
&stand.mcp_url,
&stand.mcp_key,
&revision,
REVOKED_URI,
)
.await;
assert_internal_unavailable(&denied, &format!("corrupt resolver read on {revision}"));
assert_ne!(
denied["error"]["code"].as_i64(),
Some(NOT_AVAILABLE_CODE),
"{revision}: a corrupt publication is not a missing URI"
);
}
assert_eq!(
artifact_reads(),
0,
"an integrity failure must not open artifact bytes"
);
// The whole bound snapshot is unverifiable, so every read from this Agent fails closed
// (the same rule the list page uses) instead of serving the remaining publications.
let other = mcp_read(
&stand.mcp,
&stand.mcp_url,
&stand.mcp_key,
MODERN_REVISION,
KEPT_URI,
)
.await;
assert_internal_unavailable(&other, "a read while one bound publication is corrupt");
assert_eq!(
artifact_reads(),
0,
"an unverifiable snapshot must not open any artifact bytes"
);
sqlx::query(
"update resource_publications
set size_bytes = size_bytes - 1
where workspace_id = 'ws_default' and resource_id = $1 and version = 1",
)
.bind(&stand.revoked_id)
.execute(pool)
.await
.expect("the publication row must be restorable");
let restored = mcp_read(
&stand.mcp,
&stand.mcp_url,
&stand.mcp_key,
MODERN_REVISION,
REVOKED_URI,
)
.await;
assert_eq!(read_text(&restored).as_deref(), Some(REVOKED_CANARY));
let restored_control = mcp_read(
&stand.mcp,
&stand.mcp_url,
&stand.mcp_key,
MODERN_REVISION,
KEPT_URI,
)
.await;
assert_eq!(
read_text(&restored_control).as_deref(),
Some(super::revocation::KEPT_CANARY)
);
assert!(
artifact_reads() >= 2,
"the restored reads must be observable as artifact reads"
);
}
/// One corrupt publication on the requested page must fail the whole page, not silently drop an
/// element (a partial page would look like a successful, complete answer).
#[tokio::test]
async fn story38_corrupt_list_publication_never_returns_a_partial_page() {
let stand = spawn_revocation_stand("list_corruption").await;
let pool = stand.registry.pool();
let before =
listed_uris(&mcp_list(&stand.mcp, &stand.mcp_url, &stand.mcp_key, MODERN_REVISION).await);
assert!(before.contains(&REVOKED_URI.to_owned()) && before.contains(&KEPT_URI.to_owned()));
sqlx::query(
"update resource_publications
set size_bytes = size_bytes + 1
where workspace_id = 'ws_default' and resource_id = $1",
)
.bind(&stand.kept_id)
.execute(pool)
.await
.expect("the publication row must be corruptible in the test schema");
for revision in supported_revisions() {
let listed = mcp_list(&stand.mcp, &stand.mcp_url, &stand.mcp_key, &revision).await;
assert_internal_unavailable(&listed, &format!("corrupt list on {revision}"));
assert!(
listed.get("result").is_none(),
"{revision}: a corrupt page must not be reported as a (partial) success: {listed}"
);
}
// A read of the still-consistent Resource is unaffected by the page failure.
let readable = mcp_read(
&stand.mcp,
&stand.mcp_url,
&stand.mcp_key,
MODERN_REVISION,
KEPT_URI,
)
.await;
assert_internal_unavailable(&readable, "a read of the corrupt publication");
sqlx::query(
"update resource_publications
set size_bytes = size_bytes - 1
where workspace_id = 'ws_default' and resource_id = $1",
)
.bind(&stand.kept_id)
.execute(pool)
.await
.expect("the publication row must be restorable");
let after =
listed_uris(&mcp_list(&stand.mcp, &stand.mcp_url, &stand.mcp_key, MODERN_REVISION).await);
assert_eq!(
after, before,
"after restoring the row the page must serve exactly the same elements again"
);
}
/// A failing COMMIT of the audited revoke is a sanitized 500 — never a 204 and never an
/// unaudited state change; the bounded retry then succeeds with exactly one event.
#[tokio::test]
async fn story38_commit_time_failure_never_answers_a_false_204() {
let stand = spawn_revocation_stand("commit_failure").await;
let pool = stand.registry.pool();
sqlx::raw_sql(
"create function reject_story38_revocation_commit() returns trigger language plpgsql as $$
begin
raise exception 'injected revocation commit failure' using errcode = '23514';
end $$;
create constraint trigger reject_story38_revocation_commit_trigger
after insert on resource_security_revocations
deferrable initially deferred
for each row execute function reject_story38_revocation_commit();",
)
.execute(pool)
.await
.expect("the deferred commit-time failure trigger must install");
let (status, text, body) =
read_response(revoke(&stand.admin, &stand.server, &stand.revoked_id).await).await;
assert_eq!(
status,
StatusCode::INTERNAL_SERVER_ERROR,
"a failed commit must be a sanitized storage error, not a false 204: {text}"
);
assert!(
!text.contains("injected") && !text.contains("resource_security_revocations"),
"the storage failure must not disclose SQL details: {body}"
);
assert_eq!(
super::helpers::table_row_count(pool, "resource_security_revocations").await,
0,
"the failed transaction must leave no audit event"
);
// The Resource is still readable: the failed revoke must not have partially applied.
let readable = mcp_read(
&stand.mcp,
&stand.mcp_url,
&stand.mcp_key,
MODERN_REVISION,
REVOKED_URI,
)
.await;
assert_eq!(read_text(&readable).as_deref(), Some(REVOKED_CANARY));
sqlx::raw_sql(
"drop trigger reject_story38_revocation_commit_trigger on resource_security_revocations;
drop function reject_story38_revocation_commit();",
)
.execute(pool)
.await
.expect("the failure trigger must be removable");
let retried = revoke(&stand.admin, &stand.server, &stand.revoked_id).await;
assert_eq!(
retried.status(),
StatusCode::NO_CONTENT,
"a bounded retry after a failed commit must succeed"
);
assert_eq!(
super::helpers::table_row_count(pool, "resource_security_revocations").await,
1
);
}