story 2-14-openapi-support-matrix-regression-corpus: implemented and reviewed via bmad-loop

This commit is contained in:
bsodfather committed 2026-09-29 20:48:40 +03:00
1 parent 16dc777911
commit 194cac169f
84 files changed
+23255 -1

No files matched your search

@@ -1,5 +1,34 @@
use serde_json::{Value, json};
use serial_test::serial;
use std::{sync::Arc, time::Duration};
use async_trait::async_trait;
use crank_community_mcp::session::InMemorySessionStore;
use crank_core::{
MachineAccessMode, MachineCredentialVerifier, MachineCredentialVerifierError,
OperationSecurityLevel, PlatformApiKeyScope, VerifiedMachineCredential,
};
struct CorpusMcpCredentialVerifier;
#[async_trait]
impl MachineCredentialVerifier for CorpusMcpCredentialVerifier {
async fn verify_bearer_token(
&self,
_workspace_slug: &str,
_agent_slug: &str,
token: &str,
) -> Result<Option<VerifiedMachineCredential>, MachineCredentialVerifierError> {
Ok(
(token == "corpus-test-token").then_some(VerifiedMachineCredential {
machine_access_mode: MachineAccessMode::StaticAgentKey,
max_security_level: OperationSecurityLevel::Standard,
scopes: vec![PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
platform_api_key_id: None,
}),
)
}
}
use super::super::common::*;
@@ -53,6 +82,7 @@ struct CreatedPlan {
reference: String,
digest: String,
body: Value,
preview: Value,
}
async fn create_plan(
@@ -111,6 +141,7 @@ async fn create_plan(
reference,
digest,
body,
preview,
}
}
@@ -287,6 +318,7 @@ async fn same_key_and_digest_replay_report_without_extra_draft_and_changed_diges
reference: plan.reference.clone(),
digest: "0".repeat(64),
body: plan.body.clone(),
preview: plan.preview.clone(),
};
let (changed_status, _) = apply_plan(&client, &base_url, &changed, "stable-apply-key").await;
assert_eq!(changed_status, reqwest::StatusCode::CONFLICT);
@@ -662,3 +694,153 @@ async fn disabled_auth_secret_rejects_apply_without_leaking_credentials() {
assert_eq!(after["total"], 1);
assert_eq!(after["items"][0]["name"], "beta");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn ac_4_corpus_operation_completes_the_real_http_to_mcp_workflow() {
use axum::{Json, Router};
let registry = test_registry().await;
let (request_tx, mut request_rx) = tokio::sync::mpsc::unbounded_channel();
let upstream =
Router::new().fallback(move |method: axum::http::Method, uri: axum::http::Uri| {
let request_tx = request_tx.clone();
async move {
request_tx.send((method, uri)).unwrap();
Json(json!({"id":"corpus-0"}))
}
});
let upstream_base = spawn_admin_api(upstream).await;
let storage = test_storage_root("corpus_ac4");
let base_url = spawn_admin_api(build_test_app(registry.clone(), storage)).await;
let client = authorized_client(&base_url).await;
let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../crates/crank-import/tests/corpus/representative-00.json");
let mut fixture: Value = serde_json::from_slice(&std::fs::read(fixture_path).unwrap()).unwrap();
let upstream_origin = upstream_base
.as_ref()
.split("/api/admin/workspaces/")
.next()
.unwrap();
fixture["servers"] = json!([{ "url": upstream_origin }]);
let document = serde_json::to_string(&fixture).unwrap();
let plan = create_plan(&client, &base_url, &document, full_plan()).await;
assert_eq!(plan.body["counts"]["actionable"], 1);
let (status, report) = apply_plan(&client, &base_url, &plan, "corpus-ac4-import").await;
assert_eq!(status, reqwest::StatusCode::OK, "Apply failed: {report}");
assert!(report_has_outcome_status(&report, "created"));
let operations = operation_list(&client, &base_url).await;
assert_eq!(operations["total"], 1);
assert_eq!(operations["items"][0]["status"], "draft");
let operation_id = operations["items"][0]["id"].as_str().unwrap().to_owned();
// The admin test run performs the persisted Draft validation and a real
// HTTP execution against the local fixture server before publication.
let test_run = client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.json(&json!({"version":1,"input":{}}))
.send()
.await
.unwrap();
assert_eq!(test_run.status(), reqwest::StatusCode::OK);
let test_result: Value = test_run.json().await.unwrap();
assert_eq!(test_result["ok"], true, "Draft test failed: {test_result}");
let published = client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.json(&json!({"version":1}))
.send()
.await
.unwrap();
assert_eq!(published.status(), reqwest::StatusCode::OK);
let agent = client.post(format!("{base_url}/agents")).json(&json!({
"slug":"corpus-regression-agent","display_name":"Corpus Regression Agent",
"description":"Local corpus acceptance fixture","instructions":{},"tool_selection_policy":{}
})).send().await.unwrap().json::<Value>().await.unwrap();
let agent_id = agent["agent_id"].as_str().unwrap();
let bindings = client.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, agent_etag(&client, &base_url, agent_id).await)
.json(&json!([{"operation_id":operation_id,"operation_version":1,"tool_name":"corpusOperation00","tool_title":"Synthetic operation 00","enabled":true}]))
.send().await.unwrap();
assert_eq!(bindings.status(), reqwest::StatusCode::OK);
let agent_publish = client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, agent_id).await,
)
.json(&json!({"version":1}))
.send()
.await
.unwrap();
assert_eq!(agent_publish.status(), reqwest::StatusCode::OK);
let mcp = crank_community_mcp::build_app(
registry,
Duration::ZERO,
None,
test_secret_crypto(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
crank_runtime::RequestRateLimiter::new(
crank_runtime::RequestRateLimitConfig::new(100, 100).unwrap(),
),
Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()),
Arc::new(InMemorySessionStore::default()),
Arc::new(CorpusMcpCredentialVerifier),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, mcp).await.unwrap();
});
let mcp_base = format!("http://{address}");
let mcp_url = format!("{mcp_base}/v1/default/corpus-regression-agent");
let initialized = client.post(&mcp_url).bearer_auth("corpus-test-token")
.header("accept", "application/json, text/event-stream")
.json(&json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"corpus","version":"1"}}}))
.send().await.unwrap();
assert_eq!(initialized.status(), reqwest::StatusCode::OK);
let session = initialized
.headers()
.get("mcp-session-id")
.unwrap()
.to_str()
.unwrap()
.to_owned();
let notification = client
.post(&mcp_url)
.bearer_auth("corpus-test-token")
.header("accept", "application/json, text/event-stream")
.header("mcp-session-id", &session)
.header("mcp-protocol-version", "2025-11-25")
.json(&json!({"jsonrpc":"2.0","method":"notifications/initialized","params":{}}))
.send()
.await
.unwrap();
assert_eq!(notification.status(), reqwest::StatusCode::ACCEPTED);
let called = client.post(&mcp_url).bearer_auth("corpus-test-token")
.header("accept", "application/json, text/event-stream").header("mcp-session-id", session)
.header("mcp-protocol-version", "2025-11-25")
.json(&json!({"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"corpusOperation00","arguments":{}}}))
.send().await.unwrap();
assert_eq!(called.status(), reqwest::StatusCode::OK);
let call: Value = called.json().await.unwrap();
assert_eq!(call["result"]["isError"], false, "MCP call failed: {call}");
assert_eq!(call["result"]["structuredContent"]["id"], "corpus-0");
for stage in ["Draft test run", "MCP call"] {
let (method, uri) = request_rx
.try_recv()
.unwrap_or_else(|_| panic!("{stage} missed upstream"));
assert_eq!(method, axum::http::Method::GET, "{stage}");
assert_eq!(uri.path(), "/items/0", "{stage}");
}
}
mod corpus_hostile;
@@ -0,0 +1,295 @@
use super::*;
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn ac_5_hostile_corpus_cases_are_bounded_atomic_and_redacted() {
let registry = test_registry().await;
let base_url = spawn_admin_api(build_test_app(registry, test_storage_root("corpus_ac5"))).await;
let client = authorized_client(&base_url).await;
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../crates/crank-import/tests/corpus");
let hostile_xml = std::fs::read_to_string(root.join("hostile-xml.json")).unwrap();
let before = operation_list(&client, &base_url).await;
let started = std::time::Instant::now();
let plan = create_plan(&client, &base_url, &hostile_xml, full_plan()).await;
assert!(
started.elapsed() < Duration::from_secs(5),
"hostile document exceeded bounded test budget"
);
assert!(
plan.body["counts"]["blocked"].as_u64().unwrap_or(0) > 0,
"hostile XML must be a blocker: {}",
plan.body
);
let xml_candidate = &plan.preview["preview"]["groups"][0]["operations"][0];
assert_eq!(xml_candidate["key"], "GET /xml");
assert!(
xml_candidate["findings"]
.as_array()
.is_some_and(|findings| findings.iter().any(|finding| finding["code"]
== "unsupported_xml_response_media_type"
&& finding["severity"] == "error")),
"HTTP preview must expose the exact XML blocker: {}",
plan.preview
);
assert_eq!(
plan.body["candidates"][0]["reason"], "unsupported_xml_response_media_type",
"HTTP plan must retain the XML blocker: {}",
plan.body
);
let (status, report) = apply_plan(&client, &base_url, &plan, "corpus-ac5-hostile").await;
assert_eq!(
status,
reqwest::StatusCode::CONFLICT,
"blocker must reject atomically: {report}"
);
assert_eq!(operation_list(&client, &base_url).await, before);
let secret = std::fs::read_to_string(root.join("hostile-secret.json")).unwrap();
let secret_plan = create_plan(&client, &base_url, &secret, full_plan()).await;
let exposed = format!("{}{}", secret_plan.preview, secret_plan.body);
assert!(
!exposed.contains("sk_test_corpus_secret_canary"),
"secret canary leaked into plan evidence"
);
assert_eq!(operation_list(&client, &base_url).await, before);
for file in ["hostile-malformed.json", "hostile-oversized.json"] {
let source = std::fs::read(root.join(file)).unwrap();
let started = std::time::Instant::now();
let upload = reqwest::multipart::Part::bytes(source)
.file_name(file)
.mime_str("application/json")
.unwrap();
let response = client
.post(format!("{base_url}/imports/openapi/preview"))
.multipart(reqwest::multipart::Form::new().part("file", upload))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap_or_default();
assert!(
!status.is_success(),
"{file} unexpectedly passed preview: {body}"
);
assert!(
started.elapsed() < Duration::from_secs(5),
"{file} exceeded bounded upload/parse time"
);
assert!(
!body.contains("sk_test_corpus_secret_canary"),
"hostile error response leaked a secret canary"
);
assert_eq!(
operation_list(&client, &base_url).await,
before,
"{file} mutated the operation registry"
);
}
// Real HTTP preview+plan: a root security requirement combined with an
// explicit empty operation override must leave the public operation
// actionable and block only the inherited one. No Apply mutation follows.
let mixed_security: Value = serde_json::json!({
"openapi": "3.0.3",
"info": { "title": "Mixed security plan probe", "version": "1" },
"servers": [{ "url": "https://security.example.test" }],
"components": { "securitySchemes": { "probe": { "type": "http", "scheme": "basic" } } },
"security": [{ "probe": [] }],
"paths": {
"/public": { "get": { "security": [], "operationId": "securityPublic",
"responses": { "200": { "description": "ok" } } } },
"/secured": { "get": { "operationId": "securitySecured",
"responses": { "200": { "description": "ok" } } } }
}
});
let security_mixed = serde_json::to_string(&mixed_security).unwrap();
let security_plan = create_plan(&client, &base_url, &security_mixed, full_plan()).await;
assert_eq!(
security_plan.body["counts"]["selected"], 2,
"{}",
security_plan.body
);
assert_eq!(
security_plan.body["counts"]["actionable"], 1,
"{}",
security_plan.body
);
assert_eq!(
security_plan.body["counts"]["blocked"], 1,
"{}",
security_plan.body
);
let security_candidates = security_plan.body["candidates"].as_array().unwrap();
let by_key = |key: &str| {
security_candidates
.iter()
.find(|candidate| candidate["key"] == key)
.unwrap_or_else(|| panic!("candidate {key} missing: {}", security_plan.body))
};
let public = by_key("GET /public");
let secured = by_key("GET /secured");
assert_eq!(
public["state"].as_str(),
Some("actionable"),
"public candidate with empty security override must stay actionable: {}",
security_plan.body
);
assert!(
public["reason"].is_null(),
"public candidate must not carry a security blocker: {}",
security_plan.body
);
assert_eq!(
secured["state"].as_str(),
Some("blocked"),
"inherited security requirement must block the affected candidate: {}",
security_plan.body
);
assert_eq!(
secured["reason"].as_str(),
Some("unsupported_security_requirement"),
"secured candidate must expose the exact security blocker: {}",
security_plan.body
);
assert_eq!(
operation_list(&client, &base_url).await,
before,
"security-scoped plan must not mutate the operation registry"
);
let (blocked_status, blocked_report) = apply_plan(
&client,
&base_url,
&security_plan,
"corpus-ac5-security-blocked",
)
.await;
assert_eq!(
blocked_status,
reqwest::StatusCode::CONFLICT,
"{blocked_report}"
);
assert_eq!(operation_list(&client, &base_url).await, before);
let public_plan = create_plan(
&client,
&base_url,
&security_mixed,
json!({
"selection_mode": "partial",
"selected_operation_keys": ["GET /public"]
}),
)
.await;
assert_eq!(public_plan.body["counts"]["actionable"], 1);
assert_eq!(public_plan.body["counts"]["blocked"], 0);
let (public_status, public_report) = apply_plan(
&client,
&base_url,
&public_plan,
"corpus-ac5-security-public",
)
.await;
assert_eq!(public_status, reqwest::StatusCode::OK, "{public_report}");
assert!(report_has_outcome_status(&public_report, "created"));
let after = operation_list(&client, &base_url).await;
assert_eq!(after["total"], 1, "{after}");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn p0_api_malformed_local_security_is_atomic_across_openapi_versions() {
for (version, document) in [
(
"openapi-3.1",
serde_json::json!({
"openapi": "3.1.0",
"info": { "title": "Security API regression", "version": "1" },
"servers": [{ "url": "https://security.example.test" }],
"security": [{ "probe": [] }],
"paths": {
"/bad": { "get": { "security": "basic", "operationId": "badOp",
"responses": { "200": { "description": "ok" } } } },
"/public": { "get": { "security": [], "operationId": "publicOp",
"responses": { "200": { "description": "ok" } } } }
}
}),
),
(
"swagger-2.0",
serde_json::json!({
"swagger": "2.0",
"info": { "title": "Security API regression", "version": "1" },
"host": "security.example.test",
"schemes": ["https"],
"security": [{ "apiKey": [] }],
"paths": {
"/bad": { "get": { "security": "apikey", "operationId": "badOp",
"responses": { "200": { "description": "ok" } } } },
"/public": { "get": { "security": [], "operationId": "publicOp",
"responses": { "200": { "description": "ok" } } } }
}
}),
),
] {
let registry = test_registry().await;
let base_url = spawn_admin_api(build_test_app(
registry,
test_storage_root(&format!("security_api_{version}")),
))
.await;
let client = authorized_client(&base_url).await;
let source = serde_json::to_string(&document).unwrap();
let before = operation_list(&client, &base_url).await;
let plan = create_plan(&client, &base_url, &source, full_plan()).await;
assert_eq!(
plan.body["counts"]["actionable"], 1,
"{version}: {}",
plan.body
);
assert_eq!(
plan.body["counts"]["blocked"], 1,
"{version}: {}",
plan.body
);
let candidates = plan.body["candidates"].as_array().unwrap();
let bad = candidates
.iter()
.find(|item| item["key"] == "GET /bad")
.unwrap();
let public = candidates
.iter()
.find(|item| item["key"] == "GET /public")
.unwrap();
assert_eq!(bad["state"], "blocked", "{version}: {}", plan.body);
assert_eq!(bad["reason"], "unsupported_security_requirement");
assert_eq!(public["state"], "actionable", "{version}: {}", plan.body);
assert!(public["reason"].is_null());
let (status, report) = apply_plan(&client, &base_url, &plan, "security-full").await;
assert_eq!(status, reqwest::StatusCode::CONFLICT, "{version}: {report}");
assert_eq!(operation_list(&client, &base_url).await, before);
let public_plan = create_plan(
&client,
&base_url,
&source,
json!({
"selection_mode": "partial",
"selected_operation_keys": ["GET /public"]
}),
)
.await;
assert_eq!(public_plan.body["counts"]["actionable"], 1);
assert_eq!(public_plan.body["counts"]["blocked"], 0);
let (status, report) =
apply_plan(&client, &base_url, &public_plan, "security-public").await;
assert_eq!(status, reqwest::StatusCode::OK, "{version}: {report}");
assert!(report_has_outcome_status(&report, "created"));
let after = operation_list(&client, &base_url).await;
assert_eq!(after["total"], 1, "{version}: {after}");
assert_eq!(after["items"][0]["status"], "draft");
}
}