use std::{ io, sync::{ Arc, Mutex, atomic::{AtomicUsize, Ordering}, }, time::Duration, }; use axum::{Json, Router, routing::post}; use crank_core::{PlatformApiKeyScope, WorkspaceId}; use crank_observability::{ ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity, operational_incident_total, }; use crank_registry::{ListInvocationLogsQuery, PublishRequest}; use serde_json::{Value, json}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::net::TcpListener; use tracing_subscriber::fmt::MakeWriter; #[path = "integration/common.rs"] mod common; use common::*; const CANARY_SECRET: &str = "dc08-canary-secret"; #[tokio::test] async fn preserves_mcp_result_when_postgres_rejects_invocation_history() { let registry = test_registry().await; let upstream_calls = Arc::new(AtomicUsize::new(0)); let upstream_base_url = spawn_counted_upstream(Arc::clone(&upstream_calls)).await; let operation = test_operation(&upstream_base_url, "crm_dc08"); registry .create_operation(&test_workspace_id(), &operation, Some("alice")) .await .unwrap(); registry .publish_operation(PublishRequest { workspace_id: &test_workspace_id(), operation_id: &operation.id, version: 1, published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(), published_by: Some("alice"), }) .await .unwrap(); publish_agent_for_operation(®istry, &operation, "sales-dc08").await; let api_key = create_platform_api_key( ®istry, "sales-dc08", "mcp-dc08", &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; install_history_failure_trigger(®istry).await; let writer = SharedLogWriter::default(); let subscriber = crank_observability::build_subscriber( ObservabilityConfig::new( ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(), "info", RedactionLimits::default(), ), writer.clone(), ) .unwrap(); tracing::subscriber::set_global_default(subscriber).unwrap(); let base_url = spawn_mcp_server(build_test_app( registry.clone(), Duration::from_millis(0), Some("https://crank.example.com".to_owned()), )) .await; let client = reqwest::Client::new(); let mcp_url = agent_mcp_url(&base_url, "sales-dc08"); let session_id = initialize_session(&client, &mcp_url, &api_key).await; let before = operational_incident_total(OperationalIncident::InvocationHistoryLost); let result = post_jsonrpc_response( &client, &mcp_url, &api_key, Some(&session_id), Some("req_mcp_dc08"), json!({ "jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": { "name": "crm_dc08", "arguments": { "email": format!("{CANARY_SECRET}@example.com") } } }), ) .await .json::() .await .unwrap(); assert_eq!( result["result"]["structuredContent"], json!({"id": "lead_123"}) ); assert_eq!(result["result"]["isError"], false); assert_eq!(upstream_calls.load(Ordering::SeqCst), 1); assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before); let logs = registry .list_invocation_logs(ListInvocationLogsQuery { workspace_id: &WorkspaceId::new("ws_default"), level: None, status: None, outcome_group: None, search_text: None, source: None, operation_id: Some(&operation.id), agent_id: None, created_after: None, created_before: None, cursor_created_at: None, cursor_id: None, limit: 10, }) .await .unwrap(); assert!(logs.is_empty()); let output = writer.output(); assert!(!output.contains(CANARY_SECRET)); let incidents = output .lines() .filter_map(|line| serde_json::from_str::(line).ok()) .filter(|event| event["event"] == "mcp.invocation_history.lost") .collect::>(); assert_eq!(incidents.len(), 1); assert_eq!(incidents[0]["request_id"], "req_mcp_dc08"); assert_eq!(incidents[0]["fields"]["source"], "agent_tool_call"); assert_eq!(incidents[0]["fields"]["invocation_status"], "ok"); } async fn install_history_failure_trigger(registry: &crank_registry::PostgresRegistry) { sqlx::query( r#" create function fail_dc08_invocation_history() returns trigger language plpgsql as $$ begin if new.request_id = 'req_mcp_dc08' then raise exception 'forced DC-08 invocation history failure'; end if; return new; end; $$ "#, ) .execute(registry.pool()) .await .unwrap(); sqlx::query( r#" create trigger fail_dc08_invocation_history before insert on invocation_logs for each row execute function fail_dc08_invocation_history() "#, ) .execute(registry.pool()) .await .unwrap(); } async fn spawn_counted_upstream(calls: Arc) -> String { let app = Router::new().route( "/crm/leads", post(move |Json(payload): Json| { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); Json(json!({ "id": "lead_123", "email": payload["email"] })) } }), ); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); format!("http://{address}") } #[derive(Clone, Default)] struct SharedLogWriter { buffer: Arc>>, } impl SharedLogWriter { fn output(&self) -> String { String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap() } } impl<'a> MakeWriter<'a> for SharedLogWriter { type Writer = SharedLogGuard; fn make_writer(&'a self) -> Self::Writer { SharedLogGuard { buffer: Arc::clone(&self.buffer), } } } struct SharedLogGuard { buffer: Arc>>, } impl io::Write for SharedLogGuard { fn write(&mut self, bytes: &[u8]) -> io::Result { self.buffer.lock().unwrap().extend_from_slice(bytes); Ok(bytes.len()) } fn flush(&mut self) -> io::Result<()> { Ok(()) } }