исправить: закрыть ревью сквозной корреляции
This commit is contained in:
@@ -0,0 +1,227 @@
|
||||
use std::{
|
||||
io,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{Json, Router, routing::post};
|
||||
use crank_core::{PlatformApiKeyScope, WorkspaceId};
|
||||
use crank_observability::{
|
||||
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
|
||||
operational_incident_total,
|
||||
};
|
||||
use crank_registry::{ListInvocationLogsQuery, PublishRequest};
|
||||
use serde_json::{Value, json};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::net::TcpListener;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[path = "integration/common.rs"]
|
||||
mod common;
|
||||
|
||||
use common::*;
|
||||
|
||||
const CANARY_SECRET: &str = "dc08-canary-secret";
|
||||
|
||||
#[tokio::test]
|
||||
async fn preserves_mcp_result_when_postgres_rejects_invocation_history() {
|
||||
let registry = test_registry().await;
|
||||
let upstream_calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_base_url = spawn_counted_upstream(Arc::clone(&upstream_calls)).await;
|
||||
let operation = test_operation(&upstream_base_url, "crm_dc08");
|
||||
|
||||
registry
|
||||
.create_operation(&test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent_for_operation(®istry, &operation, "sales-dc08").await;
|
||||
let api_key = create_platform_api_key(
|
||||
®istry,
|
||||
"sales-dc08",
|
||||
"mcp-dc08",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
install_history_failure_trigger(®istry).await;
|
||||
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = crank_observability::build_subscriber(
|
||||
ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
|
||||
"info",
|
||||
RedactionLimits::default(),
|
||||
),
|
||||
writer.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
tracing::subscriber::set_global_default(subscriber).unwrap();
|
||||
let base_url = spawn_mcp_server(build_test_app(
|
||||
registry.clone(),
|
||||
Duration::from_millis(0),
|
||||
Some("https://crank.example.com".to_owned()),
|
||||
))
|
||||
.await;
|
||||
let client = reqwest::Client::new();
|
||||
let mcp_url = agent_mcp_url(&base_url, "sales-dc08");
|
||||
let session_id = initialize_session(&client, &mcp_url, &api_key).await;
|
||||
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
|
||||
|
||||
let result = post_jsonrpc_response(
|
||||
&client,
|
||||
&mcp_url,
|
||||
&api_key,
|
||||
Some(&session_id),
|
||||
Some("req_mcp_dc08"),
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 3,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "crm_dc08",
|
||||
"arguments": {
|
||||
"email": format!("{CANARY_SECRET}@example.com")
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.json::<Value>()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result["result"]["structuredContent"],
|
||||
json!({"id": "lead_123"})
|
||||
);
|
||||
assert_eq!(result["result"]["isError"], false);
|
||||
assert_eq!(upstream_calls.load(Ordering::SeqCst), 1);
|
||||
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
|
||||
|
||||
let logs = registry
|
||||
.list_invocation_logs(ListInvocationLogsQuery {
|
||||
workspace_id: &WorkspaceId::new("ws_default"),
|
||||
level: None,
|
||||
search_text: None,
|
||||
source: None,
|
||||
operation_id: Some(&operation.id),
|
||||
agent_id: None,
|
||||
created_after: None,
|
||||
limit: 10,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(logs.is_empty());
|
||||
|
||||
let output = writer.output();
|
||||
assert!(!output.contains(CANARY_SECRET));
|
||||
let incidents = output
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
|
||||
.filter(|event| event["event"] == "mcp.invocation_history.lost")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(incidents.len(), 1);
|
||||
assert_eq!(incidents[0]["request_id"], "req_mcp_dc08");
|
||||
assert_eq!(incidents[0]["fields"]["source"], "agent_tool_call");
|
||||
assert_eq!(incidents[0]["fields"]["invocation_status"], "ok");
|
||||
}
|
||||
|
||||
async fn install_history_failure_trigger(registry: &crank_registry::PostgresRegistry) {
|
||||
sqlx::query(
|
||||
r#"
|
||||
create function fail_dc08_invocation_history() returns trigger
|
||||
language plpgsql
|
||||
as $$
|
||||
begin
|
||||
if new.request_id = 'req_mcp_dc08' then
|
||||
raise exception 'forced DC-08 invocation history failure';
|
||||
end if;
|
||||
return new;
|
||||
end;
|
||||
$$
|
||||
"#,
|
||||
)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
r#"
|
||||
create trigger fail_dc08_invocation_history
|
||||
before insert on invocation_logs
|
||||
for each row execute function fail_dc08_invocation_history()
|
||||
"#,
|
||||
)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn spawn_counted_upstream(calls: Arc<AtomicUsize>) -> String {
|
||||
let app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post(move |Json(payload): Json<Value>| {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Json(json!({
|
||||
"id": "lead_123",
|
||||
"email": payload["email"]
|
||||
}))
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{address}")
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user