исправить: закрыть ревью сквозной корреляции
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
use axum::{
|
||||
extract::Request,
|
||||
http::{HeaderMap, HeaderName, HeaderValue},
|
||||
http::{HeaderName, HeaderValue},
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
@@ -16,7 +16,7 @@ pub struct RequestContext {
|
||||
|
||||
pub async fn apply_request_context(mut request: Request, next: Next) -> Response {
|
||||
let context = RequestContext {
|
||||
request_id: resolve_request_id(request.headers()),
|
||||
request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
|
||||
};
|
||||
let method = request.method().clone();
|
||||
let path = request.uri().path().to_owned();
|
||||
@@ -46,15 +46,6 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
|
||||
.await
|
||||
}
|
||||
|
||||
fn resolve_request_id(headers: &HeaderMap) -> String {
|
||||
RequestId::resolve(
|
||||
headers
|
||||
.get(&REQUEST_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
)
|
||||
.into_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
|
||||
@@ -5,10 +5,10 @@ use std::sync::Arc;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use crank_core::{
|
||||
AuditSink, AuthProfile, CapabilityProfile, CommunityCapabilityProfile, EditionCapabilities,
|
||||
ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId, NoopAuditSink,
|
||||
OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine, ProductEdition, Protocol,
|
||||
ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind, ToolQualitySchemaNode,
|
||||
UsagePeriod, WorkspaceId,
|
||||
ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId,
|
||||
InvocationSource, NoopAuditSink, OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine,
|
||||
ProductEdition, Protocol, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind,
|
||||
ToolQualitySchemaNode, UsagePeriod, WorkspaceId,
|
||||
};
|
||||
use crank_mapping::{MappingRule, MappingSet};
|
||||
use crank_registry::{
|
||||
@@ -479,9 +479,7 @@ impl AdminService {
|
||||
|
||||
let history_span = crank_trace::Stage::HistoryWrite.span();
|
||||
let (outcome, db_span) = async {
|
||||
let db_span = crank_trace::Stage::DbQuery
|
||||
.db_span(crank_trace::DbOperation::InvocationHistoryWrite)
|
||||
.expect("database stage");
|
||||
let db_span = crank_trace::DbOperation::InvocationHistoryWrite.span();
|
||||
let outcome = self
|
||||
.registry
|
||||
.create_invocation_log(CreateInvocationLogRequest { log: &log })
|
||||
@@ -509,7 +507,7 @@ impl AdminService {
|
||||
outcome,
|
||||
request.request_id,
|
||||
request.status,
|
||||
"admin_test_run",
|
||||
request.source,
|
||||
);
|
||||
outcome
|
||||
}
|
||||
@@ -519,7 +517,7 @@ fn observe_invocation_history_outcome(
|
||||
outcome: InvocationHistoryWriteOutcome,
|
||||
request_id: Option<&str>,
|
||||
status: crank_core::InvocationStatus,
|
||||
source: &'static str,
|
||||
source: InvocationSource,
|
||||
) {
|
||||
let Some(loss) = outcome.loss() else {
|
||||
return;
|
||||
@@ -530,13 +528,20 @@ fn observe_invocation_history_outcome(
|
||||
tracing::warn!(
|
||||
name: "admin.invocation_history.lost",
|
||||
request_id = request_id.unwrap_or_default(),
|
||||
source,
|
||||
source = invocation_source_label(source),
|
||||
invocation_status = invocation_status_label(status),
|
||||
error_category = loss.category.as_str(),
|
||||
"invocation history was not recorded"
|
||||
);
|
||||
}
|
||||
|
||||
fn invocation_source_label(source: InvocationSource) -> &'static str {
|
||||
match source {
|
||||
InvocationSource::AdminTestRun => "admin_test_run",
|
||||
InvocationSource::AgentToolCall => "agent_tool_call",
|
||||
}
|
||||
}
|
||||
|
||||
fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
|
||||
match status {
|
||||
crank_core::InvocationStatus::Ok => "ok",
|
||||
@@ -606,9 +611,9 @@ fn new_prefixed_id(prefix: &str) -> String {
|
||||
format!("{prefix}_{}", Uuid::now_v7().simple())
|
||||
}
|
||||
|
||||
fn generate_access_secret(prefix: &str) -> String {
|
||||
fn generate_access_secret(marker: &str) -> String {
|
||||
let random = URL_SAFE_NO_PAD.encode(Uuid::now_v7().as_bytes());
|
||||
format!("{prefix}_{random}")
|
||||
format!("{marker}{random}")
|
||||
}
|
||||
|
||||
fn hash_access_secret(secret: &str) -> String {
|
||||
@@ -874,7 +879,7 @@ mod tests {
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use crank_core::InvocationStatus;
|
||||
use crank_core::{InvocationSource, InvocationStatus};
|
||||
use crank_observability::{
|
||||
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
|
||||
operational_incident_total,
|
||||
@@ -931,7 +936,7 @@ mod tests {
|
||||
}),
|
||||
Some("req_admin_dc08"),
|
||||
InvocationStatus::Error,
|
||||
"admin_test_run",
|
||||
InvocationSource::AgentToolCall,
|
||||
);
|
||||
|
||||
let output = writer.output();
|
||||
@@ -942,7 +947,7 @@ mod tests {
|
||||
.find(|event: &Value| event["event"] == "admin.invocation_history.lost")
|
||||
.unwrap();
|
||||
assert_eq!(event["request_id"], "req_admin_dc08");
|
||||
assert_eq!(event["fields"]["source"], "admin_test_run");
|
||||
assert_eq!(event["fields"]["source"], "agent_tool_call");
|
||||
assert_eq!(event["fields"]["invocation_status"], "error");
|
||||
assert_eq!(event["fields"]["error_category"], "invalid_record");
|
||||
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
|
||||
|
||||
@@ -65,10 +65,7 @@ impl AdminService {
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
let secret = generate_access_secret(match payload.key_kind {
|
||||
PlatformApiKeyKind::McpClient => "crk",
|
||||
PlatformApiKeyKind::Approval => "crk_appr",
|
||||
});
|
||||
let secret = generate_access_secret(payload.key_kind.secret_marker());
|
||||
let api_key = PlatformApiKeyRecord {
|
||||
api_key: PlatformApiKey {
|
||||
id: PlatformApiKeyId::new(new_prefixed_id("pk")),
|
||||
|
||||
+68
-1
@@ -2,11 +2,21 @@ use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::{io, sync::Mutex};
|
||||
|
||||
use axum::{Json, Router, extract::State, routing::post};
|
||||
use crank_core::{OperationId, WorkspaceId};
|
||||
use serde_json::{Value, json};
|
||||
use serial_test::serial;
|
||||
use tokio::{net::TcpListener, sync::Notify};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
use super::*;
|
||||
#[path = "integration/common.rs"]
|
||||
mod common;
|
||||
|
||||
use common::*;
|
||||
|
||||
const DEFAULT_WORKSPACE_ID: &str = "ws_default";
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
@@ -15,6 +25,17 @@ async fn preserves_external_success_when_invocation_history_is_lost() {
|
||||
let registry_for_failure = registry.clone();
|
||||
let storage_root = test_storage_root("observability_history_loss");
|
||||
let upstream = spawn_blocking_upstream_server().await;
|
||||
let log_writer = SharedLogWriter::default();
|
||||
let subscriber = crank_observability::build_subscriber(
|
||||
crank_observability::ObservabilityConfig::new(
|
||||
crank_observability::ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
|
||||
"info",
|
||||
crank_observability::RedactionLimits::default(),
|
||||
),
|
||||
log_writer.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
tracing::subscriber::set_global_default(subscriber).unwrap();
|
||||
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||
let client = authorized_client(&base_url).await;
|
||||
|
||||
@@ -85,6 +106,16 @@ async fn preserves_external_success_when_invocation_history_is_lost() {
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(logs["items"].as_array().unwrap().is_empty());
|
||||
|
||||
let output = log_writer.output();
|
||||
assert!(!output.contains("dc08-canary-secret"));
|
||||
let incident = output
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
|
||||
.find(|event| event["event"] == "admin.invocation_history.lost")
|
||||
.expect("DC-08 incident");
|
||||
assert_eq!(incident["request_id"], "req_dc08_admin");
|
||||
assert_eq!(incident["fields"]["source"], "admin_test_run");
|
||||
}
|
||||
|
||||
struct BlockingUpstream {
|
||||
@@ -139,3 +170,39 @@ async fn blocking_create_lead(
|
||||
"email": payload["email"]
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -41,8 +41,6 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
|
||||
const TEST_SESSION_SECRET: &str = "test-session-secret";
|
||||
const TEST_MASTER_KEY: &str = "test-master-key";
|
||||
|
||||
mod history_loss;
|
||||
|
||||
struct TestServer {
|
||||
base_url: String,
|
||||
shutdown: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
|
||||
@@ -7,7 +7,7 @@ use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
|
||||
use axum::{
|
||||
Router,
|
||||
body::Body,
|
||||
http::{HeaderMap, Request, StatusCode},
|
||||
http::{HeaderMap, HeaderValue, Request, StatusCode},
|
||||
routing::get,
|
||||
};
|
||||
use crank_observability::{
|
||||
@@ -16,12 +16,14 @@ use crank_observability::{
|
||||
use opentelemetry::{global, trace::TracerProvider as _};
|
||||
use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
|
||||
use tower::ServiceExt;
|
||||
use tracing::instrument::WithSubscriber;
|
||||
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
|
||||
use uuid::Version;
|
||||
|
||||
static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn logs_request_completion_and_rejects_untrusted_values() {
|
||||
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = crank_observability::build_subscriber(
|
||||
ObservabilityConfig::new(
|
||||
@@ -33,6 +35,7 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
|
||||
)
|
||||
.unwrap();
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let app = probe_app();
|
||||
|
||||
let response = app
|
||||
@@ -44,7 +47,6 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -73,7 +75,6 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch)
|
||||
.await
|
||||
.unwrap();
|
||||
let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
|
||||
@@ -88,12 +89,14 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn covers_valid_invalid_and_absent_traceparent() {
|
||||
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
let provider = SdkTracerProvider::builder().build();
|
||||
let tracer = provider.tracer("admin-request-context-test");
|
||||
let subscriber =
|
||||
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let app = trace_probe_app();
|
||||
|
||||
let valid = observed_trace_id(
|
||||
@@ -109,7 +112,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch.clone())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
@@ -122,7 +124,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch.clone())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
@@ -133,7 +134,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
@@ -145,6 +145,35 @@ async fn covers_valid_invalid_and_absent_traceparent() {
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
|
||||
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
|
||||
let dispatch = tracing::Dispatch::new(tracing_subscriber::registry());
|
||||
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let mut request = Request::builder()
|
||||
.uri("/probe")
|
||||
.body(Body::empty())
|
||||
.unwrap();
|
||||
request.headers_mut().append(
|
||||
REQUEST_ID_HEADER,
|
||||
HeaderValue::from_static("first-request-id"),
|
||||
);
|
||||
request.headers_mut().append(
|
||||
REQUEST_ID_HEADER,
|
||||
HeaderValue::from_static("second-request-id"),
|
||||
);
|
||||
|
||||
let response = probe_app().oneshot(request).await.unwrap();
|
||||
let generated = response.headers()[REQUEST_ID_HEADER].to_str().unwrap();
|
||||
|
||||
assert_ne!(generated, "first-request-id");
|
||||
assert_ne!(generated, "second-request-id");
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(generated).unwrap().get_version(),
|
||||
Some(Version::SortRand)
|
||||
);
|
||||
}
|
||||
|
||||
fn probe_app() -> Router {
|
||||
Router::new()
|
||||
.route("/probe", get(|| async { "ok" }))
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
use std::{
|
||||
io,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{Json, Router, routing::post};
|
||||
use crank_core::{PlatformApiKeyScope, WorkspaceId};
|
||||
use crank_observability::{
|
||||
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
|
||||
operational_incident_total,
|
||||
};
|
||||
use crank_registry::{ListInvocationLogsQuery, PublishRequest};
|
||||
use serde_json::{Value, json};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::net::TcpListener;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[path = "integration/common.rs"]
|
||||
mod common;
|
||||
|
||||
use common::*;
|
||||
|
||||
const CANARY_SECRET: &str = "dc08-canary-secret";
|
||||
|
||||
#[tokio::test]
|
||||
async fn preserves_mcp_result_when_postgres_rejects_invocation_history() {
|
||||
let registry = test_registry().await;
|
||||
let upstream_calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_base_url = spawn_counted_upstream(Arc::clone(&upstream_calls)).await;
|
||||
let operation = test_operation(&upstream_base_url, "crm_dc08");
|
||||
|
||||
registry
|
||||
.create_operation(&test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent_for_operation(®istry, &operation, "sales-dc08").await;
|
||||
let api_key = create_platform_api_key(
|
||||
®istry,
|
||||
"sales-dc08",
|
||||
"mcp-dc08",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
install_history_failure_trigger(®istry).await;
|
||||
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = crank_observability::build_subscriber(
|
||||
ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
|
||||
"info",
|
||||
RedactionLimits::default(),
|
||||
),
|
||||
writer.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
tracing::subscriber::set_global_default(subscriber).unwrap();
|
||||
let base_url = spawn_mcp_server(build_test_app(
|
||||
registry.clone(),
|
||||
Duration::from_millis(0),
|
||||
Some("https://crank.example.com".to_owned()),
|
||||
))
|
||||
.await;
|
||||
let client = reqwest::Client::new();
|
||||
let mcp_url = agent_mcp_url(&base_url, "sales-dc08");
|
||||
let session_id = initialize_session(&client, &mcp_url, &api_key).await;
|
||||
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
|
||||
|
||||
let result = post_jsonrpc_response(
|
||||
&client,
|
||||
&mcp_url,
|
||||
&api_key,
|
||||
Some(&session_id),
|
||||
Some("req_mcp_dc08"),
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 3,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "crm_dc08",
|
||||
"arguments": {
|
||||
"email": format!("{CANARY_SECRET}@example.com")
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.json::<Value>()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result["result"]["structuredContent"],
|
||||
json!({"id": "lead_123"})
|
||||
);
|
||||
assert_eq!(result["result"]["isError"], false);
|
||||
assert_eq!(upstream_calls.load(Ordering::SeqCst), 1);
|
||||
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
|
||||
|
||||
let logs = registry
|
||||
.list_invocation_logs(ListInvocationLogsQuery {
|
||||
workspace_id: &WorkspaceId::new("ws_default"),
|
||||
level: None,
|
||||
search_text: None,
|
||||
source: None,
|
||||
operation_id: Some(&operation.id),
|
||||
agent_id: None,
|
||||
created_after: None,
|
||||
limit: 10,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(logs.is_empty());
|
||||
|
||||
let output = writer.output();
|
||||
assert!(!output.contains(CANARY_SECRET));
|
||||
let incidents = output
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
|
||||
.filter(|event| event["event"] == "mcp.invocation_history.lost")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(incidents.len(), 1);
|
||||
assert_eq!(incidents[0]["request_id"], "req_mcp_dc08");
|
||||
assert_eq!(incidents[0]["fields"]["source"], "agent_tool_call");
|
||||
assert_eq!(incidents[0]["fields"]["invocation_status"], "ok");
|
||||
}
|
||||
|
||||
async fn install_history_failure_trigger(registry: &crank_registry::PostgresRegistry) {
|
||||
sqlx::query(
|
||||
r#"
|
||||
create function fail_dc08_invocation_history() returns trigger
|
||||
language plpgsql
|
||||
as $$
|
||||
begin
|
||||
if new.request_id = 'req_mcp_dc08' then
|
||||
raise exception 'forced DC-08 invocation history failure';
|
||||
end if;
|
||||
return new;
|
||||
end;
|
||||
$$
|
||||
"#,
|
||||
)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
r#"
|
||||
create trigger fail_dc08_invocation_history
|
||||
before insert on invocation_logs
|
||||
for each row execute function fail_dc08_invocation_history()
|
||||
"#,
|
||||
)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn spawn_counted_upstream(calls: Arc<AtomicUsize>) -> String {
|
||||
let app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post(move |Json(payload): Json<Value>| {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Json(json!({
|
||||
"id": "lead_123",
|
||||
"email": payload["email"]
|
||||
}))
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{address}")
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -518,6 +518,55 @@ async fn approval_http_endpoints_enforce_request_rate_limit() {
|
||||
assert!(limited.headers().contains_key(header::RETRY_AFTER));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unverified_session_ids_do_not_create_approval_rate_limit_buckets() {
|
||||
let registry = test_registry().await;
|
||||
let upstream_base_url = spawn_upstream_server().await;
|
||||
let operation = test_operation(&upstream_base_url, "crm_approval_session_rate_limit");
|
||||
registry
|
||||
.create_operation(&test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent_for_operation(®istry, &operation, "sales-approval-session-rate-limit").await;
|
||||
let approval_key = create_approval_platform_api_key(
|
||||
®istry,
|
||||
"sales-approval-session-rate-limit",
|
||||
"approval-session-rate-limit",
|
||||
)
|
||||
.await;
|
||||
let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
|
||||
registry,
|
||||
Duration::from_millis(0),
|
||||
Some("https://crank.example.com".to_owned()),
|
||||
RequestRateLimitConfig::new(1, 1).unwrap(),
|
||||
))
|
||||
.await;
|
||||
let approvals_url = format!(
|
||||
"{}/approvals",
|
||||
agent_mcp_url(&base_url, "sales-approval-session-rate-limit")
|
||||
);
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let allowed = client
|
||||
.get(&approvals_url)
|
||||
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
|
||||
.header("MCP-Session-Id", "unverified-session-a")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(allowed.status(), reqwest::StatusCode::OK);
|
||||
|
||||
let limited = client
|
||||
.get(&approvals_url)
|
||||
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
|
||||
.header("MCP-Session-Id", "unverified-session-b")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
assert!(limited.headers().contains_key(header::RETRY_AFTER));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_does_not_repeat_interrupted_mutating_approval() {
|
||||
let registry = test_registry().await;
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::{
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode},
|
||||
http::{HeaderValue, Request, StatusCode},
|
||||
};
|
||||
use opentelemetry::{
|
||||
global,
|
||||
@@ -23,9 +23,11 @@ use tracing_subscriber::layer::SubscriberExt;
|
||||
use super::common::{build_test_app, test_registry};
|
||||
|
||||
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
|
||||
static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
|
||||
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
let exported = Arc::new(Mutex::new(Vec::new()));
|
||||
let provider = SdkTracerProvider::builder()
|
||||
@@ -81,6 +83,37 @@ async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
|
||||
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
|
||||
let app = build_test_app(test_registry().await, Duration::ZERO, None);
|
||||
let mut request = Request::builder()
|
||||
.uri("/health")
|
||||
.body(Body::empty())
|
||||
.unwrap();
|
||||
request
|
||||
.headers_mut()
|
||||
.append("x-request-id", HeaderValue::from_static("first-request-id"));
|
||||
request.headers_mut().append(
|
||||
"x-request-id",
|
||||
HeaderValue::from_static("second-request-id"),
|
||||
);
|
||||
|
||||
let response = app
|
||||
.oneshot(request)
|
||||
.with_subscriber(tracing_subscriber::registry())
|
||||
.await
|
||||
.unwrap();
|
||||
let generated = response.headers()["x-request-id"].to_str().unwrap();
|
||||
|
||||
assert_ne!(generated, "first-request-id");
|
||||
assert_ne!(generated, "second-request-id");
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(generated).unwrap().get_version(),
|
||||
Some(uuid::Version::SortRand)
|
||||
);
|
||||
}
|
||||
|
||||
async fn send_health(
|
||||
app: axum::Router,
|
||||
traceparent: Option<&str>,
|
||||
|
||||
Reference in New Issue
Block a user