исправить: закрыть ревью сквозной корреляции
CI / Rust Checks (pull_request) Successful in 8m33s
CI / UI Checks (pull_request) Successful in 5s
CI / Frontend E2E (pull_request) Successful in 6m51s
CI / Community Image Smoke (pull_request) Failing after 9m10s
CI / Deploy (pull_request) Has been skipped

This commit is contained in:
2026-07-31 02:37:45 +03:00
parent 0e8f1ca03a
commit 9a7d60593a
28 changed files with 667 additions and 98 deletions
+2 -11
View File
@@ -1,6 +1,6 @@
use axum::{
extract::Request,
http::{HeaderMap, HeaderName, HeaderValue},
http::{HeaderName, HeaderValue},
middleware::Next,
response::Response,
};
@@ -16,7 +16,7 @@ pub struct RequestContext {
pub async fn apply_request_context(mut request: Request, next: Next) -> Response {
let context = RequestContext {
request_id: resolve_request_id(request.headers()),
request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
};
let method = request.method().clone();
let path = request.uri().path().to_owned();
@@ -46,15 +46,6 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
.await
}
fn resolve_request_id(headers: &HeaderMap) -> String {
RequestId::resolve(
headers
.get(&REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok()),
)
.into_string()
}
#[cfg(test)]
mod tests {
#[test]
+20 -15
View File
@@ -5,10 +5,10 @@ use std::sync::Arc;
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{
AuditSink, AuthProfile, CapabilityProfile, CommunityCapabilityProfile, EditionCapabilities,
ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId, NoopAuditSink,
OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine, ProductEdition, Protocol,
ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind, ToolQualitySchemaNode,
UsagePeriod, WorkspaceId,
ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId,
InvocationSource, NoopAuditSink, OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine,
ProductEdition, Protocol, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind,
ToolQualitySchemaNode, UsagePeriod, WorkspaceId,
};
use crank_mapping::{MappingRule, MappingSet};
use crank_registry::{
@@ -479,9 +479,7 @@ impl AdminService {
let history_span = crank_trace::Stage::HistoryWrite.span();
let (outcome, db_span) = async {
let db_span = crank_trace::Stage::DbQuery
.db_span(crank_trace::DbOperation::InvocationHistoryWrite)
.expect("database stage");
let db_span = crank_trace::DbOperation::InvocationHistoryWrite.span();
let outcome = self
.registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
@@ -509,7 +507,7 @@ impl AdminService {
outcome,
request.request_id,
request.status,
"admin_test_run",
request.source,
);
outcome
}
@@ -519,7 +517,7 @@ fn observe_invocation_history_outcome(
outcome: InvocationHistoryWriteOutcome,
request_id: Option<&str>,
status: crank_core::InvocationStatus,
source: &'static str,
source: InvocationSource,
) {
let Some(loss) = outcome.loss() else {
return;
@@ -530,13 +528,20 @@ fn observe_invocation_history_outcome(
tracing::warn!(
name: "admin.invocation_history.lost",
request_id = request_id.unwrap_or_default(),
source,
source = invocation_source_label(source),
invocation_status = invocation_status_label(status),
error_category = loss.category.as_str(),
"invocation history was not recorded"
);
}
fn invocation_source_label(source: InvocationSource) -> &'static str {
match source {
InvocationSource::AdminTestRun => "admin_test_run",
InvocationSource::AgentToolCall => "agent_tool_call",
}
}
fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
match status {
crank_core::InvocationStatus::Ok => "ok",
@@ -606,9 +611,9 @@ fn new_prefixed_id(prefix: &str) -> String {
format!("{prefix}_{}", Uuid::now_v7().simple())
}
fn generate_access_secret(prefix: &str) -> String {
fn generate_access_secret(marker: &str) -> String {
let random = URL_SAFE_NO_PAD.encode(Uuid::now_v7().as_bytes());
format!("{prefix}_{random}")
format!("{marker}{random}")
}
fn hash_access_secret(secret: &str) -> String {
@@ -874,7 +879,7 @@ mod tests {
sync::{Arc, Mutex},
};
use crank_core::InvocationStatus;
use crank_core::{InvocationSource, InvocationStatus};
use crank_observability::{
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
operational_incident_total,
@@ -931,7 +936,7 @@ mod tests {
}),
Some("req_admin_dc08"),
InvocationStatus::Error,
"admin_test_run",
InvocationSource::AgentToolCall,
);
let output = writer.output();
@@ -942,7 +947,7 @@ mod tests {
.find(|event: &Value| event["event"] == "admin.invocation_history.lost")
.unwrap();
assert_eq!(event["request_id"], "req_admin_dc08");
assert_eq!(event["fields"]["source"], "admin_test_run");
assert_eq!(event["fields"]["source"], "agent_tool_call");
assert_eq!(event["fields"]["invocation_status"], "error");
assert_eq!(event["fields"]["error_category"], "invalid_record");
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+1 -4
View File
@@ -65,10 +65,7 @@ impl AdminService {
),
None => None,
};
let secret = generate_access_secret(match payload.key_kind {
PlatformApiKeyKind::McpClient => "crk",
PlatformApiKeyKind::Approval => "crk_appr",
});
let secret = generate_access_secret(payload.key_kind.secret_marker());
let api_key = PlatformApiKeyRecord {
api_key: PlatformApiKey {
id: PlatformApiKeyId::new(new_prefixed_id("pk")),
@@ -2,11 +2,21 @@ use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use std::{io, sync::Mutex};
use axum::{Json, Router, extract::State, routing::post};
use crank_core::{OperationId, WorkspaceId};
use serde_json::{Value, json};
use serial_test::serial;
use tokio::{net::TcpListener, sync::Notify};
use tracing_subscriber::fmt::MakeWriter;
use super::*;
#[path = "integration/common.rs"]
mod common;
use common::*;
const DEFAULT_WORKSPACE_ID: &str = "ws_default";
#[tokio::test(flavor = "multi_thread")]
#[serial]
@@ -15,6 +25,17 @@ async fn preserves_external_success_when_invocation_history_is_lost() {
let registry_for_failure = registry.clone();
let storage_root = test_storage_root("observability_history_loss");
let upstream = spawn_blocking_upstream_server().await;
let log_writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
crank_observability::ObservabilityConfig::new(
crank_observability::ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
"info",
crank_observability::RedactionLimits::default(),
),
log_writer.clone(),
)
.unwrap();
tracing::subscriber::set_global_default(subscriber).unwrap();
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
@@ -85,6 +106,16 @@ async fn preserves_external_success_when_invocation_history_is_lost() {
.await
.unwrap();
assert!(logs["items"].as_array().unwrap().is_empty());
let output = log_writer.output();
assert!(!output.contains("dc08-canary-secret"));
let incident = output
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.find(|event| event["event"] == "admin.invocation_history.lost")
.expect("DC-08 incident");
assert_eq!(incident["request_id"], "req_dc08_admin");
assert_eq!(incident["fields"]["source"], "admin_test_run");
}
struct BlockingUpstream {
@@ -139,3 +170,39 @@ async fn blocking_create_lead(
"email": payload["email"]
}))
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
@@ -41,8 +41,6 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key";
mod history_loss;
struct TestServer {
base_url: String,
shutdown: Option<tokio::sync::oneshot::Sender<()>>,
@@ -7,7 +7,7 @@ use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
use axum::{
Router,
body::Body,
http::{HeaderMap, Request, StatusCode},
http::{HeaderMap, HeaderValue, Request, StatusCode},
routing::get,
};
use crank_observability::{
@@ -16,12 +16,14 @@ use crank_observability::{
use opentelemetry::{global, trace::TracerProvider as _};
use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
use tower::ServiceExt;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
use uuid::Version;
static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[tokio::test(flavor = "current_thread")]
async fn logs_request_completion_and_rejects_untrusted_values() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
@@ -33,6 +35,7 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
)
.unwrap();
let dispatch = tracing::Dispatch::new(subscriber);
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let app = probe_app();
let response = app
@@ -44,7 +47,6 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap();
@@ -73,7 +75,6 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch)
.await
.unwrap();
let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
@@ -88,12 +89,14 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
#[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
global::set_text_map_propagator(TraceContextPropagator::new());
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("admin-request-context-test");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber);
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let app = trace_probe_app();
let valid = observed_trace_id(
@@ -109,7 +112,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap(),
);
@@ -122,7 +124,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap(),
);
@@ -133,7 +134,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch)
.await
.unwrap(),
);
@@ -145,6 +145,35 @@ async fn covers_valid_invalid_and_absent_traceparent() {
provider.shutdown().unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let dispatch = tracing::Dispatch::new(tracing_subscriber::registry());
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let mut request = Request::builder()
.uri("/probe")
.body(Body::empty())
.unwrap();
request.headers_mut().append(
REQUEST_ID_HEADER,
HeaderValue::from_static("first-request-id"),
);
request.headers_mut().append(
REQUEST_ID_HEADER,
HeaderValue::from_static("second-request-id"),
);
let response = probe_app().oneshot(request).await.unwrap();
let generated = response.headers()[REQUEST_ID_HEADER].to_str().unwrap();
assert_ne!(generated, "first-request-id");
assert_ne!(generated, "second-request-id");
assert_eq!(
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(Version::SortRand)
);
}
fn probe_app() -> Router {
Router::new()
.route("/probe", get(|| async { "ok" }))
+227
View File
@@ -0,0 +1,227 @@
use std::{
io,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{Json, Router, routing::post};
use crank_core::{PlatformApiKeyScope, WorkspaceId};
use crank_observability::{
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
operational_incident_total,
};
use crank_registry::{ListInvocationLogsQuery, PublishRequest};
use serde_json::{Value, json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::net::TcpListener;
use tracing_subscriber::fmt::MakeWriter;
#[path = "integration/common.rs"]
mod common;
use common::*;
const CANARY_SECRET: &str = "dc08-canary-secret";
#[tokio::test]
async fn preserves_mcp_result_when_postgres_rejects_invocation_history() {
let registry = test_registry().await;
let upstream_calls = Arc::new(AtomicUsize::new(0));
let upstream_base_url = spawn_counted_upstream(Arc::clone(&upstream_calls)).await;
let operation = test_operation(&upstream_base_url, "crm_dc08");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-dc08").await;
let api_key = create_platform_api_key(
&registry,
"sales-dc08",
"mcp-dc08",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
install_history_failure_trigger(&registry).await;
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
tracing::subscriber::set_global_default(subscriber).unwrap();
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "sales-dc08");
let session_id = initialize_session(&client, &mcp_url, &api_key).await;
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
let result = post_jsonrpc_response(
&client,
&mcp_url,
&api_key,
Some(&session_id),
Some("req_mcp_dc08"),
json!({
"jsonrpc": "2.0",
"id": 3,
"method": "tools/call",
"params": {
"name": "crm_dc08",
"arguments": {
"email": format!("{CANARY_SECRET}@example.com")
}
}
}),
)
.await
.json::<Value>()
.await
.unwrap();
assert_eq!(
result["result"]["structuredContent"],
json!({"id": "lead_123"})
);
assert_eq!(result["result"]["isError"], false);
assert_eq!(upstream_calls.load(Ordering::SeqCst), 1);
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
let logs = registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &WorkspaceId::new("ws_default"),
level: None,
search_text: None,
source: None,
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
limit: 10,
})
.await
.unwrap();
assert!(logs.is_empty());
let output = writer.output();
assert!(!output.contains(CANARY_SECRET));
let incidents = output
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.filter(|event| event["event"] == "mcp.invocation_history.lost")
.collect::<Vec<_>>();
assert_eq!(incidents.len(), 1);
assert_eq!(incidents[0]["request_id"], "req_mcp_dc08");
assert_eq!(incidents[0]["fields"]["source"], "agent_tool_call");
assert_eq!(incidents[0]["fields"]["invocation_status"], "ok");
}
async fn install_history_failure_trigger(registry: &crank_registry::PostgresRegistry) {
sqlx::query(
r#"
create function fail_dc08_invocation_history() returns trigger
language plpgsql
as $$
begin
if new.request_id = 'req_mcp_dc08' then
raise exception 'forced DC-08 invocation history failure';
end if;
return new;
end;
$$
"#,
)
.execute(registry.pool())
.await
.unwrap();
sqlx::query(
r#"
create trigger fail_dc08_invocation_history
before insert on invocation_logs
for each row execute function fail_dc08_invocation_history()
"#,
)
.execute(registry.pool())
.await
.unwrap();
}
async fn spawn_counted_upstream(calls: Arc<AtomicUsize>) -> String {
let app = Router::new().route(
"/crm/leads",
post(move |Json(payload): Json<Value>| {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
Json(json!({
"id": "lead_123",
"email": payload["email"]
}))
}
}),
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://{address}")
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
@@ -518,6 +518,55 @@ async fn approval_http_endpoints_enforce_request_rate_limit() {
assert!(limited.headers().contains_key(header::RETRY_AFTER));
}
#[tokio::test]
async fn unverified_session_ids_do_not_create_approval_rate_limit_buckets() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_approval_session_rate_limit");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-approval-session-rate-limit").await;
let approval_key = create_approval_platform_api_key(
&registry,
"sales-approval-session-rate-limit",
"approval-session-rate-limit",
)
.await;
let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
registry,
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
RequestRateLimitConfig::new(1, 1).unwrap(),
))
.await;
let approvals_url = format!(
"{}/approvals",
agent_mcp_url(&base_url, "sales-approval-session-rate-limit")
);
let client = reqwest::Client::new();
let allowed = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.header("MCP-Session-Id", "unverified-session-a")
.send()
.await
.unwrap();
assert_eq!(allowed.status(), reqwest::StatusCode::OK);
let limited = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.header("MCP-Session-Id", "unverified-session-b")
.send()
.await
.unwrap();
assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
assert!(limited.headers().contains_key(header::RETRY_AFTER));
}
#[tokio::test]
async fn recovery_does_not_repeat_interrupted_mutating_approval() {
let registry = test_registry().await;
@@ -5,7 +5,7 @@ use std::{
use axum::{
body::Body,
http::{Request, StatusCode},
http::{HeaderValue, Request, StatusCode},
};
use opentelemetry::{
global,
@@ -23,9 +23,11 @@ use tracing_subscriber::layer::SubscriberExt;
use super::common::{build_test_app, test_registry};
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
global::set_text_map_propagator(TraceContextPropagator::new());
let exported = Arc::new(Mutex::new(Vec::new()));
let provider = SdkTracerProvider::builder()
@@ -81,6 +83,37 @@ async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
provider.shutdown().unwrap();
}
#[tokio::test]
async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let app = build_test_app(test_registry().await, Duration::ZERO, None);
let mut request = Request::builder()
.uri("/health")
.body(Body::empty())
.unwrap();
request
.headers_mut()
.append("x-request-id", HeaderValue::from_static("first-request-id"));
request.headers_mut().append(
"x-request-id",
HeaderValue::from_static("second-request-id"),
);
let response = app
.oneshot(request)
.with_subscriber(tracing_subscriber::registry())
.await
.unwrap();
let generated = response.headers()["x-request-id"].to_str().unwrap();
assert_ne!(generated, "first-request-id");
assert_ne!(generated, "second-request-id");
assert_eq!(
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(uuid::Version::SortRand)
);
}
async fn send_health(
app: axum::Router,
traceparent: Option<&str>,