исправить: закрыть ревью сквозной корреляции
CI / Rust Checks (pull_request) Successful in 8m33s
CI / UI Checks (pull_request) Successful in 5s
CI / Frontend E2E (pull_request) Successful in 6m51s
CI / Community Image Smoke (pull_request) Failing after 9m10s
CI / Deploy (pull_request) Has been skipped

This commit is contained in:
2026-07-31 02:37:45 +03:00
parent 0e8f1ca03a
commit 9a7d60593a
28 changed files with 667 additions and 98 deletions
+2 -11
View File
@@ -1,6 +1,6 @@
use axum::{ use axum::{
extract::Request, extract::Request,
http::{HeaderMap, HeaderName, HeaderValue}, http::{HeaderName, HeaderValue},
middleware::Next, middleware::Next,
response::Response, response::Response,
}; };
@@ -16,7 +16,7 @@ pub struct RequestContext {
pub async fn apply_request_context(mut request: Request, next: Next) -> Response { pub async fn apply_request_context(mut request: Request, next: Next) -> Response {
let context = RequestContext { let context = RequestContext {
request_id: resolve_request_id(request.headers()), request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
}; };
let method = request.method().clone(); let method = request.method().clone();
let path = request.uri().path().to_owned(); let path = request.uri().path().to_owned();
@@ -46,15 +46,6 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
.await .await
} }
fn resolve_request_id(headers: &HeaderMap) -> String {
RequestId::resolve(
headers
.get(&REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok()),
)
.into_string()
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[test] #[test]
+20 -15
View File
@@ -5,10 +5,10 @@ use std::sync::Arc;
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{ use crank_core::{
AuditSink, AuthProfile, CapabilityProfile, CommunityCapabilityProfile, EditionCapabilities, AuditSink, AuthProfile, CapabilityProfile, CommunityCapabilityProfile, EditionCapabilities,
ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId, NoopAuditSink, ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId,
OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine, ProductEdition, Protocol, InvocationSource, NoopAuditSink, OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine,
ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind, ToolQualitySchemaNode, ProductEdition, Protocol, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind,
UsagePeriod, WorkspaceId, ToolQualitySchemaNode, UsagePeriod, WorkspaceId,
}; };
use crank_mapping::{MappingRule, MappingSet}; use crank_mapping::{MappingRule, MappingSet};
use crank_registry::{ use crank_registry::{
@@ -479,9 +479,7 @@ impl AdminService {
let history_span = crank_trace::Stage::HistoryWrite.span(); let history_span = crank_trace::Stage::HistoryWrite.span();
let (outcome, db_span) = async { let (outcome, db_span) = async {
let db_span = crank_trace::Stage::DbQuery let db_span = crank_trace::DbOperation::InvocationHistoryWrite.span();
.db_span(crank_trace::DbOperation::InvocationHistoryWrite)
.expect("database stage");
let outcome = self let outcome = self
.registry .registry
.create_invocation_log(CreateInvocationLogRequest { log: &log }) .create_invocation_log(CreateInvocationLogRequest { log: &log })
@@ -509,7 +507,7 @@ impl AdminService {
outcome, outcome,
request.request_id, request.request_id,
request.status, request.status,
"admin_test_run", request.source,
); );
outcome outcome
} }
@@ -519,7 +517,7 @@ fn observe_invocation_history_outcome(
outcome: InvocationHistoryWriteOutcome, outcome: InvocationHistoryWriteOutcome,
request_id: Option<&str>, request_id: Option<&str>,
status: crank_core::InvocationStatus, status: crank_core::InvocationStatus,
source: &'static str, source: InvocationSource,
) { ) {
let Some(loss) = outcome.loss() else { let Some(loss) = outcome.loss() else {
return; return;
@@ -530,13 +528,20 @@ fn observe_invocation_history_outcome(
tracing::warn!( tracing::warn!(
name: "admin.invocation_history.lost", name: "admin.invocation_history.lost",
request_id = request_id.unwrap_or_default(), request_id = request_id.unwrap_or_default(),
source, source = invocation_source_label(source),
invocation_status = invocation_status_label(status), invocation_status = invocation_status_label(status),
error_category = loss.category.as_str(), error_category = loss.category.as_str(),
"invocation history was not recorded" "invocation history was not recorded"
); );
} }
fn invocation_source_label(source: InvocationSource) -> &'static str {
match source {
InvocationSource::AdminTestRun => "admin_test_run",
InvocationSource::AgentToolCall => "agent_tool_call",
}
}
fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str { fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
match status { match status {
crank_core::InvocationStatus::Ok => "ok", crank_core::InvocationStatus::Ok => "ok",
@@ -606,9 +611,9 @@ fn new_prefixed_id(prefix: &str) -> String {
format!("{prefix}_{}", Uuid::now_v7().simple()) format!("{prefix}_{}", Uuid::now_v7().simple())
} }
fn generate_access_secret(prefix: &str) -> String { fn generate_access_secret(marker: &str) -> String {
let random = URL_SAFE_NO_PAD.encode(Uuid::now_v7().as_bytes()); let random = URL_SAFE_NO_PAD.encode(Uuid::now_v7().as_bytes());
format!("{prefix}_{random}") format!("{marker}{random}")
} }
fn hash_access_secret(secret: &str) -> String { fn hash_access_secret(secret: &str) -> String {
@@ -874,7 +879,7 @@ mod tests {
sync::{Arc, Mutex}, sync::{Arc, Mutex},
}; };
use crank_core::InvocationStatus; use crank_core::{InvocationSource, InvocationStatus};
use crank_observability::{ use crank_observability::{
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity, ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
operational_incident_total, operational_incident_total,
@@ -931,7 +936,7 @@ mod tests {
}), }),
Some("req_admin_dc08"), Some("req_admin_dc08"),
InvocationStatus::Error, InvocationStatus::Error,
"admin_test_run", InvocationSource::AgentToolCall,
); );
let output = writer.output(); let output = writer.output();
@@ -942,7 +947,7 @@ mod tests {
.find(|event: &Value| event["event"] == "admin.invocation_history.lost") .find(|event: &Value| event["event"] == "admin.invocation_history.lost")
.unwrap(); .unwrap();
assert_eq!(event["request_id"], "req_admin_dc08"); assert_eq!(event["request_id"], "req_admin_dc08");
assert_eq!(event["fields"]["source"], "admin_test_run"); assert_eq!(event["fields"]["source"], "agent_tool_call");
assert_eq!(event["fields"]["invocation_status"], "error"); assert_eq!(event["fields"]["invocation_status"], "error");
assert_eq!(event["fields"]["error_category"], "invalid_record"); assert_eq!(event["fields"]["error_category"], "invalid_record");
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before); assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+1 -4
View File
@@ -65,10 +65,7 @@ impl AdminService {
), ),
None => None, None => None,
}; };
let secret = generate_access_secret(match payload.key_kind { let secret = generate_access_secret(payload.key_kind.secret_marker());
PlatformApiKeyKind::McpClient => "crk",
PlatformApiKeyKind::Approval => "crk_appr",
});
let api_key = PlatformApiKeyRecord { let api_key = PlatformApiKeyRecord {
api_key: PlatformApiKey { api_key: PlatformApiKey {
id: PlatformApiKeyId::new(new_prefixed_id("pk")), id: PlatformApiKeyId::new(new_prefixed_id("pk")),
@@ -2,11 +2,21 @@ use std::sync::{
Arc, Arc,
atomic::{AtomicUsize, Ordering}, atomic::{AtomicUsize, Ordering},
}; };
use std::{io, sync::Mutex};
use axum::{Json, Router, extract::State, routing::post}; use axum::{Json, Router, extract::State, routing::post};
use crank_core::{OperationId, WorkspaceId};
use serde_json::{Value, json};
use serial_test::serial;
use tokio::{net::TcpListener, sync::Notify}; use tokio::{net::TcpListener, sync::Notify};
use tracing_subscriber::fmt::MakeWriter;
use super::*; #[path = "integration/common.rs"]
mod common;
use common::*;
const DEFAULT_WORKSPACE_ID: &str = "ws_default";
#[tokio::test(flavor = "multi_thread")] #[tokio::test(flavor = "multi_thread")]
#[serial] #[serial]
@@ -15,6 +25,17 @@ async fn preserves_external_success_when_invocation_history_is_lost() {
let registry_for_failure = registry.clone(); let registry_for_failure = registry.clone();
let storage_root = test_storage_root("observability_history_loss"); let storage_root = test_storage_root("observability_history_loss");
let upstream = spawn_blocking_upstream_server().await; let upstream = spawn_blocking_upstream_server().await;
let log_writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
crank_observability::ObservabilityConfig::new(
crank_observability::ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
"info",
crank_observability::RedactionLimits::default(),
),
log_writer.clone(),
)
.unwrap();
tracing::subscriber::set_global_default(subscriber).unwrap();
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await; let client = authorized_client(&base_url).await;
@@ -85,6 +106,16 @@ async fn preserves_external_success_when_invocation_history_is_lost() {
.await .await
.unwrap(); .unwrap();
assert!(logs["items"].as_array().unwrap().is_empty()); assert!(logs["items"].as_array().unwrap().is_empty());
let output = log_writer.output();
assert!(!output.contains("dc08-canary-secret"));
let incident = output
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.find(|event| event["event"] == "admin.invocation_history.lost")
.expect("DC-08 incident");
assert_eq!(incident["request_id"], "req_dc08_admin");
assert_eq!(incident["fields"]["source"], "admin_test_run");
} }
struct BlockingUpstream { struct BlockingUpstream {
@@ -139,3 +170,39 @@ async fn blocking_create_lead(
"email": payload["email"] "email": payload["email"]
})) }))
} }
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
@@ -41,8 +41,6 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret"; const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key"; const TEST_MASTER_KEY: &str = "test-master-key";
mod history_loss;
struct TestServer { struct TestServer {
base_url: String, base_url: String,
shutdown: Option<tokio::sync::oneshot::Sender<()>>, shutdown: Option<tokio::sync::oneshot::Sender<()>>,
@@ -7,7 +7,7 @@ use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
use axum::{ use axum::{
Router, Router,
body::Body, body::Body,
http::{HeaderMap, Request, StatusCode}, http::{HeaderMap, HeaderValue, Request, StatusCode},
routing::get, routing::get,
}; };
use crank_observability::{ use crank_observability::{
@@ -16,12 +16,14 @@ use crank_observability::{
use opentelemetry::{global, trace::TracerProvider as _}; use opentelemetry::{global, trace::TracerProvider as _};
use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider}; use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
use tower::ServiceExt; use tower::ServiceExt;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt}; use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
use uuid::Version; use uuid::Version;
static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[tokio::test(flavor = "current_thread")] #[tokio::test(flavor = "current_thread")]
async fn logs_request_completion_and_rejects_untrusted_values() { async fn logs_request_completion_and_rejects_untrusted_values() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let writer = SharedLogWriter::default(); let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber( let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new( ObservabilityConfig::new(
@@ -33,6 +35,7 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
) )
.unwrap(); .unwrap();
let dispatch = tracing::Dispatch::new(subscriber); let dispatch = tracing::Dispatch::new(subscriber);
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let app = probe_app(); let app = probe_app();
let response = app let response = app
@@ -44,7 +47,6 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
.body(Body::empty()) .body(Body::empty())
.unwrap(), .unwrap(),
) )
.with_subscriber(dispatch.clone())
.await .await
.unwrap(); .unwrap();
@@ -73,7 +75,6 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
.body(Body::empty()) .body(Body::empty())
.unwrap(), .unwrap(),
) )
.with_subscriber(dispatch)
.await .await
.unwrap(); .unwrap();
let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()] let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
@@ -88,12 +89,14 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
#[tokio::test(flavor = "current_thread")] #[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent() { async fn covers_valid_invalid_and_absent_traceparent() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
global::set_text_map_propagator(TraceContextPropagator::new()); global::set_text_map_propagator(TraceContextPropagator::new());
let provider = SdkTracerProvider::builder().build(); let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("admin-request-context-test"); let tracer = provider.tracer("admin-request-context-test");
let subscriber = let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber); let dispatch = tracing::Dispatch::new(subscriber);
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let app = trace_probe_app(); let app = trace_probe_app();
let valid = observed_trace_id( let valid = observed_trace_id(
@@ -109,7 +112,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
.body(Body::empty()) .body(Body::empty())
.unwrap(), .unwrap(),
) )
.with_subscriber(dispatch.clone())
.await .await
.unwrap(), .unwrap(),
); );
@@ -122,7 +124,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
.body(Body::empty()) .body(Body::empty())
.unwrap(), .unwrap(),
) )
.with_subscriber(dispatch.clone())
.await .await
.unwrap(), .unwrap(),
); );
@@ -133,7 +134,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
.body(Body::empty()) .body(Body::empty())
.unwrap(), .unwrap(),
) )
.with_subscriber(dispatch)
.await .await
.unwrap(), .unwrap(),
); );
@@ -145,6 +145,35 @@ async fn covers_valid_invalid_and_absent_traceparent() {
provider.shutdown().unwrap(); provider.shutdown().unwrap();
} }
#[tokio::test(flavor = "current_thread")]
async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let dispatch = tracing::Dispatch::new(tracing_subscriber::registry());
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let mut request = Request::builder()
.uri("/probe")
.body(Body::empty())
.unwrap();
request.headers_mut().append(
REQUEST_ID_HEADER,
HeaderValue::from_static("first-request-id"),
);
request.headers_mut().append(
REQUEST_ID_HEADER,
HeaderValue::from_static("second-request-id"),
);
let response = probe_app().oneshot(request).await.unwrap();
let generated = response.headers()[REQUEST_ID_HEADER].to_str().unwrap();
assert_ne!(generated, "first-request-id");
assert_ne!(generated, "second-request-id");
assert_eq!(
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(Version::SortRand)
);
}
fn probe_app() -> Router { fn probe_app() -> Router {
Router::new() Router::new()
.route("/probe", get(|| async { "ok" })) .route("/probe", get(|| async { "ok" }))
+227
View File
@@ -0,0 +1,227 @@
use std::{
io,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{Json, Router, routing::post};
use crank_core::{PlatformApiKeyScope, WorkspaceId};
use crank_observability::{
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
operational_incident_total,
};
use crank_registry::{ListInvocationLogsQuery, PublishRequest};
use serde_json::{Value, json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::net::TcpListener;
use tracing_subscriber::fmt::MakeWriter;
#[path = "integration/common.rs"]
mod common;
use common::*;
const CANARY_SECRET: &str = "dc08-canary-secret";
#[tokio::test]
async fn preserves_mcp_result_when_postgres_rejects_invocation_history() {
let registry = test_registry().await;
let upstream_calls = Arc::new(AtomicUsize::new(0));
let upstream_base_url = spawn_counted_upstream(Arc::clone(&upstream_calls)).await;
let operation = test_operation(&upstream_base_url, "crm_dc08");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-dc08").await;
let api_key = create_platform_api_key(
&registry,
"sales-dc08",
"mcp-dc08",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
install_history_failure_trigger(&registry).await;
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
tracing::subscriber::set_global_default(subscriber).unwrap();
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "sales-dc08");
let session_id = initialize_session(&client, &mcp_url, &api_key).await;
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
let result = post_jsonrpc_response(
&client,
&mcp_url,
&api_key,
Some(&session_id),
Some("req_mcp_dc08"),
json!({
"jsonrpc": "2.0",
"id": 3,
"method": "tools/call",
"params": {
"name": "crm_dc08",
"arguments": {
"email": format!("{CANARY_SECRET}@example.com")
}
}
}),
)
.await
.json::<Value>()
.await
.unwrap();
assert_eq!(
result["result"]["structuredContent"],
json!({"id": "lead_123"})
);
assert_eq!(result["result"]["isError"], false);
assert_eq!(upstream_calls.load(Ordering::SeqCst), 1);
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
let logs = registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &WorkspaceId::new("ws_default"),
level: None,
search_text: None,
source: None,
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
limit: 10,
})
.await
.unwrap();
assert!(logs.is_empty());
let output = writer.output();
assert!(!output.contains(CANARY_SECRET));
let incidents = output
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.filter(|event| event["event"] == "mcp.invocation_history.lost")
.collect::<Vec<_>>();
assert_eq!(incidents.len(), 1);
assert_eq!(incidents[0]["request_id"], "req_mcp_dc08");
assert_eq!(incidents[0]["fields"]["source"], "agent_tool_call");
assert_eq!(incidents[0]["fields"]["invocation_status"], "ok");
}
async fn install_history_failure_trigger(registry: &crank_registry::PostgresRegistry) {
sqlx::query(
r#"
create function fail_dc08_invocation_history() returns trigger
language plpgsql
as $$
begin
if new.request_id = 'req_mcp_dc08' then
raise exception 'forced DC-08 invocation history failure';
end if;
return new;
end;
$$
"#,
)
.execute(registry.pool())
.await
.unwrap();
sqlx::query(
r#"
create trigger fail_dc08_invocation_history
before insert on invocation_logs
for each row execute function fail_dc08_invocation_history()
"#,
)
.execute(registry.pool())
.await
.unwrap();
}
async fn spawn_counted_upstream(calls: Arc<AtomicUsize>) -> String {
let app = Router::new().route(
"/crm/leads",
post(move |Json(payload): Json<Value>| {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
Json(json!({
"id": "lead_123",
"email": payload["email"]
}))
}
}),
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://{address}")
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
@@ -518,6 +518,55 @@ async fn approval_http_endpoints_enforce_request_rate_limit() {
assert!(limited.headers().contains_key(header::RETRY_AFTER)); assert!(limited.headers().contains_key(header::RETRY_AFTER));
} }
#[tokio::test]
async fn unverified_session_ids_do_not_create_approval_rate_limit_buckets() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_approval_session_rate_limit");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-approval-session-rate-limit").await;
let approval_key = create_approval_platform_api_key(
&registry,
"sales-approval-session-rate-limit",
"approval-session-rate-limit",
)
.await;
let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
registry,
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
RequestRateLimitConfig::new(1, 1).unwrap(),
))
.await;
let approvals_url = format!(
"{}/approvals",
agent_mcp_url(&base_url, "sales-approval-session-rate-limit")
);
let client = reqwest::Client::new();
let allowed = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.header("MCP-Session-Id", "unverified-session-a")
.send()
.await
.unwrap();
assert_eq!(allowed.status(), reqwest::StatusCode::OK);
let limited = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.header("MCP-Session-Id", "unverified-session-b")
.send()
.await
.unwrap();
assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
assert!(limited.headers().contains_key(header::RETRY_AFTER));
}
#[tokio::test] #[tokio::test]
async fn recovery_does_not_repeat_interrupted_mutating_approval() { async fn recovery_does_not_repeat_interrupted_mutating_approval() {
let registry = test_registry().await; let registry = test_registry().await;
@@ -5,7 +5,7 @@ use std::{
use axum::{ use axum::{
body::Body, body::Body,
http::{Request, StatusCode}, http::{HeaderValue, Request, StatusCode},
}; };
use opentelemetry::{ use opentelemetry::{
global, global,
@@ -23,9 +23,11 @@ use tracing_subscriber::layer::SubscriberExt;
use super::common::{build_test_app, test_registry}; use super::common::{build_test_app, test_registry};
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c"; const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[tokio::test(flavor = "current_thread")] #[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() { async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
global::set_text_map_propagator(TraceContextPropagator::new()); global::set_text_map_propagator(TraceContextPropagator::new());
let exported = Arc::new(Mutex::new(Vec::new())); let exported = Arc::new(Mutex::new(Vec::new()));
let provider = SdkTracerProvider::builder() let provider = SdkTracerProvider::builder()
@@ -81,6 +83,37 @@ async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
provider.shutdown().unwrap(); provider.shutdown().unwrap();
} }
#[tokio::test]
async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let app = build_test_app(test_registry().await, Duration::ZERO, None);
let mut request = Request::builder()
.uri("/health")
.body(Body::empty())
.unwrap();
request
.headers_mut()
.append("x-request-id", HeaderValue::from_static("first-request-id"));
request.headers_mut().append(
"x-request-id",
HeaderValue::from_static("second-request-id"),
);
let response = app
.oneshot(request)
.with_subscriber(tracing_subscriber::registry())
.await
.unwrap();
let generated = response.headers()["x-request-id"].to_str().unwrap();
assert_ne!(generated, "first-request-id");
assert_ne!(generated, "second-request-id");
assert_eq!(
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(uuid::Version::SortRand)
);
}
async fn send_health( async fn send_health(
app: axum::Router, app: axum::Router,
traceparent: Option<&str>, traceparent: Option<&str>,
+3 -7
View File
@@ -6,7 +6,7 @@ use axum::{
}; };
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{OperationSecurityLevel, PlatformApiKeyScope}; use crank_core::{OperationSecurityLevel, PlatformApiKeyScope};
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query}; use crank_trace::{DbOperation, ErrorCategory, StageOutcome, observe_db_query};
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::Instrument; use tracing::Instrument;
@@ -154,9 +154,7 @@ async fn verify_static_agent_key(
secret: &str, secret: &str,
) -> Result<Option<VerifiedMachineCredential>, MachineAccessError> { ) -> Result<Option<VerifiedMachineCredential>, MachineAccessError> {
let secret_hash = hash_access_secret(secret); let secret_hash = hash_access_secret(secret);
let read_span = Stage::DbQuery let read_span = DbOperation::MachineAccessRead.span();
.db_span(DbOperation::MachineAccessRead)
.expect("database stage");
let api_key_result = state let api_key_result = state
.registry .registry
.get_platform_api_key_by_secret_for_agent_slug( .get_platform_api_key_by_secret_for_agent_slug(
@@ -185,9 +183,7 @@ async fn verify_static_agent_key(
}; };
let used_at = OffsetDateTime::now_utc(); let used_at = OffsetDateTime::now_utc();
let touch_span = Stage::DbQuery let touch_span = DbOperation::MachineAccessTouch.span();
.db_span(DbOperation::MachineAccessTouch)
.expect("database stage");
let touch_result = state let touch_result = state
.registry .registry
.touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at) .touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at)
@@ -52,9 +52,7 @@ pub(crate) async fn persist_invocation(
let history_span = Stage::HistoryWrite.span(); let history_span = Stage::HistoryWrite.span();
let (outcome, db_span) = async { let (outcome, db_span) = async {
let db_span = Stage::DbQuery let db_span = DbOperation::InvocationHistoryWrite.span();
.db_span(DbOperation::InvocationHistoryWrite)
.expect("database stage");
let outcome = state let outcome = state
.registry .registry
.create_invocation_log(CreateInvocationLogRequest { log: &log }) .create_invocation_log(CreateInvocationLogRequest { log: &log })
@@ -82,7 +80,7 @@ pub(crate) async fn persist_invocation(
outcome, outcome,
record.request_id, record.request_id,
record.status, record.status,
"agent_tool_call", InvocationSource::AgentToolCall,
); );
outcome outcome
} }
@@ -91,7 +89,7 @@ pub(super) fn observe_invocation_history_outcome(
outcome: InvocationHistoryWriteOutcome, outcome: InvocationHistoryWriteOutcome,
request_id: Option<&str>, request_id: Option<&str>,
status: InvocationStatus, status: InvocationStatus,
source: &'static str, source: InvocationSource,
) { ) {
let Some(loss) = outcome.loss() else { let Some(loss) = outcome.loss() else {
return; return;
@@ -102,13 +100,20 @@ pub(super) fn observe_invocation_history_outcome(
warn!( warn!(
name: "mcp.invocation_history.lost", name: "mcp.invocation_history.lost",
request_id = request_id.unwrap_or_default(), request_id = request_id.unwrap_or_default(),
source, source = invocation_source_label(source),
invocation_status = invocation_status_label(status), invocation_status = invocation_status_label(status),
error_category = loss.category.as_str(), error_category = loss.category.as_str(),
"invocation history was not recorded" "invocation history was not recorded"
); );
} }
fn invocation_source_label(source: InvocationSource) -> &'static str {
match source {
InvocationSource::AdminTestRun => "admin_test_run",
InvocationSource::AgentToolCall => "agent_tool_call",
}
}
fn invocation_status_label(status: InvocationStatus) -> &'static str { fn invocation_status_label(status: InvocationStatus) -> &'static str {
match status { match status {
InvocationStatus::Ok => "ok", InvocationStatus::Ok => "ok",
+1 -1
View File
@@ -75,7 +75,7 @@ fn emits_bounded_history_loss_incident() {
}), }),
Some("req_mcp_dc08"), Some("req_mcp_dc08"),
InvocationStatus::Ok, InvocationStatus::Ok,
"agent_tool_call", crank_core::InvocationSource::AgentToolCall,
); );
let output = writer.output(); let output = writer.output();
+1 -3
View File
@@ -159,9 +159,7 @@ impl PublishedToolCatalog {
return Ok(()); return Ok(());
} }
let db_span = Stage::DbQuery let db_span = DbOperation::CatalogLoad.span();
.db_span(DbOperation::CatalogLoad)
.expect("database stage");
let catalog_result = self let catalog_result = self
.registry .registry
.get_published_agent_catalog_by_slug(workspace_slug, agent_slug) .get_published_agent_catalog_by_slug(workspace_slug, agent_slug)
+9 -1
View File
@@ -4,6 +4,7 @@ use axum::{
http::{HeaderMap, HeaderValue, StatusCode, header::RETRY_AFTER}, http::{HeaderMap, HeaderValue, StatusCode, header::RETRY_AFTER},
response::{IntoResponse, Response}, response::{IntoResponse, Response},
}; };
use crank_core::PlatformApiKeyKind;
use crank_runtime::RateLimitCheckError; use crank_runtime::RateLimitCheckError;
use serde_json::{Value, json}; use serde_json::{Value, json};
@@ -82,6 +83,13 @@ pub(super) fn rate_limited_status_response(error: RateLimitCheckError) -> Respon
} }
fn rate_limit_key(path: &AgentRoutePath, headers: &HeaderMap) -> String { fn rate_limit_key(path: &AgentRoutePath, headers: &HeaderMap) -> String {
let access_secret = bearer_token(headers);
if let Some(secret) = access_secret
&& secret.starts_with(PlatformApiKeyKind::Approval.secret_marker())
{
return format!("api_key:{}", hash_access_secret(secret));
}
if let Ok(Some(session_id)) = session_id_from_headers(headers) { if let Ok(Some(session_id)) = session_id_from_headers(headers) {
return format!( return format!(
"session:{}:{}:{}", "session:{}:{}:{}",
@@ -89,7 +97,7 @@ fn rate_limit_key(path: &AgentRoutePath, headers: &HeaderMap) -> String {
); );
} }
if let Some(secret) = bearer_token(headers) { if let Some(secret) = access_secret {
return format!("api_key:{}", hash_access_secret(secret)); return format!("api_key:{}", hash_access_secret(secret));
} }
@@ -10,13 +10,7 @@ pub(super) struct RequestContext {
} }
pub(super) async fn apply_request_context(mut request: Request, next: Next) -> Response { pub(super) async fn apply_request_context(mut request: Request, next: Next) -> Response {
let request_id = RequestId::resolve( let request_id = RequestId::resolve_from_headers(request.headers()).into_string();
request
.headers()
.get(&HEADER_X_REQUEST_ID)
.and_then(|value| value.to_str().ok()),
)
.into_string();
let context = RequestContext { let context = RequestContext {
request_id: request_id.clone(), request_id: request_id.clone(),
}; };
+15
View File
@@ -42,6 +42,15 @@ pub enum PlatformApiKeyKind {
Approval, Approval,
} }
impl PlatformApiKeyKind {
pub const fn secret_marker(self) -> &'static str {
match self {
Self::McpClient => "crk_",
Self::Approval => "crk_appr_",
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub enum PlatformApiKeyScope { pub enum PlatformApiKeyScope {
@@ -123,6 +132,12 @@ mod tests {
}; };
use crate::ids::{AgentId, PlatformApiKeyId, UserId, WorkspaceId}; use crate::ids::{AgentId, PlatformApiKeyId, UserId, WorkspaceId};
#[test]
fn api_key_kinds_own_their_secret_markers() {
assert_eq!(PlatformApiKeyKind::McpClient.secret_marker(), "crk_");
assert_eq!(PlatformApiKeyKind::Approval.secret_marker(), "crk_appr_");
}
#[test] #[test]
fn user_serializes_created_at_as_rfc3339() { fn user_serializes_created_at_as_rfc3339() {
let user = User { let user = User {
@@ -1,5 +1,6 @@
use std::fmt; use std::fmt;
use axum::http::HeaderMap;
use uuid::Uuid; use uuid::Uuid;
#[derive(Clone, Debug, Eq, Hash, PartialEq)] #[derive(Clone, Debug, Eq, Hash, PartialEq)]
@@ -7,6 +8,7 @@ pub struct RequestId(String);
impl RequestId { impl RequestId {
pub const MAX_LEN: usize = 128; pub const MAX_LEN: usize = 128;
const HEADER_NAME: &'static str = "x-request-id";
pub fn resolve(candidate: Option<&str>) -> Self { pub fn resolve(candidate: Option<&str>) -> Self {
candidate candidate
@@ -15,6 +17,16 @@ impl RequestId {
.unwrap_or_else(|| Self(Uuid::now_v7().to_string())) .unwrap_or_else(|| Self(Uuid::now_v7().to_string()))
} }
pub fn resolve_from_headers(headers: &HeaderMap) -> Self {
let mut values = headers.get_all(Self::HEADER_NAME).iter();
let candidate = values.next();
if values.next().is_some() {
return Self::resolve(None);
}
Self::resolve(candidate.and_then(|value| value.to_str().ok()))
}
pub fn is_valid(value: &str) -> bool { pub fn is_valid(value: &str) -> bool {
!value.is_empty() !value.is_empty()
&& value.len() <= Self::MAX_LEN && value.len() <= Self::MAX_LEN
+5 -5
View File
@@ -597,7 +597,7 @@ mod tests {
traces_endpoint: Some("https://traces.example.test/custom".to_owned()), traces_endpoint: Some("https://traces.example.test/custom".to_owned()),
generic_endpoint: Some("https://generic.example.test/otel".to_owned()), generic_endpoint: Some("https://generic.example.test/otel".to_owned()),
traces_protocol: Some("http/protobuf".to_owned()), traces_protocol: Some("http/protobuf".to_owned()),
generic_protocol: Some("grpc".to_owned()), generic_protocol: Some("grpc".to_owned()), // community-scope: allow=grpc
traces_timeout: Some("2500".to_owned()), traces_timeout: Some("2500".to_owned()),
generic_timeout: Some("invalid-unused-fallback".to_owned()), generic_timeout: Some("invalid-unused-fallback".to_owned()),
..OtlpEnvSettings::default() ..OtlpEnvSettings::default()
@@ -615,8 +615,8 @@ mod tests {
#[test] #[test]
fn disabled_export_ignores_inactive_settings() { fn disabled_export_ignores_inactive_settings() {
let config = OtlpEnvSettings { let config = OtlpEnvSettings {
traces_protocol: Some("grpc".to_owned()), traces_protocol: Some("grpc".to_owned()), // community-scope: allow=grpc
generic_protocol: Some("grpc".to_owned()), generic_protocol: Some("grpc".to_owned()), // community-scope: allow=grpc
traces_timeout: Some("invalid".to_owned()), traces_timeout: Some("invalid".to_owned()),
generic_timeout: Some("invalid".to_owned()), generic_timeout: Some("invalid".to_owned()),
max_queue_size: Some("invalid".to_owned()), max_queue_size: Some("invalid".to_owned()),
@@ -637,7 +637,7 @@ mod tests {
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()), traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
traces_headers: Some(String::new()), traces_headers: Some(String::new()),
generic_headers: Some( generic_headers: Some(
"authorization=Bearer%20canary-token,x-tenant=community".to_owned(), "authorization=Bearer%20canary-token,x-scope=community".to_owned(),
), ),
..OtlpEnvSettings::default() ..OtlpEnvSettings::default()
} }
@@ -645,7 +645,7 @@ mod tests {
.unwrap(); .unwrap();
assert_eq!(config.header("authorization"), Some("Bearer canary-token")); assert_eq!(config.header("authorization"), Some("Bearer canary-token"));
assert_eq!(config.header("x-tenant"), Some("community")); assert_eq!(config.header("x-scope"), Some("community"));
assert!(!format!("{config:?}").contains("canary-token")); assert!(!format!("{config:?}").contains("canary-token"));
} }
@@ -1,3 +1,4 @@
use axum::http::{HeaderMap, HeaderValue};
use crank_observability::RequestId; use crank_observability::RequestId;
use uuid::Version; use uuid::Version;
@@ -40,3 +41,34 @@ fn rejects_values_over_the_shared_limit() {
Some(Version::SortRand) Some(Version::SortRand)
); );
} }
#[test]
fn resolves_exactly_one_header_value_and_rejects_ambiguous_values() {
let mut single = HeaderMap::new();
single.insert(
"x-request-id",
HeaderValue::from_static("opaque-request-id"),
);
assert_eq!(
RequestId::resolve_from_headers(&single).as_str(),
"opaque-request-id"
);
let mut ambiguous = HeaderMap::new();
ambiguous.append("x-request-id", HeaderValue::from_static("first-request-id"));
ambiguous.append(
"x-request-id",
HeaderValue::from_static("second-request-id"),
);
let generated = RequestId::resolve_from_headers(&ambiguous);
assert_ne!(generated.as_str(), "first-request-id");
assert_ne!(generated.as_str(), "second-request-id");
assert_eq!(
uuid::Uuid::parse_str(generated.as_str())
.expect("generated UUID")
.get_version(),
Some(Version::SortRand)
);
}
+1 -1
View File
@@ -37,7 +37,7 @@ fn invalid_values_return_safe_typed_errors() {
let secret_endpoint = "https://user:canary-secret@collector.example.test/v1/traces"; let secret_endpoint = "https://user:canary-secret@collector.example.test/v1/traces";
let error = OtlpTraceConfig::try_new( let error = OtlpTraceConfig::try_new(
Some(secret_endpoint.to_owned()), Some(secret_endpoint.to_owned()),
Some("grpc".to_owned()), Some("grpc".to_owned()), // community-scope: allow=grpc
Duration::ZERO, Duration::ZERO,
OtlpBatchConfig::default(), OtlpBatchConfig::default(),
) )
+4 -3
View File
@@ -11,6 +11,7 @@ use serde_json::{Map, Value, json};
use time::OffsetDateTime; use time::OffsetDateTime;
use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use tracing::{Instrument, Span, debug}; use tracing::{Instrument, Span, debug};
use uuid::Uuid;
use crate::{ use crate::{
AdapterResponse, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeLimits, RuntimeOperation, AdapterResponse, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeLimits, RuntimeOperation,
@@ -517,9 +518,9 @@ impl RuntimeExecutor {
fn adapter_request_context( fn adapter_request_context(
request_context: Option<&RuntimeRequestContext>, request_context: Option<&RuntimeRequestContext>,
) -> crank_core::RuntimeRequestContext { ) -> crank_core::RuntimeRequestContext {
request_context request_context.map(Into::into).unwrap_or_else(|| {
.map(Into::into) crank_core::RuntimeRequestContext::from_request_id(Uuid::now_v7().to_string())
.unwrap_or_else(|| crank_core::RuntimeRequestContext::new(String::new(), String::new())) })
} }
fn map_protocol_adapter_error( fn map_protocol_adapter_error(
@@ -2,6 +2,5 @@ mod integration {
mod confirmation; mod confirmation;
mod idempotency; mod idempotency;
mod no_input_get; mod no_input_get;
mod stages;
mod valkey; mod valkey;
} }
@@ -25,8 +25,12 @@ async fn valkey_coordination_and_rate_limit_operations_are_atomic() {
.get_host_port_ipv4(6379.tcp()) .get_host_port_ipv4(6379.tcp())
.await .await
.expect("Valkey port must be mapped"); .expect("Valkey port must be mapped");
let host = container
.get_host()
.await
.expect("Docker host must be resolved");
let store = Arc::new( let store = Arc::new(
RedisCacheStore::connect(CacheBackend::Valkey, &format!("redis://127.0.0.1:{port}/0")) RedisCacheStore::connect(CacheBackend::Valkey, &format!("redis://{host}:{port}/0"))
.await .await
.expect("runtime store must connect to Valkey"), .expect("runtime store must connect to Valkey"),
); );
@@ -20,9 +20,13 @@ use serde_json::json;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::{Id, Instrument, Subscriber, field::Visit, instrument::WithSubscriber}; use tracing::{Id, Instrument, Subscriber, field::Visit, instrument::WithSubscriber};
use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan}; use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan};
use uuid::Version;
static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[tokio::test] #[tokio::test]
async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() { async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let capture = TraceCapture::default(); let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone()); let subscriber = tracing_subscriber::registry().with(capture.clone());
let executor = RuntimeExecutorBuilder::new() let executor = RuntimeExecutorBuilder::new()
@@ -82,6 +86,7 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
#[tokio::test] #[tokio::test]
async fn failed_mapping_records_closed_category_and_stops_later_stages() { async fn failed_mapping_records_closed_category_and_stops_later_stages() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let capture = TraceCapture::default(); let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone()); let subscriber = tracing_subscriber::registry().with(capture.clone());
let executor = RuntimeExecutorBuilder::new() let executor = RuntimeExecutorBuilder::new()
@@ -110,8 +115,46 @@ async fn failed_mapping_records_closed_category_and_stops_later_stages() {
); );
} }
#[tokio::test]
async fn execution_without_context_sends_generated_correlation_headers() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let captured_headers = Arc::new(Mutex::new(None));
let executor = RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(ContextCapturingAdapter {
captured_headers: Arc::clone(&captured_headers),
}))
.build();
let operation = operation().into();
let result = async {
executor
.execute(&operation, &json!({"name": "without-context"}))
.await
}
.with_subscriber(tracing_subscriber::registry())
.await;
assert_eq!(result.unwrap(), json!({"accepted": true}));
let headers = captured_headers
.lock()
.expect("captured headers lock")
.clone()
.expect("adapter headers");
let request_id = headers.get("x-request-id").expect("x-request-id");
let correlation_id = headers.get("x-correlation-id").expect("x-correlation-id");
assert!(!request_id.is_empty());
assert_eq!(correlation_id, request_id);
assert_eq!(
uuid::Uuid::parse_str(request_id)
.expect("generated UUID")
.get_version(),
Some(Version::SortRand)
);
}
#[tokio::test] #[tokio::test]
async fn approval_stage_is_present_only_when_confirmation_is_required() { async fn approval_stage_is_present_only_when_confirmation_is_required() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let capture = TraceCapture::default(); let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone()); let subscriber = tracing_subscriber::registry().with(capture.clone());
let executor = RuntimeExecutorBuilder::new() let executor = RuntimeExecutorBuilder::new()
@@ -151,6 +194,7 @@ async fn approval_stage_is_present_only_when_confirmation_is_required() {
#[tokio::test] #[tokio::test]
async fn idempotency_stage_distinguishes_execution_from_replay() { async fn idempotency_stage_distinguishes_execution_from_replay() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let capture = TraceCapture::default(); let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone()); let subscriber = tracing_subscriber::registry().with(capture.clone());
let executor = RuntimeExecutorBuilder::new() let executor = RuntimeExecutorBuilder::new()
@@ -238,6 +282,37 @@ impl ProtocolAdapter for SuccessAdapter {
} }
} }
struct ContextCapturingAdapter {
captured_headers: Arc<Mutex<Option<BTreeMap<String, String>>>>,
}
#[async_trait]
impl ProtocolAdapter for ContextCapturingAdapter {
fn protocol(&self) -> Protocol {
Protocol::Rest
}
fn supports_mode(&self, mode: ExecutionMode) -> bool {
mode == ExecutionMode::Unary
}
async fn invoke_unary(
&self,
_target: &Target,
_prepared: &crank_core::PreparedRequest,
context: &crank_core::RuntimeRequestContext,
) -> Result<AdapterResponse, ProtocolAdapterError> {
*self.captured_headers.lock().expect("captured headers lock") =
Some(context.outbound_headers());
Ok(AdapterResponse {
status_code: 200,
headers: BTreeMap::new(),
body: json!({"accepted": true}),
data: json!({"accepted": true}),
})
}
}
fn operation() -> Operation<Schema, MappingSet> { fn operation() -> Operation<Schema, MappingSet> {
Operation { Operation {
id: OperationId::new("op_stage_test"), id: OperationId::new("op_stage_test"),
+7 -12
View File
@@ -63,24 +63,13 @@ impl Stage {
), ),
} }
} }
pub fn db_span(self, operation: DbOperation) -> Option<Span> {
if self != Self::DbQuery {
return None;
}
let span = self.span();
span.record("db.operation", operation.as_str());
Some(span)
}
} }
pub async fn observe_db_query<T, E>( pub async fn observe_db_query<T, E>(
operation: DbOperation, operation: DbOperation,
future: impl Future<Output = Result<T, E>>, future: impl Future<Output = Result<T, E>>,
) -> Result<T, E> { ) -> Result<T, E> {
let span = Stage::DbQuery let span = operation.span();
.db_span(operation)
.expect("database operation requires db.query stage");
let result = future.instrument(span.clone()).await; let result = future.instrument(span.clone()).await;
match &result { match &result {
Ok(_) => StageOutcome::Success.record(&span), Ok(_) => StageOutcome::Success.record(&span),
@@ -182,6 +171,12 @@ pub enum DbOperation {
} }
impl DbOperation { impl DbOperation {
pub fn span(self) -> Span {
let span = Stage::DbQuery.span();
span.record("db.operation", self.as_str());
span
}
pub const fn as_str(self) -> &'static str { pub const fn as_str(self) -> &'static str {
match self { match self {
Self::MachineAccessRead => "machine_access.read", Self::MachineAccessRead => "machine_access.read",
+3 -9
View File
@@ -15,9 +15,7 @@ fn stage_names_and_attributes_are_closed() {
ErrorCategory::Mapping.record(&span); ErrorCategory::Mapping.record(&span);
drop(span); drop(span);
let db_span = Stage::DbQuery let db_span = DbOperation::InvocationHistoryWrite.span();
.db_span(DbOperation::InvocationHistoryWrite)
.expect("db stage accepts a db operation");
StageOutcome::Error.record(&db_span); StageOutcome::Error.record(&db_span);
drop(db_span); drop(db_span);
}); });
@@ -32,12 +30,8 @@ fn stage_names_and_attributes_are_closed() {
} }
#[test] #[test]
fn non_database_stage_rejects_database_attributes() { fn database_operation_names_are_closed() {
assert!( assert_eq!(DbOperation::CatalogLoad.as_str(), "catalog.load");
Stage::RuntimeExecute
.db_span(DbOperation::CatalogLoad)
.is_none()
);
} }
#[derive(Clone)] #[derive(Clone)]
+13
View File
@@ -20,6 +20,11 @@ DEFAULT_EXCLUDED_PREFIXES = {
"apps/ui/dist/", "apps/ui/dist/",
} }
ALLOW_DIRECTIVE = re.compile(
r"community-scope:\s*allow=([a-z0-9-]+(?:,[a-z0-9-]+)*)",
re.IGNORECASE,
)
FORBIDDEN_PATTERNS = [ FORBIDDEN_PATTERNS = [
("enterprise", re.compile(r"\benterprise\b", re.IGNORECASE)), ("enterprise", re.compile(r"\benterprise\b", re.IGNORECASE)),
("cloud", re.compile(r"\bcloud\b", re.IGNORECASE)), ("cloud", re.compile(r"\bcloud\b", re.IGNORECASE)),
@@ -100,7 +105,15 @@ def scan_file(root: Path, relative_path: str) -> list[str]:
text = data.decode("utf-8", errors="replace") text = data.decode("utf-8", errors="replace")
findings: list[str] = [] findings: list[str] = []
for line_no, line in enumerate(text.splitlines(), start=1): for line_no, line in enumerate(text.splitlines(), start=1):
directive = ALLOW_DIRECTIVE.search(line)
allowed_markers = (
{label.lower() for label in directive.group(1).split(",")}
if directive
else set()
)
for label, pattern in FORBIDDEN_PATTERNS: for label, pattern in FORBIDDEN_PATTERNS:
if label in allowed_markers:
continue
if pattern.search(line): if pattern.search(line):
findings.append(f"{relative_path}:{line_no}: forbidden marker `{label}`") findings.append(f"{relative_path}:{line_no}: forbidden marker `{label}`")
return findings return findings
+32
View File
@@ -47,6 +47,38 @@ class CommunityScopeCheckTests(unittest.TestCase):
self.assertIn("apps/ui.md:1", result.stderr) self.assertIn("apps/ui.md:1", result.stderr)
self.assertIn("graphql", result.stderr.lower()) self.assertIn("graphql", result.stderr.lower())
def test_inline_directive_allows_only_declared_marker(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "protocol.rs").write_text(
'assert_rejected_protocol("grpc"); '
"// community-scope: allow=grpc\n"
'assert_rejected_protocol("grpc");\n'
'assert_rejected_protocol("graphql"); '
"// community-scope: allow=grpc\n",
encoding="utf-8",
)
result = self.run_checker(root, ["protocol.rs"])
self.assertNotEqual(result.returncode, 0)
self.assertIn("protocol.rs:2: forbidden marker `grpc`", result.stderr)
self.assertIn("protocol.rs:3: forbidden marker `graphql`", result.stderr)
self.assertNotIn("protocol.rs:1:", result.stderr)
def test_rejects_marker_without_inline_directive(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "README.md").write_text(
"Enable grpc transport.\n",
encoding="utf-8",
)
result = self.run_checker(root, ["README.md"])
self.assertNotEqual(result.returncode, 0)
self.assertIn("forbidden marker `grpc`", result.stderr)
def test_ignores_default_excluded_paths(self) -> None: def test_ignores_default_excluded_paths(self) -> None:
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp) root = Path(tmp)