наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.
2026-07-31 01:01:14 +03:00
parent 99bd05c145
commit 0e8f1ca03a
160 changed files with 13506 additions and 1499 deletions
@@ -0,0 +1,117 @@
+use std::{sync::Arc, time::Duration};
+
+use crank_core::{
+    InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus,
+};
+use crank_registry::{
+    CreateInvocationLogRequest, InvocationHistoryWriteOutcome, PublishedAgentTool,
+};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome};
+use serde_json::Value;
+use time::OffsetDateTime;
+use tracing::{Instrument, warn};
+
+use super::AppState;
+
+pub(crate) struct InvocationRecord<'a> {
+    pub(crate) request_id: Option<&'a str>,
+    pub(crate) tool_name: &'a str,
+    pub(crate) status: InvocationStatus,
+    pub(crate) level: InvocationLevel,
+    pub(crate) message: &'a str,
+    pub(crate) status_code: Option<u16>,
+    pub(crate) error_kind: Option<&'a str>,
+    pub(crate) duration: Duration,
+    pub(crate) request_preview: Value,
+    pub(crate) response_preview: Value,
+}
+
+pub(crate) async fn persist_invocation(
+    state: &Arc<AppState>,
+    tool: &PublishedAgentTool,
+    record: InvocationRecord<'_>,
+) -> InvocationHistoryWriteOutcome {
+    let log = InvocationLog {
+        id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())),
+        workspace_id: tool.workspace_id.clone(),
+        agent_id: Some(tool.agent_id.clone()),
+        operation_id: tool.operation.id.clone(),
+        source: InvocationSource::AgentToolCall,
+        level: record.level,
+        status: record.status,
+        tool_name: record.tool_name.to_owned(),
+        message: record.message.to_owned(),
+        request_id: record.request_id.map(ToOwned::to_owned),
+        status_code: record.status_code,
+        duration_ms: u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX),
+        error_kind: record.error_kind.map(ToOwned::to_owned),
+        request_preview: record.request_preview,
+        response_preview: record.response_preview,
+        created_at: OffsetDateTime::now_utc(),
+    };
+
+    let history_span = Stage::HistoryWrite.span();
+    let (outcome, db_span) = async {
+        let db_span = Stage::DbQuery
+            .db_span(DbOperation::InvocationHistoryWrite)
+            .expect("database stage");
+        let outcome = state
+            .registry
+            .create_invocation_log(CreateInvocationLogRequest { log: &log })
+            .instrument(db_span.clone())
+            .await;
+        (outcome, db_span)
+    }
+    .instrument(history_span.clone())
+    .await;
+    match outcome {
+        InvocationHistoryWriteOutcome::Recorded => {
+            StageOutcome::Success.record(&db_span);
+            StageOutcome::Success.record(&history_span);
+        }
+        InvocationHistoryWriteOutcome::Lost(_) => {
+            StageOutcome::Error.record(&db_span);
+            ErrorCategory::Database.record(&db_span);
+            StageOutcome::Error.record(&history_span);
+            ErrorCategory::History.record(&history_span);
+        }
+    }
+    drop(db_span);
+    drop(history_span);
+    observe_invocation_history_outcome(
+        outcome,
+        record.request_id,
+        record.status,
+        "agent_tool_call",
+    );
+    outcome
+}
+
+pub(super) fn observe_invocation_history_outcome(
+    outcome: InvocationHistoryWriteOutcome,
+    request_id: Option<&str>,
+    status: InvocationStatus,
+    source: &'static str,
+) {
+    let Some(loss) = outcome.loss() else {
+        return;
+    };
+    crank_observability::record_operational_incident(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+    warn!(
+        name: "mcp.invocation_history.lost",
+        request_id = request_id.unwrap_or_default(),
+        source,
+        invocation_status = invocation_status_label(status),
+        error_category = loss.category.as_str(),
+        "invocation history was not recorded"
+    );
+}
+
+fn invocation_status_label(status: InvocationStatus) -> &'static str {
+    match status {
+        InvocationStatus::Ok => "ok",
+        InvocationStatus::Error => "error",
+    }
+}
@@ -0,0 +1,93 @@
+use std::sync::Arc;
+
+use axum::http::StatusCode;
+use serde_json::Value;
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
+
+use crate::{
+    jsonrpc::{is_notification, is_response, method_name},
+    transport::ResponseMode,
+};
+
+pub(super) struct McpRequestMetrics {
+    method: &'static str,
+    response_mode: &'static str,
+    outcome: &'static str,
+}
+
+impl McpRequestMetrics {
+    pub(super) fn new(message: &Value) -> Self {
+        Self {
+            method: normalized_mcp_method(message),
+            response_mode: "unknown",
+            outcome: "rejected",
+        }
+    }
+
+    pub(super) fn set_response_mode(&mut self, mode: ResponseMode) -> ResponseMode {
+        self.response_mode = match mode {
+            ResponseMode::Json => "json",
+            ResponseMode::Sse => "sse",
+        };
+        mode
+    }
+
+    pub(super) fn complete(&mut self, status: StatusCode) {
+        self.outcome = match status.as_u16() {
+            200..=299 => "success",
+            400..=499 => "client_error",
+            500..=599 => "server_error",
+            _ => "other",
+        };
+    }
+}
+
+impl Drop for McpRequestMetrics {
+    fn drop(&mut self) {
+        ::metrics::counter!(
+            "crank_mcp_requests_total",
+            "method" => self.method,
+            "response_mode" => self.response_mode,
+            "outcome" => self.outcome
+        )
+        .increment(1);
+    }
+}
+
+pub(super) fn normalized_mcp_method(message: &Value) -> &'static str {
+    match method_name(message) {
+        Some("initialize") => "initialize",
+        Some("notifications/initialized") => "initialized",
+        Some("ping") => "ping",
+        Some("tools/list") => "tools_list",
+        Some("tools/call") => "tools_call",
+        Some(_) if is_notification(message) => "notification",
+        Some(_) => "unsupported",
+        None if is_response(message) => "response",
+        None => "invalid",
+    }
+}
+
+pub(super) struct ActiveSessionGuard {
+    _permit: OwnedSemaphorePermit,
+}
+
+impl ActiveSessionGuard {
+    pub(super) fn try_acquire(slots: &Arc<Semaphore>) -> Result<Self, ()> {
+        let permit = Arc::clone(slots).try_acquire_owned().map_err(|_| {
+            ::metrics::counter!(
+                "crank_runtime_limit_rejections_total",
+                "stage" => "mcp_session"
+            )
+            .increment(1);
+        })?;
+        ::metrics::gauge!("crank_mcp_active_sessions").increment(1.0);
+        Ok(Self { _permit: permit })
+    }
+}
+
+impl Drop for ActiveSessionGuard {
+    fn drop(&mut self) {
+        ::metrics::gauge!("crank_mcp_active_sessions").decrement(1.0);
+    }
+}
@@ -0,0 +1,86 @@
+use std::sync::Arc;
+
+use axum::http::{HeaderMap, StatusCode};
+use crank_core::PlatformApiKeyScope;
+use crank_registry::PlatformApiKeyRecord;
+use crank_runtime::RateLimitCheckError;
+use crank_trace::{ErrorCategory, Stage, StageOutcome};
+use tracing::Instrument;
+
+use crate::{
+    access::{MachineAccessError, require_approval_access, require_machine_access},
+    app::{AgentRoutePath, AppState},
+    auth::VerifiedMachineCredential,
+    rate_limit::enforce_transport_rate_limit,
+};
+
+pub(super) async fn enforce_traced_rate_limit(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+) -> Result<(), RateLimitCheckError> {
+    let span = Stage::McpRateLimit.span();
+    let result = enforce_transport_rate_limit(state, path, headers)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(()) => StageOutcome::Allowed.record(&span),
+        Err(RateLimitCheckError::Rejected(_)) => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::RateLimit.record(&span);
+        }
+        Err(RateLimitCheckError::StoreUnavailable) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}
+
+pub(super) async fn require_traced_machine_access(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+    required_scope: PlatformApiKeyScope,
+) -> Result<VerifiedMachineCredential, MachineAccessError> {
+    let span = Stage::McpAccessCheck.span();
+    let result = require_machine_access(state, path, headers, required_scope)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(_) => StageOutcome::Allowed.record(&span),
+        Err(error) if error.is_denied() => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::Access.record(&span);
+        }
+        Err(_) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}
+
+pub(super) async fn require_traced_approval_access(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+    required_scope: PlatformApiKeyScope,
+) -> Result<PlatformApiKeyRecord, StatusCode> {
+    let span = Stage::McpAccessCheck.span();
+    let result = require_approval_access(state, path, headers, required_scope)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(_) => StageOutcome::Allowed.record(&span),
+        Err(status) if *status == StatusCode::UNAUTHORIZED || *status == StatusCode::FORBIDDEN => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::Access.record(&span);
+        }
+        Err(_) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}
@@ -0,0 +1,149 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use axum::body::to_bytes;
+use crank_core::InvocationStatus;
+use crank_observability::{
+    ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
+    operational_incident_total,
+};
+use crank_registry::{
+    InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
+};
+use serde_json::{Value, json};
+use tracing_subscriber::fmt::MakeWriter;
+
+use super::{
+    ResponseMode, metrics::normalized_mcp_method, observe_invocation_history_outcome,
+    tool_error_response,
+};
+use crate::jsonrpc::CURRENT_PROTOCOL_VERSION;
+use crate::tool_error::generic_tool_error_contract;
+
+#[tokio::test]
+async fn tool_error_response_includes_structured_context() {
+    let response = tool_error_response(
+        &json!({"jsonrpc": "2.0", "id": "req-1"}),
+        ResponseMode::Json,
+        CURRENT_PROTOCOL_VERSION,
+        generic_tool_error_contract(
+            "streaming_payload_error",
+            "request root must be an object",
+            "req-1",
+            false,
+            Some("Проверьте параметры вызова инструмента."),
+        ),
+    );
+
+    let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    let payload: Value = serde_json::from_slice(&body).unwrap();
+
+    assert_eq!(
+        payload["result"]["structuredContent"]["error"],
+        json!({
+            "code": "streaming_payload_error",
+            "error_code": "streaming_payload_error",
+            "message": "request root must be an object",
+            "recoverable": false,
+            "request_id": "req-1",
+            "suggested_action": "Проверьте параметры вызова инструмента."
+        })
+    );
+}
+
+#[test]
+fn emits_bounded_history_loss_incident() {
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _guard = tracing::dispatcher::set_default(&dispatch);
+
+    observe_invocation_history_outcome(
+        InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
+            category: InvocationHistoryLossCategory::Unavailable,
+        }),
+        Some("req_mcp_dc08"),
+        InvocationStatus::Ok,
+        "agent_tool_call",
+    );
+
+    let output = writer.output();
+    assert!(!output.contains("dc08-canary-secret"));
+    let event: Value = output
+        .lines()
+        .map(|line| serde_json::from_str(line).unwrap())
+        .find(|event: &Value| event["event"] == "mcp.invocation_history.lost")
+        .unwrap();
+    assert_eq!(event["request_id"], "req_mcp_dc08");
+    assert_eq!(event["fields"]["source"], "agent_tool_call");
+    assert_eq!(event["fields"]["invocation_status"], "ok");
+    assert_eq!(event["fields"]["error_category"], "unavailable");
+    assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+}
+
+#[test]
+fn mcp_metric_method_is_always_from_a_closed_set() {
+    assert_eq!(
+        normalized_mcp_method(&json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call"})),
+        "tools_call"
+    );
+    assert_eq!(
+        normalized_mcp_method(
+            &json!({"jsonrpc": "2.0", "id": 2, "method": "customer-controlled-method"})
+        ),
+        "unsupported"
+    );
+    assert_eq!(
+        normalized_mcp_method(
+            &json!({"jsonrpc": "2.0", "method": "customer-controlled-notification"})
+        ),
+        "notification"
+    );
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}