наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.
2026-07-31 01:01:14 +03:00
parent 99bd05c145
commit 0e8f1ca03a
160 changed files with 13506 additions and 1499 deletions
@@ -0,0 +1,117 @@
+use std::{sync::Arc, time::Duration};
+
+use crank_core::{
+    InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus,
+};
+use crank_registry::{
+    CreateInvocationLogRequest, InvocationHistoryWriteOutcome, PublishedAgentTool,
+};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome};
+use serde_json::Value;
+use time::OffsetDateTime;
+use tracing::{Instrument, warn};
+
+use super::AppState;
+
+pub(crate) struct InvocationRecord<'a> {
+    pub(crate) request_id: Option<&'a str>,
+    pub(crate) tool_name: &'a str,
+    pub(crate) status: InvocationStatus,
+    pub(crate) level: InvocationLevel,
+    pub(crate) message: &'a str,
+    pub(crate) status_code: Option<u16>,
+    pub(crate) error_kind: Option<&'a str>,
+    pub(crate) duration: Duration,
+    pub(crate) request_preview: Value,
+    pub(crate) response_preview: Value,
+}
+
+pub(crate) async fn persist_invocation(
+    state: &Arc<AppState>,
+    tool: &PublishedAgentTool,
+    record: InvocationRecord<'_>,
+) -> InvocationHistoryWriteOutcome {
+    let log = InvocationLog {
+        id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())),
+        workspace_id: tool.workspace_id.clone(),
+        agent_id: Some(tool.agent_id.clone()),
+        operation_id: tool.operation.id.clone(),
+        source: InvocationSource::AgentToolCall,
+        level: record.level,
+        status: record.status,
+        tool_name: record.tool_name.to_owned(),
+        message: record.message.to_owned(),
+        request_id: record.request_id.map(ToOwned::to_owned),
+        status_code: record.status_code,
+        duration_ms: u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX),
+        error_kind: record.error_kind.map(ToOwned::to_owned),
+        request_preview: record.request_preview,
+        response_preview: record.response_preview,
+        created_at: OffsetDateTime::now_utc(),
+    };
+
+    let history_span = Stage::HistoryWrite.span();
+    let (outcome, db_span) = async {
+        let db_span = Stage::DbQuery
+            .db_span(DbOperation::InvocationHistoryWrite)
+            .expect("database stage");
+        let outcome = state
+            .registry
+            .create_invocation_log(CreateInvocationLogRequest { log: &log })
+            .instrument(db_span.clone())
+            .await;
+        (outcome, db_span)
+    }
+    .instrument(history_span.clone())
+    .await;
+    match outcome {
+        InvocationHistoryWriteOutcome::Recorded => {
+            StageOutcome::Success.record(&db_span);
+            StageOutcome::Success.record(&history_span);
+        }
+        InvocationHistoryWriteOutcome::Lost(_) => {
+            StageOutcome::Error.record(&db_span);
+            ErrorCategory::Database.record(&db_span);
+            StageOutcome::Error.record(&history_span);
+            ErrorCategory::History.record(&history_span);
+        }
+    }
+    drop(db_span);
+    drop(history_span);
+    observe_invocation_history_outcome(
+        outcome,
+        record.request_id,
+        record.status,
+        "agent_tool_call",
+    );
+    outcome
+}
+
+pub(super) fn observe_invocation_history_outcome(
+    outcome: InvocationHistoryWriteOutcome,
+    request_id: Option<&str>,
+    status: InvocationStatus,
+    source: &'static str,
+) {
+    let Some(loss) = outcome.loss() else {
+        return;
+    };
+    crank_observability::record_operational_incident(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+    warn!(
+        name: "mcp.invocation_history.lost",
+        request_id = request_id.unwrap_or_default(),
+        source,
+        invocation_status = invocation_status_label(status),
+        error_category = loss.category.as_str(),
+        "invocation history was not recorded"
+    );
+}
+
+fn invocation_status_label(status: InvocationStatus) -> &'static str {
+    match status {
+        InvocationStatus::Ok => "ok",
+        InvocationStatus::Error => "error",
+    }
+}