наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

crates/crank-community-mcp/src/approval_execution.rs

@@ -5,11 +5,13 @@ use axum::{
     response::{IntoResponse, Response},
 };
 use crank_core::{ApprovalRequestStatus, InvocationLevel, InvocationSource, InvocationStatus};
+use crank_observability::RequestId;
 use crank_registry::{ApprovalRequestRecord, FinishApprovalRequest};
 use crank_runtime::{RuntimeExecutionRequest, RuntimeRequestContext};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use serde_json::json;
 use time::OffsetDateTime;
-use tracing::warn;
+use tracing::{Instrument, warn};
 
 use crate::{
     app::{
@@ -35,32 +37,81 @@ pub(super) fn spawn_approval_recovery(state: Arc<AppState>) {
 }
 
 async fn recover_approved_requests(state: &Arc<AppState>) {
+    fail_interrupted_requests(state).await;
+
     for _ in 0..32 {
         let now = OffsetDateTime::now_utc();
-        let approval = match state
-            .registry
-            .claim_next_recoverable_approval_request(
-                now,
-                now - RECOVERY_GRACE,
-                now - EXECUTION_LEASE,
-            )
-            .await
+        let approval = match observe_db_query(
+            DbOperation::ApprovalWrite,
+            state
+                .registry
+                .claim_next_recoverable_approval_request(now, now - RECOVERY_GRACE),
+        )
+        .await
         {
             Ok(Some(approval)) => approval,
             Ok(None) => break,
-            Err(error) => {
-                warn!(error = %error, "approval recovery query failed");
+            Err(_) => {
+                warn!(
+                    name: "mcp.approval_recovery.query_failed",
+                    error_category = "registry",
+                    "approval recovery query failed"
+                );
                 break;
             }
         };
         let Some(path) = approval_agent_path(state, &approval).await else {
             continue;
         };
-        if execute_approved_request(state, &path, approval)
-            .await
-            .is_err()
+        let recovery_span = Stage::ApprovalRecovery.span();
+        let result = execute_approved_request(state, &path, approval, None)
+            .instrument(recovery_span.clone())
+            .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&recovery_span),
+            Err(_) => {
+                StageOutcome::Error.record(&recovery_span);
+                ErrorCategory::Approval.record(&recovery_span);
+            }
+        }
+        drop(recovery_span);
+        if result.is_err() {
+            warn!(
+                name: "mcp.approval_recovery.execution_failed",
+                error_category = "runtime",
+                "recovered approval execution did not finish"
+            );
+        }
+    }
+}
+
+async fn fail_interrupted_requests(state: &Arc<AppState>) {
+    for _ in 0..32 {
+        let stale_before = OffsetDateTime::now_utc() - EXECUTION_LEASE;
+        match observe_db_query(
+            DbOperation::ApprovalWrite,
+            state
+                .registry
+                .fail_next_interrupted_approval_request(stale_before),
+        )
+        .await
         {
-            warn!("recovered approval execution did not finish");
+            Ok(Some(approval)) => {
+                warn!(
+                    name: "mcp.approval_recovery.interrupted",
+                    approval_id = approval.approval.id.as_str(),
+                    "interrupted approval execution was not retried because its outcome is unknown"
+                );
+            }
+            Ok(None) => break,
+            Err(_) => {
+                warn!(
+                    name: "mcp.approval_recovery.interrupted_query_failed",
+                    error_category = "registry",
+                    "interrupted approval recovery query failed"
+                );
+                break;
+            }
         }
     }
 }
@@ -76,8 +127,12 @@ async fn approval_agent_path(
     {
         Ok(Some(workspace)) => workspace,
         Ok(None) => return None,
-        Err(error) => {
-            warn!(error = %error, "approval workspace lookup failed");
+        Err(_) => {
+            warn!(
+                name: "mcp.approval_recovery.workspace_lookup_failed",
+                error_category = "registry",
+                "approval workspace lookup failed"
+            );
             return None;
         }
     };
@@ -88,8 +143,12 @@ async fn approval_agent_path(
     {
         Ok(Some(agent)) => agent,
         Ok(None) => return None,
-        Err(error) => {
-            warn!(error = %error, "approval agent lookup failed");
+        Err(_) => {
+            warn!(
+                name: "mcp.approval_recovery.agent_lookup_failed",
+                error_category = "registry",
+                "approval agent lookup failed"
+            );
             return None;
         }
     };
@@ -103,7 +162,9 @@ pub(super) async fn execute_approved_request(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     approval: ApprovalRequestRecord,
+    request_id: Option<&str>,
 ) -> Result<ApprovalRequestRecord, Response> {
+    let request_id = RequestId::resolve(request_id).into_string();
     let tools = state
         .catalog
         .list_tools(&path.workspace_slug, &path.agent_slug)
@@ -123,18 +184,17 @@ pub(super) async fn execute_approved_request(
         &approval.approval.request_payload,
     );
     let started_at = Instant::now();
-    let runtime_request_context =
-        RuntimeRequestContext::from_request_id(approval.approval.id.as_str().to_owned())
-            .with_response_cache_scope(
-                tool.workspace_id.as_str().to_owned(),
-                tool.agent_id.as_str().to_owned(),
-            )
-            .with_metering_context(
-                tool.workspace_id.clone(),
-                Some(tool.agent_id.clone()),
-                InvocationSource::AgentToolCall,
-            )
-            .with_approval_granted();
+    let runtime_request_context = RuntimeRequestContext::from_request_id(request_id.clone())
+        .with_response_cache_scope(
+            tool.workspace_id.as_str().to_owned(),
+            tool.agent_id.as_str().to_owned(),
+        )
+        .with_metering_context(
+            tool.workspace_id.clone(),
+            Some(tool.agent_id.clone()),
+            InvocationSource::AgentToolCall,
+        )
+        .with_approval_granted();
     let resolved_auth =
         resolve_operation_auth(state, &tool.workspace_id, &operation.execution_config).await;
     let result = match resolved_auth {
@@ -176,11 +236,11 @@ pub(super) async fn execute_approved_request(
             ),
         };
 
-    if let Err(error) = persist_invocation(
+    persist_invocation(
         state,
         &tool,
         InvocationRecord {
-            request_id: Some(approval.approval.id.as_str()),
+            request_id: Some(&request_id),
             tool_name: &tool.tool_name,
             status: invocation_status,
             level: invocation_level,
@@ -192,46 +252,49 @@ pub(super) async fn execute_approved_request(
             response_preview: response_payload.clone(),
         },
     )
-    .await
-    {
-        warn!(error = %error, "approved invocation log write failed");
-    }
+    .await;
 
-    state
-        .registry
-        .finish_approval_request(FinishApprovalRequest {
-            workspace_id: &approval.approval.workspace_id,
-            agent_id: &approval.approval.agent_id,
-            approval_id: &approval.approval.id,
-            status,
-            response_payload: Some(response_payload),
-            decision_note: None,
-        })
-        .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
-        .ok_or_else(|| StatusCode::CONFLICT.into_response())
+    observe_db_query(
+        DbOperation::ApprovalWrite,
+        state
+            .registry
+            .finish_approval_request(FinishApprovalRequest {
+                workspace_id: &approval.approval.workspace_id,
+                agent_id: &approval.approval.agent_id,
+                approval_id: &approval.approval.id,
+                status,
+                response_payload: Some(response_payload),
+                decision_note: None,
+            }),
+    )
+    .await
+    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
+    .ok_or_else(|| StatusCode::CONFLICT.into_response())
 }
 
 async fn finish_unavailable_approval(
     state: &Arc<AppState>,
     approval: &ApprovalRequestRecord,
 ) -> Result<ApprovalRequestRecord, Response> {
-    state
-        .registry
-        .finish_approval_request(FinishApprovalRequest {
-            workspace_id: &approval.approval.workspace_id,
-            agent_id: &approval.approval.agent_id,
-            approval_id: &approval.approval.id,
-            status: ApprovalRequestStatus::Failed,
-            response_payload: Some(json!({
-                "error": {
-                    "code": "approved_operation_unavailable",
-                    "message": "the approved operation version is no longer published"
-                }
-            })),
-            decision_note: None,
-        })
-        .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
-        .ok_or_else(|| StatusCode::CONFLICT.into_response())
+    observe_db_query(
+        DbOperation::ApprovalWrite,
+        state
+            .registry
+            .finish_approval_request(FinishApprovalRequest {
+                workspace_id: &approval.approval.workspace_id,
+                agent_id: &approval.approval.agent_id,
+                approval_id: &approval.approval.id,
+                status: ApprovalRequestStatus::Failed,
+                response_payload: Some(json!({
+                    "error": {
+                        "code": "approved_operation_unavailable",
+                        "message": "the approved operation version is no longer published"
+                    }
+                })),
+                decision_note: None,
+            }),
+    )
+    .await
+    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
+    .ok_or_else(|| StatusCode::CONFLICT.into_response())
 }