Усилить безопасность и надёжность выполнения операций

crates/crank-community-mcp/src/app.rs

@@ -18,9 +18,8 @@ use crank_core::{
     OperationApprovalMode, PlatformApiKeyScope, SecretId,
 };
 use crank_registry::{
-    ApprovalRequestRecord, CreateApprovalRequest, CreateInvocationLogRequest,
-    DecideApprovalRequest, ExpireApprovalRequest, FinishApprovalRequest, PostgresRegistry,
-    PublishedAgentTool,
+    CreateApprovalRequest, CreateInvocationLogRequest, DecideApprovalRequest,
+    ExpireApprovalRequest, PostgresRegistry, PublishedAgentTool,
 };
 use crank_runtime::{
     RequestRateLimiter, ResolvedAuth, RuntimeError, RuntimeExecutionRequest, RuntimeExecutor,
@@ -30,13 +29,14 @@ use futures_util::stream;
 use serde::{Deserialize, Serialize};
 use serde_json::{Value, json};
 use time::OffsetDateTime;
-use tracing::info;
+use tracing::{info, warn};
 
 use crate::{
     access::{
         credential_allows_security_level, require_approval_access, require_machine_access,
         serialize_machine_access_mode, serialize_security_level,
     },
+    approval_execution::{execute_approved_request, spawn_approval_recovery},
     auth::{SharedMachineCredentialVerifier, VerifiedMachineCredential},
     catalog::PublishedToolCatalog,
     jsonrpc::{
@@ -66,8 +66,8 @@ const TRANSPORT_SESSION_TTL_MS: u64 = 86_400_000;
 #[derive(Clone)]
 pub(super) struct AppState {
     pub(super) registry: PostgresRegistry,
-    catalog: PublishedToolCatalog,
-    runtime: RuntimeExecutor,
+    pub(super) catalog: PublishedToolCatalog,
+    pub(super) runtime: RuntimeExecutor,
     pub(super) api_rate_limiter: RequestRateLimiter,
     secret_crypto: SecretCrypto,
     sessions: SharedSessionStore,
@@ -132,6 +132,59 @@ pub fn build_app(
     coordination_store: Arc<dyn CoordinationStateStore>,
     sessions: SharedSessionStore,
     credential_verifier: SharedMachineCredentialVerifier,
+) -> Router {
+    build_app_inner(
+        registry,
+        refresh_interval,
+        public_base_url,
+        secret_crypto,
+        runtime,
+        api_rate_limiter,
+        coordination_store,
+        sessions,
+        credential_verifier,
+        false,
+    )
+}
+
+#[allow(clippy::too_many_arguments)]
+pub fn build_app_with_background_workers(
+    registry: PostgresRegistry,
+    refresh_interval: Duration,
+    public_base_url: Option<String>,
+    secret_crypto: SecretCrypto,
+    runtime: RuntimeExecutor,
+    api_rate_limiter: RequestRateLimiter,
+    coordination_store: Arc<dyn CoordinationStateStore>,
+    sessions: SharedSessionStore,
+    credential_verifier: SharedMachineCredentialVerifier,
+) -> Router {
+    build_app_inner(
+        registry,
+        refresh_interval,
+        public_base_url,
+        secret_crypto,
+        runtime,
+        api_rate_limiter,
+        coordination_store,
+        sessions,
+        credential_verifier,
+        true,
+    )
+}
+
+#[allow(clippy::too_many_arguments)]
+fn build_app_inner(
+    registry: PostgresRegistry,
+    refresh_interval: Duration,
+    public_base_url: Option<String>,
+    secret_crypto: SecretCrypto,
+    runtime: RuntimeExecutor,
+    api_rate_limiter: RequestRateLimiter,
+    coordination_store: Arc<dyn CoordinationStateStore>,
+    sessions: SharedSessionStore,
+    credential_verifier: SharedMachineCredentialVerifier,
+    start_background_workers: bool,
 ) -> Router {
     let state = Arc::new(AppState {
         registry: registry.clone(),
@@ -143,6 +196,9 @@ pub fn build_app(
         credential_verifier,
         allowed_origins: AllowedOrigins::new(public_base_url),
     });
+    if start_background_workers {
+        spawn_approval_recovery(Arc::clone(&state));
+    }
 
     Router::new()
         .route("/health", get(health))
@@ -315,7 +371,21 @@ async fn decide_approval_request(
         .await
     {
         Ok(Some(record)) if status == ApprovalRequestStatus::Approved => {
-            match execute_approved_request(&state, &agent_path, record).await {
+            let claimed = match state
+                .registry
+                .claim_approval_request(
+                    &record.approval.workspace_id,
+                    &record.approval.agent_id,
+                    &record.approval.id,
+                    OffsetDateTime::now_utc(),
+                )
+                .await
+            {
+                Ok(Some(claimed)) => claimed,
+                Ok(None) => return StatusCode::CONFLICT.into_response(),
+                Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(),
+            };
+            match execute_approved_request(&state, &agent_path, claimed).await {
                 Ok(record) => Json(json!(record)).into_response(),
                 Err(response) => response,
             }
@@ -408,103 +478,6 @@ async fn expire_approval_response(
     }
 }
 
-async fn execute_approved_request(
-    state: &Arc<AppState>,
-    path: &AgentRoutePath,
-    approval: ApprovalRequestRecord,
-) -> Result<ApprovalRequestRecord, Response> {
-    let tools = state
-        .catalog
-        .list_tools(&path.workspace_slug, &path.agent_slug)
-        .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?;
-    let Some(tool) = tools.into_iter().find(|tool| {
-        tool.operation.id == approval.approval.operation_id
-            && tool.operation.version == approval.approval.operation_version
-    }) else {
-        return Err(StatusCode::NOT_FOUND.into_response());
-    };
-
-    let operation = runtime_operation(&tool);
-    let request_preview = build_request_preview(
-        &state.runtime,
-        &operation,
-        &approval.approval.request_payload,
-    );
-    let started_at = Instant::now();
-    let resolved_auth =
-        resolve_operation_auth(state, &tool.workspace_id, &operation.execution_config).await;
-    let result = match resolved_auth {
-        Ok(resolved_auth) => {
-            state
-                .runtime
-                .execute_request(
-                    RuntimeExecutionRequest::new(&operation, &approval.approval.request_payload)
-                        .with_optional_auth(resolved_auth.as_ref()),
-                )
-                .await
-        }
-        Err(error) => Err(error),
-    };
-
-    let (status, response_payload, invocation_status, invocation_level, message, error_kind) =
-        match result {
-            Ok(output) => (
-                ApprovalRequestStatus::Completed,
-                output,
-                InvocationStatus::Ok,
-                InvocationLevel::Info,
-                "approved tool call completed",
-                None,
-            ),
-            Err(error) => (
-                ApprovalRequestStatus::Failed,
-                json!({
-                    "error": {
-                        "code": runtime_error_code(&error),
-                        "message": error.to_string(),
-                    }
-                }),
-                InvocationStatus::Error,
-                InvocationLevel::Error,
-                "approved tool call failed",
-                Some(runtime_error_code(&error)),
-            ),
-        };
-
-    let _ = persist_invocation(
-        state,
-        &tool,
-        InvocationRecord {
-            request_id: Some(approval.approval.id.as_str()),
-            tool_name: &tool.tool_name,
-            status: invocation_status,
-            level: invocation_level,
-            message,
-            status_code: None,
-            error_kind,
-            duration: started_at.elapsed(),
-            request_preview,
-            response_preview: response_payload.clone(),
-        },
-    )
-    .await;
-
-    state
-        .registry
-        .finish_approval_request(FinishApprovalRequest {
-            workspace_id: &approval.approval.workspace_id,
-            agent_id: &approval.approval.agent_id,
-            approval_id: &approval.approval.id,
-            status,
-            response_payload: Some(response_payload),
-            decision_note: None,
-        })
-        .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
-        .ok_or_else(|| StatusCode::CONFLICT.into_response())
-}
-
 async fn mcp_get(
     Path(path): Path<AgentRoutePath>,
     State(state): State<Arc<AppState>>,
@@ -857,7 +830,7 @@ async fn handle_tool_call(
     .await
 }
 
-async fn resolve_operation_auth(
+pub(super) async fn resolve_operation_auth(
     state: &Arc<AppState>,
     workspace_id: &crank_core::WorkspaceId,
     execution_config: &crank_core::ExecutionConfig,
@@ -1004,7 +977,7 @@ async fn handle_base_tool_call(
 
     match result {
         Ok(output) => {
-            let _ = persist_invocation(
+            if let Err(error) = persist_invocation(
                 &state,
                 &tool,
                 InvocationRecord {
@@ -1020,12 +993,15 @@ async fn handle_base_tool_call(
                     response_preview: output.clone(),
                 },
             )
-            .await;
+            .await
+            {
+                warn!(error = %error, "successful invocation log write failed");
+            }
 
             success_tool_response(message, response_mode, &session.protocol_version, output)
         }
         Err(error) => {
-            let _ = persist_invocation(
+            if let Err(log_error) = persist_invocation(
                 &state,
                 &tool,
                 InvocationRecord {
@@ -1041,7 +1017,10 @@ async fn handle_base_tool_call(
                     response_preview: Value::Null,
                 },
             )
-            .await;
+            .await
+            {
+                warn!(error = %log_error, "failed invocation log write failed");
+            }
 
             tool_error_response(
                 message,
@@ -1146,17 +1125,22 @@ async fn maybe_create_custom_pending_approval(
         decision_note: None,
     };
 
-    if let Err(error) = state
+    let persisted_approval = match state
         .registry
         .create_approval_request(CreateApprovalRequest {
             approval: &approval,
         })
         .await
     {
-        return Some(internal_jsonrpc_error(message, error));
-    }
+        Ok(approval) => approval,
+        Err(error) => return Some(internal_jsonrpc_error(message, error)),
+    };
+    let response_payload = persisted_approval
+        .approval
+        .response_payload
+        .unwrap_or(response_payload);
 
-    let _ = persist_invocation(
+    if let Err(error) = persist_invocation(
         state,
         tool,
         InvocationRecord {
@@ -1172,7 +1156,10 @@ async fn maybe_create_custom_pending_approval(
             response_preview: response_payload.clone(),
         },
     )
-    .await;
+    .await
+    {
+        warn!(error = %error, "pending approval invocation log write failed");
+    }
 
     Some(success_tool_response(
         message,
@@ -1404,7 +1391,7 @@ fn take_confirmation_token(arguments: &mut Value) -> Option<String> {
         .filter(|value| !value.trim().is_empty())
 }
 
-fn build_request_preview(
+pub(super) fn build_request_preview(
     runtime: &RuntimeExecutor,
     operation: &RuntimeOperation,
     arguments: &Value,
@@ -1420,20 +1407,20 @@ fn build_request_preview(
     }
 }
 
-struct InvocationRecord<'a> {
-    request_id: Option<&'a str>,
-    tool_name: &'a str,
-    status: InvocationStatus,
-    level: InvocationLevel,
-    message: &'a str,
-    status_code: Option<u16>,
-    error_kind: Option<&'a str>,
-    duration: Duration,
-    request_preview: Value,
-    response_preview: Value,
+pub(super) struct InvocationRecord<'a> {
+    pub(super) request_id: Option<&'a str>,
+    pub(super) tool_name: &'a str,
+    pub(super) status: InvocationStatus,
+    pub(super) level: InvocationLevel,
+    pub(super) message: &'a str,
+    pub(super) status_code: Option<u16>,
+    pub(super) error_kind: Option<&'a str>,
+    pub(super) duration: Duration,
+    pub(super) request_preview: Value,
+    pub(super) response_preview: Value,
 }
 
-async fn persist_invocation(
+pub(super) async fn persist_invocation(
     state: &Arc<AppState>,
     tool: &PublishedAgentTool,
     record: InvocationRecord<'_>,
@@ -1543,7 +1530,7 @@ fn resolve_generated_tool(
     None
 }
 
-fn runtime_operation(tool: &PublishedAgentTool) -> RuntimeOperation {
+pub(super) fn runtime_operation(tool: &PublishedAgentTool) -> RuntimeOperation {
     let mut operation = RuntimeOperation::from(tool.operation.clone());
     operation.tool_name = tool.tool_name.clone();
     operation.tool_description.title = tool.tool_title.clone();