наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

apps/mcp-server/tests/integration/catalog_access/approval_access.rs

@@ -89,11 +89,16 @@ async fn approval_key_lists_and_decides_pending_requests() {
     let approved = client
         .post(&approve_url)
         .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .header("x-request-id", "req_approval_execute_123")
         .json(&json!({ "approve": "yes", "note": "confirmed by test" }))
         .send()
         .await
         .unwrap();
     assert_eq!(approved.status(), reqwest::StatusCode::OK);
+    assert_eq!(
+        approved.headers()["x-request-id"].to_str().unwrap(),
+        "req_approval_execute_123"
+    );
     let approved_body = approved.json::<Value>().await.unwrap();
     assert_eq!(
         approved_body["approval"]["status"],
@@ -169,6 +174,10 @@ async fn approval_key_lists_and_decides_pending_requests() {
         .await
         .unwrap();
     assert_eq!(logs.len(), 1);
+    assert_eq!(
+        logs[0].log.request_id.as_deref(),
+        Some("req_approval_execute_123")
+    );
 }
 
 #[tokio::test]
@@ -401,22 +410,23 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
     let mcp_url = agent_mcp_url(&base_url, "sales-gated");
     let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
 
+    let tool_call = json!({
+        "jsonrpc": "2.0",
+        "id": 9,
+        "method": "tools/call",
+        "params": {
+            "name": "crm_requires_human_approval",
+            "arguments": {
+                "email": "ada@example.com"
+            }
+        }
+    });
     let tool_result = post_jsonrpc(
         &client,
         &mcp_url,
         &api_key,
         Some(&initialized_session),
-        json!({
-            "jsonrpc": "2.0",
-            "id": 9,
-            "method": "tools/call",
-            "params": {
-                "name": "crm_requires_human_approval",
-                "arguments": {
-                    "email": "ada@example.com"
-                }
-            }
-        }),
+        tool_call.clone(),
     )
     .await;
 
@@ -430,6 +440,19 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
         .unwrap();
     assert!(approval_id.starts_with("approval_"));
 
+    let repeated_tool_result = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&initialized_session),
+        tool_call,
+    )
+    .await;
+    assert_eq!(
+        repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id,
+        "deduplicated tools/call must return the persisted approval id",
+    );
+
     let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated"));
     let pending = client
         .get(&approvals_url)
@@ -448,6 +471,189 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
     );
 }
 
+#[tokio::test]
+async fn approval_http_endpoints_enforce_request_rate_limit() {
+    let registry = test_registry().await;
+    let upstream_base_url = spawn_upstream_server().await;
+    let operation = test_operation(&upstream_base_url, "crm_approval_rate_limit");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-approval-rate-limit").await;
+    let approval_key = create_approval_platform_api_key(
+        &registry,
+        "sales-approval-rate-limit",
+        "approval-rate-limit",
+    )
+    .await;
+    let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
+        registry,
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+        RequestRateLimitConfig::new(1, 1).unwrap(),
+    ))
+    .await;
+    let approvals_url = format!(
+        "{}/approvals",
+        agent_mcp_url(&base_url, "sales-approval-rate-limit")
+    );
+    let client = reqwest::Client::new();
+
+    let allowed = client
+        .get(&approvals_url)
+        .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(allowed.status(), reqwest::StatusCode::OK);
+
+    let limited = client
+        .get(&approvals_url)
+        .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
+    assert!(limited.headers().contains_key(header::RETRY_AFTER));
+}
+
+#[tokio::test]
+async fn recovery_does_not_repeat_interrupted_mutating_approval() {
+    let registry = test_registry().await;
+    let (upstream_base_url, upstream_calls) = spawn_counted_approval_upstream().await;
+    let operation = test_operation(&upstream_base_url, "crm_interrupted_approval");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-interrupted-approval").await;
+    let approval_key_name = "approval-interrupted";
+    create_approval_platform_api_key(&registry, "sales-interrupted-approval", approval_key_name)
+        .await;
+    let now = OffsetDateTime::now_utc();
+    let approval = ApprovalRequest {
+        id: ApprovalRequestId::new("approval_interrupted_mutation"),
+        workspace_id: test_workspace_id(),
+        agent_id: test_agent_id("sales-interrupted-approval"),
+        operation_id: operation.id.clone(),
+        operation_version: operation.version,
+        status: ApprovalRequestStatus::Pending,
+        risk_level: OperationApprovalRiskLevel::Dangerous,
+        request_payload: json!({"email": "interrupted@example.com"}),
+        response_payload: None,
+        created_at: now - time::Duration::minutes(10),
+        expires_at: now + time::Duration::minutes(5),
+        decided_at: None,
+        decided_by_key_id: None,
+        decision_note: None,
+    };
+    registry
+        .create_approval_request(CreateApprovalRequest {
+            approval: &approval,
+        })
+        .await
+        .unwrap();
+    let approval_key_id = PlatformApiKeyId::new(format!("pk_{approval_key_name}"));
+    registry
+        .decide_approval_request(crank_registry::DecideApprovalRequest {
+            workspace_id: &approval.workspace_id,
+            agent_id: &approval.agent_id,
+            approval_id: &approval.id,
+            status: ApprovalRequestStatus::Approved,
+            decided_at: now - time::Duration::minutes(10),
+            decided_by_key_id: &approval_key_id,
+            response_payload: Some(json!({"approve": "yes"})),
+            decision_note: None,
+        })
+        .await
+        .unwrap()
+        .unwrap();
+    registry
+        .claim_approval_request(
+            &approval.workspace_id,
+            &approval.agent_id,
+            &approval.id,
+            now - time::Duration::minutes(7),
+        )
+        .await
+        .unwrap()
+        .unwrap();
+
+    let _app = build_test_app_with_approval_recovery(registry.clone());
+    let failed = tokio::time::timeout(Duration::from_secs(2), async {
+        loop {
+            let current = registry
+                .get_approval_request_for_agent(
+                    &approval.workspace_id,
+                    &approval.agent_id,
+                    &approval.id,
+                )
+                .await
+                .unwrap()
+                .unwrap();
+            if current.approval.status == ApprovalRequestStatus::Failed {
+                break current;
+            }
+            tokio::time::sleep(Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("recovery must quarantine interrupted execution");
+
+    assert_eq!(
+        failed.approval.response_payload.unwrap()["error"]["code"],
+        "approval_execution_outcome_unknown"
+    );
+    assert_eq!(
+        upstream_calls.load(std::sync::atomic::Ordering::SeqCst),
+        0,
+        "recovery must not repeat a mutating upstream request"
+    );
+}
+
+fn build_test_app_with_approval_recovery(registry: PostgresRegistry) -> Router {
+    crank_community_mcp::build_app_with_background_workers(
+        registry,
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+        SecretCrypto::new("test-master-key").unwrap(),
+        crank_runtime::community_with_outbound_policy(
+            crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
+        )
+        .build(),
+        RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
+        Arc::new(InMemoryCoordinationStateStore::default()),
+        Arc::new(InMemorySessionStore::default()),
+        Arc::new(CommunityMachineCredentialVerifier),
+    )
+}
+
+async fn spawn_counted_approval_upstream() -> (String, Arc<std::sync::atomic::AtomicUsize>) {
+    let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let handler_calls = Arc::clone(&calls);
+    let app = Router::new().route(
+        "/crm/leads",
+        post(move |Json(payload): Json<Value>| {
+            let calls = Arc::clone(&handler_calls);
+            async move {
+                calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                Json(json!({
+                    "id": "lead_123",
+                    "email": payload["email"]
+                }))
+            }
+        }),
+    );
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    (format!("http://{address}"), calls)
+}
+
 #[tokio::test]
 async fn elicitation_approval_requires_client_capability() {
     let registry = test_registry().await;