наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.
2026-07-31 01:01:14 +03:00
parent 99bd05c145
commit 0e8f1ca03a
160 changed files with 13506 additions and 1499 deletions
@@ -12,16 +12,18 @@ use crank_core::{
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_registry::{
-    AgentSummary, CreateInvocationLogRequest, OperationAgentRef, OperationSummary,
-    OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
+    AgentSummary, CreateInvocationLogRequest, InvocationHistoryWriteOutcome, OperationAgentRef,
+    OperationSummary, OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
 };
 use crank_runtime::{
    OutboundHttpPolicy, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
 };
 use crank_schema::{Schema, SchemaKind};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use serde_json::{Value, json};
 use sha2::{Digest, Sha256};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tracing::Instrument;
 use uuid::Uuid;

 mod agents;
@@ -74,6 +76,11 @@ pub struct AdminServiceBuilder {
 pub use crate::dto::*;

 impl AdminService {
+    pub async fn readiness(&self) -> Result<(), ApiError> {
+        self.registry.ping().await?;
+        Ok(())
+    }
+
    #[cfg(test)]
    pub fn new(
        registry: PostgresRegistry,
@@ -199,16 +206,33 @@ impl AdminServiceBuilder {
 }

 impl AdminService {
-    pub async fn export_workspace(
+    pub async fn export_workspace_catalog_snapshot(
        &self,
        workspace_id: &WorkspaceId,
-    ) -> Result<WorkspaceExportResponse, ApiError> {
+    ) -> Result<WorkspaceCatalogSnapshotResponse, ApiError> {
        let workspace = self.get_workspace(workspace_id).await?;
        let operations = self.list_operations(workspace_id).await?;
        let agents = self.list_agents(workspace_id).await?;
        let platform_api_keys = self.registry.list_platform_api_keys(workspace_id).await?;

-        Ok(WorkspaceExportResponse {
+        Ok(WorkspaceCatalogSnapshotResponse {
+            kind: "workspace_catalog_snapshot".to_owned(),
+            format_version: "1".to_owned(),
+            restorable: false,
+            included: vec![
+                "workspace_settings".to_owned(),
+                "operation_summaries".to_owned(),
+                "agent_summaries".to_owned(),
+                "platform_api_key_metadata".to_owned(),
+            ],
+            excluded: vec![
+                "operation_versions_and_samples".to_owned(),
+                "agent_versions_and_bindings".to_owned(),
+                "secret_metadata_and_values".to_owned(),
+                "secret_values".to_owned(),
+                "invocation_logs_and_usage".to_owned(),
+                "authentication_sessions".to_owned(),
+            ],
            workspace,
            operations,
            agents,
@@ -239,9 +263,13 @@ impl AdminService {
            return Ok(None);
        };

-        let auth_profile = self
-            .registry
-            .get_auth_profile(workspace_id, auth_profile_id)
+        let span = Stage::AuthResolve.span();
+        let result = async {
+            let auth_profile = observe_db_query(
+                DbOperation::AuthProfileRead,
+                self.registry
+                    .get_auth_profile(workspace_id, auth_profile_id),
+            )
            .await
            .map_err(|error| RuntimeError::SecretCrypto {
                operation: "load auth profile",
@@ -251,9 +279,20 @@ impl AdminService {
                auth_profile_id: auth_profile_id.as_str().to_owned(),
            })?;

-        self.resolve_auth_profile(workspace_id, &auth_profile)
-            .await
-            .map(Some)
+            self.resolve_auth_profile(workspace_id, &auth_profile)
+                .await
+                .map(Some)
+        }
+        .instrument(span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&span),
+            Err(_) => {
+                StageOutcome::Error.record(&span);
+                ErrorCategory::Configuration.record(&span);
+            }
+        }
+        result
    }

    async fn resolve_auth_profile(
@@ -265,40 +304,46 @@ impl AdminService {
        let used_at = OffsetDateTime::now_utc();

        for secret_id in auth_profile.config.secret_ids() {
-            let secret = self
-                .registry
-                .get_secret(workspace_id, secret_id)
-                .await
-                .map_err(|error| RuntimeError::SecretCrypto {
-                    operation: "load secret",
-                    details: error.to_string(),
-                })?
-                .ok_or_else(|| RuntimeError::MissingSecret {
-                    secret_id: secret_id.as_str().to_owned(),
-                })?;
-            let version = self
-                .registry
-                .get_current_secret_version(workspace_id, secret_id)
-                .await
-                .map_err(|error| RuntimeError::SecretCrypto {
-                    operation: "load current secret version",
-                    details: error.to_string(),
-                })?
-                .ok_or_else(|| RuntimeError::MissingSecretVersion {
-                    secret_id: secret_id.as_str().to_owned(),
-                    version: secret.secret.current_version,
-                })?;
+            let secret = observe_db_query(
+                DbOperation::SecretRead,
+                self.registry.get_secret(workspace_id, secret_id),
+            )
+            .await
+            .map_err(|error| RuntimeError::SecretCrypto {
+                operation: "load secret",
+                details: error.to_string(),
+            })?
+            .ok_or_else(|| RuntimeError::MissingSecret {
+                secret_id: secret_id.as_str().to_owned(),
+            })?;
+            let version = observe_db_query(
+                DbOperation::SecretRead,
+                self.registry
+                    .get_current_secret_version(workspace_id, secret_id),
+            )
+            .await
+            .map_err(|error| RuntimeError::SecretCrypto {
+                operation: "load current secret version",
+                details: error.to_string(),
+            })?
+            .ok_or_else(|| RuntimeError::MissingSecretVersion {
+                secret_id: secret_id.as_str().to_owned(),
+                version: secret.secret.current_version,
+            })?;
            let plaintext = self.secret_crypto.decrypt(
                &version.secret_version.key_version,
                &version.secret_version.ciphertext,
            )?;
-            self.registry
-                .touch_secret(workspace_id, secret_id, &used_at)
-                .await
-                .map_err(|error| RuntimeError::SecretCrypto {
-                    operation: "touch secret",
-                    details: error.to_string(),
-                })?;
+            observe_db_query(
+                DbOperation::SecretTouch,
+                self.registry
+                    .touch_secret(workspace_id, secret_id, &used_at),
+            )
+            .await
+            .map_err(|error| RuntimeError::SecretCrypto {
+                operation: "touch secret",
+                details: error.to_string(),
+            })?;
            secrets.insert(secret_id.clone(), plaintext);
        }

@@ -412,7 +457,7 @@ impl AdminService {
    async fn record_invocation(
        &self,
        request: InvocationRecordRequest<'_>,
-    ) -> Result<(), ApiError> {
+    ) -> InvocationHistoryWriteOutcome {
        let log = InvocationLog {
            id: InvocationLogId::new(new_prefixed_id("log")),
            workspace_id: request.workspace_id.clone(),
@@ -432,11 +477,70 @@ impl AdminService {
            created_at: OffsetDateTime::now_utc(),
        };

-        self.registry
-            .create_invocation_log(CreateInvocationLogRequest { log: &log })
-            .await?;
+        let history_span = crank_trace::Stage::HistoryWrite.span();
+        let (outcome, db_span) = async {
+            let db_span = crank_trace::Stage::DbQuery
+                .db_span(crank_trace::DbOperation::InvocationHistoryWrite)
+                .expect("database stage");
+            let outcome = self
+                .registry
+                .create_invocation_log(CreateInvocationLogRequest { log: &log })
+                .instrument(db_span.clone())
+                .await;
+            (outcome, db_span)
+        }
+        .instrument(history_span.clone())
+        .await;
+        match outcome {
+            InvocationHistoryWriteOutcome::Recorded => {
+                crank_trace::StageOutcome::Success.record(&db_span);
+                crank_trace::StageOutcome::Success.record(&history_span);
+            }
+            InvocationHistoryWriteOutcome::Lost(_) => {
+                crank_trace::StageOutcome::Error.record(&db_span);
+                crank_trace::ErrorCategory::Database.record(&db_span);
+                crank_trace::StageOutcome::Error.record(&history_span);
+                crank_trace::ErrorCategory::History.record(&history_span);
+            }
+        }
+        drop(db_span);
+        drop(history_span);
+        observe_invocation_history_outcome(
+            outcome,
+            request.request_id,
+            request.status,
+            "admin_test_run",
+        );
+        outcome
+    }
+}

-        Ok(())
+fn observe_invocation_history_outcome(
+    outcome: InvocationHistoryWriteOutcome,
+    request_id: Option<&str>,
+    status: crank_core::InvocationStatus,
+    source: &'static str,
+) {
+    let Some(loss) = outcome.loss() else {
+        return;
+    };
+    crank_observability::record_operational_incident(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+    tracing::warn!(
+        name: "admin.invocation_history.lost",
+        request_id = request_id.unwrap_or_default(),
+        source,
+        invocation_status = invocation_status_label(status),
+        error_category = loss.category.as_str(),
+        "invocation history was not recorded"
+    );
+}
+
+fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
+    match status {
+        crank_core::InvocationStatus::Ok => "ok",
+        crank_core::InvocationStatus::Error => "error",
    }
 }

@@ -543,6 +647,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
        RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
        RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
        RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
        RuntimeError::RestAdapter(_) => "rest_error",
        RuntimeError::ProtocolAdapter(_) => "adapter_error",
        RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
@@ -761,7 +869,25 @@ fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule {
 #[cfg(test)]
 #[allow(clippy::items_after_test_module)]
 mod tests {
-    use super::{validate_profile_display_name, validate_profile_email};
+    use std::{
+        io,
+        sync::{Arc, Mutex},
+    };
+
+    use crank_core::InvocationStatus;
+    use crank_observability::{
+        ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
+        operational_incident_total,
+    };
+    use crank_registry::{
+        InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
+    };
+    use serde_json::Value;
+    use tracing_subscriber::fmt::MakeWriter;
+
+    use super::{
+        observe_invocation_history_outcome, validate_profile_display_name, validate_profile_email,
+    };

    #[test]
    fn validates_profile_identity_fields() {
@@ -782,6 +908,81 @@ mod tests {
        assert!(validate_profile_display_name(&"x".repeat(81)).is_err());
        assert!(validate_profile_email("owner <html>@crank.local").is_err());
    }
+
+    #[test]
+    fn emits_bounded_history_loss_incident() {
+        let writer = SharedLogWriter::default();
+        let subscriber = crank_observability::build_subscriber(
+            ObservabilityConfig::new(
+                ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+                "info",
+                RedactionLimits::default(),
+            ),
+            writer.clone(),
+        )
+        .unwrap();
+        let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let _guard = tracing::dispatcher::set_default(&dispatch);
+
+        observe_invocation_history_outcome(
+            InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
+                category: InvocationHistoryLossCategory::InvalidRecord,
+            }),
+            Some("req_admin_dc08"),
+            InvocationStatus::Error,
+            "admin_test_run",
+        );
+
+        let output = writer.output();
+        assert!(!output.contains("dc08-canary-secret"));
+        let event: Value = output
+            .lines()
+            .map(|line| serde_json::from_str(line).unwrap())
+            .find(|event: &Value| event["event"] == "admin.invocation_history.lost")
+            .unwrap();
+        assert_eq!(event["request_id"], "req_admin_dc08");
+        assert_eq!(event["fields"]["source"], "admin_test_run");
+        assert_eq!(event["fields"]["invocation_status"], "error");
+        assert_eq!(event["fields"]["error_category"], "invalid_record");
+        assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+    }
+
+    #[derive(Clone, Default)]
+    struct SharedLogWriter {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl SharedLogWriter {
+        fn output(&self) -> String {
+            String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+        }
+    }
+
+    impl<'a> MakeWriter<'a> for SharedLogWriter {
+        type Writer = SharedLogGuard;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            SharedLogGuard {
+                buffer: Arc::clone(&self.buffer),
+            }
+        }
+    }
+
+    struct SharedLogGuard {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl io::Write for SharedLogGuard {
+        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+            self.buffer.lock().unwrap().extend_from_slice(bytes);
+            Ok(bytes.len())
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
 }

 fn enrich_operation_summary(