наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

apps/admin-api/tests/integration/community_access_usage.rs

@@ -23,6 +23,7 @@ use crank_schema::{Schema, SchemaKind};
 use serde_json::{Value, json};
 use serial_test::serial;
 use tokio::net::TcpListener;
+use uuid::Version;
 
 use admin_api::{
     app::build_app,
@@ -40,6 +41,8 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
 const TEST_SESSION_SECRET: &str = "test-session-secret";
 const TEST_MASTER_KEY: &str = "test-master-key";
 
+mod history_loss;
+
 struct TestServer {
     base_url: String,
     shutdown: Option<tokio::sync::oneshot::Sender<()>>,
@@ -385,6 +388,30 @@ async fn exports_single_workspace_but_rejects_access_lifecycle() {
         exported["workspace"]["workspace"]["id"],
         DEFAULT_WORKSPACE_ID
     );
+    assert_eq!(exported["kind"], "workspace_catalog_snapshot");
+    assert_eq!(exported["format_version"], "1");
+    assert_eq!(exported["restorable"], false);
+    assert_eq!(
+        exported["included"],
+        json!([
+            "workspace_settings",
+            "operation_summaries",
+            "agent_summaries",
+            "platform_api_key_metadata"
+        ])
+    );
+    assert!(
+        exported["excluded"]
+            .as_array()
+            .unwrap()
+            .contains(&json!("secret_values"))
+    );
+    assert!(
+        exported["excluded"]
+            .as_array()
+            .unwrap()
+            .contains(&json!("invocation_logs_and_usage"))
+    );
     assert!(exported.get("memberships").is_none());
     assert!(exported.get("invitations").is_none());
 
@@ -574,6 +601,25 @@ async fn updates_profile_and_changes_password() {
         .unwrap()
         .to_owned();
     let client = authorized_client(&base_url).await;
+    let second_client = reqwest::Client::builder()
+        .cookie_store(true)
+        .build()
+        .unwrap();
+    let second_login = second_client
+        .post(format!("{root_url}/api/auth/login"))
+        .json(&json!({
+            "email": TEST_AUTH_EMAIL,
+            "password": TEST_AUTH_PASSWORD,
+        }))
+        .send()
+        .await
+        .unwrap();
+    let second_login_status = second_login.status();
+    let second_login_body = second_login.text().await.unwrap();
+    assert!(
+        second_login_status.is_success(),
+        "second login failed with {second_login_status}: {second_login_body}"
+    );
 
     let profile = assert_success_json(
         client
@@ -615,6 +661,21 @@ async fn updates_profile_and_changes_password() {
         .status();
     assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT);
 
+    let current_session_status = client
+        .get(format!("{root_url}/api/auth/profile"))
+        .send()
+        .await
+        .unwrap()
+        .status();
+    let other_session_status = second_client
+        .get(format!("{root_url}/api/auth/profile"))
+        .send()
+        .await
+        .unwrap()
+        .status();
+    assert_eq!(current_session_status, reqwest::StatusCode::OK);
+    assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED);
+
     let relogin_client = reqwest::Client::builder()
         .cookie_store(true)
         .build()
@@ -855,7 +916,10 @@ async fn generates_request_id_for_test_run_invocations() {
         .unwrap()
         .to_owned();
 
-    assert!(!request_id.is_empty());
+    assert_eq!(
+        uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
     response.error_for_status().unwrap();
 
     let logs = client

apps/admin-api/tests/integration/community_access_usage/history_loss.rs

@@ -0,0 +1,141 @@
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+
+use axum::{Json, Router, extract::State, routing::post};
+use tokio::{net::TcpListener, sync::Notify};
+
+use super::*;
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn preserves_external_success_when_invocation_history_is_lost() {
+    let registry = test_registry().await;
+    let registry_for_failure = registry.clone();
+    let storage_root = test_storage_root("observability_history_loss");
+    let upstream = spawn_blocking_upstream_server().await;
+    let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
+    let client = authorized_client(&base_url).await;
+
+    let created = client
+        .post(format!("{base_url}/operations"))
+        .json(&test_operation_payload(
+            &upstream.base_url,
+            "crm_history_loss",
+        ))
+        .send()
+        .await
+        .unwrap()
+        .json::<Value>()
+        .await
+        .unwrap();
+    let operation_id = created["operation_id"].as_str().unwrap().to_owned();
+    let request_client = client.clone();
+    let request_url = format!("{base_url}/operations/{operation_id}/test-runs");
+    let before = crank_observability::operational_incident_total(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+
+    let request = tokio::spawn(async move {
+        request_client
+            .post(request_url)
+            .header("x-request-id", "req_dc08_admin")
+            .json(&json!({
+                "version": 1,
+                "input": { "email": "dc08-canary-secret@example.com" }
+            }))
+            .send()
+            .await
+            .unwrap()
+    });
+
+    upstream.started.notified().await;
+    registry_for_failure
+        .delete_operation(
+            &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
+            &OperationId::new(operation_id.clone()),
+        )
+        .await
+        .unwrap();
+    upstream.release.notify_one();
+
+    let response = request.await.unwrap();
+    assert_eq!(response.status(), reqwest::StatusCode::OK);
+    assert_eq!(
+        response.headers()["x-request-id"].to_str().unwrap(),
+        "req_dc08_admin"
+    );
+    let body = response.json::<Value>().await.unwrap();
+    assert_eq!(body["ok"], true);
+    assert_eq!(body["response_preview"]["id"], "lead_123");
+    assert_eq!(upstream.calls.load(Ordering::SeqCst), 1);
+    assert!(
+        crank_observability::operational_incident_total(
+            crank_observability::OperationalIncident::InvocationHistoryLost
+        ) > before
+    );
+
+    let logs = client
+        .get(format!("{base_url}/logs?period=7d"))
+        .send()
+        .await
+        .unwrap()
+        .json::<Value>()
+        .await
+        .unwrap();
+    assert!(logs["items"].as_array().unwrap().is_empty());
+}
+
+struct BlockingUpstream {
+    base_url: String,
+    started: Arc<Notify>,
+    release: Arc<Notify>,
+    calls: Arc<AtomicUsize>,
+}
+
+#[derive(Clone)]
+struct BlockingUpstreamState {
+    started: Arc<Notify>,
+    release: Arc<Notify>,
+    calls: Arc<AtomicUsize>,
+}
+
+async fn spawn_blocking_upstream_server() -> BlockingUpstream {
+    let state = BlockingUpstreamState {
+        started: Arc::new(Notify::new()),
+        release: Arc::new(Notify::new()),
+        calls: Arc::new(AtomicUsize::new(0)),
+    };
+    let app = Router::new()
+        .route("/crm/leads", post(blocking_create_lead))
+        .with_state(state.clone());
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    BlockingUpstream {
+        base_url: format!("http://{address}"),
+        started: state.started,
+        release: state.release,
+        calls: state.calls,
+    }
+}
+
+async fn blocking_create_lead(
+    State(state): State<BlockingUpstreamState>,
+    Json(payload): Json<Value>,
+) -> Json<Value> {
+    state.calls.fetch_add(1, Ordering::SeqCst);
+    state.started.notify_one();
+    state.release.notified().await;
+
+    Json(json!({
+        "id": "lead_123",
+        "status": "created",
+        "email": payload["email"]
+    }))
+}

apps/admin-api/tests/integration/openapi_import.rs

@@ -1,5 +1,6 @@
 use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload};
 use crank_core::WorkspaceId;
+use crank_registry::ImportJobStatus;
 use serial_test::serial;
 
 use super::common::{
@@ -42,7 +43,7 @@ paths:
 async fn previews_openapi_and_creates_draft_operations() {
     let registry = test_registry().await;
     let service = test_service(
-        registry,
+        registry.clone(),
         test_storage_root("openapi_import"),
         test_auth_settings(),
         test_secret_crypto(),
@@ -64,6 +65,12 @@ async fn previews_openapi_and_creates_draft_operations() {
         preview.preview.groups[0].operations[0].suggested_name,
         "latest_rates"
     );
+    let preview_job = registry
+        .get_import_job(&workspace_id, &preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(preview_job.status, ImportJobStatus::Pending);
 
     let created = service
         .create_openapi_import(
@@ -112,10 +119,19 @@ async fn previews_openapi_and_creates_draft_operations() {
             .any(|finding| finding.code == "openapi_import.weak_tool_description")
     );
 
+    let skip_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
     let skipped = service
         .create_openapi_import(
             &workspace_id,
-            &preview.job_id.as_str().into(),
+            &skip_preview.job_id.as_str().into(),
             OpenApiImportCreatePayload {
                 selected_operation_keys: vec!["GET /v2/latest".to_owned()],
                 server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -130,10 +146,19 @@ async fn previews_openapi_and_creates_draft_operations() {
     assert_eq!(skipped.skipped[0].name, "latest_rates");
     assert_eq!(skipped.findings[0].code, "operation_name_conflict");
 
+    let rename_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
     let renamed = service
         .create_openapi_import(
             &workspace_id,
-            &preview.job_id.as_str().into(),
+            &rename_preview.job_id.as_str().into(),
             OpenApiImportCreatePayload {
                 selected_operation_keys: vec!["GET /v2/latest".to_owned()],
                 server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -147,3 +172,63 @@ async fn previews_openapi_and_creates_draft_operations() {
     assert_eq!(renamed.created[0].name, "latest_rates_2");
     assert_eq!(renamed.findings[0].code, "operation_name_renamed");
 }
+
+#[tokio::test]
+#[serial]
+async fn concurrent_openapi_import_replays_the_same_atomic_result() {
+    let registry = test_registry().await;
+    let service = test_service(
+        registry,
+        test_storage_root("openapi_import_replay"),
+        test_auth_settings(),
+        test_secret_crypto(),
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    let job_id = preview.job_id.as_str().into();
+    let payload = OpenApiImportCreatePayload {
+        selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+        server_url: Some("https://api.frankfurter.dev".to_owned()),
+        conflict_mode: "rename".to_owned(),
+    };
+
+    let (first, second) = tokio::join!(
+        service.create_openapi_import(&workspace_id, &job_id, payload.clone()),
+        service.create_openapi_import(&workspace_id, &job_id, payload),
+    );
+    let first = first.unwrap();
+    let second = second.unwrap();
+
+    assert_eq!(first.created.len(), 1);
+    assert_eq!(second.created.len(), 1);
+    assert_eq!(
+        first.created[0].operation_id,
+        second.created[0].operation_id
+    );
+    assert_eq!(first.created[0].name, second.created[0].name);
+    assert_eq!(
+        service.list_operations(&workspace_id).await.unwrap().len(),
+        1
+    );
+
+    let conflicting_replay = service
+        .create_openapi_import(
+            &workspace_id,
+            &job_id,
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+                server_url: Some("https://api.frankfurter.dev".to_owned()),
+                conflict_mode: "skip".to_owned(),
+            },
+        )
+        .await;
+    assert!(conflicting_replay.is_err());
+}

apps/admin-api/tests/integration/request_context.rs

@@ -0,0 +1,211 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
+use axum::{
+    Router,
+    body::Body,
+    http::{HeaderMap, Request, StatusCode},
+    routing::get,
+};
+use crank_observability::{
+    ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
+};
+use opentelemetry::{global, trace::TracerProvider as _};
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
+use tower::ServiceExt;
+use tracing::instrument::WithSubscriber;
+use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
+use uuid::Version;
+
+#[tokio::test(flavor = "current_thread")]
+async fn logs_request_completion_and_rejects_untrusted_values() {
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let app = probe_app();
+
+    let response = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri("/probe")
+                .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch.clone())
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(
+        response.headers()[REQUEST_ID_HEADER.as_str()]
+            .to_str()
+            .unwrap(),
+        "req_admin_trace_123"
+    );
+    let event: serde_json::Value = writer
+        .output()
+        .lines()
+        .map(|line| serde_json::from_str(line).unwrap())
+        .find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
+        .unwrap();
+    assert_eq!(event["request_id"], "req_admin_trace_123");
+    assert_eq!(event["fields"]["status"], 200);
+
+    let invalid_response = app
+        .oneshot(
+            Request::builder()
+                .uri("/probe")
+                .header(REQUEST_ID_HEADER.as_str(), "bad,value")
+                .header("traceparent", "canary-invalid-traceparent")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch)
+        .await
+        .unwrap();
+    let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
+        .to_str()
+        .unwrap();
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
+    assert!(!writer.output().contains("canary-invalid-traceparent"));
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn covers_valid_invalid_and_absent_traceparent() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder().build();
+    let tracer = provider.tracer("admin-request-context-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let app = trace_probe_app();
+
+    let valid = observed_trace_id(
+        app.clone()
+            .oneshot(
+                Request::builder()
+                    .uri("/trace")
+                    .header(
+                        "traceparent",
+                        "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
+                    )
+                    .header(REQUEST_ID_HEADER.as_str(), "request-id-is-separate")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .with_subscriber(dispatch.clone())
+            .await
+            .unwrap(),
+    );
+    let invalid = observed_trace_id(
+        app.clone()
+            .oneshot(
+                Request::builder()
+                    .uri("/trace")
+                    .header("traceparent", "canary-invalid-traceparent")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .with_subscriber(dispatch.clone())
+            .await
+            .unwrap(),
+    );
+    let absent = observed_trace_id(
+        app.oneshot(
+            Request::builder()
+                .uri("/trace")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch)
+        .await
+        .unwrap(),
+    );
+
+    assert_eq!(valid, "0af7651916cd43dd8448eb211c80319c");
+    assert_ne!(invalid, valid);
+    assert_ne!(absent, valid);
+    assert_ne!(invalid, absent);
+    provider.shutdown().unwrap();
+}
+
+fn probe_app() -> Router {
+    Router::new()
+        .route("/probe", get(|| async { "ok" }))
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
+fn trace_probe_app() -> Router {
+    Router::new()
+        .route("/trace", get(observed_traceparent))
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
+async fn observed_traceparent() -> HeaderMap {
+    let mut trace_headers = HeaderMap::new();
+    inject_current_trace_context(&mut trace_headers);
+    let mut response_headers = HeaderMap::new();
+    if let Some(traceparent) = trace_headers.remove("traceparent") {
+        response_headers.insert("x-observed-traceparent", traceparent);
+    }
+    response_headers
+}
+
+fn observed_trace_id(response: axum::response::Response) -> String {
+    let traceparent = response.headers()["x-observed-traceparent"]
+        .to_str()
+        .unwrap();
+    traceparent[3..35].to_owned()
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

apps/admin-api/tests/request_context.rs

@@ -0,0 +1,2 @@
+#[path = "integration/request_context.rs"]
+mod request_context;