наблюдаемость: ввести безопасный контракт метрик

2026-07-31 05:04:01 +03:00
parent ec2453c00f
commit 9b1a739e39
50 changed files with 3066 additions and 433 deletions
@@ -0,0 +1,89 @@
+#[path = "integration/common.rs"]
+mod common;
+
+use std::{collections::BTreeSet, time::Duration};
+
+use common::*;
+use metrics_util::debugging::DebuggingRecorder;
+
+#[tokio::test(flavor = "multi_thread")]
+async fn real_admin_routes_and_postgres_sampler_emit_bounded_metrics() {
+    let recorder = DebuggingRecorder::new();
+    let snapshotter = recorder.snapshotter();
+    recorder
+        .install()
+        .expect("isolated integration test recorder");
+
+    let registry = test_registry().await;
+    let sampler = admin_api::pool_metrics::spawn_postgres_pool_metrics(registry.pool().clone());
+    let server = spawn_admin_api(build_test_app(
+        registry,
+        test_storage_root("product_metrics"),
+    ))
+    .await;
+    let client = authorized_client(&server).await;
+    client
+        .get(format!("{server}/operations"))
+        .send()
+        .await
+        .unwrap()
+        .error_for_status()
+        .unwrap();
+
+    let snapshot = wait_for_metrics(&snapshotter).await;
+    assert!(snapshot.iter().any(|(key, _, _, _)| {
+        key.key().name() == "crank_http_requests_total"
+            && has_label(
+                key.key(),
+                "route",
+                "/api/admin/workspaces/{workspace_id}/operations",
+            )
+            && has_label(key.key(), "method", "GET")
+            && has_label(key.key(), "status_class", "2xx")
+    }));
+    let pool_states = snapshot
+        .iter()
+        .filter(|(key, _, _, _)| key.key().name() == "crank_db_pool_connections")
+        .filter_map(|(key, _, _, _)| {
+            key.key()
+                .labels()
+                .find(|label| label.key() == "state")
+                .map(|label| label.value().to_owned())
+        })
+        .collect::<BTreeSet<_>>();
+    assert_eq!(
+        pool_states,
+        BTreeSet::from(["idle".to_owned(), "used".to_owned()])
+    );
+
+    sampler.abort();
+}
+
+async fn wait_for_metrics(
+    snapshotter: &metrics_util::debugging::Snapshotter,
+) -> Vec<(
+    metrics_util::CompositeKey,
+    Option<metrics::Unit>,
+    Option<metrics::SharedString>,
+    metrics_util::debugging::DebugValue,
+)> {
+    for _ in 0..50 {
+        let snapshot = snapshotter.snapshot().into_vec();
+        let has_http = snapshot
+            .iter()
+            .any(|(key, _, _, _)| key.key().name() == "crank_http_requests_total");
+        let has_pool = snapshot
+            .iter()
+            .any(|(key, _, _, _)| key.key().name() == "crank_db_pool_connections");
+        if has_http && has_pool {
+            return snapshot;
+        }
+        tokio::time::sleep(Duration::from_millis(10)).await;
+    }
+    panic!("product metrics did not become visible");
+}
+
+fn has_label(key: &metrics::Key, name: &str, value: &str) -> bool {
+    key.labels()
+        .any(|label| label.key() == name && label.value() == value)
+}