Files
crank/apps/admin-api/tests/resource_metrics_http/support.rs
T
Hermes DevOps 75fe5a123c [skip ci] wip: preserve Hermes crank development checkpoint
Owner-authorized recovery of accumulated source changes.
Includes accepted work and in-progress changes; NOT a release or acceptance.
Parent baseline 1b6db09bd1. Runtime data, build caches and local credentials excluded.
2026-10-25 10:00:22 +00:00

480 lines
17 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Общая фикстура и оракулы таргета `resource_metrics_http`: изолированный
//! дренирующий рекордер, дельта-снимки и точные проверки delta/HTTP-хелперы.
use std::sync::OnceLock;
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
use crank_metrics::DURATION_BUCKETS_SECONDS;
use crank_registry::{CreateWorkspaceRequest, PostgresRegistry};
use metrics::{SharedString, Unit};
use metrics_util::CompositeKey;
use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
use reqwest::StatusCode;
use reqwest::multipart::{Form, Part};
use serde_json::{Value, json};
use time::OffsetDateTime;
use crate::common::{
TestServer, authorized_client, build_test_app, spawn_admin_api, test_registry,
test_storage_root,
};
// ---------------------------------------------------------------------------
// Точные Resource-шаблоны (canonical admin-форма) и метрики.
// ---------------------------------------------------------------------------
pub const RESOURCE_COLLECTION: &str = "/api/admin/workspaces/{workspace_id}/resources";
pub const RESOURCE_ITEM: &str = "/api/admin/workspaces/{workspace_id}/resources/{resource_id}";
pub const RESOURCE_PREVIEW: &str =
"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/preview";
pub const RESOURCE_CONTENT: &str =
"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/content";
pub const UNMATCHED: &str = "unmatched";
pub const HTTP_REQUESTS: &str = "crank_http_requests_total";
pub const HTTP_DURATION: &str = "crank_http_request_duration_seconds";
pub const LOGIN_ROUTE: &str = "/api/auth/login";
pub const OPERATIONS_ROUTE: &str = "/api/admin/workspaces/{workspace_id}/operations";
pub const FOREIGN_WORKSPACE_ID: &str = "ws_story36_metrics_foreign";
pub const CONTENT_TOO_LARGE: usize = 256 * 1024 + 1;
pub type Samples = Vec<(CompositeKey, Option<Unit>, Option<SharedString>, DebugValue)>;
/// Единственный рекордер процесса, установленный при первом обращении.
pub fn snapshotter() -> &'static Snapshotter {
static SNAPSHOTTER: OnceLock<Snapshotter> = OnceLock::new();
SNAPSHOTTER.get_or_init(|| {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
recorder
.install()
.expect("isolated Resource metrics recorder (single install per process)");
snapshotter
})
}
/// Дренирующий дельта-снимок: возвращает накопленное и обнуляет его.
pub fn drain() -> Samples {
snapshotter().snapshot().into_vec()
}
/// Окно измерения запроса: `let _ = drain();` перед запросом, `drain_delta()` после.
pub fn drain_delta() -> Delta {
Delta::from_samples(drain())
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CounterSeries {
pub route: String,
pub method: String,
pub status_class: String,
pub count: u64,
}
#[derive(Clone, Debug, PartialEq)]
pub struct DurationSeries {
pub route: String,
pub method: String,
pub samples: Vec<f64>,
}
#[derive(Clone, Debug, Default)]
pub struct Delta {
pub counters: Vec<CounterSeries>,
pub durations: Vec<DurationSeries>,
}
impl Delta {
pub fn from_samples(samples: Samples) -> Self {
let mut counters = Vec::new();
let mut durations = Vec::new();
for (key, _, _, value) in samples {
let label = |name: &str| {
key.key()
.labels()
.find(|label| label.key() == name)
.map(|label| label.value().to_owned())
.unwrap_or_default()
};
match (key.key().name(), value) {
(HTTP_REQUESTS, DebugValue::Counter(count)) => counters.push(CounterSeries {
route: label("route"),
method: label("method"),
status_class: label("status_class"),
count,
}),
(HTTP_DURATION, DebugValue::Histogram(values)) => durations.push(DurationSeries {
route: label("route"),
method: label("method"),
samples: values.iter().map(|value| value.0).collect(),
}),
_ => {}
}
}
counters.sort_by(|left, right| {
(&left.route, &left.method, &left.status_class).cmp(&(
&right.route,
&right.method,
&right.status_class,
))
});
Self {
counters,
durations,
}
}
pub fn counter(&self, route: &str, method: &str, status_class: &str) -> u64 {
self.counters
.iter()
.find(|series| {
series.route == route
&& series.method == method
&& series.status_class == status_class
})
.map(|series| series.count)
.unwrap_or(0)
}
pub fn nonzero_counters(&self) -> Vec<CounterSeries> {
self.counters
.iter()
.filter(|series| series.count > 0)
.cloned()
.collect()
}
pub fn nonzero_durations(&self) -> Vec<DurationSeries> {
self.durations
.iter()
.filter(|series| !series.samples.is_empty())
.cloned()
.collect()
}
pub fn durations_for(&self, route: &str, method: &str) -> Vec<f64> {
self.durations
.iter()
.filter(|series| series.route == route && series.method == method)
.flat_map(|series| series.samples.iter().copied())
.collect()
}
pub fn counter_routes(&self, method: &str, status_class: &str) -> String {
let routes = self
.counters
.iter()
.filter(|series| series.method == method && series.status_class == status_class)
.map(|series| format!("{}={}", series.route, series.count))
.collect::<Vec<_>>();
format!("{routes:?}")
}
}
/// Измерение TD-H7c: фиксируется как evidence, но не является отдельным GREEN.
pub fn record_measurement(name: &str, value: &str) {
println!("TD-H7c MEASURE {name}={value}");
if let Ok(path) = std::env::var("CRANK_STORY36_MEASUREMENTS") {
use std::io::Write as _;
if let Ok(mut file) = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
{
let _ = writeln!(file, "{name}={value}");
}
}
}
/// Non-panicking precise oracle of one routed request: exactly one counter series
/// and exactly one duration series on the **exact** template, with the expected
/// delta and a declared bucket `le`.
pub fn check_exact_request_delta(
delta: &Delta,
route: &str,
method: &str,
status_class: &str,
expected_count: u64,
context: &str,
) -> Result<(), String> {
let expected = CounterSeries {
route: route.to_owned(),
method: method.to_owned(),
status_class: status_class.to_owned(),
count: expected_count,
};
let observed = delta.nonzero_counters();
if observed != vec![expected.clone()] {
return Err(format!(
"{context}: ожидалась ровно одна counter series {expected:?}; observed {observed:#?}"
));
}
let durations = delta.nonzero_durations();
if durations.len() != 1 {
return Err(format!(
"{context}: ожидалась ровно одна duration series; observed {durations:#?}"
));
}
if durations[0].route != route || durations[0].method != method {
return Err(format!(
"{context}: duration обязана быть на точном route={route} method={method}; observed {durations:#?}"
));
}
if durations[0].samples.len() != usize::try_from(expected_count).unwrap_or(usize::MAX) {
return Err(format!(
"{context}: duration обязан получить ровно {expected_count} наблюдений; observed {:?}",
durations[0].samples
));
}
// Каждое наблюдение обязано попасть в объявленный bucket с известным `le`
// (а не «на глаз» в +Inf): леджер считается по frozen DURATION_BUCKETS_SECONDS.
for sample in &durations[0].samples {
if !sample.is_finite() || *sample < 0.0 {
return Err(format!(
"{context}: duration обязана быть конечной и неотрицательной, observed {sample}"
));
}
let declared = DURATION_BUCKETS_SECONDS.iter().any(|bound| sample <= bound);
if !declared {
return Err(format!(
"{context}: duration {sample}s вышла за объявленные buckets {DURATION_BUCKETS_SECONDS:?}"
));
}
}
Ok(())
}
#[track_caller]
pub fn assert_exact_request_delta(
delta: &Delta,
route: &str,
method: &str,
status_class: &str,
expected_count: u64,
context: &str,
) {
if let Err(message) =
check_exact_request_delta(delta, route, method, status_class, expected_count, context)
{
panic!("{message}");
}
}
/// Non-panicking precise oracle of a router-generated refusal: the method keeps its
/// own closed-enum value, the observed status class is a client refusal and the
/// counter/duration delta is exactly one. Returns the observed `route` label.
pub fn check_method_delta(
delta: &Delta,
method: &str,
status_class: &str,
context: &str,
) -> Result<String, String> {
let observed = delta.nonzero_counters();
if observed.len() != 1 {
return Err(format!(
"{context}: ожидалась ровно одна counter series; observed {observed:#?}"
));
}
if observed[0].method != method {
return Err(format!(
"{context}: method обязан остаться `{method}`; observed {:#?}",
observed[0]
));
}
if observed[0].status_class != status_class {
return Err(format!(
"{context}: status_class обязан быть `{status_class}`; observed {:#?}",
observed[0]
));
}
if observed[0].count != 1 {
return Err(format!(
"{context}: counter delta обязан быть 1; observed {:#?}",
observed[0]
));
}
let durations = delta.nonzero_durations();
if durations.len() != 1 || durations[0].method != method || durations[0].samples.len() != 1 {
return Err(format!(
"{context}: ожидалась ровно одна duration series с delta 1 для method `{method}`; observed {durations:#?}"
));
}
Ok(observed[0].route.clone())
}
/// Canary: raw concrete id / uri / content / query не становится значением label.
pub fn check_no_raw_tokens(delta: &Delta, tokens: &[&str], context: &str) -> Result<(), String> {
for series in &delta.counters {
for value in [&series.route, &series.method, &series.status_class] {
for token in tokens {
if value.contains(token) {
return Err(format!(
"{context}: label не может нести raw значение `{token}`: {value}"
));
}
}
}
}
for series in &delta.durations {
for value in [&series.route, &series.method] {
for token in tokens {
if value.contains(token) {
return Err(format!(
"{context}: label не может нести raw значение `{token}`: {value}"
));
}
}
}
}
Ok(())
}
#[track_caller]
pub fn assert_no_raw_tokens(delta: &Delta, tokens: &[&str], context: &str) {
if let Err(message) = check_no_raw_tokens(delta, tokens, context) {
panic!("{message}");
}
}
/// Класс статуса в терминах закрытого домена метрики.
pub fn status_class(status: StatusCode) -> String {
crank_metrics::HttpStatusClass::from_status(status.as_u16())
.as_str()
.to_owned()
}
// ---------------------------------------------------------------------------
// Фикстура
// ---------------------------------------------------------------------------
pub fn prime_metrics_fixture() {
// Рекордер и service binding обязаны существовать ДО первого запроса
// (включая `login` внутри `authorized_client`): `DebuggingRecorder::snapshot()`
// — дренирующий дельта-снимок, поэтому ленивая установка рекордера после
// login теряла login-наблюдение (TL finding F1).
let _ = snapshotter();
crank_metrics::bind_metric_service(crank_metrics::MetricService::AdminApi)
.expect("admin-api metrics binding for the isolated Resource fixture");
}
pub async fn spawn_stack(storage_name: &str) -> (TestServer, reqwest::Client) {
prime_metrics_fixture();
let registry = test_registry().await;
let app = build_test_app(registry, test_storage_root(storage_name));
let server = spawn_admin_api(app).await;
let client = authorized_client(&server).await;
(server, client)
}
pub fn api_root(server: &str) -> String {
server
.split("/api/admin/workspaces/")
.next()
.expect("workspace base url must contain the admin workspace prefix")
.to_owned()
}
pub fn draft_metadata(uri: &str, name: &str) -> Value {
json!({
"uri": uri,
"name": name,
"description": "",
"kind": "text",
"mime_type": "text/plain",
"sensitivity": "internal",
})
}
pub fn draft_form(metadata: &Value, content: &[u8]) -> Form {
Form::new()
.part(
"metadata",
Part::bytes(serde_json::to_vec(metadata).expect("metadata must serialize"))
.file_name("metadata.json".to_owned())
.mime_str("application/json")
.expect("json part mime"),
)
.part(
"content",
Part::bytes(content.to_vec())
.file_name("draft.txt".to_owned())
.mime_str("text/plain")
.expect("content part mime"),
)
}
pub async fn create_draft(
client: &reqwest::Client,
server: &TestServer,
uri: &str,
name: &str,
content: &[u8],
) -> String {
let response = client
.post(format!("{server}/resources"))
.multipart(draft_form(&draft_metadata(uri, name), content))
.send()
.await
.expect("create request must complete");
let status = response.status();
let body = response.text().await.unwrap_or_default();
assert_eq!(
status,
StatusCode::CREATED,
"POST /resources must create a Draft; observed {status} body={body}"
);
let card: Value = serde_json::from_str(&body).expect("card must be json");
card["resource"]["resource_id"]
.as_str()
.expect("201 response must carry the created Resource identifier")
.to_owned()
}
pub fn updated_at(card: &Value) -> String {
for pointer in ["/resource/updated_at", "/version/updated_at", "/updated_at"] {
if let Some(value) = card.pointer(pointer).and_then(Value::as_str) {
return value.to_owned();
}
}
panic!("Draft card must expose the precondition timestamp `updated_at`; body={card}");
}
pub async fn get_card(client: &reqwest::Client, server: &TestServer, resource_id: &str) -> Value {
let response = client
.get(format!("{server}/resources/{resource_id}"))
.send()
.await
.expect("card request must complete");
assert_eq!(response.status(), StatusCode::OK);
serde_json::from_str(&response.text().await.expect("card body")).expect("card json")
}
pub fn counter_series(route: &str, method: &str, status_class: &str, count: u64) -> CounterSeries {
CounterSeries {
route: route.to_owned(),
method: method.to_owned(),
status_class: status_class.to_owned(),
count,
}
}
/// Synthetic workspace без membership вызывающего аккаунта (негатив F36).
pub async fn create_foreign_workspace(registry: &PostgresRegistry) {
let now = OffsetDateTime::now_utc();
registry
.create_workspace(CreateWorkspaceRequest {
workspace: &Workspace {
id: WorkspaceId::new(FOREIGN_WORKSPACE_ID),
slug: "story36-metrics-foreign".to_owned(),
display_name: "Story 3.6 Metrics Foreign".to_owned(),
status: WorkspaceStatus::Active,
settings: json!({}),
created_at: now,
updated_at: now,
},
})
.await
.expect("foreign workspace must be created");
}