Owner-authorized recovery of accumulated source changes.
Includes accepted work and in-progress changes; NOT a release or acceptance.
Parent baseline 1b6db09bd1. Runtime data, build caches and local credentials excluded.
480 lines
17 KiB
Rust
480 lines
17 KiB
Rust
//! Общая фикстура и оракулы таргета `resource_metrics_http`: изолированный
|
||
//! дренирующий рекордер, дельта-снимки и точные проверки delta/HTTP-хелперы.
|
||
|
||
use std::sync::OnceLock;
|
||
|
||
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
|
||
use crank_metrics::DURATION_BUCKETS_SECONDS;
|
||
use crank_registry::{CreateWorkspaceRequest, PostgresRegistry};
|
||
use metrics::{SharedString, Unit};
|
||
use metrics_util::CompositeKey;
|
||
use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
|
||
use reqwest::StatusCode;
|
||
use reqwest::multipart::{Form, Part};
|
||
use serde_json::{Value, json};
|
||
use time::OffsetDateTime;
|
||
|
||
use crate::common::{
|
||
TestServer, authorized_client, build_test_app, spawn_admin_api, test_registry,
|
||
test_storage_root,
|
||
};
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Точные Resource-шаблоны (canonical admin-форма) и метрики.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
pub const RESOURCE_COLLECTION: &str = "/api/admin/workspaces/{workspace_id}/resources";
|
||
pub const RESOURCE_ITEM: &str = "/api/admin/workspaces/{workspace_id}/resources/{resource_id}";
|
||
pub const RESOURCE_PREVIEW: &str =
|
||
"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/preview";
|
||
pub const RESOURCE_CONTENT: &str =
|
||
"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/content";
|
||
pub const UNMATCHED: &str = "unmatched";
|
||
pub const HTTP_REQUESTS: &str = "crank_http_requests_total";
|
||
pub const HTTP_DURATION: &str = "crank_http_request_duration_seconds";
|
||
pub const LOGIN_ROUTE: &str = "/api/auth/login";
|
||
pub const OPERATIONS_ROUTE: &str = "/api/admin/workspaces/{workspace_id}/operations";
|
||
pub const FOREIGN_WORKSPACE_ID: &str = "ws_story36_metrics_foreign";
|
||
pub const CONTENT_TOO_LARGE: usize = 256 * 1024 + 1;
|
||
|
||
pub type Samples = Vec<(CompositeKey, Option<Unit>, Option<SharedString>, DebugValue)>;
|
||
|
||
/// Единственный рекордер процесса, установленный при первом обращении.
|
||
pub fn snapshotter() -> &'static Snapshotter {
|
||
static SNAPSHOTTER: OnceLock<Snapshotter> = OnceLock::new();
|
||
SNAPSHOTTER.get_or_init(|| {
|
||
let recorder = DebuggingRecorder::new();
|
||
let snapshotter = recorder.snapshotter();
|
||
recorder
|
||
.install()
|
||
.expect("isolated Resource metrics recorder (single install per process)");
|
||
snapshotter
|
||
})
|
||
}
|
||
|
||
/// Дренирующий дельта-снимок: возвращает накопленное и обнуляет его.
|
||
pub fn drain() -> Samples {
|
||
snapshotter().snapshot().into_vec()
|
||
}
|
||
|
||
/// Окно измерения запроса: `let _ = drain();` перед запросом, `drain_delta()` после.
|
||
pub fn drain_delta() -> Delta {
|
||
Delta::from_samples(drain())
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||
pub struct CounterSeries {
|
||
pub route: String,
|
||
pub method: String,
|
||
pub status_class: String,
|
||
pub count: u64,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
pub struct DurationSeries {
|
||
pub route: String,
|
||
pub method: String,
|
||
pub samples: Vec<f64>,
|
||
}
|
||
|
||
#[derive(Clone, Debug, Default)]
|
||
pub struct Delta {
|
||
pub counters: Vec<CounterSeries>,
|
||
pub durations: Vec<DurationSeries>,
|
||
}
|
||
|
||
impl Delta {
|
||
pub fn from_samples(samples: Samples) -> Self {
|
||
let mut counters = Vec::new();
|
||
let mut durations = Vec::new();
|
||
for (key, _, _, value) in samples {
|
||
let label = |name: &str| {
|
||
key.key()
|
||
.labels()
|
||
.find(|label| label.key() == name)
|
||
.map(|label| label.value().to_owned())
|
||
.unwrap_or_default()
|
||
};
|
||
match (key.key().name(), value) {
|
||
(HTTP_REQUESTS, DebugValue::Counter(count)) => counters.push(CounterSeries {
|
||
route: label("route"),
|
||
method: label("method"),
|
||
status_class: label("status_class"),
|
||
count,
|
||
}),
|
||
(HTTP_DURATION, DebugValue::Histogram(values)) => durations.push(DurationSeries {
|
||
route: label("route"),
|
||
method: label("method"),
|
||
samples: values.iter().map(|value| value.0).collect(),
|
||
}),
|
||
_ => {}
|
||
}
|
||
}
|
||
counters.sort_by(|left, right| {
|
||
(&left.route, &left.method, &left.status_class).cmp(&(
|
||
&right.route,
|
||
&right.method,
|
||
&right.status_class,
|
||
))
|
||
});
|
||
Self {
|
||
counters,
|
||
durations,
|
||
}
|
||
}
|
||
|
||
pub fn counter(&self, route: &str, method: &str, status_class: &str) -> u64 {
|
||
self.counters
|
||
.iter()
|
||
.find(|series| {
|
||
series.route == route
|
||
&& series.method == method
|
||
&& series.status_class == status_class
|
||
})
|
||
.map(|series| series.count)
|
||
.unwrap_or(0)
|
||
}
|
||
|
||
pub fn nonzero_counters(&self) -> Vec<CounterSeries> {
|
||
self.counters
|
||
.iter()
|
||
.filter(|series| series.count > 0)
|
||
.cloned()
|
||
.collect()
|
||
}
|
||
|
||
pub fn nonzero_durations(&self) -> Vec<DurationSeries> {
|
||
self.durations
|
||
.iter()
|
||
.filter(|series| !series.samples.is_empty())
|
||
.cloned()
|
||
.collect()
|
||
}
|
||
|
||
pub fn durations_for(&self, route: &str, method: &str) -> Vec<f64> {
|
||
self.durations
|
||
.iter()
|
||
.filter(|series| series.route == route && series.method == method)
|
||
.flat_map(|series| series.samples.iter().copied())
|
||
.collect()
|
||
}
|
||
|
||
pub fn counter_routes(&self, method: &str, status_class: &str) -> String {
|
||
let routes = self
|
||
.counters
|
||
.iter()
|
||
.filter(|series| series.method == method && series.status_class == status_class)
|
||
.map(|series| format!("{}={}", series.route, series.count))
|
||
.collect::<Vec<_>>();
|
||
format!("{routes:?}")
|
||
}
|
||
}
|
||
|
||
/// Измерение TD-H7c: фиксируется как evidence, но не является отдельным GREEN.
|
||
pub fn record_measurement(name: &str, value: &str) {
|
||
println!("TD-H7c MEASURE {name}={value}");
|
||
if let Ok(path) = std::env::var("CRANK_STORY36_MEASUREMENTS") {
|
||
use std::io::Write as _;
|
||
if let Ok(mut file) = std::fs::OpenOptions::new()
|
||
.create(true)
|
||
.append(true)
|
||
.open(path)
|
||
{
|
||
let _ = writeln!(file, "{name}={value}");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Non-panicking precise oracle of one routed request: exactly one counter series
|
||
/// and exactly one duration series on the **exact** template, with the expected
|
||
/// delta and a declared bucket `le`.
|
||
pub fn check_exact_request_delta(
|
||
delta: &Delta,
|
||
route: &str,
|
||
method: &str,
|
||
status_class: &str,
|
||
expected_count: u64,
|
||
context: &str,
|
||
) -> Result<(), String> {
|
||
let expected = CounterSeries {
|
||
route: route.to_owned(),
|
||
method: method.to_owned(),
|
||
status_class: status_class.to_owned(),
|
||
count: expected_count,
|
||
};
|
||
let observed = delta.nonzero_counters();
|
||
if observed != vec![expected.clone()] {
|
||
return Err(format!(
|
||
"{context}: ожидалась ровно одна counter series {expected:?}; observed {observed:#?}"
|
||
));
|
||
}
|
||
|
||
let durations = delta.nonzero_durations();
|
||
if durations.len() != 1 {
|
||
return Err(format!(
|
||
"{context}: ожидалась ровно одна duration series; observed {durations:#?}"
|
||
));
|
||
}
|
||
if durations[0].route != route || durations[0].method != method {
|
||
return Err(format!(
|
||
"{context}: duration обязана быть на точном route={route} method={method}; observed {durations:#?}"
|
||
));
|
||
}
|
||
if durations[0].samples.len() != usize::try_from(expected_count).unwrap_or(usize::MAX) {
|
||
return Err(format!(
|
||
"{context}: duration обязан получить ровно {expected_count} наблюдений; observed {:?}",
|
||
durations[0].samples
|
||
));
|
||
}
|
||
|
||
// Каждое наблюдение обязано попасть в объявленный bucket с известным `le`
|
||
// (а не «на глаз» в +Inf): леджер считается по frozen DURATION_BUCKETS_SECONDS.
|
||
for sample in &durations[0].samples {
|
||
if !sample.is_finite() || *sample < 0.0 {
|
||
return Err(format!(
|
||
"{context}: duration обязана быть конечной и неотрицательной, observed {sample}"
|
||
));
|
||
}
|
||
let declared = DURATION_BUCKETS_SECONDS.iter().any(|bound| sample <= bound);
|
||
if !declared {
|
||
return Err(format!(
|
||
"{context}: duration {sample}s вышла за объявленные buckets {DURATION_BUCKETS_SECONDS:?}"
|
||
));
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[track_caller]
|
||
pub fn assert_exact_request_delta(
|
||
delta: &Delta,
|
||
route: &str,
|
||
method: &str,
|
||
status_class: &str,
|
||
expected_count: u64,
|
||
context: &str,
|
||
) {
|
||
if let Err(message) =
|
||
check_exact_request_delta(delta, route, method, status_class, expected_count, context)
|
||
{
|
||
panic!("{message}");
|
||
}
|
||
}
|
||
|
||
/// Non-panicking precise oracle of a router-generated refusal: the method keeps its
|
||
/// own closed-enum value, the observed status class is a client refusal and the
|
||
/// counter/duration delta is exactly one. Returns the observed `route` label.
|
||
pub fn check_method_delta(
|
||
delta: &Delta,
|
||
method: &str,
|
||
status_class: &str,
|
||
context: &str,
|
||
) -> Result<String, String> {
|
||
let observed = delta.nonzero_counters();
|
||
if observed.len() != 1 {
|
||
return Err(format!(
|
||
"{context}: ожидалась ровно одна counter series; observed {observed:#?}"
|
||
));
|
||
}
|
||
if observed[0].method != method {
|
||
return Err(format!(
|
||
"{context}: method обязан остаться `{method}`; observed {:#?}",
|
||
observed[0]
|
||
));
|
||
}
|
||
if observed[0].status_class != status_class {
|
||
return Err(format!(
|
||
"{context}: status_class обязан быть `{status_class}`; observed {:#?}",
|
||
observed[0]
|
||
));
|
||
}
|
||
if observed[0].count != 1 {
|
||
return Err(format!(
|
||
"{context}: counter delta обязан быть 1; observed {:#?}",
|
||
observed[0]
|
||
));
|
||
}
|
||
let durations = delta.nonzero_durations();
|
||
if durations.len() != 1 || durations[0].method != method || durations[0].samples.len() != 1 {
|
||
return Err(format!(
|
||
"{context}: ожидалась ровно одна duration series с delta 1 для method `{method}`; observed {durations:#?}"
|
||
));
|
||
}
|
||
Ok(observed[0].route.clone())
|
||
}
|
||
|
||
/// Canary: raw concrete id / uri / content / query не становится значением label.
|
||
pub fn check_no_raw_tokens(delta: &Delta, tokens: &[&str], context: &str) -> Result<(), String> {
|
||
for series in &delta.counters {
|
||
for value in [&series.route, &series.method, &series.status_class] {
|
||
for token in tokens {
|
||
if value.contains(token) {
|
||
return Err(format!(
|
||
"{context}: label не может нести raw значение `{token}`: {value}"
|
||
));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for series in &delta.durations {
|
||
for value in [&series.route, &series.method] {
|
||
for token in tokens {
|
||
if value.contains(token) {
|
||
return Err(format!(
|
||
"{context}: label не может нести raw значение `{token}`: {value}"
|
||
));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[track_caller]
|
||
pub fn assert_no_raw_tokens(delta: &Delta, tokens: &[&str], context: &str) {
|
||
if let Err(message) = check_no_raw_tokens(delta, tokens, context) {
|
||
panic!("{message}");
|
||
}
|
||
}
|
||
|
||
/// Класс статуса в терминах закрытого домена метрики.
|
||
pub fn status_class(status: StatusCode) -> String {
|
||
crank_metrics::HttpStatusClass::from_status(status.as_u16())
|
||
.as_str()
|
||
.to_owned()
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Фикстура
|
||
// ---------------------------------------------------------------------------
|
||
|
||
pub fn prime_metrics_fixture() {
|
||
// Рекордер и service binding обязаны существовать ДО первого запроса
|
||
// (включая `login` внутри `authorized_client`): `DebuggingRecorder::snapshot()`
|
||
// — дренирующий дельта-снимок, поэтому ленивая установка рекордера после
|
||
// login теряла login-наблюдение (TL finding F1).
|
||
let _ = snapshotter();
|
||
crank_metrics::bind_metric_service(crank_metrics::MetricService::AdminApi)
|
||
.expect("admin-api metrics binding for the isolated Resource fixture");
|
||
}
|
||
|
||
pub async fn spawn_stack(storage_name: &str) -> (TestServer, reqwest::Client) {
|
||
prime_metrics_fixture();
|
||
let registry = test_registry().await;
|
||
let app = build_test_app(registry, test_storage_root(storage_name));
|
||
let server = spawn_admin_api(app).await;
|
||
let client = authorized_client(&server).await;
|
||
(server, client)
|
||
}
|
||
|
||
pub fn api_root(server: &str) -> String {
|
||
server
|
||
.split("/api/admin/workspaces/")
|
||
.next()
|
||
.expect("workspace base url must contain the admin workspace prefix")
|
||
.to_owned()
|
||
}
|
||
|
||
pub fn draft_metadata(uri: &str, name: &str) -> Value {
|
||
json!({
|
||
"uri": uri,
|
||
"name": name,
|
||
"description": "",
|
||
"kind": "text",
|
||
"mime_type": "text/plain",
|
||
"sensitivity": "internal",
|
||
})
|
||
}
|
||
|
||
pub fn draft_form(metadata: &Value, content: &[u8]) -> Form {
|
||
Form::new()
|
||
.part(
|
||
"metadata",
|
||
Part::bytes(serde_json::to_vec(metadata).expect("metadata must serialize"))
|
||
.file_name("metadata.json".to_owned())
|
||
.mime_str("application/json")
|
||
.expect("json part mime"),
|
||
)
|
||
.part(
|
||
"content",
|
||
Part::bytes(content.to_vec())
|
||
.file_name("draft.txt".to_owned())
|
||
.mime_str("text/plain")
|
||
.expect("content part mime"),
|
||
)
|
||
}
|
||
|
||
pub async fn create_draft(
|
||
client: &reqwest::Client,
|
||
server: &TestServer,
|
||
uri: &str,
|
||
name: &str,
|
||
content: &[u8],
|
||
) -> String {
|
||
let response = client
|
||
.post(format!("{server}/resources"))
|
||
.multipart(draft_form(&draft_metadata(uri, name), content))
|
||
.send()
|
||
.await
|
||
.expect("create request must complete");
|
||
let status = response.status();
|
||
let body = response.text().await.unwrap_or_default();
|
||
assert_eq!(
|
||
status,
|
||
StatusCode::CREATED,
|
||
"POST /resources must create a Draft; observed {status} body={body}"
|
||
);
|
||
let card: Value = serde_json::from_str(&body).expect("card must be json");
|
||
card["resource"]["resource_id"]
|
||
.as_str()
|
||
.expect("201 response must carry the created Resource identifier")
|
||
.to_owned()
|
||
}
|
||
|
||
pub fn updated_at(card: &Value) -> String {
|
||
for pointer in ["/resource/updated_at", "/version/updated_at", "/updated_at"] {
|
||
if let Some(value) = card.pointer(pointer).and_then(Value::as_str) {
|
||
return value.to_owned();
|
||
}
|
||
}
|
||
panic!("Draft card must expose the precondition timestamp `updated_at`; body={card}");
|
||
}
|
||
|
||
pub async fn get_card(client: &reqwest::Client, server: &TestServer, resource_id: &str) -> Value {
|
||
let response = client
|
||
.get(format!("{server}/resources/{resource_id}"))
|
||
.send()
|
||
.await
|
||
.expect("card request must complete");
|
||
assert_eq!(response.status(), StatusCode::OK);
|
||
serde_json::from_str(&response.text().await.expect("card body")).expect("card json")
|
||
}
|
||
|
||
pub fn counter_series(route: &str, method: &str, status_class: &str, count: u64) -> CounterSeries {
|
||
CounterSeries {
|
||
route: route.to_owned(),
|
||
method: method.to_owned(),
|
||
status_class: status_class.to_owned(),
|
||
count,
|
||
}
|
||
}
|
||
|
||
/// Synthetic workspace без membership вызывающего аккаунта (негатив F36).
|
||
pub async fn create_foreign_workspace(registry: &PostgresRegistry) {
|
||
let now = OffsetDateTime::now_utc();
|
||
registry
|
||
.create_workspace(CreateWorkspaceRequest {
|
||
workspace: &Workspace {
|
||
id: WorkspaceId::new(FOREIGN_WORKSPACE_ID),
|
||
slug: "story36-metrics-foreign".to_owned(),
|
||
display_name: "Story 3.6 Metrics Foreign".to_owned(),
|
||
status: WorkspaceStatus::Active,
|
||
settings: json!({}),
|
||
created_at: now,
|
||
updated_at: now,
|
||
},
|
||
})
|
||
.await
|
||
.expect("foreign workspace must be created");
|
||
}
|