CI / Rust Checks (pull_request) Failing after 13m41s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
Community Release / release (push) Failing after 13m33s
Recovery delivery of the accumulated, TL-accepted Epic 3 work that had never been
committed to Git (stories 3.6-3.17, see the crank board). Story 3.18 is not accepted
yet and is deliberately excluded: its logging surface, its capability registration,
its oracle projections and its tests are not part of this tree.
Accepted scope
- 3.6 typed static Resource Drafts; 3.7 publication and exact binding of static
Resource Versions; 3.8 MCP discovery/read and revocation of static Resources;
- 3.9 Resource import/export in API and UI plus TLS dependency maintenance;
3.10 REST-backed Resource templates; 3.11 shared REST read runtime and the MCP
revision/capability foundation;
- 3.12 parameterised Prompt Drafts with preview; 3.13 Prompt publication and exact
version binding to Agents; 3.14 MCP prompts; 3.15 Prompt import/export;
- 3.16 composite Prompts with published Resources (dependency content scan plus
dynamic read through the accepted 3.11 runtime, admin preview and MCP prompts/get);
- 3.17 bounded MCP completion suggestions from typed published schemas and the
resources/list admission fix for bound template publications.
Accepted evidence being reused (unchanged code paths)
- 3.16 candidate: core 7, admin-api 22, registry 4, mcp-server 4, UI 6; one final
`just verify` run 1651 passed / 0 failed / 2 ignored.
- 3.17 candidate: full run 1672 passed / 0 failed / 2 ignored, QA targeted reruns and
a real MCP Inspector 2.12.0 client smoke.
Delivery engineering in this commit
- CI: one event per change (feature branches are gated by their pull request, push
runs only on main), story events run the changed packages, the full workspace suite
stays the feature/release gate; no cache/sharding without a measured runner check.
- Release: the packaged archives, checksums and SBOM are now uploaded to the Gitea
release entry of the tag instead of dying with the runner workspace.
Not included on purpose: Story 3.18 work, build caches, runtime databases, local
credentials and scratch evidence.
Refs: crank kanban t_e39d7263; baseline 1b6db09bd1.
557 lines
19 KiB
Rust
557 lines
19 KiB
Rust
use std::{
|
||
collections::HashMap,
|
||
net::SocketAddr,
|
||
sync::{
|
||
Arc,
|
||
atomic::{AtomicUsize, Ordering},
|
||
},
|
||
};
|
||
|
||
use axum::{
|
||
Router,
|
||
extract::{Request, State},
|
||
http::{
|
||
HeaderMap, StatusCode,
|
||
header::{self, HeaderValue},
|
||
},
|
||
middleware::{self, Next},
|
||
response::{IntoResponse, Response},
|
||
routing::get,
|
||
};
|
||
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle, PrometheusRecorder};
|
||
use sha2::{Digest, Sha256};
|
||
use subtle::ConstantTimeEq;
|
||
use thiserror::Error;
|
||
use tokio::net::TcpListener;
|
||
|
||
use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity};
|
||
use crank_metrics::{ExemplarObservation, MAX_EXPOSITION_BYTES, MetricService, exemplar_snapshot};
|
||
|
||
const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
|
||
const OPENMETRICS_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
|
||
const EXPOSITION_TOO_LARGE: &str = "metrics exposition exceeds configured bound\n";
|
||
const TOO_MANY_SCRAPES: &str = "metrics scrape concurrency limit exceeded\n";
|
||
const MAX_CONCURRENT_SCRAPES: usize = 2;
|
||
|
||
#[derive(Clone)]
|
||
pub struct MetricsConfig {
|
||
enabled: bool,
|
||
bind_addr: SocketAddr,
|
||
token_digest: Option<[u8; 32]>,
|
||
}
|
||
|
||
impl std::fmt::Debug for MetricsConfig {
|
||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
formatter
|
||
.debug_struct("MetricsConfig")
|
||
.field("enabled", &self.enabled)
|
||
.field("bind_addr", &self.bind_addr)
|
||
.field("authentication_configured", &self.token_digest.is_some())
|
||
.finish()
|
||
}
|
||
}
|
||
|
||
impl MetricsConfig {
|
||
pub fn new(
|
||
enabled: bool,
|
||
bind_addr: SocketAddr,
|
||
bearer_token: Option<String>,
|
||
) -> Result<Self, MetricsConfigError> {
|
||
let token_digest = bearer_token
|
||
.filter(|token| !token.is_empty())
|
||
.map(|token| token_digest(token.as_bytes()));
|
||
|
||
if enabled && !bind_addr.ip().is_loopback() && token_digest.is_none() {
|
||
return Err(MetricsConfigError::MissingTokenForExternalBind);
|
||
}
|
||
|
||
Ok(Self {
|
||
enabled,
|
||
bind_addr,
|
||
token_digest,
|
||
})
|
||
}
|
||
|
||
pub fn enabled(&self) -> bool {
|
||
self.enabled
|
||
}
|
||
|
||
pub fn bind_addr(&self) -> SocketAddr {
|
||
self.bind_addr
|
||
}
|
||
|
||
pub fn requires_authentication(&self) -> bool {
|
||
!self.bind_addr.ip().is_loopback()
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Error)]
|
||
pub enum MetricsConfigError {
|
||
#[error("metrics environment variable is not valid UTF-8: {field}")]
|
||
InvalidEnvironmentEncoding { field: &'static str },
|
||
#[error("metrics bind address is invalid: {field}")]
|
||
InvalidBindAddress { field: &'static str },
|
||
#[error("metrics enabled flag must be one of true, false, 1, 0")]
|
||
InvalidEnabledFlag,
|
||
#[error("external metrics bind requires a bearer token")]
|
||
MissingTokenForExternalBind,
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
struct MetricsState {
|
||
handle: PrometheusHandle,
|
||
token_digest: Option<[u8; 32]>,
|
||
requires_authentication: bool,
|
||
active_scrapes: Arc<AtomicUsize>,
|
||
}
|
||
|
||
pub struct MetricsSurface {
|
||
config: MetricsConfig,
|
||
state: MetricsState,
|
||
_recorder: Option<PrometheusRecorder>,
|
||
}
|
||
|
||
impl MetricsSurface {
|
||
pub(crate) fn new(config: MetricsConfig, handle: PrometheusHandle) -> Self {
|
||
Self {
|
||
state: MetricsState {
|
||
handle,
|
||
token_digest: config.token_digest,
|
||
requires_authentication: config.requires_authentication(),
|
||
active_scrapes: Arc::new(AtomicUsize::new(0)),
|
||
},
|
||
config,
|
||
_recorder: None,
|
||
}
|
||
}
|
||
|
||
pub fn for_test(
|
||
config: MetricsConfig,
|
||
identity: ServiceIdentity,
|
||
) -> Result<Self, MetricsSurfaceError> {
|
||
let recorder = prometheus_builder(&identity)?.build_recorder();
|
||
let handle = recorder.handle();
|
||
let mut surface = Self::new(config, handle);
|
||
surface._recorder = Some(recorder);
|
||
Ok(surface)
|
||
}
|
||
|
||
pub fn router(&self) -> Router {
|
||
Router::new()
|
||
.route("/metrics", get(render_metrics))
|
||
.route("/health", get(metrics_health))
|
||
.route_layer(middleware::from_fn_with_state(
|
||
self.state.clone(),
|
||
authorize_metrics,
|
||
))
|
||
.with_state(self.state.clone())
|
||
}
|
||
|
||
pub async fn bind(self) -> Result<MetricsServer, MetricsServeError> {
|
||
let listener = TcpListener::bind(self.config.bind_addr)
|
||
.await
|
||
.map_err(|_| MetricsServeError::Bind)?;
|
||
Ok(MetricsServer {
|
||
listener,
|
||
router: self.router(),
|
||
})
|
||
}
|
||
}
|
||
|
||
pub struct MetricsServer {
|
||
listener: TcpListener,
|
||
router: Router,
|
||
}
|
||
|
||
impl MetricsServer {
|
||
pub fn local_addr(&self) -> Result<SocketAddr, MetricsServeError> {
|
||
self.listener
|
||
.local_addr()
|
||
.map_err(|_| MetricsServeError::LocalAddress)
|
||
}
|
||
|
||
pub async fn serve(self) -> Result<(), MetricsServeError> {
|
||
axum::serve(self.listener, self.router)
|
||
.await
|
||
.map_err(|_| MetricsServeError::Serve)
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Error)]
|
||
pub enum MetricsSurfaceError {
|
||
#[error("failed to configure Prometheus recorder")]
|
||
RecorderConfiguration,
|
||
}
|
||
|
||
#[derive(Debug, Error)]
|
||
pub enum MetricsServeError {
|
||
#[error("failed to bind metrics listener")]
|
||
Bind,
|
||
#[error("metrics listener stopped unexpectedly")]
|
||
Serve,
|
||
#[error("failed to read metrics listener address")]
|
||
LocalAddress,
|
||
}
|
||
|
||
pub(crate) fn install_prometheus_recorder(
|
||
identity: &ServiceIdentity,
|
||
) -> Result<PrometheusHandle, MetricsSurfaceError> {
|
||
let service = MetricService::parse(identity.service())
|
||
.ok_or(MetricsSurfaceError::RecorderConfiguration)?;
|
||
if crank_metrics::bound_metric_service() != Some(service) {
|
||
return Err(MetricsSurfaceError::RecorderConfiguration);
|
||
}
|
||
prometheus_builder(identity)?
|
||
.install_recorder()
|
||
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
|
||
}
|
||
|
||
fn prometheus_builder(
|
||
identity: &ServiceIdentity,
|
||
) -> Result<PrometheusBuilder, MetricsSurfaceError> {
|
||
MetricService::parse(identity.service()).ok_or(MetricsSurfaceError::RecorderConfiguration)?;
|
||
PrometheusBuilder::new()
|
||
.set_buckets(DURATION_BUCKETS_SECONDS)
|
||
.map(|builder| {
|
||
builder
|
||
.add_global_label("service", identity.service())
|
||
.add_global_label("version", identity.version())
|
||
.add_global_label("environment", identity.environment())
|
||
})
|
||
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
|
||
}
|
||
|
||
async fn render_metrics(State(state): State<MetricsState>, headers: HeaderMap) -> Response {
|
||
let Some(_permit) = ScrapePermit::try_acquire(&state.active_scrapes) else {
|
||
return (StatusCode::SERVICE_UNAVAILABLE, TOO_MANY_SCRAPES).into_response();
|
||
};
|
||
// cfg(test)-only точка синхронизации у permit seam'а: в production-сборке
|
||
// строки нет, поэтому поведение не меняется.
|
||
#[cfg(test)]
|
||
crate::prometheus::scrape_barrier::hold().await;
|
||
let legacy = state.handle.render();
|
||
let openmetrics = headers
|
||
.get(header::ACCEPT)
|
||
.and_then(|value| value.to_str().ok())
|
||
.is_some_and(prefers_openmetrics);
|
||
let body = if openmetrics {
|
||
render_openmetrics(&legacy, &exemplar_snapshot())
|
||
} else {
|
||
legacy
|
||
};
|
||
exposition_response(body, openmetrics)
|
||
}
|
||
|
||
fn exposition_response(body: String, openmetrics: bool) -> Response {
|
||
if body.len() > MAX_EXPOSITION_BYTES {
|
||
return (StatusCode::SERVICE_UNAVAILABLE, EXPOSITION_TOO_LARGE).into_response();
|
||
}
|
||
let mut response = body.into_response();
|
||
response.headers_mut().insert(
|
||
header::CONTENT_TYPE,
|
||
HeaderValue::from_static(if openmetrics {
|
||
OPENMETRICS_CONTENT_TYPE
|
||
} else {
|
||
PROMETHEUS_CONTENT_TYPE
|
||
}),
|
||
);
|
||
response
|
||
}
|
||
|
||
fn render_openmetrics(legacy: &str, exemplars: &[Arc<ExemplarObservation>]) -> String {
|
||
let index: HashMap<String, &ExemplarObservation> = exemplars
|
||
.iter()
|
||
.map(|exemplar| (exemplar_key(exemplar), exemplar.as_ref()))
|
||
.collect();
|
||
let mut output = String::with_capacity(legacy.len() + exemplars.len().saturating_mul(96) + 6);
|
||
for line in legacy.lines() {
|
||
output.push_str(line);
|
||
if let Some(exemplar) = sample_key(line).and_then(|key| index.get(&key).copied()) {
|
||
output.push_str(" # {trace_id=\"");
|
||
output.push_str(exemplar.trace_id.as_str());
|
||
output.push_str("\"} ");
|
||
output.push_str(&exemplar.value.to_string());
|
||
}
|
||
output.push('\n');
|
||
}
|
||
output.push_str("# EOF\n");
|
||
output
|
||
}
|
||
|
||
fn exemplar_key(exemplar: &ExemplarObservation) -> String {
|
||
let mut labels = exemplar.labels.clone();
|
||
labels.sort_unstable();
|
||
let bound = exemplar
|
||
.bucket_upper_bound
|
||
.map_or_else(|| "+Inf".to_owned(), |value| value.to_string());
|
||
format!("{}|{:?}|{bound}", exemplar.metric, labels)
|
||
}
|
||
|
||
fn sample_key(line: &str) -> Option<String> {
|
||
let (head, _) = line.split_once("} ")?;
|
||
let (metric, raw_labels) = head.split_once("_bucket{")?;
|
||
let mut labels = Vec::new();
|
||
let mut bound = None;
|
||
for item in raw_labels.split(',') {
|
||
let (name, value) = item.split_once("=\"")?;
|
||
let value = value.strip_suffix('"')?;
|
||
match name {
|
||
"le" => bound = Some(value),
|
||
"service" | "version" | "environment" => {}
|
||
_ => labels.push((name, value)),
|
||
}
|
||
}
|
||
labels.sort_unstable();
|
||
Some(format!("{metric}|{labels:?}|{}", bound?))
|
||
}
|
||
|
||
fn prefers_openmetrics(value: &str) -> bool {
|
||
let mut open_q = 0.0_f32;
|
||
let mut legacy_q = 0.0_f32;
|
||
for range in value.split(',') {
|
||
let mut parts = range.trim().split(';');
|
||
let media = parts.next().unwrap_or_default().trim();
|
||
let mut q = 1.0_f32;
|
||
let mut supported_version = true;
|
||
for parameter in parts {
|
||
let Some((name, raw)) = parameter.trim().split_once('=') else {
|
||
continue;
|
||
};
|
||
match name.trim() {
|
||
"q" => q = raw.trim().parse().unwrap_or(0.0),
|
||
"version" if media == "application/openmetrics-text" => {
|
||
supported_version = raw.trim().trim_matches('"') == "1.0.0";
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
if media == "application/openmetrics-text" && supported_version {
|
||
open_q = open_q.max(q.clamp(0.0, 1.0));
|
||
} else if media == "text/plain" || media == "*/*" {
|
||
legacy_q = legacy_q.max(q.clamp(0.0, 1.0));
|
||
}
|
||
}
|
||
open_q > 0.0 && open_q >= legacy_q
|
||
}
|
||
|
||
struct ScrapePermit(Arc<AtomicUsize>);
|
||
|
||
impl ScrapePermit {
|
||
fn try_acquire(active: &Arc<AtomicUsize>) -> Option<Self> {
|
||
active
|
||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| {
|
||
(count < MAX_CONCURRENT_SCRAPES).then_some(count + 1)
|
||
})
|
||
.ok()
|
||
.map(|_| Self(Arc::clone(active)))
|
||
}
|
||
}
|
||
|
||
impl Drop for ScrapePermit {
|
||
fn drop(&mut self) {
|
||
self.0.fetch_sub(1, Ordering::Release);
|
||
}
|
||
}
|
||
|
||
async fn metrics_health() -> impl IntoResponse {
|
||
(StatusCode::OK, "ok\n")
|
||
}
|
||
|
||
async fn authorize_metrics(
|
||
State(state): State<MetricsState>,
|
||
request: Request,
|
||
next: Next,
|
||
) -> Response {
|
||
if !state.requires_authentication {
|
||
return next.run(request).await;
|
||
}
|
||
|
||
let authorized = bearer_token(request.headers())
|
||
.map(token_digest)
|
||
.zip(state.token_digest)
|
||
.is_some_and(|(actual, expected)| bool::from(actual.ct_eq(&expected)));
|
||
|
||
if authorized {
|
||
next.run(request).await
|
||
} else {
|
||
StatusCode::UNAUTHORIZED.into_response()
|
||
}
|
||
}
|
||
|
||
fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
|
||
let value = headers.get(header::AUTHORIZATION)?.as_bytes();
|
||
let separator = value.iter().position(|byte| *byte == b' ')?;
|
||
let (scheme, token_with_spaces) = value.split_at(separator);
|
||
let token_start = token_with_spaces.iter().position(|byte| *byte != b' ')?;
|
||
let token = &token_with_spaces[token_start..];
|
||
|
||
scheme
|
||
.eq_ignore_ascii_case(b"bearer")
|
||
.then_some(token)
|
||
.filter(|token| !token.is_empty())
|
||
}
|
||
|
||
fn token_digest(token: &[u8]) -> [u8; 32] {
|
||
Sha256::digest(token).into()
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod rendering_tests;
|
||
|
||
/// cfg(test)-only барьер у scrape-permit seam'а: позволяет детерминированно
|
||
/// доказать, что допускаются ровно два одновременных scrape, третий
|
||
/// отвергается, а после release permit освобождается. В production-сборке
|
||
/// (и в интеграционных тест-бинар��х, где crate компилируется как dependency)
|
||
/// модуля нет, поэтому семантика не меняется.
|
||
#[cfg(test)]
|
||
pub(crate) mod scrape_barrier {
|
||
use std::sync::{Mutex, MutexGuard, OnceLock};
|
||
|
||
static STATE: OnceLock<Mutex<State>> = OnceLock::new();
|
||
|
||
#[derive(Default)]
|
||
struct State {
|
||
enabled: bool,
|
||
release: bool,
|
||
parked: usize,
|
||
}
|
||
|
||
fn lock() -> MutexGuard<'static, State> {
|
||
STATE
|
||
.get_or_init(|| Mutex::new(State::default()))
|
||
.lock()
|
||
.unwrap_or_else(|error| error.into_inner())
|
||
}
|
||
|
||
/// Включает барьер для одного детерминированного кейса.
|
||
pub(crate) fn enable() {
|
||
let mut state = lock();
|
||
state.enabled = true;
|
||
state.release = false;
|
||
state.parked = 0;
|
||
}
|
||
|
||
/// Парковка после успешного acquire permit'а. Если барьер выключен —
|
||
/// немедленный возврат, поэтому прочие unit-тесты не затрагиваются.
|
||
/// Ожидание асинхронное (yield), поэтому worker'ы runtime'а не блокируются и
|
||
/// обслуживание следующих соединений остаётся детерминированным.
|
||
pub(crate) async fn hold() {
|
||
{
|
||
let mut state = lock();
|
||
if !state.enabled {
|
||
return;
|
||
}
|
||
state.parked += 1;
|
||
}
|
||
while !lock().release {
|
||
tokio::task::yield_now().await;
|
||
}
|
||
}
|
||
|
||
/// Ожидание `count` припаркованных scrape'ов; `false` — истёк deadline.
|
||
/// Вызывается из blocking-пула, поэтому не расходует worker'ы runtime'а.
|
||
pub(crate) fn wait_for_parked(count: usize, timeout: std::time::Duration) -> bool {
|
||
let deadline = std::time::Instant::now() + timeout;
|
||
while std::time::Instant::now() < deadline {
|
||
if lock().parked >= count {
|
||
return true;
|
||
}
|
||
std::thread::sleep(std::time::Duration::from_millis(2));
|
||
}
|
||
lock().parked >= count
|
||
}
|
||
|
||
pub(crate) fn parked() -> usize {
|
||
lock().parked
|
||
}
|
||
|
||
/// Освобождает все припаркованные scrape'ы.
|
||
pub(crate) fn release() {
|
||
lock().release = true;
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod recorder_guard_tests {
|
||
use std::process::Command;
|
||
|
||
use super::{MetricsSurfaceError, install_prometheus_recorder};
|
||
use crate::ServiceIdentity;
|
||
|
||
const GUARD_ROLE_PREFIX: &str = "crank-observability-guard-role-";
|
||
|
||
fn current_role() -> Option<String> {
|
||
std::env::current_dir()
|
||
.ok()?
|
||
.file_name()?
|
||
.to_str()?
|
||
.strip_prefix(GUARD_ROLE_PREFIX)
|
||
.map(str::to_owned)
|
||
}
|
||
|
||
/// F7 — приватный `install_prometheus_recorder` обязан отвергать identity,
|
||
/// не совпадающую с опубликованным binding'ом, **до** любой установки
|
||
/// recorder'а; ошибка типизированная, а positive control доказывает, что
|
||
/// отклонённый вызов ничего не установил.
|
||
///
|
||
/// Свежий процесс обязателен: outer lifecycle-тест падает раньше (уже
|
||
/// занятый global recorder/binding), поэтому приватный guard он не
|
||
/// доказывает.
|
||
#[test]
|
||
fn private_recorder_guard_rejects_identity_mismatch_before_installing() {
|
||
if current_role().is_none() {
|
||
let role = "guard";
|
||
let directory = std::env::temp_dir().join(format!("{GUARD_ROLE_PREFIX}{role}"));
|
||
std::fs::create_dir_all(&directory).expect("guard role dir");
|
||
let output = Command::new(std::env::current_exe().expect("current test binary"))
|
||
.args([
|
||
"prometheus::recorder_guard_tests::private_recorder_guard_rejects_identity_mismatch_before_installing",
|
||
"--exact",
|
||
"--nocapture",
|
||
])
|
||
.current_dir(&directory)
|
||
.output()
|
||
.expect("re-executed guard child");
|
||
assert!(
|
||
output.status.success(),
|
||
"guard child failed:\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&output.stdout),
|
||
String::from_utf8_lossy(&output.stderr)
|
||
);
|
||
return;
|
||
}
|
||
|
||
// Свежий процесс: ни recorder, ни binding ещё не установлены.
|
||
assert_eq!(crank_metrics::bound_metric_service(), None);
|
||
crank_metrics::bind_metric_service(crank_metrics::MetricService::AdminApi)
|
||
.expect("admin binding");
|
||
|
||
let mismatched = ServiceIdentity::try_new("mcp-server", "0.3.1", "test")
|
||
.expect("valid mismatched identity");
|
||
match install_prometheus_recorder(&mismatched) {
|
||
Err(MetricsSurfaceError::RecorderConfiguration) => {}
|
||
other => panic!(
|
||
"identity другого сервиса обязана быть отвергнута приватным guard'ом: {other:?}"
|
||
),
|
||
}
|
||
|
||
// Positive control: matching identity проходит, значит предыдущий отказ
|
||
// пришёл из guard'а, а не из заранее установленного recorder'а.
|
||
let matching = ServiceIdentity::try_new("admin-api", "0.3.1", "test")
|
||
.expect("valid matching identity");
|
||
let handle = install_prometheus_recorder(&matching)
|
||
.expect("matching identity обязан установить recorder");
|
||
crank_metrics::record_http_request(
|
||
crank_metrics::HttpRoute::from_matched_path("/health"),
|
||
crank_metrics::HttpMethod::Get,
|
||
crank_metrics::HttpStatusClass::Success,
|
||
std::time::Duration::from_millis(1),
|
||
);
|
||
let exposition = handle.render();
|
||
assert!(
|
||
exposition.contains("crank_http_requests_total"),
|
||
"установленный recorder обязан отдавать записанные series: {exposition}"
|
||
);
|
||
}
|
||
}
|