Files
crank/crates/crank-observability/src/prometheus.rs
T
bsodfather 54ce770682
CI / Rust Checks (pull_request) Failing after 13m41s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
Community Release / release (push) Failing after 13m33s
feat(epic3): deliver the accepted Epic 3 scope through Story 3.17
Recovery delivery of the accumulated, TL-accepted Epic 3 work that had never been
committed to Git (stories 3.6-3.17, see the crank board). Story 3.18 is not accepted
yet and is deliberately excluded: its logging surface, its capability registration,
its oracle projections and its tests are not part of this tree.

Accepted scope
- 3.6 typed static Resource Drafts; 3.7 publication and exact binding of static
  Resource Versions; 3.8 MCP discovery/read and revocation of static Resources;
- 3.9 Resource import/export in API and UI plus TLS dependency maintenance;
  3.10 REST-backed Resource templates; 3.11 shared REST read runtime and the MCP
  revision/capability foundation;
- 3.12 parameterised Prompt Drafts with preview; 3.13 Prompt publication and exact
  version binding to Agents; 3.14 MCP prompts; 3.15 Prompt import/export;
- 3.16 composite Prompts with published Resources (dependency content scan plus
  dynamic read through the accepted 3.11 runtime, admin preview and MCP prompts/get);
- 3.17 bounded MCP completion suggestions from typed published schemas and the
  resources/list admission fix for bound template publications.

Accepted evidence being reused (unchanged code paths)
- 3.16 candidate: core 7, admin-api 22, registry 4, mcp-server 4, UI 6; one final
  `just verify` run 1651 passed / 0 failed / 2 ignored.
- 3.17 candidate: full run 1672 passed / 0 failed / 2 ignored, QA targeted reruns and
  a real MCP Inspector 2.12.0 client smoke.

Delivery engineering in this commit
- CI: one event per change (feature branches are gated by their pull request, push
  runs only on main), story events run the changed packages, the full workspace suite
  stays the feature/release gate; no cache/sharding without a measured runner check.
- Release: the packaged archives, checksums and SBOM are now uploaded to the Gitea
  release entry of the tag instead of dying with the runner workspace.

Not included on purpose: Story 3.18 work, build caches, runtime databases, local
credentials and scratch evidence.

Refs: crank kanban t_e39d7263; baseline 1b6db09bd1.
2026-10-25 09:08:22 +00:00

557 lines
19 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use std::{
collections::HashMap,
net::SocketAddr,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
use axum::{
Router,
extract::{Request, State},
http::{
HeaderMap, StatusCode,
header::{self, HeaderValue},
},
middleware::{self, Next},
response::{IntoResponse, Response},
routing::get,
};
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle, PrometheusRecorder};
use sha2::{Digest, Sha256};
use subtle::ConstantTimeEq;
use thiserror::Error;
use tokio::net::TcpListener;
use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity};
use crank_metrics::{ExemplarObservation, MAX_EXPOSITION_BYTES, MetricService, exemplar_snapshot};
const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
const OPENMETRICS_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
const EXPOSITION_TOO_LARGE: &str = "metrics exposition exceeds configured bound\n";
const TOO_MANY_SCRAPES: &str = "metrics scrape concurrency limit exceeded\n";
const MAX_CONCURRENT_SCRAPES: usize = 2;
#[derive(Clone)]
pub struct MetricsConfig {
enabled: bool,
bind_addr: SocketAddr,
token_digest: Option<[u8; 32]>,
}
impl std::fmt::Debug for MetricsConfig {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("MetricsConfig")
.field("enabled", &self.enabled)
.field("bind_addr", &self.bind_addr)
.field("authentication_configured", &self.token_digest.is_some())
.finish()
}
}
impl MetricsConfig {
pub fn new(
enabled: bool,
bind_addr: SocketAddr,
bearer_token: Option<String>,
) -> Result<Self, MetricsConfigError> {
let token_digest = bearer_token
.filter(|token| !token.is_empty())
.map(|token| token_digest(token.as_bytes()));
if enabled && !bind_addr.ip().is_loopback() && token_digest.is_none() {
return Err(MetricsConfigError::MissingTokenForExternalBind);
}
Ok(Self {
enabled,
bind_addr,
token_digest,
})
}
pub fn enabled(&self) -> bool {
self.enabled
}
pub fn bind_addr(&self) -> SocketAddr {
self.bind_addr
}
pub fn requires_authentication(&self) -> bool {
!self.bind_addr.ip().is_loopback()
}
}
#[derive(Debug, Error)]
pub enum MetricsConfigError {
#[error("metrics environment variable is not valid UTF-8: {field}")]
InvalidEnvironmentEncoding { field: &'static str },
#[error("metrics bind address is invalid: {field}")]
InvalidBindAddress { field: &'static str },
#[error("metrics enabled flag must be one of true, false, 1, 0")]
InvalidEnabledFlag,
#[error("external metrics bind requires a bearer token")]
MissingTokenForExternalBind,
}
#[derive(Clone)]
struct MetricsState {
handle: PrometheusHandle,
token_digest: Option<[u8; 32]>,
requires_authentication: bool,
active_scrapes: Arc<AtomicUsize>,
}
pub struct MetricsSurface {
config: MetricsConfig,
state: MetricsState,
_recorder: Option<PrometheusRecorder>,
}
impl MetricsSurface {
pub(crate) fn new(config: MetricsConfig, handle: PrometheusHandle) -> Self {
Self {
state: MetricsState {
handle,
token_digest: config.token_digest,
requires_authentication: config.requires_authentication(),
active_scrapes: Arc::new(AtomicUsize::new(0)),
},
config,
_recorder: None,
}
}
pub fn for_test(
config: MetricsConfig,
identity: ServiceIdentity,
) -> Result<Self, MetricsSurfaceError> {
let recorder = prometheus_builder(&identity)?.build_recorder();
let handle = recorder.handle();
let mut surface = Self::new(config, handle);
surface._recorder = Some(recorder);
Ok(surface)
}
pub fn router(&self) -> Router {
Router::new()
.route("/metrics", get(render_metrics))
.route("/health", get(metrics_health))
.route_layer(middleware::from_fn_with_state(
self.state.clone(),
authorize_metrics,
))
.with_state(self.state.clone())
}
pub async fn bind(self) -> Result<MetricsServer, MetricsServeError> {
let listener = TcpListener::bind(self.config.bind_addr)
.await
.map_err(|_| MetricsServeError::Bind)?;
Ok(MetricsServer {
listener,
router: self.router(),
})
}
}
pub struct MetricsServer {
listener: TcpListener,
router: Router,
}
impl MetricsServer {
pub fn local_addr(&self) -> Result<SocketAddr, MetricsServeError> {
self.listener
.local_addr()
.map_err(|_| MetricsServeError::LocalAddress)
}
pub async fn serve(self) -> Result<(), MetricsServeError> {
axum::serve(self.listener, self.router)
.await
.map_err(|_| MetricsServeError::Serve)
}
}
#[derive(Debug, Error)]
pub enum MetricsSurfaceError {
#[error("failed to configure Prometheus recorder")]
RecorderConfiguration,
}
#[derive(Debug, Error)]
pub enum MetricsServeError {
#[error("failed to bind metrics listener")]
Bind,
#[error("metrics listener stopped unexpectedly")]
Serve,
#[error("failed to read metrics listener address")]
LocalAddress,
}
pub(crate) fn install_prometheus_recorder(
identity: &ServiceIdentity,
) -> Result<PrometheusHandle, MetricsSurfaceError> {
let service = MetricService::parse(identity.service())
.ok_or(MetricsSurfaceError::RecorderConfiguration)?;
if crank_metrics::bound_metric_service() != Some(service) {
return Err(MetricsSurfaceError::RecorderConfiguration);
}
prometheus_builder(identity)?
.install_recorder()
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
}
fn prometheus_builder(
identity: &ServiceIdentity,
) -> Result<PrometheusBuilder, MetricsSurfaceError> {
MetricService::parse(identity.service()).ok_or(MetricsSurfaceError::RecorderConfiguration)?;
PrometheusBuilder::new()
.set_buckets(DURATION_BUCKETS_SECONDS)
.map(|builder| {
builder
.add_global_label("service", identity.service())
.add_global_label("version", identity.version())
.add_global_label("environment", identity.environment())
})
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
}
async fn render_metrics(State(state): State<MetricsState>, headers: HeaderMap) -> Response {
let Some(_permit) = ScrapePermit::try_acquire(&state.active_scrapes) else {
return (StatusCode::SERVICE_UNAVAILABLE, TOO_MANY_SCRAPES).into_response();
};
// cfg(test)-only точка синхронизации у permit seam'а: в production-сборке
// строки нет, поэтому поведение не меняется.
#[cfg(test)]
crate::prometheus::scrape_barrier::hold().await;
let legacy = state.handle.render();
let openmetrics = headers
.get(header::ACCEPT)
.and_then(|value| value.to_str().ok())
.is_some_and(prefers_openmetrics);
let body = if openmetrics {
render_openmetrics(&legacy, &exemplar_snapshot())
} else {
legacy
};
exposition_response(body, openmetrics)
}
fn exposition_response(body: String, openmetrics: bool) -> Response {
if body.len() > MAX_EXPOSITION_BYTES {
return (StatusCode::SERVICE_UNAVAILABLE, EXPOSITION_TOO_LARGE).into_response();
}
let mut response = body.into_response();
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static(if openmetrics {
OPENMETRICS_CONTENT_TYPE
} else {
PROMETHEUS_CONTENT_TYPE
}),
);
response
}
fn render_openmetrics(legacy: &str, exemplars: &[Arc<ExemplarObservation>]) -> String {
let index: HashMap<String, &ExemplarObservation> = exemplars
.iter()
.map(|exemplar| (exemplar_key(exemplar), exemplar.as_ref()))
.collect();
let mut output = String::with_capacity(legacy.len() + exemplars.len().saturating_mul(96) + 6);
for line in legacy.lines() {
output.push_str(line);
if let Some(exemplar) = sample_key(line).and_then(|key| index.get(&key).copied()) {
output.push_str(" # {trace_id=\"");
output.push_str(exemplar.trace_id.as_str());
output.push_str("\"} ");
output.push_str(&exemplar.value.to_string());
}
output.push('\n');
}
output.push_str("# EOF\n");
output
}
fn exemplar_key(exemplar: &ExemplarObservation) -> String {
let mut labels = exemplar.labels.clone();
labels.sort_unstable();
let bound = exemplar
.bucket_upper_bound
.map_or_else(|| "+Inf".to_owned(), |value| value.to_string());
format!("{}|{:?}|{bound}", exemplar.metric, labels)
}
fn sample_key(line: &str) -> Option<String> {
let (head, _) = line.split_once("} ")?;
let (metric, raw_labels) = head.split_once("_bucket{")?;
let mut labels = Vec::new();
let mut bound = None;
for item in raw_labels.split(',') {
let (name, value) = item.split_once("=\"")?;
let value = value.strip_suffix('"')?;
match name {
"le" => bound = Some(value),
"service" | "version" | "environment" => {}
_ => labels.push((name, value)),
}
}
labels.sort_unstable();
Some(format!("{metric}|{labels:?}|{}", bound?))
}
fn prefers_openmetrics(value: &str) -> bool {
let mut open_q = 0.0_f32;
let mut legacy_q = 0.0_f32;
for range in value.split(',') {
let mut parts = range.trim().split(';');
let media = parts.next().unwrap_or_default().trim();
let mut q = 1.0_f32;
let mut supported_version = true;
for parameter in parts {
let Some((name, raw)) = parameter.trim().split_once('=') else {
continue;
};
match name.trim() {
"q" => q = raw.trim().parse().unwrap_or(0.0),
"version" if media == "application/openmetrics-text" => {
supported_version = raw.trim().trim_matches('"') == "1.0.0";
}
_ => {}
}
}
if media == "application/openmetrics-text" && supported_version {
open_q = open_q.max(q.clamp(0.0, 1.0));
} else if media == "text/plain" || media == "*/*" {
legacy_q = legacy_q.max(q.clamp(0.0, 1.0));
}
}
open_q > 0.0 && open_q >= legacy_q
}
struct ScrapePermit(Arc<AtomicUsize>);
impl ScrapePermit {
fn try_acquire(active: &Arc<AtomicUsize>) -> Option<Self> {
active
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| {
(count < MAX_CONCURRENT_SCRAPES).then_some(count + 1)
})
.ok()
.map(|_| Self(Arc::clone(active)))
}
}
impl Drop for ScrapePermit {
fn drop(&mut self) {
self.0.fetch_sub(1, Ordering::Release);
}
}
async fn metrics_health() -> impl IntoResponse {
(StatusCode::OK, "ok\n")
}
async fn authorize_metrics(
State(state): State<MetricsState>,
request: Request,
next: Next,
) -> Response {
if !state.requires_authentication {
return next.run(request).await;
}
let authorized = bearer_token(request.headers())
.map(token_digest)
.zip(state.token_digest)
.is_some_and(|(actual, expected)| bool::from(actual.ct_eq(&expected)));
if authorized {
next.run(request).await
} else {
StatusCode::UNAUTHORIZED.into_response()
}
}
fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
let value = headers.get(header::AUTHORIZATION)?.as_bytes();
let separator = value.iter().position(|byte| *byte == b' ')?;
let (scheme, token_with_spaces) = value.split_at(separator);
let token_start = token_with_spaces.iter().position(|byte| *byte != b' ')?;
let token = &token_with_spaces[token_start..];
scheme
.eq_ignore_ascii_case(b"bearer")
.then_some(token)
.filter(|token| !token.is_empty())
}
fn token_digest(token: &[u8]) -> [u8; 32] {
Sha256::digest(token).into()
}
#[cfg(test)]
mod rendering_tests;
/// cfg(test)-only барьер у scrape-permit seam'а: позволяет детерминированно
/// доказать, что допускаются ровно два одновременных scrape, третий
/// отвергается, а после release permit освобождается. В production-сборке
/// (и в интеграционных тест-бинар��х, где crate компилируется как dependency)
/// модуля нет, поэтому семантика не меняется.
#[cfg(test)]
pub(crate) mod scrape_barrier {
use std::sync::{Mutex, MutexGuard, OnceLock};
static STATE: OnceLock<Mutex<State>> = OnceLock::new();
#[derive(Default)]
struct State {
enabled: bool,
release: bool,
parked: usize,
}
fn lock() -> MutexGuard<'static, State> {
STATE
.get_or_init(|| Mutex::new(State::default()))
.lock()
.unwrap_or_else(|error| error.into_inner())
}
/// Включает барьер для одного детерминированного кейса.
pub(crate) fn enable() {
let mut state = lock();
state.enabled = true;
state.release = false;
state.parked = 0;
}
/// Парковка после успешного acquire permit'а. Если барьер выключен —
/// немедленный возврат, поэтому прочие unit-тесты не затрагиваются.
/// Ожидание асинхронное (yield), поэтому worker'ы runtime'а не блокируются и
/// обслуживание следующих соединений остаётся детерминированным.
pub(crate) async fn hold() {
{
let mut state = lock();
if !state.enabled {
return;
}
state.parked += 1;
}
while !lock().release {
tokio::task::yield_now().await;
}
}
/// Ожидание `count` припаркованных scrape'ов; `false` — истёк deadline.
/// Вызывается из blocking-пула, поэтому не расходует worker'ы runtime'а.
pub(crate) fn wait_for_parked(count: usize, timeout: std::time::Duration) -> bool {
let deadline = std::time::Instant::now() + timeout;
while std::time::Instant::now() < deadline {
if lock().parked >= count {
return true;
}
std::thread::sleep(std::time::Duration::from_millis(2));
}
lock().parked >= count
}
pub(crate) fn parked() -> usize {
lock().parked
}
/// Освобождает все припаркованные scrape'ы.
pub(crate) fn release() {
lock().release = true;
}
}
#[cfg(test)]
mod recorder_guard_tests {
use std::process::Command;
use super::{MetricsSurfaceError, install_prometheus_recorder};
use crate::ServiceIdentity;
const GUARD_ROLE_PREFIX: &str = "crank-observability-guard-role-";
fn current_role() -> Option<String> {
std::env::current_dir()
.ok()?
.file_name()?
.to_str()?
.strip_prefix(GUARD_ROLE_PREFIX)
.map(str::to_owned)
}
/// F7 — приватный `install_prometheus_recorder` обязан отвергать identity,
/// не совпадающую с опубликованным binding'ом, **до** любой установки
/// recorder'а; ошибка типизированная, а positive control доказывает, что
/// отклонённый вызов ничего не установил.
///
/// Свежий процесс обязателен: outer lifecycle-тест падает раньше (уже
/// занятый global recorder/binding), поэтому приватный guard он не
/// доказывает.
#[test]
fn private_recorder_guard_rejects_identity_mismatch_before_installing() {
if current_role().is_none() {
let role = "guard";
let directory = std::env::temp_dir().join(format!("{GUARD_ROLE_PREFIX}{role}"));
std::fs::create_dir_all(&directory).expect("guard role dir");
let output = Command::new(std::env::current_exe().expect("current test binary"))
.args([
"prometheus::recorder_guard_tests::private_recorder_guard_rejects_identity_mismatch_before_installing",
"--exact",
"--nocapture",
])
.current_dir(&directory)
.output()
.expect("re-executed guard child");
assert!(
output.status.success(),
"guard child failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
return;
}
// Свежий процесс: ни recorder, ни binding ещё не установлены.
assert_eq!(crank_metrics::bound_metric_service(), None);
crank_metrics::bind_metric_service(crank_metrics::MetricService::AdminApi)
.expect("admin binding");
let mismatched = ServiceIdentity::try_new("mcp-server", "0.3.1", "test")
.expect("valid mismatched identity");
match install_prometheus_recorder(&mismatched) {
Err(MetricsSurfaceError::RecorderConfiguration) => {}
other => panic!(
"identity другого сервиса обязана быть отвергнута приватным guard'ом: {other:?}"
),
}
// Positive control: matching identity проходит, значит предыдущий отказ
// пришёл из guard'а, а не из заранее установленного recorder'а.
let matching = ServiceIdentity::try_new("admin-api", "0.3.1", "test")
.expect("valid matching identity");
let handle = install_prometheus_recorder(&matching)
.expect("matching identity обязан установить recorder");
crank_metrics::record_http_request(
crank_metrics::HttpRoute::from_matched_path("/health"),
crank_metrics::HttpMethod::Get,
crank_metrics::HttpStatusClass::Success,
std::time::Duration::from_millis(1),
);
let exposition = handle.render();
assert!(
exposition.contains("crank_http_requests_total"),
"установленный recorder обязан отдавать записанные series: {exposition}"
);
}
}