Files
crank/crates/crank-metrics/src/record.rs
T
bsodfather 54ce770682
CI / Rust Checks (pull_request) Failing after 13m41s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
Community Release / release (push) Failing after 13m33s
feat(epic3): deliver the accepted Epic 3 scope through Story 3.17
Recovery delivery of the accumulated, TL-accepted Epic 3 work that had never been
committed to Git (stories 3.6-3.17, see the crank board). Story 3.18 is not accepted
yet and is deliberately excluded: its logging surface, its capability registration,
its oracle projections and its tests are not part of this tree.

Accepted scope
- 3.6 typed static Resource Drafts; 3.7 publication and exact binding of static
  Resource Versions; 3.8 MCP discovery/read and revocation of static Resources;
- 3.9 Resource import/export in API and UI plus TLS dependency maintenance;
  3.10 REST-backed Resource templates; 3.11 shared REST read runtime and the MCP
  revision/capability foundation;
- 3.12 parameterised Prompt Drafts with preview; 3.13 Prompt publication and exact
  version binding to Agents; 3.14 MCP prompts; 3.15 Prompt import/export;
- 3.16 composite Prompts with published Resources (dependency content scan plus
  dynamic read through the accepted 3.11 runtime, admin preview and MCP prompts/get);
- 3.17 bounded MCP completion suggestions from typed published schemas and the
  resources/list admission fix for bound template publications.

Accepted evidence being reused (unchanged code paths)
- 3.16 candidate: core 7, admin-api 22, registry 4, mcp-server 4, UI 6; one final
  `just verify` run 1651 passed / 0 failed / 2 ignored.
- 3.17 candidate: full run 1672 passed / 0 failed / 2 ignored, QA targeted reruns and
  a real MCP Inspector 2.12.0 client smoke.

Delivery engineering in this commit
- CI: one event per change (feature branches are gated by their pull request, push
  runs only on main), story events run the changed packages, the full workspace suite
  stays the feature/release gate; no cache/sharding without a measured runner check.
- Release: the packaged archives, checksums and SBOM are now uploaded to the Gitea
  release entry of the tag instead of dying with the runner workspace.

Not included on purpose: Story 3.18 work, build caches, runtime databases, local
credentials and scratch evidence.

Refs: crank kanban t_e39d7263; baseline 1b6db09bd1.
2026-10-25 09:08:22 +00:00

494 lines
14 KiB
Rust

use std::sync::OnceLock;
use std::time::{Duration, Instant};
use metrics::Gauge;
use crate::schema::{MetricDefinition, metric_schema};
use crate::{
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
static BINDING: OnceLock<MetricService> = OnceLock::new();
/// Publishes the closed process identity exactly once, before the recorder,
/// gauges, producers and any listener exist. Re-binding the same service is
/// idempotent; a different service is a typed conflict and never wins twice.
pub fn bind_metric_service(service: MetricService) -> Result<(), MetricBindingError> {
crate::schema::validate_all_projections().map_err(|_| MetricBindingError::InvalidSchema)?;
// `OnceLock::set` returns `Err(value)` (the value passed in, not the stored
// one), so the idempotency check must read the published binding back.
if BINDING.set(service).is_ok() {
return Ok(());
}
match BINDING.get() {
Some(existing) if *existing == service => Ok(()),
_ => Err(MetricBindingError::ConflictingService),
}
}
pub fn bound_metric_service() -> Option<MetricService> {
BINDING.get().copied()
}
/// Normalises one route against the bound service: own and shared routes are
/// preserved, a known route owned by another service degrades to `unmatched`.
pub fn normalize_http_route(service: MetricService, route: HttpRoute) -> HttpRoute {
match crate::http_route_processes(route.as_str()) {
Some(owners) if owners.contains(&service.process()) => route,
_ => HttpRoute::unmatched(),
}
}
pub(crate) fn family_applies(service: MetricService, name: &'static str) -> bool {
metric_schema()
.iter()
.find(|definition| definition.name == name)
.is_some_and(|definition: &MetricDefinition| {
definition.processes.contains(&service.process())
})
}
/// Unbound facades (plain unit tests or libraries without observability) are a
/// safe no-op; the process never registers a metric handle or exemplar entry.
fn bound_family(name: &'static str) -> Option<MetricService> {
let service = BINDING.get().copied()?;
family_applies(service, name).then_some(service)
}
pub fn record_http_request(
route: HttpRoute,
method: HttpMethod,
status: HttpStatusClass,
duration: Duration,
) {
record_http_request_with_exemplar(route, method, status, duration, None);
}
pub fn record_http_request_with_exemplar(
route: HttpRoute,
method: HttpMethod,
status: HttpStatusClass,
duration: Duration,
exemplar: Option<ExemplarTraceId>,
) {
let Some(service) = bound_family("crank_http_requests_total") else {
return;
};
let route = normalize_http_route(service, route);
metrics::counter!(
"crank_http_requests_total",
"route" => route.as_str(),
"method" => method.as_str(),
"status_class" => status.as_str()
)
.increment(1);
metrics::histogram!(
"crank_http_request_duration_seconds",
"route" => route.as_str(),
"method" => method.as_str()
)
.record(duration.as_secs_f64());
crate::exemplar::record_exemplar(
service,
"crank_http_request_duration_seconds",
vec![("route", route.as_str()), ("method", method.as_str())],
duration.as_secs_f64(),
exemplar,
);
}
pub fn record_mcp_request(
method: McpMethod,
response_mode: McpResponseMode,
outcome: McpOutcome,
duration: Duration,
) {
record_mcp_request_with_exemplar(method, response_mode, outcome, duration, None);
}
pub fn record_mcp_request_with_exemplar(
method: McpMethod,
response_mode: McpResponseMode,
outcome: McpOutcome,
duration: Duration,
exemplar: Option<ExemplarTraceId>,
) {
let Some(service) = bound_family("crank_mcp_requests_total") else {
return;
};
metrics::counter!(
"crank_mcp_requests_total",
"method" => method.as_str(),
"response_mode" => response_mode.as_str(),
"outcome" => outcome.as_str()
)
.increment(1);
metrics::histogram!(
"crank_mcp_request_duration_seconds",
"method" => method.as_str(),
"outcome" => outcome.as_str()
)
.record(duration.as_secs_f64());
crate::exemplar::record_exemplar(
service,
"crank_mcp_request_duration_seconds",
vec![("method", method.as_str()), ("outcome", outcome.as_str())],
duration.as_secs_f64(),
exemplar,
);
}
pub fn set_mcp_active_sessions(count: u64) {
if bound_family("crank_mcp_active_sessions").is_none() {
return;
}
metrics::gauge!("crank_mcp_active_sessions").set(count as f64);
}
pub fn set_mcp_session_metrics_fresh(fresh: bool) {
if bound_family("crank_mcp_session_metrics_fresh").is_none() {
return;
}
metrics::gauge!("crank_mcp_session_metrics_fresh").set(if fresh { 1.0 } else { 0.0 });
}
pub fn record_tool_invocation(
source: InvocationSource,
outcome: ToolOutcome,
error_kind: ToolErrorKind,
duration: Duration,
) {
record_tool_invocation_with_exemplar(source, outcome, error_kind, duration, None);
}
pub fn record_tool_invocation_with_exemplar(
source: InvocationSource,
outcome: ToolOutcome,
error_kind: ToolErrorKind,
duration: Duration,
exemplar: Option<ExemplarTraceId>,
) {
let Some(service) = bound_family("crank_tool_invocations_total") else {
return;
};
let error_kind = match outcome {
ToolOutcome::Success => ToolErrorKind::None,
ToolOutcome::Aborted => ToolErrorKind::Aborted,
ToolOutcome::Error
if matches!(error_kind, ToolErrorKind::None | ToolErrorKind::Aborted) =>
{
ToolErrorKind::ProtocolAdapter
}
ToolOutcome::Error => error_kind,
};
metrics::counter!(
"crank_tool_invocations_total",
"source" => source.as_str(),
"outcome" => outcome.as_str(),
"error_kind" => error_kind.as_str()
)
.increment(1);
metrics::histogram!(
"crank_tool_invocation_duration_seconds",
"source" => source.as_str(),
"outcome" => outcome.as_str()
)
.record(duration.as_secs_f64());
crate::exemplar::record_exemplar(
service,
"crank_tool_invocation_duration_seconds",
vec![("source", source.as_str()), ("outcome", outcome.as_str())],
duration.as_secs_f64(),
exemplar,
);
}
pub fn record_upstream_request(
operation_kind: UpstreamOperationKind,
outcome: UpstreamOutcome,
duration: Duration,
) {
record_upstream_request_with_exemplar(operation_kind, outcome, duration, None);
}
pub fn record_upstream_request_with_exemplar(
operation_kind: UpstreamOperationKind,
outcome: UpstreamOutcome,
duration: Duration,
exemplar: Option<ExemplarTraceId>,
) {
let Some(service) = bound_family("crank_upstream_requests_total") else {
return;
};
metrics::counter!(
"crank_upstream_requests_total",
"operation_kind" => operation_kind.as_str(),
"outcome" => outcome.as_str()
)
.increment(1);
metrics::histogram!(
"crank_upstream_request_duration_seconds",
"operation_kind" => operation_kind.as_str(),
"outcome" => outcome.as_str()
)
.record(duration.as_secs_f64());
crate::exemplar::record_exemplar(
service,
"crank_upstream_request_duration_seconds",
vec![
("operation_kind", operation_kind.as_str()),
("outcome", outcome.as_str()),
],
duration.as_secs_f64(),
exemplar,
);
}
pub struct ToolInvocationMetrics {
source: InvocationSource,
started_at: Option<Instant>,
exemplar: Option<ExemplarTraceId>,
active: bool,
}
impl ToolInvocationMetrics {
pub fn start(source: InvocationSource) -> Self {
Self::start_with_exemplar(source, None)
}
pub fn start_with_exemplar(
source: InvocationSource,
exemplar: Option<ExemplarTraceId>,
) -> Self {
let active = bound_family("crank_tool_invocations_total").is_some();
Self {
source,
started_at: active.then(Instant::now),
exemplar,
active,
}
}
pub fn complete(mut self, outcome: ToolOutcome, error_kind: ToolErrorKind) {
self.finish(outcome, error_kind);
}
fn finish(&mut self, outcome: ToolOutcome, error_kind: ToolErrorKind) {
if let Some(started_at) = self.started_at.take() {
record_tool_invocation_with_exemplar(
self.source,
outcome,
error_kind,
started_at.elapsed(),
self.exemplar,
);
}
}
}
impl Drop for ToolInvocationMetrics {
fn drop(&mut self) {
if self.active {
self.finish(ToolOutcome::Aborted, ToolErrorKind::Aborted);
}
}
}
pub struct UpstreamRequestMetrics {
operation_kind: UpstreamOperationKind,
started_at: Option<Instant>,
exemplar: Option<ExemplarTraceId>,
active: bool,
}
impl UpstreamRequestMetrics {
pub fn start(operation_kind: UpstreamOperationKind) -> Self {
Self::start_with_exemplar(operation_kind, None)
}
pub fn start_with_exemplar(
operation_kind: UpstreamOperationKind,
exemplar: Option<ExemplarTraceId>,
) -> Self {
let active = bound_family("crank_upstream_requests_total").is_some();
Self {
operation_kind,
started_at: active.then(Instant::now),
exemplar,
active,
}
}
pub fn complete(mut self, outcome: UpstreamOutcome) {
self.finish(outcome);
}
fn finish(&mut self, outcome: UpstreamOutcome) {
if let Some(started_at) = self.started_at.take() {
record_upstream_request_with_exemplar(
self.operation_kind,
outcome,
started_at.elapsed(),
self.exemplar,
);
}
}
}
impl Drop for UpstreamRequestMetrics {
fn drop(&mut self) {
if self.active {
self.finish(UpstreamOutcome::Aborted);
}
}
}
pub fn record_limit_rejection(stage: LimitStage) {
if bound_family("crank_runtime_limit_rejections_total").is_none() {
return;
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => stage.as_str()
)
.increment(1);
}
pub fn record_cache_outcome(outcome: CacheOutcome) {
if bound_family("crank_runtime_cache_total").is_none() {
return;
}
metrics::counter!(
"crank_runtime_cache_total",
"outcome" => outcome.as_str()
)
.increment(1);
}
pub fn record_idempotency_outcome(outcome: IdempotencyOutcome) {
if bound_family("crank_idempotency_total").is_none() {
return;
}
metrics::counter!(
"crank_idempotency_total",
"outcome" => outcome.as_str()
)
.increment(1);
}
pub fn record_confirmation_outcome(outcome: ConfirmationOutcome) {
if bound_family("crank_confirmation_total").is_none() {
return;
}
metrics::counter!(
"crank_confirmation_total",
"outcome" => outcome.as_str()
)
.increment(1);
}
pub fn set_db_pool_connections(state: DbPoolState, count: u32) {
if bound_family("crank_db_pool_connections").is_none() {
return;
}
metrics::gauge!(
"crank_db_pool_connections",
"state" => state.as_str()
)
.set(f64::from(count));
}
pub fn set_catalog(tool_count: usize, estimated_context_tokens: usize, warnings: usize) {
if bound_family("crank_catalog_tools").is_none() {
return;
}
metrics::gauge!("crank_catalog_tools").set(tool_count as f64);
metrics::gauge!("crank_catalog_estimated_context_tokens").set(estimated_context_tokens as f64);
metrics::gauge!("crank_catalog_warnings").set(warnings as f64);
}
pub fn record_invocation_history_lost() {
if bound_family("crank_invocation_history_lost_total").is_none() {
return;
}
metrics::counter!("crank_invocation_history_lost_total").increment(1);
}
pub fn record_export_failure(signal: SignalType, exporter: Exporter) {
if bound_family("crank_telemetry_export_failures_total").is_none() {
return;
}
metrics::counter!(
"crank_telemetry_export_failures_total",
"signal_type" => signal.as_str(),
"exporter" => exporter.as_str()
)
.increment(1);
}
/// Gauges are initialised only for the bound service; any other service (or an
/// unbound process) is a no-op and never registers a handle.
pub fn initialize_gauges(service: MetricService) {
if bound_metric_service() != Some(service) {
return;
}
if family_applies(service, "crank_http_inflight") {
metrics::gauge!("crank_http_inflight").set(0.0);
}
if family_applies(service, "crank_runtime_inflight") {
metrics::gauge!("crank_runtime_inflight").set(0.0);
}
set_db_pool_connections(DbPoolState::Idle, 0);
set_db_pool_connections(DbPoolState::Used, 0);
if service == MetricService::McpServer {
set_mcp_active_sessions(0);
set_mcp_session_metrics_fresh(false);
if family_applies(service, "crank_mcp_active_streams") {
metrics::gauge!("crank_mcp_active_streams").set(0.0);
}
set_catalog(0, 0, 0);
}
}
pub struct InFlightGuard {
gauge: Option<Gauge>,
}
impl InFlightGuard {
pub fn http() -> Self {
Self::increment("crank_http_inflight")
}
pub fn runtime() -> Self {
Self::increment("crank_runtime_inflight")
}
pub fn mcp_stream() -> Self {
Self::increment("crank_mcp_active_streams")
}
fn increment(name: &'static str) -> Self {
if bound_family(name).is_none() {
return Self { gauge: None };
}
let gauge = match name {
"crank_http_inflight" => metrics::gauge!("crank_http_inflight"),
"crank_runtime_inflight" => metrics::gauge!("crank_runtime_inflight"),
_ => metrics::gauge!("crank_mcp_active_streams"),
};
gauge.increment(1.0);
Self { gauge: Some(gauge) }
}
}
impl Drop for InFlightGuard {
fn drop(&mut self) {
if let Some(gauge) = &self.gauge {
gauge.decrement(1.0);
}
}
}