CI / Rust Checks (pull_request) Failing after 13m41s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
Community Release / release (push) Failing after 13m33s
Recovery delivery of the accumulated, TL-accepted Epic 3 work that had never been
committed to Git (stories 3.6-3.17, see the crank board). Story 3.18 is not accepted
yet and is deliberately excluded: its logging surface, its capability registration,
its oracle projections and its tests are not part of this tree.
Accepted scope
- 3.6 typed static Resource Drafts; 3.7 publication and exact binding of static
Resource Versions; 3.8 MCP discovery/read and revocation of static Resources;
- 3.9 Resource import/export in API and UI plus TLS dependency maintenance;
3.10 REST-backed Resource templates; 3.11 shared REST read runtime and the MCP
revision/capability foundation;
- 3.12 parameterised Prompt Drafts with preview; 3.13 Prompt publication and exact
version binding to Agents; 3.14 MCP prompts; 3.15 Prompt import/export;
- 3.16 composite Prompts with published Resources (dependency content scan plus
dynamic read through the accepted 3.11 runtime, admin preview and MCP prompts/get);
- 3.17 bounded MCP completion suggestions from typed published schemas and the
resources/list admission fix for bound template publications.
Accepted evidence being reused (unchanged code paths)
- 3.16 candidate: core 7, admin-api 22, registry 4, mcp-server 4, UI 6; one final
`just verify` run 1651 passed / 0 failed / 2 ignored.
- 3.17 candidate: full run 1672 passed / 0 failed / 2 ignored, QA targeted reruns and
a real MCP Inspector 2.12.0 client smoke.
Delivery engineering in this commit
- CI: one event per change (feature branches are gated by their pull request, push
runs only on main), story events run the changed packages, the full workspace suite
stays the feature/release gate; no cache/sharding without a measured runner check.
- Release: the packaged archives, checksums and SBOM are now uploaded to the Gitea
release entry of the tag instead of dying with the runner workspace.
Not included on purpose: Story 3.18 work, build caches, runtime databases, local
credentials and scratch evidence.
Refs: crank kanban t_e39d7263; baseline 1b6db09bd1.
494 lines
14 KiB
Rust
494 lines
14 KiB
Rust
use std::sync::OnceLock;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use metrics::Gauge;
|
|
|
|
use crate::schema::{MetricDefinition, metric_schema};
|
|
use crate::{
|
|
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
|
|
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
|
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
|
|
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
|
};
|
|
|
|
static BINDING: OnceLock<MetricService> = OnceLock::new();
|
|
|
|
/// Publishes the closed process identity exactly once, before the recorder,
|
|
/// gauges, producers and any listener exist. Re-binding the same service is
|
|
/// idempotent; a different service is a typed conflict and never wins twice.
|
|
pub fn bind_metric_service(service: MetricService) -> Result<(), MetricBindingError> {
|
|
crate::schema::validate_all_projections().map_err(|_| MetricBindingError::InvalidSchema)?;
|
|
// `OnceLock::set` returns `Err(value)` (the value passed in, not the stored
|
|
// one), so the idempotency check must read the published binding back.
|
|
if BINDING.set(service).is_ok() {
|
|
return Ok(());
|
|
}
|
|
match BINDING.get() {
|
|
Some(existing) if *existing == service => Ok(()),
|
|
_ => Err(MetricBindingError::ConflictingService),
|
|
}
|
|
}
|
|
|
|
pub fn bound_metric_service() -> Option<MetricService> {
|
|
BINDING.get().copied()
|
|
}
|
|
|
|
/// Normalises one route against the bound service: own and shared routes are
|
|
/// preserved, a known route owned by another service degrades to `unmatched`.
|
|
pub fn normalize_http_route(service: MetricService, route: HttpRoute) -> HttpRoute {
|
|
match crate::http_route_processes(route.as_str()) {
|
|
Some(owners) if owners.contains(&service.process()) => route,
|
|
_ => HttpRoute::unmatched(),
|
|
}
|
|
}
|
|
|
|
pub(crate) fn family_applies(service: MetricService, name: &'static str) -> bool {
|
|
metric_schema()
|
|
.iter()
|
|
.find(|definition| definition.name == name)
|
|
.is_some_and(|definition: &MetricDefinition| {
|
|
definition.processes.contains(&service.process())
|
|
})
|
|
}
|
|
|
|
/// Unbound facades (plain unit tests or libraries without observability) are a
|
|
/// safe no-op; the process never registers a metric handle or exemplar entry.
|
|
fn bound_family(name: &'static str) -> Option<MetricService> {
|
|
let service = BINDING.get().copied()?;
|
|
family_applies(service, name).then_some(service)
|
|
}
|
|
|
|
pub fn record_http_request(
|
|
route: HttpRoute,
|
|
method: HttpMethod,
|
|
status: HttpStatusClass,
|
|
duration: Duration,
|
|
) {
|
|
record_http_request_with_exemplar(route, method, status, duration, None);
|
|
}
|
|
|
|
pub fn record_http_request_with_exemplar(
|
|
route: HttpRoute,
|
|
method: HttpMethod,
|
|
status: HttpStatusClass,
|
|
duration: Duration,
|
|
exemplar: Option<ExemplarTraceId>,
|
|
) {
|
|
let Some(service) = bound_family("crank_http_requests_total") else {
|
|
return;
|
|
};
|
|
let route = normalize_http_route(service, route);
|
|
metrics::counter!(
|
|
"crank_http_requests_total",
|
|
"route" => route.as_str(),
|
|
"method" => method.as_str(),
|
|
"status_class" => status.as_str()
|
|
)
|
|
.increment(1);
|
|
metrics::histogram!(
|
|
"crank_http_request_duration_seconds",
|
|
"route" => route.as_str(),
|
|
"method" => method.as_str()
|
|
)
|
|
.record(duration.as_secs_f64());
|
|
crate::exemplar::record_exemplar(
|
|
service,
|
|
"crank_http_request_duration_seconds",
|
|
vec![("route", route.as_str()), ("method", method.as_str())],
|
|
duration.as_secs_f64(),
|
|
exemplar,
|
|
);
|
|
}
|
|
|
|
pub fn record_mcp_request(
|
|
method: McpMethod,
|
|
response_mode: McpResponseMode,
|
|
outcome: McpOutcome,
|
|
duration: Duration,
|
|
) {
|
|
record_mcp_request_with_exemplar(method, response_mode, outcome, duration, None);
|
|
}
|
|
|
|
pub fn record_mcp_request_with_exemplar(
|
|
method: McpMethod,
|
|
response_mode: McpResponseMode,
|
|
outcome: McpOutcome,
|
|
duration: Duration,
|
|
exemplar: Option<ExemplarTraceId>,
|
|
) {
|
|
let Some(service) = bound_family("crank_mcp_requests_total") else {
|
|
return;
|
|
};
|
|
metrics::counter!(
|
|
"crank_mcp_requests_total",
|
|
"method" => method.as_str(),
|
|
"response_mode" => response_mode.as_str(),
|
|
"outcome" => outcome.as_str()
|
|
)
|
|
.increment(1);
|
|
metrics::histogram!(
|
|
"crank_mcp_request_duration_seconds",
|
|
"method" => method.as_str(),
|
|
"outcome" => outcome.as_str()
|
|
)
|
|
.record(duration.as_secs_f64());
|
|
crate::exemplar::record_exemplar(
|
|
service,
|
|
"crank_mcp_request_duration_seconds",
|
|
vec![("method", method.as_str()), ("outcome", outcome.as_str())],
|
|
duration.as_secs_f64(),
|
|
exemplar,
|
|
);
|
|
}
|
|
|
|
pub fn set_mcp_active_sessions(count: u64) {
|
|
if bound_family("crank_mcp_active_sessions").is_none() {
|
|
return;
|
|
}
|
|
metrics::gauge!("crank_mcp_active_sessions").set(count as f64);
|
|
}
|
|
|
|
pub fn set_mcp_session_metrics_fresh(fresh: bool) {
|
|
if bound_family("crank_mcp_session_metrics_fresh").is_none() {
|
|
return;
|
|
}
|
|
metrics::gauge!("crank_mcp_session_metrics_fresh").set(if fresh { 1.0 } else { 0.0 });
|
|
}
|
|
|
|
pub fn record_tool_invocation(
|
|
source: InvocationSource,
|
|
outcome: ToolOutcome,
|
|
error_kind: ToolErrorKind,
|
|
duration: Duration,
|
|
) {
|
|
record_tool_invocation_with_exemplar(source, outcome, error_kind, duration, None);
|
|
}
|
|
|
|
pub fn record_tool_invocation_with_exemplar(
|
|
source: InvocationSource,
|
|
outcome: ToolOutcome,
|
|
error_kind: ToolErrorKind,
|
|
duration: Duration,
|
|
exemplar: Option<ExemplarTraceId>,
|
|
) {
|
|
let Some(service) = bound_family("crank_tool_invocations_total") else {
|
|
return;
|
|
};
|
|
let error_kind = match outcome {
|
|
ToolOutcome::Success => ToolErrorKind::None,
|
|
ToolOutcome::Aborted => ToolErrorKind::Aborted,
|
|
ToolOutcome::Error
|
|
if matches!(error_kind, ToolErrorKind::None | ToolErrorKind::Aborted) =>
|
|
{
|
|
ToolErrorKind::ProtocolAdapter
|
|
}
|
|
ToolOutcome::Error => error_kind,
|
|
};
|
|
metrics::counter!(
|
|
"crank_tool_invocations_total",
|
|
"source" => source.as_str(),
|
|
"outcome" => outcome.as_str(),
|
|
"error_kind" => error_kind.as_str()
|
|
)
|
|
.increment(1);
|
|
metrics::histogram!(
|
|
"crank_tool_invocation_duration_seconds",
|
|
"source" => source.as_str(),
|
|
"outcome" => outcome.as_str()
|
|
)
|
|
.record(duration.as_secs_f64());
|
|
crate::exemplar::record_exemplar(
|
|
service,
|
|
"crank_tool_invocation_duration_seconds",
|
|
vec![("source", source.as_str()), ("outcome", outcome.as_str())],
|
|
duration.as_secs_f64(),
|
|
exemplar,
|
|
);
|
|
}
|
|
|
|
pub fn record_upstream_request(
|
|
operation_kind: UpstreamOperationKind,
|
|
outcome: UpstreamOutcome,
|
|
duration: Duration,
|
|
) {
|
|
record_upstream_request_with_exemplar(operation_kind, outcome, duration, None);
|
|
}
|
|
|
|
pub fn record_upstream_request_with_exemplar(
|
|
operation_kind: UpstreamOperationKind,
|
|
outcome: UpstreamOutcome,
|
|
duration: Duration,
|
|
exemplar: Option<ExemplarTraceId>,
|
|
) {
|
|
let Some(service) = bound_family("crank_upstream_requests_total") else {
|
|
return;
|
|
};
|
|
metrics::counter!(
|
|
"crank_upstream_requests_total",
|
|
"operation_kind" => operation_kind.as_str(),
|
|
"outcome" => outcome.as_str()
|
|
)
|
|
.increment(1);
|
|
metrics::histogram!(
|
|
"crank_upstream_request_duration_seconds",
|
|
"operation_kind" => operation_kind.as_str(),
|
|
"outcome" => outcome.as_str()
|
|
)
|
|
.record(duration.as_secs_f64());
|
|
crate::exemplar::record_exemplar(
|
|
service,
|
|
"crank_upstream_request_duration_seconds",
|
|
vec![
|
|
("operation_kind", operation_kind.as_str()),
|
|
("outcome", outcome.as_str()),
|
|
],
|
|
duration.as_secs_f64(),
|
|
exemplar,
|
|
);
|
|
}
|
|
|
|
pub struct ToolInvocationMetrics {
|
|
source: InvocationSource,
|
|
started_at: Option<Instant>,
|
|
exemplar: Option<ExemplarTraceId>,
|
|
active: bool,
|
|
}
|
|
|
|
impl ToolInvocationMetrics {
|
|
pub fn start(source: InvocationSource) -> Self {
|
|
Self::start_with_exemplar(source, None)
|
|
}
|
|
|
|
pub fn start_with_exemplar(
|
|
source: InvocationSource,
|
|
exemplar: Option<ExemplarTraceId>,
|
|
) -> Self {
|
|
let active = bound_family("crank_tool_invocations_total").is_some();
|
|
Self {
|
|
source,
|
|
started_at: active.then(Instant::now),
|
|
exemplar,
|
|
active,
|
|
}
|
|
}
|
|
|
|
pub fn complete(mut self, outcome: ToolOutcome, error_kind: ToolErrorKind) {
|
|
self.finish(outcome, error_kind);
|
|
}
|
|
|
|
fn finish(&mut self, outcome: ToolOutcome, error_kind: ToolErrorKind) {
|
|
if let Some(started_at) = self.started_at.take() {
|
|
record_tool_invocation_with_exemplar(
|
|
self.source,
|
|
outcome,
|
|
error_kind,
|
|
started_at.elapsed(),
|
|
self.exemplar,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Drop for ToolInvocationMetrics {
|
|
fn drop(&mut self) {
|
|
if self.active {
|
|
self.finish(ToolOutcome::Aborted, ToolErrorKind::Aborted);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub struct UpstreamRequestMetrics {
|
|
operation_kind: UpstreamOperationKind,
|
|
started_at: Option<Instant>,
|
|
exemplar: Option<ExemplarTraceId>,
|
|
active: bool,
|
|
}
|
|
|
|
impl UpstreamRequestMetrics {
|
|
pub fn start(operation_kind: UpstreamOperationKind) -> Self {
|
|
Self::start_with_exemplar(operation_kind, None)
|
|
}
|
|
|
|
pub fn start_with_exemplar(
|
|
operation_kind: UpstreamOperationKind,
|
|
exemplar: Option<ExemplarTraceId>,
|
|
) -> Self {
|
|
let active = bound_family("crank_upstream_requests_total").is_some();
|
|
Self {
|
|
operation_kind,
|
|
started_at: active.then(Instant::now),
|
|
exemplar,
|
|
active,
|
|
}
|
|
}
|
|
|
|
pub fn complete(mut self, outcome: UpstreamOutcome) {
|
|
self.finish(outcome);
|
|
}
|
|
|
|
fn finish(&mut self, outcome: UpstreamOutcome) {
|
|
if let Some(started_at) = self.started_at.take() {
|
|
record_upstream_request_with_exemplar(
|
|
self.operation_kind,
|
|
outcome,
|
|
started_at.elapsed(),
|
|
self.exemplar,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Drop for UpstreamRequestMetrics {
|
|
fn drop(&mut self) {
|
|
if self.active {
|
|
self.finish(UpstreamOutcome::Aborted);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn record_limit_rejection(stage: LimitStage) {
|
|
if bound_family("crank_runtime_limit_rejections_total").is_none() {
|
|
return;
|
|
}
|
|
metrics::counter!(
|
|
"crank_runtime_limit_rejections_total",
|
|
"stage" => stage.as_str()
|
|
)
|
|
.increment(1);
|
|
}
|
|
|
|
pub fn record_cache_outcome(outcome: CacheOutcome) {
|
|
if bound_family("crank_runtime_cache_total").is_none() {
|
|
return;
|
|
}
|
|
metrics::counter!(
|
|
"crank_runtime_cache_total",
|
|
"outcome" => outcome.as_str()
|
|
)
|
|
.increment(1);
|
|
}
|
|
|
|
pub fn record_idempotency_outcome(outcome: IdempotencyOutcome) {
|
|
if bound_family("crank_idempotency_total").is_none() {
|
|
return;
|
|
}
|
|
metrics::counter!(
|
|
"crank_idempotency_total",
|
|
"outcome" => outcome.as_str()
|
|
)
|
|
.increment(1);
|
|
}
|
|
|
|
pub fn record_confirmation_outcome(outcome: ConfirmationOutcome) {
|
|
if bound_family("crank_confirmation_total").is_none() {
|
|
return;
|
|
}
|
|
metrics::counter!(
|
|
"crank_confirmation_total",
|
|
"outcome" => outcome.as_str()
|
|
)
|
|
.increment(1);
|
|
}
|
|
|
|
pub fn set_db_pool_connections(state: DbPoolState, count: u32) {
|
|
if bound_family("crank_db_pool_connections").is_none() {
|
|
return;
|
|
}
|
|
metrics::gauge!(
|
|
"crank_db_pool_connections",
|
|
"state" => state.as_str()
|
|
)
|
|
.set(f64::from(count));
|
|
}
|
|
|
|
pub fn set_catalog(tool_count: usize, estimated_context_tokens: usize, warnings: usize) {
|
|
if bound_family("crank_catalog_tools").is_none() {
|
|
return;
|
|
}
|
|
metrics::gauge!("crank_catalog_tools").set(tool_count as f64);
|
|
metrics::gauge!("crank_catalog_estimated_context_tokens").set(estimated_context_tokens as f64);
|
|
metrics::gauge!("crank_catalog_warnings").set(warnings as f64);
|
|
}
|
|
|
|
pub fn record_invocation_history_lost() {
|
|
if bound_family("crank_invocation_history_lost_total").is_none() {
|
|
return;
|
|
}
|
|
metrics::counter!("crank_invocation_history_lost_total").increment(1);
|
|
}
|
|
|
|
pub fn record_export_failure(signal: SignalType, exporter: Exporter) {
|
|
if bound_family("crank_telemetry_export_failures_total").is_none() {
|
|
return;
|
|
}
|
|
metrics::counter!(
|
|
"crank_telemetry_export_failures_total",
|
|
"signal_type" => signal.as_str(),
|
|
"exporter" => exporter.as_str()
|
|
)
|
|
.increment(1);
|
|
}
|
|
|
|
/// Gauges are initialised only for the bound service; any other service (or an
|
|
/// unbound process) is a no-op and never registers a handle.
|
|
pub fn initialize_gauges(service: MetricService) {
|
|
if bound_metric_service() != Some(service) {
|
|
return;
|
|
}
|
|
if family_applies(service, "crank_http_inflight") {
|
|
metrics::gauge!("crank_http_inflight").set(0.0);
|
|
}
|
|
if family_applies(service, "crank_runtime_inflight") {
|
|
metrics::gauge!("crank_runtime_inflight").set(0.0);
|
|
}
|
|
set_db_pool_connections(DbPoolState::Idle, 0);
|
|
set_db_pool_connections(DbPoolState::Used, 0);
|
|
if service == MetricService::McpServer {
|
|
set_mcp_active_sessions(0);
|
|
set_mcp_session_metrics_fresh(false);
|
|
if family_applies(service, "crank_mcp_active_streams") {
|
|
metrics::gauge!("crank_mcp_active_streams").set(0.0);
|
|
}
|
|
set_catalog(0, 0, 0);
|
|
}
|
|
}
|
|
|
|
pub struct InFlightGuard {
|
|
gauge: Option<Gauge>,
|
|
}
|
|
|
|
impl InFlightGuard {
|
|
pub fn http() -> Self {
|
|
Self::increment("crank_http_inflight")
|
|
}
|
|
|
|
pub fn runtime() -> Self {
|
|
Self::increment("crank_runtime_inflight")
|
|
}
|
|
|
|
pub fn mcp_stream() -> Self {
|
|
Self::increment("crank_mcp_active_streams")
|
|
}
|
|
|
|
fn increment(name: &'static str) -> Self {
|
|
if bound_family(name).is_none() {
|
|
return Self { gauge: None };
|
|
}
|
|
let gauge = match name {
|
|
"crank_http_inflight" => metrics::gauge!("crank_http_inflight"),
|
|
"crank_runtime_inflight" => metrics::gauge!("crank_runtime_inflight"),
|
|
_ => metrics::gauge!("crank_mcp_active_streams"),
|
|
};
|
|
gauge.increment(1.0);
|
|
Self { gauge: Some(gauge) }
|
|
}
|
|
}
|
|
|
|
impl Drop for InFlightGuard {
|
|
fn drop(&mut self) {
|
|
if let Some(gauge) = &self.gauge {
|
|
gauge.decrement(1.0);
|
|
}
|
|
}
|
|
}
|