Files
crank/apps/mcp-server/tests/integration/static_resources/discovery_diagnostic.rs
T
bsodfather 9567bbf3aa
CI / Rust Checks (pull_request) Canceled after 38m52s
CI / UI Checks (pull_request) Canceled after 0s
CI / Frontend E2E (pull_request) Canceled after 0s
CI / Community Image Smoke (pull_request) Canceled after 0s
CI / Deploy (pull_request) Canceled after 0s
Community Release / release (push) Canceled after 38m39s
fix(ci): pass the community scope guard and publish release artifacts before registry steps
- the community scope guard rejected the innocuous English comment 'One timed
  resources/list' in the Story 3.8 discovery diagnostic (its 'one-time-token'
  marker matches any 'one time' phrase). Reword the comment; no behaviour change.
- move the release artifact publication ahead of the registry/OpenBao steps, so a
  credential problem cannot leave the archives, checksums and SBOM unpublished.
2026-10-25 09:27:55 +00:00

618 lines
24 KiB
Rust

//! Story 3.8 · NFR-P4 diagnostics: real `resources/list` latency for both supported MCP revisions.
//!
//! This is an **opt-in local diagnostic**, not a default-suite gate and not Reference evidence.
//! It drives the same real loopback HTTP endpoint, real PostgreSQL registry and real artifact store
//! the accepted AT-3.8-01..05 suite uses (`standard_endpoint`), measures `resources/list` until the
//! complete response body is consumed on a monotonic clock, validates every wire result against the
//! exact bound Resource snapshot, and writes raw samples plus nearest-rank percentiles under
//! `W/.tmp/t_b5624309` (override with `CRANK_DISCOVERY_DIAGNOSTIC_DIR`).
//!
//! Run explicitly:
//!
//! ```text
//! SQLX_OFFLINE=true CARGO_BUILD_JOBS=1 CARGO_INCREMENTAL=0 \
//! cargo test -p mcp-server --test integration \
//! integration::static_resources::discovery_diagnostic -- --ignored --nocapture --test-threads=1
//! ```
//!
//! It never asserts a latency threshold: the accepted ≤150 ms p95 NFR-P4 limit is reported as a
//! comparison only, because the dataset here is the bounded existing fixture (see `dataset` in the
//! summary), not the PRD Reference dataset, and the host is a shared 4 GiB test server. It does
//! assert protocol correctness (exactly the bound snapshot, correct per-revision result shape) so a
//! broken wire response can never be reported as a latency sample.
use std::{
fs,
path::PathBuf,
time::{Duration, Instant},
};
use reqwest::header;
use serde_json::{Value, json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use super::super::common::initialize_session;
use super::fixtures::*;
/// Hard run bounds agreed for this bounded diagnostic.
const WARMUP_REQUESTS_PER_REVISION: usize = 20;
const MEASURED_REQUESTS_PER_REVISION: usize = 200;
/// At most 5 MCP requests/second with one request in flight: 250 ms between request starts.
const MIN_REQUEST_INTERVAL: Duration = Duration::from_millis(250);
const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
const NFR_P4_P95_LIMIT_MS: f64 = 150.0;
const JSONRPC_LIST_ID: u64 = 41;
#[derive(Clone, Copy, PartialEq, Eq)]
enum Phase {
Warmup,
Measured,
}
impl Phase {
fn as_str(self) -> &'static str {
match self {
Phase::Warmup => "warmup",
Phase::Measured => "measured",
}
}
}
/// One HTTP attempt: latency to the fully consumed body, plus whatever failed.
struct WireCall {
latency_ms: f64,
http_status: Option<u16>,
body_bytes: Option<usize>,
body: Option<String>,
transport_error: Option<String>,
}
impl WireCall {
fn failure(&self, outcome: &'static str, detail: impl Into<String>) -> Sample {
Sample {
revision: "",
phase: Phase::Measured,
index: 0,
latency_ms: self.latency_ms,
http_status: self.http_status,
body_bytes: self.body_bytes,
outcome,
detail: detail.into(),
}
}
}
struct Sample {
revision: &'static str,
phase: Phase,
index: usize,
latency_ms: f64,
http_status: Option<u16>,
body_bytes: Option<usize>,
outcome: &'static str,
detail: String,
}
impl Sample {
fn as_json(&self) -> Value {
json!({
"revision": self.revision,
"phase": self.phase.as_str(),
"index": self.index,
"latency_ms": self.latency_ms,
"http_status": self.http_status,
"body_bytes": self.body_bytes,
"outcome": self.outcome,
"detail": self.detail,
})
}
}
fn now_rfc3339() -> String {
OffsetDateTime::now_utc()
.format(&Rfc3339)
.unwrap_or_else(|error| format!("unformatted-utc ({error})"))
}
/// A single timed `resources/list` call, built directly on `reqwest` so transport/timeout/HTTP
/// failures are recorded as outcomes instead of aborting the run.
async fn timed_list(
endpoint: &ResourceEndpoint,
revision: &'static str,
session: Option<&str>,
) -> WireCall {
let payload = if revision == LEGACY_REVISION {
json!({
"jsonrpc": "2.0",
"id": JSONRPC_LIST_ID,
"method": "resources/list",
"params": {}
})
} else {
let mut params = json!({});
params["_meta"] = modern_meta(revision);
json!({
"jsonrpc": "2.0",
"id": JSONRPC_LIST_ID,
"method": "resources/list",
"params": params
})
};
let mut request = endpoint
.client
.post(&endpoint.mcp_url)
.header(header::ACCEPT, "application/json, text/event-stream")
.header(
header::AUTHORIZATION,
format!("Bearer {}", endpoint.api_key),
);
if revision == LEGACY_REVISION {
request = request
.header("MCP-Protocol-Version", LEGACY_REVISION)
.header("MCP-Session-Id", session.unwrap_or_default());
} else {
request = request
.header("MCP-Protocol-Version", revision)
.header("Mcp-Method", "resources/list");
}
let started = Instant::now();
let response = match request.json(&payload).send().await {
Ok(response) => response,
Err(error) => {
return WireCall {
latency_ms: started.elapsed().as_secs_f64() * 1000.0,
http_status: None,
body_bytes: None,
body: None,
transport_error: Some(error.to_string()),
};
}
};
let status = response.status().as_u16();
match response.text().await {
Ok(body) => {
let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
let body_bytes = body.len();
WireCall {
latency_ms,
http_status: Some(status),
body_bytes: Some(body_bytes),
body: Some(body),
transport_error: None,
}
}
Err(error) => {
let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
WireCall {
latency_ms,
http_status: Some(status),
body_bytes: None,
body: None,
transport_error: Some(format!("reading the response body failed: {error}")),
}
}
}
}
/// Validate one wire result against the exact bound snapshot and the pinned per-revision shape.
/// Returns the number of listed Resources on success.
fn validate_list_body(
body: &str,
revision: &'static str,
endpoint: &ResourceEndpoint,
) -> Result<usize, String> {
let parsed: Value = serde_json::from_str(body)
.map_err(|error| format!("response body is not JSON ({error}): {body:.400}"))?;
let result = parsed
.get("result")
.ok_or_else(|| format!("no JSON-RPC result: {body:.400}"))?;
let resources = result
.get("resources")
.and_then(Value::as_array)
.ok_or_else(|| format!("result carries no resources array: {result}"))?;
if revision == LEGACY_REVISION {
for field in ["resultType", "ttlMs", "cacheScope"] {
if result.get(field).is_some() {
return Err(format!("legacy result must not carry {field}: {result}"));
}
}
} else if result.get("resultType").and_then(Value::as_str) != Some("complete") {
return Err(format!(
"modern result must carry resultType=complete: {result}"
));
}
if result.get("nextCursor").is_some() {
return Err(format!(
"a 3-Resource snapshot must not advertise a continuation: {result}"
));
}
let expected: Vec<(&str, &str)> = [&endpoint.text, &endpoint.json, &endpoint.binary]
.into_iter()
.map(|bound| (bound.uri.as_str(), bound.name.as_str()))
.collect();
if resources.len() != expected.len() {
return Err(format!(
"expected exactly {} bound Resources, got {}: {result}",
expected.len(),
resources.len()
));
}
for resource in resources {
let uri = resource
.get("uri")
.and_then(Value::as_str)
.ok_or_else(|| format!("listed Resource without uri: {resource}"))?;
let Some((_, name)) = expected
.iter()
.find(|(expected_uri, _)| *expected_uri == uri)
else {
return Err(format!("unexpected listed Resource {uri}: {result}"));
};
if resource.get("name").and_then(Value::as_str) != Some(*name) {
return Err(format!(
"listed Resource {uri} must carry the published name {name}: {resource}"
));
}
if resource
.get("mimeType")
.and_then(Value::as_str)
.is_none_or(|mime| mime.is_empty())
{
return Err(format!(
"listed Resource {uri} must carry a MIME type: {resource}"
));
}
}
Ok(resources.len())
}
fn evaluate(call: WireCall, revision: &'static str, endpoint: &ResourceEndpoint) -> Sample {
let Some(body) = call.body.as_deref() else {
return call.failure(
"transport-error",
call.transport_error.clone().unwrap_or_default(),
);
};
if call.http_status != Some(200) {
return call.failure(
"http-status",
format!("HTTP {}: {body:.400}", call.http_status.unwrap_or(0)),
);
}
match validate_list_body(body, revision, endpoint) {
Ok(_) => Sample {
revision,
phase: Phase::Measured,
index: 0,
latency_ms: call.latency_ms,
http_status: call.http_status,
body_bytes: call.body_bytes,
outcome: "ok",
detail: String::new(),
},
Err(detail) => call.failure("wire-invalid", detail),
}
}
fn percentile_nearest_rank(sorted: &[f64], fraction: f64) -> f64 {
if sorted.is_empty() {
return f64::NAN;
}
let rank = (fraction * sorted.len() as f64).ceil() as usize;
sorted[rank.saturating_sub(1).min(sorted.len() - 1)]
}
fn summarize(
revision: &'static str,
samples: &[Sample],
wall: Duration,
init_ms: Option<f64>,
) -> Value {
let mut latencies: Vec<f64> = samples
.iter()
.filter(|sample| sample.outcome == "ok")
.map(|sample| sample.latency_ms)
.collect();
latencies.sort_by(f64::total_cmp);
let mut error_counts: std::collections::BTreeMap<&'static str, usize> =
std::collections::BTreeMap::new();
for sample in samples.iter().filter(|sample| sample.outcome != "ok") {
*error_counts.entry(sample.outcome).or_default() += 1;
}
let mean = if latencies.is_empty() {
f64::NAN
} else {
latencies.iter().sum::<f64>() / latencies.len() as f64
};
let p95 = percentile_nearest_rank(&latencies, 0.95);
let wall_seconds = wall.as_secs_f64();
json!({
"revision": revision,
"measured_requests": samples.len(),
"ok": latencies.len(),
"errors": error_counts,
"p50_ms": percentile_nearest_rank(&latencies, 0.50),
"p95_ms": p95,
"max_ms": latencies.last().copied().unwrap_or(f64::NAN),
"min_ms": latencies.first().copied().unwrap_or(f64::NAN),
"mean_ms": mean,
"wall_seconds": wall_seconds,
"achieved_requests_per_second": if wall_seconds > 0.0 {
samples.len() as f64 / wall_seconds
} else {
f64::NAN
},
"body_bytes_min": samples.iter().filter_map(|s| s.body_bytes).min(),
"body_bytes_max": samples.iter().filter_map(|s| s.body_bytes).max(),
"session_init_ms": init_ms,
"nfr_p4_p95_limit_ms": NFR_P4_P95_LIMIT_MS,
"p95_within_150ms_local_comparison_only": p95.is_finite() && p95 <= NFR_P4_P95_LIMIT_MS,
})
}
fn git(workdir: &str, args: &[&str]) -> String {
std::process::Command::new("git")
.args(args)
.current_dir(workdir)
.output()
.ok()
.filter(|output| output.status.success())
.and_then(|output| String::from_utf8(output.stdout).ok())
.map(|stdout| stdout.trim().to_owned())
.unwrap_or_default()
}
fn environment_snapshot() -> Value {
let loadavg = fs::read_to_string("/proc/loadavg").unwrap_or_default();
let meminfo = fs::read_to_string("/proc/meminfo").unwrap_or_default();
let mem_available_kib = meminfo
.lines()
.find(|line| line.starts_with("MemAvailable:"))
.and_then(|line| line.split_whitespace().nth(1))
.and_then(|value| value.parse::<u64>().ok());
json!({
"os": std::env::consts::OS,
"arch": std::env::consts::ARCH,
"logical_cpus": std::thread::available_parallelism().map(|n| n.get()).unwrap_or(0),
"loadavg": loadavg.trim(),
"mem_available_kib": mem_available_kib,
"build_profile": if cfg!(debug_assertions) { "debug" } else { "release" },
"clock": "std::time::Instant (monotonic)",
"shared_host_note": "shared 4 vCPU / 4 GiB test server; loadavg recorded at measurement start and end",
})
}
async fn db_counts(registry: &crank_registry::PostgresRegistry) -> Value {
let pool = registry.pool();
let mut counts = serde_json::Map::new();
for (label, sql) in [
("agents", "select count(*) from agents"),
("published_agents", "select count(*) from published_agents"),
("operations", "select count(*) from operations"),
("resources", "select count(*) from resources"),
(
"resource_publications",
"select count(*) from resource_publications",
),
(
"agent_resource_bindings",
"select count(*) from agent_resource_bindings",
),
] {
let value = sqlx::query_scalar::<_, i64>(sql)
.fetch_one(pool)
.await
.map_err(|error| format!("{label}: {error}"));
match value {
Ok(count) => {
counts.insert(label.to_owned(), json!(count));
}
Err(detail) => {
counts.insert(label.to_owned(), json!({ "error": detail }));
}
}
}
Value::Object(counts)
}
fn output_dir() -> PathBuf {
std::env::var_os("CRANK_DISCOVERY_DIAGNOSTIC_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.tmp/t_b5624309"))
}
/// Opt-in: `#[ignore]` keeps an unproven timing diagnostic out of the default suite. The console
/// `test result: ignored` line is the visible signal that this measurement did not run.
#[tokio::test]
#[ignore = "opt-in NFR-P4 discovery latency diagnostic (real HTTP + PostgreSQL), not a default-suite gate"]
async fn discovery_latency_diagnostic_p50_p95_both_revisions() {
let mut endpoint = standard_endpoint("nfr_p4_discovery").await;
// The accepted fixture resyncs the published catalog on every request (`refresh_interval` = 0),
// so each sample measures a cold-catalog discovery request: the production read path over HTTP.
endpoint.client = reqwest::Client::builder()
.timeout(REQUEST_TIMEOUT)
.build()
.expect("diagnostic HTTP client must build");
let revisions = supported_revisions();
for revision in [MODERN_REVISION, LEGACY_REVISION] {
assert!(
revisions.iter().any(|supported| supported == revision),
"the checked-in revision oracle must contain {revision}: {revisions:?}"
);
}
let worktree = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
let worktree = worktree.display().to_string();
let environment_start = environment_snapshot();
let counts = db_counts(&endpoint.registry).await;
let started_at = now_rfc3339();
let run_started = Instant::now();
let mut all_samples: Vec<Value> = Vec::new();
let mut summaries = serde_json::Map::new();
for revision in [MODERN_REVISION, LEGACY_REVISION] {
// Session setup is deliberately outside the timed list span: legacy only.
let (init_ms, session) = if revision == LEGACY_REVISION {
let init_started = Instant::now();
let session =
initialize_session(&endpoint.client, &endpoint.mcp_url, &endpoint.api_key).await;
(
Some(init_started.elapsed().as_secs_f64() * 1000.0),
Some(session),
)
} else {
(None, None)
};
let mut measured: Vec<Sample> = Vec::new();
let total = WARMUP_REQUESTS_PER_REVISION + MEASURED_REQUESTS_PER_REVISION;
let measured_started = Instant::now();
for index in 0..total {
let phase = if index < WARMUP_REQUESTS_PER_REVISION {
Phase::Warmup
} else {
Phase::Measured
};
let request_started = Instant::now();
let call = timed_list(&endpoint, revision, session.as_deref()).await;
let mut sample = evaluate(call, revision, &endpoint);
sample.revision = revision;
sample.phase = phase;
sample.index = index;
all_samples.push(sample.as_json());
if phase == Phase::Measured {
measured.push(sample);
}
let elapsed = request_started.elapsed();
if elapsed < MIN_REQUEST_INTERVAL {
tokio::time::sleep(MIN_REQUEST_INTERVAL - elapsed).await;
}
}
let wall = measured_started.elapsed();
let summary = summarize(revision, &measured, wall, init_ms);
println!(
"[discovery-diagnostic] revision={revision} samples={} ok={} errors={} p50={:.3}ms p95={:.3}ms max={:.3}ms wall={:.2}s",
summary["measured_requests"],
summary["ok"],
summary["errors"],
summary["p50_ms"].as_f64().unwrap_or(f64::NAN),
summary["p95_ms"].as_f64().unwrap_or(f64::NAN),
summary["max_ms"].as_f64().unwrap_or(f64::NAN),
summary["wall_seconds"].as_f64().unwrap_or(f64::NAN),
);
summaries.insert(revision.to_owned(), summary);
}
let out_dir = output_dir();
fs::create_dir_all(&out_dir).expect("diagnostic output directory must be creatable");
let raw_path = out_dir.join("discovery-samples.jsonl");
let mut raw = String::new();
for sample in &all_samples {
raw.push_str(&sample.to_string());
raw.push('\n');
}
fs::write(&raw_path, raw).expect("raw samples must be written");
let max_payload_bytes = [&endpoint.text, &endpoint.json, &endpoint.binary]
.into_iter()
.map(|bound| bound.bytes.len())
.max()
.unwrap_or(0);
let summary = json!({
"diagnostic": "story38_nfr_p4_resource_discovery_latency",
"generated_at_utc": now_rfc3339(),
"started_at_utc": started_at,
"candidate": {
"head": git(&worktree, &["rev-parse", "HEAD"]),
"branch": git(&worktree, &["rev-parse", "--abbrev-ref", "HEAD"]),
"worktree": worktree,
"dirty_tree": !git(&worktree, &["status", "--porcelain"]).is_empty(),
"note": "HEAD alone is insufficient in this dirty worktree; source identity is recorded in .tmp/t_b5624309/identity-*.json",
},
"scope": {
"measures": "MCP resources/list response latency over the real HTTP endpoint",
"does_not_measure": [
"resources/read latency",
"tools/call or platform execution overhead",
"mixed load, soak, burst or release Reference workload",
"representative real MCP client applications (none used; reqwest is not a client application)",
],
"nfr_p4_p95_limit_ms": NFR_P4_P95_LIMIT_MS,
"qualifies_nfr_p4": false,
"qualification_reason": "bounded existing fixture dataset on a shared 4 GiB host, single in-process test, not the PRD Reference Deployment dataset/load generator",
},
"method": {
"transport": "loopback HTTP (127.0.0.1, ephemeral port) served by the production axum router",
"registry": "real PostgreSQL 16 (testcontainers, own schema) with the real migration authority",
"artifact_store": "real immutable test-owned ArtifactStore root",
"revisions": [MODERN_REVISION, LEGACY_REVISION],
"warmup_requests_per_revision": WARMUP_REQUESTS_PER_REVISION,
"measured_requests_per_revision": MEASURED_REQUESTS_PER_REVISION,
"concurrency": 1,
"min_request_interval_ms": MIN_REQUEST_INTERVAL.as_millis(),
"request_timeout_ms": REQUEST_TIMEOUT.as_millis(),
"timed_span": "request start → complete response body consumed",
"percentile": "nearest-rank",
"warm_cold_policy": "process and fixtures warm after construction; first measured revision runs after its own 20-request warmup; catalog refresh interval is 0 ms, so every request reloads the published catalog from PostgreSQL (cold-catalog path)",
"session_policy": "legacy 2025-11-25: one initialize/initialized handshake before warmup, timed separately; modern 2026-07-28: stateless per request",
"error_policy": "transport, timeout, HTTP-status and wire-validation failures are counted per revision and excluded from percentiles; any error fails the test",
},
"dataset": {
"fixture": "static_resources::fixtures::standard_endpoint (accepted AT-3.8 suite)",
"bound_resources": 3,
"resource_kinds": ["text", "json", "binary"],
"max_resource_payload_bytes": max_payload_bytes,
"page": "no limit parameter: all 3 bound Resources in one page (server max page 100)",
"published_agents_in_request_scope": 1,
"published_operations_in_request_scope": 1,
"seeded_but_not_in_scope": "1 additional draft Agent, 1 published-but-unbound Resource, 1 draft Resource, 1 foreign-workspace Resource",
"db_row_counts": counts,
"dataset_gap": "PRD Reference dataset (200 Agents / 2 000 Operations, 32 static Resources of ≤4 KiB) is NOT seeded: the accepted fixture capabilities cannot seed it without a larger new harness, so this run measures the minimal existing set",
},
"environment": environment_start,
"environment_at_end": environment_snapshot(),
"run_wall_seconds": run_started.elapsed().as_secs_f64(),
"revisions": summaries,
"raw_samples_file": raw_path.display().to_string(),
"total_requests": all_samples.len(),
});
let summary_path = out_dir.join("discovery-summary.json");
fs::write(
&summary_path,
format!(
"{}\n",
serde_json::to_string_pretty(&summary).expect("summary serializes")
),
)
.expect("summary must be written");
println!("[discovery-diagnostic] summary: {}", summary_path.display());
for revision in [MODERN_REVISION, LEGACY_REVISION] {
let entry = &summaries[revision];
let errors = entry["errors"]
.as_object()
.map(|map| map.len())
.unwrap_or(0);
assert_eq!(
entry["measured_requests"].as_u64(),
Some(MEASURED_REQUESTS_PER_REVISION as u64),
"every revision must produce the planned measured sample count: {entry}"
);
assert_eq!(
errors, 0,
"a working local endpoint must not answer transport/HTTP/wire failures on {revision}: {entry}"
);
assert_eq!(
entry["ok"].as_u64(),
Some(MEASURED_REQUESTS_PER_REVISION as u64),
"every measured response must validate against the bound snapshot on {revision}: {entry}"
);
}
}