CI / Rust Checks (pull_request) Canceled after 38m52s
CI / UI Checks (pull_request) Canceled after 0s
CI / Frontend E2E (pull_request) Canceled after 0s
CI / Community Image Smoke (pull_request) Canceled after 0s
CI / Deploy (pull_request) Canceled after 0s
Community Release / release (push) Canceled after 38m39s
- the community scope guard rejected the innocuous English comment 'One timed resources/list' in the Story 3.8 discovery diagnostic (its 'one-time-token' marker matches any 'one time' phrase). Reword the comment; no behaviour change. - move the release artifact publication ahead of the registry/OpenBao steps, so a credential problem cannot leave the archives, checksums and SBOM unpublished.
618 lines
24 KiB
Rust
618 lines
24 KiB
Rust
//! Story 3.8 · NFR-P4 diagnostics: real `resources/list` latency for both supported MCP revisions.
|
|
//!
|
|
//! This is an **opt-in local diagnostic**, not a default-suite gate and not Reference evidence.
|
|
//! It drives the same real loopback HTTP endpoint, real PostgreSQL registry and real artifact store
|
|
//! the accepted AT-3.8-01..05 suite uses (`standard_endpoint`), measures `resources/list` until the
|
|
//! complete response body is consumed on a monotonic clock, validates every wire result against the
|
|
//! exact bound Resource snapshot, and writes raw samples plus nearest-rank percentiles under
|
|
//! `W/.tmp/t_b5624309` (override with `CRANK_DISCOVERY_DIAGNOSTIC_DIR`).
|
|
//!
|
|
//! Run explicitly:
|
|
//!
|
|
//! ```text
|
|
//! SQLX_OFFLINE=true CARGO_BUILD_JOBS=1 CARGO_INCREMENTAL=0 \
|
|
//! cargo test -p mcp-server --test integration \
|
|
//! integration::static_resources::discovery_diagnostic -- --ignored --nocapture --test-threads=1
|
|
//! ```
|
|
//!
|
|
//! It never asserts a latency threshold: the accepted ≤150 ms p95 NFR-P4 limit is reported as a
|
|
//! comparison only, because the dataset here is the bounded existing fixture (see `dataset` in the
|
|
//! summary), not the PRD Reference dataset, and the host is a shared 4 GiB test server. It does
|
|
//! assert protocol correctness (exactly the bound snapshot, correct per-revision result shape) so a
|
|
//! broken wire response can never be reported as a latency sample.
|
|
|
|
use std::{
|
|
fs,
|
|
path::PathBuf,
|
|
time::{Duration, Instant},
|
|
};
|
|
|
|
use reqwest::header;
|
|
use serde_json::{Value, json};
|
|
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
|
|
|
use super::super::common::initialize_session;
|
|
use super::fixtures::*;
|
|
|
|
/// Hard run bounds agreed for this bounded diagnostic.
|
|
const WARMUP_REQUESTS_PER_REVISION: usize = 20;
|
|
const MEASURED_REQUESTS_PER_REVISION: usize = 200;
|
|
/// At most 5 MCP requests/second with one request in flight: 250 ms between request starts.
|
|
const MIN_REQUEST_INTERVAL: Duration = Duration::from_millis(250);
|
|
const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
|
|
const NFR_P4_P95_LIMIT_MS: f64 = 150.0;
|
|
const JSONRPC_LIST_ID: u64 = 41;
|
|
|
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
|
enum Phase {
|
|
Warmup,
|
|
Measured,
|
|
}
|
|
|
|
impl Phase {
|
|
fn as_str(self) -> &'static str {
|
|
match self {
|
|
Phase::Warmup => "warmup",
|
|
Phase::Measured => "measured",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One HTTP attempt: latency to the fully consumed body, plus whatever failed.
|
|
struct WireCall {
|
|
latency_ms: f64,
|
|
http_status: Option<u16>,
|
|
body_bytes: Option<usize>,
|
|
body: Option<String>,
|
|
transport_error: Option<String>,
|
|
}
|
|
|
|
impl WireCall {
|
|
fn failure(&self, outcome: &'static str, detail: impl Into<String>) -> Sample {
|
|
Sample {
|
|
revision: "",
|
|
phase: Phase::Measured,
|
|
index: 0,
|
|
latency_ms: self.latency_ms,
|
|
http_status: self.http_status,
|
|
body_bytes: self.body_bytes,
|
|
outcome,
|
|
detail: detail.into(),
|
|
}
|
|
}
|
|
}
|
|
|
|
struct Sample {
|
|
revision: &'static str,
|
|
phase: Phase,
|
|
index: usize,
|
|
latency_ms: f64,
|
|
http_status: Option<u16>,
|
|
body_bytes: Option<usize>,
|
|
outcome: &'static str,
|
|
detail: String,
|
|
}
|
|
|
|
impl Sample {
|
|
fn as_json(&self) -> Value {
|
|
json!({
|
|
"revision": self.revision,
|
|
"phase": self.phase.as_str(),
|
|
"index": self.index,
|
|
"latency_ms": self.latency_ms,
|
|
"http_status": self.http_status,
|
|
"body_bytes": self.body_bytes,
|
|
"outcome": self.outcome,
|
|
"detail": self.detail,
|
|
})
|
|
}
|
|
}
|
|
|
|
fn now_rfc3339() -> String {
|
|
OffsetDateTime::now_utc()
|
|
.format(&Rfc3339)
|
|
.unwrap_or_else(|error| format!("unformatted-utc ({error})"))
|
|
}
|
|
|
|
/// A single timed `resources/list` call, built directly on `reqwest` so transport/timeout/HTTP
|
|
/// failures are recorded as outcomes instead of aborting the run.
|
|
async fn timed_list(
|
|
endpoint: &ResourceEndpoint,
|
|
revision: &'static str,
|
|
session: Option<&str>,
|
|
) -> WireCall {
|
|
let payload = if revision == LEGACY_REVISION {
|
|
json!({
|
|
"jsonrpc": "2.0",
|
|
"id": JSONRPC_LIST_ID,
|
|
"method": "resources/list",
|
|
"params": {}
|
|
})
|
|
} else {
|
|
let mut params = json!({});
|
|
params["_meta"] = modern_meta(revision);
|
|
json!({
|
|
"jsonrpc": "2.0",
|
|
"id": JSONRPC_LIST_ID,
|
|
"method": "resources/list",
|
|
"params": params
|
|
})
|
|
};
|
|
|
|
let mut request = endpoint
|
|
.client
|
|
.post(&endpoint.mcp_url)
|
|
.header(header::ACCEPT, "application/json, text/event-stream")
|
|
.header(
|
|
header::AUTHORIZATION,
|
|
format!("Bearer {}", endpoint.api_key),
|
|
);
|
|
if revision == LEGACY_REVISION {
|
|
request = request
|
|
.header("MCP-Protocol-Version", LEGACY_REVISION)
|
|
.header("MCP-Session-Id", session.unwrap_or_default());
|
|
} else {
|
|
request = request
|
|
.header("MCP-Protocol-Version", revision)
|
|
.header("Mcp-Method", "resources/list");
|
|
}
|
|
|
|
let started = Instant::now();
|
|
let response = match request.json(&payload).send().await {
|
|
Ok(response) => response,
|
|
Err(error) => {
|
|
return WireCall {
|
|
latency_ms: started.elapsed().as_secs_f64() * 1000.0,
|
|
http_status: None,
|
|
body_bytes: None,
|
|
body: None,
|
|
transport_error: Some(error.to_string()),
|
|
};
|
|
}
|
|
};
|
|
let status = response.status().as_u16();
|
|
match response.text().await {
|
|
Ok(body) => {
|
|
let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
|
|
let body_bytes = body.len();
|
|
WireCall {
|
|
latency_ms,
|
|
http_status: Some(status),
|
|
body_bytes: Some(body_bytes),
|
|
body: Some(body),
|
|
transport_error: None,
|
|
}
|
|
}
|
|
Err(error) => {
|
|
let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
|
|
WireCall {
|
|
latency_ms,
|
|
http_status: Some(status),
|
|
body_bytes: None,
|
|
body: None,
|
|
transport_error: Some(format!("reading the response body failed: {error}")),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Validate one wire result against the exact bound snapshot and the pinned per-revision shape.
|
|
/// Returns the number of listed Resources on success.
|
|
fn validate_list_body(
|
|
body: &str,
|
|
revision: &'static str,
|
|
endpoint: &ResourceEndpoint,
|
|
) -> Result<usize, String> {
|
|
let parsed: Value = serde_json::from_str(body)
|
|
.map_err(|error| format!("response body is not JSON ({error}): {body:.400}"))?;
|
|
let result = parsed
|
|
.get("result")
|
|
.ok_or_else(|| format!("no JSON-RPC result: {body:.400}"))?;
|
|
let resources = result
|
|
.get("resources")
|
|
.and_then(Value::as_array)
|
|
.ok_or_else(|| format!("result carries no resources array: {result}"))?;
|
|
|
|
if revision == LEGACY_REVISION {
|
|
for field in ["resultType", "ttlMs", "cacheScope"] {
|
|
if result.get(field).is_some() {
|
|
return Err(format!("legacy result must not carry {field}: {result}"));
|
|
}
|
|
}
|
|
} else if result.get("resultType").and_then(Value::as_str) != Some("complete") {
|
|
return Err(format!(
|
|
"modern result must carry resultType=complete: {result}"
|
|
));
|
|
}
|
|
if result.get("nextCursor").is_some() {
|
|
return Err(format!(
|
|
"a 3-Resource snapshot must not advertise a continuation: {result}"
|
|
));
|
|
}
|
|
|
|
let expected: Vec<(&str, &str)> = [&endpoint.text, &endpoint.json, &endpoint.binary]
|
|
.into_iter()
|
|
.map(|bound| (bound.uri.as_str(), bound.name.as_str()))
|
|
.collect();
|
|
if resources.len() != expected.len() {
|
|
return Err(format!(
|
|
"expected exactly {} bound Resources, got {}: {result}",
|
|
expected.len(),
|
|
resources.len()
|
|
));
|
|
}
|
|
for resource in resources {
|
|
let uri = resource
|
|
.get("uri")
|
|
.and_then(Value::as_str)
|
|
.ok_or_else(|| format!("listed Resource without uri: {resource}"))?;
|
|
let Some((_, name)) = expected
|
|
.iter()
|
|
.find(|(expected_uri, _)| *expected_uri == uri)
|
|
else {
|
|
return Err(format!("unexpected listed Resource {uri}: {result}"));
|
|
};
|
|
if resource.get("name").and_then(Value::as_str) != Some(*name) {
|
|
return Err(format!(
|
|
"listed Resource {uri} must carry the published name {name}: {resource}"
|
|
));
|
|
}
|
|
if resource
|
|
.get("mimeType")
|
|
.and_then(Value::as_str)
|
|
.is_none_or(|mime| mime.is_empty())
|
|
{
|
|
return Err(format!(
|
|
"listed Resource {uri} must carry a MIME type: {resource}"
|
|
));
|
|
}
|
|
}
|
|
Ok(resources.len())
|
|
}
|
|
|
|
fn evaluate(call: WireCall, revision: &'static str, endpoint: &ResourceEndpoint) -> Sample {
|
|
let Some(body) = call.body.as_deref() else {
|
|
return call.failure(
|
|
"transport-error",
|
|
call.transport_error.clone().unwrap_or_default(),
|
|
);
|
|
};
|
|
if call.http_status != Some(200) {
|
|
return call.failure(
|
|
"http-status",
|
|
format!("HTTP {}: {body:.400}", call.http_status.unwrap_or(0)),
|
|
);
|
|
}
|
|
match validate_list_body(body, revision, endpoint) {
|
|
Ok(_) => Sample {
|
|
revision,
|
|
phase: Phase::Measured,
|
|
index: 0,
|
|
latency_ms: call.latency_ms,
|
|
http_status: call.http_status,
|
|
body_bytes: call.body_bytes,
|
|
outcome: "ok",
|
|
detail: String::new(),
|
|
},
|
|
Err(detail) => call.failure("wire-invalid", detail),
|
|
}
|
|
}
|
|
|
|
fn percentile_nearest_rank(sorted: &[f64], fraction: f64) -> f64 {
|
|
if sorted.is_empty() {
|
|
return f64::NAN;
|
|
}
|
|
let rank = (fraction * sorted.len() as f64).ceil() as usize;
|
|
sorted[rank.saturating_sub(1).min(sorted.len() - 1)]
|
|
}
|
|
|
|
fn summarize(
|
|
revision: &'static str,
|
|
samples: &[Sample],
|
|
wall: Duration,
|
|
init_ms: Option<f64>,
|
|
) -> Value {
|
|
let mut latencies: Vec<f64> = samples
|
|
.iter()
|
|
.filter(|sample| sample.outcome == "ok")
|
|
.map(|sample| sample.latency_ms)
|
|
.collect();
|
|
latencies.sort_by(f64::total_cmp);
|
|
let mut error_counts: std::collections::BTreeMap<&'static str, usize> =
|
|
std::collections::BTreeMap::new();
|
|
for sample in samples.iter().filter(|sample| sample.outcome != "ok") {
|
|
*error_counts.entry(sample.outcome).or_default() += 1;
|
|
}
|
|
let mean = if latencies.is_empty() {
|
|
f64::NAN
|
|
} else {
|
|
latencies.iter().sum::<f64>() / latencies.len() as f64
|
|
};
|
|
let p95 = percentile_nearest_rank(&latencies, 0.95);
|
|
let wall_seconds = wall.as_secs_f64();
|
|
json!({
|
|
"revision": revision,
|
|
"measured_requests": samples.len(),
|
|
"ok": latencies.len(),
|
|
"errors": error_counts,
|
|
"p50_ms": percentile_nearest_rank(&latencies, 0.50),
|
|
"p95_ms": p95,
|
|
"max_ms": latencies.last().copied().unwrap_or(f64::NAN),
|
|
"min_ms": latencies.first().copied().unwrap_or(f64::NAN),
|
|
"mean_ms": mean,
|
|
"wall_seconds": wall_seconds,
|
|
"achieved_requests_per_second": if wall_seconds > 0.0 {
|
|
samples.len() as f64 / wall_seconds
|
|
} else {
|
|
f64::NAN
|
|
},
|
|
"body_bytes_min": samples.iter().filter_map(|s| s.body_bytes).min(),
|
|
"body_bytes_max": samples.iter().filter_map(|s| s.body_bytes).max(),
|
|
"session_init_ms": init_ms,
|
|
"nfr_p4_p95_limit_ms": NFR_P4_P95_LIMIT_MS,
|
|
"p95_within_150ms_local_comparison_only": p95.is_finite() && p95 <= NFR_P4_P95_LIMIT_MS,
|
|
})
|
|
}
|
|
|
|
fn git(workdir: &str, args: &[&str]) -> String {
|
|
std::process::Command::new("git")
|
|
.args(args)
|
|
.current_dir(workdir)
|
|
.output()
|
|
.ok()
|
|
.filter(|output| output.status.success())
|
|
.and_then(|output| String::from_utf8(output.stdout).ok())
|
|
.map(|stdout| stdout.trim().to_owned())
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
fn environment_snapshot() -> Value {
|
|
let loadavg = fs::read_to_string("/proc/loadavg").unwrap_or_default();
|
|
let meminfo = fs::read_to_string("/proc/meminfo").unwrap_or_default();
|
|
let mem_available_kib = meminfo
|
|
.lines()
|
|
.find(|line| line.starts_with("MemAvailable:"))
|
|
.and_then(|line| line.split_whitespace().nth(1))
|
|
.and_then(|value| value.parse::<u64>().ok());
|
|
json!({
|
|
"os": std::env::consts::OS,
|
|
"arch": std::env::consts::ARCH,
|
|
"logical_cpus": std::thread::available_parallelism().map(|n| n.get()).unwrap_or(0),
|
|
"loadavg": loadavg.trim(),
|
|
"mem_available_kib": mem_available_kib,
|
|
"build_profile": if cfg!(debug_assertions) { "debug" } else { "release" },
|
|
"clock": "std::time::Instant (monotonic)",
|
|
"shared_host_note": "shared 4 vCPU / 4 GiB test server; loadavg recorded at measurement start and end",
|
|
})
|
|
}
|
|
|
|
async fn db_counts(registry: &crank_registry::PostgresRegistry) -> Value {
|
|
let pool = registry.pool();
|
|
let mut counts = serde_json::Map::new();
|
|
for (label, sql) in [
|
|
("agents", "select count(*) from agents"),
|
|
("published_agents", "select count(*) from published_agents"),
|
|
("operations", "select count(*) from operations"),
|
|
("resources", "select count(*) from resources"),
|
|
(
|
|
"resource_publications",
|
|
"select count(*) from resource_publications",
|
|
),
|
|
(
|
|
"agent_resource_bindings",
|
|
"select count(*) from agent_resource_bindings",
|
|
),
|
|
] {
|
|
let value = sqlx::query_scalar::<_, i64>(sql)
|
|
.fetch_one(pool)
|
|
.await
|
|
.map_err(|error| format!("{label}: {error}"));
|
|
match value {
|
|
Ok(count) => {
|
|
counts.insert(label.to_owned(), json!(count));
|
|
}
|
|
Err(detail) => {
|
|
counts.insert(label.to_owned(), json!({ "error": detail }));
|
|
}
|
|
}
|
|
}
|
|
Value::Object(counts)
|
|
}
|
|
|
|
fn output_dir() -> PathBuf {
|
|
std::env::var_os("CRANK_DISCOVERY_DIAGNOSTIC_DIR")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.tmp/t_b5624309"))
|
|
}
|
|
|
|
/// Opt-in: `#[ignore]` keeps an unproven timing diagnostic out of the default suite. The console
|
|
/// `test result: ignored` line is the visible signal that this measurement did not run.
|
|
#[tokio::test]
|
|
#[ignore = "opt-in NFR-P4 discovery latency diagnostic (real HTTP + PostgreSQL), not a default-suite gate"]
|
|
async fn discovery_latency_diagnostic_p50_p95_both_revisions() {
|
|
let mut endpoint = standard_endpoint("nfr_p4_discovery").await;
|
|
// The accepted fixture resyncs the published catalog on every request (`refresh_interval` = 0),
|
|
// so each sample measures a cold-catalog discovery request: the production read path over HTTP.
|
|
endpoint.client = reqwest::Client::builder()
|
|
.timeout(REQUEST_TIMEOUT)
|
|
.build()
|
|
.expect("diagnostic HTTP client must build");
|
|
|
|
let revisions = supported_revisions();
|
|
for revision in [MODERN_REVISION, LEGACY_REVISION] {
|
|
assert!(
|
|
revisions.iter().any(|supported| supported == revision),
|
|
"the checked-in revision oracle must contain {revision}: {revisions:?}"
|
|
);
|
|
}
|
|
|
|
let worktree = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
|
|
let worktree = worktree.display().to_string();
|
|
let environment_start = environment_snapshot();
|
|
let counts = db_counts(&endpoint.registry).await;
|
|
let started_at = now_rfc3339();
|
|
let run_started = Instant::now();
|
|
let mut all_samples: Vec<Value> = Vec::new();
|
|
let mut summaries = serde_json::Map::new();
|
|
|
|
for revision in [MODERN_REVISION, LEGACY_REVISION] {
|
|
// Session setup is deliberately outside the timed list span: legacy only.
|
|
let (init_ms, session) = if revision == LEGACY_REVISION {
|
|
let init_started = Instant::now();
|
|
let session =
|
|
initialize_session(&endpoint.client, &endpoint.mcp_url, &endpoint.api_key).await;
|
|
(
|
|
Some(init_started.elapsed().as_secs_f64() * 1000.0),
|
|
Some(session),
|
|
)
|
|
} else {
|
|
(None, None)
|
|
};
|
|
|
|
let mut measured: Vec<Sample> = Vec::new();
|
|
let total = WARMUP_REQUESTS_PER_REVISION + MEASURED_REQUESTS_PER_REVISION;
|
|
let measured_started = Instant::now();
|
|
for index in 0..total {
|
|
let phase = if index < WARMUP_REQUESTS_PER_REVISION {
|
|
Phase::Warmup
|
|
} else {
|
|
Phase::Measured
|
|
};
|
|
let request_started = Instant::now();
|
|
let call = timed_list(&endpoint, revision, session.as_deref()).await;
|
|
let mut sample = evaluate(call, revision, &endpoint);
|
|
sample.revision = revision;
|
|
sample.phase = phase;
|
|
sample.index = index;
|
|
all_samples.push(sample.as_json());
|
|
if phase == Phase::Measured {
|
|
measured.push(sample);
|
|
}
|
|
let elapsed = request_started.elapsed();
|
|
if elapsed < MIN_REQUEST_INTERVAL {
|
|
tokio::time::sleep(MIN_REQUEST_INTERVAL - elapsed).await;
|
|
}
|
|
}
|
|
let wall = measured_started.elapsed();
|
|
let summary = summarize(revision, &measured, wall, init_ms);
|
|
println!(
|
|
"[discovery-diagnostic] revision={revision} samples={} ok={} errors={} p50={:.3}ms p95={:.3}ms max={:.3}ms wall={:.2}s",
|
|
summary["measured_requests"],
|
|
summary["ok"],
|
|
summary["errors"],
|
|
summary["p50_ms"].as_f64().unwrap_or(f64::NAN),
|
|
summary["p95_ms"].as_f64().unwrap_or(f64::NAN),
|
|
summary["max_ms"].as_f64().unwrap_or(f64::NAN),
|
|
summary["wall_seconds"].as_f64().unwrap_or(f64::NAN),
|
|
);
|
|
summaries.insert(revision.to_owned(), summary);
|
|
}
|
|
|
|
let out_dir = output_dir();
|
|
fs::create_dir_all(&out_dir).expect("diagnostic output directory must be creatable");
|
|
let raw_path = out_dir.join("discovery-samples.jsonl");
|
|
let mut raw = String::new();
|
|
for sample in &all_samples {
|
|
raw.push_str(&sample.to_string());
|
|
raw.push('\n');
|
|
}
|
|
fs::write(&raw_path, raw).expect("raw samples must be written");
|
|
|
|
let max_payload_bytes = [&endpoint.text, &endpoint.json, &endpoint.binary]
|
|
.into_iter()
|
|
.map(|bound| bound.bytes.len())
|
|
.max()
|
|
.unwrap_or(0);
|
|
|
|
let summary = json!({
|
|
"diagnostic": "story38_nfr_p4_resource_discovery_latency",
|
|
"generated_at_utc": now_rfc3339(),
|
|
"started_at_utc": started_at,
|
|
"candidate": {
|
|
"head": git(&worktree, &["rev-parse", "HEAD"]),
|
|
"branch": git(&worktree, &["rev-parse", "--abbrev-ref", "HEAD"]),
|
|
"worktree": worktree,
|
|
"dirty_tree": !git(&worktree, &["status", "--porcelain"]).is_empty(),
|
|
"note": "HEAD alone is insufficient in this dirty worktree; source identity is recorded in .tmp/t_b5624309/identity-*.json",
|
|
},
|
|
"scope": {
|
|
"measures": "MCP resources/list response latency over the real HTTP endpoint",
|
|
"does_not_measure": [
|
|
"resources/read latency",
|
|
"tools/call or platform execution overhead",
|
|
"mixed load, soak, burst or release Reference workload",
|
|
"representative real MCP client applications (none used; reqwest is not a client application)",
|
|
],
|
|
"nfr_p4_p95_limit_ms": NFR_P4_P95_LIMIT_MS,
|
|
"qualifies_nfr_p4": false,
|
|
"qualification_reason": "bounded existing fixture dataset on a shared 4 GiB host, single in-process test, not the PRD Reference Deployment dataset/load generator",
|
|
},
|
|
"method": {
|
|
"transport": "loopback HTTP (127.0.0.1, ephemeral port) served by the production axum router",
|
|
"registry": "real PostgreSQL 16 (testcontainers, own schema) with the real migration authority",
|
|
"artifact_store": "real immutable test-owned ArtifactStore root",
|
|
"revisions": [MODERN_REVISION, LEGACY_REVISION],
|
|
"warmup_requests_per_revision": WARMUP_REQUESTS_PER_REVISION,
|
|
"measured_requests_per_revision": MEASURED_REQUESTS_PER_REVISION,
|
|
"concurrency": 1,
|
|
"min_request_interval_ms": MIN_REQUEST_INTERVAL.as_millis(),
|
|
"request_timeout_ms": REQUEST_TIMEOUT.as_millis(),
|
|
"timed_span": "request start → complete response body consumed",
|
|
"percentile": "nearest-rank",
|
|
"warm_cold_policy": "process and fixtures warm after construction; first measured revision runs after its own 20-request warmup; catalog refresh interval is 0 ms, so every request reloads the published catalog from PostgreSQL (cold-catalog path)",
|
|
"session_policy": "legacy 2025-11-25: one initialize/initialized handshake before warmup, timed separately; modern 2026-07-28: stateless per request",
|
|
"error_policy": "transport, timeout, HTTP-status and wire-validation failures are counted per revision and excluded from percentiles; any error fails the test",
|
|
},
|
|
"dataset": {
|
|
"fixture": "static_resources::fixtures::standard_endpoint (accepted AT-3.8 suite)",
|
|
"bound_resources": 3,
|
|
"resource_kinds": ["text", "json", "binary"],
|
|
"max_resource_payload_bytes": max_payload_bytes,
|
|
"page": "no limit parameter: all 3 bound Resources in one page (server max page 100)",
|
|
"published_agents_in_request_scope": 1,
|
|
"published_operations_in_request_scope": 1,
|
|
"seeded_but_not_in_scope": "1 additional draft Agent, 1 published-but-unbound Resource, 1 draft Resource, 1 foreign-workspace Resource",
|
|
"db_row_counts": counts,
|
|
"dataset_gap": "PRD Reference dataset (200 Agents / 2 000 Operations, 32 static Resources of ≤4 KiB) is NOT seeded: the accepted fixture capabilities cannot seed it without a larger new harness, so this run measures the minimal existing set",
|
|
},
|
|
"environment": environment_start,
|
|
"environment_at_end": environment_snapshot(),
|
|
"run_wall_seconds": run_started.elapsed().as_secs_f64(),
|
|
"revisions": summaries,
|
|
"raw_samples_file": raw_path.display().to_string(),
|
|
"total_requests": all_samples.len(),
|
|
});
|
|
let summary_path = out_dir.join("discovery-summary.json");
|
|
fs::write(
|
|
&summary_path,
|
|
format!(
|
|
"{}\n",
|
|
serde_json::to_string_pretty(&summary).expect("summary serializes")
|
|
),
|
|
)
|
|
.expect("summary must be written");
|
|
println!("[discovery-diagnostic] summary: {}", summary_path.display());
|
|
|
|
for revision in [MODERN_REVISION, LEGACY_REVISION] {
|
|
let entry = &summaries[revision];
|
|
let errors = entry["errors"]
|
|
.as_object()
|
|
.map(|map| map.len())
|
|
.unwrap_or(0);
|
|
assert_eq!(
|
|
entry["measured_requests"].as_u64(),
|
|
Some(MEASURED_REQUESTS_PER_REVISION as u64),
|
|
"every revision must produce the planned measured sample count: {entry}"
|
|
);
|
|
assert_eq!(
|
|
errors, 0,
|
|
"a working local endpoint must not answer transport/HTTP/wire failures on {revision}: {entry}"
|
|
);
|
|
assert_eq!(
|
|
entry["ok"].as_u64(),
|
|
Some(MEASURED_REQUESTS_PER_REVISION as u64),
|
|
"every measured response must validate against the bound snapshot on {revision}: {entry}"
|
|
);
|
|
}
|
|
}
|