Files
crank/apps/admin-api/tests/config_startup.rs
T
bsodfather 54ce770682
CI / Rust Checks (pull_request) Failing after 13m41s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
Community Release / release (push) Failing after 13m33s
feat(epic3): deliver the accepted Epic 3 scope through Story 3.17
Recovery delivery of the accumulated, TL-accepted Epic 3 work that had never been
committed to Git (stories 3.6-3.17, see the crank board). Story 3.18 is not accepted
yet and is deliberately excluded: its logging surface, its capability registration,
its oracle projections and its tests are not part of this tree.

Accepted scope
- 3.6 typed static Resource Drafts; 3.7 publication and exact binding of static
  Resource Versions; 3.8 MCP discovery/read and revocation of static Resources;
- 3.9 Resource import/export in API and UI plus TLS dependency maintenance;
  3.10 REST-backed Resource templates; 3.11 shared REST read runtime and the MCP
  revision/capability foundation;
- 3.12 parameterised Prompt Drafts with preview; 3.13 Prompt publication and exact
  version binding to Agents; 3.14 MCP prompts; 3.15 Prompt import/export;
- 3.16 composite Prompts with published Resources (dependency content scan plus
  dynamic read through the accepted 3.11 runtime, admin preview and MCP prompts/get);
- 3.17 bounded MCP completion suggestions from typed published schemas and the
  resources/list admission fix for bound template publications.

Accepted evidence being reused (unchanged code paths)
- 3.16 candidate: core 7, admin-api 22, registry 4, mcp-server 4, UI 6; one final
  `just verify` run 1651 passed / 0 failed / 2 ignored.
- 3.17 candidate: full run 1672 passed / 0 failed / 2 ignored, QA targeted reruns and
  a real MCP Inspector 2.12.0 client smoke.

Delivery engineering in this commit
- CI: one event per change (feature branches are gated by their pull request, push
  runs only on main), story events run the changed packages, the full workspace suite
  stays the feature/release gate; no cache/sharding without a measured runner check.
- Release: the packaged archives, checksums and SBOM are now uploaded to the Gitea
  release entry of the tag instead of dying with the runner workspace.

Not included on purpose: Story 3.18 work, build caches, runtime databases, local
credentials and scratch evidence.

Refs: crank kanban t_e39d7263; baseline 1b6db09bd1.
2026-10-25 09:08:22 +00:00

604 lines
22 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use std::{
fs,
io::Read,
os::unix::fs::PermissionsExt,
process::{Child, Command, Output, Stdio},
sync::atomic::{AtomicU64, Ordering},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use axum::{body::Body, http::Request};
use crank_observability::{
MetricService, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle, OtlpTraceConfig,
RedactionLimits, SentryConfig, ServiceIdentity, bound_metric_service,
};
use crank_registry::{MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresRegistry};
use crank_runtime::SecretCrypto;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tower::ServiceExt;
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
const REGISTERED_MASTER_KEY: &str = "registered-master-key-CANARY_SECRET_VALUE-00000000";
const WRONG_MASTER_KEY: &str = "wrong-master-key-CANARY_SECRET_VALUE-0000000000000";
static NEXT_STORAGE_ROOT: AtomicU64 = AtomicU64::new(0);
fn isolated_metrics_bind() -> String {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("ephemeral metrics port");
let addr = listener.local_addr().unwrap().to_string();
drop(listener);
addr
}
fn run_with(entries: &[(&str, &str)]) -> String {
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
command.envs([
("CRANK_MASTER_KEY", TEST_MASTER_KEY),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_METRICS_BIND", isolated_metrics_bind().as_str()),
]);
for (name, value) in entries {
command.env(name, value);
}
let output = command.output().expect("admin binary executes");
assert!(!output.status.success());
let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8");
assert!(!stderr.contains("CANARY_SECRET_VALUE"));
assert!(!stderr.contains("connection refused"));
assert!(stderr.len() <= 65_536);
stderr
}
#[test]
fn invalid_config_fails_before_database_or_listener_side_effects() {
for (entries, code) in [
(
vec![("CRANK_CONFIG_CANARY_UNKNOWN", "CANARY_SECRET_VALUE")],
"config.unknown_field",
),
(vec![("POSTGRES_PORT", "bad")], "config.invalid_type"),
(
vec![
("CRANK_DATABASE_URL", "postgres://db/crank"),
("POSTGRES_HOST", "other"),
],
"config.conflict",
),
(vec![("CRANK_LOG_LEVEL", "[")], "config.invalid_type"),
] {
let stderr = run_with(&entries);
assert!(stderr.contains(code), "{stderr}");
}
}
#[test]
fn database_driver_failures_are_normalized_and_redacted() {
let stderr = run_with(&[(
"CRANK_DATABASE_URL",
"postgres://CANARY_SECRET_VALUE:CANARY_SECRET_VALUE@127.0.0.1:1/crank",
)]);
assert!(stderr.contains("startup_failed"), "{stderr}");
}
#[tokio::test]
async fn fresh_database_startup_is_read_only() {
let database_url = crank_test_support::postgres_schema_url("admin_startup_read_only").await;
let stderr = run_with(&[("CRANK_DATABASE_URL", &database_url)]);
assert!(stderr.contains("schema_missing"), "{stderr}");
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let present: bool = sqlx::query_scalar(
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
)
.fetch_one(&pool)
.await
.unwrap();
assert!(!present);
}
#[tokio::test]
async fn real_admin_startup_runs_immediate_import_maintenance() {
let unique = NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed);
let epoch = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let database_url =
crank_test_support::postgres_schema_url(&format!("startup_maint_{epoch}_{unique}")).await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
assert_eq!(
sqlx::query_scalar::<_, Option<i64>>("select max(version) from __crank_migrations")
.fetch_one(&pool)
.await
.unwrap(),
Some(23),
"migration binary must install the current canonical ledger",
);
sqlx::query(
"insert into import_jobs (
id, workspace_id, kind, source_format, source_version, status,
preview_payload, created_operation_ids, error_text, created_at, expires_at, finished_at
) values (
'imp_startup_expired', 'ws_default', 'openapi', 'openapi', null, 'pending',
'{\"source\": {\"source_id\": 1}}'::jsonb, '[]'::jsonb, null,
now() - interval '2 hours', now() - interval '1 hour', null
)",
)
.execute(&pool)
.await
.unwrap();
let storage_root = std::env::temp_dir().join(format!(
"crank-admin-startup-maintenance-{}-{epoch}-{unique}",
std::process::id()
));
fs::create_dir(&storage_root).unwrap();
fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let bind_addr = listener.local_addr().unwrap().to_string();
drop(listener);
let metrics_bind = isolated_metrics_bind();
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
let mut child = command
.envs([
("CRANK_DATABASE_URL", database_url.as_str()),
("CRANK_MASTER_KEY", TEST_MASTER_KEY),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_BIND", bind_addr.as_str()),
("CRANK_ADMIN_METRICS_BIND", metrics_bind.as_str()),
("CRANK_STORAGE_ROOT", storage_root.to_str().unwrap()),
])
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("admin binary starts");
let mut cleaned = false;
for _ in 0..200 {
if let Some(status) = child.try_wait().unwrap() {
let mut stderr = String::new();
if let Some(mut stream) = child.stderr.take() {
stream.read_to_string(&mut stderr).unwrap();
}
panic!("admin binary exited before startup maintenance: {status}: {stderr}");
}
let remaining: i64 =
sqlx::query_scalar("select count(*) from import_jobs where id = 'imp_startup_expired'")
.fetch_one(&pool)
.await
.unwrap();
if remaining == 0 {
cleaned = true;
break;
}
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
child.kill().expect("admin binary can be stopped");
child.wait().expect("admin binary can be reaped");
fs::remove_dir_all(&storage_root).unwrap();
assert!(cleaned, "real startup did not clean the expired import job");
}
#[tokio::test]
async fn master_key_mismatch_blocks_startup_with_safe_diagnostic() {
let database_url = crank_test_support::postgres_schema_url("admin_master_key_mismatch").await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::parse("2026-08-21T00:00:00Z", &Rfc3339).unwrap(),
})
.await
.unwrap();
let stderr = run_with(&[
("CRANK_DATABASE_URL", &database_url),
("CRANK_MASTER_KEY", WRONG_MASTER_KEY),
]);
assert!(stderr.contains("master_key_identity_mismatch"), "{stderr}");
assert!(!stderr.contains("registered-master-key"));
assert!(!stderr.contains("wrong-master-key"));
}
#[tokio::test]
async fn populated_database_without_identity_rejects_wrong_first_key() {
let database_url =
crank_test_support::postgres_schema_url("admin_master_key_first_registration_wrong").await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let old_crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap();
let ciphertext = old_crypto
.encrypt(&serde_json::json!({"token": "CANARY_SECRET_VALUE"}))
.unwrap();
sqlx::query(
"insert into secrets (
id, workspace_id, name, kind, status, current_version, created_at, updated_at
) values (
'legacy_secret', 'ws_default', 'legacy secret', 'token', 'active', 1, now(), now()
)",
)
.execute(&pool)
.await
.unwrap();
sqlx::query(
"insert into secret_versions (
secret_id, version, ciphertext, key_version, master_key_epoch, created_at, created_by
) values (
'legacy_secret', 1, $1, $2, 1, now(), null
)",
)
.bind(ciphertext)
.bind(old_crypto.key_version())
.execute(&pool)
.await
.unwrap();
let stderr = run_with(&[
("CRANK_DATABASE_URL", &database_url),
("CRANK_MASTER_KEY", WRONG_MASTER_KEY),
]);
assert!(stderr.contains("startup_failed"), "{stderr}");
assert!(!stderr.contains("CANARY_SECRET_VALUE"));
let identities: i64 = sqlx::query_scalar("select count(*) from master_key_identities")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(identities, 0);
}
// --- TD-B6/TD-E6/TD-E8: реальный startup-порядок binding → recorder → listeners ---
struct AdminProcess {
child: Child,
base_url: String,
storage_root: std::path::PathBuf,
}
impl AdminProcess {
fn stop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
let _ = fs::remove_dir_all(&self.storage_root);
}
}
impl Drop for AdminProcess {
fn drop(&mut self) {
self.stop();
}
}
async fn migrated_database_url(prefix: &str) -> String {
let database_url = crank_test_support::postgres_schema_url(prefix).await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"migration must apply: {}",
String::from_utf8_lossy(&applied.stderr)
);
database_url
}
fn admin_command(
database_url: &str,
product_addr: &str,
metrics_addr: &str,
metrics_enabled: bool,
storage_root: &std::path::Path,
) -> Command {
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
command.envs([
("CRANK_DATABASE_URL", database_url),
("CRANK_MASTER_KEY", TEST_MASTER_KEY),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_BIND", product_addr),
("CRANK_ADMIN_METRICS_BIND", metrics_addr),
(
"CRANK_METRICS_ENABLED",
if metrics_enabled { "true" } else { "false" },
),
("CRANK_LOG_LEVEL", "info"),
]);
command.env("CRANK_STORAGE_ROOT", storage_root);
command
}
fn spawn_admin(
database_url: &str,
product_addr: &str,
metrics_addr: &str,
metrics_enabled: bool,
) -> AdminProcess {
let storage_root = std::env::temp_dir().join(format!(
"crank-admin-f4-{}-{}-{metrics_enabled}",
std::process::id(),
NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir(&storage_root).unwrap();
fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap();
let child = admin_command(
database_url,
product_addr,
metrics_addr,
metrics_enabled,
&storage_root,
)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("admin binary starts");
AdminProcess {
child,
base_url: format!("http://{product_addr}"),
storage_root,
}
}
fn run_admin_once(
database_url: &str,
product_addr: &str,
metrics_addr: &str,
metrics_enabled: bool,
) -> Output {
let storage_root = std::env::temp_dir().join(format!(
"crank-admin-f4-terminal-{}-{}",
std::process::id(),
NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir(&storage_root).unwrap();
fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap();
let output = admin_command(
database_url,
product_addr,
metrics_addr,
metrics_enabled,
&storage_root,
)
.output()
.expect("admin binary executes");
let _ = fs::remove_dir_all(&storage_root);
output
}
async fn wait_until_healthy(client: &reqwest::Client, process: &mut AdminProcess) {
let deadline = Instant::now() + Duration::from_secs(60);
loop {
if let Some(status) = process.child.try_wait().expect("child status is readable") {
panic!("admin binary exited before readiness: {status}");
}
if client
.get(format!("{}/health", process.base_url))
.timeout(Duration::from_millis(500))
.send()
.await
.is_ok_and(|response| response.status().is_success())
{
return;
}
assert!(Instant::now() < deadline, "admin readiness deadline");
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
/// TD-B6 — реальный бинарник: service binding и recorder публикуются **до**
/// product- и metrics-listener'ов. Product-запрос обязан быть записан (при
/// неопубликованном binding'е recording — no-op), а metrics-listener обязан
/// отдавать exposition с process-constant identity.
#[tokio::test]
async fn binding_precedes_recorder_and_listeners() {
let database_url = migrated_database_url("admin_startup_binding_precedes").await;
let product_addr = isolated_metrics_bind();
let metrics_addr = isolated_metrics_bind();
let mut process = spawn_admin(&database_url, &product_addr, &metrics_addr, true);
let client = reqwest::Client::new();
wait_until_healthy(&client, &mut process).await;
let health = client
.get(format!("{}/health", process.base_url))
.send()
.await
.expect("product health request");
assert_eq!(health.status(), reqwest::StatusCode::OK);
let response = client
.get(format!("http://{metrics_addr}/metrics"))
.send()
.await
.expect("metrics scrape");
assert_eq!(response.status(), reqwest::StatusCode::OK);
let content_type = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
assert_eq!(content_type, "text/plain; version=0.0.4; charset=utf-8");
let exposition = response.text().await.expect("metrics body");
assert!(
exposition.contains("service=\"admin-api\""),
"process-constant identity обязана присутствовать в exposition"
);
assert!(
exposition.contains("crank_http_requests_total{"),
"recorder обязан отдавать записанные series: {}",
&exposition[..exposition.len().min(300)]
);
assert!(
exposition.contains("route=\"/health\""),
"запись product-запроса обязана быть под normalized route: {}",
&exposition[..exposition.len().min(300)]
);
process.stop();
}
/// TD-E6 — `enabled=false` выключает metrics-listener, но не binding и не
/// локальную запись: реальный бинарник стартует даже с уже занятым
/// metrics-адресом, а in-process lifecycle сохраняет опубликованный binding и
/// отдаёт записанные series без открытия listener'а.
#[tokio::test]
async fn disabled_listener_still_binds_service() {
// (1) Реальный бинарник: metrics disabled, metrics-адрес занят тестом.
let occupied = std::net::TcpListener::bind("127.0.0.1:0").expect("occupied metrics port");
let metrics_addr = occupied.local_addr().unwrap().to_string();
let database_url = migrated_database_url("admin_startup_metrics_disabled").await;
let product_addr = isolated_metrics_bind();
let mut process = spawn_admin(&database_url, &product_addr, &metrics_addr, false);
let client = reqwest::Client::new();
wait_until_healthy(&client, &mut process).await;
assert_eq!(
client
.get(format!("{}/health", process.base_url))
.send()
.await
.expect("product health request")
.status(),
reqwest::StatusCode::OK,
"disabled metrics listener не делает startup terminal"
);
process.stop();
drop(occupied);
// (2) In-process: disabled MetricsConfig сохраняет binding и локальную запись,
// но не создаёт listener.
let bind_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let identity =
ServiceIdentity::try_new("admin-api", env!("CARGO_PKG_VERSION"), "test").expect("identity");
let config = ObservabilityConfig::try_new(identity, "off", RedactionLimits::default())
.expect("observability config");
let lifecycle = ObservabilityLifecycle::init_with_exporters(
config,
SentryConfig::parse(None).expect("sentry config"),
OtlpTraceConfig::default(),
)
.expect("observability lifecycle");
assert_eq!(
bound_metric_service(),
Some(MetricService::AdminApi),
"binding обязателен и при disabled listener"
);
let metrics_config = MetricsConfig::new(false, bind_addr, None).expect("metrics config");
assert!(!metrics_config.enabled(), "listener обязан быть выключен");
let surface = lifecycle.metrics_surface(metrics_config);
let probe =
std::net::TcpListener::bind(bind_addr).expect("disabled listener не занимает адрес");
drop(probe);
crank_metrics::record_http_request(
crank_metrics::HttpRoute::from_matched_path("/health"),
crank_metrics::HttpMethod::Get,
crank_metrics::HttpStatusClass::Success,
Duration::from_millis(1),
);
let response = surface
.router()
.oneshot(
Request::builder()
.uri("/metrics")
.body(Body::empty())
.unwrap(),
)
.await
.expect("in-process metrics request");
assert_eq!(response.status(), axum::http::StatusCode::OK);
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.expect("metrics body");
let exposition = String::from_utf8(body.to_vec()).expect("utf-8 exposition");
assert!(
exposition.contains("crank_http_requests_total")
&& exposition.contains("service=\"admin-api\""),
"локальная запись при disabled listener обязана сохраняться: {}",
&exposition[..exposition.len().min(200)]
);
}
/// TD-E8 — ошибка binding'а metrics-listener'а terminal: тот же конфиг со
/// свободным портом стартует, а с занятым — падает до открытия product
/// listener'а (product-адрес остаётся свободным, listening-лог не появляется).
#[tokio::test]
async fn startup_binding_failures_are_terminal() {
let database_url = migrated_database_url("admin_startup_binding_terminal").await;
// Positive control: конфигурация со свободным metrics-портом стартует.
let free_metrics = isolated_metrics_bind();
let product_addr = isolated_metrics_bind();
let mut process = spawn_admin(&database_url, &product_addr, &free_metrics, true);
let client = reqwest::Client::new();
wait_until_healthy(&client, &mut process).await;
process.stop();
// Тот же конфиг с занятым metrics-адресом обязан упасть ДО product listener.
let occupied = std::net::TcpListener::bind("127.0.0.1:0").expect("occupied metrics port");
let occupied_addr = occupied.local_addr().unwrap().to_string();
let output = run_admin_once(&database_url, &product_addr, &occupied_addr, true);
assert!(
!output.status.success(),
"bind-ошибка metrics-listener'а обязана быть terminal"
);
let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8");
assert!(stderr.contains("startup_failed"), "{stderr}");
assert!(
!stderr.contains("admin.server.listening"),
"product listener не мог быть открыт до terminal-ошибки: {stderr}"
);
assert!(
std::net::TcpListener::bind(&product_addr).is_ok(),
"product listener процесса не открывался"
);
drop(occupied);
}