CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с fence-токеном, зашифрованный input и повторная проверка доступа до сетевого вызова, checkpoint с AttemptID до dispatch и восстановление только по PG. Устаревший worker не может зафиксировать результат. Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса: стоп в окне после checkpoint и до permit завершает задачу как failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи остаются доступными. Retry-попытки получают отдельные дочерние spans с durable current_attempt_id. Shared admission/fairness из Story 4.1 сохранены. Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47, bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA и исходный runtime-прогон), точный exact-head CI остаётся обязательным.
978 lines
35 KiB
Rust
978 lines
35 KiB
Rust
#[path = "crank_migrate/admin_auth_command.rs"]
|
|
mod admin_auth_command;
|
|
#[path = "crank_migrate/db_connect.rs"]
|
|
mod db_connect;
|
|
#[path = "crank_migrate/reporting.rs"]
|
|
mod reporting;
|
|
|
|
use reporting::{active_rotation, ensure_no_options, read_required_key_file, rotation_json};
|
|
|
|
use crank_config::{ConfigSource, parse_migrator};
|
|
use crank_registry::{
|
|
BackfillPolicy, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
|
|
MasterKeyRotationRecord, MigrationApplyResult, MigrationAuthority, MigrationPreflight,
|
|
PostgresRegistry, RegistryError, SecretVersionRecord,
|
|
};
|
|
use crank_runtime::SecretCrypto;
|
|
use serde_json::json;
|
|
use std::process::ExitCode;
|
|
use time::OffsetDateTime;
|
|
const MASTER_KEY_ROTATION_PAGE_SIZE: i64 = 1_000;
|
|
#[tokio::main]
|
|
async fn main() -> ExitCode {
|
|
match run().await {
|
|
Ok(code) => code,
|
|
Err(error) => {
|
|
eprintln!(
|
|
"{}",
|
|
json!({
|
|
"status": "error",
|
|
"code": error.code,
|
|
"stage": error.stage,
|
|
"version": error.version,
|
|
"recovery": error.recovery,
|
|
})
|
|
);
|
|
ExitCode::FAILURE
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
struct CliError {
|
|
code: &'static str,
|
|
stage: &'static str,
|
|
recovery: &'static str,
|
|
version: Option<i64>,
|
|
}
|
|
impl CliError {
|
|
const fn new(code: &'static str, stage: &'static str, recovery: &'static str) -> Self {
|
|
Self {
|
|
code,
|
|
stage,
|
|
recovery,
|
|
version: None,
|
|
}
|
|
}
|
|
fn from_migration(error: crank_registry::MigrationError) -> Self {
|
|
Self {
|
|
code: error.code(),
|
|
stage: error.stage(),
|
|
recovery: error.recovery(),
|
|
version: error.version(),
|
|
}
|
|
}
|
|
fn from_registry(error: RegistryError) -> Self {
|
|
match error {
|
|
RegistryError::MasterKeyIdentityMismatch { .. } => Self::new(
|
|
"master_key_identity_mismatch",
|
|
"master_key.identity",
|
|
"use_matching_master_key",
|
|
),
|
|
RegistryError::InvalidMasterKeyIdentity => Self::new(
|
|
"master_key_identity_invalid",
|
|
"master_key.identity",
|
|
"verify_operator_input",
|
|
),
|
|
RegistryError::MasterKeyRotationInProgress => Self::new(
|
|
"master_key_rotation_in_progress",
|
|
"master_key.rotation",
|
|
"resume_verify_promote_or_abort_rotation",
|
|
),
|
|
RegistryError::MasterKeyRotationNotFound { .. } => Self::new(
|
|
"master_key_rotation_not_found",
|
|
"master_key.rotation",
|
|
"run_status",
|
|
),
|
|
RegistryError::MasterKeyRotationConflict => Self::new(
|
|
"master_key_rotation_conflict",
|
|
"master_key.rotation",
|
|
"run_status",
|
|
),
|
|
RegistryError::MasterKeyRotationVerificationFailed => Self::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
),
|
|
RegistryError::AdminBootstrapUnavailable => Self::new(
|
|
"admin_bootstrap_unavailable",
|
|
"admin_auth.bootstrap",
|
|
"use_existing_active_contract_or_wait_until_expired",
|
|
),
|
|
RegistryError::AdminBootstrapRejected => Self::new(
|
|
"admin_bootstrap_rejected",
|
|
"admin_auth.bootstrap",
|
|
"create_new_local_bootstrap_contract",
|
|
),
|
|
RegistryError::AdminRecoveryRejected => Self::new(
|
|
"admin_recovery_rejected",
|
|
"admin_auth.recovery",
|
|
"verify_local_inputs_and_master_key",
|
|
),
|
|
RegistryError::AdminLoginRateLimited { .. } => Self::new(
|
|
"admin_login_rate_limited",
|
|
"admin_auth.login",
|
|
"retry_after_delay",
|
|
),
|
|
RegistryError::AdminCsrfRejected => {
|
|
Self::new("admin_csrf_rejected", "admin_auth.csrf", "refresh_session")
|
|
}
|
|
RegistryError::Storage(_) => {
|
|
Self::new("storage_unavailable", "database.query", "contact_operator")
|
|
}
|
|
_ => Self::new("registry_error", "registry.operation", "contact_operator"),
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn run() -> Result<ExitCode, CliError> {
|
|
let args = std::env::args().skip(1).collect::<Vec<_>>();
|
|
if args.first().map(String::as_str) == Some("master-key") {
|
|
return run_master_key(&args[1..]).await;
|
|
}
|
|
if args.first().map(String::as_str) == Some("admin-auth") {
|
|
return admin_auth_command::run_admin_auth(&args[1..]).await;
|
|
}
|
|
let requested = args.first().map(String::as_str);
|
|
let option = args.get(1).map(String::as_str);
|
|
if args.len() > 2 {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_preflight",
|
|
));
|
|
}
|
|
let command = match requested {
|
|
None | Some("preflight") => "preflight",
|
|
Some("plan") => "plan",
|
|
Some("apply") => "apply",
|
|
Some(_) => {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_preflight",
|
|
));
|
|
}
|
|
};
|
|
|
|
if command == "plan" {
|
|
MigrationAuthority::validate_sequence().map_err(CliError::from_migration)?;
|
|
let sequence = MigrationAuthority::sequence()
|
|
.into_iter()
|
|
.map(|migration| {
|
|
let backfill = match migration.backfill {
|
|
BackfillPolicy::None => json!({ "kind": "none" }),
|
|
BackfillPolicy::Bounded {
|
|
max_batch_rows,
|
|
max_batch_ms,
|
|
resumable,
|
|
} => json!({
|
|
"kind": "bounded",
|
|
"max_batch_rows": max_batch_rows,
|
|
"max_batch_ms": max_batch_ms,
|
|
"resumable": resumable,
|
|
}),
|
|
};
|
|
json!({
|
|
"version": migration.version,
|
|
"name": migration.name,
|
|
"checksum": migration.checksum,
|
|
"source_digest": migration.source_digest,
|
|
"phase": migration.phase,
|
|
"compatibility": migration.compatibility,
|
|
"owner": migration.owner,
|
|
"transactional": migration.transactional,
|
|
"backfill": backfill,
|
|
"readable_schema_min": migration.readable_schema_min,
|
|
"readable_schema_max": migration.readable_schema_max,
|
|
"contract_evidence": migration.contract_evidence,
|
|
})
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let plan = json!({ "schema_version": 1, "sequence": sequence });
|
|
if option == Some("--target-version") {
|
|
let target_version = plan["sequence"]
|
|
.as_array()
|
|
.and_then(|sequence| sequence.last())
|
|
.and_then(|migration| migration["version"].as_i64())
|
|
.ok_or(CliError::new(
|
|
"migration_sequence_invalid",
|
|
"plan.target_version",
|
|
"contact_operator",
|
|
))?;
|
|
println!("{target_version}");
|
|
return Ok(ExitCode::SUCCESS);
|
|
}
|
|
if option == Some("--check") {
|
|
let bytes = std::fs::read("docs/schemas/migration-sequence.json")
|
|
.map_err(|_| CliError::new("contract_drift", "plan.read", "contact_operator"))?;
|
|
if bytes.len() > 65_536 {
|
|
return Err(CliError::new(
|
|
"contract_drift",
|
|
"plan.size",
|
|
"contact_operator",
|
|
));
|
|
}
|
|
let committed: serde_json::Value = serde_json::from_slice(&bytes)
|
|
.map_err(|_| CliError::new("contract_drift", "plan.parse", "contact_operator"))?;
|
|
if committed != plan {
|
|
return Err(CliError::new(
|
|
"contract_drift",
|
|
"plan.compare",
|
|
"contact_operator",
|
|
));
|
|
}
|
|
println!("{}", json!({ "status": "contract_current" }));
|
|
return Ok(ExitCode::SUCCESS);
|
|
}
|
|
if option.is_some() {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_preflight",
|
|
));
|
|
}
|
|
println!("{plan}");
|
|
return Ok(ExitCode::SUCCESS);
|
|
}
|
|
if option.is_some() {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_preflight",
|
|
));
|
|
}
|
|
|
|
let config = parse_migrator(
|
|
ConfigSource::from_os_for_migrator()
|
|
.map_err(|_| CliError::new("config_invalid", "config.source", "run_preflight"))?,
|
|
)
|
|
.map_err(|_| CliError::new("config_invalid", "config.validate", "run_preflight"))?;
|
|
let pool = db_connect::connect(&config.database).await?;
|
|
|
|
if command == "apply" {
|
|
let result = MigrationAuthority::apply(&pool)
|
|
.await
|
|
.map_err(CliError::from_migration)?;
|
|
let (status, from, to) = match result {
|
|
MigrationApplyResult::Applied { from, to } => ("applied", from, to),
|
|
MigrationApplyResult::AlreadyCurrent { version } => {
|
|
("already_current", version, version)
|
|
}
|
|
};
|
|
println!(
|
|
"{}",
|
|
json!({ "status": status, "from_version": from, "to_version": to })
|
|
);
|
|
return Ok(ExitCode::SUCCESS);
|
|
}
|
|
|
|
match MigrationAuthority::preflight(&pool)
|
|
.await
|
|
.map_err(CliError::from_migration)?
|
|
{
|
|
MigrationPreflight::Current { version } => {
|
|
println!("{}", json!({ "status": "current", "version": version }));
|
|
Ok(ExitCode::SUCCESS)
|
|
}
|
|
MigrationPreflight::MigrationRequired { current, target } => {
|
|
println!(
|
|
"{}",
|
|
json!({
|
|
"status": "migration_required",
|
|
"current_version": current,
|
|
"target_version": target,
|
|
"recovery": "run_controlled_migration",
|
|
})
|
|
);
|
|
Ok(ExitCode::from(2))
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn run_master_key(arguments: &[String]) -> Result<ExitCode, CliError> {
|
|
let Some(command) = arguments.first().map(String::as_str) else {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_master_key_status",
|
|
));
|
|
};
|
|
let options = MasterKeyOptions::parse(&arguments[1..])?;
|
|
let config = parse_migrator(
|
|
ConfigSource::from_os_for_migrator()
|
|
.map_err(|_| CliError::new("config_invalid", "config.source", "run_preflight"))?,
|
|
)
|
|
.map_err(|_| CliError::new("config_invalid", "config.validate", "run_preflight"))?;
|
|
let registry = db_connect::connect_registry(&config.database).await?;
|
|
|
|
match command {
|
|
"status" => {
|
|
ensure_no_options(&options)?;
|
|
let status = registry
|
|
.master_key_rotation_status()
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
println!(
|
|
"{}",
|
|
json!({
|
|
"status": "ok",
|
|
"active_epoch": status.active_identity.as_ref().map(|identity| identity.epoch),
|
|
"rotation_count": status.rotations.len(),
|
|
"rotations": status.rotations.iter().map(rotation_json).collect::<Vec<_>>(),
|
|
})
|
|
);
|
|
}
|
|
"preflight" => {
|
|
let preflight = master_key_preflight(®istry, &options).await?;
|
|
println!(
|
|
"{}",
|
|
json!({
|
|
"status": "preflight_ok",
|
|
"source_epoch": preflight.source_epoch,
|
|
"target_epoch": preflight.target_epoch,
|
|
"affected_secret_versions": preflight.affected_secret_versions,
|
|
"backup_ref": preflight.backup_ref.map(|_| "configured"),
|
|
})
|
|
);
|
|
}
|
|
"rotate" => {
|
|
let (rotation, current_crypto, target_crypto) =
|
|
if let Some(rotation) = active_rotation(®istry, &["running"]).await? {
|
|
let (current_crypto, target_crypto) =
|
|
rotation_crypto_for_resume(®istry, &options, &rotation).await?;
|
|
(rotation, current_crypto, target_crypto)
|
|
} else {
|
|
let preflight = master_key_preflight(®istry, &options).await?;
|
|
let now = OffsetDateTime::now_utc();
|
|
let rotation = registry
|
|
.begin_master_key_rotation(
|
|
preflight.source_epoch,
|
|
preflight.target_epoch,
|
|
preflight.target_crypto.master_key_fingerprint(),
|
|
preflight.backup_ref.as_deref(),
|
|
&now,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
(rotation, preflight.current_crypto, preflight.target_crypto)
|
|
};
|
|
let processed = process_rotation_batch(
|
|
®istry,
|
|
&rotation,
|
|
¤t_crypto,
|
|
&target_crypto,
|
|
options.max_versions,
|
|
)
|
|
.await?;
|
|
let status = if rotation.processed_secret_versions + processed
|
|
>= rotation.total_secret_versions
|
|
{
|
|
let now = OffsetDateTime::now_utc();
|
|
registry
|
|
.finish_master_key_rotation_batches(&rotation.id, &now)
|
|
.await
|
|
.map_err(CliError::from_registry)?
|
|
} else {
|
|
registry
|
|
.master_key_rotation_status()
|
|
.await
|
|
.map_err(CliError::from_registry)?
|
|
.rotations
|
|
.into_iter()
|
|
.find(|candidate| candidate.id == rotation.id)
|
|
.ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_rotation_not_found",
|
|
"master_key.rotation",
|
|
"run_status",
|
|
)
|
|
})?
|
|
};
|
|
println!(
|
|
"{}",
|
|
json!({
|
|
"status": status.state,
|
|
"rotation_id": status.id,
|
|
"source_epoch": status.source_epoch,
|
|
"target_epoch": status.target_epoch,
|
|
"processed_secret_versions": status.processed_secret_versions,
|
|
"total_secret_versions": status.total_secret_versions,
|
|
})
|
|
);
|
|
}
|
|
"verify" => {
|
|
let target_key = read_required_key_file(options.target_key_file.as_deref())?;
|
|
let rotation = active_rotation(®istry, &["verifying"])
|
|
.await?
|
|
.ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_rotation_conflict",
|
|
"master_key.rotation",
|
|
"run_rotate",
|
|
)
|
|
})?;
|
|
let target_crypto = SecretCrypto::with_epoch(&target_key, rotation.target_epoch)
|
|
.map_err(|_| {
|
|
CliError::new(
|
|
"master_key_invalid",
|
|
"master_key.input",
|
|
"verify_operator_input",
|
|
)
|
|
})?;
|
|
if target_crypto.master_key_fingerprint() != rotation.target_fingerprint {
|
|
return Err(CliError::new(
|
|
"master_key_identity_mismatch",
|
|
"master_key.identity",
|
|
"use_matching_target_key",
|
|
));
|
|
}
|
|
let verified = verify_staged_targets(®istry, &rotation, &target_crypto).await?;
|
|
let now = OffsetDateTime::now_utc();
|
|
let status = registry
|
|
.verify_master_key_rotation(&rotation.id, verified, &now)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
println!(
|
|
"{}",
|
|
json!({
|
|
"status": status.state,
|
|
"rotation_id": status.id,
|
|
"verified_secret_versions": status.verified_secret_versions,
|
|
"total_secret_versions": status.total_secret_versions,
|
|
})
|
|
);
|
|
}
|
|
"promote" => {
|
|
let target_key = read_required_key_file(options.target_key_file.as_deref())?;
|
|
let rotation = active_rotation(®istry, &["verified"])
|
|
.await?
|
|
.ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_rotation_conflict",
|
|
"master_key.rotation",
|
|
"run_verify",
|
|
)
|
|
})?;
|
|
let target_crypto = SecretCrypto::with_epoch(&target_key, rotation.target_epoch)
|
|
.map_err(|_| {
|
|
CliError::new(
|
|
"master_key_invalid",
|
|
"master_key.input",
|
|
"verify_operator_input",
|
|
)
|
|
})?;
|
|
if target_crypto.master_key_fingerprint() != rotation.target_fingerprint {
|
|
return Err(CliError::new(
|
|
"master_key_identity_mismatch",
|
|
"master_key.identity",
|
|
"use_matching_target_key",
|
|
));
|
|
}
|
|
verify_staged_targets(®istry, &rotation, &target_crypto).await?;
|
|
let now = OffsetDateTime::now_utc();
|
|
let status = registry
|
|
.promote_master_key_rotation(
|
|
&rotation.id,
|
|
MasterKeyIdentityCandidate {
|
|
epoch: rotation.target_epoch,
|
|
fingerprint: target_crypto.master_key_fingerprint(),
|
|
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &now,
|
|
},
|
|
&now,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
println!(
|
|
"{}",
|
|
json!({
|
|
"status": status.state,
|
|
"rotation_id": status.id,
|
|
"active_epoch": status.target_epoch,
|
|
})
|
|
);
|
|
}
|
|
"abort" => {
|
|
let rotation_id = options
|
|
.rotation_id
|
|
.as_deref()
|
|
.ok_or_else(|| CliError::new("invalid_command", "cli.arguments", "run_status"))?;
|
|
let now = OffsetDateTime::now_utc();
|
|
let status = registry
|
|
.abort_master_key_rotation(rotation_id, &now)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
println!(
|
|
"{}",
|
|
json!({
|
|
"status": status.state,
|
|
"rotation_id": status.id,
|
|
"active_epoch": status.source_epoch,
|
|
})
|
|
);
|
|
}
|
|
_ => {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_master_key_status",
|
|
));
|
|
}
|
|
}
|
|
Ok(ExitCode::SUCCESS)
|
|
}
|
|
#[derive(Default)]
|
|
struct MasterKeyOptions {
|
|
current_key_file: Option<String>,
|
|
target_key_file: Option<String>,
|
|
backup_ref: Option<String>,
|
|
rotation_id: Option<String>,
|
|
max_versions: Option<usize>,
|
|
}
|
|
impl MasterKeyOptions {
|
|
fn parse(arguments: &[String]) -> Result<Self, CliError> {
|
|
let mut options = Self::default();
|
|
let mut index = 0;
|
|
while index < arguments.len() {
|
|
let key = arguments[index].as_str();
|
|
let Some(value) = arguments.get(index + 1) else {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_master_key_status",
|
|
));
|
|
};
|
|
match key {
|
|
"--current-key-file" => options.current_key_file = Some(value.clone()),
|
|
"--target-key-file" => options.target_key_file = Some(value.clone()),
|
|
"--backup-ref" => options.backup_ref = Some(value.clone()),
|
|
"--rotation-id" => options.rotation_id = Some(value.clone()),
|
|
"--max-versions" => {
|
|
let parsed = value.parse::<usize>().map_err(|_| {
|
|
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
|
|
})?;
|
|
if parsed == 0 || parsed > 10_000 {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_master_key_status",
|
|
));
|
|
}
|
|
options.max_versions = Some(parsed);
|
|
}
|
|
_ => {
|
|
return Err(CliError::new(
|
|
"invalid_command",
|
|
"cli.arguments",
|
|
"run_master_key_status",
|
|
));
|
|
}
|
|
}
|
|
index += 2;
|
|
}
|
|
Ok(options)
|
|
}
|
|
}
|
|
|
|
struct MasterKeyPreflight {
|
|
source_epoch: i64,
|
|
target_epoch: i64,
|
|
affected_secret_versions: usize,
|
|
backup_ref: Option<String>,
|
|
current_crypto: SecretCrypto,
|
|
target_crypto: SecretCrypto,
|
|
}
|
|
|
|
async fn master_key_preflight(
|
|
registry: &PostgresRegistry,
|
|
options: &MasterKeyOptions,
|
|
) -> Result<MasterKeyPreflight, CliError> {
|
|
let current_key = read_required_key_file(options.current_key_file.as_deref())?;
|
|
let target_key = read_required_key_file(options.target_key_file.as_deref())?;
|
|
let status = registry
|
|
.master_key_rotation_status()
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
if status.rotations.iter().any(|rotation| {
|
|
matches!(
|
|
rotation.state.as_str(),
|
|
"running" | "verifying" | "verified"
|
|
)
|
|
}) {
|
|
return Err(CliError::new(
|
|
"master_key_rotation_in_progress",
|
|
"master_key.rotation",
|
|
"resume_verify_promote_or_abort_rotation",
|
|
));
|
|
}
|
|
let active = status.active_identity.ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_identity_missing",
|
|
"master_key.identity",
|
|
"start_secret_process_once",
|
|
)
|
|
})?;
|
|
let current_crypto = SecretCrypto::with_epoch(¤t_key, active.epoch).map_err(|_| {
|
|
CliError::new(
|
|
"master_key_invalid",
|
|
"master_key.input",
|
|
"verify_operator_input",
|
|
)
|
|
})?;
|
|
if current_crypto.master_key_fingerprint() != active.fingerprint {
|
|
return Err(CliError::new(
|
|
"master_key_identity_mismatch",
|
|
"master_key.identity",
|
|
"use_matching_current_key",
|
|
));
|
|
}
|
|
let target_epoch = active.epoch + 1;
|
|
let target_crypto = SecretCrypto::with_epoch(&target_key, target_epoch).map_err(|_| {
|
|
CliError::new(
|
|
"master_key_invalid",
|
|
"master_key.input",
|
|
"verify_operator_input",
|
|
)
|
|
})?;
|
|
if target_crypto.master_key_fingerprint() == current_crypto.master_key_fingerprint()
|
|
|| registry
|
|
.master_key_fingerprint_exists(target_crypto.master_key_fingerprint())
|
|
.await
|
|
.map_err(CliError::from_registry)?
|
|
{
|
|
return Err(CliError::new(
|
|
"master_key_rotation_conflict",
|
|
"master_key.rotation",
|
|
"choose_new_target_key",
|
|
));
|
|
}
|
|
let affected_secret_versions =
|
|
count_and_verify_current_versions(registry, active.epoch, ¤t_crypto).await?;
|
|
Ok(MasterKeyPreflight {
|
|
source_epoch: active.epoch,
|
|
target_epoch,
|
|
affected_secret_versions,
|
|
backup_ref: options.backup_ref.clone(),
|
|
current_crypto,
|
|
target_crypto,
|
|
})
|
|
}
|
|
|
|
async fn rotation_crypto_for_resume(
|
|
registry: &PostgresRegistry,
|
|
options: &MasterKeyOptions,
|
|
rotation: &MasterKeyRotationRecord,
|
|
) -> Result<(SecretCrypto, SecretCrypto), CliError> {
|
|
let current_key = read_required_key_file(options.current_key_file.as_deref())?;
|
|
let target_key = read_required_key_file(options.target_key_file.as_deref())?;
|
|
let active = registry
|
|
.active_master_key_identity()
|
|
.await
|
|
.map_err(CliError::from_registry)?
|
|
.ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_identity_missing",
|
|
"master_key.identity",
|
|
"start_secret_process_once",
|
|
)
|
|
})?;
|
|
let current_crypto =
|
|
SecretCrypto::with_epoch(¤t_key, rotation.source_epoch).map_err(|_| {
|
|
CliError::new(
|
|
"master_key_invalid",
|
|
"master_key.input",
|
|
"verify_operator_input",
|
|
)
|
|
})?;
|
|
if active.epoch != rotation.source_epoch
|
|
|| active.fingerprint != current_crypto.master_key_fingerprint()
|
|
{
|
|
return Err(CliError::new(
|
|
"master_key_identity_mismatch",
|
|
"master_key.identity",
|
|
"use_matching_current_key",
|
|
));
|
|
}
|
|
let target_crypto =
|
|
SecretCrypto::with_epoch(&target_key, rotation.target_epoch).map_err(|_| {
|
|
CliError::new(
|
|
"master_key_invalid",
|
|
"master_key.input",
|
|
"verify_operator_input",
|
|
)
|
|
})?;
|
|
if target_crypto.master_key_fingerprint() != rotation.target_fingerprint {
|
|
return Err(CliError::new(
|
|
"master_key_identity_mismatch",
|
|
"master_key.identity",
|
|
"use_matching_target_key",
|
|
));
|
|
}
|
|
Ok((current_crypto, target_crypto))
|
|
}
|
|
|
|
async fn process_rotation_batch(
|
|
registry: &PostgresRegistry,
|
|
rotation: &MasterKeyRotationRecord,
|
|
current_crypto: &SecretCrypto,
|
|
target_crypto: &SecretCrypto,
|
|
max_versions: Option<usize>,
|
|
) -> Result<i64, CliError> {
|
|
let mut processed = 0_i64;
|
|
let mut after_secret_id: Option<String> = None;
|
|
let mut after_version: Option<u32> = None;
|
|
loop {
|
|
let versions = registry
|
|
.list_secret_versions_for_master_key_epoch_page(
|
|
rotation.source_epoch,
|
|
after_secret_id.as_deref(),
|
|
after_version,
|
|
MASTER_KEY_ROTATION_PAGE_SIZE,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
if versions.is_empty() {
|
|
break;
|
|
}
|
|
for version in versions {
|
|
after_secret_id = Some(version.secret_version.secret_id.as_str().to_owned());
|
|
after_version = Some(version.secret_version.version);
|
|
if version.target_master_key_epoch == Some(rotation.target_epoch) {
|
|
continue;
|
|
}
|
|
if max_versions.is_some_and(|limit| processed as usize >= limit) {
|
|
return Ok(processed);
|
|
}
|
|
let plaintext = decrypt_current(&version, current_crypto)?;
|
|
let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
let now = OffsetDateTime::now_utc();
|
|
registry
|
|
.stage_master_key_rotation_ciphertext(
|
|
&rotation.id,
|
|
&version.secret_version.secret_id,
|
|
version.secret_version.version,
|
|
rotation.source_epoch,
|
|
&target_ciphertext,
|
|
target_crypto.key_version(),
|
|
rotation.target_epoch,
|
|
&now,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
processed += 1;
|
|
}
|
|
}
|
|
let mut after_task_id: Option<String> = None;
|
|
loop {
|
|
let tasks = registry
|
|
.list_tasks_for_master_key_epoch_page(
|
|
rotation.source_epoch,
|
|
after_task_id.as_deref(),
|
|
MASTER_KEY_ROTATION_PAGE_SIZE,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
if tasks.is_empty() {
|
|
break;
|
|
}
|
|
for task in tasks {
|
|
after_task_id = Some(task.task_id.clone());
|
|
if task.target_epoch == Some(rotation.target_epoch) {
|
|
continue;
|
|
}
|
|
if max_versions.is_some_and(|limit| processed as usize >= limit) {
|
|
return Ok(processed);
|
|
}
|
|
let plaintext = current_crypto
|
|
.decrypt_for_epoch(&task.key_version, task.epoch, &task.ciphertext)
|
|
.map_err(|_| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
let now = OffsetDateTime::now_utc();
|
|
registry
|
|
.stage_master_key_rotation_task_ciphertext(
|
|
&rotation.id,
|
|
&task.task_id,
|
|
rotation.source_epoch,
|
|
&target_ciphertext,
|
|
target_crypto.key_version(),
|
|
rotation.target_epoch,
|
|
&now,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
processed += 1;
|
|
}
|
|
}
|
|
Ok(processed)
|
|
}
|
|
|
|
async fn verify_staged_targets(
|
|
registry: &PostgresRegistry,
|
|
rotation: &MasterKeyRotationRecord,
|
|
target_crypto: &SecretCrypto,
|
|
) -> Result<i64, CliError> {
|
|
let mut verified = 0_i64;
|
|
let mut after_secret_id: Option<String> = None;
|
|
let mut after_version: Option<u32> = None;
|
|
loop {
|
|
let versions = registry
|
|
.list_target_secret_versions_for_master_key_rotation_page(
|
|
rotation.target_epoch,
|
|
after_secret_id.as_deref(),
|
|
after_version,
|
|
MASTER_KEY_ROTATION_PAGE_SIZE,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
if versions.is_empty() {
|
|
break;
|
|
}
|
|
for version in versions {
|
|
after_secret_id = Some(version.secret_version.secret_id.as_str().to_owned());
|
|
after_version = Some(version.secret_version.version);
|
|
let ciphertext = version.target_ciphertext.as_deref().ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
let key_version = version.target_key_version.as_deref().ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
target_crypto
|
|
.decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext)
|
|
.map_err(|_| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
verified += 1;
|
|
}
|
|
}
|
|
let mut after_task_id: Option<String> = None;
|
|
loop {
|
|
let tasks = registry
|
|
.list_target_tasks_for_master_key_rotation_page(
|
|
rotation.target_epoch,
|
|
after_task_id.as_deref(),
|
|
MASTER_KEY_ROTATION_PAGE_SIZE,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
if tasks.is_empty() {
|
|
break;
|
|
}
|
|
for task in tasks {
|
|
after_task_id = Some(task.task_id.clone());
|
|
let ciphertext = task.target_ciphertext.as_deref().ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
let key_version = task.target_key_version.as_deref().ok_or_else(|| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
target_crypto
|
|
.decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext)
|
|
.map_err(|_| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})?;
|
|
verified += 1;
|
|
}
|
|
}
|
|
if verified != rotation.total_secret_versions {
|
|
return Err(CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
));
|
|
}
|
|
Ok(verified)
|
|
}
|
|
|
|
async fn count_and_verify_current_versions(
|
|
registry: &PostgresRegistry,
|
|
epoch: i64,
|
|
current_crypto: &SecretCrypto,
|
|
) -> Result<usize, CliError> {
|
|
let mut count = 0_usize;
|
|
let mut after_secret_id: Option<String> = None;
|
|
let mut after_version: Option<u32> = None;
|
|
loop {
|
|
let versions = registry
|
|
.list_secret_versions_for_master_key_epoch_page(
|
|
epoch,
|
|
after_secret_id.as_deref(),
|
|
after_version,
|
|
MASTER_KEY_ROTATION_PAGE_SIZE,
|
|
)
|
|
.await
|
|
.map_err(CliError::from_registry)?;
|
|
if versions.is_empty() {
|
|
break;
|
|
}
|
|
for version in versions {
|
|
after_secret_id = Some(version.secret_version.secret_id.as_str().to_owned());
|
|
after_version = Some(version.secret_version.version);
|
|
decrypt_current(&version, current_crypto)?;
|
|
count += 1;
|
|
}
|
|
}
|
|
Ok(count)
|
|
}
|
|
|
|
fn decrypt_current(
|
|
version: &SecretVersionRecord,
|
|
current_crypto: &SecretCrypto,
|
|
) -> Result<serde_json::Value, CliError> {
|
|
current_crypto
|
|
.decrypt_for_epoch(
|
|
&version.secret_version.key_version,
|
|
version.master_key_epoch,
|
|
&version.secret_version.ciphertext,
|
|
)
|
|
.map_err(|_| {
|
|
CliError::new(
|
|
"master_key_rotation_verification_failed",
|
|
"master_key.rotation",
|
|
"rerun_rotation_or_abort",
|
|
)
|
|
})
|
|
}
|