CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable worker приостанавливает Task как input_required до dispatch, если закреплённая схема требует отсутствующее поле; authorized schema-validated update атомарно и с аудитом возобновляет тот же Task ровно один раз. Replay/conflict/TTL безопасны, ожидание удерживает admission permit. Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
382 lines
16 KiB
Rust
382 lines
16 KiB
Rust
//! Application-level durable Task handler and attempt authority (Story 4.3/4.4).
|
|
//!
|
|
//! The handler owns everything that touches product authority: the exact published pin, the typed
|
|
//! Platform API key, the approval policy, the encrypted executable input and the resolved upstream
|
|
//! auth. The runtime owns the single bounded retry loop and the durable attempt checkpoint/resolution;
|
|
//! this module only revalidates and prepares one attempt at a time.
|
|
|
|
use std::time::{Duration, Instant};
|
|
|
|
use async_trait::async_trait;
|
|
use crank_core::{
|
|
AgentId, ExecutionOrigin, MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
|
|
TaskError, TaskLease, TaskOwnerKind, TraceContext, VerifiedMachineCredential, WorkspaceId,
|
|
current_unix_ms,
|
|
};
|
|
use crank_registry::PostgresRegistry;
|
|
use crank_registry::PublishedAgentTool;
|
|
use crank_runtime::{
|
|
ExecutionAuthorization, PrepareAttempt, RuntimeExecutor, RuntimeOperation, SecretCrypto,
|
|
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
|
|
TaskAttemptRefusal, TaskHandler, TaskService, TaskWorkerError,
|
|
};
|
|
use sha2::{Digest, Sha256};
|
|
|
|
use super::TaskApplication;
|
|
use super::auth_resolution::resolve_operation_auth_with;
|
|
use super::tool_resolution::runtime_operation;
|
|
use crate::access::{allows_scope, credential_allows_security_level};
|
|
use crate::catalog::PublishedToolCatalog;
|
|
|
|
/// Server-owned evidence that one published operation deduplicates repeated sends of the same stable
|
|
/// idempotency key.
|
|
///
|
|
/// The descriptor is constructed by the application for an integration whose upstream contract was
|
|
/// actually qualified (key delivery, dedup scope, canonical input binding and a dedup window that
|
|
/// covers the pinned retry budget). It is never deserialized from Task input, so a caller key or
|
|
/// header alone can never make a write retryable. Production wiring passes an empty set until such a
|
|
/// contract exists.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct VerifiedIdempotentWrite {
|
|
pub operation_id: String,
|
|
pub operation_version: u32,
|
|
/// Header through which the stable key is delivered to this exact upstream.
|
|
pub key_header: String,
|
|
/// Dedup window of the upstream contract, which must cover the pinned retry budget.
|
|
pub validity_ms: i64,
|
|
}
|
|
|
|
pub struct TaskWorkerHandler {
|
|
registry: PostgresRegistry,
|
|
catalog: PublishedToolCatalog,
|
|
runtime: RuntimeExecutor,
|
|
secret_crypto: SecretCrypto,
|
|
service: TaskService,
|
|
verified_writes: Vec<VerifiedIdempotentWrite>,
|
|
}
|
|
|
|
impl TaskWorkerHandler {
|
|
/// Production construction from the same application entry, registry and runtime the MCP
|
|
/// process serves requests with. The worker needs no HTTP state, only these authoritative
|
|
/// dependencies.
|
|
pub fn from_application(
|
|
tasks: &TaskApplication,
|
|
registry: PostgresRegistry,
|
|
runtime: RuntimeExecutor,
|
|
secret_crypto: SecretCrypto,
|
|
) -> Self {
|
|
Self {
|
|
registry,
|
|
catalog: tasks.catalog(),
|
|
runtime,
|
|
secret_crypto,
|
|
service: tasks.task_service(),
|
|
verified_writes: Vec::new(),
|
|
}
|
|
}
|
|
|
|
/// Registers server-owned verified-idempotency evidence of qualified integrations.
|
|
pub fn with_verified_writes(mut self, verified_writes: Vec<VerifiedIdempotentWrite>) -> Self {
|
|
self.verified_writes = verified_writes;
|
|
self
|
|
}
|
|
|
|
/// Authoritative pre-dispatch authorization of one leased Task (Story 4.3).
|
|
///
|
|
/// Resolves the exact pinned Published Version in the current publication, repeats the approval
|
|
/// policy and revalidates the typed persisted Platform API key identity against the registry:
|
|
/// the key must still be active, unexpired and bound to this workspace/agent, and its exact
|
|
/// scopes/security level must still authorize the operation. Any denial is a typed error, so the
|
|
/// worker finishes the attempt without dispatch (fail closed before any network use).
|
|
///
|
|
/// An administrator Task has no authoritative Task revalidation API in the registry, so it stays
|
|
/// fail-closed here instead of inventing authority.
|
|
async fn authorize(&self, lease: &TaskLease) -> Result<PublishedAgentTool, TaskError> {
|
|
if lease.record.owner_kind != TaskOwnerKind::Agent {
|
|
return Err(TaskError::invalid(
|
|
"admin task authority revalidation is not implemented",
|
|
));
|
|
}
|
|
let Some(key_id) = lease.record.platform_api_key_id.as_ref() else {
|
|
return Err(TaskError::invalid("task authority reference is missing"));
|
|
};
|
|
|
|
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
|
|
let agent_id = AgentId::new(lease.record.owner_id.as_str());
|
|
let workspace = self
|
|
.registry
|
|
.get_workspace(&workspace_id)
|
|
.await
|
|
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?
|
|
.ok_or_else(|| TaskError::invalid("task workspace no longer exists"))?;
|
|
let agents = self
|
|
.registry
|
|
.list_agents(&workspace_id)
|
|
.await
|
|
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
|
|
let agent = agents
|
|
.into_iter()
|
|
.find(|agent| agent.id.as_str() == lease.record.owner_id.as_str())
|
|
.ok_or_else(|| TaskError::invalid("task agent no longer exists"))?;
|
|
|
|
let catalog = self
|
|
.catalog
|
|
.get_catalog(&workspace.workspace.slug, &agent.slug)
|
|
.await
|
|
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
|
|
let tool = catalog
|
|
.tools
|
|
.into_iter()
|
|
.find(|tool| {
|
|
tool.operation.id.as_str() == lease.record.operation_id.as_str()
|
|
&& tool.operation.version == lease.record.pins.operation_version
|
|
})
|
|
.ok_or_else(|| TaskError::invalid("task published operation pin no longer matches"))?;
|
|
|
|
if tool.workspace_id.as_str() != lease.record.workspace_id.as_str() {
|
|
return Err(TaskError::invalid("task workspace binding mismatch"));
|
|
}
|
|
if tool.agent_id.as_str() != lease.record.owner_id.as_str() {
|
|
return Err(TaskError::invalid("task agent binding mismatch"));
|
|
}
|
|
if tool
|
|
.operation
|
|
.execution_config
|
|
.approval_policy
|
|
.as_ref()
|
|
.is_some_and(|policy| policy.required)
|
|
{
|
|
return Err(TaskError::invalid(
|
|
"missing approval proof for task operation",
|
|
));
|
|
}
|
|
|
|
// Authoritative revalidation of the exact immutable key identity. A revoked, expired,
|
|
// foreign-workspace or foreign-agent key is absent here and never reaches the network.
|
|
let key = self
|
|
.registry
|
|
.revalidate_platform_api_key(&workspace_id, &agent_id, key_id)
|
|
.await
|
|
.map_err(|_| TaskError::unavailable("task authority store unavailable"))?
|
|
.ok_or_else(|| TaskError::invalid("task authority is no longer active"))?;
|
|
let credential = VerifiedMachineCredential {
|
|
machine_access_mode: MachineAccessMode::StaticAgentKey,
|
|
max_security_level: OperationSecurityLevel::Standard,
|
|
scopes: key.api_key.scopes.clone(),
|
|
platform_api_key_id: Some(key.api_key.id.clone()),
|
|
};
|
|
if !allows_scope(&credential.scopes, PlatformApiKeyScope::Write) {
|
|
return Err(TaskError::invalid("task authority scope is insufficient"));
|
|
}
|
|
if !credential_allows_security_level(&credential, tool.operation.security_level) {
|
|
return Err(TaskError::invalid(
|
|
"task authority security level is insufficient",
|
|
));
|
|
}
|
|
Ok(tool)
|
|
}
|
|
|
|
/// Safe-retry classification of one attempt and the stable upstream key digest.
|
|
///
|
|
/// A read is always safe to repeat. A write is only safe to repeat when this process holds
|
|
/// server-owned evidence for the exact published operation and that evidence's dedup window still
|
|
/// covers the accepted retry budget; the presence of a client key or an idempotency header alone
|
|
/// never qualifies.
|
|
fn retry_classification(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &serde_json::Value,
|
|
retry_window_ms: i64,
|
|
) -> (bool, Option<String>) {
|
|
if matches!(
|
|
&operation.target,
|
|
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
|
|
) {
|
|
return (true, None);
|
|
}
|
|
let Some(policy) = operation.execution_config.idempotency.as_ref() else {
|
|
return (false, None);
|
|
};
|
|
let Some(field) = policy
|
|
.input_field
|
|
.as_deref()
|
|
.filter(|field| !field.trim().is_empty())
|
|
else {
|
|
return (false, None);
|
|
};
|
|
let Some(header) = policy
|
|
.header_name
|
|
.as_deref()
|
|
.filter(|value| !value.is_empty())
|
|
else {
|
|
return (false, None);
|
|
};
|
|
let key = match input.get(field) {
|
|
Some(serde_json::Value::String(value)) if !value.trim().is_empty() => value.clone(),
|
|
Some(serde_json::Value::Number(value)) => value.to_string(),
|
|
_ => return (false, None),
|
|
};
|
|
let evidence = self.verified_writes.iter().find(|evidence| {
|
|
evidence.operation_id == operation.operation_id.as_str()
|
|
&& evidence.operation_version == operation.operation_version
|
|
&& evidence.key_header.eq_ignore_ascii_case(header)
|
|
&& evidence.validity_ms >= retry_window_ms
|
|
});
|
|
let Some(evidence) = evidence else {
|
|
return (false, None);
|
|
};
|
|
// Only a digest of the stable key is persisted; the key itself stays in the encrypted input.
|
|
let mut digest = Sha256::new();
|
|
digest.update(evidence.key_header.as_bytes());
|
|
digest.update([0u8]);
|
|
digest.update(key.as_bytes());
|
|
(true, Some(format!("{:x}", digest.finalize())))
|
|
}
|
|
|
|
/// Story 4.5 producer: the pinned operation still lacks a required input item, so the Task must be
|
|
/// suspended before any dispatch. The schema snapshot and its presentation projection are frozen
|
|
/// here; the TTL never exceeds the accepted absolute execution deadline.
|
|
fn input_request(
|
|
&self,
|
|
tool: &PublishedAgentTool,
|
|
lease: &TaskLease,
|
|
missing_path: &str,
|
|
expires_at_unix_ms: i64,
|
|
) -> Result<crank_core::TaskInputRequest, TaskAttemptRefusal> {
|
|
let schema_snapshot = serde_json::to_value(&tool.operation.input_schema).map_err(|_| {
|
|
TaskAttemptRefusal::Failed {
|
|
code: "task_input_schema_unavailable".to_owned(),
|
|
}
|
|
})?;
|
|
let request = crank_core::TaskInputRequest {
|
|
wait_id: format!("wait-{}", uuid::Uuid::now_v7()),
|
|
expected_version: lease.record.version.saturating_add(1),
|
|
schema_revision: tool.operation.version,
|
|
schema_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
|
schema_snapshot,
|
|
projected_schema: crate::manifest::schema_to_json_schema(&tool.operation.input_schema),
|
|
prompt_code: "required_input_missing".to_owned(),
|
|
missing_fields: vec![missing_path.to_owned()],
|
|
expires_at_unix_ms,
|
|
fingerprint_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
|
};
|
|
request.validate().map_err(|_| TaskAttemptRefusal::Failed {
|
|
code: "task_input_request_invalid".to_owned(),
|
|
})?;
|
|
Ok(request)
|
|
}
|
|
|
|
/// Maps a preparation failure onto the attempt refusal vocabulary: a transient authority-store
|
|
/// outage leaves the Task leased for recovery, everything else is a proven pre-dispatch refusal.
|
|
fn refusal(error: TaskError) -> TaskAttemptRefusal {
|
|
match error {
|
|
TaskError::Unavailable { .. } => TaskAttemptRefusal::Stopped,
|
|
_ => TaskAttemptRefusal::Failed {
|
|
code: "task_authority_rejected".to_owned(),
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl TaskHandler for TaskWorkerHandler {
|
|
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
|
self.authorize(lease)
|
|
.await
|
|
.map_err(TaskWorkerError::Store)?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome {
|
|
self.runtime.execute_task_outcome(&context, self).await
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl TaskAttemptAuthority for TaskWorkerHandler {
|
|
async fn prepare_attempt(
|
|
&self,
|
|
lease: &TaskLease,
|
|
) -> Result<PrepareAttempt, TaskAttemptRefusal> {
|
|
let tool = self.authorize(lease).await.map_err(Self::refusal)?;
|
|
let input = self
|
|
.service
|
|
.executable_input(lease)
|
|
.await
|
|
.map_err(Self::refusal)?;
|
|
let operation = runtime_operation(&tool);
|
|
let now_ms = current_unix_ms();
|
|
let deadline_ms = lease.record.pins.execution_deadline_unix_ms;
|
|
if deadline_ms <= now_ms {
|
|
// The accepted absolute deadline already elapsed: no further attempt may be dispatched,
|
|
// and any previous uncertainty was already recorded as such.
|
|
return Err(TaskAttemptRefusal::Failed {
|
|
code: "task_deadline_elapsed".to_owned(),
|
|
});
|
|
}
|
|
// Only a missing required item of the pinned schema becomes a typed input request. Any other
|
|
// schema violation keeps the existing pre-dispatch attempt validation, and a generation that
|
|
// already accepted a continuation can never open a second wait for the same Task.
|
|
if let Some(missing_path) =
|
|
crank_runtime::missing_required_input(&tool.operation.input_schema, &input)
|
|
{
|
|
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
|
|
return Err(TaskAttemptRefusal::Failed {
|
|
code: "task_input_still_incomplete".to_owned(),
|
|
});
|
|
}
|
|
return Ok(PrepareAttempt::InputRequired(self.input_request(
|
|
&tool,
|
|
lease,
|
|
&missing_path,
|
|
deadline_ms,
|
|
)?));
|
|
}
|
|
let deadline = Instant::now() + Duration::from_millis((deadline_ms - now_ms) as u64);
|
|
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
|
|
let agent_id = AgentId::new(lease.record.owner_id.as_str());
|
|
let resolved_auth = resolve_operation_auth_with(
|
|
&self.registry,
|
|
&self.secret_crypto,
|
|
&workspace_id,
|
|
&operation.execution_config,
|
|
)
|
|
.await
|
|
.map_err(|_| TaskAttemptRefusal::Failed {
|
|
code: "task_auth_resolution_failed".to_owned(),
|
|
})?;
|
|
// Continue the parent trace persisted with the accepted Task; every attempt adds its own
|
|
// child `adapter_attempt` span carrying the committed durable attempt ID.
|
|
let trace_context =
|
|
TraceContext::parse(&lease.record.support.traceparent).map_err(|_| {
|
|
TaskAttemptRefusal::Failed {
|
|
code: "task_trace_parent_invalid".to_owned(),
|
|
}
|
|
})?;
|
|
let request_context = crank_runtime::RuntimeRequestContext::new(
|
|
lease.record.support.request_id.clone(),
|
|
trace_context,
|
|
)
|
|
.with_metering_context(
|
|
workspace_id.clone(),
|
|
Some(agent_id.clone()),
|
|
crank_core::InvocationSource::AgentToolCall,
|
|
);
|
|
let (retry_allowed, upstream_key_digest) =
|
|
self.retry_classification(&operation, &input, deadline_ms - now_ms);
|
|
Ok(PrepareAttempt::Ready(Box::new(TaskAttemptPlan {
|
|
workspace_id,
|
|
origin: ExecutionOrigin::AgentSnapshot,
|
|
agent_id: Some(agent_id),
|
|
operation,
|
|
input,
|
|
authorization: ExecutionAuthorization::Authorized,
|
|
resolved_auth,
|
|
request_context,
|
|
deadline,
|
|
retry_allowed,
|
|
upstream_key_digest,
|
|
})))
|
|
}
|
|
}
|