Files
crank/apps/admin-api/src/service/import_export.rs
T

592 lines
22 KiB
Rust

use crank_core::{OperationStatus, Target, WorkspaceId};
use crank_registry::OperationStateExpectation;
use serde::de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor};
use serde_yaml::Value as YamlValue;
use std::fmt;
use tracing::{info, instrument};
use crate::{
error::ApiError,
import_guidance::import_guidance_warnings,
service::{
AdminService, ExportQuery, ImportMode, ImportQuery, ImportResponse,
LegacyYamlOperationDocument, NewVersionPayload, OperationPayload, PortableOperation,
YamlOperationDocument,
},
};
const MAX_YAML_BYTES: usize = 256 * 1024;
const MAX_YAML_LINE_OR_SCALAR_BYTES: usize = 64 * 1024;
const MAX_YAML_DEPTH: usize = 64;
const MAX_YAML_NODES: usize = 20_000;
const MAX_YAML_COLLECTION_ITEMS: usize = 4_096;
impl AdminService {
#[instrument(skip(self), fields(operation_id = %operation_id.as_str(), version = query.version.unwrap_or_default(), mode = ?query.mode))]
pub async fn export_operation(
&self,
workspace_id: &WorkspaceId,
operation_id: &crank_core::OperationId,
query: ExportQuery,
) -> Result<String, ApiError> {
let version = match query.version {
Some(version) => version,
None => {
self.get_operation(workspace_id, operation_id)
.await?
.current_draft_version
}
};
let record = self
.get_operation_version(workspace_id, operation_id, version)
.await?;
let aggregate = self.get_operation(workspace_id, operation_id).await?;
let exportable = record.status == OperationStatus::Published
|| (aggregate.status != OperationStatus::Archived
&& record.status == OperationStatus::Draft
&& aggregate.current_draft_version == version);
if !exportable {
return Err(ApiError::conflict_with_context(
"only the active current Draft or a Published Version can be exported",
serde_json::json!({
"error_code": "operation_invalid_transition",
"version": version
}),
));
}
let operation = PortableOperation::from_registry(&record.snapshot);
reject_credential_material(&operation)?;
let document = YamlOperationDocument {
format_version: "2".to_owned(),
kind: "operation".to_owned(),
operation,
};
serde_yaml::to_string(&document)
.map_err(|_| ApiError::internal("portable yaml serialization failed"))
}
#[instrument(skip(self, yaml_document), fields(mode = ?query.mode))]
pub async fn import_operation(
&self,
workspace_id: &WorkspaceId,
query: ImportQuery,
yaml_document: &str,
if_match: Option<&str>,
) -> Result<ImportResponse, ApiError> {
reject_yaml_anchors_and_aliases(yaml_document)?;
let parsed = parse_bounded_yaml(yaml_document)?;
let format_version = parsed
.as_mapping()
.and_then(|mapping| mapping.get(YamlValue::String("format_version".to_owned())))
.and_then(YamlValue::as_str)
.ok_or_else(yaml_invalid)?;
let (operation, warnings) = match format_version {
"2" => {
let document: YamlOperationDocument =
serde_yaml::from_value(parsed).map_err(|_| yaml_invalid())?;
if document.kind != "operation" || document.format_version != "2" {
return Err(yaml_unsupported());
}
(document.operation, Vec::new())
}
"1" => {
let document: LegacyYamlOperationDocument =
serde_yaml::from_value(parsed).map_err(|_| yaml_invalid())?;
if document.kind != "operation" || document.format_version != "1" {
return Err(yaml_unsupported());
}
let warnings = import_guidance_warnings(&document.operation);
(
PortableOperation::from_registry(&document.operation),
warnings,
)
}
_ => return Err(yaml_unsupported()),
};
reject_credential_material(&operation)?;
let payload = operation.to_payload();
match query.mode {
ImportMode::Create => {
let created = self.create_operation(workspace_id, payload).await?;
Ok(ImportResponse {
operation_id: created.operation_id,
workspace_id: created.workspace_id,
version: created.version,
import_mode: ImportMode::Create,
warnings,
})
}
ImportMode::Upsert => {
if let Some(existing) = self
.find_operation_by_name(workspace_id, &operation.name)
.await?
{
if existing.status == OperationStatus::Archived {
return Err(crank_registry::RegistryError::OperationArchived {
operation_id: existing.id.as_str().to_owned(),
}
.into());
}
let current = self
.get_operation_version(
workspace_id,
&existing.id,
existing.current_draft_version,
)
.await?;
if operation.matches_registry(&current.snapshot) {
let expected_state = OperationStateExpectation {
current_draft_version: existing.current_draft_version,
status: existing.status,
latest_published_version: existing.latest_published_version,
};
self.registry
.verify_operation_state(workspace_id, &existing.id, &expected_state)
.await?;
return Ok(ImportResponse {
operation_id: existing.id.as_str().to_owned(),
workspace_id: workspace_id.as_str().to_owned(),
version: existing.current_draft_version,
import_mode: ImportMode::Upsert,
warnings,
});
}
let current_detail = self.get_operation(workspace_id, &existing.id).await?;
let expected_state = OperationStateExpectation {
current_draft_version: current_detail.current_draft_version,
status: current_detail.status,
latest_published_version: current_detail.latest_published_version,
};
let expected_etag = Self::operation_state_etag(&current_detail);
let provided_etag = if_match.ok_or_else(|| {
ApiError::precondition_required_with_context(
"If-Match is required when YAML upsert changes an existing Operation",
serde_json::json!({
"error_code": "operation_precondition_required",
"current_version": existing.current_draft_version,
"recovery": "reload"
}),
)
})?;
if provided_etag != expected_etag {
return Err(ApiError::conflict_with_context(
"Operation state changed; reload before retrying YAML upsert",
serde_json::json!({
"error_code": "operation_stale_version",
"current_version": existing.current_draft_version,
"recovery": "reload"
}),
));
}
let created = self
.create_version(
workspace_id,
&existing.id,
NewVersionPayload {
operation: payload,
change_note: Some("yaml upsert".to_owned()),
},
&expected_state,
)
.await?;
let response = ImportResponse {
operation_id: created.operation_id,
workspace_id: created.workspace_id,
version: created.version,
import_mode: ImportMode::Upsert,
warnings,
};
info!(
name: "admin.operation.imported",
operation_id = %response.operation_id,
version = response.version,
"operation imported by upsert"
);
Ok(response)
} else {
let created = self.create_operation(workspace_id, payload).await?;
Ok(ImportResponse {
operation_id: created.operation_id,
workspace_id: created.workspace_id,
version: created.version,
import_mode: ImportMode::Upsert,
warnings,
})
}
}
}
}
}
impl PortableOperation {
fn from_registry(operation: &crank_registry::RegistryOperation) -> Self {
Self {
name: operation.name.clone(),
display_name: operation.display_name.clone(),
category: operation.category.clone(),
protocol: operation.protocol,
security_level: operation.security_level,
target: operation.target.clone(),
input_schema: operation.input_schema.clone(),
output_schema: operation.output_schema.clone(),
input_mapping: operation.input_mapping.clone(),
output_mapping: operation.output_mapping.clone(),
execution_config: operation.execution_config.clone(),
tool_description: operation.tool_description.clone(),
}
}
fn to_payload(&self) -> OperationPayload {
OperationPayload {
name: self.name.clone(),
display_name: self.display_name.clone(),
category: self.category.clone(),
protocol: self.protocol,
security_level: self.security_level,
target: self.target.clone(),
input_schema: self.input_schema.clone(),
output_schema: self.output_schema.clone(),
input_mapping: self.input_mapping.clone(),
output_mapping: self.output_mapping.clone(),
execution_config: self.execution_config.clone(),
tool_description: self.tool_description.clone(),
wizard_state: None,
}
}
fn matches_registry(&self, operation: &crank_registry::RegistryOperation) -> bool {
self.name == operation.name
&& self.display_name == operation.display_name
&& self.category == operation.category
&& self.protocol == operation.protocol
&& self.security_level == operation.security_level
&& self.target == operation.target
&& self.input_schema == operation.input_schema
&& self.output_schema == operation.output_schema
&& self.input_mapping == operation.input_mapping
&& self.output_mapping == operation.output_mapping
&& self.execution_config == operation.execution_config
&& self.tool_description == operation.tool_description
}
}
fn parse_bounded_yaml(document: &str) -> Result<YamlValue, ApiError> {
if document.is_empty() {
return Err(yaml_invalid());
}
if document.len() > MAX_YAML_BYTES {
return Err(ApiError::payload_too_large_with_context(
"operation yaml exceeds the 256 KiB limit",
serde_json::json!({ "error_code": "operation_yaml_too_large" }),
));
}
if document
.lines()
.any(|line| line.len() > MAX_YAML_LINE_OR_SCALAR_BYTES)
{
return Err(yaml_unsupported());
}
let separators = document.lines().filter(|line| line.trim() == "---").count();
if separators > 1 || document.lines().any(|line| line.trim() == "...") {
return Err(yaml_unsupported());
}
DuplicateChecked::deserialize(serde_yaml::Deserializer::from_str(document))
.map_err(|_| yaml_invalid())?;
let parsed: YamlValue = serde_yaml::from_str(document).map_err(|_| yaml_invalid())?;
let mut nodes = 0;
validate_yaml_value(&parsed, 0, &mut nodes)?;
Ok(parsed)
}
/// A bounded preflight pass which rejects duplicate mapping keys before
/// `serde_yaml::Value` can collapse them. Values are deliberately discarded:
/// the authoritative typed parse follows only after this structural check.
struct DuplicateChecked;
impl<'de> Deserialize<'de> for DuplicateChecked {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_any(DuplicateCheckedVisitor)
}
}
struct DuplicateCheckedVisitor;
impl<'de> Visitor<'de> for DuplicateCheckedVisitor {
type Value = DuplicateChecked;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a YAML value without duplicate mapping keys")
}
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
where
A: MapAccess<'de>,
{
let mut keys = Vec::<YamlValue>::new();
while let Some(key) = map.next_key::<YamlValue>()? {
if keys.iter().any(|existing| existing == &key) {
return Err(de::Error::custom("duplicate mapping key"));
}
keys.push(key);
map.next_value::<DuplicateChecked>()?;
}
Ok(DuplicateChecked)
}
fn visit_seq<A>(self, mut sequence: A) -> Result<Self::Value, A::Error>
where
A: SeqAccess<'de>,
{
while sequence.next_element::<DuplicateChecked>()?.is_some() {}
Ok(DuplicateChecked)
}
fn visit_bool<E>(self, _: bool) -> Result<Self::Value, E> {
Ok(DuplicateChecked)
}
fn visit_i64<E>(self, _: i64) -> Result<Self::Value, E> {
Ok(DuplicateChecked)
}
fn visit_u64<E>(self, _: u64) -> Result<Self::Value, E> {
Ok(DuplicateChecked)
}
fn visit_f64<E>(self, _: f64) -> Result<Self::Value, E> {
Ok(DuplicateChecked)
}
fn visit_str<E>(self, _: &str) -> Result<Self::Value, E> {
Ok(DuplicateChecked)
}
fn visit_string<E>(self, _: String) -> Result<Self::Value, E> {
Ok(DuplicateChecked)
}
fn visit_none<E>(self) -> Result<Self::Value, E> {
Ok(DuplicateChecked)
}
fn visit_unit<E>(self) -> Result<Self::Value, E> {
Ok(DuplicateChecked)
}
fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
where
D: Deserializer<'de>,
{
DuplicateChecked::deserialize(deserializer)
}
fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
where
D: Deserializer<'de>,
{
DuplicateChecked::deserialize(deserializer)
}
}
fn validate_yaml_value(value: &YamlValue, depth: usize, nodes: &mut usize) -> Result<(), ApiError> {
if depth > MAX_YAML_DEPTH {
return Err(yaml_unsupported());
}
*nodes = nodes.saturating_add(1);
if *nodes > MAX_YAML_NODES {
return Err(yaml_unsupported());
}
match value {
YamlValue::Sequence(values) => {
if values.len() > MAX_YAML_COLLECTION_ITEMS {
return Err(yaml_unsupported());
}
for value in values {
validate_yaml_value(value, depth + 1, nodes)?;
}
}
YamlValue::Mapping(values) => {
if values.len() > MAX_YAML_COLLECTION_ITEMS {
return Err(yaml_unsupported());
}
for (key, value) in values {
validate_yaml_value(key, depth + 1, nodes)?;
validate_yaml_value(value, depth + 1, nodes)?;
}
}
YamlValue::String(value) if value.len() > MAX_YAML_LINE_OR_SCALAR_BYTES => {
return Err(yaml_unsupported());
}
YamlValue::Tagged(_) => return Err(yaml_unsupported()),
_ => {}
}
Ok(())
}
fn reject_credential_material(operation: &PortableOperation) -> Result<(), ApiError> {
let sensitive_name = |name: &str| {
let normalized = name.to_ascii_lowercase();
let segments = normalized.split(['-', '_', '.']).collect::<Vec<_>>();
matches!(
normalized.as_str(),
"authorization" | "proxy-authorization" | "cookie" | "set-cookie"
) || segments
.iter()
.any(|segment| matches!(*segment, "auth" | "token" | "secret" | "cookie"))
|| segments.windows(2).any(|parts| parts == ["api", "key"])
};
let sensitive_value = |value: &str| {
let normalized = value.trim().to_ascii_lowercase();
normalized.starts_with("bearer ")
|| normalized.starts_with("basic ")
|| normalized.starts_with("digest ")
|| normalized.contains("api_key=")
|| normalized.contains("access_token=")
};
let target_headers = match &operation.target {
Target::Rest(target) => &target.static_headers,
};
if target_headers
.iter()
.any(|(name, value)| sensitive_name(name) || sensitive_value(value))
|| operation
.execution_config
.headers
.iter()
.any(|(name, value)| sensitive_name(name) || sensitive_value(value))
{
return Err(ApiError::unprocessable_with_context(
"portable yaml contains credential-bearing headers",
serde_json::json!({ "error_code": "operation_yaml_invalid" }),
));
}
Ok(())
}
fn reject_yaml_anchors_and_aliases(document: &str) -> Result<(), ApiError> {
let mut single_quote = false;
let mut double_quote = false;
let mut escaped = false;
let mut comment = false;
let mut previous = '\n';
let chars = document.chars().collect::<Vec<_>>();
for (index, character) in chars.iter().copied().enumerate() {
if character == '\n' {
comment = false;
previous = character;
continue;
}
if comment {
previous = character;
continue;
}
if double_quote {
if escaped {
escaped = false;
} else if character == '\\' {
escaped = true;
} else if character == '"' {
double_quote = false;
}
previous = character;
continue;
}
if single_quote {
if character == '\'' {
single_quote = false;
}
previous = character;
continue;
}
match character {
'#' => comment = true,
'"' => double_quote = true,
'\'' => single_quote = true,
'&' | '*'
if (previous.is_whitespace() || matches!(previous, ':' | '[' | '{' | ','))
&& chars.get(index + 1).is_some_and(|next| {
next.is_ascii_alphanumeric() || matches!(next, '_' | '-')
}) =>
{
return Err(yaml_unsupported());
}
_ => {}
}
previous = character;
}
Ok(())
}
fn yaml_invalid() -> ApiError {
ApiError::unprocessable_with_context(
"operation yaml is invalid",
serde_json::json!({ "error_code": "operation_yaml_invalid" }),
)
}
fn yaml_unsupported() -> ApiError {
ApiError::unprocessable_with_context(
"operation yaml uses an unsupported construct",
serde_json::json!({ "error_code": "operation_yaml_unsupported" }),
)
}
#[cfg(test)]
mod tests {
use super::parse_bounded_yaml;
use serde_json::Value;
#[test]
fn portable_v2_schema_freezes_the_closed_top_level_contract() {
let schema: Value = serde_json::from_str(include_str!(
"../../../../docs/schemas/operation-export-v2.schema.json"
))
.unwrap();
assert_eq!(schema["properties"]["format_version"]["const"], "2");
assert_eq!(schema["properties"]["kind"]["const"], "operation");
assert_eq!(schema["additionalProperties"], false);
assert_eq!(
schema["$defs"]["portable_operation"]["additionalProperties"],
false
);
assert_eq!(
schema["$defs"]["portable_operation"]["properties"]["security_level"]["enum"],
serde_json::json!(["standard"])
);
for name in [
"rest_target",
"schema",
"mapping_set",
"mapping_rule",
"execution_config",
"tool_description",
] {
assert_eq!(schema["$defs"][name]["additionalProperties"], false);
}
assert_eq!(
schema["$defs"]["portable_operation"]["properties"]["target"]["$ref"],
"#/$defs/rest_target"
);
}
#[test]
fn bounded_yaml_rejects_duplicate_keys_before_typed_parsing() {
let document =
"format_version: '2'\nkind: operation\nkind: attacker_override\noperation: {}\n";
let error = parse_bounded_yaml(document).expect_err("duplicate key must fail closed");
match error {
crate::error::ApiError::Unprocessable { context, .. } => assert_eq!(
context.and_then(|value| value["error_code"].as_str().map(str::to_owned)),
Some("operation_yaml_invalid".to_owned())
),
other => panic!("unexpected error: {other:?}"),
}
}
}