Compare commits
23 Commits
bc03c33387
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 083974c27f | |||
| bbb9394911 | |||
| 2759fde81f | |||
| 527efb510f | |||
| 66b9bfd875 | |||
| 6c5d0c62d0 | |||
| dd5dbba1f6 | |||
| 5818255cf5 | |||
| 343bb99bff | |||
| dc8171d052 | |||
| d92f817e73 | |||
| ad8390e297 | |||
| 466659c506 | |||
| 8375c7bbf0 | |||
| 2ccaba897d | |||
| 619394f509 | |||
| 501e0931c5 | |||
| c3188637b3 | |||
| 32cde544da | |||
| 10641faf43 | |||
| c2556ec236 | |||
| 55209a9bbc | |||
| 6c2a3712d8 |
@@ -25,6 +25,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS=
|
|||||||
CRANK_OUTBOUND_DENIED_HOSTS=
|
CRANK_OUTBOUND_DENIED_HOSTS=
|
||||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
||||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES=
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000
|
||||||
CRANK_ENVIRONMENT=development
|
CRANK_ENVIRONMENT=development
|
||||||
CRANK_LOG_LEVEL=
|
CRANK_LOG_LEVEL=
|
||||||
CRANK_SENTRY_DSN=
|
CRANK_SENTRY_DSN=
|
||||||
|
|||||||
+31
-4
@@ -232,7 +232,7 @@ jobs:
|
|||||||
--required-test 'operations page imports OpenAPI methods as drafts' \
|
--required-test 'operations page imports OpenAPI methods as drafts' \
|
||||||
--required-test 'OpenAPI upload rejects invalid files locally and restores focus after Escape' \
|
--required-test 'OpenAPI upload rejects invalid files locally and restores focus after Escape' \
|
||||||
--required-test 'OpenAPI upload recovers from pagehide and a preview server error' \
|
--required-test 'OpenAPI upload recovers from pagehide and a preview server error' \
|
||||||
--required-test 'OpenAPI upload invalidates active draft creation after language or workspace changes' \
|
--required-test 'OpenAPI apply preserves job authority after language or workspace changes' \
|
||||||
--required-test 'OpenAPI upload only renders the latest selected file and clears reset or close races' \
|
--required-test 'OpenAPI upload only renders the latest selected file and clears reset or close races' \
|
||||||
--required-test 'OpenAPI upload ignores a stale failure and renders only correlation identifiers'
|
--required-test 'OpenAPI upload ignores a stale failure and renders only correlation identifiers'
|
||||||
python3 ../../scripts/validate-capability-run.py \
|
python3 ../../scripts/validate-capability-run.py \
|
||||||
@@ -286,17 +286,44 @@ jobs:
|
|||||||
CRANK_SESSION_SECRET=ci-session-secret
|
CRANK_SESSION_SECRET=ci-session-secret
|
||||||
CRANK_PASSWORD_PEPPER=ci-password-pepper
|
CRANK_PASSWORD_PEPPER=ci-password-pepper
|
||||||
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
|
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
|
||||||
CRANK_BOOTSTRAP_ADMIN_PASSWORD=ci-admin-password
|
|
||||||
CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME=CI Owner
|
CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME=CI Owner
|
||||||
CRANK_BASE_URL=http://127.0.0.1
|
CRANK_BASE_URL=http://127.0.0.1
|
||||||
|
CRANK_ENVIRONMENT=ci
|
||||||
|
CRANK_OUTBOUND_ALLOWED_HOSTS=admin-api
|
||||||
CRANK_PUBLISH_BIND=127.0.0.1
|
CRANK_PUBLISH_BIND=127.0.0.1
|
||||||
CRANK_ADMIN_PUBLISH_PORT=0
|
CRANK_ADMIN_PUBLISH_PORT=0
|
||||||
CRANK_MCP_PUBLISH_PORT=0
|
CRANK_MCP_PUBLISH_PORT=0
|
||||||
CRANK_UI_PUBLISH_PORT=0
|
CRANK_UI_PUBLISH_PORT=0
|
||||||
CRANK_DEMO_SEED=true
|
CRANK_DEMO_SEED=true
|
||||||
EOF
|
EOF
|
||||||
docker compose -f deploy/community/docker-compose.images.yml \
|
compose=(docker compose -f deploy/community/docker-compose.images.yml \
|
||||||
--env-file .tmp/community-smoke.env --profile local-db up -d --wait
|
--env-file .tmp/community-smoke.env --profile local-db)
|
||||||
|
"${compose[@]}" up -d --wait postgres
|
||||||
|
"${compose[@]}" run --rm migrate
|
||||||
|
bootstrap_json="$("${compose[@]}" run --rm --no-deps \
|
||||||
|
--entrypoint crank-migrate migrate admin-auth bootstrap-create \
|
||||||
|
--email owner@crank.test --display-name 'CI Owner')"
|
||||||
|
bootstrap_token="$(python3 -c \
|
||||||
|
'import json,sys; print(json.loads(sys.stdin.read())["bootstrap_token"])' \
|
||||||
|
<<<"$bootstrap_json")"
|
||||||
|
install -d -m 700 .tmp/community-bootstrap
|
||||||
|
printf '%s' "$bootstrap_token" > .tmp/community-bootstrap/token
|
||||||
|
printf '%s' 'ci-admin-password' > .tmp/community-bootstrap/password
|
||||||
|
printf '%s' 'ci-password-pepper' > .tmp/community-bootstrap/password-pepper
|
||||||
|
chmod 600 .tmp/community-bootstrap/token \
|
||||||
|
.tmp/community-bootstrap/password \
|
||||||
|
.tmp/community-bootstrap/password-pepper
|
||||||
|
"${compose[@]}" run --rm --no-deps \
|
||||||
|
-v "$PWD/.tmp/community-bootstrap:/run/bootstrap:ro" \
|
||||||
|
--entrypoint crank-migrate migrate admin-auth bootstrap-complete \
|
||||||
|
--token-file /run/bootstrap/token \
|
||||||
|
--password-file /run/bootstrap/password \
|
||||||
|
--password-pepper-file /run/bootstrap/password-pepper
|
||||||
|
rm -f .tmp/community-bootstrap/token \
|
||||||
|
.tmp/community-bootstrap/password \
|
||||||
|
.tmp/community-bootstrap/password-pepper
|
||||||
|
rmdir .tmp/community-bootstrap
|
||||||
|
"${compose[@]}" up -d --wait
|
||||||
|
|
||||||
- name: Run authenticated Community image smoke
|
- name: Run authenticated Community image smoke
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -139,7 +139,7 @@ jobs:
|
|||||||
--required-test 'operations page imports OpenAPI methods as drafts' \
|
--required-test 'operations page imports OpenAPI methods as drafts' \
|
||||||
--required-test 'OpenAPI upload rejects invalid files locally and restores focus after Escape' \
|
--required-test 'OpenAPI upload rejects invalid files locally and restores focus after Escape' \
|
||||||
--required-test 'OpenAPI upload recovers from pagehide and a preview server error' \
|
--required-test 'OpenAPI upload recovers from pagehide and a preview server error' \
|
||||||
--required-test 'OpenAPI upload invalidates active draft creation after language or workspace changes' \
|
--required-test 'OpenAPI apply preserves job authority after language or workspace changes' \
|
||||||
--required-test 'OpenAPI upload only renders the latest selected file and clears reset or close races' \
|
--required-test 'OpenAPI upload only renders the latest selected file and clears reset or close races' \
|
||||||
--required-test 'OpenAPI upload ignores a stale failure and renders only correlation identifiers'
|
--required-test 'OpenAPI upload ignores a stale failure and renders only correlation identifiers'
|
||||||
python3 ../../scripts/validate-capability-run.py \
|
python3 ../../scripts/validate-capability-run.py \
|
||||||
@@ -217,8 +217,8 @@ jobs:
|
|||||||
CRANK_SESSION_SECRET=release-smoke-session
|
CRANK_SESSION_SECRET=release-smoke-session
|
||||||
CRANK_PASSWORD_PEPPER=release-smoke-pepper
|
CRANK_PASSWORD_PEPPER=release-smoke-pepper
|
||||||
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
|
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
|
||||||
CRANK_BOOTSTRAP_ADMIN_PASSWORD=release-smoke-password
|
|
||||||
CRANK_BASE_URL=http://127.0.0.1
|
CRANK_BASE_URL=http://127.0.0.1
|
||||||
|
CRANK_ENVIRONMENT=release-smoke
|
||||||
CRANK_PUBLISH_BIND=127.0.0.1
|
CRANK_PUBLISH_BIND=127.0.0.1
|
||||||
CRANK_ADMIN_PUBLISH_PORT=0
|
CRANK_ADMIN_PUBLISH_PORT=0
|
||||||
CRANK_MCP_PUBLISH_PORT=0
|
CRANK_MCP_PUBLISH_PORT=0
|
||||||
|
|||||||
@@ -16,17 +16,17 @@ RUN --mount=type=cache,id=crank-admin-cargo-registry,target=/usr/local/cargo/reg
|
|||||||
|
|
||||||
FROM debian:bookworm-slim
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
RUN apt-get update \
|
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||||
&& apt-get install -y --no-install-recommends ca-certificates curl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY --from=builder /tmp/admin-api /usr/local/bin/admin-api
|
COPY --from=builder /tmp/admin-api /usr/local/bin/admin-api
|
||||||
COPY --from=builder /tmp/crank-migrate /usr/local/bin/crank-migrate
|
COPY --from=builder /tmp/crank-migrate /usr/local/bin/crank-migrate
|
||||||
COPY apps/admin-api/docker-entrypoint.sh /usr/local/bin/crank-admin-entrypoint
|
COPY apps/admin-api/docker-entrypoint.sh /usr/local/bin/crank-admin-entrypoint
|
||||||
|
COPY scripts/docker-http-healthcheck.sh /usr/local/bin/crank-http-healthcheck
|
||||||
|
|
||||||
RUN chmod 0755 /usr/local/bin/crank-admin-entrypoint
|
RUN test -s /etc/ssl/certs/ca-certificates.crt \
|
||||||
|
&& chmod 0755 /usr/local/bin/crank-admin-entrypoint /usr/local/bin/crank-http-healthcheck
|
||||||
|
|
||||||
ENV CRANK_ADMIN_BIND=0.0.0.0:3001
|
ENV CRANK_ADMIN_BIND=0.0.0.0:3001
|
||||||
ENV CRANK_STORAGE_ROOT=/var/lib/crank/storage
|
ENV CRANK_STORAGE_ROOT=/var/lib/crank/storage
|
||||||
|
|||||||
@@ -227,6 +227,7 @@ async fn run(
|
|||||||
.with_artifact_store(artifact_store.clone())
|
.with_artifact_store(artifact_store.clone())
|
||||||
.with_public_base_url(base_url)
|
.with_public_base_url(base_url)
|
||||||
.with_outbound_http_policy(outbound_http_policy)
|
.with_outbound_http_policy(outbound_http_policy)
|
||||||
|
.with_external_reference_import(&config.external_references)?
|
||||||
.with_identity_provider(std::sync::Arc::new(identity_provider))
|
.with_identity_provider(std::sync::Arc::new(identity_provider))
|
||||||
.build();
|
.build();
|
||||||
if config.demo_seed {
|
if config.demo_seed {
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use axum::{
|
|||||||
};
|
};
|
||||||
use crank_core::{CorrelationContext, RequestId, TraceContext};
|
use crank_core::{CorrelationContext, RequestId, TraceContext};
|
||||||
use crank_metrics::ExemplarTraceId;
|
use crank_metrics::ExemplarTraceId;
|
||||||
use crank_observability::{set_remote_trace_parent, with_request_correlation};
|
use crank_observability::with_request_correlation;
|
||||||
use tracing::{Instrument, info, info_span};
|
use tracing::{Instrument, info, info_span};
|
||||||
|
|
||||||
pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
|
pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
|
||||||
@@ -124,11 +124,7 @@ fn one_auxiliary_header_within_budget(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn set_canonical_parent(span: &tracing::Span, context: &TraceContext) {
|
fn set_canonical_parent(span: &tracing::Span, context: &TraceContext) {
|
||||||
let mut headers = axum::http::HeaderMap::new();
|
crank_trace::set_parent_from_trace_context(span, context);
|
||||||
if let Ok(value) = HeaderValue::from_str(context.traceparent()) {
|
|
||||||
headers.insert("traceparent", value);
|
|
||||||
set_remote_trace_parent(span, &headers);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||||
use crank_artifacts::ArtifactStore;
|
use crank_artifacts::ArtifactStore;
|
||||||
|
use crank_config::ExternalReferenceSettings;
|
||||||
use crank_core::{
|
use crank_core::{
|
||||||
AuditActor, AuditEvent, AuditEventId, AuditSink, AuditTarget, AuditTargetKind, AuthProfile,
|
AuditActor, AuditEvent, AuditEventId, AuditSink, AuditTarget, AuditTargetKind, AuthProfile,
|
||||||
CapabilityProfile, CommunityCapabilityProfile, CorrelationContext, EditionCapabilities,
|
CapabilityProfile, CommunityCapabilityProfile, CorrelationContext, EditionCapabilities,
|
||||||
@@ -19,7 +20,8 @@ use crank_registry::{
|
|||||||
UsageBucket,
|
UsageBucket,
|
||||||
};
|
};
|
||||||
use crank_runtime::{
|
use crank_runtime::{
|
||||||
OutboundHttpPolicy, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
|
ExternalReferenceFetcher, OutboundHttpPolicy, ResolvedAuth, RuntimeError, RuntimeExecutor,
|
||||||
|
SecretCrypto,
|
||||||
};
|
};
|
||||||
use crank_schema::{Schema, SchemaKind};
|
use crank_schema::{Schema, SchemaKind};
|
||||||
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
|
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
|
||||||
@@ -68,6 +70,9 @@ pub struct AdminService {
|
|||||||
audit_sink: Arc<dyn AuditSink>,
|
audit_sink: Arc<dyn AuditSink>,
|
||||||
capability_profile: Arc<dyn CapabilityProfile>,
|
capability_profile: Arc<dyn CapabilityProfile>,
|
||||||
outbound_http_policy: OutboundHttpPolicy,
|
outbound_http_policy: OutboundHttpPolicy,
|
||||||
|
external_reference_fetcher: ExternalReferenceFetcher,
|
||||||
|
external_reference_normalization: crank_import::rest::NormalizationConfig,
|
||||||
|
external_reference_materialization_timeout: std::time::Duration,
|
||||||
public_base_url: String,
|
public_base_url: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,6 +88,9 @@ pub struct AdminServiceBuilder {
|
|||||||
audit_sink: Option<Arc<dyn AuditSink>>,
|
audit_sink: Option<Arc<dyn AuditSink>>,
|
||||||
capability_profile: Option<Arc<dyn CapabilityProfile>>,
|
capability_profile: Option<Arc<dyn CapabilityProfile>>,
|
||||||
outbound_http_policy: OutboundHttpPolicy,
|
outbound_http_policy: OutboundHttpPolicy,
|
||||||
|
external_reference_fetcher: ExternalReferenceFetcher,
|
||||||
|
external_reference_normalization: crank_import::rest::NormalizationConfig,
|
||||||
|
external_reference_settings: ExternalReferenceSettings,
|
||||||
public_base_url: String,
|
public_base_url: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,6 +228,22 @@ impl AdminServiceBuilder {
|
|||||||
secret_crypto: SecretCrypto,
|
secret_crypto: SecretCrypto,
|
||||||
runtime: RuntimeExecutor,
|
runtime: RuntimeExecutor,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
|
let outbound_http_policy = OutboundHttpPolicy::default();
|
||||||
|
let external_reference_settings = ExternalReferenceSettings {
|
||||||
|
allowed_url_prefixes: Vec::new(),
|
||||||
|
max_depth: 8,
|
||||||
|
max_documents: 32,
|
||||||
|
max_fetch_bytes: 256 * 1024,
|
||||||
|
fetch_timeout_ms: 5_000,
|
||||||
|
max_expanded_nodes: 10_000,
|
||||||
|
};
|
||||||
|
let external_reference_fetcher = ExternalReferenceFetcher::try_new(
|
||||||
|
outbound_http_policy.clone(),
|
||||||
|
external_reference_settings.allowed_url_prefixes.clone(),
|
||||||
|
external_reference_settings.max_fetch_bytes,
|
||||||
|
std::time::Duration::from_millis(external_reference_settings.fetch_timeout_ms),
|
||||||
|
)
|
||||||
|
.expect("static external reference defaults are valid");
|
||||||
Self {
|
Self {
|
||||||
registry,
|
registry,
|
||||||
storage_root,
|
storage_root,
|
||||||
@@ -231,7 +255,10 @@ impl AdminServiceBuilder {
|
|||||||
policy_engine: None,
|
policy_engine: None,
|
||||||
audit_sink: None,
|
audit_sink: None,
|
||||||
capability_profile: None,
|
capability_profile: None,
|
||||||
outbound_http_policy: OutboundHttpPolicy::default(),
|
outbound_http_policy,
|
||||||
|
external_reference_fetcher,
|
||||||
|
external_reference_normalization: Default::default(),
|
||||||
|
external_reference_settings,
|
||||||
public_base_url: "http://localhost:3000".to_owned(),
|
public_base_url: "http://localhost:3000".to_owned(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -242,10 +269,41 @@ impl AdminServiceBuilder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn with_outbound_http_policy(mut self, policy: OutboundHttpPolicy) -> Self {
|
pub fn with_outbound_http_policy(mut self, policy: OutboundHttpPolicy) -> Self {
|
||||||
|
self.external_reference_fetcher = ExternalReferenceFetcher::try_new(
|
||||||
|
policy.clone(),
|
||||||
|
self.external_reference_settings
|
||||||
|
.allowed_url_prefixes
|
||||||
|
.clone(),
|
||||||
|
self.external_reference_settings.max_fetch_bytes,
|
||||||
|
std::time::Duration::from_millis(self.external_reference_settings.fetch_timeout_ms),
|
||||||
|
)
|
||||||
|
.expect("validated external reference settings remain valid");
|
||||||
self.outbound_http_policy = policy;
|
self.outbound_http_policy = policy;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn with_external_reference_import(
|
||||||
|
mut self,
|
||||||
|
settings: &ExternalReferenceSettings,
|
||||||
|
) -> Result<Self, crank_runtime::ExternalReferenceFetchError> {
|
||||||
|
self.external_reference_fetcher = ExternalReferenceFetcher::try_new(
|
||||||
|
self.outbound_http_policy.clone(),
|
||||||
|
settings.allowed_url_prefixes.clone(),
|
||||||
|
settings.max_fetch_bytes,
|
||||||
|
std::time::Duration::from_millis(settings.fetch_timeout_ms),
|
||||||
|
)?;
|
||||||
|
self.external_reference_normalization.max_reference_depth = settings.max_depth;
|
||||||
|
self.external_reference_normalization
|
||||||
|
.max_reference_documents = settings.max_documents;
|
||||||
|
self.external_reference_normalization
|
||||||
|
.max_external_document_bytes = settings.max_fetch_bytes;
|
||||||
|
self.external_reference_normalization.max_expanded_nodes = settings.max_expanded_nodes;
|
||||||
|
self.external_reference_normalization
|
||||||
|
.external_references_enabled = !settings.allowed_url_prefixes.is_empty();
|
||||||
|
self.external_reference_settings = settings.clone();
|
||||||
|
Ok(self)
|
||||||
|
}
|
||||||
|
|
||||||
/// Reuses the process-wide immutable artifact authority for OpenAPI
|
/// Reuses the process-wide immutable artifact authority for OpenAPI
|
||||||
/// ingress and reconciliation. Tests may omit this and use their private
|
/// ingress and reconciliation. Tests may omit this and use their private
|
||||||
/// storage root instead.
|
/// storage root instead.
|
||||||
@@ -299,6 +357,11 @@ impl AdminServiceBuilder {
|
|||||||
.capability_profile
|
.capability_profile
|
||||||
.unwrap_or_else(|| Arc::new(CommunityCapabilityProfile)),
|
.unwrap_or_else(|| Arc::new(CommunityCapabilityProfile)),
|
||||||
outbound_http_policy: self.outbound_http_policy,
|
outbound_http_policy: self.outbound_http_policy,
|
||||||
|
external_reference_fetcher: self.external_reference_fetcher,
|
||||||
|
external_reference_normalization: self.external_reference_normalization,
|
||||||
|
external_reference_materialization_timeout: std::time::Duration::from_millis(
|
||||||
|
self.external_reference_settings.fetch_timeout_ms,
|
||||||
|
),
|
||||||
public_base_url: self.public_base_url,
|
public_base_url: self.public_base_url,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,11 +6,12 @@ use crank_core::{
|
|||||||
WorkspaceId,
|
WorkspaceId,
|
||||||
};
|
};
|
||||||
use crank_import::rest::{
|
use crank_import::rest::{
|
||||||
ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate,
|
ImportFinding, ImportFindingSeverity, ImportOperationCandidate, NORMALIZER_VERSION,
|
||||||
|
PROJECTION_VERSION, operation_draft_from_candidate,
|
||||||
};
|
};
|
||||||
use crank_registry::{
|
use crank_registry::{
|
||||||
ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity,
|
ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity,
|
||||||
CreateArtifactSourceRequest, CreateImportJobRequest, DetachArtifactSourceRequest,
|
CreateArtifactSourceRequest, CreateImportJobRequest, FinishImportJobRequest,
|
||||||
ImportConflictMode, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope,
|
ImportConflictMode, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope,
|
||||||
ImportJobStatus, ImportOperationDraft, RegistryError,
|
ImportJobStatus, ImportOperationDraft, RegistryError,
|
||||||
};
|
};
|
||||||
@@ -28,6 +29,13 @@ use crate::{
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod external_references;
|
||||||
|
mod job_contract;
|
||||||
|
use external_references::ImportReplayContext;
|
||||||
|
use job_contract::{
|
||||||
|
SourceDetachGuard, import_job_dependencies, import_job_normalization_config, import_job_source,
|
||||||
|
};
|
||||||
|
|
||||||
const IMPORT_JOB_TTL_HOURS: i64 = 24;
|
const IMPORT_JOB_TTL_HOURS: i64 = 24;
|
||||||
const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30);
|
const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30);
|
||||||
const OPENAPI_PARSE_CONCURRENCY: usize = 4;
|
const OPENAPI_PARSE_CONCURRENCY: usize = 4;
|
||||||
@@ -94,18 +102,33 @@ impl AdminService {
|
|||||||
detach_guard.detach_now().await;
|
detach_guard.detach_now().await;
|
||||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||||
}
|
}
|
||||||
let preview = match parse_verified_preview(verified.bytes, locale).await {
|
let mut dependencies = self
|
||||||
|
.materialize_external_reference_snapshots(workspace_id, &verified.bytes, now)
|
||||||
|
.await;
|
||||||
|
let parsed = match parse_verified_preview(
|
||||||
|
verified.bytes,
|
||||||
|
artifact.artifact_ref().digest_hex().to_owned(),
|
||||||
|
dependencies.snapshots.clone(),
|
||||||
|
self.external_reference_normalization.clone(),
|
||||||
|
locale,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(preview) => preview,
|
Ok(preview) => preview,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
|
dependencies.detach_all().await;
|
||||||
detach_guard.detach_now().await;
|
detach_guard.detach_now().await;
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if preview
|
if parsed
|
||||||
|
.preview
|
||||||
.groups
|
.groups
|
||||||
.iter()
|
.iter()
|
||||||
.all(|group| group.operations.is_empty())
|
.all(|group| group.operations.is_empty())
|
||||||
{
|
{
|
||||||
|
dependencies.detach_all().await;
|
||||||
detach_guard.detach_now().await;
|
detach_guard.detach_now().await;
|
||||||
return Err(ApiError::openapi_upload(locale, "no_methods"));
|
return Err(ApiError::openapi_upload(locale, "no_methods"));
|
||||||
}
|
}
|
||||||
@@ -113,7 +136,7 @@ impl AdminService {
|
|||||||
source_id,
|
source_id,
|
||||||
digest: artifact.artifact_ref().clone(),
|
digest: artifact.artifact_ref().clone(),
|
||||||
};
|
};
|
||||||
let preview_value = serde_json::to_value(&preview)
|
let preview_value = serde_json::to_value(&parsed.preview)
|
||||||
.map_err(|error| ApiError::internal(error.to_string()))?;
|
.map_err(|error| ApiError::internal(error.to_string()))?;
|
||||||
let preview_digest = preview_digest(&preview_value)?;
|
let preview_digest = preview_digest(&preview_value)?;
|
||||||
let preview_payload = json!({
|
let preview_payload = json!({
|
||||||
@@ -121,8 +144,16 @@ impl AdminService {
|
|||||||
"source_id": source_envelope.source_id.as_str(),
|
"source_id": source_envelope.source_id.as_str(),
|
||||||
"digest": source_envelope.digest.as_str(),
|
"digest": source_envelope.digest.as_str(),
|
||||||
},
|
},
|
||||||
|
"dependencies": dependencies.payload(),
|
||||||
|
"dependency_snapshots": dependencies.snapshot_payload(),
|
||||||
"preview": preview_value,
|
"preview": preview_value,
|
||||||
"preview_digest": preview_digest,
|
"preview_digest": preview_digest,
|
||||||
|
"normalization": {
|
||||||
|
"normalizer_version": NORMALIZER_VERSION,
|
||||||
|
"projection_version": PROJECTION_VERSION,
|
||||||
|
"ir_fingerprint": parsed.ir_fingerprint,
|
||||||
|
"config": self.external_reference_normalization,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if let Err(error) = self
|
if let Err(error) = self
|
||||||
@@ -131,8 +162,8 @@ impl AdminService {
|
|||||||
id: &job_id,
|
id: &job_id,
|
||||||
workspace_id,
|
workspace_id,
|
||||||
kind: ImportJobKind::OpenApi,
|
kind: ImportJobKind::OpenApi,
|
||||||
source_format: &preview.source.format,
|
source_format: &parsed.preview.source.format,
|
||||||
source_version: preview.source.version.as_deref(),
|
source_version: parsed.preview.source.version.as_deref(),
|
||||||
status: ImportJobStatus::Pending,
|
status: ImportJobStatus::Pending,
|
||||||
source: &source_envelope,
|
source: &source_envelope,
|
||||||
preview_payload: &preview_payload,
|
preview_payload: &preview_payload,
|
||||||
@@ -141,9 +172,11 @@ impl AdminService {
|
|||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
dependencies.detach_all().await;
|
||||||
detach_guard.detach_now().await;
|
detach_guard.detach_now().await;
|
||||||
return Err(ApiError::from(error));
|
return Err(ApiError::from(error));
|
||||||
}
|
}
|
||||||
|
dependencies.disarm();
|
||||||
detach_guard.disarm();
|
detach_guard.disarm();
|
||||||
|
|
||||||
Ok(OpenApiImportPreviewResponse {
|
Ok(OpenApiImportPreviewResponse {
|
||||||
@@ -151,7 +184,7 @@ impl AdminService {
|
|||||||
expires_at: expires_at
|
expires_at: expires_at
|
||||||
.format(&Rfc3339)
|
.format(&Rfc3339)
|
||||||
.map_err(|error| ApiError::internal(error.to_string()))?,
|
.map_err(|error| ApiError::internal(error.to_string()))?,
|
||||||
preview,
|
preview: parsed.preview,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,6 +253,13 @@ impl AdminService {
|
|||||||
} else {
|
} else {
|
||||||
ImportConflictMode::Rename
|
ImportConflictMode::Rename
|
||||||
};
|
};
|
||||||
|
let replay_context = ImportReplayContext {
|
||||||
|
workspace_id,
|
||||||
|
job_id,
|
||||||
|
application_key: &application_key,
|
||||||
|
conflict_mode,
|
||||||
|
finished_at: &finished_at,
|
||||||
|
};
|
||||||
if job.status == ImportJobStatus::Completed {
|
if job.status == ImportJobStatus::Completed {
|
||||||
let applied = self
|
let applied = self
|
||||||
.registry
|
.registry
|
||||||
@@ -236,7 +276,18 @@ impl AdminService {
|
|||||||
return Ok(openapi_import_response(applied));
|
return Ok(openapi_import_response(applied));
|
||||||
}
|
}
|
||||||
|
|
||||||
let source = import_job_source(&job.preview_payload, locale)?;
|
let source = match import_job_source(&job.preview_payload, locale) {
|
||||||
|
Ok(source) => source,
|
||||||
|
Err(error) => {
|
||||||
|
return self
|
||||||
|
.fail_openapi_import_or_replay(
|
||||||
|
&replay_context,
|
||||||
|
"import_source_verification_failed",
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
};
|
||||||
let verified = match self
|
let verified = match self
|
||||||
.registry
|
.registry
|
||||||
.read_artifact_source(
|
.read_artifact_source(
|
||||||
@@ -250,37 +301,126 @@ impl AdminService {
|
|||||||
Err(
|
Err(
|
||||||
error @ (RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable),
|
error @ (RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable),
|
||||||
) => {
|
) => {
|
||||||
// Another request may have finished and detached the source
|
return self
|
||||||
// after our unlocked Pending read. Re-read the job and ask
|
.fail_openapi_import_or_replay(
|
||||||
// the registry for its canonical, locked replay instead of
|
&replay_context,
|
||||||
// exposing a false `source_unavailable` outcome.
|
"import_source_verification_failed",
|
||||||
let latest = self.registry.get_import_job(workspace_id, job_id).await?;
|
openapi_source_error(locale, error),
|
||||||
if latest.is_some_and(|latest| latest.status == ImportJobStatus::Completed) {
|
)
|
||||||
let applied = self
|
.await;
|
||||||
.registry
|
}
|
||||||
.apply_import_job(ApplyImportJobRequest {
|
Err(error @ RegistryError::SourceIntegrity) => {
|
||||||
id: job_id,
|
return self
|
||||||
workspace_id,
|
.fail_openapi_import_or_replay(
|
||||||
application_key: &application_key,
|
&replay_context,
|
||||||
conflict_mode,
|
"import_source_verification_failed",
|
||||||
operations: &[],
|
openapi_source_error(locale, error),
|
||||||
pre_skipped: &[],
|
)
|
||||||
finished_at: &finished_at,
|
.await;
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
return Ok(openapi_import_response(applied));
|
|
||||||
}
|
|
||||||
return Err(openapi_source_error(locale, error));
|
|
||||||
}
|
}
|
||||||
Err(error) => return Err(openapi_source_error(locale, error)),
|
Err(error) => return Err(openapi_source_error(locale, error)),
|
||||||
};
|
};
|
||||||
if verified.source.blob.artifact_ref != source.digest {
|
if verified.source.blob.artifact_ref != source.digest {
|
||||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
return self
|
||||||
|
.fail_openapi_import_or_replay(
|
||||||
|
&replay_context,
|
||||||
|
"import_source_verification_failed",
|
||||||
|
ApiError::openapi_upload(locale, "source_integrity"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
let replay_contract = match import_replay_contract(&job.preview_payload, locale) {
|
||||||
|
Ok(contract) => contract,
|
||||||
|
Err(error) => {
|
||||||
|
return self
|
||||||
|
.fail_openapi_import_or_replay(
|
||||||
|
&replay_context,
|
||||||
|
"import_replay_verification_failed",
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let parsed = match replay_contract {
|
||||||
|
ImportReplayContract::PersistedV2(parsed) => parsed,
|
||||||
|
ImportReplayContract::LegacyV1 | ImportReplayContract::Current => {
|
||||||
|
let legacy_v1 = matches!(replay_contract, ImportReplayContract::LegacyV1);
|
||||||
|
let dependency_snapshots = match self
|
||||||
|
.read_import_job_dependencies(workspace_id, &job.preview_payload, locale)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(snapshots) => snapshots,
|
||||||
|
Err(error) => {
|
||||||
|
return self
|
||||||
|
.fail_openapi_import_or_replay(
|
||||||
|
&replay_context,
|
||||||
|
"import_dependency_verification_failed",
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let normalization_config = match import_job_normalization_config(
|
||||||
|
&job.preview_payload,
|
||||||
|
&self.external_reference_normalization,
|
||||||
|
legacy_v1,
|
||||||
|
locale,
|
||||||
|
) {
|
||||||
|
Ok(config) => config,
|
||||||
|
Err(error) => {
|
||||||
|
return self
|
||||||
|
.fail_openapi_import_or_replay(
|
||||||
|
&replay_context,
|
||||||
|
"import_replay_verification_failed",
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let parsed = match parse_verified_preview(
|
||||||
|
verified.bytes,
|
||||||
|
source.digest.digest_hex().to_owned(),
|
||||||
|
dependency_snapshots,
|
||||||
|
normalization_config,
|
||||||
|
locale,
|
||||||
|
legacy_v1,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(parsed) => parsed,
|
||||||
|
Err(error) => {
|
||||||
|
return self
|
||||||
|
.fail_openapi_import_or_replay(
|
||||||
|
&replay_context,
|
||||||
|
"import_replay_verification_failed",
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Err(error) = verify_preview_contract(&job.preview_payload, &parsed, locale) {
|
||||||
|
return self
|
||||||
|
.fail_openapi_import_or_replay(
|
||||||
|
&replay_context,
|
||||||
|
"import_replay_verification_failed",
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
parsed
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if preview_has_blocker(&parsed.preview, &selected) {
|
||||||
|
return self
|
||||||
|
.fail_openapi_import_or_replay(
|
||||||
|
&replay_context,
|
||||||
|
"reference_resolution_blocked",
|
||||||
|
ApiError::openapi_upload(locale, "invalid_document"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
let preview = parse_verified_preview(verified.bytes, locale).await?;
|
|
||||||
verify_preview_contract(&job.preview_payload, &preview, locale)?;
|
|
||||||
let mut candidates = BTreeMap::new();
|
let mut candidates = BTreeMap::new();
|
||||||
for group in &preview.groups {
|
for group in &parsed.preview.groups {
|
||||||
for operation in &group.operations {
|
for operation in &group.operations {
|
||||||
candidates.insert(operation.key.clone(), operation);
|
candidates.insert(operation.key.clone(), operation);
|
||||||
}
|
}
|
||||||
@@ -348,6 +488,24 @@ impl AdminService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn canonical_external_document_uri(base: Option<&str>, reference: &str) -> Option<String> {
|
||||||
|
if reference.starts_with('#') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut url = match base {
|
||||||
|
Some(base) => url::Url::parse(base).ok()?.join(reference).ok()?,
|
||||||
|
None => url::Url::parse(reference).ok()?,
|
||||||
|
};
|
||||||
|
if !matches!(url.scheme(), "http" | "https")
|
||||||
|
|| !url.username().is_empty()
|
||||||
|
|| url.password().is_some()
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
url.set_fragment(None);
|
||||||
|
Some(url.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse {
|
fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse {
|
||||||
let created = applied
|
let created = applied
|
||||||
.created
|
.created
|
||||||
@@ -441,10 +599,80 @@ fn validate_openapi_upload(upload: &OpenApiUpload) -> Result<(), ApiError> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct ParsedOpenApiPreview {
|
||||||
|
preview: crank_import::rest::ImportPreview,
|
||||||
|
ir_fingerprint: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ImportReplayContract {
|
||||||
|
LegacyV1,
|
||||||
|
PersistedV2(ParsedOpenApiPreview),
|
||||||
|
Current,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn import_replay_contract(
|
||||||
|
payload: &serde_json::Value,
|
||||||
|
locale: OpenApiUploadLocale,
|
||||||
|
) -> Result<ImportReplayContract, ApiError> {
|
||||||
|
let Some(normalization) = payload.get("normalization") else {
|
||||||
|
return Ok(ImportReplayContract::LegacyV1);
|
||||||
|
};
|
||||||
|
let normalizer = normalization
|
||||||
|
.get("normalizer_version")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
let projection = normalization
|
||||||
|
.get("projection_version")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
let ir_fingerprint = normalization
|
||||||
|
.get("ir_fingerprint")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.filter(|value| is_lower_sha256(value))
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
let expected_preview_digest = payload
|
||||||
|
.get("preview_digest")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.filter(|value| is_lower_sha256(value))
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
|
||||||
|
if normalizer == NORMALIZER_VERSION && projection == PROJECTION_VERSION {
|
||||||
|
return Ok(ImportReplayContract::Current);
|
||||||
|
}
|
||||||
|
if normalizer != "normalized-ir-v2" || projection != "preview-v2" {
|
||||||
|
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let preview_value = payload
|
||||||
|
.get("preview")
|
||||||
|
.cloned()
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
if preview_digest(&preview_value)? != expected_preview_digest {
|
||||||
|
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||||
|
}
|
||||||
|
let preview = serde_json::from_value(preview_value)
|
||||||
|
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
Ok(ImportReplayContract::PersistedV2(ParsedOpenApiPreview {
|
||||||
|
preview,
|
||||||
|
ir_fingerprint: ir_fingerprint.to_owned(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_lower_sha256(value: &str) -> bool {
|
||||||
|
value.len() == 64
|
||||||
|
&& value
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
}
|
||||||
|
|
||||||
async fn parse_verified_preview(
|
async fn parse_verified_preview(
|
||||||
bytes: Vec<u8>,
|
bytes: Vec<u8>,
|
||||||
|
digest: String,
|
||||||
|
snapshots: Vec<crank_import::rest::ExternalDocumentSnapshot>,
|
||||||
|
normalization_config: crank_import::rest::NormalizationConfig,
|
||||||
locale: OpenApiUploadLocale,
|
locale: OpenApiUploadLocale,
|
||||||
) -> Result<crank_import::rest::ImportPreview, ApiError> {
|
legacy_v1: bool,
|
||||||
|
) -> Result<ParsedOpenApiPreview, ApiError> {
|
||||||
let started = tokio::time::Instant::now();
|
let started = tokio::time::Instant::now();
|
||||||
let permit = tokio::time::timeout(OPENAPI_PARSE_DEADLINE, OPENAPI_PARSE_SLOTS.acquire())
|
let permit = tokio::time::timeout(OPENAPI_PARSE_DEADLINE, OPENAPI_PARSE_SLOTS.acquire())
|
||||||
.await
|
.await
|
||||||
@@ -457,8 +685,36 @@ async fn parse_verified_preview(
|
|||||||
let _permit = permit;
|
let _permit = permit;
|
||||||
let document = std::str::from_utf8(&bytes)
|
let document = std::str::from_utf8(&bytes)
|
||||||
.map_err(|_| ApiError::openapi_upload(locale, "invalid_utf8"))?;
|
.map_err(|_| ApiError::openapi_upload(locale, "invalid_utf8"))?;
|
||||||
crank_import::rest::preview_document(document)
|
if legacy_v1 {
|
||||||
.map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))
|
return crank_import::rest::preview_document_legacy_v1(document)
|
||||||
|
.map(|preview| ParsedOpenApiPreview {
|
||||||
|
preview,
|
||||||
|
ir_fingerprint: String::new(),
|
||||||
|
})
|
||||||
|
.map_err(|_| ApiError::openapi_upload(locale, "invalid_document"));
|
||||||
|
}
|
||||||
|
let digest = crank_import::rest::SourceDigest::parse(digest)
|
||||||
|
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
let ir = crank_import::rest::normalize_verified_bundle(
|
||||||
|
document,
|
||||||
|
digest,
|
||||||
|
&snapshots,
|
||||||
|
&normalization_config,
|
||||||
|
)
|
||||||
|
.map_err(|error| match error {
|
||||||
|
crank_import::rest::ImportParseError::NoMethods => {
|
||||||
|
ApiError::openapi_upload(locale, "no_methods")
|
||||||
|
}
|
||||||
|
_ => ApiError::openapi_upload(locale, "invalid_document"),
|
||||||
|
})?;
|
||||||
|
let ir_fingerprint = preview_digest(
|
||||||
|
&serde_json::to_value(&ir)
|
||||||
|
.map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))?,
|
||||||
|
)?;
|
||||||
|
Ok::<_, ApiError>(ParsedOpenApiPreview {
|
||||||
|
preview: crank_import::rest::preview_from_ir(&ir),
|
||||||
|
ir_fingerprint,
|
||||||
|
})
|
||||||
});
|
});
|
||||||
let remaining = OPENAPI_PARSE_DEADLINE
|
let remaining = OPENAPI_PARSE_DEADLINE
|
||||||
.checked_sub(started.elapsed())
|
.checked_sub(started.elapsed())
|
||||||
@@ -499,121 +755,64 @@ fn preview_digest(preview: &serde_json::Value) -> Result<String, ApiError> {
|
|||||||
|
|
||||||
fn verify_preview_contract(
|
fn verify_preview_contract(
|
||||||
payload: &serde_json::Value,
|
payload: &serde_json::Value,
|
||||||
preview: &crank_import::rest::ImportPreview,
|
parsed: &ParsedOpenApiPreview,
|
||||||
locale: OpenApiUploadLocale,
|
locale: OpenApiUploadLocale,
|
||||||
) -> Result<(), ApiError> {
|
) -> Result<(), ApiError> {
|
||||||
// Pre-fingerprint jobs are legacy rolling-upgrade records. They retain the
|
// Pre-fingerprint jobs are legacy rolling-upgrade records. They retain the
|
||||||
// old reparse behavior; new jobs fail closed if parser output drifts or a
|
// old reparse behavior; new jobs fail closed if parser output drifts or a
|
||||||
// persisted preview has been changed.
|
// persisted preview has been changed.
|
||||||
let Some(expected) = payload
|
let expected = payload
|
||||||
.get("preview_digest")
|
.get("preview_digest")
|
||||||
.and_then(serde_json::Value::as_str)
|
.and_then(serde_json::Value::as_str);
|
||||||
else {
|
if payload.get("normalization").is_some() && expected.is_none() {
|
||||||
|
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||||
|
}
|
||||||
|
let Some(expected) = expected else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
let actual = preview_digest(
|
let actual = preview_digest(
|
||||||
&serde_json::to_value(preview).map_err(|error| ApiError::internal(error.to_string()))?,
|
&serde_json::to_value(&parsed.preview)
|
||||||
|
.map_err(|error| ApiError::internal(error.to_string()))?,
|
||||||
)?;
|
)?;
|
||||||
if actual == expected {
|
if actual == expected {
|
||||||
|
if let Some(normalization) = payload.get("normalization") {
|
||||||
|
let normalizer = normalization
|
||||||
|
.get("normalizer_version")
|
||||||
|
.and_then(serde_json::Value::as_str);
|
||||||
|
let projection = normalization
|
||||||
|
.get("projection_version")
|
||||||
|
.and_then(serde_json::Value::as_str);
|
||||||
|
let fingerprint = normalization
|
||||||
|
.get("ir_fingerprint")
|
||||||
|
.and_then(serde_json::Value::as_str);
|
||||||
|
if normalizer != Some(NORMALIZER_VERSION)
|
||||||
|
|| projection != Some(PROJECTION_VERSION)
|
||||||
|
|| fingerprint != Some(parsed.ir_fingerprint.as_str())
|
||||||
|
{
|
||||||
|
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
Err(ApiError::openapi_upload(locale, "source_integrity"))
|
Err(ApiError::openapi_upload(locale, "source_integrity"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn import_job_source(
|
fn preview_has_blocker(
|
||||||
payload: &serde_json::Value,
|
preview: &crank_import::rest::ImportPreview,
|
||||||
locale: OpenApiUploadLocale,
|
selected_operation_keys: &BTreeSet<String>,
|
||||||
) -> Result<ImportJobSourceEnvelope, ApiError> {
|
) -> bool {
|
||||||
let source = payload
|
preview
|
||||||
.get("source")
|
.findings
|
||||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
.iter()
|
||||||
let source_id = source
|
.any(|finding| finding.severity == ImportFindingSeverity::Error)
|
||||||
.get("source_id")
|
|| preview
|
||||||
.and_then(serde_json::Value::as_str)
|
.groups
|
||||||
.filter(|value| value.len() <= 132)
|
.iter()
|
||||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
.flat_map(|group| group.operations.iter())
|
||||||
let digest = source
|
.filter(|operation| selected_operation_keys.contains(&operation.key))
|
||||||
.get("digest")
|
.flat_map(|operation| operation.findings.iter())
|
||||||
.and_then(serde_json::Value::as_str)
|
.any(|finding| finding.severity == ImportFindingSeverity::Error)
|
||||||
.and_then(|value| value.parse().ok())
|
|
||||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
|
||||||
Ok(ImportJobSourceEnvelope {
|
|
||||||
source_id: ArtifactSourceId::new(source_id),
|
|
||||||
digest,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
struct SourceDetachGuard {
|
|
||||||
registry: crank_registry::PostgresRegistry,
|
|
||||||
workspace_id: WorkspaceId,
|
|
||||||
source_id: ArtifactSourceId,
|
|
||||||
expected_updated_at: OffsetDateTime,
|
|
||||||
armed: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SourceDetachGuard {
|
|
||||||
fn new(
|
|
||||||
registry: crank_registry::PostgresRegistry,
|
|
||||||
workspace_id: WorkspaceId,
|
|
||||||
source_id: ArtifactSourceId,
|
|
||||||
expected_updated_at: OffsetDateTime,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
|
||||||
registry,
|
|
||||||
workspace_id,
|
|
||||||
source_id,
|
|
||||||
expected_updated_at,
|
|
||||||
armed: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn disarm(&mut self) {
|
|
||||||
self.armed = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn detach_now(&mut self) {
|
|
||||||
if !self.armed {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
self.armed = false;
|
|
||||||
// This is the normal error path: await the database transition so a
|
|
||||||
// caller receives a completed cleanup before it can retry. `Drop`
|
|
||||||
// remains only the cancellation/shutdown safety net.
|
|
||||||
let _ = self
|
|
||||||
.registry
|
|
||||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
|
||||||
workspace_id: &self.workspace_id,
|
|
||||||
source_id: &self.source_id,
|
|
||||||
expected_updated_at: Some(self.expected_updated_at),
|
|
||||||
detached_at: OffsetDateTime::now_utc(),
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for SourceDetachGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if !self.armed {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let registry = self.registry.clone();
|
|
||||||
let workspace_id = self.workspace_id.clone();
|
|
||||||
let source_id = self.source_id.clone();
|
|
||||||
let expected_updated_at = Some(self.expected_updated_at);
|
|
||||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
|
||||||
handle.spawn(async move {
|
|
||||||
let _ = registry
|
|
||||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
|
||||||
workspace_id: &workspace_id,
|
|
||||||
source_id: &source_id,
|
|
||||||
expected_updated_at,
|
|
||||||
detached_at: OffsetDateTime::now_utc(),
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
|
fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
|
||||||
|
|||||||
@@ -0,0 +1,416 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
|
pub(super) struct MaterializedDependencies {
|
||||||
|
pub(super) snapshots: Vec<crank_import::rest::ExternalDocumentSnapshot>,
|
||||||
|
envelopes: Vec<ImportJobSourceEnvelope>,
|
||||||
|
canonical_uris: Vec<String>,
|
||||||
|
guards: Vec<SourceDetachGuard>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) struct ImportReplayContext<'a> {
|
||||||
|
pub(super) workspace_id: &'a WorkspaceId,
|
||||||
|
pub(super) job_id: &'a ImportJobId,
|
||||||
|
pub(super) application_key: &'a str,
|
||||||
|
pub(super) conflict_mode: ImportConflictMode,
|
||||||
|
pub(super) finished_at: &'a OffsetDateTime,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MaterializedDependencies {
|
||||||
|
fn empty() -> Self {
|
||||||
|
Self {
|
||||||
|
snapshots: Vec::new(),
|
||||||
|
envelopes: Vec::new(),
|
||||||
|
canonical_uris: Vec::new(),
|
||||||
|
guards: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn payload(&self) -> Vec<serde_json::Value> {
|
||||||
|
self.envelopes
|
||||||
|
.iter()
|
||||||
|
.zip(&self.canonical_uris)
|
||||||
|
.map(|(dependency, canonical_uri)| {
|
||||||
|
json!({
|
||||||
|
"source_id": dependency.source_id.as_str(),
|
||||||
|
"digest": dependency.digest.as_str(),
|
||||||
|
"canonical_uri": canonical_uri,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn snapshot_payload(&self) -> Vec<serde_json::Value> {
|
||||||
|
self.snapshots
|
||||||
|
.iter()
|
||||||
|
.filter_map(|snapshot| {
|
||||||
|
self.envelopes
|
||||||
|
.iter()
|
||||||
|
.find(|dependency| dependency.digest.digest_hex() == snapshot.digest.as_str())
|
||||||
|
.map(|dependency| {
|
||||||
|
json!({
|
||||||
|
"source_id": dependency.source_id.as_str(),
|
||||||
|
"digest": dependency.digest.as_str(),
|
||||||
|
"canonical_uri": snapshot.canonical_uri,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn disarm(&mut self) {
|
||||||
|
for guard in &mut self.guards {
|
||||||
|
guard.disarm();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn detach_all(&mut self) {
|
||||||
|
for guard in &mut self.guards {
|
||||||
|
guard.detach_now().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AdminService {
|
||||||
|
pub(super) async fn fail_openapi_import_or_replay(
|
||||||
|
&self,
|
||||||
|
context: &ImportReplayContext<'_>,
|
||||||
|
error_text: &'static str,
|
||||||
|
error: ApiError,
|
||||||
|
) -> Result<OpenApiImportCreateResponse, ApiError> {
|
||||||
|
let empty = json!([]);
|
||||||
|
self.registry
|
||||||
|
.finish_import_job(FinishImportJobRequest {
|
||||||
|
id: context.job_id,
|
||||||
|
status: ImportJobStatus::Failed,
|
||||||
|
created_operation_ids: &empty,
|
||||||
|
error_text: Some(error_text),
|
||||||
|
finished_at: context.finished_at,
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
// `finish_import_job` preserves an already Completed row under its
|
||||||
|
// lock. This read distinguishes that race from the terminal Failed
|
||||||
|
// transition and returns the immutable canonical application result.
|
||||||
|
let latest = self
|
||||||
|
.registry
|
||||||
|
.get_import_job(context.workspace_id, context.job_id)
|
||||||
|
.await?;
|
||||||
|
if latest.is_some_and(|job| job.status == ImportJobStatus::Completed) {
|
||||||
|
let applied = self
|
||||||
|
.registry
|
||||||
|
.apply_import_job(ApplyImportJobRequest {
|
||||||
|
id: context.job_id,
|
||||||
|
workspace_id: context.workspace_id,
|
||||||
|
application_key: context.application_key,
|
||||||
|
conflict_mode: context.conflict_mode,
|
||||||
|
operations: &[],
|
||||||
|
pre_skipped: &[],
|
||||||
|
finished_at: context.finished_at,
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
return Ok(openapi_import_response(applied));
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn materialize_external_reference_snapshots(
|
||||||
|
&self,
|
||||||
|
workspace_id: &WorkspaceId,
|
||||||
|
primary_bytes: &[u8],
|
||||||
|
created_at: OffsetDateTime,
|
||||||
|
) -> MaterializedDependencies {
|
||||||
|
let materialized_count = std::sync::atomic::AtomicUsize::new(0);
|
||||||
|
match tokio::time::timeout(
|
||||||
|
self.external_reference_materialization_timeout,
|
||||||
|
self.materialize_external_reference_snapshots_inner(
|
||||||
|
workspace_id,
|
||||||
|
primary_bytes,
|
||||||
|
created_at,
|
||||||
|
&materialized_count,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(dependencies) => dependencies,
|
||||||
|
Err(_) => {
|
||||||
|
materialization_failure(
|
||||||
|
"chain",
|
||||||
|
"timeout",
|
||||||
|
materialized_count.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
|
);
|
||||||
|
MaterializedDependencies::empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn materialize_external_reference_snapshots_inner(
|
||||||
|
&self,
|
||||||
|
workspace_id: &WorkspaceId,
|
||||||
|
primary_bytes: &[u8],
|
||||||
|
created_at: OffsetDateTime,
|
||||||
|
materialized_count: &std::sync::atomic::AtomicUsize,
|
||||||
|
) -> MaterializedDependencies {
|
||||||
|
if !self
|
||||||
|
.external_reference_normalization
|
||||||
|
.external_references_enabled
|
||||||
|
{
|
||||||
|
return MaterializedDependencies::empty();
|
||||||
|
}
|
||||||
|
let Ok(primary) = std::str::from_utf8(primary_bytes) else {
|
||||||
|
materialization_failure("primary_decode", "invalid_utf8", 0);
|
||||||
|
return MaterializedDependencies::empty();
|
||||||
|
};
|
||||||
|
let Ok(primary_references) =
|
||||||
|
crank_import::rest::reference_uris(primary, &self.external_reference_normalization)
|
||||||
|
else {
|
||||||
|
materialization_failure("primary_scan", "invalid_document", 0);
|
||||||
|
return MaterializedDependencies::empty();
|
||||||
|
};
|
||||||
|
let mut queue = VecDeque::new();
|
||||||
|
for reference in primary_references {
|
||||||
|
if let Some(uri) = canonical_external_document_uri(None, &reference) {
|
||||||
|
queue.push_back((uri, 1usize));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut seen = BTreeSet::new();
|
||||||
|
let mut result = MaterializedDependencies::empty();
|
||||||
|
let mut dependency_by_digest = BTreeMap::<String, usize>::new();
|
||||||
|
while let Some((canonical_uri, depth)) = queue.pop_front() {
|
||||||
|
if !seen.insert(canonical_uri.clone())
|
||||||
|
|| depth > self.external_reference_normalization.max_reference_depth
|
||||||
|
|| seen.len()
|
||||||
|
> self
|
||||||
|
.external_reference_normalization
|
||||||
|
.max_reference_documents
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bytes = match self.external_reference_fetcher.get(&canonical_uri).await {
|
||||||
|
Ok(bytes) => bytes,
|
||||||
|
Err(error) => {
|
||||||
|
materialization_failure(
|
||||||
|
"fetch",
|
||||||
|
external_fetch_error_code(&error),
|
||||||
|
result.snapshots.len(),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let Ok(document) = std::str::from_utf8(&bytes).map(str::to_owned) else {
|
||||||
|
materialization_failure(
|
||||||
|
"dependency_decode",
|
||||||
|
"invalid_utf8",
|
||||||
|
result.snapshots.len(),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Ok(references) = crank_import::rest::external_reference_uris(
|
||||||
|
&document,
|
||||||
|
&self.external_reference_normalization,
|
||||||
|
) else {
|
||||||
|
materialization_failure(
|
||||||
|
"dependency_scan",
|
||||||
|
"invalid_document",
|
||||||
|
result.snapshots.len(),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let store = self.artifact_store.clone();
|
||||||
|
let artifact_bytes = bytes.clone();
|
||||||
|
let Ok(Ok(artifact)) =
|
||||||
|
tokio::task::spawn_blocking(move || store.put_registered(&artifact_bytes)).await
|
||||||
|
else {
|
||||||
|
materialization_failure(
|
||||||
|
"artifact_store",
|
||||||
|
"storage_unavailable",
|
||||||
|
result.snapshots.len(),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let digest_key = artifact.artifact_ref().as_str().to_owned();
|
||||||
|
let envelope_index = if let Some(index) = dependency_by_digest.get(&digest_key) {
|
||||||
|
*index
|
||||||
|
} else {
|
||||||
|
let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi_dep"));
|
||||||
|
let Ok(source) = self
|
||||||
|
.registry
|
||||||
|
.create_artifact_source(CreateArtifactSourceRequest {
|
||||||
|
workspace_id,
|
||||||
|
source_id: &source_id,
|
||||||
|
artifact: &artifact,
|
||||||
|
mime_type: "application/octet-stream",
|
||||||
|
sensitivity: ArtifactSourceSensitivity::Internal,
|
||||||
|
created_at,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
else {
|
||||||
|
materialization_failure(
|
||||||
|
"source_create",
|
||||||
|
"registry_unavailable",
|
||||||
|
result.snapshots.len(),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let mut guard = SourceDetachGuard::new(
|
||||||
|
self.registry.clone(),
|
||||||
|
workspace_id.clone(),
|
||||||
|
source_id.clone(),
|
||||||
|
source.updated_at,
|
||||||
|
);
|
||||||
|
let Ok(verified) = self
|
||||||
|
.registry
|
||||||
|
.read_artifact_source(
|
||||||
|
std::sync::Arc::clone(&self.artifact_store),
|
||||||
|
workspace_id,
|
||||||
|
&source_id,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
else {
|
||||||
|
materialization_failure(
|
||||||
|
"source_verify",
|
||||||
|
"source_unavailable",
|
||||||
|
result.snapshots.len(),
|
||||||
|
);
|
||||||
|
guard.detach_now().await;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if verified.source.blob.artifact_ref != *artifact.artifact_ref() {
|
||||||
|
materialization_failure(
|
||||||
|
"source_verify",
|
||||||
|
"source_integrity",
|
||||||
|
result.snapshots.len(),
|
||||||
|
);
|
||||||
|
guard.detach_now().await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let index = result.envelopes.len();
|
||||||
|
result.envelopes.push(ImportJobSourceEnvelope {
|
||||||
|
source_id: source_id.clone(),
|
||||||
|
digest: artifact.artifact_ref().clone(),
|
||||||
|
});
|
||||||
|
result.guards.push(guard);
|
||||||
|
dependency_by_digest.insert(digest_key, index);
|
||||||
|
index
|
||||||
|
};
|
||||||
|
let dependency = &result.envelopes[envelope_index];
|
||||||
|
let snapshot_digest = match crank_import::rest::SourceDigest::parse(
|
||||||
|
dependency.digest.digest_hex().to_owned(),
|
||||||
|
) {
|
||||||
|
Ok(digest) => digest,
|
||||||
|
Err(_) => {
|
||||||
|
materialization_failure("snapshot", "source_integrity", result.snapshots.len());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
result
|
||||||
|
.snapshots
|
||||||
|
.push(crank_import::rest::ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: canonical_uri.clone(),
|
||||||
|
digest: snapshot_digest,
|
||||||
|
document,
|
||||||
|
});
|
||||||
|
materialized_count.store(result.snapshots.len(), std::sync::atomic::Ordering::Relaxed);
|
||||||
|
result.canonical_uris.push(canonical_uri.clone());
|
||||||
|
for reference in references {
|
||||||
|
if let Some(uri) = canonical_external_document_uri(Some(&canonical_uri), &reference)
|
||||||
|
{
|
||||||
|
queue.push_back((uri, depth.saturating_add(1)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Registry ownership is one envelope per immutable digest, while the
|
||||||
|
// resolver may map several canonical URIs to that same snapshot. Keep
|
||||||
|
// payload rows aligned with envelopes and add aliases separately.
|
||||||
|
if result.envelopes.len() != result.canonical_uris.len() {
|
||||||
|
let mut canonical_by_digest = BTreeMap::new();
|
||||||
|
for snapshot in &result.snapshots {
|
||||||
|
canonical_by_digest
|
||||||
|
.entry(snapshot.digest.as_str().to_owned())
|
||||||
|
.or_insert_with(|| snapshot.canonical_uri.clone());
|
||||||
|
}
|
||||||
|
result.canonical_uris = result
|
||||||
|
.envelopes
|
||||||
|
.iter()
|
||||||
|
.map(|envelope| {
|
||||||
|
canonical_by_digest
|
||||||
|
.get(envelope.digest.digest_hex())
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_default()
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn read_import_job_dependencies(
|
||||||
|
&self,
|
||||||
|
workspace_id: &WorkspaceId,
|
||||||
|
payload: &serde_json::Value,
|
||||||
|
locale: OpenApiUploadLocale,
|
||||||
|
) -> Result<Vec<crank_import::rest::ExternalDocumentSnapshot>, ApiError> {
|
||||||
|
let dependencies = import_job_dependencies(payload, locale)?;
|
||||||
|
let mut snapshots = Vec::with_capacity(dependencies.len());
|
||||||
|
for (canonical_uri, dependency) in dependencies {
|
||||||
|
let verified = self
|
||||||
|
.registry
|
||||||
|
.read_artifact_source(
|
||||||
|
std::sync::Arc::clone(&self.artifact_store),
|
||||||
|
workspace_id,
|
||||||
|
&dependency.source_id,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| openapi_source_error(locale, error))?;
|
||||||
|
if verified.source.blob.artifact_ref != dependency.digest {
|
||||||
|
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||||
|
}
|
||||||
|
let document = String::from_utf8(verified.bytes)
|
||||||
|
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
let digest =
|
||||||
|
crank_import::rest::SourceDigest::parse(dependency.digest.digest_hex().to_owned())
|
||||||
|
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
snapshots.push(crank_import::rest::ExternalDocumentSnapshot {
|
||||||
|
canonical_uri,
|
||||||
|
digest,
|
||||||
|
document,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(snapshots)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn external_fetch_error_code(error: &crank_runtime::ExternalReferenceFetchError) -> &'static str {
|
||||||
|
match error {
|
||||||
|
crank_runtime::ExternalReferenceFetchError::Disabled => "disabled",
|
||||||
|
crank_runtime::ExternalReferenceFetchError::InvalidUrl => "invalid_url",
|
||||||
|
crank_runtime::ExternalReferenceFetchError::TargetNotAllowed => "target_not_allowed",
|
||||||
|
crank_runtime::ExternalReferenceFetchError::RedirectNotAllowed => "redirect_not_allowed",
|
||||||
|
crank_runtime::ExternalReferenceFetchError::ResponseTooLarge { .. } => "response_too_large",
|
||||||
|
crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { .. } => "unexpected_status",
|
||||||
|
crank_runtime::ExternalReferenceFetchError::Transport { timeout: true, .. } => "timeout",
|
||||||
|
crank_runtime::ExternalReferenceFetchError::Transport { .. } => "transport",
|
||||||
|
crank_runtime::ExternalReferenceFetchError::InvalidConfiguration => "invalid_configuration",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn materialization_failure(stage: &'static str, error_code: &'static str, count: usize) {
|
||||||
|
warn!(
|
||||||
|
name: "admin.openapi_import.materialization_failed",
|
||||||
|
stage,
|
||||||
|
error_code,
|
||||||
|
count,
|
||||||
|
"external OpenAPI materialization failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::external_fetch_error_code;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unexpected_status_has_a_stable_safe_error_code() {
|
||||||
|
let error = crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { status: 500 };
|
||||||
|
assert_eq!(external_fetch_error_code(&error), "unexpected_status");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
|
use crank_core::WorkspaceId;
|
||||||
|
use crank_registry::{ArtifactSourceId, DetachArtifactSourceRequest, ImportJobSourceEnvelope};
|
||||||
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
|
use crate::{error::ApiError, service::OpenApiUploadLocale};
|
||||||
|
|
||||||
|
use super::canonical_external_document_uri;
|
||||||
|
|
||||||
|
const MAX_IMPORT_JOB_DOCUMENTS: usize = 32;
|
||||||
|
|
||||||
|
pub(super) fn import_job_source(
|
||||||
|
payload: &serde_json::Value,
|
||||||
|
locale: OpenApiUploadLocale,
|
||||||
|
) -> Result<ImportJobSourceEnvelope, ApiError> {
|
||||||
|
let source = payload
|
||||||
|
.get("source")
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||||
|
let source_id = source
|
||||||
|
.get("source_id")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.filter(|value| value.len() <= 132)
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||||
|
let digest = source
|
||||||
|
.get("digest")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.and_then(|value| value.parse().ok())
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
Ok(ImportJobSourceEnvelope {
|
||||||
|
source_id: ArtifactSourceId::new(source_id),
|
||||||
|
digest,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn import_job_dependencies(
|
||||||
|
payload: &serde_json::Value,
|
||||||
|
locale: OpenApiUploadLocale,
|
||||||
|
) -> Result<Vec<(String, ImportJobSourceEnvelope)>, ApiError> {
|
||||||
|
let empty = Vec::new();
|
||||||
|
let items = payload
|
||||||
|
.get("dependency_snapshots")
|
||||||
|
.and_then(serde_json::Value::as_array)
|
||||||
|
.unwrap_or(&empty);
|
||||||
|
if items.len() > MAX_IMPORT_JOB_DOCUMENTS {
|
||||||
|
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||||
|
}
|
||||||
|
let mut canonical_uris = BTreeSet::new();
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.map(|item| {
|
||||||
|
let canonical_uri = item
|
||||||
|
.get("canonical_uri")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.filter(|value| {
|
||||||
|
canonical_external_document_uri(None, value).as_deref() == Some(*value)
|
||||||
|
})
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
if !canonical_uris.insert(canonical_uri.to_owned()) {
|
||||||
|
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||||
|
}
|
||||||
|
let source_id = item
|
||||||
|
.get("source_id")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.filter(|value| value.len() <= 132)
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
let digest = item
|
||||||
|
.get("digest")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.and_then(|value| value.parse().ok())
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||||
|
Ok((
|
||||||
|
canonical_uri.to_owned(),
|
||||||
|
ImportJobSourceEnvelope {
|
||||||
|
source_id: ArtifactSourceId::new(source_id),
|
||||||
|
digest,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn import_job_normalization_config(
|
||||||
|
payload: &serde_json::Value,
|
||||||
|
current: &crank_import::rest::NormalizationConfig,
|
||||||
|
legacy_v1: bool,
|
||||||
|
locale: OpenApiUploadLocale,
|
||||||
|
) -> Result<crank_import::rest::NormalizationConfig, ApiError> {
|
||||||
|
if legacy_v1 {
|
||||||
|
return Ok(current.clone());
|
||||||
|
}
|
||||||
|
payload
|
||||||
|
.pointer("/normalization/config")
|
||||||
|
.cloned()
|
||||||
|
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))
|
||||||
|
.and_then(|value| {
|
||||||
|
serde_json::from_value(value)
|
||||||
|
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) struct SourceDetachGuard {
|
||||||
|
registry: crank_registry::PostgresRegistry,
|
||||||
|
workspace_id: WorkspaceId,
|
||||||
|
source_id: ArtifactSourceId,
|
||||||
|
expected_updated_at: OffsetDateTime,
|
||||||
|
armed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SourceDetachGuard {
|
||||||
|
pub(super) fn new(
|
||||||
|
registry: crank_registry::PostgresRegistry,
|
||||||
|
workspace_id: WorkspaceId,
|
||||||
|
source_id: ArtifactSourceId,
|
||||||
|
expected_updated_at: OffsetDateTime,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
registry,
|
||||||
|
workspace_id,
|
||||||
|
source_id,
|
||||||
|
expected_updated_at,
|
||||||
|
armed: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn disarm(&mut self) {
|
||||||
|
self.armed = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn detach_now(&mut self) {
|
||||||
|
if !self.armed {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.armed = false;
|
||||||
|
let _ = self
|
||||||
|
.registry
|
||||||
|
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||||
|
workspace_id: &self.workspace_id,
|
||||||
|
source_id: &self.source_id,
|
||||||
|
expected_updated_at: Some(self.expected_updated_at),
|
||||||
|
detached_at: OffsetDateTime::now_utc(),
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for SourceDetachGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.armed {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let registry = self.registry.clone();
|
||||||
|
let workspace_id = self.workspace_id.clone();
|
||||||
|
let source_id = self.source_id.clone();
|
||||||
|
let expected_updated_at = Some(self.expected_updated_at);
|
||||||
|
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||||
|
handle.spawn(async move {
|
||||||
|
let _ = registry
|
||||||
|
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||||
|
workspace_id: &workspace_id,
|
||||||
|
source_id: &source_id,
|
||||||
|
expected_updated_at,
|
||||||
|
detached_at: OffsetDateTime::now_utc(),
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -108,6 +108,28 @@ pub(super) fn build_test_app(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_test_app_with_external_references(
|
||||||
|
registry: PostgresRegistry,
|
||||||
|
storage_root: std::path::PathBuf,
|
||||||
|
allowed_url_prefixes: Vec<String>,
|
||||||
|
) -> Router {
|
||||||
|
build_app(AppState {
|
||||||
|
service: test_service_with_external_references_timeout(
|
||||||
|
registry,
|
||||||
|
storage_root,
|
||||||
|
allowed_url_prefixes,
|
||||||
|
// This HTTP fixture asserts classification of an immediate
|
||||||
|
// upstream status. Leave headroom for CI scheduling jitter so the
|
||||||
|
// unrelated chain-timeout branch cannot win first.
|
||||||
|
10_000,
|
||||||
|
),
|
||||||
|
api_rate_limiter: crank_runtime::RequestRateLimiter::new(
|
||||||
|
crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(),
|
||||||
|
),
|
||||||
|
trusted_proxy_ips: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn build_test_app_with_audit_sink(
|
pub(super) fn build_test_app_with_audit_sink(
|
||||||
registry: PostgresRegistry,
|
registry: PostgresRegistry,
|
||||||
storage_root: std::path::PathBuf,
|
storage_root: std::path::PathBuf,
|
||||||
@@ -152,6 +174,47 @@ pub(super) fn test_service(
|
|||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn test_service_with_external_references(
|
||||||
|
registry: PostgresRegistry,
|
||||||
|
storage_root: std::path::PathBuf,
|
||||||
|
allowed_url_prefixes: Vec<String>,
|
||||||
|
) -> AdminService {
|
||||||
|
test_service_with_external_references_timeout(
|
||||||
|
registry,
|
||||||
|
storage_root,
|
||||||
|
allowed_url_prefixes,
|
||||||
|
2_000,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_service_with_external_references_timeout(
|
||||||
|
registry: PostgresRegistry,
|
||||||
|
storage_root: std::path::PathBuf,
|
||||||
|
allowed_url_prefixes: Vec<String>,
|
||||||
|
fetch_timeout_ms: u64,
|
||||||
|
) -> AdminService {
|
||||||
|
let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
|
||||||
|
let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
|
||||||
|
AdminServiceBuilder::new(
|
||||||
|
registry,
|
||||||
|
storage_root,
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
runtime,
|
||||||
|
)
|
||||||
|
.with_external_reference_import(&crank_config::ExternalReferenceSettings {
|
||||||
|
allowed_url_prefixes,
|
||||||
|
max_depth: 8,
|
||||||
|
max_documents: 32,
|
||||||
|
max_fetch_bytes: 64 * 1024,
|
||||||
|
fetch_timeout_ms,
|
||||||
|
max_expanded_nodes: 10_000,
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.with_outbound_http_policy(outbound_policy)
|
||||||
|
.build()
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) async fn spawn_upstream_server() -> String {
|
pub(super) async fn spawn_upstream_server() -> String {
|
||||||
let app = Router::new().route("/crm/leads", post(create_lead));
|
let app = Router::new().route("/crm/leads", post(create_lead));
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
|
use admin_api::service::{
|
||||||
|
AdminServiceBuilder, OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale,
|
||||||
|
};
|
||||||
use crank_core::WorkspaceId;
|
use crank_core::WorkspaceId;
|
||||||
use crank_registry::ImportJobStatus;
|
use crank_registry::ImportJobStatus;
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
@@ -7,6 +9,8 @@ use super::common::{
|
|||||||
test_auth_settings, test_registry, test_secret_crypto, test_service, test_storage_root,
|
test_auth_settings, test_registry, test_secret_crypto, test_service, test_storage_root,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod external_references;
|
||||||
|
|
||||||
const OPENAPI3: &str = r#"
|
const OPENAPI3: &str = r#"
|
||||||
openapi: 3.0.3
|
openapi: 3.0.3
|
||||||
info:
|
info:
|
||||||
@@ -60,12 +64,26 @@ async fn previews_openapi_and_creates_draft_operations() {
|
|||||||
preview.preview.groups[0].operations[0].suggested_name,
|
preview.preview.groups[0].operations[0].suggested_name,
|
||||||
"latest_rates"
|
"latest_rates"
|
||||||
);
|
);
|
||||||
|
let public_response = serde_json::to_string(&preview).unwrap();
|
||||||
|
assert!(!public_response.contains("normalizer_version"));
|
||||||
|
assert!(!public_response.contains("projection_version"));
|
||||||
|
assert!(!public_response.contains("ir_fingerprint"));
|
||||||
|
assert!(!public_response.contains("source_identity"));
|
||||||
let preview_job = registry
|
let preview_job = registry
|
||||||
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||||||
.await
|
.await
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(preview_job.status, ImportJobStatus::Pending);
|
assert_eq!(preview_job.status, ImportJobStatus::Pending);
|
||||||
|
assert_eq!(
|
||||||
|
preview_job.preview_payload["normalization"]["normalizer_version"],
|
||||||
|
"normalized-ir-v3"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
preview_job.preview_payload["normalization"]["projection_version"],
|
||||||
|
"preview-v3"
|
||||||
|
);
|
||||||
|
assert!(preview_job.preview_payload["normalization"]["ir_fingerprint"].is_string());
|
||||||
|
|
||||||
let created = service
|
let created = service
|
||||||
.create_openapi_import(
|
.create_openapi_import(
|
||||||
@@ -222,7 +240,7 @@ async fn unknown_selected_operations_are_persisted_in_the_canonical_replay() {
|
|||||||
async fn concurrent_openapi_import_replays_the_same_atomic_result() {
|
async fn concurrent_openapi_import_replays_the_same_atomic_result() {
|
||||||
let registry = test_registry().await;
|
let registry = test_registry().await;
|
||||||
let service = test_service(
|
let service = test_service(
|
||||||
registry,
|
registry.clone(),
|
||||||
test_storage_root("openapi_import_replay"),
|
test_storage_root("openapi_import_replay"),
|
||||||
test_auth_settings(),
|
test_auth_settings(),
|
||||||
test_secret_crypto(),
|
test_secret_crypto(),
|
||||||
@@ -257,6 +275,31 @@ async fn concurrent_openapi_import_replays_the_same_atomic_result() {
|
|||||||
service.list_operations(&workspace_id).await.unwrap().len(),
|
service.list_operations(&workspace_id).await.unwrap().len(),
|
||||||
1
|
1
|
||||||
);
|
);
|
||||||
|
let completed_before = registry
|
||||||
|
.get_import_job(&workspace_id, &job_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let empty = serde_json::json!([]);
|
||||||
|
let failed_at = time::OffsetDateTime::now_utc();
|
||||||
|
registry
|
||||||
|
.finish_import_job(crank_registry::FinishImportJobRequest {
|
||||||
|
id: &job_id,
|
||||||
|
status: ImportJobStatus::Failed,
|
||||||
|
created_operation_ids: &empty,
|
||||||
|
error_text: Some("late_verification_failure"),
|
||||||
|
finished_at: &failed_at,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let completed_after = registry
|
||||||
|
.get_import_job(&workspace_id, &job_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(completed_after.status, ImportJobStatus::Completed);
|
||||||
|
assert_eq!(completed_after.error_text, completed_before.error_text);
|
||||||
|
assert_eq!(completed_after.finished_at, completed_before.finished_at);
|
||||||
|
|
||||||
let conflicting_replay = service
|
let conflicting_replay = service
|
||||||
.create_openapi_import(
|
.create_openapi_import(
|
||||||
@@ -312,6 +355,13 @@ async fn apply_fails_closed_when_the_preview_parser_contract_drifts() {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
|
assert_failed_job_and_detached_sources(
|
||||||
|
®istry,
|
||||||
|
&workspace_id,
|
||||||
|
&job_id,
|
||||||
|
"import_replay_verification_failed",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
assert!(
|
assert!(
|
||||||
service
|
service
|
||||||
.list_operations(&workspace_id)
|
.list_operations(&workspace_id)
|
||||||
@@ -321,6 +371,366 @@ async fn apply_fails_closed_when_the_preview_parser_contract_drifts() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn apply_fails_closed_for_each_normalization_contract_field_and_digest_shape() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let service = test_service(
|
||||||
|
registry.clone(),
|
||||||
|
test_storage_root("openapi_import_contract_fields"),
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
for case in [
|
||||||
|
"normalizer_version",
|
||||||
|
"projection_version",
|
||||||
|
"ir_fingerprint",
|
||||||
|
"missing_preview_digest",
|
||||||
|
"non_string_preview_digest",
|
||||||
|
] {
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(&workspace_id, openapi_upload())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||||||
|
let query = match case {
|
||||||
|
"normalizer_version" => sqlx::query(
|
||||||
|
"update import_jobs set preview_payload = jsonb_set(preview_payload, '{normalization,normalizer_version}', to_jsonb('future'::text)) where id = $1",
|
||||||
|
),
|
||||||
|
"projection_version" => sqlx::query(
|
||||||
|
"update import_jobs set preview_payload = jsonb_set(preview_payload, '{normalization,projection_version}', to_jsonb('future'::text)) where id = $1",
|
||||||
|
),
|
||||||
|
"ir_fingerprint" => sqlx::query(
|
||||||
|
"update import_jobs set preview_payload = jsonb_set(preview_payload, '{normalization,ir_fingerprint}', to_jsonb('bad'::text)) where id = $1",
|
||||||
|
),
|
||||||
|
"missing_preview_digest" => sqlx::query(
|
||||||
|
"update import_jobs set preview_payload = preview_payload - 'preview_digest' where id = $1",
|
||||||
|
),
|
||||||
|
"non_string_preview_digest" => sqlx::query(
|
||||||
|
"update import_jobs set preview_payload = jsonb_set(preview_payload, '{preview_digest}', '42'::jsonb) where id = $1",
|
||||||
|
),
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
query
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.execute(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&job_id,
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||||
|
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||||
|
conflict_mode: "rename".to_owned(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_failed_job_and_detached_sources(
|
||||||
|
®istry,
|
||||||
|
&workspace_id,
|
||||||
|
&job_id,
|
||||||
|
"import_replay_verification_failed",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
service
|
||||||
|
.list_operations(&workspace_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn legacy_job_without_normalization_contract_replays_until_expiry() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let service = test_service(
|
||||||
|
registry.clone(),
|
||||||
|
test_storage_root("openapi_import_legacy_replay"),
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(&workspace_id, openapi_upload())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||||||
|
sqlx::query(
|
||||||
|
"update import_jobs set preview_payload = preview_payload - 'normalization' where id = $1",
|
||||||
|
)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.execute(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let result = service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&job_id,
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||||
|
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||||
|
conflict_mode: "skip".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.created.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn blocker_finding_keeps_valid_preview_but_prevents_draft_mutation() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let service = test_service(
|
||||||
|
registry.clone(),
|
||||||
|
test_storage_root("openapi_import_blocker"),
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let upload = OpenApiUpload {
|
||||||
|
bytes: br#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Partial }
|
||||||
|
paths:
|
||||||
|
/valid:
|
||||||
|
get:
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
/broken: not-a-path-item
|
||||||
|
"#
|
||||||
|
.to_vec(),
|
||||||
|
mime_type: "application/yaml".to_owned(),
|
||||||
|
locale: OpenApiUploadLocale::En,
|
||||||
|
};
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(&workspace_id, upload)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(preview.preview.groups[0].operations.len(), 1);
|
||||||
|
assert!(
|
||||||
|
preview
|
||||||
|
.preview
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.severity == crank_import::rest::ImportFindingSeverity::Error)
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&preview.job_id.as_str().into(),
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /valid".to_owned()],
|
||||||
|
server_url: None,
|
||||||
|
conflict_mode: "skip".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
service
|
||||||
|
.list_operations(&workspace_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert_failed_job_and_detached_sources(
|
||||||
|
®istry,
|
||||||
|
&workspace_id,
|
||||||
|
&preview.job_id.as_str().into(),
|
||||||
|
"reference_resolution_blocked",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn operation_blockers_apply_only_to_selected_keys_and_full_selection_fails_atomically() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let service = test_service(
|
||||||
|
registry.clone(),
|
||||||
|
test_storage_root("openapi_import_selected_blockers"),
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let upload = OpenApiUpload {
|
||||||
|
bytes: br#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Selected blockers }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/valid:
|
||||||
|
get:
|
||||||
|
operationId: validOperation
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
/broken:
|
||||||
|
get:
|
||||||
|
operationId: brokenOperation
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: unresolved external schema
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: 'https://schemas.example.test/missing.yaml#/Result' }
|
||||||
|
"#
|
||||||
|
.to_vec(),
|
||||||
|
mime_type: "application/yaml".to_owned(),
|
||||||
|
locale: OpenApiUploadLocale::En,
|
||||||
|
};
|
||||||
|
|
||||||
|
let valid_preview = service
|
||||||
|
.preview_openapi_import(&workspace_id, upload.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
valid_preview.preview.findings.iter().all(|finding| {
|
||||||
|
finding.severity != crank_import::rest::ImportFindingSeverity::Error
|
||||||
|
})
|
||||||
|
);
|
||||||
|
let broken = valid_preview
|
||||||
|
.preview
|
||||||
|
.groups
|
||||||
|
.iter()
|
||||||
|
.flat_map(|group| &group.operations)
|
||||||
|
.find(|operation| operation.key == "GET /broken")
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
broken.findings.iter().any(|finding| {
|
||||||
|
finding.severity == crank_import::rest::ImportFindingSeverity::Error
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
let valid = service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&valid_preview.job_id.as_str().into(),
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /valid".to_owned()],
|
||||||
|
server_url: None,
|
||||||
|
conflict_mode: "skip".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(valid.created.len(), 1);
|
||||||
|
|
||||||
|
let full_preview = service
|
||||||
|
.preview_openapi_import(&workspace_id, upload)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let result = service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&full_preview.job_id.as_str().into(),
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /valid".to_owned(), "GET /broken".to_owned()],
|
||||||
|
server_url: None,
|
||||||
|
conflict_mode: "rename".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(
|
||||||
|
service.list_operations(&workspace_id).await.unwrap().len(),
|
||||||
|
1,
|
||||||
|
"the full selection must not create either selected Draft"
|
||||||
|
);
|
||||||
|
let failed = registry
|
||||||
|
.get_import_job(&workspace_id, &full_preview.job_id.as_str().into())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(failed.status, ImportJobStatus::Failed);
|
||||||
|
assert_eq!(
|
||||||
|
failed.error_text.as_deref(),
|
||||||
|
Some("reference_resolution_blocked")
|
||||||
|
);
|
||||||
|
assert!(failed.finished_at.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn local_operation_reference_resolves_and_creates_draft() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let service = test_service(
|
||||||
|
registry,
|
||||||
|
test_storage_root("openapi_import_operation_blocker"),
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let upload = OpenApiUpload {
|
||||||
|
bytes: br#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: References }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/referenced:
|
||||||
|
get:
|
||||||
|
operationId: referenced
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/Result' }
|
||||||
|
components:
|
||||||
|
schemas:
|
||||||
|
Result: { type: object, properties: { id: { type: string } } }
|
||||||
|
"#
|
||||||
|
.to_vec(),
|
||||||
|
mime_type: "application/yaml".to_owned(),
|
||||||
|
locale: OpenApiUploadLocale::En,
|
||||||
|
};
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(&workspace_id, upload)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
preview.preview.findings.iter().all(|finding| {
|
||||||
|
finding.severity != crank_import::rest::ImportFindingSeverity::Error
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
preview.preview.groups[0].operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.all(|finding| finding.code != "unresolved_reference")
|
||||||
|
);
|
||||||
|
assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1);
|
||||||
|
|
||||||
|
let created = service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&preview.job_id.as_str().into(),
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /referenced".to_owned()],
|
||||||
|
server_url: None,
|
||||||
|
conflict_mode: "skip".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(created.created.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
service.list_operations(&workspace_id).await.unwrap().len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn openapi_upload() -> OpenApiUpload {
|
fn openapi_upload() -> OpenApiUpload {
|
||||||
OpenApiUpload {
|
OpenApiUpload {
|
||||||
bytes: OPENAPI3.as_bytes().to_vec(),
|
bytes: OPENAPI3.as_bytes().to_vec(),
|
||||||
@@ -328,3 +738,44 @@ fn openapi_upload() -> OpenApiUpload {
|
|||||||
locale: OpenApiUploadLocale::En,
|
locale: OpenApiUploadLocale::En,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn assert_failed_job_and_detached_sources(
|
||||||
|
registry: &crank_registry::PostgresRegistry,
|
||||||
|
workspace_id: &WorkspaceId,
|
||||||
|
job_id: &crank_registry::ImportJobId,
|
||||||
|
expected_error: &str,
|
||||||
|
) {
|
||||||
|
let job = registry
|
||||||
|
.get_import_job(workspace_id, job_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(job.status, ImportJobStatus::Failed);
|
||||||
|
assert!(job.finished_at.is_some());
|
||||||
|
assert_eq!(job.error_text.as_deref(), Some(expected_error));
|
||||||
|
|
||||||
|
let mut source_ids = vec![
|
||||||
|
job.preview_payload["source"]["source_id"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.to_owned(),
|
||||||
|
];
|
||||||
|
source_ids.extend(
|
||||||
|
job.preview_payload["dependencies"]
|
||||||
|
.as_array()
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.map(|dependency| dependency["source_id"].as_str().unwrap().to_owned()),
|
||||||
|
);
|
||||||
|
for source_id in source_ids {
|
||||||
|
let lifecycle: String = sqlx::query_scalar(
|
||||||
|
"select lifecycle from artifact_sources where workspace_id = $1 and source_id = $2",
|
||||||
|
)
|
||||||
|
.bind(workspace_id.as_str())
|
||||||
|
.bind(source_id)
|
||||||
|
.fetch_one(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(lifecycle, "detached");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,538 @@
|
|||||||
|
use super::super::common::test_service_with_external_references;
|
||||||
|
use super::*;
|
||||||
|
use axum::{Router, routing::get};
|
||||||
|
use std::sync::{
|
||||||
|
Arc,
|
||||||
|
atomic::{AtomicUsize, Ordering},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn external_relative_chain_survives_reversed_builder_order_and_apply_never_refetches() {
|
||||||
|
let fetches = Arc::new(AtomicUsize::new(0));
|
||||||
|
let root_fetches = Arc::clone(&fetches);
|
||||||
|
let child_fetches = Arc::clone(&fetches);
|
||||||
|
let app = Router::new()
|
||||||
|
.route(
|
||||||
|
"/root.yaml",
|
||||||
|
get(move || {
|
||||||
|
let fetches = Arc::clone(&root_fetches);
|
||||||
|
async move {
|
||||||
|
fetches.fetch_add(1, Ordering::SeqCst);
|
||||||
|
"Item: { $ref: './child.yaml#/Item' }"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/child.yaml",
|
||||||
|
get(move || {
|
||||||
|
let fetches = Arc::clone(&child_fetches);
|
||||||
|
async move {
|
||||||
|
fetches.fetch_add(1, Ordering::SeqCst);
|
||||||
|
"Item: { type: object, required: [id], properties: { id: { type: string } } }"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||||
|
let origin = format!("http://{address}");
|
||||||
|
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let service = test_service_with_external_references(
|
||||||
|
registry.clone(),
|
||||||
|
test_storage_root("openapi_import_external_snapshot"),
|
||||||
|
vec![format!("{origin}/")],
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let upload = OpenApiUpload {
|
||||||
|
bytes: format!(
|
||||||
|
r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: {{ title: External }}
|
||||||
|
servers: [{{ url: https://api.example.test }}]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: {{ $ref: '{origin}/root.yaml#/Item' }}
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.into_bytes(),
|
||||||
|
mime_type: "application/yaml".to_owned(),
|
||||||
|
locale: OpenApiUploadLocale::En,
|
||||||
|
};
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(&workspace_id, upload)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||||||
|
assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1);
|
||||||
|
let job = registry
|
||||||
|
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
job.preview_payload["dependencies"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
job.preview_payload["dependency_snapshots"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
|
||||||
|
let applied = service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&preview.job_id.as_str().into(),
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /items".to_owned()],
|
||||||
|
server_url: None,
|
||||||
|
conflict_mode: "skip".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(applied.created.len(), 1);
|
||||||
|
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||||||
|
let active_sources: i64 = sqlx::query_scalar(
|
||||||
|
"select count(*) from artifact_sources
|
||||||
|
where source_id like 'src_openapi_%' and lifecycle = 'active'",
|
||||||
|
)
|
||||||
|
.fetch_one(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(active_sources, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn missing_external_snapshot_fails_closed_before_draft_mutation() {
|
||||||
|
let app = Router::new().route(
|
||||||
|
"/schemas.yaml",
|
||||||
|
get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||||
|
let origin = format!("http://{address}");
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let storage_root = test_storage_root("openapi_import_missing_external_snapshot");
|
||||||
|
let service = test_service_with_external_references(
|
||||||
|
registry.clone(),
|
||||||
|
storage_root.clone(),
|
||||||
|
vec![format!("{origin}/")],
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
OpenApiUpload {
|
||||||
|
bytes: format!(
|
||||||
|
r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: {{ title: External integrity }}
|
||||||
|
servers: [{{ url: https://api.example.test }}]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.into_bytes(),
|
||||||
|
mime_type: "application/yaml".to_owned(),
|
||||||
|
locale: OpenApiUploadLocale::En,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let job = registry
|
||||||
|
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let dependency_source_id = job.preview_payload["dependencies"][0]["source_id"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"update artifact_sources
|
||||||
|
set lifecycle = 'detached', updated_at = now(), detached_at = now()
|
||||||
|
where workspace_id = $1 and source_id = $2",
|
||||||
|
)
|
||||||
|
.bind(workspace_id.as_str())
|
||||||
|
.bind(dependency_source_id)
|
||||||
|
.execute(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&preview.job_id.as_str().into(),
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /items".to_owned()],
|
||||||
|
server_url: None,
|
||||||
|
conflict_mode: "skip".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
service
|
||||||
|
.list_operations(&workspace_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert_failed_job_and_detached_sources(
|
||||||
|
®istry,
|
||||||
|
&workspace_id,
|
||||||
|
&preview.job_id.as_str().into(),
|
||||||
|
"import_dependency_verification_failed",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let corrupt_preview = service
|
||||||
|
.preview_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
OpenApiUpload {
|
||||||
|
bytes: format!(
|
||||||
|
r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: {{ title: Corrupt external integrity }}
|
||||||
|
servers: [{{ url: https://api.example.test }}]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItemsCorrupt
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.into_bytes(),
|
||||||
|
mime_type: "application/yaml".to_owned(),
|
||||||
|
locale: OpenApiUploadLocale::En,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let corrupt_job = registry
|
||||||
|
.get_import_job(&workspace_id, &corrupt_preview.job_id.as_str().into())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let dependency_ref: crank_artifacts::ArtifactRef = corrupt_job.preview_payload["dependencies"]
|
||||||
|
[0]["digest"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.parse()
|
||||||
|
.unwrap();
|
||||||
|
let dependency_path = storage_root
|
||||||
|
.join("sha256")
|
||||||
|
.join(&dependency_ref.digest_hex()[..2])
|
||||||
|
.join(dependency_ref.digest_hex());
|
||||||
|
std::fs::set_permissions(
|
||||||
|
&dependency_path,
|
||||||
|
std::os::unix::fs::PermissionsExt::from_mode(0o600),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(&dependency_path, b"corrupt external dependency").unwrap();
|
||||||
|
std::fs::set_permissions(
|
||||||
|
&dependency_path,
|
||||||
|
std::os::unix::fs::PermissionsExt::from_mode(0o400),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&corrupt_preview.job_id.as_str().into(),
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /items".to_owned()],
|
||||||
|
server_url: None,
|
||||||
|
conflict_mode: "skip".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_failed_job_and_detached_sources(
|
||||||
|
®istry,
|
||||||
|
&workspace_id,
|
||||||
|
&corrupt_preview.job_id.as_str().into(),
|
||||||
|
"import_dependency_verification_failed",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn external_materialization_uses_one_wall_clock_timeout_and_cleans_cancelled_sources() {
|
||||||
|
let fetches = Arc::new(AtomicUsize::new(0));
|
||||||
|
let root_fetches = Arc::clone(&fetches);
|
||||||
|
let child_fetches = Arc::clone(&fetches);
|
||||||
|
let app = Router::new()
|
||||||
|
.route(
|
||||||
|
"/root.yaml",
|
||||||
|
get(move || {
|
||||||
|
let fetches = Arc::clone(&root_fetches);
|
||||||
|
async move {
|
||||||
|
fetches.fetch_add(1, Ordering::SeqCst);
|
||||||
|
"Item: { $ref: './slow-child.yaml#/Item' }"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/slow-child.yaml",
|
||||||
|
get(move || {
|
||||||
|
let fetches = Arc::clone(&child_fetches);
|
||||||
|
async move {
|
||||||
|
fetches.fetch_add(1, Ordering::SeqCst);
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||||
|
"Item: { type: object, properties: { id: { type: string } } }"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||||
|
let origin = format!("http://{address}");
|
||||||
|
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
|
||||||
|
let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
|
||||||
|
let service = AdminServiceBuilder::new(
|
||||||
|
registry.clone(),
|
||||||
|
test_storage_root("openapi_import_chain_timeout"),
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
runtime,
|
||||||
|
)
|
||||||
|
.with_external_reference_import(&crank_config::ExternalReferenceSettings {
|
||||||
|
allowed_url_prefixes: vec![format!("{origin}/")],
|
||||||
|
max_depth: 8,
|
||||||
|
max_documents: 32,
|
||||||
|
max_fetch_bytes: 64 * 1024,
|
||||||
|
fetch_timeout_ms: 200,
|
||||||
|
max_expanded_nodes: 10_000,
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.with_outbound_http_policy(outbound_policy)
|
||||||
|
.build();
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let started = tokio::time::Instant::now();
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
OpenApiUpload {
|
||||||
|
bytes: format!(
|
||||||
|
r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: {{ title: Timed chain }}
|
||||||
|
servers: [{{ url: https://api.example.test }}]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: timedItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content: {{ application/json: {{ schema: {{ $ref: '{origin}/root.yaml#/Item' }} }} }}
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.into_bytes(),
|
||||||
|
mime_type: "application/yaml".to_owned(),
|
||||||
|
locale: OpenApiUploadLocale::En,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(started.elapsed() < std::time::Duration::from_millis(800));
|
||||||
|
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||||||
|
let job = registry
|
||||||
|
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
job.preview_payload["dependencies"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
|
||||||
|
for _ in 0..100 {
|
||||||
|
let active_dependencies: i64 = sqlx::query_scalar(
|
||||||
|
"select count(*) from artifact_sources
|
||||||
|
where source_id like 'src_openapi_dep_%' and lifecycle = 'active'",
|
||||||
|
)
|
||||||
|
.fetch_one(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
if active_dependencies == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
let detached_dependencies: i64 = sqlx::query_scalar(
|
||||||
|
"select count(*) from artifact_sources
|
||||||
|
where source_id like 'src_openapi_dep_%' and lifecycle = 'detached'",
|
||||||
|
)
|
||||||
|
.fetch_one(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(detached_dependencies, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn expired_job_detaches_primary_and_external_dependency_without_orphan() {
|
||||||
|
let app = Router::new().route(
|
||||||
|
"/schemas.yaml",
|
||||||
|
get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||||
|
let origin = format!("http://{address}");
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let service = test_service_with_external_references(
|
||||||
|
registry.clone(),
|
||||||
|
test_storage_root("openapi_import_external_expiry"),
|
||||||
|
vec![format!("{origin}/")],
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
OpenApiUpload {
|
||||||
|
bytes: format!(
|
||||||
|
r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: {{ title: Expiring external }}
|
||||||
|
servers: [{{ url: https://api.example.test }}]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.into_bytes(),
|
||||||
|
mime_type: "application/yaml".to_owned(),
|
||||||
|
locale: OpenApiUploadLocale::En,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query("update import_jobs set expires_at = now() - interval '1 second' where id = $1")
|
||||||
|
.bind(preview.job_id.as_str())
|
||||||
|
.execute(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let report = registry.cleanup_expired_import_jobs(16).await.unwrap();
|
||||||
|
assert_eq!(report.deleted_jobs, 1);
|
||||||
|
assert_eq!(report.detached_sources, 2);
|
||||||
|
let active_sources: i64 = sqlx::query_scalar(
|
||||||
|
"select count(*) from artifact_sources
|
||||||
|
where source_id like 'src_openapi_%' and lifecycle = 'active'",
|
||||||
|
)
|
||||||
|
.fetch_one(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(active_sources, 0);
|
||||||
|
}
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn exact_v2_contract_replays_persisted_preview_without_v3_or_dependency_reads() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let service = test_service(
|
||||||
|
registry.clone(),
|
||||||
|
test_storage_root("openapi_import_v2_compatibility"),
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
);
|
||||||
|
let workspace_id = WorkspaceId::new("ws_default");
|
||||||
|
let preview = service
|
||||||
|
.preview_openapi_import(&workspace_id, openapi_upload())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||||||
|
let missing_dependency = serde_json::json!([{
|
||||||
|
"source_id": "src_openapi_dep_v2_must_not_be_read",
|
||||||
|
"digest": format!("sha256:{}", "0".repeat(64)),
|
||||||
|
"canonical_uri": "https://schemas.example.test/v2.yaml"
|
||||||
|
}]);
|
||||||
|
sqlx::query(
|
||||||
|
"update import_jobs
|
||||||
|
set preview_payload = jsonb_set(
|
||||||
|
jsonb_set(
|
||||||
|
jsonb_set(
|
||||||
|
jsonb_set(
|
||||||
|
preview_payload,
|
||||||
|
'{normalization,normalizer_version}',
|
||||||
|
to_jsonb('normalized-ir-v2'::text)
|
||||||
|
),
|
||||||
|
'{normalization,projection_version}',
|
||||||
|
to_jsonb('preview-v2'::text)
|
||||||
|
),
|
||||||
|
'{normalization,ir_fingerprint}',
|
||||||
|
to_jsonb($1::text)
|
||||||
|
),
|
||||||
|
'{dependency_snapshots}',
|
||||||
|
$2::jsonb
|
||||||
|
)
|
||||||
|
where id = $3",
|
||||||
|
)
|
||||||
|
.bind("a".repeat(64))
|
||||||
|
.bind(missing_dependency)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.execute(registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let result = service
|
||||||
|
.create_openapi_import(
|
||||||
|
&workspace_id,
|
||||||
|
&job_id,
|
||||||
|
OpenApiImportCreatePayload {
|
||||||
|
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||||
|
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||||
|
conflict_mode: "skip".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.created.len(), 1);
|
||||||
|
let completed = registry
|
||||||
|
.get_import_job(&workspace_id, &job_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(completed.status, ImportJobStatus::Completed);
|
||||||
|
}
|
||||||
@@ -1,30 +1,18 @@
|
|||||||
use std::{
|
|
||||||
io,
|
|
||||||
sync::{Arc, Mutex},
|
|
||||||
};
|
|
||||||
|
|
||||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
|
|
||||||
use crank_artifacts::MAX_ARTIFACT_BYTES;
|
|
||||||
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
|
|
||||||
use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest};
|
|
||||||
use metrics_util::debugging::DebuggingRecorder;
|
|
||||||
use opentelemetry::trace::TracerProvider as _;
|
|
||||||
use opentelemetry_sdk::{
|
|
||||||
error::OTelSdkResult,
|
|
||||||
trace::{SdkTracerProvider, SpanData, SpanExporter},
|
|
||||||
};
|
|
||||||
use reqwest::multipart::{Form, Part};
|
|
||||||
use serde_json::Value;
|
|
||||||
use serial_test::serial;
|
|
||||||
use time::{Duration, OffsetDateTime};
|
|
||||||
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
|
|
||||||
|
|
||||||
use super::common::{
|
use super::common::{
|
||||||
authorized_client, build_test_app, spawn_admin_api, test_auth_settings, test_registry,
|
authorized_client, build_test_app, spawn_admin_api, test_auth_settings, test_registry,
|
||||||
test_secret_crypto, test_service, test_storage_root,
|
test_secret_crypto, test_service, test_storage_root,
|
||||||
};
|
};
|
||||||
|
use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
|
||||||
|
use crank_artifacts::MAX_ARTIFACT_BYTES;
|
||||||
|
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
|
||||||
|
use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest};
|
||||||
|
use reqwest::multipart::{Form, Part};
|
||||||
|
use serde_json::Value;
|
||||||
|
use serial_test::serial;
|
||||||
|
use time::{Duration, OffsetDateTime};
|
||||||
|
|
||||||
mod apply_failures;
|
mod apply_failures;
|
||||||
|
mod telemetry;
|
||||||
|
|
||||||
const OPENAPI: &str = r#"
|
const OPENAPI: &str = r#"
|
||||||
openapi: 3.0.3
|
openapi: 3.0.3
|
||||||
@@ -136,78 +124,6 @@ async fn multipart_requires_an_owner_membership_before_reading_the_file() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
#[serial]
|
|
||||||
async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() {
|
|
||||||
const CANARY: &str = "openapi-telemetry-secret-canary";
|
|
||||||
|
|
||||||
let recorder = DebuggingRecorder::new();
|
|
||||||
let snapshotter = recorder.snapshotter();
|
|
||||||
recorder
|
|
||||||
.install()
|
|
||||||
.expect("isolated integration test metrics recorder");
|
|
||||||
|
|
||||||
let writer = SharedLogWriter::default();
|
|
||||||
let exported = Arc::new(Mutex::new(Vec::new()));
|
|
||||||
let provider = SdkTracerProvider::builder()
|
|
||||||
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
|
|
||||||
.build();
|
|
||||||
let tracer = provider.tracer("admin-openapi-source-test");
|
|
||||||
let subscriber = tracing_subscriber::registry()
|
|
||||||
.with(tracing_subscriber::fmt::layer().with_writer(writer.clone()))
|
|
||||||
.with(tracing_opentelemetry::layer().with_tracer(tracer));
|
|
||||||
let dispatch = tracing::Dispatch::new(subscriber);
|
|
||||||
tracing::dispatcher::set_global_default(dispatch)
|
|
||||||
.expect("isolated integration test tracing subscriber");
|
|
||||||
|
|
||||||
let registry = test_registry().await;
|
|
||||||
let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary"));
|
|
||||||
let server = spawn_admin_api(app).await;
|
|
||||||
let client = authorized_client(&server).await;
|
|
||||||
let document = format!(
|
|
||||||
"openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n /broken:\n get:\n description: {CANARY}\n responses: ["
|
|
||||||
);
|
|
||||||
let response = client
|
|
||||||
.post(format!("{server}/imports/openapi/preview"))
|
|
||||||
.multipart(Form::new().part(
|
|
||||||
"file",
|
|
||||||
file_part(document.as_bytes(), "openapi.yaml", "application/yaml"),
|
|
||||||
))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
|
|
||||||
let trace_id = response
|
|
||||||
.headers()
|
|
||||||
.get("x-trace-id")
|
|
||||||
.unwrap()
|
|
||||||
.to_str()
|
|
||||||
.unwrap()
|
|
||||||
.to_owned();
|
|
||||||
let body = response.text().await.unwrap();
|
|
||||||
assert!(!body.contains(CANARY));
|
|
||||||
let diagnostics: Value = serde_json::from_str(&body).unwrap();
|
|
||||||
assert_eq!(diagnostics["error"]["trace_id"], trace_id);
|
|
||||||
|
|
||||||
provider.force_flush().unwrap();
|
|
||||||
let logs = writer.output();
|
|
||||||
assert!(!logs.contains(CANARY));
|
|
||||||
assert!(logs.contains(&trace_id));
|
|
||||||
let spans = exported.lock().unwrap();
|
|
||||||
let rendered_spans = format!("{spans:?}");
|
|
||||||
assert!(!rendered_spans.contains(CANARY));
|
|
||||||
drop(spans);
|
|
||||||
for (key, _, _, _) in snapshotter.snapshot().into_vec() {
|
|
||||||
assert!(!key.key().name().contains(CANARY));
|
|
||||||
assert!(
|
|
||||||
!key.key()
|
|
||||||
.labels()
|
|
||||||
.any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) })
|
|
||||||
);
|
|
||||||
}
|
|
||||||
provider.shutdown().unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn multipart_boundary_rejections_are_localized_and_leave_no_entities() {
|
async fn multipart_boundary_rejections_are_localized_and_leave_no_entities() {
|
||||||
@@ -939,49 +855,3 @@ async fn wait_for_specific_source_lifecycle(
|
|||||||
}
|
}
|
||||||
panic!("artifact source {source_id:?} did not reach {expected}");
|
panic!("artifact source {source_id:?} did not reach {expected}");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Default)]
|
|
||||||
struct SharedLogWriter {
|
|
||||||
buffer: Arc<Mutex<Vec<u8>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SharedLogWriter {
|
|
||||||
fn output(&self) -> String {
|
|
||||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
|
||||||
type Writer = SharedLogGuard;
|
|
||||||
|
|
||||||
fn make_writer(&'a self) -> Self::Writer {
|
|
||||||
SharedLogGuard {
|
|
||||||
buffer: Arc::clone(&self.buffer),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
struct SharedLogGuard {
|
|
||||||
buffer: Arc<Mutex<Vec<u8>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl io::Write for SharedLogGuard {
|
|
||||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
|
||||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
|
||||||
Ok(bytes.len())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn flush(&mut self) -> io::Result<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
|
||||||
|
|
||||||
impl SpanExporter for CapturingExporter {
|
|
||||||
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
|
|
||||||
self.0.lock().unwrap().extend(batch);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,237 @@
|
|||||||
|
use super::super::common::build_test_app_with_external_references;
|
||||||
|
use super::*;
|
||||||
|
use std::{
|
||||||
|
io,
|
||||||
|
sync::{Arc, Mutex},
|
||||||
|
};
|
||||||
|
|
||||||
|
use metrics_util::debugging::DebuggingRecorder;
|
||||||
|
use opentelemetry::trace::TracerProvider as _;
|
||||||
|
use opentelemetry_sdk::{
|
||||||
|
error::OTelSdkResult,
|
||||||
|
trace::{SdkTracerProvider, SpanData, SpanExporter},
|
||||||
|
};
|
||||||
|
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
|
async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() {
|
||||||
|
const CANARY: &str = "openapi-telemetry-secret-canary";
|
||||||
|
|
||||||
|
let recorder = DebuggingRecorder::new();
|
||||||
|
let snapshotter = recorder.snapshotter();
|
||||||
|
recorder
|
||||||
|
.install()
|
||||||
|
.expect("isolated integration test metrics recorder");
|
||||||
|
|
||||||
|
let writer = SharedLogWriter::default();
|
||||||
|
let exported = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
let provider = SdkTracerProvider::builder()
|
||||||
|
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
|
||||||
|
.build();
|
||||||
|
let tracer = provider.tracer("admin-openapi-source-test");
|
||||||
|
let subscriber = tracing_subscriber::registry()
|
||||||
|
.with(
|
||||||
|
tracing_subscriber::fmt::layer()
|
||||||
|
.with_ansi(false)
|
||||||
|
.with_writer(writer.clone()),
|
||||||
|
)
|
||||||
|
.with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||||
|
let dispatch = tracing::Dispatch::new(subscriber);
|
||||||
|
tracing::dispatcher::set_global_default(dispatch)
|
||||||
|
.expect("isolated integration test tracing subscriber");
|
||||||
|
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary"));
|
||||||
|
let server = spawn_admin_api(app).await;
|
||||||
|
let client = authorized_client(&server).await;
|
||||||
|
let document = format!(
|
||||||
|
"openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n /broken:\n get:\n description: {CANARY}\n responses: ["
|
||||||
|
);
|
||||||
|
let response = client
|
||||||
|
.post(format!("{server}/imports/openapi/preview"))
|
||||||
|
.multipart(Form::new().part(
|
||||||
|
"file",
|
||||||
|
file_part(document.as_bytes(), "openapi.yaml", "application/yaml"),
|
||||||
|
))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||||
|
let trace_id = response
|
||||||
|
.headers()
|
||||||
|
.get("x-trace-id")
|
||||||
|
.unwrap()
|
||||||
|
.to_str()
|
||||||
|
.unwrap()
|
||||||
|
.to_owned();
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
assert!(!body.contains(CANARY));
|
||||||
|
let diagnostics: Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(diagnostics["error"]["trace_id"], trace_id);
|
||||||
|
|
||||||
|
let route = format!("/{CANARY}.yaml");
|
||||||
|
let external = axum::Router::new().route(
|
||||||
|
&route,
|
||||||
|
axum::routing::get(|| async { (axum::http::StatusCode::INTERNAL_SERVER_ERROR, CANARY) }),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
tokio::spawn(async move { axum::serve(listener, external).await.unwrap() });
|
||||||
|
let origin = format!("http://{address}");
|
||||||
|
let external_registry = test_registry().await;
|
||||||
|
let external_app = build_test_app_with_external_references(
|
||||||
|
external_registry,
|
||||||
|
test_storage_root("openapi_materialization_telemetry_canary"),
|
||||||
|
vec![format!("{origin}/")],
|
||||||
|
);
|
||||||
|
let external_server = spawn_admin_api(external_app).await;
|
||||||
|
let external_client = authorized_client(&external_server).await;
|
||||||
|
let external_document = format!(
|
||||||
|
r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: {{ title: Safe materialization }}
|
||||||
|
servers: [{{ url: https://api.example.test }}]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: {{ $ref: '{origin}/{CANARY}.yaml#/Item' }}
|
||||||
|
"#
|
||||||
|
);
|
||||||
|
let response = external_client
|
||||||
|
.post(format!("{external_server}/imports/openapi/preview"))
|
||||||
|
.multipart(Form::new().part(
|
||||||
|
"file",
|
||||||
|
file_part(
|
||||||
|
external_document.as_bytes(),
|
||||||
|
"external.yaml",
|
||||||
|
"application/yaml",
|
||||||
|
),
|
||||||
|
))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||||
|
|
||||||
|
provider.force_flush().unwrap();
|
||||||
|
let logs = writer.output();
|
||||||
|
assert!(!logs.contains(CANARY));
|
||||||
|
assert!(logs.contains(&trace_id));
|
||||||
|
let safe_materialization_failure = logs.lines().any(is_safe_materialization_failure);
|
||||||
|
assert!(
|
||||||
|
safe_materialization_failure,
|
||||||
|
"expected one bounded, sanitized materialization failure"
|
||||||
|
);
|
||||||
|
let spans = exported.lock().unwrap();
|
||||||
|
let rendered_spans = format!("{spans:?}");
|
||||||
|
assert!(!rendered_spans.contains(CANARY));
|
||||||
|
drop(spans);
|
||||||
|
for (key, _, _, _) in snapshotter.snapshot().into_vec() {
|
||||||
|
assert!(!key.key().name().contains(CANARY));
|
||||||
|
assert!(
|
||||||
|
!key.key()
|
||||||
|
.labels()
|
||||||
|
.any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) })
|
||||||
|
);
|
||||||
|
}
|
||||||
|
provider.shutdown().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_safe_materialization_failure(line: &str) -> bool {
|
||||||
|
if !line.contains("external OpenAPI materialization failed") || !line.contains("count=0") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let immediate_status = contains_log_field(line, "stage", "fetch")
|
||||||
|
&& contains_log_field(line, "error_code", "unexpected_status");
|
||||||
|
let bounded_timeout = contains_log_field(line, "stage", "chain")
|
||||||
|
&& contains_log_field(line, "error_code", "timeout");
|
||||||
|
immediate_status || bounded_timeout
|
||||||
|
}
|
||||||
|
|
||||||
|
fn contains_log_field(line: &str, name: &str, value: &str) -> bool {
|
||||||
|
line.contains(&format!("{name}=\"{value}\"")) || line.contains(&format!("{name}={value}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn materialization_log_accepts_only_coherent_safe_outcomes() {
|
||||||
|
for line in [
|
||||||
|
r#"external OpenAPI materialization failed stage="fetch" error_code="unexpected_status" count=0"#,
|
||||||
|
"external OpenAPI materialization failed stage=chain error_code=timeout count=0",
|
||||||
|
] {
|
||||||
|
assert!(is_safe_materialization_failure(line));
|
||||||
|
}
|
||||||
|
assert!(!is_safe_materialization_failure(
|
||||||
|
r#"external OpenAPI materialization failed stage="fetch" error_code="timeout" count=0"#
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct SharedLogWriter {
|
||||||
|
buffer: Arc<Mutex<Vec<u8>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SharedLogWriter {
|
||||||
|
fn output(&self) -> String {
|
||||||
|
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||||
|
type Writer = SharedLogGuard;
|
||||||
|
|
||||||
|
fn make_writer(&'a self) -> Self::Writer {
|
||||||
|
SharedLogGuard {
|
||||||
|
buffer: Arc::clone(&self.buffer),
|
||||||
|
pending: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct SharedLogGuard {
|
||||||
|
buffer: Arc<Mutex<Vec<u8>>>,
|
||||||
|
pending: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl io::Write for SharedLogGuard {
|
||||||
|
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||||
|
self.pending.extend_from_slice(bytes);
|
||||||
|
Ok(bytes.len())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush(&mut self) -> io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for SharedLogGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.buffer.lock().unwrap().extend_from_slice(&self.pending);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn shared_log_writer_publishes_complete_records_only() {
|
||||||
|
let writer = SharedLogWriter::default();
|
||||||
|
let mut guard = writer.make_writer();
|
||||||
|
io::Write::write_all(&mut guard, b"message").unwrap();
|
||||||
|
assert!(writer.output().is_empty());
|
||||||
|
|
||||||
|
io::Write::write_all(&mut guard, b" stage=\"fetch\"\n").unwrap();
|
||||||
|
drop(guard);
|
||||||
|
assert_eq!(writer.output(), "message stage=\"fetch\"\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
||||||
|
|
||||||
|
impl SpanExporter for CapturingExporter {
|
||||||
|
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
|
||||||
|
self.0.lock().unwrap().extend(batch);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,7 +40,7 @@ fn plan_is_deterministic_and_committed_contract_is_current() {
|
|||||||
);
|
);
|
||||||
assert_eq!(first.stdout, second.stdout);
|
assert_eq!(first.stdout, second.stdout);
|
||||||
let plan: serde_json::Value = serde_json::from_slice(&first.stdout).unwrap();
|
let plan: serde_json::Value = serde_json::from_slice(&first.stdout).unwrap();
|
||||||
assert_eq!(plan["sequence"].as_array().unwrap().len(), 12);
|
assert_eq!(plan["sequence"].as_array().unwrap().len(), 13);
|
||||||
|
|
||||||
let checked = command(&["plan", "--check"], None);
|
let checked = command(&["plan", "--check"], None);
|
||||||
assert!(
|
assert!(
|
||||||
@@ -82,7 +82,7 @@ async fn database_only_config_can_apply_and_preflight_a_fresh_schema() {
|
|||||||
);
|
);
|
||||||
let result: serde_json::Value = serde_json::from_slice(&preflight.stdout).unwrap();
|
let result: serde_json::Value = serde_json::from_slice(&preflight.stdout).unwrap();
|
||||||
assert_eq!(result["status"], "current");
|
assert_eq!(result["status"], "current");
|
||||||
assert_eq!(result["version"], 12);
|
assert_eq!(result["version"], 13);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
use admin_api::request_context::{TRACE_ID_HEADER, apply_request_context};
|
||||||
|
use axum::{Router, body::Body, http::Request, routing::get};
|
||||||
|
use opentelemetry::trace::TracerProvider as _;
|
||||||
|
use opentelemetry_sdk::trace::SdkTracerProvider;
|
||||||
|
use tower::ServiceExt;
|
||||||
|
use tracing_subscriber::layer::SubscriberExt;
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn preserves_remote_trace_id_with_an_active_tracer_and_no_global_propagator() {
|
||||||
|
let provider = SdkTracerProvider::builder().build();
|
||||||
|
let tracer = provider.tracer("admin-request-context-parent-test");
|
||||||
|
let subscriber =
|
||||||
|
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||||
|
let dispatch = tracing::Dispatch::new(subscriber);
|
||||||
|
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
|
||||||
|
let app = Router::new()
|
||||||
|
.route("/probe", get(|| async { "ok" }))
|
||||||
|
.layer(axum::middleware::from_fn(apply_request_context));
|
||||||
|
|
||||||
|
let response = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.uri("/probe")
|
||||||
|
.header(
|
||||||
|
"traceparent",
|
||||||
|
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
|
||||||
|
)
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()[TRACE_ID_HEADER],
|
||||||
|
"0af7651916cd43dd8448eb211c80319c"
|
||||||
|
);
|
||||||
|
provider.shutdown().unwrap();
|
||||||
|
}
|
||||||
@@ -53,13 +53,15 @@ RUN --mount=type=cache,id=crank-mcp-cargo-registry,target=/usr/local/cargo/regis
|
|||||||
|
|
||||||
FROM debian:bookworm-slim
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
RUN apt-get update \
|
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||||
&& apt-get install -y --no-install-recommends ca-certificates curl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY --from=builder /tmp/mcp-server /usr/local/bin/mcp-server
|
COPY --from=builder /tmp/mcp-server /usr/local/bin/mcp-server
|
||||||
|
COPY scripts/docker-http-healthcheck.sh /usr/local/bin/crank-http-healthcheck
|
||||||
|
|
||||||
|
RUN test -s /etc/ssl/certs/ca-certificates.crt \
|
||||||
|
&& chmod 0755 /usr/local/bin/crank-http-healthcheck
|
||||||
|
|
||||||
ENV CRANK_MCP_BIND=0.0.0.0:3002
|
ENV CRANK_MCP_BIND=0.0.0.0:3002
|
||||||
|
|
||||||
|
|||||||
+5
-1
@@ -10,7 +10,11 @@ COPY crank-community.png ./crank-community.png
|
|||||||
|
|
||||||
RUN npm run build
|
RUN npm run build
|
||||||
|
|
||||||
FROM nginx:1.27-alpine
|
FROM nginx:1.30.4-alpine3.24-slim
|
||||||
|
|
||||||
|
RUN apk add --no-cache --upgrade \
|
||||||
|
'libcrypto3>=3.5.8-r0' \
|
||||||
|
'libssl3>=3.5.8-r0'
|
||||||
|
|
||||||
COPY apps/ui/nginx.conf /etc/nginx/conf.d/default.conf
|
COPY apps/ui/nginx.conf /etc/nginx/conf.d/default.conf
|
||||||
COPY --from=build /app/dist /usr/share/nginx/html
|
COPY --from=build /app/dist /usr/share/nginx/html
|
||||||
|
|||||||
@@ -72,6 +72,17 @@
|
|||||||
body:has(.drawer.open) .onboarding-trigger,
|
body:has(.drawer.open) .onboarding-trigger,
|
||||||
body:has(.drawer.open) .onboarding-panel { z-index: 149; }
|
body:has(.drawer.open) .onboarding-panel { z-index: 149; }
|
||||||
|
|
||||||
|
/* The wizard owns a fixed bottom action bar. Keep this optional helper above
|
||||||
|
it so the primary Continue action remains both visible and clickable. */
|
||||||
|
body.wizard-page .onboarding-trigger {
|
||||||
|
bottom: calc(96px + env(safe-area-inset-bottom, 0px));
|
||||||
|
}
|
||||||
|
|
||||||
|
body.wizard-page .onboarding-panel {
|
||||||
|
bottom: calc(150px + env(safe-area-inset-bottom, 0px));
|
||||||
|
max-height: min(700px, calc(100vh - 178px));
|
||||||
|
}
|
||||||
|
|
||||||
@media (max-width: 640px) {
|
@media (max-width: 640px) {
|
||||||
.onboarding-trigger { right: 14px; bottom: 14px; }
|
.onboarding-trigger { right: 14px; bottom: 14px; }
|
||||||
.onboarding-panel { right: 14px; bottom: 68px; }
|
.onboarding-panel { right: 14px; bottom: 68px; }
|
||||||
|
|||||||
@@ -17,6 +17,30 @@ POSTGRES_HOST="${CRANK_E2E_POSTGRES_HOST:-127.0.0.1}"
|
|||||||
USE_EXTERNAL_POSTGRES="${CRANK_E2E_USE_EXTERNAL_POSTGRES:-0}"
|
USE_EXTERNAL_POSTGRES="${CRANK_E2E_USE_EXTERNAL_POSTGRES:-0}"
|
||||||
ADMIN_EMAIL="${CRANK_E2E_ADMIN_EMAIL:-owner@crank.local}"
|
ADMIN_EMAIL="${CRANK_E2E_ADMIN_EMAIL:-owner@crank.local}"
|
||||||
ADMIN_PASSWORD="${CRANK_E2E_ADMIN_PASSWORD:-change-me-admin-password}"
|
ADMIN_PASSWORD="${CRANK_E2E_ADMIN_PASSWORD:-change-me-admin-password}"
|
||||||
|
STARTUP_TIMEOUT_SECONDS="${CRANK_E2E_STARTUP_TIMEOUT_SECONDS:-120}"
|
||||||
|
|
||||||
|
if [[ ! "$STARTUP_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
|
||||||
|
echo "CRANK_E2E_STARTUP_TIMEOUT_SECONDS must be a positive integer" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# These variables configure only this orchestration script. The application
|
||||||
|
# rejects unknown variables in the owned CRANK_* namespace, so never leak the
|
||||||
|
# E2E control plane into production process configuration.
|
||||||
|
unset \
|
||||||
|
CRANK_E2E_ADMIN_EMAIL \
|
||||||
|
CRANK_E2E_ADMIN_PASSWORD \
|
||||||
|
CRANK_E2E_ADMIN_PORT \
|
||||||
|
CRANK_E2E_MCP_PORT \
|
||||||
|
CRANK_E2E_POSTGRES_DB \
|
||||||
|
CRANK_E2E_POSTGRES_HOST \
|
||||||
|
CRANK_E2E_POSTGRES_PASSWORD \
|
||||||
|
CRANK_E2E_POSTGRES_PORT \
|
||||||
|
CRANK_E2E_POSTGRES_USER \
|
||||||
|
CRANK_E2E_STARTUP_TIMEOUT_SECONDS \
|
||||||
|
CRANK_E2E_STREAM_FIXTURE_PORT \
|
||||||
|
CRANK_E2E_UI_PORT \
|
||||||
|
CRANK_E2E_USE_EXTERNAL_POSTGRES
|
||||||
|
|
||||||
if [[ -f "$HOME/.cargo/env" ]]; then
|
if [[ -f "$HOME/.cargo/env" ]]; then
|
||||||
. "$HOME/.cargo/env"
|
. "$HOME/.cargo/env"
|
||||||
@@ -32,6 +56,7 @@ if [[ -z "$PYTHON_BIN" ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
mkdir -p "$LOG_DIR"
|
mkdir -p "$LOG_DIR"
|
||||||
|
find "$LOG_DIR" -maxdepth 1 -type f -name '*.log' -delete
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR/apps/ui"
|
cd "$ROOT_DIR/apps/ui"
|
||||||
@@ -77,10 +102,23 @@ trap 'exit 130' INT TERM
|
|||||||
|
|
||||||
cleanup
|
cleanup
|
||||||
|
|
||||||
|
show_startup_log() {
|
||||||
|
local log_file="$1"
|
||||||
|
if [[ -f "$log_file" ]]; then
|
||||||
|
echo "---- $log_file ----" >&2
|
||||||
|
tail -n 120 "$log_file" >&2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
wait_for_port() {
|
wait_for_port() {
|
||||||
local host="$1"
|
local label="$1"
|
||||||
local port="$2"
|
local host="$2"
|
||||||
until "$PYTHON_BIN" - "$host" "$port" <<'PY'
|
local port="$3"
|
||||||
|
local deadline=$((SECONDS + STARTUP_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
echo "Waiting for $label on $host:$port"
|
||||||
|
while true; do
|
||||||
|
if "$PYTHON_BIN" - "$host" "$port" <<'PY'
|
||||||
import socket, sys
|
import socket, sys
|
||||||
sock = socket.socket()
|
sock = socket.socket()
|
||||||
sock.settimeout(0.5)
|
sock.settimeout(0.5)
|
||||||
@@ -91,13 +129,75 @@ except OSError:
|
|||||||
finally:
|
finally:
|
||||||
sock.close()
|
sock.close()
|
||||||
PY
|
PY
|
||||||
do
|
then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "Timed out waiting for $label on $host:$port after ${STARTUP_TIMEOUT_SECONDS}s" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_http() {
|
||||||
|
local label="$1"
|
||||||
|
local url="$2"
|
||||||
|
local pid_file="$3"
|
||||||
|
local log_file="$4"
|
||||||
|
local deadline=$((SECONDS + STARTUP_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
echo "Waiting for $label at $url"
|
||||||
|
while true; do
|
||||||
|
if "$PYTHON_BIN" - "$url" <<'PY'
|
||||||
|
import sys, urllib.request
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(sys.argv[1], timeout=0.5) as response:
|
||||||
|
sys.exit(0 if 200 <= response.status < 400 else 1)
|
||||||
|
except Exception:
|
||||||
|
sys.exit(1)
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local pid=""
|
||||||
|
if [[ -f "$pid_file" ]]; then
|
||||||
|
pid="$(cat "$pid_file")"
|
||||||
|
fi
|
||||||
|
if [[ -z "$pid" ]] || ! kill -0 "$pid" >/dev/null 2>&1; then
|
||||||
|
echo "$label exited before becoming ready at $url" >&2
|
||||||
|
show_startup_log "$log_file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "Timed out waiting for $label at $url after ${STARTUP_TIMEOUT_SECONDS}s" >&2
|
||||||
|
show_startup_log "$log_file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_postgres_container() {
|
||||||
|
local deadline=$((SECONDS + STARTUP_TIMEOUT_SECONDS))
|
||||||
|
echo "Waiting for PostgreSQL container $POSTGRES_CONTAINER"
|
||||||
|
while ! docker exec "$POSTGRES_CONTAINER" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do
|
||||||
|
if ! docker inspect "$POSTGRES_CONTAINER" >/dev/null 2>&1; then
|
||||||
|
echo "PostgreSQL container $POSTGRES_CONTAINER exited before becoming ready" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "Timed out waiting for PostgreSQL container after ${STARTUP_TIMEOUT_SECONDS}s" >&2
|
||||||
|
docker logs "$POSTGRES_CONTAINER" >&2 || true
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
if [[ "$USE_EXTERNAL_POSTGRES" = "1" ]]; then
|
if [[ "$USE_EXTERNAL_POSTGRES" = "1" ]]; then
|
||||||
wait_for_port "$POSTGRES_HOST" "$POSTGRES_PORT"
|
wait_for_port "PostgreSQL" "$POSTGRES_HOST" "$POSTGRES_PORT" || exit 1
|
||||||
else
|
else
|
||||||
docker run -d --rm \
|
docker run -d --rm \
|
||||||
--name "$POSTGRES_CONTAINER" \
|
--name "$POSTGRES_CONTAINER" \
|
||||||
@@ -107,9 +207,7 @@ else
|
|||||||
-p "$POSTGRES_PORT:5432" \
|
-p "$POSTGRES_PORT:5432" \
|
||||||
postgres:16-alpine >/dev/null
|
postgres:16-alpine >/dev/null
|
||||||
|
|
||||||
until docker exec "$POSTGRES_CONTAINER" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do
|
wait_for_postgres_container || exit 1
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
export POSTGRES_HOST
|
export POSTGRES_HOST
|
||||||
@@ -141,40 +239,58 @@ chmod 700 "$CRANK_STORAGE_ROOT"
|
|||||||
) &
|
) &
|
||||||
echo $! > "$TMP_DIR/http-fixture.pid"
|
echo $! > "$TMP_DIR/http-fixture.pid"
|
||||||
|
|
||||||
until curl -fsS "http://127.0.0.1:$STREAM_FIXTURE_PORT/health" >/dev/null 2>&1; do
|
wait_for_http \
|
||||||
sleep 1
|
"Playwright HTTP fixture" \
|
||||||
done
|
"http://127.0.0.1:$STREAM_FIXTURE_PORT/health" \
|
||||||
|
"$TMP_DIR/http-fixture.pid" \
|
||||||
|
"$LOG_DIR/http-fixture.log" || exit 1
|
||||||
|
|
||||||
(
|
if ! (
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
cargo run -p admin-api --bin crank-migrate -- apply >"$LOG_DIR/migrate.log" 2>&1
|
cargo run -p admin-api --bin crank-migrate -- apply >"$LOG_DIR/migrate.log" 2>&1
|
||||||
)
|
); then
|
||||||
|
echo "Failed to apply E2E database migrations" >&2
|
||||||
|
show_startup_log "$LOG_DIR/migrate.log"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
BOOTSTRAP_JSON="$(
|
if ! (
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
cargo run -p admin-api --bin crank-migrate -- admin-auth bootstrap-create \
|
cargo run -p admin-api --bin crank-migrate -- admin-auth bootstrap-create \
|
||||||
--email "$ADMIN_EMAIL" \
|
--email "$ADMIN_EMAIL" \
|
||||||
--display-name "Crank E2E" \
|
--display-name "Crank E2E" \
|
||||||
>"$LOG_DIR/bootstrap-create.log" 2>&1
|
>"$LOG_DIR/bootstrap-create.log" 2>&1
|
||||||
tail -n 1 "$LOG_DIR/bootstrap-create.log"
|
); then
|
||||||
)"
|
echo "Failed to create the E2E bootstrap contract" >&2
|
||||||
BOOTSTRAP_TOKEN="$("$PYTHON_BIN" - <<'PY' "$BOOTSTRAP_JSON"
|
show_startup_log "$LOG_DIR/bootstrap-create.log"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
BOOTSTRAP_JSON="$(tail -n 1 "$LOG_DIR/bootstrap-create.log")"
|
||||||
|
if ! BOOTSTRAP_TOKEN="$("$PYTHON_BIN" - <<'PY' "$BOOTSTRAP_JSON"
|
||||||
import json, sys
|
import json, sys
|
||||||
print(json.loads(sys.argv[1])["bootstrap_token"])
|
print(json.loads(sys.argv[1])["bootstrap_token"])
|
||||||
PY
|
PY
|
||||||
)"
|
)"; then
|
||||||
|
echo "Failed to parse the E2E bootstrap token" >&2
|
||||||
|
show_startup_log "$LOG_DIR/bootstrap-create.log"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
printf '%s' "$BOOTSTRAP_TOKEN" >"$TMP_DIR/bootstrap-token.txt"
|
printf '%s' "$BOOTSTRAP_TOKEN" >"$TMP_DIR/bootstrap-token.txt"
|
||||||
printf '%s' "$ADMIN_PASSWORD" >"$TMP_DIR/admin-password.txt"
|
printf '%s' "$ADMIN_PASSWORD" >"$TMP_DIR/admin-password.txt"
|
||||||
printf '%s' "$CRANK_PASSWORD_PEPPER" >"$TMP_DIR/password-pepper.txt"
|
printf '%s' "$CRANK_PASSWORD_PEPPER" >"$TMP_DIR/password-pepper.txt"
|
||||||
|
|
||||||
(
|
if ! (
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
cargo run -p admin-api --bin crank-migrate -- admin-auth bootstrap-complete \
|
cargo run -p admin-api --bin crank-migrate -- admin-auth bootstrap-complete \
|
||||||
--token-file "$TMP_DIR/bootstrap-token.txt" \
|
--token-file "$TMP_DIR/bootstrap-token.txt" \
|
||||||
--password-file "$TMP_DIR/admin-password.txt" \
|
--password-file "$TMP_DIR/admin-password.txt" \
|
||||||
--password-pepper-file "$TMP_DIR/password-pepper.txt" \
|
--password-pepper-file "$TMP_DIR/password-pepper.txt" \
|
||||||
>"$LOG_DIR/bootstrap-complete.log" 2>&1
|
>"$LOG_DIR/bootstrap-complete.log" 2>&1
|
||||||
)
|
); then
|
||||||
|
echo "Failed to complete the E2E admin bootstrap" >&2
|
||||||
|
show_startup_log "$LOG_DIR/bootstrap-complete.log"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
@@ -183,9 +299,11 @@ printf '%s' "$CRANK_PASSWORD_PEPPER" >"$TMP_DIR/password-pepper.txt"
|
|||||||
) &
|
) &
|
||||||
echo $! > "$TMP_DIR/admin-api.pid"
|
echo $! > "$TMP_DIR/admin-api.pid"
|
||||||
|
|
||||||
until curl -fsS "http://127.0.0.1:$ADMIN_PORT/health" >/dev/null 2>&1; do
|
wait_for_http \
|
||||||
sleep 1
|
"admin-api" \
|
||||||
done
|
"http://127.0.0.1:$ADMIN_PORT/health" \
|
||||||
|
"$TMP_DIR/admin-api.pid" \
|
||||||
|
"$LOG_DIR/admin-api.log" || exit 1
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
@@ -196,9 +314,11 @@ done
|
|||||||
) &
|
) &
|
||||||
echo $! > "$TMP_DIR/mcp-server.pid"
|
echo $! > "$TMP_DIR/mcp-server.pid"
|
||||||
|
|
||||||
until curl -fsS "http://127.0.0.1:$MCP_PORT/health" >/dev/null 2>&1; do
|
wait_for_http \
|
||||||
sleep 1
|
"mcp-server" \
|
||||||
done
|
"http://127.0.0.1:$MCP_PORT/health" \
|
||||||
|
"$TMP_DIR/mcp-server.pid" \
|
||||||
|
"$LOG_DIR/mcp-server.log" || exit 1
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR/apps/ui"
|
cd "$ROOT_DIR/apps/ui"
|
||||||
@@ -206,9 +326,11 @@ done
|
|||||||
) &
|
) &
|
||||||
echo $! > "$TMP_DIR/ui-server.pid"
|
echo $! > "$TMP_DIR/ui-server.pid"
|
||||||
|
|
||||||
until curl -fsS "http://127.0.0.1:$UI_PORT/login" >/dev/null 2>&1; do
|
wait_for_http \
|
||||||
sleep 1
|
"Playwright UI server" \
|
||||||
done
|
"http://127.0.0.1:$UI_PORT/login" \
|
||||||
|
"$TMP_DIR/ui-server.pid" \
|
||||||
|
"$LOG_DIR/ui-server.log" || exit 1
|
||||||
|
|
||||||
echo "Crank UI e2e stack is ready on http://127.0.0.1:$UI_PORT"
|
echo "Crank UI e2e stack is ready on http://127.0.0.1:$UI_PORT"
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,25 @@
|
|||||||
const { test, expect } = require('@playwright/test');
|
const { test, expect } = require('@playwright/test');
|
||||||
const { getCurrentWorkspace, login, localized } = require('./helpers');
|
const { getCurrentWorkspace, login, localized } = require('./helpers');
|
||||||
|
|
||||||
|
async function expectOnboardingOutsideWizardNavigation(page) {
|
||||||
|
await expect(page.getByTestId('onboarding-trigger')).toBeVisible();
|
||||||
|
|
||||||
|
const overlapsContinue = await page.locator('#btn-continue').evaluate((continueButton) => {
|
||||||
|
const trigger = document.querySelector('[data-testid="onboarding-trigger"]');
|
||||||
|
if (!trigger) return false;
|
||||||
|
const continueRect = continueButton.getBoundingClientRect();
|
||||||
|
const triggerRect = trigger.getBoundingClientRect();
|
||||||
|
return !(
|
||||||
|
triggerRect.right <= continueRect.left
|
||||||
|
|| triggerRect.left >= continueRect.right
|
||||||
|
|| triggerRect.bottom <= continueRect.top
|
||||||
|
|| triggerRect.top >= continueRect.bottom
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(overlapsContinue).toBe(false);
|
||||||
|
}
|
||||||
|
|
||||||
test('mobile wizard progress connector crosses the indicator centers', async ({ page }) => {
|
test('mobile wizard progress connector crosses the indicator centers', async ({ page }) => {
|
||||||
await page.setViewportSize({ width: 720, height: 900 });
|
await page.setViewportSize({ width: 720, height: 900 });
|
||||||
await login(page);
|
await login(page);
|
||||||
@@ -24,6 +43,19 @@ test('mobile wizard progress connector crosses the indicator centers', async ({
|
|||||||
expect(geometry.topDelta).toBeLessThan(1);
|
expect(geometry.topDelta).toBeLessThan(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('onboarding helper does not cover wizard navigation', async ({ page }) => {
|
||||||
|
await login(page);
|
||||||
|
await page.goto('/wizard/');
|
||||||
|
await expectOnboardingOutsideWizardNavigation(page);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('onboarding helper does not cover wizard navigation on mobile', async ({ page }) => {
|
||||||
|
await page.setViewportSize({ width: 390, height: 844 });
|
||||||
|
await login(page);
|
||||||
|
await page.goto('/wizard/');
|
||||||
|
await expectOnboardingOutsideWizardNavigation(page);
|
||||||
|
});
|
||||||
|
|
||||||
test('wizard loads and protocol selection updates flow', async ({ page }) => {
|
test('wizard loads and protocol selection updates flow', async ({ page }) => {
|
||||||
await login(page);
|
await login(page);
|
||||||
await page.goto('/wizard/');
|
await page.goto('/wizard/');
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ use serde_json::Value;
|
|||||||
use tracing::{Instrument, Span};
|
use tracing::{Instrument, Span};
|
||||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||||
|
|
||||||
use crate::{RestAdapterError, RestRequest, RestResponse};
|
use crate::{ExternalReferenceFetchError, RestAdapterError, RestRequest, RestResponse};
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct RestAdapter {
|
pub struct RestAdapter {
|
||||||
@@ -33,6 +33,18 @@ pub struct RestAdapter {
|
|||||||
policy: OutboundHttpPolicy,
|
policy: OutboundHttpPolicy,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A deliberately separate, GET-only boundary for materializing external
|
||||||
|
/// OpenAPI documents. It has no execution metrics, tracing propagation, or
|
||||||
|
/// request construction semantics from [`RestAdapter`].
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct ExternalReferenceFetcher {
|
||||||
|
client: Result<Client, Arc<str>>,
|
||||||
|
policy: OutboundHttpPolicy,
|
||||||
|
allowed_url_prefixes: Vec<String>,
|
||||||
|
max_response_bytes: usize,
|
||||||
|
timeout: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub struct OutboundHttpPolicy {
|
pub struct OutboundHttpPolicy {
|
||||||
allowed_hosts: Vec<String>,
|
allowed_hosts: Vec<String>,
|
||||||
@@ -56,15 +68,7 @@ impl RestAdapter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn with_policy(policy: OutboundHttpPolicy) -> Self {
|
pub fn with_policy(policy: OutboundHttpPolicy) -> Self {
|
||||||
let resolver = Arc::new(PolicyDnsResolver {
|
let client = outbound_client(&policy);
|
||||||
policy: policy.clone(),
|
|
||||||
});
|
|
||||||
let client = Client::builder()
|
|
||||||
.redirect(redirect::Policy::none())
|
|
||||||
.no_proxy()
|
|
||||||
.dns_resolver(resolver)
|
|
||||||
.build()
|
|
||||||
.map_err(|error| Arc::<str>::from(error.to_string()));
|
|
||||||
|
|
||||||
Self { client, policy }
|
Self { client, policy }
|
||||||
}
|
}
|
||||||
@@ -195,6 +199,153 @@ impl RestAdapter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ExternalReferenceFetcher {
|
||||||
|
pub fn try_new(
|
||||||
|
policy: OutboundHttpPolicy,
|
||||||
|
allowed_url_prefixes: Vec<String>,
|
||||||
|
max_response_bytes: usize,
|
||||||
|
timeout: Duration,
|
||||||
|
) -> Result<Self, ExternalReferenceFetchError> {
|
||||||
|
if max_response_bytes == 0 || timeout.is_zero() {
|
||||||
|
return Err(ExternalReferenceFetchError::InvalidConfiguration);
|
||||||
|
}
|
||||||
|
let allowed_url_prefixes = allowed_url_prefixes
|
||||||
|
.into_iter()
|
||||||
|
.map(|prefix| canonical_external_reference_prefix(&prefix))
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
Ok(Self {
|
||||||
|
client: outbound_client(&policy),
|
||||||
|
policy,
|
||||||
|
allowed_url_prefixes,
|
||||||
|
max_response_bytes,
|
||||||
|
timeout,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetches one document with a bounded, headerless `GET`.
|
||||||
|
///
|
||||||
|
/// URL fragments are stripped because they address JSON Pointer targets in
|
||||||
|
/// the fetched document rather than a network resource.
|
||||||
|
pub async fn get(&self, url: &str) -> Result<Vec<u8>, ExternalReferenceFetchError> {
|
||||||
|
if self.allowed_url_prefixes.is_empty() {
|
||||||
|
return Err(ExternalReferenceFetchError::Disabled);
|
||||||
|
}
|
||||||
|
let mut url =
|
||||||
|
reqwest::Url::parse(url).map_err(|_| ExternalReferenceFetchError::InvalidUrl)?;
|
||||||
|
url.set_fragment(None);
|
||||||
|
if !self
|
||||||
|
.allowed_url_prefixes
|
||||||
|
.iter()
|
||||||
|
.any(|prefix| matches_external_reference_prefix(&url, prefix))
|
||||||
|
{
|
||||||
|
return Err(ExternalReferenceFetchError::TargetNotAllowed);
|
||||||
|
}
|
||||||
|
self.policy
|
||||||
|
.validate_url(&url)
|
||||||
|
.map_err(external_policy_error)?;
|
||||||
|
let client = self
|
||||||
|
.client
|
||||||
|
.as_ref()
|
||||||
|
.map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?;
|
||||||
|
let response = client
|
||||||
|
.get(url)
|
||||||
|
.timeout(self.timeout)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(external_transport_error)?;
|
||||||
|
let status = response.status();
|
||||||
|
if status.is_redirection() {
|
||||||
|
return Err(ExternalReferenceFetchError::RedirectNotAllowed);
|
||||||
|
}
|
||||||
|
if !status.is_success() {
|
||||||
|
return Err(ExternalReferenceFetchError::UnexpectedStatus {
|
||||||
|
status: status.as_u16(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
read_external_response_bytes(response, self.max_response_bytes).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn outbound_client(policy: &OutboundHttpPolicy) -> Result<Client, Arc<str>> {
|
||||||
|
let resolver = Arc::new(PolicyDnsResolver {
|
||||||
|
policy: policy.clone(),
|
||||||
|
});
|
||||||
|
Client::builder()
|
||||||
|
.redirect(redirect::Policy::none())
|
||||||
|
.no_proxy()
|
||||||
|
.dns_resolver(resolver)
|
||||||
|
.build()
|
||||||
|
.map_err(|error| Arc::<str>::from(error.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn canonical_external_reference_prefix(
|
||||||
|
prefix: &str,
|
||||||
|
) -> Result<String, ExternalReferenceFetchError> {
|
||||||
|
let url = reqwest::Url::parse(prefix)
|
||||||
|
.map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?;
|
||||||
|
if !matches!(url.scheme(), "http" | "https")
|
||||||
|
|| url.host_str().is_none()
|
||||||
|
|| !url.username().is_empty()
|
||||||
|
|| url.password().is_some()
|
||||||
|
|| url.query().is_some()
|
||||||
|
|| url.fragment().is_some()
|
||||||
|
{
|
||||||
|
return Err(ExternalReferenceFetchError::InvalidConfiguration);
|
||||||
|
}
|
||||||
|
Ok(url.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matches_external_reference_prefix(url: &reqwest::Url, prefix: &str) -> bool {
|
||||||
|
let url = url.as_str();
|
||||||
|
if !url.starts_with(prefix) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Some(next) = url.as_bytes().get(prefix.len()) else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
prefix.ends_with('/') || matches!(next, b'/' | b'?')
|
||||||
|
}
|
||||||
|
|
||||||
|
fn external_policy_error(error: RestAdapterError) -> ExternalReferenceFetchError {
|
||||||
|
match error {
|
||||||
|
RestAdapterError::TargetNotAllowed { .. } => ExternalReferenceFetchError::TargetNotAllowed,
|
||||||
|
_ => ExternalReferenceFetchError::InvalidConfiguration,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn external_transport_error(error: reqwest::Error) -> ExternalReferenceFetchError {
|
||||||
|
ExternalReferenceFetchError::Transport {
|
||||||
|
timeout: error.is_timeout(),
|
||||||
|
connect: error.is_connect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read_external_response_bytes(
|
||||||
|
response: reqwest::Response,
|
||||||
|
max_response_bytes: usize,
|
||||||
|
) -> Result<Vec<u8>, ExternalReferenceFetchError> {
|
||||||
|
if response
|
||||||
|
.content_length()
|
||||||
|
.is_some_and(|length| length > max_response_bytes as u64)
|
||||||
|
{
|
||||||
|
return Err(ExternalReferenceFetchError::ResponseTooLarge {
|
||||||
|
limit_bytes: max_response_bytes,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let mut stream = response.bytes_stream();
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
let chunk = chunk.map_err(external_transport_error)?;
|
||||||
|
if bytes.len().saturating_add(chunk.len()) > max_response_bytes {
|
||||||
|
return Err(ExternalReferenceFetchError::ResponseTooLarge {
|
||||||
|
limit_bytes: max_response_bytes,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
bytes.extend_from_slice(&chunk);
|
||||||
|
}
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
fn upstream_outcome(error: &RestAdapterError) -> UpstreamOutcome {
|
fn upstream_outcome(error: &RestAdapterError) -> UpstreamOutcome {
|
||||||
match error {
|
match error {
|
||||||
RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
|
RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
|
||||||
|
|||||||
@@ -1,6 +1,26 @@
|
|||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
pub enum ExternalReferenceFetchError {
|
||||||
|
#[error("external references are disabled")]
|
||||||
|
Disabled,
|
||||||
|
#[error("external reference URL is invalid")]
|
||||||
|
InvalidUrl,
|
||||||
|
#[error("external reference target is not allowed")]
|
||||||
|
TargetNotAllowed,
|
||||||
|
#[error("external reference redirects are not allowed")]
|
||||||
|
RedirectNotAllowed,
|
||||||
|
#[error("external reference response exceeds the configured limit of {limit_bytes} bytes")]
|
||||||
|
ResponseTooLarge { limit_bytes: usize },
|
||||||
|
#[error("external reference endpoint returned status {status}")]
|
||||||
|
UnexpectedStatus { status: u16 },
|
||||||
|
#[error("external reference request failed")]
|
||||||
|
Transport { timeout: bool, connect: bool },
|
||||||
|
#[error("external reference fetch configuration is invalid")]
|
||||||
|
InvalidConfiguration,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum RestAdapterError {
|
pub enum RestAdapterError {
|
||||||
#[error("invalid base url: {url}")]
|
#[error("invalid base url: {url}")]
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ use crank_core::{
|
|||||||
ProtocolAdapterError, RestTarget, RuntimeRequestContext, Target,
|
ProtocolAdapterError, RestTarget, RuntimeRequestContext, Target,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub use client::{OutboundHttpPolicy, RestAdapter};
|
pub use client::{ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter};
|
||||||
pub use error::RestAdapterError;
|
pub use error::{ExternalReferenceFetchError, RestAdapterError};
|
||||||
pub use model::{RestRequest, RestResponse};
|
pub use model::{RestRequest, RestResponse};
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
mod integration {
|
mod integration {
|
||||||
mod client;
|
mod client;
|
||||||
|
mod external_reference_fetcher;
|
||||||
mod outbound_security;
|
mod outbound_security;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,151 @@
|
|||||||
|
use std::{
|
||||||
|
sync::{
|
||||||
|
Arc,
|
||||||
|
atomic::{AtomicUsize, Ordering},
|
||||||
|
},
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
use axum::{Router, http::StatusCode, response::Redirect, routing::get};
|
||||||
|
use crank_adapter_rest::{
|
||||||
|
ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy,
|
||||||
|
};
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn external_references_are_default_off_before_any_request() {
|
||||||
|
let requests = Arc::new(AtomicUsize::new(0));
|
||||||
|
let base_url = spawn_server(Arc::clone(&requests)).await;
|
||||||
|
let fetcher = ExternalReferenceFetcher::try_new(
|
||||||
|
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||||
|
Vec::new(),
|
||||||
|
1024,
|
||||||
|
Duration::from_secs(1),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = fetcher
|
||||||
|
.get(&format!("{base_url}/document"))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
assert!(matches!(error, ExternalReferenceFetchError::Disabled));
|
||||||
|
assert_eq!(requests.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn fetcher_uses_prefix_and_actual_address_policy_then_returns_bounded_bytes() {
|
||||||
|
let base_url = spawn_server(Arc::new(AtomicUsize::new(0))).await;
|
||||||
|
let fetcher = ExternalReferenceFetcher::try_new(
|
||||||
|
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||||
|
vec![base_url.clone()],
|
||||||
|
8,
|
||||||
|
Duration::from_secs(1),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
fetcher
|
||||||
|
.get(&format!("{base_url}/document#/components/schemas/A"))
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
b"openapi".to_vec()
|
||||||
|
);
|
||||||
|
let error = fetcher.get(&format!("{base_url}/large")).await.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
ExternalReferenceFetchError::ResponseTooLarge { limit_bytes: 8 }
|
||||||
|
));
|
||||||
|
|
||||||
|
let private_without_explicit_outbound_allow = ExternalReferenceFetcher::try_new(
|
||||||
|
OutboundHttpPolicy::default(),
|
||||||
|
vec![base_url.clone()],
|
||||||
|
1024,
|
||||||
|
Duration::from_secs(1),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let error = private_without_explicit_outbound_allow
|
||||||
|
.get(&format!("{base_url}/document"))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
ExternalReferenceFetchError::TargetNotAllowed
|
||||||
|
));
|
||||||
|
|
||||||
|
let exact_path_fetcher = ExternalReferenceFetcher::try_new(
|
||||||
|
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||||
|
vec![format!("{base_url}/document")],
|
||||||
|
1024,
|
||||||
|
Duration::from_secs(1),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let error = exact_path_fetcher
|
||||||
|
.get(&format!("{base_url}/document-unrelated"))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
ExternalReferenceFetchError::TargetNotAllowed
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn fetcher_rejects_redirects_and_userinfo_without_exposing_the_url() {
|
||||||
|
let requests = Arc::new(AtomicUsize::new(0));
|
||||||
|
let base_url = spawn_server(Arc::clone(&requests)).await;
|
||||||
|
let fetcher = ExternalReferenceFetcher::try_new(
|
||||||
|
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||||
|
vec![base_url.clone()],
|
||||||
|
1024,
|
||||||
|
Duration::from_secs(1),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = fetcher
|
||||||
|
.get(&format!("{base_url}/redirect"))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
ExternalReferenceFetchError::RedirectNotAllowed
|
||||||
|
));
|
||||||
|
|
||||||
|
let userinfo_url = base_url.replacen("http://", "http://user:credential@", 1);
|
||||||
|
let error = fetcher
|
||||||
|
.get(&format!("{userinfo_url}/document"))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
ExternalReferenceFetchError::TargetNotAllowed
|
||||||
|
));
|
||||||
|
let rendered = format!("{error:?} {error}");
|
||||||
|
assert!(!rendered.contains("credential"));
|
||||||
|
assert_eq!(requests.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn spawn_server(requests: Arc<AtomicUsize>) -> String {
|
||||||
|
let app = Router::new()
|
||||||
|
.route(
|
||||||
|
"/document",
|
||||||
|
get({
|
||||||
|
let requests = Arc::clone(&requests);
|
||||||
|
move || {
|
||||||
|
let requests = Arc::clone(&requests);
|
||||||
|
async move {
|
||||||
|
requests.fetch_add(1, Ordering::SeqCst);
|
||||||
|
(StatusCode::OK, "openapi")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route("/large", get(|| async { "response too large" }))
|
||||||
|
.route("/redirect", get(|| async { Redirect::to("/document") }));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
});
|
||||||
|
format!("http://{address}")
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ use std::{
|
|||||||
Arc,
|
Arc,
|
||||||
atomic::{AtomicUsize, Ordering},
|
atomic::{AtomicUsize, Ordering},
|
||||||
},
|
},
|
||||||
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
use axum::{
|
use axum::{
|
||||||
@@ -14,7 +15,9 @@ use axum::{
|
|||||||
http::StatusCode,
|
http::StatusCode,
|
||||||
routing::{any, post},
|
routing::{any, post},
|
||||||
};
|
};
|
||||||
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest};
|
use crank_adapter_rest::{
|
||||||
|
ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest,
|
||||||
|
};
|
||||||
use crank_core::{HttpMethod, ProtocolAdapterError, RestTarget};
|
use crank_core::{HttpMethod, ProtocolAdapterError, RestTarget};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
@@ -94,7 +97,7 @@ async fn proxy_environment_is_ignored_by_default() {
|
|||||||
let adapter = RestAdapter::default();
|
let adapter = RestAdapter::default();
|
||||||
let request = json_request(json!({"payload": "proxy-env-canary"}));
|
let request = json_request(json!({"payload": "proxy-env-canary"}));
|
||||||
let rest_target = RestTarget {
|
let rest_target = RestTarget {
|
||||||
base_url: target,
|
base_url: target.clone(),
|
||||||
method: HttpMethod::Post,
|
method: HttpMethod::Post,
|
||||||
path_template: "/capture".to_owned(),
|
path_template: "/capture".to_owned(),
|
||||||
static_headers: BTreeMap::new(),
|
static_headers: BTreeMap::new(),
|
||||||
@@ -102,6 +105,17 @@ async fn proxy_environment_is_ignored_by_default() {
|
|||||||
let _ = adapter.execute(&rest_target, &request).await.expect_err(
|
let _ = adapter.execute(&rest_target, &request).await.expect_err(
|
||||||
"unresolvable target should fail locally instead of being sent through proxy env",
|
"unresolvable target should fail locally instead of being sent through proxy env",
|
||||||
);
|
);
|
||||||
|
let fetcher = ExternalReferenceFetcher::try_new(
|
||||||
|
OutboundHttpPolicy::default(),
|
||||||
|
vec!["http://public.example.test/".to_owned()],
|
||||||
|
1024,
|
||||||
|
Duration::from_secs(1),
|
||||||
|
)
|
||||||
|
.expect("valid external reference fetcher");
|
||||||
|
let _ = fetcher
|
||||||
|
.get(&target)
|
||||||
|
.await
|
||||||
|
.expect_err("external reference fetcher must not use proxy environment variables");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
use std::{
|
use std::{
|
||||||
fmt,
|
fmt,
|
||||||
os::fd::{AsRawFd, FromRawFd, OwnedFd},
|
os::fd::{AsRawFd, FromRawFd, OwnedFd},
|
||||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::temp_scan::{list_names_after, valid_temp_name};
|
use crate::temp_scan::valid_temp_name;
|
||||||
use crate::{
|
use crate::{
|
||||||
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
|
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
|
||||||
ReconciliationScan, ReconciliationScanStop,
|
ReconciliationScan, ReconciliationScanStop,
|
||||||
@@ -15,6 +14,10 @@ use crate::{
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod stale_temp;
|
||||||
|
|
||||||
|
pub use stale_temp::{StaleTemp, TempScan, TempScanCursor};
|
||||||
|
|
||||||
/// Opaque bounded-scan continuation, valid only for an unchanged namespace.
|
/// Opaque bounded-scan continuation, valid only for an unchanged namespace.
|
||||||
/// Discard it after any put, quarantine, delete, or external mutation.
|
/// Discard it after any put, quarantine, delete, or external mutation.
|
||||||
pub struct ReconciliationCursor {
|
pub struct ReconciliationCursor {
|
||||||
@@ -80,49 +83,6 @@ pub struct ReconciliationCandidate {
|
|||||||
pub(crate) modified_nanoseconds: i64,
|
pub(crate) modified_nanoseconds: i64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Opaque single-use evidence for a stale temporary inode under the pinned root.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct StaleTemp {
|
|
||||||
name: String,
|
|
||||||
shard: String,
|
|
||||||
root_dev: u64,
|
|
||||||
root_ino: u64,
|
|
||||||
dev: u64,
|
|
||||||
ino: u64,
|
|
||||||
modified_seconds: i64,
|
|
||||||
modified_nanoseconds: i64,
|
|
||||||
grace: Duration,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Opaque, inode-bound progress marker for the bounded stale-temp sweeper.
|
|
||||||
///
|
|
||||||
/// It deliberately advances by shard instead of retaining a directory offset:
|
|
||||||
/// directory offsets are invalidated by a concurrent writer, while round-robin
|
|
||||||
/// shard progress prevents a busy low-numbered shard from starving all others.
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
pub struct TempScanCursor {
|
|
||||||
root_dev: u64,
|
|
||||||
root_ino: u64,
|
|
||||||
next_shard: u8,
|
|
||||||
shard_continuation: Option<TempShardContinuation>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
struct TempShardContinuation {
|
|
||||||
shard: u8,
|
|
||||||
dev: u64,
|
|
||||||
ino: u64,
|
|
||||||
cookie: i64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
|
||||||
pub struct TempScan {
|
|
||||||
pub scanned: usize,
|
|
||||||
pub omitted: usize,
|
|
||||||
/// True when this page completed a full round-robin traversal.
|
|
||||||
pub complete: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ArtifactStore {
|
impl ArtifactStore {
|
||||||
pub(crate) fn scan_reconciliation_bounded(
|
pub(crate) fn scan_reconciliation_bounded(
|
||||||
&self,
|
&self,
|
||||||
@@ -340,229 +300,6 @@ impl ArtifactStore {
|
|||||||
Err(_) => Ok(ReconciliationMutation::Retryable),
|
Err(_) => Ok(ReconciliationMutation::Retryable),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// Scans at most `scan_budget` entries and returns `result_limit` stale capabilities.
|
|
||||||
pub fn scan_stale_temps(
|
|
||||||
&self,
|
|
||||||
grace: Duration,
|
|
||||||
scan_budget: usize,
|
|
||||||
result_limit: usize,
|
|
||||||
) -> Result<(TempScan, Vec<StaleTemp>), ArtifactError> {
|
|
||||||
self.scan_stale_temps_after(grace, None, scan_budget, result_limit)
|
|
||||||
.map(|(report, candidates, _)| (report, candidates))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Resumes stale-temp cleanup from an opaque round-robin marker. A
|
|
||||||
/// continuation is valid only for this exact pinned root.
|
|
||||||
pub fn scan_stale_temps_after(
|
|
||||||
&self,
|
|
||||||
grace: Duration,
|
|
||||||
continuation: Option<TempScanCursor>,
|
|
||||||
scan_budget: usize,
|
|
||||||
result_limit: usize,
|
|
||||||
) -> Result<(TempScan, Vec<StaleTemp>, TempScanCursor), ArtifactError> {
|
|
||||||
let root = self.root()?;
|
|
||||||
let _lock = RootLock::shared(root)?;
|
|
||||||
ensure_root_unchanged(root)?;
|
|
||||||
let sha = match open_existing_dir(root.fd.as_raw_fd(), b"sha256") {
|
|
||||||
Ok(fd) => fd,
|
|
||||||
Err(ArtifactError::NotFound) => {
|
|
||||||
return Ok((
|
|
||||||
TempScan {
|
|
||||||
complete: true,
|
|
||||||
..TempScan::default()
|
|
||||||
},
|
|
||||||
Vec::new(),
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard: 0,
|
|
||||||
shard_continuation: None,
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Err(error) => return Err(error),
|
|
||||||
};
|
|
||||||
let (start_shard, mut shard_continuation) = match continuation {
|
|
||||||
Some(cursor) if cursor.root_dev == root.dev && cursor.root_ino == root.ino => {
|
|
||||||
(cursor.next_shard, cursor.shard_continuation)
|
|
||||||
}
|
|
||||||
Some(_) => return Err(ArtifactError::UnsafeRoot),
|
|
||||||
None => (0, None),
|
|
||||||
};
|
|
||||||
let mut report = TempScan::default();
|
|
||||||
let mut remaining = scan_budget;
|
|
||||||
let mut result = Vec::new();
|
|
||||||
if remaining == 0 {
|
|
||||||
return Ok((
|
|
||||||
report,
|
|
||||||
result,
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard: start_shard,
|
|
||||||
shard_continuation,
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let mut next_shard = start_shard;
|
|
||||||
for offset in 0_u16..=255 {
|
|
||||||
if remaining == 0 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
let shard_number = start_shard.wrapping_add(offset as u8);
|
|
||||||
next_shard = shard_number.wrapping_add(1);
|
|
||||||
let shard = format!("{shard_number:02x}");
|
|
||||||
let shard_fd = match open_existing_dir(sha.as_raw_fd(), shard.as_bytes()) {
|
|
||||||
Ok(fd) => fd,
|
|
||||||
Err(ArtifactError::NotFound) => {
|
|
||||||
shard_continuation = None;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Err(error) => return Err(error),
|
|
||||||
};
|
|
||||||
let stat = stat_fd(shard_fd.as_raw_fd())?;
|
|
||||||
let resume = shard_continuation.take().and_then(|continuation| {
|
|
||||||
if continuation.shard == shard_number
|
|
||||||
&& continuation.dev == stat.st_dev
|
|
||||||
&& continuation.ino == stat.st_ino
|
|
||||||
{
|
|
||||||
Some(continuation.cookie)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let page = list_names_after(shard_fd.as_raw_fd(), &mut remaining, &mut report, resume)?;
|
|
||||||
for entry in page.names {
|
|
||||||
let name = entry.name;
|
|
||||||
if !valid_temp_name(&name) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
#[cfg(debug_assertions)]
|
|
||||||
let _ = crate::test_support::checkpoint("housekeeping_before_stat");
|
|
||||||
let stat = match nofollow_stat(shard_fd.as_raw_fd(), &name) {
|
|
||||||
Ok(stat) => stat,
|
|
||||||
Err(ArtifactError::NotFound) => continue,
|
|
||||||
Err(error) => return Err(error),
|
|
||||||
};
|
|
||||||
if (stat.st_mode & libc::S_IFMT) != libc::S_IFREG || stat.st_nlink != 1 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let modified = UNIX_EPOCH
|
|
||||||
.checked_add(Duration::new(
|
|
||||||
stat.st_mtime.max(0) as u64,
|
|
||||||
stat.st_mtime_nsec.max(0) as u32,
|
|
||||||
))
|
|
||||||
.unwrap_or(SystemTime::UNIX_EPOCH);
|
|
||||||
if SystemTime::now()
|
|
||||||
.duration_since(modified)
|
|
||||||
.is_ok_and(|age| age >= grace)
|
|
||||||
{
|
|
||||||
if result.len() == result_limit {
|
|
||||||
report.omitted += 1;
|
|
||||||
if result_limit > 0 {
|
|
||||||
return Ok((
|
|
||||||
report,
|
|
||||||
result,
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard: shard_number,
|
|
||||||
shard_continuation: Some(TempShardContinuation {
|
|
||||||
shard: shard_number,
|
|
||||||
dev: stat.st_dev,
|
|
||||||
ino: stat.st_ino,
|
|
||||||
cookie: entry.cookie_before,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
result.push(StaleTemp {
|
|
||||||
name,
|
|
||||||
shard: shard.clone(),
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
dev: stat.st_dev,
|
|
||||||
ino: stat.st_ino,
|
|
||||||
modified_seconds: stat.st_mtime,
|
|
||||||
modified_nanoseconds: stat.st_mtime_nsec,
|
|
||||||
grace,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(cookie) = page.continuation {
|
|
||||||
return Ok((
|
|
||||||
report,
|
|
||||||
result,
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard: shard_number,
|
|
||||||
shard_continuation: Some(TempShardContinuation {
|
|
||||||
shard: shard_number,
|
|
||||||
dev: stat.st_dev,
|
|
||||||
ino: stat.st_ino,
|
|
||||||
cookie,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
report.complete = true;
|
|
||||||
Ok((
|
|
||||||
report,
|
|
||||||
result,
|
|
||||||
TempScanCursor {
|
|
||||||
root_dev: root.dev,
|
|
||||||
root_ino: root.ino,
|
|
||||||
next_shard,
|
|
||||||
shard_continuation: None,
|
|
||||||
},
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Deletes a revalidated stale inode under an exclusive lock, then fsyncs.
|
|
||||||
pub fn delete_stale_temp(&self, candidate: StaleTemp) -> Result<(), ArtifactError> {
|
|
||||||
let root = self.root()?;
|
|
||||||
let _lock = RootLock::exclusive(root)?;
|
|
||||||
ensure_root_unchanged(root)?;
|
|
||||||
if candidate.root_dev != root.dev || candidate.root_ino != root.ino {
|
|
||||||
return Err(ArtifactError::UnsafeRoot);
|
|
||||||
}
|
|
||||||
if !valid_temp_name(&candidate.name)
|
|
||||||
|| candidate.shard.len() != 2
|
|
||||||
|| !candidate.shard.bytes().all(is_lower_hex)
|
|
||||||
{
|
|
||||||
return Err(ArtifactError::UnsafeRoot);
|
|
||||||
}
|
|
||||||
let sha = open_existing_dir(root.fd.as_raw_fd(), b"sha256")?;
|
|
||||||
let shard = open_existing_dir(sha.as_raw_fd(), candidate.shard.as_bytes())?;
|
|
||||||
let stat = nofollow_stat(shard.as_raw_fd(), &candidate.name)?;
|
|
||||||
if stat.st_dev != candidate.dev
|
|
||||||
|| stat.st_ino != candidate.ino
|
|
||||||
|| stat.st_mtime != candidate.modified_seconds
|
|
||||||
|| stat.st_mtime_nsec != candidate.modified_nanoseconds
|
|
||||||
|| (stat.st_mode & libc::S_IFMT) != libc::S_IFREG
|
|
||||||
|| stat.st_nlink != 1
|
|
||||||
{
|
|
||||||
return Err(ArtifactError::UnsafeRoot);
|
|
||||||
}
|
|
||||||
let modified = UNIX_EPOCH
|
|
||||||
.checked_add(Duration::new(
|
|
||||||
stat.st_mtime.max(0) as u64,
|
|
||||||
stat.st_mtime_nsec.max(0) as u32,
|
|
||||||
))
|
|
||||||
.unwrap_or(SystemTime::UNIX_EPOCH);
|
|
||||||
if !SystemTime::now()
|
|
||||||
.duration_since(modified)
|
|
||||||
.is_ok_and(|age| age >= candidate.grace)
|
|
||||||
{
|
|
||||||
return Err(ArtifactError::UnsafeRoot);
|
|
||||||
}
|
|
||||||
unlinkat(shard.as_raw_fd(), candidate.name.as_bytes())?;
|
|
||||||
fsync_fd(shard.as_raw_fd())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reconciliation_cursor(
|
fn reconciliation_cursor(
|
||||||
|
|||||||
@@ -0,0 +1,280 @@
|
|||||||
|
use std::{
|
||||||
|
os::fd::AsRawFd,
|
||||||
|
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::{is_lower_hex, nofollow_stat};
|
||||||
|
use crate::{
|
||||||
|
ArtifactError, ArtifactStore,
|
||||||
|
store::{RootLock, ensure_root_unchanged, fsync_fd, open_existing_dir, stat_fd, unlinkat},
|
||||||
|
temp_scan::{list_names_after, valid_temp_name},
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Opaque single-use evidence for a stale temporary inode under the pinned root.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct StaleTemp {
|
||||||
|
name: String,
|
||||||
|
shard: String,
|
||||||
|
root_dev: u64,
|
||||||
|
root_ino: u64,
|
||||||
|
dev: u64,
|
||||||
|
ino: u64,
|
||||||
|
modified_seconds: i64,
|
||||||
|
modified_nanoseconds: i64,
|
||||||
|
grace: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opaque, inode-bound progress marker for the bounded stale-temp sweeper.
|
||||||
|
///
|
||||||
|
/// It deliberately advances by shard instead of retaining a directory offset:
|
||||||
|
/// directory offsets are invalidated by a concurrent writer, while round-robin
|
||||||
|
/// shard progress prevents a busy low-numbered shard from starving all others.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct TempScanCursor {
|
||||||
|
root_dev: u64,
|
||||||
|
root_ino: u64,
|
||||||
|
next_shard: u8,
|
||||||
|
shard_continuation: Option<TempShardContinuation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct TempShardContinuation {
|
||||||
|
shard: u8,
|
||||||
|
dev: u64,
|
||||||
|
ino: u64,
|
||||||
|
cookie: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||||
|
pub struct TempScan {
|
||||||
|
pub scanned: usize,
|
||||||
|
pub omitted: usize,
|
||||||
|
/// True when this page completed a full round-robin traversal.
|
||||||
|
pub complete: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ArtifactStore {
|
||||||
|
/// Scans at most `scan_budget` entries and returns `result_limit` stale capabilities.
|
||||||
|
pub fn scan_stale_temps(
|
||||||
|
&self,
|
||||||
|
grace: Duration,
|
||||||
|
scan_budget: usize,
|
||||||
|
result_limit: usize,
|
||||||
|
) -> Result<(TempScan, Vec<StaleTemp>), ArtifactError> {
|
||||||
|
self.scan_stale_temps_after(grace, None, scan_budget, result_limit)
|
||||||
|
.map(|(report, candidates, _)| (report, candidates))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resumes stale-temp cleanup from an opaque round-robin marker. A
|
||||||
|
/// continuation is valid only for this exact pinned root.
|
||||||
|
pub fn scan_stale_temps_after(
|
||||||
|
&self,
|
||||||
|
grace: Duration,
|
||||||
|
continuation: Option<TempScanCursor>,
|
||||||
|
scan_budget: usize,
|
||||||
|
result_limit: usize,
|
||||||
|
) -> Result<(TempScan, Vec<StaleTemp>, TempScanCursor), ArtifactError> {
|
||||||
|
let root = self.root()?;
|
||||||
|
let _lock = RootLock::shared(root)?;
|
||||||
|
ensure_root_unchanged(root)?;
|
||||||
|
let sha = match open_existing_dir(root.fd.as_raw_fd(), b"sha256") {
|
||||||
|
Ok(fd) => fd,
|
||||||
|
Err(ArtifactError::NotFound) => {
|
||||||
|
return Ok((
|
||||||
|
TempScan {
|
||||||
|
complete: true,
|
||||||
|
..TempScan::default()
|
||||||
|
},
|
||||||
|
Vec::new(),
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard: 0,
|
||||||
|
shard_continuation: None,
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
};
|
||||||
|
let (start_shard, mut shard_continuation) = match continuation {
|
||||||
|
Some(cursor) if cursor.root_dev == root.dev && cursor.root_ino == root.ino => {
|
||||||
|
(cursor.next_shard, cursor.shard_continuation)
|
||||||
|
}
|
||||||
|
Some(_) => return Err(ArtifactError::UnsafeRoot),
|
||||||
|
None => (0, None),
|
||||||
|
};
|
||||||
|
let mut report = TempScan::default();
|
||||||
|
let mut remaining = scan_budget;
|
||||||
|
let mut result = Vec::new();
|
||||||
|
if remaining == 0 {
|
||||||
|
return Ok((
|
||||||
|
report,
|
||||||
|
result,
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard: start_shard,
|
||||||
|
shard_continuation,
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut next_shard = start_shard;
|
||||||
|
for offset in 0_u16..=255 {
|
||||||
|
if remaining == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let shard_number = start_shard.wrapping_add(offset as u8);
|
||||||
|
next_shard = shard_number.wrapping_add(1);
|
||||||
|
let shard = format!("{shard_number:02x}");
|
||||||
|
let shard_fd = match open_existing_dir(sha.as_raw_fd(), shard.as_bytes()) {
|
||||||
|
Ok(fd) => fd,
|
||||||
|
Err(ArtifactError::NotFound) => {
|
||||||
|
shard_continuation = None;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
};
|
||||||
|
let stat = stat_fd(shard_fd.as_raw_fd())?;
|
||||||
|
let resume = shard_continuation.take().and_then(|continuation| {
|
||||||
|
if continuation.shard == shard_number
|
||||||
|
&& continuation.dev == stat.st_dev
|
||||||
|
&& continuation.ino == stat.st_ino
|
||||||
|
{
|
||||||
|
Some(continuation.cookie)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let page = list_names_after(shard_fd.as_raw_fd(), &mut remaining, &mut report, resume)?;
|
||||||
|
for entry in page.names {
|
||||||
|
let name = entry.name;
|
||||||
|
if !valid_temp_name(&name) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
let _ = crate::test_support::checkpoint("housekeeping_before_stat");
|
||||||
|
let stat = match nofollow_stat(shard_fd.as_raw_fd(), &name) {
|
||||||
|
Ok(stat) => stat,
|
||||||
|
Err(ArtifactError::NotFound) => continue,
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
};
|
||||||
|
if (stat.st_mode & libc::S_IFMT) != libc::S_IFREG || stat.st_nlink != 1 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let modified = UNIX_EPOCH
|
||||||
|
.checked_add(Duration::new(
|
||||||
|
stat.st_mtime.max(0) as u64,
|
||||||
|
stat.st_mtime_nsec.max(0) as u32,
|
||||||
|
))
|
||||||
|
.unwrap_or(SystemTime::UNIX_EPOCH);
|
||||||
|
if SystemTime::now()
|
||||||
|
.duration_since(modified)
|
||||||
|
.is_ok_and(|age| age >= grace)
|
||||||
|
{
|
||||||
|
if result.len() == result_limit {
|
||||||
|
report.omitted += 1;
|
||||||
|
if result_limit > 0 {
|
||||||
|
return Ok((
|
||||||
|
report,
|
||||||
|
result,
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard: shard_number,
|
||||||
|
shard_continuation: Some(TempShardContinuation {
|
||||||
|
shard: shard_number,
|
||||||
|
dev: stat.st_dev,
|
||||||
|
ino: stat.st_ino,
|
||||||
|
cookie: entry.cookie_before,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
result.push(StaleTemp {
|
||||||
|
name,
|
||||||
|
shard: shard.clone(),
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
dev: stat.st_dev,
|
||||||
|
ino: stat.st_ino,
|
||||||
|
modified_seconds: stat.st_mtime,
|
||||||
|
modified_nanoseconds: stat.st_mtime_nsec,
|
||||||
|
grace,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(cookie) = page.continuation {
|
||||||
|
return Ok((
|
||||||
|
report,
|
||||||
|
result,
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard: shard_number,
|
||||||
|
shard_continuation: Some(TempShardContinuation {
|
||||||
|
shard: shard_number,
|
||||||
|
dev: stat.st_dev,
|
||||||
|
ino: stat.st_ino,
|
||||||
|
cookie,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.complete = true;
|
||||||
|
Ok((
|
||||||
|
report,
|
||||||
|
result,
|
||||||
|
TempScanCursor {
|
||||||
|
root_dev: root.dev,
|
||||||
|
root_ino: root.ino,
|
||||||
|
next_shard,
|
||||||
|
shard_continuation: None,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deletes a revalidated stale inode under an exclusive lock, then fsyncs.
|
||||||
|
pub fn delete_stale_temp(&self, candidate: StaleTemp) -> Result<(), ArtifactError> {
|
||||||
|
let root = self.root()?;
|
||||||
|
let _lock = RootLock::exclusive(root)?;
|
||||||
|
ensure_root_unchanged(root)?;
|
||||||
|
if candidate.root_dev != root.dev || candidate.root_ino != root.ino {
|
||||||
|
return Err(ArtifactError::UnsafeRoot);
|
||||||
|
}
|
||||||
|
if !valid_temp_name(&candidate.name)
|
||||||
|
|| candidate.shard.len() != 2
|
||||||
|
|| !candidate.shard.bytes().all(is_lower_hex)
|
||||||
|
{
|
||||||
|
return Err(ArtifactError::UnsafeRoot);
|
||||||
|
}
|
||||||
|
let sha = open_existing_dir(root.fd.as_raw_fd(), b"sha256")?;
|
||||||
|
let shard = open_existing_dir(sha.as_raw_fd(), candidate.shard.as_bytes())?;
|
||||||
|
let stat = nofollow_stat(shard.as_raw_fd(), &candidate.name)?;
|
||||||
|
if stat.st_dev != candidate.dev
|
||||||
|
|| stat.st_ino != candidate.ino
|
||||||
|
|| stat.st_mtime != candidate.modified_seconds
|
||||||
|
|| stat.st_mtime_nsec != candidate.modified_nanoseconds
|
||||||
|
|| (stat.st_mode & libc::S_IFMT) != libc::S_IFREG
|
||||||
|
|| stat.st_nlink != 1
|
||||||
|
{
|
||||||
|
return Err(ArtifactError::UnsafeRoot);
|
||||||
|
}
|
||||||
|
let modified = UNIX_EPOCH
|
||||||
|
.checked_add(Duration::new(
|
||||||
|
stat.st_mtime.max(0) as u64,
|
||||||
|
stat.st_mtime_nsec.max(0) as u32,
|
||||||
|
))
|
||||||
|
.unwrap_or(SystemTime::UNIX_EPOCH);
|
||||||
|
if !SystemTime::now()
|
||||||
|
.duration_since(modified)
|
||||||
|
.is_ok_and(|age| age >= candidate.grace)
|
||||||
|
{
|
||||||
|
return Err(ArtifactError::UnsafeRoot);
|
||||||
|
}
|
||||||
|
unlinkat(shard.as_raw_fd(), candidate.name.as_bytes())?;
|
||||||
|
fsync_fd(shard.as_raw_fd())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
use std::{
|
use std::{
|
||||||
fs,
|
fs,
|
||||||
io::{self, Read},
|
|
||||||
os::unix::fs::PermissionsExt,
|
os::unix::fs::PermissionsExt,
|
||||||
path::PathBuf,
|
path::PathBuf,
|
||||||
process::{Child, Command, ExitStatus},
|
process::{Child, Command, ExitStatus},
|
||||||
@@ -13,7 +12,7 @@ use std::{
|
|||||||
use crank_artifacts::test_support::{
|
use crank_artifacts::test_support::{
|
||||||
FaultAction, clear_checkpoint, set_checkpoint, wait_until_held,
|
FaultAction, clear_checkpoint, set_checkpoint, wait_until_held,
|
||||||
};
|
};
|
||||||
use crank_artifacts::{ArtifactError, ArtifactRef, ArtifactStore, MAX_ARTIFACT_BYTES, StaleTemp};
|
use crank_artifacts::{ArtifactError, ArtifactRef, ArtifactStore, MAX_ARTIFACT_BYTES};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
#[cfg(debug_assertions)]
|
#[cfg(debug_assertions)]
|
||||||
use std::sync::{Mutex, OnceLock};
|
use std::sync::{Mutex, OnceLock};
|
||||||
@@ -786,318 +785,3 @@ fn legacy_oversize_is_rejected() {
|
|||||||
ArtifactError::Integrity
|
ArtifactError::Integrity
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fifo_entries_fail_without_blocking_reads() {
|
|
||||||
let root = TestRoot::new("fifo");
|
|
||||||
let expected =
|
|
||||||
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"fifo bytes"))).unwrap();
|
|
||||||
let sha = root.0.join("sha256");
|
|
||||||
let shard = sha.join(&expected.digest_hex()[..2]);
|
|
||||||
fs::create_dir(&sha).unwrap();
|
|
||||||
fs::create_dir(&shard).unwrap();
|
|
||||||
fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap();
|
|
||||||
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
|
|
||||||
let blob_fifo = shard.join(expected.digest_hex());
|
|
||||||
let legacy = root.0.join("legacy");
|
|
||||||
fs::create_dir(&legacy).unwrap();
|
|
||||||
fs::set_permissions(&legacy, fs::Permissions::from_mode(0o700)).unwrap();
|
|
||||||
let legacy_fifo = legacy.join("source");
|
|
||||||
for path in [&blob_fifo, &legacy_fifo] {
|
|
||||||
let path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap();
|
|
||||||
assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o400) }, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
let executable = std::env::current_exe().unwrap();
|
|
||||||
for kind in ["blob", "legacy"] {
|
|
||||||
let child = Command::new(&executable)
|
|
||||||
.args(["--exact", "fifo_read_child", "--nocapture"])
|
|
||||||
.env("CRANK_ARTIFACTS_FIFO_ROOT", &root.0)
|
|
||||||
.env("CRANK_ARTIFACTS_FIFO_KIND", kind)
|
|
||||||
.env("CRANK_ARTIFACTS_FIFO_REF", expected.as_str())
|
|
||||||
.spawn()
|
|
||||||
.unwrap();
|
|
||||||
assert!(wait_for_child(child, Duration::from_secs(2)).success());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fifo_read_child() {
|
|
||||||
let Some(root) = std::env::var_os("CRANK_ARTIFACTS_FIFO_ROOT") else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
let root = PathBuf::from(root);
|
|
||||||
let kind = std::env::var("CRANK_ARTIFACTS_FIFO_KIND").unwrap();
|
|
||||||
let expected = ArtifactRef::parse(&std::env::var("CRANK_ARTIFACTS_FIFO_REF").unwrap()).unwrap();
|
|
||||||
let store = ArtifactStore::open(&root).unwrap();
|
|
||||||
let result = if kind == "blob" {
|
|
||||||
store.read(&expected)
|
|
||||||
} else {
|
|
||||||
store
|
|
||||||
.with_legacy_root(root.join("legacy"))
|
|
||||||
.read_legacy_file_url(
|
|
||||||
&format!("file://{}/legacy/source", root.display()),
|
|
||||||
&expected,
|
|
||||||
)
|
|
||||||
};
|
|
||||||
assert_eq!(result.unwrap_err(), ArtifactError::Integrity);
|
|
||||||
}
|
|
||||||
|
|
||||||
struct ShortReader {
|
|
||||||
bytes: Vec<u8>,
|
|
||||||
offset: usize,
|
|
||||||
interrupted: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct FailingReader;
|
|
||||||
|
|
||||||
struct HousekeepingReader {
|
|
||||||
store: ArtifactStore,
|
|
||||||
candidate: Option<StaleTemp>,
|
|
||||||
bytes: io::Cursor<Vec<u8>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Read for FailingReader {
|
|
||||||
fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
|
|
||||||
Err(io::Error::other("private reader detail"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
impl Read for HousekeepingReader {
|
|
||||||
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
|
|
||||||
if let Some(candidate) = self.candidate.take() {
|
|
||||||
self.store.delete_stale_temp(candidate).unwrap();
|
|
||||||
}
|
|
||||||
self.bytes.read(buffer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
impl Read for ShortReader {
|
|
||||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
|
||||||
if !self.interrupted {
|
|
||||||
self.interrupted = true;
|
|
||||||
return Err(io::Error::from(io::ErrorKind::Interrupted));
|
|
||||||
}
|
|
||||||
if self.offset == self.bytes.len() {
|
|
||||||
return Ok(0);
|
|
||||||
}
|
|
||||||
let amount = 1.min(buf.len()).min(self.bytes.len() - self.offset);
|
|
||||||
buf[..amount].copy_from_slice(&self.bytes[self.offset..self.offset + amount]);
|
|
||||||
self.offset += amount;
|
|
||||||
Ok(amount)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn streaming_input_handles_short_reads_and_interruption() {
|
|
||||||
let root = TestRoot::new("stream");
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let mut reader = ShortReader {
|
|
||||||
bytes: b"short chunk source".to_vec(),
|
|
||||||
offset: 0,
|
|
||||||
interrupted: false,
|
|
||||||
};
|
|
||||||
let stored = store.put_reader(&mut reader).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
store.read(&stored.artifact_ref).unwrap(),
|
|
||||||
b"short chunk source"
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
store.put_reader(&mut FailingReader).unwrap_err(),
|
|
||||||
ArtifactError::Storage
|
|
||||||
);
|
|
||||||
for error in [
|
|
||||||
ArtifactError::InvalidReference,
|
|
||||||
ArtifactError::Integrity,
|
|
||||||
ArtifactError::Storage,
|
|
||||||
ArtifactError::UnsafeRoot,
|
|
||||||
] {
|
|
||||||
let diagnostic = format!("{error} {error:?}");
|
|
||||||
assert!(!diagnostic.contains("private reader detail"));
|
|
||||||
assert!(!diagnostic.contains(std::env::temp_dir().to_string_lossy().as_ref()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn user_reader_can_run_housekeeping_without_self_deadlock() {
|
|
||||||
use std::sync::mpsc;
|
|
||||||
|
|
||||||
let root = TestRoot::new("reader-housekeeping");
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let stored = store.put(b"reader housekeeping shard").unwrap();
|
|
||||||
let temp = root
|
|
||||||
.0
|
|
||||||
.join("sha256")
|
|
||||||
.join(&stored.artifact_ref.digest_hex()[..2])
|
|
||||||
.join(".crank-artifact-tmp-v1-00000000000000000000000000000000-1-10");
|
|
||||||
fs::write(&temp, b"partial").unwrap();
|
|
||||||
let (_, candidates) = store.scan_stale_temps(Duration::ZERO, 128, 1).unwrap();
|
|
||||||
let mut reader = HousekeepingReader {
|
|
||||||
store: store.clone(),
|
|
||||||
candidate: candidates.into_iter().next(),
|
|
||||||
bytes: io::Cursor::new(b"reader bytes".to_vec()),
|
|
||||||
};
|
|
||||||
let writer_store = store.clone();
|
|
||||||
let (sender, receiver) = mpsc::channel();
|
|
||||||
thread::spawn(move || {
|
|
||||||
sender.send(writer_store.put_reader(&mut reader)).unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let stored = receiver
|
|
||||||
.recv_timeout(Duration::from_secs(2))
|
|
||||||
.expect("put_reader deadlocked while its reader ran housekeeping")
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(store.read(&stored.artifact_ref).unwrap(), b"reader bytes");
|
|
||||||
assert!(!temp.exists());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
#[cfg(debug_assertions)]
|
|
||||||
fn forked_crash_checkpoints_publish_only_complete_or_absent_blobs() {
|
|
||||||
let _guard = fault_guard();
|
|
||||||
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
|
||||||
set_checkpoint(
|
|
||||||
std::env::var("CRANK_ARTIFACTS_TEST_STAGE").unwrap(),
|
|
||||||
FaultAction::Exit,
|
|
||||||
);
|
|
||||||
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
|
||||||
let _ = store.put(b"crash-consistent bytes");
|
|
||||||
panic!("checkpoint did not terminate the child");
|
|
||||||
}
|
|
||||||
|
|
||||||
for stage in ["write", "file_fsync", "publish", "directory_fsync"] {
|
|
||||||
let root = TestRoot::new(stage);
|
|
||||||
let status = Command::new(std::env::current_exe().unwrap())
|
|
||||||
.args([
|
|
||||||
"--exact",
|
|
||||||
"forked_crash_checkpoints_publish_only_complete_or_absent_blobs",
|
|
||||||
"--nocapture",
|
|
||||||
])
|
|
||||||
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
|
||||||
.env("CRANK_ARTIFACTS_TEST_STAGE", stage)
|
|
||||||
.status()
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(status.code(), Some(86), "stage={stage}");
|
|
||||||
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let expected = ArtifactRef::from_digest_hex(&format!(
|
|
||||||
"{:x}",
|
|
||||||
Sha256::digest(b"crash-consistent bytes")
|
|
||||||
))
|
|
||||||
.unwrap();
|
|
||||||
match store.read(&expected) {
|
|
||||||
Ok(bytes) => assert_eq!(bytes, b"crash-consistent bytes", "stage={stage}"),
|
|
||||||
Err(ArtifactError::NotFound) => {}
|
|
||||||
Err(error) => panic!("stage={stage} exposed an invalid final blob: {error}"),
|
|
||||||
}
|
|
||||||
let stored = store.put(b"crash-consistent bytes").unwrap();
|
|
||||||
assert_eq!(stored.artifact_ref, expected, "stage={stage}");
|
|
||||||
assert_eq!(
|
|
||||||
store.read(&expected).unwrap(),
|
|
||||||
b"crash-consistent bytes",
|
|
||||||
"stage={stage}"
|
|
||||||
);
|
|
||||||
|
|
||||||
let (_, candidates) = store.scan_stale_temps(Duration::ZERO, 128, 16).unwrap();
|
|
||||||
for candidate in candidates {
|
|
||||||
store.delete_stale_temp(candidate).unwrap();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
#[cfg(debug_assertions)]
|
|
||||||
fn disk_full_fault_seams_leave_a_retryable_store() {
|
|
||||||
let _guard = fault_guard();
|
|
||||||
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
|
||||||
set_checkpoint(
|
|
||||||
std::env::var("CRANK_ARTIFACTS_TEST_STAGE").unwrap(),
|
|
||||||
FaultAction::Fail,
|
|
||||||
);
|
|
||||||
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
store.put(b"disk-full seam bytes").unwrap_err(),
|
|
||||||
ArtifactError::Storage
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for stage in ["write", "file_fsync", "publish", "directory_fsync"] {
|
|
||||||
let root = TestRoot::new(&format!("full-{stage}"));
|
|
||||||
let status = Command::new(std::env::current_exe().unwrap())
|
|
||||||
.args([
|
|
||||||
"--exact",
|
|
||||||
"disk_full_fault_seams_leave_a_retryable_store",
|
|
||||||
"--nocapture",
|
|
||||||
])
|
|
||||||
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
|
||||||
.env("CRANK_ARTIFACTS_TEST_STAGE", stage)
|
|
||||||
.status()
|
|
||||||
.unwrap();
|
|
||||||
assert!(status.success(), "stage={stage}");
|
|
||||||
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let expected =
|
|
||||||
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"disk-full seam bytes")))
|
|
||||||
.unwrap();
|
|
||||||
match store.read(&expected) {
|
|
||||||
Ok(bytes) => assert_eq!(bytes, b"disk-full seam bytes", "stage={stage}"),
|
|
||||||
Err(ArtifactError::NotFound) => {}
|
|
||||||
Err(error) => panic!("stage={stage} exposed an invalid final blob: {error}"),
|
|
||||||
}
|
|
||||||
assert_eq!(
|
|
||||||
store.put(b"disk-full seam bytes").unwrap().artifact_ref,
|
|
||||||
expected,
|
|
||||||
"stage={stage}"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
store.read(&expected).unwrap(),
|
|
||||||
b"disk-full seam bytes",
|
|
||||||
"stage={stage}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn real_process_write_limit_leaves_no_partial_final_blob() {
|
|
||||||
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
|
||||||
// Keep the test's host filesystem untouched while making the actual
|
|
||||||
// write syscall fail as it would for a quota/disk-full condition.
|
|
||||||
unsafe {
|
|
||||||
assert_ne!(libc::signal(libc::SIGXFSZ, libc::SIG_IGN), libc::SIG_ERR);
|
|
||||||
let limit = libc::rlimit {
|
|
||||||
rlim_cur: 1,
|
|
||||||
rlim_max: 1,
|
|
||||||
};
|
|
||||||
assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0);
|
|
||||||
}
|
|
||||||
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
store.put(b"real write limit bytes").unwrap_err(),
|
|
||||||
ArtifactError::Storage
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let root = TestRoot::new("real-write-limit");
|
|
||||||
let status = Command::new(std::env::current_exe().unwrap())
|
|
||||||
.args([
|
|
||||||
"--exact",
|
|
||||||
"real_process_write_limit_leaves_no_partial_final_blob",
|
|
||||||
"--nocapture",
|
|
||||||
])
|
|
||||||
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
|
||||||
.status()
|
|
||||||
.unwrap();
|
|
||||||
assert!(status.success());
|
|
||||||
|
|
||||||
let store = ArtifactStore::open(&root.0).unwrap();
|
|
||||||
let expected =
|
|
||||||
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"real write limit bytes")))
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(store.read(&expected).unwrap_err(), ArtifactError::NotFound);
|
|
||||||
assert_eq!(
|
|
||||||
store.put(b"real write limit bytes").unwrap().artifact_ref,
|
|
||||||
expected
|
|
||||||
);
|
|
||||||
assert_eq!(store.read(&expected).unwrap(), b"real write limit bytes");
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,379 @@
|
|||||||
|
use std::{
|
||||||
|
fs,
|
||||||
|
io::{self, Read},
|
||||||
|
os::unix::fs::PermissionsExt,
|
||||||
|
path::PathBuf,
|
||||||
|
process::{Child, Command, ExitStatus},
|
||||||
|
sync::atomic::{AtomicU64, Ordering},
|
||||||
|
thread,
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
use crank_artifacts::test_support::{FaultAction, set_checkpoint};
|
||||||
|
use crank_artifacts::{ArtifactError, ArtifactRef, ArtifactStore, StaleTemp};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
|
||||||
|
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
|
||||||
|
static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
|
||||||
|
GUARD.get_or_init(|| Mutex::new(())).lock().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
struct TestRoot(PathBuf);
|
||||||
|
|
||||||
|
impl TestRoot {
|
||||||
|
fn new(name: &str) -> Self {
|
||||||
|
let path = std::env::temp_dir().join(format!(
|
||||||
|
"crank-artifacts-{name}-{}-{}",
|
||||||
|
std::process::id(),
|
||||||
|
NEXT_ROOT.fetch_add(1, Ordering::Relaxed)
|
||||||
|
));
|
||||||
|
fs::create_dir(&path).unwrap();
|
||||||
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
Self(path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for TestRoot {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700));
|
||||||
|
let _ = fs::remove_dir_all(&self.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_for_child(mut child: Child, timeout: Duration) -> ExitStatus {
|
||||||
|
let deadline = std::time::Instant::now() + timeout;
|
||||||
|
loop {
|
||||||
|
if let Some(status) = child.try_wait().unwrap() {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
if std::time::Instant::now() >= deadline {
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
panic!("child process did not finish within {timeout:?}");
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fifo_entries_fail_without_blocking_reads() {
|
||||||
|
let root = TestRoot::new("fifo");
|
||||||
|
let expected =
|
||||||
|
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"fifo bytes"))).unwrap();
|
||||||
|
let sha = root.0.join("sha256");
|
||||||
|
let shard = sha.join(&expected.digest_hex()[..2]);
|
||||||
|
fs::create_dir(&sha).unwrap();
|
||||||
|
fs::create_dir(&shard).unwrap();
|
||||||
|
fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
let blob_fifo = shard.join(expected.digest_hex());
|
||||||
|
let legacy = root.0.join("legacy");
|
||||||
|
fs::create_dir(&legacy).unwrap();
|
||||||
|
fs::set_permissions(&legacy, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
let legacy_fifo = legacy.join("source");
|
||||||
|
for path in [&blob_fifo, &legacy_fifo] {
|
||||||
|
let path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap();
|
||||||
|
assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o400) }, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let executable = std::env::current_exe().unwrap();
|
||||||
|
for kind in ["blob", "legacy"] {
|
||||||
|
let child = Command::new(&executable)
|
||||||
|
.args(["--exact", "fifo_read_child", "--nocapture"])
|
||||||
|
.env("CRANK_ARTIFACTS_FIFO_ROOT", &root.0)
|
||||||
|
.env("CRANK_ARTIFACTS_FIFO_KIND", kind)
|
||||||
|
.env("CRANK_ARTIFACTS_FIFO_REF", expected.as_str())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
assert!(wait_for_child(child, Duration::from_secs(2)).success());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fifo_read_child() {
|
||||||
|
let Some(root) = std::env::var_os("CRANK_ARTIFACTS_FIFO_ROOT") else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let root = PathBuf::from(root);
|
||||||
|
let kind = std::env::var("CRANK_ARTIFACTS_FIFO_KIND").unwrap();
|
||||||
|
let expected = ArtifactRef::parse(&std::env::var("CRANK_ARTIFACTS_FIFO_REF").unwrap()).unwrap();
|
||||||
|
let store = ArtifactStore::open(&root).unwrap();
|
||||||
|
let result = if kind == "blob" {
|
||||||
|
store.read(&expected)
|
||||||
|
} else {
|
||||||
|
store
|
||||||
|
.with_legacy_root(root.join("legacy"))
|
||||||
|
.read_legacy_file_url(
|
||||||
|
&format!("file://{}/legacy/source", root.display()),
|
||||||
|
&expected,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert_eq!(result.unwrap_err(), ArtifactError::Integrity);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ShortReader {
|
||||||
|
bytes: Vec<u8>,
|
||||||
|
offset: usize,
|
||||||
|
interrupted: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FailingReader;
|
||||||
|
|
||||||
|
struct HousekeepingReader {
|
||||||
|
store: ArtifactStore,
|
||||||
|
candidate: Option<StaleTemp>,
|
||||||
|
bytes: io::Cursor<Vec<u8>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Read for FailingReader {
|
||||||
|
fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
|
||||||
|
Err(io::Error::other("private reader detail"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Read for HousekeepingReader {
|
||||||
|
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
|
||||||
|
if let Some(candidate) = self.candidate.take() {
|
||||||
|
self.store.delete_stale_temp(candidate).unwrap();
|
||||||
|
}
|
||||||
|
self.bytes.read(buffer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Read for ShortReader {
|
||||||
|
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||||
|
if !self.interrupted {
|
||||||
|
self.interrupted = true;
|
||||||
|
return Err(io::Error::from(io::ErrorKind::Interrupted));
|
||||||
|
}
|
||||||
|
if self.offset == self.bytes.len() {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
let amount = 1.min(buf.len()).min(self.bytes.len() - self.offset);
|
||||||
|
buf[..amount].copy_from_slice(&self.bytes[self.offset..self.offset + amount]);
|
||||||
|
self.offset += amount;
|
||||||
|
Ok(amount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn streaming_input_handles_short_reads_and_interruption() {
|
||||||
|
let root = TestRoot::new("stream");
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let mut reader = ShortReader {
|
||||||
|
bytes: b"short chunk source".to_vec(),
|
||||||
|
offset: 0,
|
||||||
|
interrupted: false,
|
||||||
|
};
|
||||||
|
let stored = store.put_reader(&mut reader).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store.read(&stored.artifact_ref).unwrap(),
|
||||||
|
b"short chunk source"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
store.put_reader(&mut FailingReader).unwrap_err(),
|
||||||
|
ArtifactError::Storage
|
||||||
|
);
|
||||||
|
for error in [
|
||||||
|
ArtifactError::InvalidReference,
|
||||||
|
ArtifactError::Integrity,
|
||||||
|
ArtifactError::Storage,
|
||||||
|
ArtifactError::UnsafeRoot,
|
||||||
|
] {
|
||||||
|
let diagnostic = format!("{error} {error:?}");
|
||||||
|
assert!(!diagnostic.contains("private reader detail"));
|
||||||
|
assert!(!diagnostic.contains(std::env::temp_dir().to_string_lossy().as_ref()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn user_reader_can_run_housekeeping_without_self_deadlock() {
|
||||||
|
use std::sync::mpsc;
|
||||||
|
|
||||||
|
let root = TestRoot::new("reader-housekeeping");
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let stored = store.put(b"reader housekeeping shard").unwrap();
|
||||||
|
let temp = root
|
||||||
|
.0
|
||||||
|
.join("sha256")
|
||||||
|
.join(&stored.artifact_ref.digest_hex()[..2])
|
||||||
|
.join(".crank-artifact-tmp-v1-00000000000000000000000000000000-1-10");
|
||||||
|
fs::write(&temp, b"partial").unwrap();
|
||||||
|
let (_, candidates) = store.scan_stale_temps(Duration::ZERO, 128, 1).unwrap();
|
||||||
|
let mut reader = HousekeepingReader {
|
||||||
|
store: store.clone(),
|
||||||
|
candidate: candidates.into_iter().next(),
|
||||||
|
bytes: io::Cursor::new(b"reader bytes".to_vec()),
|
||||||
|
};
|
||||||
|
let writer_store = store.clone();
|
||||||
|
let (sender, receiver) = mpsc::channel();
|
||||||
|
thread::spawn(move || {
|
||||||
|
sender.send(writer_store.put_reader(&mut reader)).unwrap();
|
||||||
|
});
|
||||||
|
|
||||||
|
let stored = receiver
|
||||||
|
.recv_timeout(Duration::from_secs(2))
|
||||||
|
.expect("put_reader deadlocked while its reader ran housekeeping")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(store.read(&stored.artifact_ref).unwrap(), b"reader bytes");
|
||||||
|
assert!(!temp.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
fn forked_crash_checkpoints_publish_only_complete_or_absent_blobs() {
|
||||||
|
let _guard = fault_guard();
|
||||||
|
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
||||||
|
set_checkpoint(
|
||||||
|
std::env::var("CRANK_ARTIFACTS_TEST_STAGE").unwrap(),
|
||||||
|
FaultAction::Exit,
|
||||||
|
);
|
||||||
|
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
||||||
|
let _ = store.put(b"crash-consistent bytes");
|
||||||
|
panic!("checkpoint did not terminate the child");
|
||||||
|
}
|
||||||
|
|
||||||
|
for stage in ["write", "file_fsync", "publish", "directory_fsync"] {
|
||||||
|
let root = TestRoot::new(stage);
|
||||||
|
let status = Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"forked_crash_checkpoints_publish_only_complete_or_absent_blobs",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
||||||
|
.env("CRANK_ARTIFACTS_TEST_STAGE", stage)
|
||||||
|
.status()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(status.code(), Some(86), "stage={stage}");
|
||||||
|
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let expected = ArtifactRef::from_digest_hex(&format!(
|
||||||
|
"{:x}",
|
||||||
|
Sha256::digest(b"crash-consistent bytes")
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
match store.read(&expected) {
|
||||||
|
Ok(bytes) => assert_eq!(bytes, b"crash-consistent bytes", "stage={stage}"),
|
||||||
|
Err(ArtifactError::NotFound) => {}
|
||||||
|
Err(error) => panic!("stage={stage} exposed an invalid final blob: {error}"),
|
||||||
|
}
|
||||||
|
let stored = store.put(b"crash-consistent bytes").unwrap();
|
||||||
|
assert_eq!(stored.artifact_ref, expected, "stage={stage}");
|
||||||
|
assert_eq!(
|
||||||
|
store.read(&expected).unwrap(),
|
||||||
|
b"crash-consistent bytes",
|
||||||
|
"stage={stage}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let (_, candidates) = store.scan_stale_temps(Duration::ZERO, 128, 16).unwrap();
|
||||||
|
for candidate in candidates {
|
||||||
|
store.delete_stale_temp(candidate).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
fn disk_full_fault_seams_leave_a_retryable_store() {
|
||||||
|
let _guard = fault_guard();
|
||||||
|
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
||||||
|
set_checkpoint(
|
||||||
|
std::env::var("CRANK_ARTIFACTS_TEST_STAGE").unwrap(),
|
||||||
|
FaultAction::Fail,
|
||||||
|
);
|
||||||
|
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store.put(b"disk-full seam bytes").unwrap_err(),
|
||||||
|
ArtifactError::Storage
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for stage in ["write", "file_fsync", "publish", "directory_fsync"] {
|
||||||
|
let root = TestRoot::new(&format!("full-{stage}"));
|
||||||
|
let status = Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"disk_full_fault_seams_leave_a_retryable_store",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
||||||
|
.env("CRANK_ARTIFACTS_TEST_STAGE", stage)
|
||||||
|
.status()
|
||||||
|
.unwrap();
|
||||||
|
assert!(status.success(), "stage={stage}");
|
||||||
|
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let expected =
|
||||||
|
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"disk-full seam bytes")))
|
||||||
|
.unwrap();
|
||||||
|
match store.read(&expected) {
|
||||||
|
Ok(bytes) => assert_eq!(bytes, b"disk-full seam bytes", "stage={stage}"),
|
||||||
|
Err(ArtifactError::NotFound) => {}
|
||||||
|
Err(error) => panic!("stage={stage} exposed an invalid final blob: {error}"),
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
store.put(b"disk-full seam bytes").unwrap().artifact_ref,
|
||||||
|
expected,
|
||||||
|
"stage={stage}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.read(&expected).unwrap(),
|
||||||
|
b"disk-full seam bytes",
|
||||||
|
"stage={stage}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn real_process_write_limit_leaves_no_partial_final_blob() {
|
||||||
|
if let Some(root) = std::env::var_os("CRANK_ARTIFACTS_CHILD_ROOT") {
|
||||||
|
// Keep the test's host filesystem untouched while making the actual
|
||||||
|
// write syscall fail as it would for a quota/disk-full condition.
|
||||||
|
unsafe {
|
||||||
|
assert_ne!(libc::signal(libc::SIGXFSZ, libc::SIG_IGN), libc::SIG_ERR);
|
||||||
|
let limit = libc::rlimit {
|
||||||
|
rlim_cur: 1,
|
||||||
|
rlim_max: 1,
|
||||||
|
};
|
||||||
|
assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0);
|
||||||
|
}
|
||||||
|
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store.put(b"real write limit bytes").unwrap_err(),
|
||||||
|
ArtifactError::Storage
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let root = TestRoot::new("real-write-limit");
|
||||||
|
let status = Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"real_process_write_limit_leaves_no_partial_final_blob",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env("CRANK_ARTIFACTS_CHILD_ROOT", &root.0)
|
||||||
|
.status()
|
||||||
|
.unwrap();
|
||||||
|
assert!(status.success());
|
||||||
|
|
||||||
|
let store = ArtifactStore::open(&root.0).unwrap();
|
||||||
|
let expected =
|
||||||
|
ArtifactRef::from_digest_hex(&format!("{:x}", Sha256::digest(b"real write limit bytes")))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(store.read(&expected).unwrap_err(), ArtifactError::NotFound);
|
||||||
|
assert_eq!(
|
||||||
|
store.put(b"real write limit bytes").unwrap().artifact_ref,
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
assert_eq!(store.read(&expected).unwrap(), b"real write limit bytes");
|
||||||
|
}
|
||||||
@@ -1,8 +1,9 @@
|
|||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
AdminProcessConfig, CacheSettings, DatabaseSettings, McpProcessConfig, MetricsSettings,
|
AdminProcessConfig, CacheSettings, DatabaseSettings, ExternalReferenceSettings,
|
||||||
MigratorConfig, ObservabilitySettings, OtlpSettings, OutboundSettings, RuntimeSettings,
|
McpProcessConfig, MetricsSettings, MigratorConfig, ObservabilitySettings, OtlpSettings,
|
||||||
|
OutboundSettings, RuntimeSettings,
|
||||||
};
|
};
|
||||||
|
|
||||||
impl fmt::Debug for DatabaseSettings {
|
impl fmt::Debug for DatabaseSettings {
|
||||||
@@ -32,6 +33,18 @@ impl fmt::Debug for OutboundSettings {
|
|||||||
.finish()
|
.finish()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
impl fmt::Debug for ExternalReferenceSettings {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
f.debug_struct("ExternalReferenceSettings")
|
||||||
|
.field("allowed_url_prefix_count", &self.allowed_url_prefixes.len())
|
||||||
|
.field("max_depth", &self.max_depth)
|
||||||
|
.field("max_documents", &self.max_documents)
|
||||||
|
.field("max_fetch_bytes", &self.max_fetch_bytes)
|
||||||
|
.field("fetch_timeout_ms", &self.fetch_timeout_ms)
|
||||||
|
.field("max_expanded_nodes", &self.max_expanded_nodes)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
impl fmt::Debug for RuntimeSettings {
|
impl fmt::Debug for RuntimeSettings {
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
f.debug_struct("RuntimeSettings")
|
f.debug_struct("RuntimeSettings")
|
||||||
@@ -93,6 +106,7 @@ impl fmt::Debug for AdminProcessConfig {
|
|||||||
f.debug_struct("AdminProcessConfig")
|
f.debug_struct("AdminProcessConfig")
|
||||||
.field("database", &self.database)
|
.field("database", &self.database)
|
||||||
.field("runtime", &self.runtime)
|
.field("runtime", &self.runtime)
|
||||||
|
.field("external_references", &self.external_references)
|
||||||
.field("observability", &self.observability)
|
.field("observability", &self.observability)
|
||||||
.field("storage_root", &"configured")
|
.field("storage_root", &"configured")
|
||||||
.field("session_secret", &self.session_secret)
|
.field("session_secret", &self.session_secret)
|
||||||
|
|||||||
@@ -15,8 +15,9 @@ pub use diagnostic::{ConfigError, Diagnostic, DiagnosticCode};
|
|||||||
pub use migrator::{MigratorConfig, parse_migrator};
|
pub use migrator::{MigratorConfig, parse_migrator};
|
||||||
pub use process::{
|
pub use process::{
|
||||||
AdminProcessConfig, CacheBackend, CacheSettings, DatabaseSettings, DeprecationRecord,
|
AdminProcessConfig, CacheBackend, CacheSettings, DatabaseSettings, DeprecationRecord,
|
||||||
EffectiveConfig, McpProcessConfig, MetricsSettings, ObservabilitySettings, OtlpSettings,
|
EffectiveConfig, ExternalReferenceSettings, McpProcessConfig, MetricsSettings,
|
||||||
OutboundSettings, PoolSettings, ProcessKind, RateLimitSettings, RuntimeSettings, parse_process,
|
ObservabilitySettings, OtlpSettings, OutboundSettings, PoolSettings, ProcessKind,
|
||||||
|
RateLimitSettings, RuntimeSettings, parse_process,
|
||||||
};
|
};
|
||||||
pub use schema::{
|
pub use schema::{
|
||||||
FieldMode, FieldSpec, ProcessScope, Sensitivity, deployment_field_registry, field_registry,
|
FieldMode, FieldSpec, ProcessScope, Sensitivity, deployment_field_registry, field_registry,
|
||||||
|
|||||||
@@ -12,7 +12,9 @@ use std::{
|
|||||||
path::PathBuf,
|
path::PathBuf,
|
||||||
};
|
};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
mod external_references;
|
||||||
mod list_parsers;
|
mod list_parsers;
|
||||||
|
pub use external_references::ExternalReferenceSettings;
|
||||||
const MAX_ENV_VALUE_BYTES: usize = 8_192;
|
const MAX_ENV_VALUE_BYTES: usize = 8_192;
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub enum ProcessKind {
|
pub enum ProcessKind {
|
||||||
@@ -127,6 +129,7 @@ pub struct ObservabilitySettings {
|
|||||||
pub struct AdminProcessConfig {
|
pub struct AdminProcessConfig {
|
||||||
pub database: DatabaseSettings,
|
pub database: DatabaseSettings,
|
||||||
pub runtime: RuntimeSettings,
|
pub runtime: RuntimeSettings,
|
||||||
|
pub external_references: ExternalReferenceSettings,
|
||||||
pub observability: ObservabilitySettings,
|
pub observability: ObservabilitySettings,
|
||||||
pub bind_addr: SocketAddr,
|
pub bind_addr: SocketAddr,
|
||||||
pub storage_root: PathBuf,
|
pub storage_root: PathBuf,
|
||||||
@@ -154,8 +157,8 @@ pub struct McpProcessConfig {
|
|||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
enum Projection {
|
enum Projection {
|
||||||
Admin(AdminProcessConfig),
|
Admin(Box<AdminProcessConfig>),
|
||||||
Mcp(McpProcessConfig),
|
Mcp(Box<McpProcessConfig>),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -790,9 +793,10 @@ pub fn parse_process(
|
|||||||
{
|
{
|
||||||
parser.push(DiagnosticCode::UnsafeCombination, "admin.exposure.tls");
|
parser.push(DiagnosticCode::UnsafeCombination, "admin.exposure.tls");
|
||||||
}
|
}
|
||||||
Projection::Admin(AdminProcessConfig {
|
Projection::Admin(Box::new(AdminProcessConfig {
|
||||||
database,
|
database,
|
||||||
runtime,
|
runtime,
|
||||||
|
external_references: external_references::parse(&mut parser),
|
||||||
observability,
|
observability,
|
||||||
bind_addr,
|
bind_addr,
|
||||||
storage_root: parser.absolute_path("CRANK_STORAGE_ROOT", "/var/lib/crank/storage"),
|
storage_root: parser.absolute_path("CRANK_STORAGE_ROOT", "/var/lib/crank/storage"),
|
||||||
@@ -811,7 +815,7 @@ pub fn parse_process(
|
|||||||
bootstrap_display_name: parser
|
bootstrap_display_name: parser
|
||||||
.string("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", Some("Crank Owner")),
|
.string("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", Some("Crank Owner")),
|
||||||
demo_seed: parser.boolean("CRANK_DEMO_SEED"),
|
demo_seed: parser.boolean("CRANK_DEMO_SEED"),
|
||||||
})
|
}))
|
||||||
}
|
}
|
||||||
ProcessKind::McpServer => {
|
ProcessKind::McpServer => {
|
||||||
let rps = parser.number("CRANK_MCP_RATE_LIMIT_RPS") as u32;
|
let rps = parser.number("CRANK_MCP_RATE_LIMIT_RPS") as u32;
|
||||||
@@ -819,7 +823,7 @@ pub fn parse_process(
|
|||||||
if burst < rps {
|
if burst < rps {
|
||||||
parser.push(DiagnosticCode::UnsafeCombination, "mcp.rate_limit.burst");
|
parser.push(DiagnosticCode::UnsafeCombination, "mcp.rate_limit.burst");
|
||||||
}
|
}
|
||||||
Projection::Mcp(McpProcessConfig {
|
Projection::Mcp(Box::new(McpProcessConfig {
|
||||||
database,
|
database,
|
||||||
runtime,
|
runtime,
|
||||||
observability,
|
observability,
|
||||||
@@ -829,7 +833,7 @@ pub fn parse_process(
|
|||||||
requests_per_second: rps,
|
requests_per_second: rps,
|
||||||
burst,
|
burst,
|
||||||
},
|
},
|
||||||
})
|
}))
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -867,6 +871,18 @@ fn fingerprint_parts(kind: ProcessKind, projection: &Projection) -> Vec<String>
|
|||||||
format!("session_ttl={}", config.session_ttl_hours),
|
format!("session_ttl={}", config.session_ttl_hours),
|
||||||
format!("trusted_proxies={:?}", config.trusted_proxy_ips),
|
format!("trusted_proxies={:?}", config.trusted_proxy_ips),
|
||||||
format!("demo={}", config.demo_seed),
|
format!("demo={}", config.demo_seed),
|
||||||
|
format!(
|
||||||
|
"external_reference_prefixes={}",
|
||||||
|
config.external_references.allowed_url_prefixes.join(",")
|
||||||
|
),
|
||||||
|
format!(
|
||||||
|
"external_reference_limits={}:{}:{}:{}:{}",
|
||||||
|
config.external_references.max_depth,
|
||||||
|
config.external_references.max_documents,
|
||||||
|
config.external_references.max_fetch_bytes,
|
||||||
|
config.external_references.fetch_timeout_ms,
|
||||||
|
config.external_references.max_expanded_nodes,
|
||||||
|
),
|
||||||
"storage=path-configured".to_owned(),
|
"storage=path-configured".to_owned(),
|
||||||
format!("session_secret={}", config.session_secret.is_configured()),
|
format!("session_secret={}", config.session_secret.is_configured()),
|
||||||
format!("pepper={}", config.password_pepper.is_configured()),
|
format!("pepper={}", config.password_pepper.is_configured()),
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#[derive(Clone, Eq, PartialEq)]
|
||||||
|
pub struct ExternalReferenceSettings {
|
||||||
|
/// Canonical HTTP(S) URL prefixes which opt an operator into remote `$ref` fetches.
|
||||||
|
/// An empty list is a deliberate default-deny switch.
|
||||||
|
pub allowed_url_prefixes: Vec<String>,
|
||||||
|
pub max_depth: usize,
|
||||||
|
pub max_documents: usize,
|
||||||
|
pub max_fetch_bytes: usize,
|
||||||
|
pub fetch_timeout_ms: u64,
|
||||||
|
pub max_expanded_nodes: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn parse(parser: &mut super::Parser<'_>) -> ExternalReferenceSettings {
|
||||||
|
ExternalReferenceSettings {
|
||||||
|
allowed_url_prefixes: parser
|
||||||
|
.url_prefix_list("CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES"),
|
||||||
|
max_depth: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH") as usize,
|
||||||
|
max_documents: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS") as usize,
|
||||||
|
max_fetch_bytes: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES") as usize,
|
||||||
|
fetch_timeout_ms: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS"),
|
||||||
|
max_expanded_nodes: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES")
|
||||||
|
as usize,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ use crate::{
|
|||||||
DiagnosticCode,
|
DiagnosticCode,
|
||||||
validation::{parse_host_list, parse_ip_list},
|
validation::{parse_host_list, parse_ip_list},
|
||||||
};
|
};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
impl super::Parser<'_> {
|
impl super::Parser<'_> {
|
||||||
pub(super) fn host_list(&mut self, name: &'static str) -> Vec<String> {
|
pub(super) fn host_list(&mut self, name: &'static str) -> Vec<String> {
|
||||||
@@ -33,4 +34,41 @@ impl super::Parser<'_> {
|
|||||||
}
|
}
|
||||||
parsed.items
|
parsed.items
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn url_prefix_list(&mut self, name: &'static str) -> Vec<String> {
|
||||||
|
let Some(raw) = self.optional(name) else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
let mut prefixes = Vec::new();
|
||||||
|
let mut invalid = false;
|
||||||
|
for item in raw.split(',') {
|
||||||
|
let item = item.trim();
|
||||||
|
let Ok(url) = Url::parse(item) else {
|
||||||
|
invalid = true;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !matches!(url.scheme(), "http" | "https")
|
||||||
|
|| url.host_str().is_none()
|
||||||
|
|| !url.username().is_empty()
|
||||||
|
|| url.password().is_some()
|
||||||
|
|| url.query().is_some()
|
||||||
|
|| url.fragment().is_some()
|
||||||
|
{
|
||||||
|
invalid = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let canonical = url.to_string();
|
||||||
|
if !prefixes.contains(&canonical) {
|
||||||
|
prefixes.push(canonical);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if invalid {
|
||||||
|
self.push(DiagnosticCode::InvalidType, name);
|
||||||
|
}
|
||||||
|
if prefixes.len() > 64 {
|
||||||
|
self.push(DiagnosticCode::OutOfRange, name);
|
||||||
|
prefixes.truncate(64);
|
||||||
|
}
|
||||||
|
prefixes
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ macro_rules! f {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
static FIELDS: [FieldSpec; 59] = [
|
static FIELDS: [FieldSpec; 65] = [
|
||||||
FieldSpec {
|
FieldSpec {
|
||||||
compatibility: Some("legacy URL form"),
|
compatibility: Some("legacy URL form"),
|
||||||
rules: &[
|
rules: &[
|
||||||
@@ -339,6 +339,78 @@ static FIELDS: [FieldSpec; 59] = [
|
|||||||
Some(67108864),
|
Some(67108864),
|
||||||
Public
|
Public
|
||||||
),
|
),
|
||||||
|
FieldSpec {
|
||||||
|
rules: &[
|
||||||
|
"empty list disables external OpenAPI reference fetching",
|
||||||
|
"each prefix must be canonical HTTP(S) without userinfo, query, or fragment",
|
||||||
|
],
|
||||||
|
..f!(
|
||||||
|
"import.external_references.allowed_url_prefixes",
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES",
|
||||||
|
AdminApi,
|
||||||
|
"url_prefix_list",
|
||||||
|
None,
|
||||||
|
Some(""),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Internal
|
||||||
|
)
|
||||||
|
},
|
||||||
|
f!(
|
||||||
|
"import.external_references.max_depth",
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH",
|
||||||
|
AdminApi,
|
||||||
|
"u32",
|
||||||
|
Some("edges"),
|
||||||
|
Some("8"),
|
||||||
|
Some(1),
|
||||||
|
Some(32),
|
||||||
|
Public
|
||||||
|
),
|
||||||
|
f!(
|
||||||
|
"import.external_references.max_documents",
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS",
|
||||||
|
AdminApi,
|
||||||
|
"u32",
|
||||||
|
Some("documents"),
|
||||||
|
Some("32"),
|
||||||
|
Some(1),
|
||||||
|
Some(32),
|
||||||
|
Public
|
||||||
|
),
|
||||||
|
f!(
|
||||||
|
"import.external_references.max_fetch_bytes",
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES",
|
||||||
|
AdminApi,
|
||||||
|
"u64",
|
||||||
|
Some("bytes"),
|
||||||
|
Some("262144"),
|
||||||
|
Some(1),
|
||||||
|
Some(4194304),
|
||||||
|
Public
|
||||||
|
),
|
||||||
|
f!(
|
||||||
|
"import.external_references.fetch_timeout_ms",
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS",
|
||||||
|
AdminApi,
|
||||||
|
"u64",
|
||||||
|
Some("milliseconds"),
|
||||||
|
Some("10000"),
|
||||||
|
Some(1),
|
||||||
|
Some(300000),
|
||||||
|
Public
|
||||||
|
),
|
||||||
|
f!(
|
||||||
|
"import.external_references.max_expanded_nodes",
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES",
|
||||||
|
AdminApi,
|
||||||
|
"u32",
|
||||||
|
Some("nodes"),
|
||||||
|
Some("10000"),
|
||||||
|
Some(1),
|
||||||
|
Some(100000),
|
||||||
|
Public
|
||||||
|
),
|
||||||
f!(
|
f!(
|
||||||
"observability.environment",
|
"observability.environment",
|
||||||
"CRANK_ENVIRONMENT",
|
"CRANK_ENVIRONMENT",
|
||||||
|
|||||||
@@ -108,9 +108,64 @@ fn source_for(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn registry_covers_exactly_the_59_observed_runtime_names() {
|
fn external_reference_contract_is_default_off_and_validates_prefixes_and_limits() {
|
||||||
|
let config = parse_process(
|
||||||
|
ProcessKind::AdminApi,
|
||||||
|
ConfigSource::from_utf8(required_admin()),
|
||||||
|
)
|
||||||
|
.expect("the external reference contract has safe defaults");
|
||||||
|
let references = &config.admin().unwrap().external_references;
|
||||||
|
assert!(references.allowed_url_prefixes.is_empty());
|
||||||
|
assert_eq!(references.max_depth, 8);
|
||||||
|
assert_eq!(references.max_documents, 32);
|
||||||
|
assert_eq!(references.max_fetch_bytes, 262_144);
|
||||||
|
assert_eq!(references.fetch_timeout_ms, 10_000);
|
||||||
|
assert_eq!(references.max_expanded_nodes, 10_000);
|
||||||
|
|
||||||
|
let mut allowed = required_admin();
|
||||||
|
allowed.insert(
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES".to_owned(),
|
||||||
|
"https://schemas.example.test/openapi/,https://schemas.example.test/openapi/".to_owned(),
|
||||||
|
);
|
||||||
|
let config = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(allowed)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
config
|
||||||
|
.admin()
|
||||||
|
.unwrap()
|
||||||
|
.external_references
|
||||||
|
.allowed_url_prefixes,
|
||||||
|
vec!["https://schemas.example.test/openapi/".to_owned()]
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut invalid = required_admin();
|
||||||
|
invalid.insert(
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES".to_owned(),
|
||||||
|
"https://user:secret@schemas.example.test/openapi/".to_owned(),
|
||||||
|
);
|
||||||
|
let error = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(invalid))
|
||||||
|
.expect_err("userinfo must never become an operator allow rule");
|
||||||
|
assert!(error.diagnostics().iter().any(|diagnostic| {
|
||||||
|
diagnostic.code == DiagnosticCode::InvalidType
|
||||||
|
&& diagnostic.field == "import.external_references.allowed_url_prefixes"
|
||||||
|
}));
|
||||||
|
|
||||||
|
let mut out_of_range = required_admin();
|
||||||
|
out_of_range.insert(
|
||||||
|
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS".to_owned(),
|
||||||
|
"33".to_owned(),
|
||||||
|
);
|
||||||
|
let error = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(out_of_range))
|
||||||
|
.expect_err("graph document limit must be bounded");
|
||||||
|
assert!(error.diagnostics().iter().any(|diagnostic| {
|
||||||
|
diagnostic.code == DiagnosticCode::OutOfRange
|
||||||
|
&& diagnostic.field == "import.external_references.max_documents"
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registry_covers_exactly_the_65_observed_runtime_names() {
|
||||||
let registry = field_registry();
|
let registry = field_registry();
|
||||||
assert_eq!(registry.len(), 59);
|
assert_eq!(registry.len(), 65);
|
||||||
let unique = registry
|
let unique = registry
|
||||||
.iter()
|
.iter()
|
||||||
.map(|field| field.env_name)
|
.map(|field| field.env_name)
|
||||||
|
|||||||
@@ -30,6 +30,9 @@ fn generated_reference_distinguishes_required_and_optional_fields() {
|
|||||||
assert!(reference.contains(
|
assert!(reference.contains(
|
||||||
"| `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` |"
|
"| `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` |"
|
||||||
));
|
));
|
||||||
|
assert!(reference.contains(
|
||||||
|
"| `CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES` | `import.external_references.allowed_url_prefixes` | `AdminApi` | `url_prefix_list/-` | `` |"
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -2,16 +2,28 @@ mod mapping;
|
|||||||
pub mod model;
|
pub mod model;
|
||||||
mod naming;
|
mod naming;
|
||||||
mod normalize;
|
mod normalize;
|
||||||
|
mod normalize_coverage;
|
||||||
|
mod normalize_limits;
|
||||||
|
mod normalize_schema;
|
||||||
mod openapi3;
|
mod openapi3;
|
||||||
mod payload;
|
mod payload;
|
||||||
mod recommendations;
|
mod recommendations;
|
||||||
|
mod reference;
|
||||||
mod schema;
|
mod schema;
|
||||||
mod swagger2;
|
mod swagger2;
|
||||||
|
|
||||||
pub use model::{
|
pub use model::{
|
||||||
ImportFinding, ImportFindingSeverity, ImportGroupPreview, ImportOperationCandidate,
|
ExternalDocumentSnapshot, ImportFinding, ImportFindingSeverity, ImportGroupPreview,
|
||||||
ImportPreview, ImportSourcePreview, RestImportCandidate, RestImportDocument,
|
ImportOperationCandidate, ImportPreview, ImportSourcePreview, NORMALIZER_VERSION,
|
||||||
RestImportOperation, RestImportParameter, RestParameterLocation,
|
NormalizationConfig, NormalizedFinding, NormalizedIr, NormalizedOperation, NormalizedParameter,
|
||||||
|
NormalizedReference, NormalizedSchema, NormalizedSchemaConstraints, NormalizedSchemaKind,
|
||||||
|
PROJECTION_VERSION, ResolvedReferenceEdge, ResolvedReferenceGraph, ResolvedReferenceNode,
|
||||||
|
RestImportCandidate, RestImportDocument, RestImportOperation, RestImportParameter,
|
||||||
|
RestParameterLocation, SourceDigest, SourceIdentity, SourceLocation, UnresolvedReference,
|
||||||
|
};
|
||||||
|
pub use normalize::{
|
||||||
|
ImportParseError, external_reference_uris, normalize_verified_bundle,
|
||||||
|
normalize_verified_document, preview_document, preview_document_legacy_v1, preview_from_ir,
|
||||||
|
reference_uris, validate_normalized_ir,
|
||||||
};
|
};
|
||||||
pub use normalize::preview_document;
|
|
||||||
pub use payload::operation_draft_from_candidate;
|
pub use payload::operation_draft_from_candidate;
|
||||||
|
|||||||
@@ -1,9 +1,372 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use crank_core::{HttpMethod, RestTarget, ToolDescription, WizardState};
|
use crank_core::{HttpMethod, RestTarget, ToolDescription, WizardState};
|
||||||
use crank_mapping::MappingSet;
|
use crank_mapping::MappingSet;
|
||||||
use crank_schema::Schema;
|
use crank_schema::Schema;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Deserializer, Serialize, de};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
|
/// The immutable contract used to normalize an OpenAPI source. These names
|
||||||
|
/// deliberately travel with an import job: changing either contract must not
|
||||||
|
/// silently reinterpret a pending preview.
|
||||||
|
pub const NORMALIZER_VERSION: &str = "normalized-ir-v3";
|
||||||
|
pub const PROJECTION_VERSION: &str = "preview-v3";
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
|
||||||
|
pub struct SourceDigest(String);
|
||||||
|
|
||||||
|
impl SourceDigest {
|
||||||
|
pub fn parse(value: impl Into<String>) -> Result<Self, &'static str> {
|
||||||
|
let value = value.into();
|
||||||
|
if value.len() == 64
|
||||||
|
&& value
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
{
|
||||||
|
Ok(Self(value))
|
||||||
|
} else {
|
||||||
|
Err("source digest must be a lowercase SHA-256 hex string")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn as_str(&self) -> &str {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'de> Deserialize<'de> for SourceDigest {
|
||||||
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
let value = String::deserialize(deserializer)?;
|
||||||
|
Self::parse(value).map_err(de::Error::custom)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct SourceIdentity {
|
||||||
|
pub digest: SourceDigest,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum SourceSyntax {
|
||||||
|
Json,
|
||||||
|
Yaml,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct CanonicalSourceNode {
|
||||||
|
pub construct_id: String,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
pub value: CanonicalSourceValue,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case", tag = "kind", content = "value")]
|
||||||
|
pub enum CanonicalSourceValue {
|
||||||
|
Null,
|
||||||
|
Boolean(bool),
|
||||||
|
Number(String),
|
||||||
|
String(String),
|
||||||
|
Array(Vec<CanonicalSourceNode>),
|
||||||
|
Object(BTreeMap<String, CanonicalSourceNode>),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedApiMetadata {
|
||||||
|
pub title: String,
|
||||||
|
pub version: Option<String>,
|
||||||
|
pub description: Option<String>,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedPath {
|
||||||
|
pub construct_id: String,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
pub path: String,
|
||||||
|
pub operation_ids: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizationConfig {
|
||||||
|
pub normalizer_version: String,
|
||||||
|
pub projection_version: String,
|
||||||
|
pub max_bytes: usize,
|
||||||
|
pub max_depth: usize,
|
||||||
|
pub max_nodes: usize,
|
||||||
|
pub max_collection_items: usize,
|
||||||
|
pub max_aliases: usize,
|
||||||
|
pub max_scalar_bytes: usize,
|
||||||
|
/// Maximum number of reference hops followed from one source location.
|
||||||
|
pub max_reference_depth: usize,
|
||||||
|
/// Maximum number of `$ref` occurrences inspected across the bundle.
|
||||||
|
pub max_references: usize,
|
||||||
|
/// Maximum number of immutable external documents supplied to the pure resolver.
|
||||||
|
pub max_reference_documents: usize,
|
||||||
|
/// Maximum number of nodes copied while expanding resolved references.
|
||||||
|
pub max_expanded_nodes: usize,
|
||||||
|
pub max_external_document_bytes: usize,
|
||||||
|
/// Signals that orchestration enabled external fetching. It never permits
|
||||||
|
/// I/O in this crate; it only distinguishes default-deny from a missing or
|
||||||
|
/// rejected supplied snapshot in exact findings.
|
||||||
|
pub external_references_enabled: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for NormalizationConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
normalizer_version: NORMALIZER_VERSION.to_owned(),
|
||||||
|
projection_version: PROJECTION_VERSION.to_owned(),
|
||||||
|
max_bytes: 256 * 1024,
|
||||||
|
max_depth: 64,
|
||||||
|
max_nodes: 50_000,
|
||||||
|
max_collection_items: 10_000,
|
||||||
|
max_aliases: 128,
|
||||||
|
max_scalar_bytes: 256 * 1024,
|
||||||
|
max_reference_depth: 32,
|
||||||
|
max_references: 4_096,
|
||||||
|
max_reference_documents: 32,
|
||||||
|
max_expanded_nodes: 100_000,
|
||||||
|
max_external_document_bytes: 256 * 1024,
|
||||||
|
external_references_enabled: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NormalizationConfig {
|
||||||
|
pub fn validate_versions(&self) -> Result<(), &'static str> {
|
||||||
|
if self.normalizer_version == NORMALIZER_VERSION
|
||||||
|
&& self.projection_version == PROJECTION_VERSION
|
||||||
|
{
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err("unsupported normalization contract version")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct SourceLocation {
|
||||||
|
/// RFC 6901 JSON Pointer. It is intentionally the only source location
|
||||||
|
/// exposed by the normalizer: no excerpts or parser diagnostics leak.
|
||||||
|
pub pointer: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct UnresolvedReference {
|
||||||
|
pub uri: String,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Immutable external input for the pure reference resolver. The caller owns
|
||||||
|
/// URL policy and I/O; `crank-import` only consumes already verified bytes.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct ExternalDocumentSnapshot {
|
||||||
|
pub canonical_uri: String,
|
||||||
|
pub digest: SourceDigest,
|
||||||
|
pub document: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct ResolvedReferenceNode {
|
||||||
|
pub snapshot_digest: SourceDigest,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct ResolvedReferenceEdge {
|
||||||
|
pub source: ResolvedReferenceNode,
|
||||||
|
pub target: ResolvedReferenceNode,
|
||||||
|
pub recursive: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct ResolvedReferenceGraph {
|
||||||
|
/// Sorted, deduplicated immutable dependency identities. Canonical URLs
|
||||||
|
/// deliberately do not enter the IR or public diagnostics.
|
||||||
|
#[serde(default)]
|
||||||
|
pub dependency_digests: Vec<SourceDigest>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub edges: Vec<ResolvedReferenceEdge>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An unresolved `$ref` preserved from the decoded source. This is deliberately
|
||||||
|
/// broader than schema references: path items, reusable parameters and other
|
||||||
|
/// object-level references remain available to a later resolution phase.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedReference {
|
||||||
|
pub construct_id: String,
|
||||||
|
pub uri: String,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedFinding {
|
||||||
|
pub code: String,
|
||||||
|
pub severity: ImportFindingSeverity,
|
||||||
|
pub message: String,
|
||||||
|
pub construct_id: String,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub operation_key: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CoverageDisposition {
|
||||||
|
Mapped,
|
||||||
|
Finding,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct CoverageEntry {
|
||||||
|
pub construct_id: String,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
pub disposition: CoverageDisposition,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedOperation {
|
||||||
|
pub stable_id: String,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
pub key: String,
|
||||||
|
pub method: HttpMethod,
|
||||||
|
pub path: String,
|
||||||
|
pub operation_id: Option<String>,
|
||||||
|
pub summary: Option<String>,
|
||||||
|
pub description: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub tags: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub parameters: Vec<NormalizedParameter>,
|
||||||
|
pub request_body_schema: Option<NormalizedSchema>,
|
||||||
|
pub response_schema: Option<NormalizedSchema>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub servers: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub findings: Vec<NormalizedFinding>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedParameter {
|
||||||
|
pub name: String,
|
||||||
|
pub location: RestParameterLocation,
|
||||||
|
pub required: bool,
|
||||||
|
pub description: Option<String>,
|
||||||
|
pub schema: Option<NormalizedSchema>,
|
||||||
|
pub construct_id: String,
|
||||||
|
pub source_location: SourceLocation,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedSchema {
|
||||||
|
pub construct_id: String,
|
||||||
|
pub location: SourceLocation,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub description: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub discriminator: Option<NormalizedDiscriminator>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub constraints: NormalizedSchemaConstraints,
|
||||||
|
pub kind: NormalizedSchemaKind,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedSchemaConstraints {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub minimum: Option<f64>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub maximum: Option<f64>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub min_length: Option<u64>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub max_length: Option<u64>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub pattern: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedDiscriminator {
|
||||||
|
pub property_name: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub mapping: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case", tag = "type")]
|
||||||
|
pub enum NormalizedSchemaKind {
|
||||||
|
Unknown,
|
||||||
|
Scalar {
|
||||||
|
scalar_type: NormalizedScalarKind,
|
||||||
|
format: Option<String>,
|
||||||
|
nullable: bool,
|
||||||
|
default_value: Option<NormalizedLiteral>,
|
||||||
|
enum_values: Vec<NormalizedLiteral>,
|
||||||
|
},
|
||||||
|
Object {
|
||||||
|
properties: BTreeMap<String, NormalizedSchema>,
|
||||||
|
required: Vec<String>,
|
||||||
|
},
|
||||||
|
Array {
|
||||||
|
items: Option<Box<NormalizedSchema>>,
|
||||||
|
},
|
||||||
|
Reference {
|
||||||
|
reference: UnresolvedReference,
|
||||||
|
},
|
||||||
|
Composition {
|
||||||
|
operator: String,
|
||||||
|
variants: Vec<NormalizedSchema>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum NormalizedScalarKind {
|
||||||
|
String,
|
||||||
|
Integer,
|
||||||
|
Number,
|
||||||
|
Boolean,
|
||||||
|
Null,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case", tag = "kind", content = "value")]
|
||||||
|
pub enum NormalizedLiteral {
|
||||||
|
String(String),
|
||||||
|
Integer(i64),
|
||||||
|
Unsigned(u64),
|
||||||
|
Number(f64),
|
||||||
|
Boolean(bool),
|
||||||
|
Null,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure, canonical representation between parsing and preview projection.
|
||||||
|
/// `operations`, `findings` and `coverage` are sorted before construction.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct NormalizedIr {
|
||||||
|
pub normalizer_version: String,
|
||||||
|
pub projection_version: String,
|
||||||
|
pub source_identity: SourceIdentity,
|
||||||
|
pub source_syntax: SourceSyntax,
|
||||||
|
pub source_tree: CanonicalSourceNode,
|
||||||
|
pub metadata: NormalizedApiMetadata,
|
||||||
|
#[serde(default)]
|
||||||
|
pub base_path_candidates: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub paths: Vec<NormalizedPath>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub unresolved_references: Vec<NormalizedReference>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub reference_graph: ResolvedReferenceGraph,
|
||||||
|
pub source: ImportSourcePreview,
|
||||||
|
#[serde(default)]
|
||||||
|
pub operations: Vec<NormalizedOperation>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub findings: Vec<NormalizedFinding>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub coverage: Vec<CoverageEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum ImportFindingSeverity {
|
pub enum ImportFindingSeverity {
|
||||||
@@ -80,7 +443,7 @@ pub struct RestImportCandidate {
|
|||||||
pub wizard_state: Option<WizardState>,
|
pub wizard_state: Option<WizardState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq)]
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct RestImportDocument {
|
pub struct RestImportDocument {
|
||||||
pub format: String,
|
pub format: String,
|
||||||
pub version: Option<String>,
|
pub version: Option<String>,
|
||||||
@@ -88,9 +451,11 @@ pub struct RestImportDocument {
|
|||||||
pub servers: Vec<String>,
|
pub servers: Vec<String>,
|
||||||
pub operations: Vec<RestImportOperation>,
|
pub operations: Vec<RestImportOperation>,
|
||||||
pub findings: Vec<ImportFinding>,
|
pub findings: Vec<ImportFinding>,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub internal_finding_locations: Vec<Option<SourceLocation>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq)]
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct RestImportOperation {
|
pub struct RestImportOperation {
|
||||||
pub key: String,
|
pub key: String,
|
||||||
pub method: HttpMethod,
|
pub method: HttpMethod,
|
||||||
@@ -101,21 +466,36 @@ pub struct RestImportOperation {
|
|||||||
pub tags: Vec<String>,
|
pub tags: Vec<String>,
|
||||||
pub parameters: Vec<RestImportParameter>,
|
pub parameters: Vec<RestImportParameter>,
|
||||||
pub request_body_schema: Option<Value>,
|
pub request_body_schema: Option<Value>,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub request_body_schema_location: Option<SourceLocation>,
|
||||||
pub response_schema: Option<Value>,
|
pub response_schema: Option<Value>,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub response_schema_location: Option<SourceLocation>,
|
||||||
pub servers: Vec<String>,
|
pub servers: Vec<String>,
|
||||||
pub findings: Vec<ImportFinding>,
|
pub findings: Vec<ImportFinding>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq)]
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct RestImportParameter {
|
pub struct RestImportParameter {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
pub location: RestParameterLocation,
|
pub location: RestParameterLocation,
|
||||||
pub required: bool,
|
pub required: bool,
|
||||||
pub description: Option<String>,
|
pub description: Option<String>,
|
||||||
pub schema: Option<Value>,
|
pub schema: Option<Value>,
|
||||||
|
#[serde(skip, default = "empty_source_location")]
|
||||||
|
pub source_location: SourceLocation,
|
||||||
|
#[serde(skip, default)]
|
||||||
|
pub schema_source_location: Option<SourceLocation>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
fn empty_source_location() -> SourceLocation {
|
||||||
|
SourceLocation {
|
||||||
|
pointer: String::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum RestParameterLocation {
|
pub enum RestParameterLocation {
|
||||||
Path,
|
Path,
|
||||||
Query,
|
Query,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,887 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::rest::model::{
|
||||||
|
CanonicalSourceNode, CanonicalSourceValue, CoverageDisposition, CoverageEntry,
|
||||||
|
ImportSourcePreview, NORMALIZER_VERSION, NormalizedIr, NormalizedOperation, NormalizedPath,
|
||||||
|
NormalizedReference, NormalizedSchema, NormalizedSchemaKind, PROJECTION_VERSION,
|
||||||
|
SourceLocation,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::ImportParseError;
|
||||||
|
|
||||||
|
pub(super) use super::normalize_schema::{has_reference_schema, typed_schema};
|
||||||
|
|
||||||
|
pub(super) fn validate_coverage(ir: &NormalizedIr) -> Result<(), ImportParseError> {
|
||||||
|
if ir.normalizer_version != NORMALIZER_VERSION || ir.projection_version != PROJECTION_VERSION {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
let mut source_nodes = BTreeMap::new();
|
||||||
|
index_source_tree(&ir.source_tree, "", &mut source_nodes)?;
|
||||||
|
let version = validate_source_contract(ir, &source_nodes)?;
|
||||||
|
|
||||||
|
let mut expected = BTreeMap::<String, ExpectedCoverage>::new();
|
||||||
|
for node in source_nodes.values() {
|
||||||
|
insert_expected(
|
||||||
|
&mut expected,
|
||||||
|
node.construct_id.clone(),
|
||||||
|
node.location.clone(),
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
insert_real_expected(
|
||||||
|
&mut expected,
|
||||||
|
&source_nodes,
|
||||||
|
"document".to_owned(),
|
||||||
|
SourceLocation {
|
||||||
|
pointer: String::new(),
|
||||||
|
},
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
for (index, location) in server_locations(ir, &source_nodes)?.into_iter().enumerate() {
|
||||||
|
insert_real_expected(
|
||||||
|
&mut expected,
|
||||||
|
&source_nodes,
|
||||||
|
format!("{version}:server:{index}"),
|
||||||
|
location,
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let expected_references = references_from_source_tree(&ir.source_tree);
|
||||||
|
if ir.unresolved_references != expected_references {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
for reference in &ir.unresolved_references {
|
||||||
|
insert_real_expected(
|
||||||
|
&mut expected,
|
||||||
|
&source_nodes,
|
||||||
|
reference.construct_id.clone(),
|
||||||
|
reference.location.clone(),
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let expected_paths = paths_from_operations(&ir.operations, &version);
|
||||||
|
if ir.paths != expected_paths {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
for path in &ir.paths {
|
||||||
|
insert_real_expected(
|
||||||
|
&mut expected,
|
||||||
|
&source_nodes,
|
||||||
|
path.construct_id.clone(),
|
||||||
|
path.location.clone(),
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_operation_order(&ir.operations)?;
|
||||||
|
for operation in &ir.operations {
|
||||||
|
collect_operation_expectations(operation, &version, &source_nodes, &mut expected)?;
|
||||||
|
}
|
||||||
|
collect_finding_only_expectations(&version, ir, &source_nodes, &mut expected)?;
|
||||||
|
|
||||||
|
let findings = ir
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.chain(
|
||||||
|
ir.operations
|
||||||
|
.iter()
|
||||||
|
.flat_map(|operation| &operation.findings),
|
||||||
|
)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
for finding in &findings {
|
||||||
|
let Some(target) = expected.get(&finding.construct_id) else {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
};
|
||||||
|
if target.location != finding.location {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if expected.iter().any(|(construct_id, target)| {
|
||||||
|
target.disposition == CoverageDisposition::Finding
|
||||||
|
&& !findings.iter().any(|finding| {
|
||||||
|
finding.construct_id == *construct_id && finding.location == target.location
|
||||||
|
})
|
||||||
|
}) {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut actual = BTreeMap::new();
|
||||||
|
for entry in &ir.coverage {
|
||||||
|
if actual
|
||||||
|
.insert(
|
||||||
|
entry.construct_id.clone(),
|
||||||
|
ExpectedCoverage {
|
||||||
|
location: entry.location.clone(),
|
||||||
|
disposition: entry.disposition.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if actual != expected {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
struct ExpectedCoverage {
|
||||||
|
location: SourceLocation,
|
||||||
|
disposition: CoverageDisposition,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_expected(
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
construct_id: String,
|
||||||
|
location: SourceLocation,
|
||||||
|
disposition: CoverageDisposition,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
let value = ExpectedCoverage {
|
||||||
|
location,
|
||||||
|
disposition,
|
||||||
|
};
|
||||||
|
if expected
|
||||||
|
.get(&construct_id)
|
||||||
|
.is_some_and(|existing| existing != &value)
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
expected.entry(construct_id).or_insert(value);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_real_expected(
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
source_nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
construct_id: String,
|
||||||
|
location: SourceLocation,
|
||||||
|
disposition: CoverageDisposition,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
if !source_nodes.contains_key(&location.pointer) {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
insert_expected(expected, construct_id, location, disposition)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn index_source_tree<'a>(
|
||||||
|
node: &'a CanonicalSourceNode,
|
||||||
|
expected_pointer: &str,
|
||||||
|
nodes: &mut BTreeMap<String, &'a CanonicalSourceNode>,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
if node.location.pointer != expected_pointer
|
||||||
|
|| node.construct_id != format!("source:{expected_pointer}")
|
||||||
|
|| nodes.insert(expected_pointer.to_owned(), node).is_some()
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
match &node.value {
|
||||||
|
CanonicalSourceValue::Array(items) => {
|
||||||
|
for (index, child) in items.iter().enumerate() {
|
||||||
|
index_source_tree(child, &format!("{expected_pointer}/{index}"), nodes)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CanonicalSourceValue::Object(items) => {
|
||||||
|
for (key, child) in items {
|
||||||
|
index_source_tree(
|
||||||
|
child,
|
||||||
|
&format!("{expected_pointer}/{}", escape_pointer(key)),
|
||||||
|
nodes,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_source_contract(
|
||||||
|
ir: &NormalizedIr,
|
||||||
|
nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
) -> Result<String, ImportParseError> {
|
||||||
|
let version = ir
|
||||||
|
.source
|
||||||
|
.version
|
||||||
|
.as_deref()
|
||||||
|
.ok_or(ImportParseError::InvalidDocument)?;
|
||||||
|
let valid = match ir.source.format.as_str() {
|
||||||
|
"openapi" => string_at(nodes, "/openapi") == Some(version) && supported_oas(version),
|
||||||
|
"swagger" => version == "2.0" && string_at(nodes, "/swagger") == Some("2.0"),
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
if !valid
|
||||||
|
|| ir.metadata.title != ir.source.title
|
||||||
|
|| ir.source.title != string_at(nodes, "/info/title").unwrap_or("Imported API")
|
||||||
|
|| ir.metadata.version != string_at(nodes, "/info/version").map(ToOwned::to_owned)
|
||||||
|
|| ir.metadata.description != string_at(nodes, "/info/description").map(ToOwned::to_owned)
|
||||||
|
|| ir.base_path_candidates
|
||||||
|
!= string_at(nodes, "/basePath")
|
||||||
|
.map(|value| vec![value.to_owned()])
|
||||||
|
.unwrap_or_default()
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
Ok(version.to_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn supported_oas(version: &str) -> bool {
|
||||||
|
["3.0.", "3.1."].into_iter().any(|prefix| {
|
||||||
|
version.strip_prefix(prefix).is_some_and(|patch| {
|
||||||
|
!patch.is_empty() && patch.bytes().all(|byte| byte.is_ascii_digit())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn string_at<'a>(
|
||||||
|
nodes: &'a BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
pointer: &str,
|
||||||
|
) -> Option<&'a str> {
|
||||||
|
match &nodes.get(pointer)?.value {
|
||||||
|
CanonicalSourceValue::String(value) => Some(value),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn server_locations(
|
||||||
|
ir: &NormalizedIr,
|
||||||
|
nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
) -> Result<Vec<SourceLocation>, ImportParseError> {
|
||||||
|
server_locations_for_source(&ir.source, nodes)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn server_locations_for_source(
|
||||||
|
source: &ImportSourcePreview,
|
||||||
|
nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
) -> Result<Vec<SourceLocation>, ImportParseError> {
|
||||||
|
let (urls, locations) = if source.format == "openapi" {
|
||||||
|
openapi_servers(nodes)
|
||||||
|
} else {
|
||||||
|
swagger_servers(nodes)
|
||||||
|
};
|
||||||
|
if urls != source.servers {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
Ok(locations)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn server_coverage(
|
||||||
|
source: &ImportSourcePreview,
|
||||||
|
source_tree: &CanonicalSourceNode,
|
||||||
|
version: &str,
|
||||||
|
) -> Result<Vec<CoverageEntry>, ImportParseError> {
|
||||||
|
let mut nodes = BTreeMap::new();
|
||||||
|
index_source_tree(source_tree, "", &mut nodes)?;
|
||||||
|
Ok(server_locations_for_source(source, &nodes)?
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, location)| CoverageEntry {
|
||||||
|
construct_id: format!("{version}:server:{index}"),
|
||||||
|
location,
|
||||||
|
disposition: CoverageDisposition::Mapped,
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn tag_locations(
|
||||||
|
root: &Value,
|
||||||
|
operation_pointer: &str,
|
||||||
|
tags: &[String],
|
||||||
|
) -> Result<Vec<SourceLocation>, ImportParseError> {
|
||||||
|
let source_tags = root
|
||||||
|
.pointer(&format!("{operation_pointer}/tags"))
|
||||||
|
.and_then(Value::as_array);
|
||||||
|
let locations = source_tags
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, value)| {
|
||||||
|
value.as_str().map(|value| {
|
||||||
|
(
|
||||||
|
value,
|
||||||
|
SourceLocation {
|
||||||
|
pointer: format!("{operation_pointer}/tags/{index}"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if locations
|
||||||
|
.iter()
|
||||||
|
.map(|(value, _)| *value)
|
||||||
|
.ne(tags.iter().map(String::as_str))
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
Ok(locations
|
||||||
|
.into_iter()
|
||||||
|
.map(|(_, location)| location)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn openapi_servers(
|
||||||
|
nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
) -> (Vec<String>, Vec<SourceLocation>) {
|
||||||
|
let Some(CanonicalSourceNode {
|
||||||
|
value: CanonicalSourceValue::Array(items),
|
||||||
|
..
|
||||||
|
}) = nodes.get("/servers").copied()
|
||||||
|
else {
|
||||||
|
return (Vec::new(), Vec::new());
|
||||||
|
};
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.filter_map(|item| {
|
||||||
|
let CanonicalSourceValue::Object(fields) = &item.value else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
let CanonicalSourceValue::String(url) = &fields.get("url")?.value else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
Some((url.trim_end_matches('/').to_owned(), item.location.clone()))
|
||||||
|
})
|
||||||
|
.unzip()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn swagger_servers(
|
||||||
|
nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
) -> (Vec<String>, Vec<SourceLocation>) {
|
||||||
|
let Some(host) = string_at(nodes, "/host") else {
|
||||||
|
return (Vec::new(), Vec::new());
|
||||||
|
};
|
||||||
|
let base_path = string_at(nodes, "/basePath").unwrap_or("");
|
||||||
|
let schemes = nodes
|
||||||
|
.get("/schemes")
|
||||||
|
.and_then(|node| match &node.value {
|
||||||
|
CanonicalSourceValue::Array(items) => Some(
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.filter_map(|item| match &item.value {
|
||||||
|
CanonicalSourceValue::String(value) => Some((value.as_str(), item)),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.filter(|items| !items.is_empty());
|
||||||
|
let schemes = schemes.unwrap_or_else(|| {
|
||||||
|
vec![(
|
||||||
|
"https",
|
||||||
|
*nodes.get("/host").expect("host was checked above"),
|
||||||
|
)]
|
||||||
|
});
|
||||||
|
schemes
|
||||||
|
.into_iter()
|
||||||
|
.map(|(scheme, node)| {
|
||||||
|
(
|
||||||
|
format!("{scheme}://{host}{base_path}")
|
||||||
|
.trim_end_matches('/')
|
||||||
|
.to_owned(),
|
||||||
|
node.location.clone(),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unzip()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_operation_order(operations: &[NormalizedOperation]) -> Result<(), ImportParseError> {
|
||||||
|
if operations
|
||||||
|
.windows(2)
|
||||||
|
.any(|pair| operation_sort_key(&pair[0]) >= operation_sort_key(&pair[1]))
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operation_sort_key(operation: &NormalizedOperation) -> (&str, u8, &str) {
|
||||||
|
(
|
||||||
|
&operation.path,
|
||||||
|
method_rank(operation.method),
|
||||||
|
&operation.location.pointer,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn method_rank(method: crank_core::HttpMethod) -> u8 {
|
||||||
|
match method {
|
||||||
|
crank_core::HttpMethod::Get => 0,
|
||||||
|
crank_core::HttpMethod::Post => 1,
|
||||||
|
crank_core::HttpMethod::Put => 2,
|
||||||
|
crank_core::HttpMethod::Patch => 3,
|
||||||
|
crank_core::HttpMethod::Delete => 4,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn method_name(method: crank_core::HttpMethod) -> &'static str {
|
||||||
|
match method {
|
||||||
|
crank_core::HttpMethod::Get => "get",
|
||||||
|
crank_core::HttpMethod::Post => "post",
|
||||||
|
crank_core::HttpMethod::Put => "put",
|
||||||
|
crank_core::HttpMethod::Patch => "patch",
|
||||||
|
crank_core::HttpMethod::Delete => "delete",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_operation_expectations(
|
||||||
|
operation: &NormalizedOperation,
|
||||||
|
version: &str,
|
||||||
|
source_nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
let method = method_name(operation.method);
|
||||||
|
let pointer = format!("/paths/{}/{method}", escape_pointer(&operation.path));
|
||||||
|
let stable_id = format!("{version}:{method}:{pointer}");
|
||||||
|
if operation.location.pointer != pointer
|
||||||
|
|| operation.stable_id != stable_id
|
||||||
|
|| operation.key != format!("{} {}", method.to_ascii_uppercase(), operation.path)
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
insert_real_expected(
|
||||||
|
expected,
|
||||||
|
source_nodes,
|
||||||
|
stable_id.clone(),
|
||||||
|
operation.location.clone(),
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let operation_id_location = SourceLocation {
|
||||||
|
pointer: format!("{pointer}/operationId"),
|
||||||
|
};
|
||||||
|
let operation_id_disposition = match operation
|
||||||
|
.operation_id
|
||||||
|
.as_deref()
|
||||||
|
.filter(|value| !value.trim().is_empty())
|
||||||
|
{
|
||||||
|
Some(value) if string_at(source_nodes, &operation_id_location.pointer) == Some(value) => {
|
||||||
|
CoverageDisposition::Mapped
|
||||||
|
}
|
||||||
|
Some(_) => return Err(ImportParseError::InvalidDocument),
|
||||||
|
None => CoverageDisposition::Finding,
|
||||||
|
};
|
||||||
|
if operation_id_disposition == CoverageDisposition::Mapped
|
||||||
|
|| source_nodes.contains_key(&operation_id_location.pointer)
|
||||||
|
{
|
||||||
|
insert_real_expected(
|
||||||
|
expected,
|
||||||
|
source_nodes,
|
||||||
|
format!("{stable_id}:operation_id"),
|
||||||
|
operation_id_location,
|
||||||
|
operation_id_disposition,
|
||||||
|
)?;
|
||||||
|
} else {
|
||||||
|
insert_expected(
|
||||||
|
expected,
|
||||||
|
format!("{stable_id}:operation_id"),
|
||||||
|
operation_id_location,
|
||||||
|
CoverageDisposition::Finding,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let tag_locations = normalized_tag_locations(source_nodes, &pointer, &operation.tags)?;
|
||||||
|
for (index, location) in tag_locations.into_iter().enumerate() {
|
||||||
|
insert_real_expected(
|
||||||
|
expected,
|
||||||
|
source_nodes,
|
||||||
|
format!("{stable_id}:tag:{index}"),
|
||||||
|
location,
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
for parameter in &operation.parameters {
|
||||||
|
if parameter.construct_id
|
||||||
|
!= format!(
|
||||||
|
"parameter:{}",
|
||||||
|
escape_pointer(¶meter.source_location.pointer)
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
insert_real_expected(
|
||||||
|
expected,
|
||||||
|
source_nodes,
|
||||||
|
parameter.construct_id.clone(),
|
||||||
|
parameter.source_location.clone(),
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
if let Some(schema) = ¶meter.schema {
|
||||||
|
collect_schema_expectations(schema, source_nodes, expected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(schema) = &operation.request_body_schema {
|
||||||
|
collect_schema_expectations(schema, source_nodes, expected)?;
|
||||||
|
}
|
||||||
|
if let Some(schema) = &operation.response_schema {
|
||||||
|
collect_schema_expectations(schema, source_nodes, expected)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_tag_locations(
|
||||||
|
nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
operation_pointer: &str,
|
||||||
|
tags: &[String],
|
||||||
|
) -> Result<Vec<SourceLocation>, ImportParseError> {
|
||||||
|
let tags_pointer = format!("{operation_pointer}/tags");
|
||||||
|
let source_tags = nodes.get(&tags_pointer).and_then(|node| match &node.value {
|
||||||
|
CanonicalSourceValue::Array(items) => Some(items),
|
||||||
|
_ => None,
|
||||||
|
});
|
||||||
|
let locations = source_tags
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.filter_map(|node| match &node.value {
|
||||||
|
CanonicalSourceValue::String(value) => Some((value, node.location.clone())),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if locations.iter().map(|(value, _)| *value).ne(tags.iter()) {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
Ok(locations
|
||||||
|
.into_iter()
|
||||||
|
.map(|(_, location)| location)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_schema_expectations(
|
||||||
|
schema: &NormalizedSchema,
|
||||||
|
source_nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
if schema.construct_id != format!("schema:{}", escape_pointer(&schema.location.pointer)) {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
insert_real_expected(
|
||||||
|
expected,
|
||||||
|
source_nodes,
|
||||||
|
schema.construct_id.clone(),
|
||||||
|
schema.location.clone(),
|
||||||
|
CoverageDisposition::Mapped,
|
||||||
|
)?;
|
||||||
|
if schema.description
|
||||||
|
!= string_at(
|
||||||
|
source_nodes,
|
||||||
|
&format!("{}/description", schema.location.pointer),
|
||||||
|
)
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
match &schema.kind {
|
||||||
|
NormalizedSchemaKind::Object { properties, .. } => {
|
||||||
|
for child in properties.values() {
|
||||||
|
collect_schema_expectations(child, source_nodes, expected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
NormalizedSchemaKind::Array { items: Some(child) } => {
|
||||||
|
collect_schema_expectations(child, source_nodes, expected)?;
|
||||||
|
}
|
||||||
|
NormalizedSchemaKind::Composition { variants, .. } => {
|
||||||
|
for child in variants {
|
||||||
|
collect_schema_expectations(child, source_nodes, expected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_finding_only_expectations(
|
||||||
|
version: &str,
|
||||||
|
ir: &NormalizedIr,
|
||||||
|
source_nodes: &BTreeMap<String, &CanonicalSourceNode>,
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
if ir.source.format == "openapi" && ir.source.servers.len() > 1 {
|
||||||
|
let location = SourceLocation {
|
||||||
|
pointer: "/servers".to_owned(),
|
||||||
|
};
|
||||||
|
insert_real_expected(
|
||||||
|
expected,
|
||||||
|
source_nodes,
|
||||||
|
format!("{version}:/servers"),
|
||||||
|
location,
|
||||||
|
CoverageDisposition::Finding,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
if ir.source.format == "openapi"
|
||||||
|
&& let Some(servers) = source_nodes.get("/servers").copied()
|
||||||
|
{
|
||||||
|
collect_invalid_openapi_server_findings(version, servers, expected)?;
|
||||||
|
}
|
||||||
|
let Some(CanonicalSourceNode {
|
||||||
|
value: CanonicalSourceValue::Object(paths),
|
||||||
|
..
|
||||||
|
}) = source_nodes.get("/paths").copied()
|
||||||
|
else {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
};
|
||||||
|
for path_item in paths.values() {
|
||||||
|
let CanonicalSourceValue::Object(methods) = &path_item.value else {
|
||||||
|
insert_finding_pointer(version, path_item, expected)?;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if ir.source.format == "openapi"
|
||||||
|
&& let Some(servers) = methods.get("servers")
|
||||||
|
{
|
||||||
|
collect_invalid_openapi_server_findings(version, servers, expected)?;
|
||||||
|
}
|
||||||
|
if let Some(parameters) = methods.get("parameters") {
|
||||||
|
collect_dropped_parameter_findings(
|
||||||
|
version,
|
||||||
|
parameters,
|
||||||
|
ir.source.format.as_str(),
|
||||||
|
expected,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
for method in ["head", "options", "trace", "connect"] {
|
||||||
|
if let Some(node) = methods.get(method) {
|
||||||
|
insert_finding_pointer(version, node, expected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for method in ["get", "post", "put", "patch", "delete"] {
|
||||||
|
if let Some(node) = methods.get(method) {
|
||||||
|
match &node.value {
|
||||||
|
CanonicalSourceValue::Object(operation) => {
|
||||||
|
if ir.source.format == "openapi"
|
||||||
|
&& let Some(servers) = operation.get("servers")
|
||||||
|
{
|
||||||
|
collect_invalid_openapi_server_findings(version, servers, expected)?;
|
||||||
|
}
|
||||||
|
if let Some(tags) = operation.get("tags") {
|
||||||
|
collect_invalid_tag_findings(version, tags, expected)?;
|
||||||
|
}
|
||||||
|
if let Some(parameters) = operation.get("parameters") {
|
||||||
|
collect_dropped_parameter_findings(
|
||||||
|
version,
|
||||||
|
parameters,
|
||||||
|
ir.source.format.as_str(),
|
||||||
|
expected,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => insert_finding_pointer(version, node, expected)?,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_invalid_openapi_server_findings(
|
||||||
|
version: &str,
|
||||||
|
servers: &CanonicalSourceNode,
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
let CanonicalSourceValue::Array(items) = &servers.value else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
for server in items {
|
||||||
|
let valid = matches!(
|
||||||
|
&server.value,
|
||||||
|
CanonicalSourceValue::Object(fields)
|
||||||
|
if matches!(fields.get("url").map(|node| &node.value), Some(CanonicalSourceValue::String(_)))
|
||||||
|
);
|
||||||
|
if !valid {
|
||||||
|
insert_finding_pointer(version, server, expected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_invalid_tag_findings(
|
||||||
|
version: &str,
|
||||||
|
tags: &CanonicalSourceNode,
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
let CanonicalSourceValue::Array(items) = &tags.value else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
for tag in items {
|
||||||
|
if !matches!(tag.value, CanonicalSourceValue::String(_)) {
|
||||||
|
insert_finding_pointer(version, tag, expected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_dropped_parameter_findings(
|
||||||
|
version: &str,
|
||||||
|
parameters: &CanonicalSourceNode,
|
||||||
|
format: &str,
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
let CanonicalSourceValue::Array(items) = ¶meters.value else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
for parameter in items {
|
||||||
|
let CanonicalSourceValue::Object(fields) = ¶meter.value else {
|
||||||
|
insert_finding_pointer(version, parameter, expected)?;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if fields.contains_key("$ref") {
|
||||||
|
insert_finding_pointer(version, parameter, expected)?;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
fields.get("name").map(|node| &node.value),
|
||||||
|
Some(CanonicalSourceValue::String(_))
|
||||||
|
) || !matches!(
|
||||||
|
fields.get("in").map(|node| &node.value),
|
||||||
|
Some(CanonicalSourceValue::String(_))
|
||||||
|
) {
|
||||||
|
insert_finding_pointer(version, parameter, expected)?;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(CanonicalSourceNode {
|
||||||
|
value: CanonicalSourceValue::String(location),
|
||||||
|
..
|
||||||
|
}) = fields.get("in")
|
||||||
|
else {
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
};
|
||||||
|
let supported = match format {
|
||||||
|
"openapi" => matches!(location.as_str(), "path" | "query" | "header"),
|
||||||
|
"swagger" => matches!(location.as_str(), "path" | "query" | "header" | "body"),
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
if !supported {
|
||||||
|
insert_finding_pointer(version, parameter, expected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_finding_pointer(
|
||||||
|
version: &str,
|
||||||
|
node: &CanonicalSourceNode,
|
||||||
|
expected: &mut BTreeMap<String, ExpectedCoverage>,
|
||||||
|
) -> Result<(), ImportParseError> {
|
||||||
|
insert_expected(
|
||||||
|
expected,
|
||||||
|
format!("{version}:{}", node.location.pointer),
|
||||||
|
node.location.clone(),
|
||||||
|
CoverageDisposition::Finding,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn references_from_source_tree(
|
||||||
|
source_tree: &CanonicalSourceNode,
|
||||||
|
) -> Vec<NormalizedReference> {
|
||||||
|
let mut references = Vec::new();
|
||||||
|
collect_references(source_tree, &mut references);
|
||||||
|
references.sort_by(|left, right| left.construct_id.cmp(&right.construct_id));
|
||||||
|
references
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_references(node: &CanonicalSourceNode, references: &mut Vec<NormalizedReference>) {
|
||||||
|
match &node.value {
|
||||||
|
CanonicalSourceValue::Array(items) => {
|
||||||
|
for item in items {
|
||||||
|
collect_references(item, references);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CanonicalSourceValue::Object(items) => {
|
||||||
|
if let Some(CanonicalSourceNode {
|
||||||
|
value: CanonicalSourceValue::String(uri),
|
||||||
|
location,
|
||||||
|
..
|
||||||
|
}) = items.get("$ref")
|
||||||
|
{
|
||||||
|
references.push(NormalizedReference {
|
||||||
|
construct_id: format!("reference:{}", escape_pointer(&location.pointer)),
|
||||||
|
uri: uri.clone(),
|
||||||
|
location: location.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for item in items.values() {
|
||||||
|
collect_references(item, references);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn canonical_source_tree(
|
||||||
|
value: &Value,
|
||||||
|
pointer: &str,
|
||||||
|
coverage: &mut Vec<CoverageEntry>,
|
||||||
|
) -> CanonicalSourceNode {
|
||||||
|
let id = format!("source:{pointer}");
|
||||||
|
let location = SourceLocation {
|
||||||
|
pointer: pointer.to_owned(),
|
||||||
|
};
|
||||||
|
coverage.push(CoverageEntry {
|
||||||
|
construct_id: id.clone(),
|
||||||
|
location: location.clone(),
|
||||||
|
disposition: CoverageDisposition::Mapped,
|
||||||
|
});
|
||||||
|
let value = match value {
|
||||||
|
Value::Null => CanonicalSourceValue::Null,
|
||||||
|
Value::Bool(value) => CanonicalSourceValue::Boolean(*value),
|
||||||
|
Value::Number(value) => CanonicalSourceValue::Number(value.to_string()),
|
||||||
|
Value::String(value) => CanonicalSourceValue::String(value.clone()),
|
||||||
|
Value::Array(items) => CanonicalSourceValue::Array(
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, value)| {
|
||||||
|
canonical_source_tree(value, &format!("{pointer}/{index}"), coverage)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
Value::Object(items) => CanonicalSourceValue::Object(
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.map(|(key, value)| {
|
||||||
|
(
|
||||||
|
key.clone(),
|
||||||
|
canonical_source_tree(
|
||||||
|
value,
|
||||||
|
&format!("{pointer}/{}", escape_pointer(key)),
|
||||||
|
coverage,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
CanonicalSourceNode {
|
||||||
|
construct_id: id,
|
||||||
|
location,
|
||||||
|
value,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn paths_from_operations(
|
||||||
|
operations: &[NormalizedOperation],
|
||||||
|
version: &str,
|
||||||
|
) -> Vec<NormalizedPath> {
|
||||||
|
let mut paths = BTreeMap::<String, Vec<String>>::new();
|
||||||
|
for operation in operations {
|
||||||
|
paths
|
||||||
|
.entry(operation.path.clone())
|
||||||
|
.or_default()
|
||||||
|
.push(operation.stable_id.clone());
|
||||||
|
}
|
||||||
|
paths
|
||||||
|
.into_iter()
|
||||||
|
.map(|(path, operation_ids)| NormalizedPath {
|
||||||
|
construct_id: format!("{version}:path:{path}"),
|
||||||
|
location: SourceLocation {
|
||||||
|
pointer: format!("/paths/{}", escape_pointer(&path)),
|
||||||
|
},
|
||||||
|
path,
|
||||||
|
operation_ids,
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn escape_pointer(value: &str) -> String {
|
||||||
|
value.replace('~', "~0").replace('/', "~1")
|
||||||
|
}
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
enum AliasQuote {
|
||||||
|
Single,
|
||||||
|
Double,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
struct BlockScalar {
|
||||||
|
parent_indent: usize,
|
||||||
|
content_indent: Option<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn alias_count(document: &str) -> usize {
|
||||||
|
// YAML lexical preflight: aliases are counted before serde_yaml can expand
|
||||||
|
// them. This is deliberately a small lexer rather than a second YAML
|
||||||
|
// parser: it understands quote escapes, comments, and flow aliases while
|
||||||
|
// keeping the scan bounded by the source size.
|
||||||
|
//
|
||||||
|
// Block scalars are the one intentionally conservative exception. Their
|
||||||
|
// contents are opaque YAML text, and reproducing YAML's indentation rules
|
||||||
|
// here would create a second parser with its own correctness risks. If a
|
||||||
|
// content line contains `*`, fail closed by returning usize::MAX; this
|
||||||
|
// prevents an alias-looking token from being hidden in a literal block at
|
||||||
|
// the cost of rejecting that block before decode.
|
||||||
|
let mut count = 0;
|
||||||
|
let mut quote = None;
|
||||||
|
let mut block_scalar: Option<BlockScalar> = None;
|
||||||
|
|
||||||
|
for line in document.split('\n') {
|
||||||
|
if let Some(block) = block_scalar {
|
||||||
|
if is_blank_line(line) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let indent = leading_spaces(line);
|
||||||
|
let is_content = match block.content_indent {
|
||||||
|
Some(content_indent) => indent >= content_indent,
|
||||||
|
None => indent > block.parent_indent,
|
||||||
|
};
|
||||||
|
if is_content {
|
||||||
|
if line.as_bytes().contains(&b'*') {
|
||||||
|
return usize::MAX;
|
||||||
|
}
|
||||||
|
if block.content_indent.is_none() {
|
||||||
|
block_scalar = Some(BlockScalar {
|
||||||
|
content_indent: Some(indent),
|
||||||
|
..block
|
||||||
|
});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
block_scalar = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let bytes = line.as_bytes();
|
||||||
|
let line_indent = leading_spaces(line);
|
||||||
|
let mut comment = false;
|
||||||
|
let mut index = 0;
|
||||||
|
while index < bytes.len() {
|
||||||
|
let byte = bytes[index];
|
||||||
|
if comment {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if let Some(current) = quote {
|
||||||
|
match current {
|
||||||
|
AliasQuote::Single if byte == b'\'' => {
|
||||||
|
// YAML escapes a single quote by doubling it. Consume
|
||||||
|
// both bytes so the second quote cannot reopen a
|
||||||
|
// scalar and desynchronise the scan.
|
||||||
|
if bytes.get(index + 1) == Some(&b'\'') {
|
||||||
|
index += 2;
|
||||||
|
} else {
|
||||||
|
quote = None;
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
AliasQuote::Double if byte == b'\\' => {
|
||||||
|
// A backslash escapes the following byte, including a
|
||||||
|
// quote. At end of line it only folds the YAML line;
|
||||||
|
// the quote remains open for the next line.
|
||||||
|
index += if index + 1 < bytes.len() { 2 } else { 1 };
|
||||||
|
}
|
||||||
|
AliasQuote::Double if byte == b'"' => {
|
||||||
|
quote = None;
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
_ => index += 1,
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
match byte {
|
||||||
|
b'\'' => {
|
||||||
|
quote = Some(AliasQuote::Single);
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
b'"' => {
|
||||||
|
quote = Some(AliasQuote::Double);
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
b'#' if index == 0
|
||||||
|
|| bytes
|
||||||
|
.get(index - 1)
|
||||||
|
.is_some_and(|previous| previous.is_ascii_whitespace()) =>
|
||||||
|
{
|
||||||
|
comment = true;
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
b'*' => {
|
||||||
|
let previous = index.checked_sub(1).and_then(|i| bytes.get(i)).copied();
|
||||||
|
let next = bytes.get(index + 1).copied();
|
||||||
|
if previous
|
||||||
|
.is_none_or(|byte| byte.is_ascii_whitespace() || b"[:,[{".contains(&byte))
|
||||||
|
&& next.is_some_and(is_alias_name_byte)
|
||||||
|
{
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
b'|' | b'>' if is_block_scalar_indicator(bytes, index) => {
|
||||||
|
block_scalar = Some(BlockScalar {
|
||||||
|
parent_indent: line_indent,
|
||||||
|
content_indent: block_scalar_indent(bytes, index, line_indent),
|
||||||
|
});
|
||||||
|
// The rest of a block-scalar header cannot contain YAML
|
||||||
|
// aliases; its content starts on the next line.
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
_ => index += 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
count
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_alias_name_byte(byte: u8) -> bool {
|
||||||
|
byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-'
|
||||||
|
}
|
||||||
|
|
||||||
|
fn leading_spaces(line: &str) -> usize {
|
||||||
|
line.bytes().take_while(|byte| *byte == b' ').count()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_blank_line(line: &str) -> bool {
|
||||||
|
line.bytes()
|
||||||
|
.all(|byte| byte == b' ' || byte == b'\t' || byte == b'\r')
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_block_scalar_indicator(bytes: &[u8], index: usize) -> bool {
|
||||||
|
let mut previous_position = index;
|
||||||
|
while previous_position > 0
|
||||||
|
&& bytes
|
||||||
|
.get(previous_position - 1)
|
||||||
|
.is_some_and(|byte| byte.is_ascii_whitespace())
|
||||||
|
{
|
||||||
|
previous_position -= 1;
|
||||||
|
}
|
||||||
|
let previous_is_value_boundary = previous_position == 0
|
||||||
|
|| bytes
|
||||||
|
.get(previous_position - 1)
|
||||||
|
.is_some_and(|byte| *byte == b':' || *byte == b'-');
|
||||||
|
let next = bytes.get(index + 1).copied();
|
||||||
|
let next_is_header_suffix = next.is_none_or(|byte| {
|
||||||
|
byte.is_ascii_whitespace()
|
||||||
|
|| byte == b'#'
|
||||||
|
|| byte == b'+'
|
||||||
|
|| byte == b'-'
|
||||||
|
|| byte.is_ascii_digit()
|
||||||
|
});
|
||||||
|
previous_is_value_boundary && next_is_header_suffix
|
||||||
|
}
|
||||||
|
|
||||||
|
fn block_scalar_indent(bytes: &[u8], index: usize, parent_indent: usize) -> Option<usize> {
|
||||||
|
let mut position = index + 1;
|
||||||
|
while let Some(byte) = bytes.get(position).copied() {
|
||||||
|
if byte == b'+' || byte == b'-' {
|
||||||
|
position += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if byte.is_ascii_digit() && byte != b'0' {
|
||||||
|
return Some(parent_indent + usize::from(byte - b'0'));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::rest::model::{
|
||||||
|
CoverageDisposition, CoverageEntry, NormalizedDiscriminator, NormalizedLiteral,
|
||||||
|
NormalizedScalarKind, NormalizedSchema, NormalizedSchemaConstraints, NormalizedSchemaKind,
|
||||||
|
SourceLocation, UnresolvedReference,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::normalize_coverage::escape_pointer;
|
||||||
|
|
||||||
|
pub(super) fn typed_schema(
|
||||||
|
value: &Value,
|
||||||
|
pointer: &str,
|
||||||
|
_id: &str,
|
||||||
|
coverage: &mut Vec<CoverageEntry>,
|
||||||
|
) -> NormalizedSchema {
|
||||||
|
// IDs are derived from RFC 6901 locations rather than user-controlled
|
||||||
|
// property names, so escaped keys cannot collide with one another.
|
||||||
|
let id = format!("schema:{}", escape_pointer(pointer));
|
||||||
|
let location = SourceLocation {
|
||||||
|
pointer: pointer.to_owned(),
|
||||||
|
};
|
||||||
|
coverage.push(CoverageEntry {
|
||||||
|
construct_id: id.clone(),
|
||||||
|
location: location.clone(),
|
||||||
|
disposition: CoverageDisposition::Mapped,
|
||||||
|
});
|
||||||
|
let kind = if let Some(reference) = value.get("$ref").and_then(Value::as_str) {
|
||||||
|
NormalizedSchemaKind::Reference {
|
||||||
|
reference: UnresolvedReference {
|
||||||
|
uri: reference.to_owned(),
|
||||||
|
location: SourceLocation {
|
||||||
|
pointer: format!("{pointer}/$ref"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} else if let Some((operator, variants)) =
|
||||||
|
["allOf", "oneOf", "anyOf"]
|
||||||
|
.into_iter()
|
||||||
|
.find_map(|operator| {
|
||||||
|
value
|
||||||
|
.get(operator)
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| (operator, items))
|
||||||
|
})
|
||||||
|
{
|
||||||
|
NormalizedSchemaKind::Composition {
|
||||||
|
operator: operator.to_owned(),
|
||||||
|
variants: variants
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, item)| {
|
||||||
|
typed_schema(
|
||||||
|
item,
|
||||||
|
&format!("{pointer}/{operator}/{index}"),
|
||||||
|
&format!(
|
||||||
|
"schema:{}",
|
||||||
|
escape_pointer(&format!("{pointer}/{operator}/{index}"))
|
||||||
|
),
|
||||||
|
coverage,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
}
|
||||||
|
} else if value.get("properties").is_some()
|
||||||
|
|| value.get("type").and_then(Value::as_str) == Some("object")
|
||||||
|
{
|
||||||
|
let required = value
|
||||||
|
.get("required")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.filter_map(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let properties = value
|
||||||
|
.get("properties")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.map(|properties| {
|
||||||
|
properties
|
||||||
|
.iter()
|
||||||
|
.map(|(name, schema)| {
|
||||||
|
(
|
||||||
|
name.clone(),
|
||||||
|
typed_schema(
|
||||||
|
schema,
|
||||||
|
&format!("{pointer}/properties/{}", escape_pointer(name)),
|
||||||
|
&format!(
|
||||||
|
"schema:{}",
|
||||||
|
escape_pointer(&format!(
|
||||||
|
"{pointer}/properties/{}",
|
||||||
|
escape_pointer(name)
|
||||||
|
))
|
||||||
|
),
|
||||||
|
coverage,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
NormalizedSchemaKind::Object {
|
||||||
|
properties,
|
||||||
|
required,
|
||||||
|
}
|
||||||
|
} else if value.get("type").and_then(Value::as_str) == Some("array") {
|
||||||
|
NormalizedSchemaKind::Array {
|
||||||
|
items: value.get("items").map(|items| {
|
||||||
|
Box::new(typed_schema(
|
||||||
|
items,
|
||||||
|
&format!("{pointer}/items"),
|
||||||
|
&format!("schema:{}", escape_pointer(&format!("{pointer}/items"))),
|
||||||
|
coverage,
|
||||||
|
))
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
} else if let Some(raw_type) = value.get("type").and_then(Value::as_str) {
|
||||||
|
NormalizedSchemaKind::Scalar {
|
||||||
|
scalar_type: match raw_type {
|
||||||
|
"integer" => NormalizedScalarKind::Integer,
|
||||||
|
"number" => NormalizedScalarKind::Number,
|
||||||
|
"boolean" => NormalizedScalarKind::Boolean,
|
||||||
|
"null" => NormalizedScalarKind::Null,
|
||||||
|
_ => NormalizedScalarKind::String,
|
||||||
|
},
|
||||||
|
format: value
|
||||||
|
.get("format")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
nullable: value
|
||||||
|
.get("nullable")
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
.unwrap_or(false),
|
||||||
|
default_value: value.get("default").and_then(normalized_literal),
|
||||||
|
enum_values: value
|
||||||
|
.get("enum")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| items.iter().filter_map(normalized_literal).collect())
|
||||||
|
.unwrap_or_default(),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
NormalizedSchemaKind::Unknown
|
||||||
|
};
|
||||||
|
NormalizedSchema {
|
||||||
|
construct_id: id,
|
||||||
|
location,
|
||||||
|
description: value
|
||||||
|
.get("description")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
discriminator: value.get("discriminator").and_then(|value| {
|
||||||
|
let property_name = value.get("propertyName")?.as_str()?.to_owned();
|
||||||
|
let mapping = value
|
||||||
|
.get("mapping")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.map(|mapping| {
|
||||||
|
mapping
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(key, value)| {
|
||||||
|
value.as_str().map(|value| (key.clone(), value.to_owned()))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
Some(NormalizedDiscriminator {
|
||||||
|
property_name,
|
||||||
|
mapping,
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
constraints: NormalizedSchemaConstraints {
|
||||||
|
minimum: value.get("minimum").and_then(Value::as_f64),
|
||||||
|
maximum: value.get("maximum").and_then(Value::as_f64),
|
||||||
|
min_length: value.get("minLength").and_then(Value::as_u64),
|
||||||
|
max_length: value.get("maxLength").and_then(Value::as_u64),
|
||||||
|
pattern: value
|
||||||
|
.get("pattern")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
},
|
||||||
|
kind,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn has_reference_schema(schema: Option<&NormalizedSchema>) -> bool {
|
||||||
|
match schema.map(|schema| &schema.kind) {
|
||||||
|
Some(NormalizedSchemaKind::Reference { .. }) => true,
|
||||||
|
Some(NormalizedSchemaKind::Object { properties, .. }) => properties
|
||||||
|
.values()
|
||||||
|
.any(|schema| has_reference_schema(Some(schema))),
|
||||||
|
Some(NormalizedSchemaKind::Array { items }) => has_reference_schema(items.as_deref()),
|
||||||
|
Some(NormalizedSchemaKind::Composition { variants, .. }) => variants
|
||||||
|
.iter()
|
||||||
|
.any(|schema| has_reference_schema(Some(schema))),
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_literal(value: &Value) -> Option<NormalizedLiteral> {
|
||||||
|
match value {
|
||||||
|
Value::String(value) => Some(NormalizedLiteral::String(value.clone())),
|
||||||
|
Value::Bool(value) => Some(NormalizedLiteral::Boolean(*value)),
|
||||||
|
Value::Null => Some(NormalizedLiteral::Null),
|
||||||
|
Value::Number(value) => value
|
||||||
|
.as_i64()
|
||||||
|
.map(NormalizedLiteral::Integer)
|
||||||
|
.or_else(|| value.as_u64().map(NormalizedLiteral::Unsigned))
|
||||||
|
.or_else(|| value.as_f64().map(NormalizedLiteral::Number)),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,14 +3,18 @@ use serde_json::Value;
|
|||||||
|
|
||||||
use crate::rest::{
|
use crate::rest::{
|
||||||
model::{
|
model::{
|
||||||
ImportFinding, RestImportDocument, RestImportOperation, RestImportParameter,
|
ImportFinding, ImportFindingSeverity, RestImportDocument, RestImportOperation,
|
||||||
RestParameterLocation,
|
RestImportParameter, RestParameterLocation,
|
||||||
},
|
},
|
||||||
normalize::{ImportParseError, resolve_local_ref},
|
normalize::ImportParseError,
|
||||||
recommendations::document_finding,
|
recommendations::{document_blocker, document_finding},
|
||||||
};
|
};
|
||||||
|
|
||||||
pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
||||||
|
parse_document_v2(root)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
||||||
let version = root
|
let version = root
|
||||||
.get("openapi")
|
.get("openapi")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
@@ -20,17 +24,204 @@ pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseErr
|
|||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.unwrap_or("Imported API")
|
.unwrap_or("Imported API")
|
||||||
.to_owned();
|
.to_owned();
|
||||||
let servers = root
|
let mut findings = Vec::new();
|
||||||
.get("servers")
|
let mut internal_finding_locations = Vec::new();
|
||||||
.and_then(Value::as_array)
|
let servers = parse_servers(root.get("servers"), "/servers");
|
||||||
.map(|items| {
|
append_findings_with_locations(
|
||||||
items
|
&mut findings,
|
||||||
.iter()
|
&mut internal_finding_locations,
|
||||||
.filter_map(|item| item.get("url").and_then(Value::as_str))
|
servers.findings,
|
||||||
.map(|url| url.trim_end_matches('/').to_owned())
|
);
|
||||||
.collect::<Vec<_>>()
|
let servers = servers.servers;
|
||||||
})
|
if servers.is_empty() {
|
||||||
.unwrap_or_default();
|
findings.push(document_finding(
|
||||||
|
"missing_servers",
|
||||||
|
"В документе не указаны servers, base URL нужно будет выбрать вручную.",
|
||||||
|
));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: String::new(),
|
||||||
|
}));
|
||||||
|
} else if servers.len() > 1 {
|
||||||
|
findings.push(document_finding(
|
||||||
|
"multiple_servers",
|
||||||
|
"В документе несколько servers, при импорте нужно выбрать нужный base URL.",
|
||||||
|
));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: "/servers".to_owned(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut operations = Vec::new();
|
||||||
|
let paths = root
|
||||||
|
.get("paths")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.ok_or(ImportParseError::UnsupportedDocument)?;
|
||||||
|
|
||||||
|
for (path, path_item) in paths {
|
||||||
|
if !path_item.is_object() {
|
||||||
|
findings.push(path_item_blocker(path));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("/paths/{}", path.replace('~', "~0").replace('/', "~1")),
|
||||||
|
}));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for method_name in ["head", "options", "trace", "connect"] {
|
||||||
|
if path_item.get(method_name).is_some() {
|
||||||
|
findings.push(document_blocker(
|
||||||
|
"unsupported_http_method",
|
||||||
|
format!("Метод {method_name} для пути {path} пока не поддерживается."),
|
||||||
|
));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"/paths/{}/{}",
|
||||||
|
path.replace('~', "~0").replace('/', "~1"),
|
||||||
|
method_name
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let path_pointer = format!("/paths/{}", path.replace('~', "~0").replace('/', "~1"));
|
||||||
|
let path_parameters = parameters(
|
||||||
|
path_item.get("parameters"),
|
||||||
|
&format!("{path_pointer}/parameters"),
|
||||||
|
);
|
||||||
|
append_findings_with_locations(
|
||||||
|
&mut findings,
|
||||||
|
&mut internal_finding_locations,
|
||||||
|
path_parameters.findings,
|
||||||
|
);
|
||||||
|
let path_parameters = path_parameters.parameters;
|
||||||
|
let path_servers =
|
||||||
|
parse_servers(path_item.get("servers"), &format!("{path_pointer}/servers"));
|
||||||
|
append_findings_with_locations(
|
||||||
|
&mut findings,
|
||||||
|
&mut internal_finding_locations,
|
||||||
|
path_servers.findings,
|
||||||
|
);
|
||||||
|
let path_servers = path_servers.servers;
|
||||||
|
for method_name in ["get", "post", "put", "patch", "delete"] {
|
||||||
|
let Some(operation_value) = path_item.get(method_name) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !operation_value.is_object() {
|
||||||
|
findings.push(document_blocker(
|
||||||
|
"invalid_operation",
|
||||||
|
"Операция имеет неверную структуру и не была интерпретирована.",
|
||||||
|
));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"/paths/{}/{}",
|
||||||
|
path.replace('~', "~0").replace('/', "~1"),
|
||||||
|
method_name
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(method) = method_from_lower(method_name) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let operation_pointer = format!("{path_pointer}/{method_name}");
|
||||||
|
|
||||||
|
let mut operation_parameters = path_parameters.clone();
|
||||||
|
let parsed_parameters = parameters(
|
||||||
|
operation_value.get("parameters"),
|
||||||
|
&format!("{path_pointer}/{method_name}/parameters"),
|
||||||
|
);
|
||||||
|
append_findings_with_locations(
|
||||||
|
&mut findings,
|
||||||
|
&mut internal_finding_locations,
|
||||||
|
parsed_parameters.findings,
|
||||||
|
);
|
||||||
|
operation_parameters.extend(parsed_parameters.parameters);
|
||||||
|
deduplicate_parameters(&mut operation_parameters);
|
||||||
|
let operation_servers = parse_servers(
|
||||||
|
operation_value.get("servers"),
|
||||||
|
&format!("{operation_pointer}/servers"),
|
||||||
|
);
|
||||||
|
append_findings_with_locations(
|
||||||
|
&mut findings,
|
||||||
|
&mut internal_finding_locations,
|
||||||
|
operation_servers.findings,
|
||||||
|
);
|
||||||
|
let operation_servers = operation_servers.servers;
|
||||||
|
let tags = tags(
|
||||||
|
operation_value.get("tags"),
|
||||||
|
&format!("{operation_pointer}/tags"),
|
||||||
|
);
|
||||||
|
append_findings_with_locations(
|
||||||
|
&mut findings,
|
||||||
|
&mut internal_finding_locations,
|
||||||
|
tags.findings,
|
||||||
|
);
|
||||||
|
|
||||||
|
operations.push(RestImportOperation {
|
||||||
|
key: format!("{} {}", method_name.to_uppercase(), path),
|
||||||
|
method,
|
||||||
|
path: path.clone(),
|
||||||
|
operation_id: operation_value
|
||||||
|
.get("operationId")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
summary: operation_value
|
||||||
|
.get("summary")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
description: operation_value
|
||||||
|
.get("description")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
tags: tags.tags,
|
||||||
|
parameters: operation_parameters,
|
||||||
|
request_body_schema: request_body_schema(operation_value, &operation_pointer)
|
||||||
|
.map(|(schema, _)| schema),
|
||||||
|
request_body_schema_location: request_body_schema(
|
||||||
|
operation_value,
|
||||||
|
&operation_pointer,
|
||||||
|
)
|
||||||
|
.map(|(_, location)| location),
|
||||||
|
response_schema: response_schema(operation_value, &operation_pointer)
|
||||||
|
.map(|(schema, _)| schema),
|
||||||
|
response_schema_location: response_schema(operation_value, &operation_pointer)
|
||||||
|
.map(|(_, location)| location),
|
||||||
|
servers: if operation_servers.is_empty() {
|
||||||
|
path_servers.clone()
|
||||||
|
} else {
|
||||||
|
operation_servers
|
||||||
|
},
|
||||||
|
findings: operation_findings(operation_value),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(RestImportDocument {
|
||||||
|
format: "openapi".to_owned(),
|
||||||
|
version,
|
||||||
|
title,
|
||||||
|
servers,
|
||||||
|
operations,
|
||||||
|
findings,
|
||||||
|
internal_finding_locations,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn deduplicate_parameters(parameters: &mut Vec<RestImportParameter>) {
|
||||||
|
let mut seen = std::collections::BTreeSet::new();
|
||||||
|
parameters.reverse();
|
||||||
|
parameters.retain(|parameter| seen.insert((parameter.name.clone(), parameter.location)));
|
||||||
|
parameters.reverse();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse_document_legacy_v1(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
||||||
|
let version = root
|
||||||
|
.get("openapi")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned);
|
||||||
|
let title = root
|
||||||
|
.pointer("/info/title")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("Imported API")
|
||||||
|
.to_owned();
|
||||||
|
let servers = legacy_servers(root.get("servers"));
|
||||||
let mut findings = Vec::new();
|
let mut findings = Vec::new();
|
||||||
if servers.is_empty() {
|
if servers.is_empty() {
|
||||||
findings.push(document_finding(
|
findings.push(document_finding(
|
||||||
@@ -49,67 +240,54 @@ pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseErr
|
|||||||
.get("paths")
|
.get("paths")
|
||||||
.and_then(Value::as_object)
|
.and_then(Value::as_object)
|
||||||
.ok_or(ImportParseError::UnsupportedDocument)?;
|
.ok_or(ImportParseError::UnsupportedDocument)?;
|
||||||
|
|
||||||
for (path, path_item) in paths {
|
for (path, path_item) in paths {
|
||||||
let path_parameters = parameters(root, path_item.get("parameters"));
|
let path_pointer = format!("/paths/{}", path.replace('~', "~0").replace('/', "~1"));
|
||||||
|
let path_parameters = legacy_parameters(
|
||||||
|
root,
|
||||||
|
path_item.get("parameters"),
|
||||||
|
&format!("{path_pointer}/parameters"),
|
||||||
|
);
|
||||||
for method_name in ["get", "post", "put", "patch", "delete"] {
|
for method_name in ["get", "post", "put", "patch", "delete"] {
|
||||||
let Some(operation_value) = path_item.get(method_name) else {
|
let Some(operation) = path_item.get(method_name) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let Some(method) = method_from_lower(method_name) else {
|
let Some(method) = method_from_lower(method_name) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
let operation_pointer = format!("{path_pointer}/{method_name}");
|
||||||
let mut operation_parameters = path_parameters.clone();
|
let mut parameters = path_parameters.clone();
|
||||||
operation_parameters.extend(parameters(root, operation_value.get("parameters")));
|
parameters.extend(legacy_parameters(
|
||||||
let operation_servers = operation_value
|
root,
|
||||||
.get("servers")
|
operation.get("parameters"),
|
||||||
.and_then(Value::as_array)
|
&format!("{operation_pointer}/parameters"),
|
||||||
.map(|items| {
|
));
|
||||||
items
|
|
||||||
.iter()
|
|
||||||
.filter_map(|item| item.get("url").and_then(Value::as_str))
|
|
||||||
.map(|url| url.trim_end_matches('/').to_owned())
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
})
|
|
||||||
.unwrap_or_default();
|
|
||||||
|
|
||||||
operations.push(RestImportOperation {
|
operations.push(RestImportOperation {
|
||||||
key: format!("{} {}", method_name.to_uppercase(), path),
|
key: format!("{} {}", method_name.to_uppercase(), path),
|
||||||
method,
|
method,
|
||||||
path: path.clone(),
|
path: path.clone(),
|
||||||
operation_id: operation_value
|
operation_id: operation
|
||||||
.get("operationId")
|
.get("operationId")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(ToOwned::to_owned),
|
.map(ToOwned::to_owned),
|
||||||
summary: operation_value
|
summary: operation
|
||||||
.get("summary")
|
.get("summary")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(ToOwned::to_owned),
|
.map(ToOwned::to_owned),
|
||||||
description: operation_value
|
description: operation
|
||||||
.get("description")
|
.get("description")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(ToOwned::to_owned),
|
.map(ToOwned::to_owned),
|
||||||
tags: operation_value
|
tags: legacy_tags(operation.get("tags")),
|
||||||
.get("tags")
|
parameters,
|
||||||
.and_then(Value::as_array)
|
request_body_schema: legacy_request_body_schema(root, operation),
|
||||||
.map(|items| {
|
request_body_schema_location: None,
|
||||||
items
|
response_schema: legacy_response_schema(root, operation),
|
||||||
.iter()
|
response_schema_location: None,
|
||||||
.filter_map(Value::as_str)
|
servers: legacy_servers(operation.get("servers")),
|
||||||
.map(ToOwned::to_owned)
|
findings: legacy_operation_findings(operation),
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.unwrap_or_default(),
|
|
||||||
parameters: operation_parameters,
|
|
||||||
request_body_schema: request_body_schema(root, operation_value),
|
|
||||||
response_schema: response_schema(root, operation_value),
|
|
||||||
servers: operation_servers,
|
|
||||||
findings: operation_findings(operation_value),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(RestImportDocument {
|
Ok(RestImportDocument {
|
||||||
format: "openapi".to_owned(),
|
format: "openapi".to_owned(),
|
||||||
version,
|
version,
|
||||||
@@ -117,17 +295,49 @@ pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseErr
|
|||||||
servers,
|
servers,
|
||||||
operations,
|
operations,
|
||||||
findings,
|
findings,
|
||||||
|
internal_finding_locations: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parameters(root: &Value, value: Option<&Value>) -> Vec<RestImportParameter> {
|
fn legacy_servers(value: Option<&Value>) -> Vec<String> {
|
||||||
value
|
value
|
||||||
.and_then(Value::as_array)
|
.and_then(Value::as_array)
|
||||||
.map(|items| {
|
.map(|items| {
|
||||||
items
|
items
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|item| {
|
.filter_map(|item| item.get("url").and_then(Value::as_str))
|
||||||
let item = resolve_local_ref(root, item, 0);
|
.map(|url| url.trim_end_matches('/').to_owned())
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn legacy_tags(value: Option<&Value>) -> Vec<String> {
|
||||||
|
value
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.filter_map(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn legacy_parameters(
|
||||||
|
root: &Value,
|
||||||
|
value: Option<&Value>,
|
||||||
|
base_pointer: &str,
|
||||||
|
) -> Vec<RestImportParameter> {
|
||||||
|
value
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, item)| {
|
||||||
|
let item = crate::rest::normalize::resolve_local_ref(root, item, 0);
|
||||||
let location = match item.get("in").and_then(Value::as_str)? {
|
let location = match item.get("in").and_then(Value::as_str)? {
|
||||||
"path" => RestParameterLocation::Path,
|
"path" => RestParameterLocation::Path,
|
||||||
"query" => RestParameterLocation::Query,
|
"query" => RestParameterLocation::Query,
|
||||||
@@ -146,9 +356,13 @@ fn parameters(root: &Value, value: Option<&Value>) -> Vec<RestImportParameter> {
|
|||||||
.get("description")
|
.get("description")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(ToOwned::to_owned),
|
.map(ToOwned::to_owned),
|
||||||
schema: item
|
schema: item.get("schema").map(|schema| {
|
||||||
.get("schema")
|
crate::rest::normalize::resolve_local_ref(root, schema, 0)
|
||||||
.map(|schema| resolve_local_ref(root, schema, 0)),
|
}),
|
||||||
|
source_location: crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("{base_pointer}/{index}"),
|
||||||
|
},
|
||||||
|
schema_source_location: None,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
@@ -156,31 +370,31 @@ fn parameters(root: &Value, value: Option<&Value>) -> Vec<RestImportParameter> {
|
|||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn request_body_schema(root: &Value, operation: &Value) -> Option<Value> {
|
fn legacy_request_body_schema(root: &Value, operation: &Value) -> Option<Value> {
|
||||||
let body = resolve_local_ref(root, operation.get("requestBody")?, 0);
|
let body = crate::rest::normalize::resolve_local_ref(root, operation.get("requestBody")?, 0);
|
||||||
let content = body.get("content")?.as_object()?;
|
let content = body.get("content")?.as_object()?;
|
||||||
for content_type in ["application/json", "application/*+json"] {
|
for content_type in ["application/json", "application/*+json"] {
|
||||||
if let Some(schema) = content
|
if let Some(schema) = content
|
||||||
.get(content_type)
|
.get(content_type)
|
||||||
.and_then(|media| media.get("schema"))
|
.and_then(|media| media.get("schema"))
|
||||||
{
|
{
|
||||||
return Some(resolve_local_ref(root, schema, 0));
|
return Some(crate::rest::normalize::resolve_local_ref(root, schema, 0));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
content
|
content
|
||||||
.iter()
|
.iter()
|
||||||
.find(|(content_type, _)| content_type.contains("json"))
|
.find(|(content_type, _)| content_type.contains("json"))
|
||||||
.and_then(|(_, media)| media.get("schema"))
|
.and_then(|(_, media)| media.get("schema"))
|
||||||
.map(|schema| resolve_local_ref(root, schema, 0))
|
.map(|schema| crate::rest::normalize::resolve_local_ref(root, schema, 0))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn response_schema(root: &Value, operation: &Value) -> Option<Value> {
|
fn legacy_response_schema(root: &Value, operation: &Value) -> Option<Value> {
|
||||||
let responses = operation.get("responses")?.as_object()?;
|
let responses = operation.get("responses")?.as_object()?;
|
||||||
for code in ["200", "201", "202", "default"] {
|
for code in ["200", "201", "202", "default"] {
|
||||||
let Some(response) = responses.get(code) else {
|
let Some(response) = responses.get(code) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let response = resolve_local_ref(root, response, 0);
|
let response = crate::rest::normalize::resolve_local_ref(root, response, 0);
|
||||||
let Some(content) = response.get("content").and_then(Value::as_object) else {
|
let Some(content) = response.get("content").and_then(Value::as_object) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
@@ -189,7 +403,7 @@ fn response_schema(root: &Value, operation: &Value) -> Option<Value> {
|
|||||||
.get(content_type)
|
.get(content_type)
|
||||||
.and_then(|media| media.get("schema"))
|
.and_then(|media| media.get("schema"))
|
||||||
{
|
{
|
||||||
return Some(resolve_local_ref(root, schema, 0));
|
return Some(crate::rest::normalize::resolve_local_ref(root, schema, 0));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(schema) = content
|
if let Some(schema) = content
|
||||||
@@ -197,17 +411,350 @@ fn response_schema(root: &Value, operation: &Value) -> Option<Value> {
|
|||||||
.find(|(content_type, _)| content_type.contains("json"))
|
.find(|(content_type, _)| content_type.contains("json"))
|
||||||
.and_then(|(_, media)| media.get("schema"))
|
.and_then(|(_, media)| media.get("schema"))
|
||||||
{
|
{
|
||||||
return Some(resolve_local_ref(root, schema, 0));
|
return Some(crate::rest::normalize::resolve_local_ref(root, schema, 0));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn legacy_operation_findings(operation: &Value) -> Vec<ImportFinding> {
|
||||||
|
if operation.get("requestBody").is_some()
|
||||||
|
&& legacy_request_body_schema(&Value::Null, operation).is_none()
|
||||||
|
{
|
||||||
|
vec![document_finding(
|
||||||
|
"unsupported_request_body",
|
||||||
|
"У метода есть requestBody, но JSON schema не найдена.",
|
||||||
|
)]
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ParsedParameters {
|
||||||
|
parameters: Vec<RestImportParameter>,
|
||||||
|
findings: Vec<(ImportFinding, crate::rest::model::SourceLocation)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ParsedServers {
|
||||||
|
servers: Vec<String>,
|
||||||
|
findings: Vec<(ImportFinding, crate::rest::model::SourceLocation)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_servers(value: Option<&Value>, base_pointer: &str) -> ParsedServers {
|
||||||
|
let mut findings = Vec::new();
|
||||||
|
let servers = value
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, item)| {
|
||||||
|
let pointer = format!("{base_pointer}/{index}");
|
||||||
|
let Some(object) = item.as_object() else {
|
||||||
|
findings.push(server_blocker(&pointer));
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
let Some(url) = object.get("url").and_then(Value::as_str) else {
|
||||||
|
findings.push(server_blocker(&pointer));
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
Some(url.trim_end_matches('/').to_owned())
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
ParsedServers { servers, findings }
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ParsedTags {
|
||||||
|
tags: Vec<String>,
|
||||||
|
findings: Vec<(ImportFinding, crate::rest::model::SourceLocation)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tags(value: Option<&Value>, base_pointer: &str) -> ParsedTags {
|
||||||
|
let mut findings = Vec::new();
|
||||||
|
let tags = value
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, item)| match item.as_str() {
|
||||||
|
Some(tag) => Some(tag.to_owned()),
|
||||||
|
None => {
|
||||||
|
findings.push((
|
||||||
|
document_finding(
|
||||||
|
"invalid_tag",
|
||||||
|
"Тег должен быть строкой и не был импортирован.",
|
||||||
|
),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("{base_pointer}/{index}"),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
ParsedTags { tags, findings }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn server_blocker(pointer: &str) -> (ImportFinding, crate::rest::model::SourceLocation) {
|
||||||
|
(
|
||||||
|
document_blocker(
|
||||||
|
"invalid_server",
|
||||||
|
"Server должен быть объектом со строковым url и не был импортирован.",
|
||||||
|
),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: pointer.to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parameters(value: Option<&Value>, base_pointer: &str) -> ParsedParameters {
|
||||||
|
let mut findings = Vec::new();
|
||||||
|
let parameters = value
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, item)| {
|
||||||
|
let pointer = format!("{base_pointer}/{index}");
|
||||||
|
let Some(object) = item.as_object() else {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"invalid_parameter",
|
||||||
|
"Параметр должен быть объектом и не был интерпретирован.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
if object.contains_key("$ref") {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"unresolved_parameter_reference",
|
||||||
|
"Параметр по $ref требует разрешения перед импортом.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if !object.get("name").is_some_and(Value::is_string) {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"missing_parameter_name",
|
||||||
|
"У параметра отсутствует строковое поле name.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if !object.get("in").is_some_and(Value::is_string) {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"missing_parameter_location",
|
||||||
|
"У параметра отсутствует строковое поле in.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let item = item.clone();
|
||||||
|
let location = match item.get("in").and_then(Value::as_str)? {
|
||||||
|
"path" => RestParameterLocation::Path,
|
||||||
|
"query" => RestParameterLocation::Query,
|
||||||
|
"header" => RestParameterLocation::Header,
|
||||||
|
"cookie" => {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"Cookie parameter пока не поддерживается и не был импортирован.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"Параметр использует неподдерживаемое значение in и не был импортирован.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Some(RestImportParameter {
|
||||||
|
name: item.get("name").and_then(Value::as_str)?.to_owned(),
|
||||||
|
location,
|
||||||
|
required: item
|
||||||
|
.get("required")
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
.unwrap_or(false)
|
||||||
|
|| location == RestParameterLocation::Path,
|
||||||
|
description: item
|
||||||
|
.get("description")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
schema: item.get("schema").cloned(),
|
||||||
|
schema_source_location: item.get("schema").map(|_| {
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("{base_pointer}/{index}/schema"),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
source_location: crate::rest::model::SourceLocation {
|
||||||
|
pointer,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
ParsedParameters {
|
||||||
|
parameters,
|
||||||
|
findings,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parameter_blocker(
|
||||||
|
code: &str,
|
||||||
|
message: &str,
|
||||||
|
pointer: &str,
|
||||||
|
) -> (ImportFinding, crate::rest::model::SourceLocation) {
|
||||||
|
(
|
||||||
|
document_blocker(code, message),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: pointer.to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn append_findings_with_locations(
|
||||||
|
findings: &mut Vec<ImportFinding>,
|
||||||
|
locations: &mut Vec<Option<crate::rest::model::SourceLocation>>,
|
||||||
|
parameter_findings: Vec<(ImportFinding, crate::rest::model::SourceLocation)>,
|
||||||
|
) {
|
||||||
|
for (finding, location) in parameter_findings {
|
||||||
|
findings.push(finding);
|
||||||
|
locations.push(Some(location));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_body_schema(
|
||||||
|
operation: &Value,
|
||||||
|
operation_pointer: &str,
|
||||||
|
) -> Option<(Value, crate::rest::model::SourceLocation)> {
|
||||||
|
let body = operation.get("requestBody")?;
|
||||||
|
let content = body.get("content")?.as_object()?;
|
||||||
|
for content_type in ["application/json", "application/*+json"] {
|
||||||
|
if let Some(schema) = content
|
||||||
|
.get(content_type)
|
||||||
|
.and_then(|media| media.get("schema"))
|
||||||
|
{
|
||||||
|
return Some((
|
||||||
|
schema.clone(),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"{operation_pointer}/requestBody/content/{}/schema",
|
||||||
|
content_type.replace('~', "~0").replace('/', "~1")
|
||||||
|
),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
content
|
||||||
|
.iter()
|
||||||
|
.find(|(content_type, _)| content_type.contains("json"))
|
||||||
|
.and_then(|(content_type, media)| {
|
||||||
|
media.get("schema").cloned().map(|schema| {
|
||||||
|
(
|
||||||
|
schema,
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"{operation_pointer}/requestBody/content/{}/schema",
|
||||||
|
content_type.replace('~', "~0").replace('/', "~1")
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn response_schema(
|
||||||
|
operation: &Value,
|
||||||
|
operation_pointer: &str,
|
||||||
|
) -> Option<(Value, crate::rest::model::SourceLocation)> {
|
||||||
|
let responses = operation.get("responses")?.as_object()?;
|
||||||
|
let mut numeric = responses
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(code, response)| {
|
||||||
|
(code.len() == 3)
|
||||||
|
.then(|| code.parse::<u16>().ok())
|
||||||
|
.flatten()
|
||||||
|
.filter(|status| (200..300).contains(status))
|
||||||
|
.map(|status| (status, code.as_str(), response))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
numeric.sort_by(|left, right| (left.0, left.1).cmp(&(right.0, right.1)));
|
||||||
|
for (_, code, response) in numeric {
|
||||||
|
if let Some(schema) = response_json_schema(response, operation_pointer, code) {
|
||||||
|
return Some(schema);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut wildcard = responses
|
||||||
|
.iter()
|
||||||
|
.filter(|(code, _)| code.eq_ignore_ascii_case("2xx"))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
wildcard.sort_by(|left, right| left.0.cmp(right.0));
|
||||||
|
for (code, response) in wildcard {
|
||||||
|
if let Some(schema) = response_json_schema(response, operation_pointer, code) {
|
||||||
|
return Some(schema);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
responses
|
||||||
|
.get("default")
|
||||||
|
.and_then(|response| response_json_schema(response, operation_pointer, "default"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn response_json_schema(
|
||||||
|
response: &Value,
|
||||||
|
operation_pointer: &str,
|
||||||
|
code: &str,
|
||||||
|
) -> Option<(Value, crate::rest::model::SourceLocation)> {
|
||||||
|
let content = response.get("content")?.as_object()?;
|
||||||
|
for content_type in ["application/json", "application/*+json"] {
|
||||||
|
if let Some(schema) = content
|
||||||
|
.get(content_type)
|
||||||
|
.and_then(|media| media.get("schema"))
|
||||||
|
{
|
||||||
|
return Some((
|
||||||
|
schema.clone(),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"{operation_pointer}/responses/{}/content/{}/schema",
|
||||||
|
code.replace('~', "~0").replace('/', "~1"),
|
||||||
|
content_type.replace('~', "~0").replace('/', "~1")
|
||||||
|
),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
content
|
||||||
|
.iter()
|
||||||
|
.find(|(content_type, _)| content_type.contains("json"))
|
||||||
|
.and_then(|(content_type, media)| {
|
||||||
|
media.get("schema").map(|schema| {
|
||||||
|
(
|
||||||
|
schema.clone(),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"{operation_pointer}/responses/{}/content/{}/schema",
|
||||||
|
code.replace('~', "~0").replace('/', "~1"),
|
||||||
|
content_type.replace('~', "~0").replace('/', "~1")
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn operation_findings(operation: &Value) -> Vec<ImportFinding> {
|
fn operation_findings(operation: &Value) -> Vec<ImportFinding> {
|
||||||
let mut findings = Vec::new();
|
let mut findings = Vec::new();
|
||||||
if operation.get("requestBody").is_some()
|
if operation.get("requestBody").is_some() && request_body_schema(operation, "").is_none() {
|
||||||
&& request_body_schema(&Value::Null, operation).is_none()
|
|
||||||
{
|
|
||||||
findings.push(document_finding(
|
findings.push(document_finding(
|
||||||
"unsupported_request_body",
|
"unsupported_request_body",
|
||||||
"У метода есть requestBody, но JSON schema не найдена.",
|
"У метода есть requestBody, но JSON schema не найдена.",
|
||||||
@@ -226,3 +773,12 @@ fn method_from_lower(value: &str) -> Option<HttpMethod> {
|
|||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn path_item_blocker(_path: &str) -> ImportFinding {
|
||||||
|
ImportFinding {
|
||||||
|
code: "invalid_path_item".to_owned(),
|
||||||
|
severity: ImportFindingSeverity::Error,
|
||||||
|
message: "Path Item имеет неверную структуру и не был интерпретирован.".to_owned(),
|
||||||
|
operation_key: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,6 +11,15 @@ pub fn document_finding(code: &str, message: impl Into<String>) -> ImportFinding
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn document_blocker(code: &str, message: impl Into<String>) -> ImportFinding {
|
||||||
|
ImportFinding {
|
||||||
|
code: code.to_owned(),
|
||||||
|
severity: ImportFindingSeverity::Error,
|
||||||
|
message: message.into(),
|
||||||
|
operation_key: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn operation_finding(
|
pub fn operation_finding(
|
||||||
operation_key: &str,
|
operation_key: &str,
|
||||||
code: &str,
|
code: &str,
|
||||||
@@ -45,6 +54,9 @@ pub fn operation_recommendations(operation: &RestImportOperation) -> Vec<ImportF
|
|||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.trim()
|
.trim()
|
||||||
.is_empty()
|
.is_empty()
|
||||||
|
&& !findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "missing_operation_id")
|
||||||
{
|
{
|
||||||
findings.push(operation_finding(
|
findings.push(operation_finding(
|
||||||
&operation.key,
|
&operation.key,
|
||||||
|
|||||||
@@ -0,0 +1,872 @@
|
|||||||
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
|
||||||
|
use serde_json::{Map, Value};
|
||||||
|
|
||||||
|
use crate::rest::model::{
|
||||||
|
ExternalDocumentSnapshot, ImportFindingSeverity, NormalizationConfig, NormalizedFinding,
|
||||||
|
ResolvedReferenceEdge, ResolvedReferenceGraph, ResolvedReferenceNode, SourceDigest,
|
||||||
|
SourceLocation,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::{ImportParseError, normalize_coverage::escape_pointer};
|
||||||
|
|
||||||
|
pub(super) struct ResolutionResult {
|
||||||
|
pub root: Value,
|
||||||
|
pub graph: ResolvedReferenceGraph,
|
||||||
|
pub findings: Vec<NormalizedFinding>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn reference_uris(
|
||||||
|
document: &str,
|
||||||
|
config: &NormalizationConfig,
|
||||||
|
) -> Result<Vec<String>, ImportParseError> {
|
||||||
|
reference_uris_with_max_bytes(document, config, config.max_bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn external_reference_uris(
|
||||||
|
document: &str,
|
||||||
|
config: &NormalizationConfig,
|
||||||
|
) -> Result<Vec<String>, ImportParseError> {
|
||||||
|
reference_uris_with_max_bytes(document, config, config.max_external_document_bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reference_uris_with_max_bytes(
|
||||||
|
document: &str,
|
||||||
|
config: &NormalizationConfig,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<Vec<String>, ImportParseError> {
|
||||||
|
fn collect(value: &Value, uris: &mut BTreeSet<String>) {
|
||||||
|
match value {
|
||||||
|
Value::Object(object) => {
|
||||||
|
if let Some(reference) = object.get("$ref").and_then(Value::as_str) {
|
||||||
|
uris.insert(reference.to_owned());
|
||||||
|
}
|
||||||
|
for (key, child) in object {
|
||||||
|
if is_literal_payload_key(key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
collect(child, uris);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Array(items) => {
|
||||||
|
for child in items {
|
||||||
|
collect(child, uris);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if document.len() > max_bytes
|
||||||
|
|| super::normalize_limits::alias_count(document) > config.max_aliases
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::LimitExceeded);
|
||||||
|
}
|
||||||
|
let root = decode_snapshot(document)?;
|
||||||
|
super::normalize::validate_value_limits(&root, config)?;
|
||||||
|
let mut uris = BTreeSet::new();
|
||||||
|
collect(&root, &mut uris);
|
||||||
|
Ok(uris.into_iter().collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Document {
|
||||||
|
uri: Option<String>,
|
||||||
|
digest: SourceDigest,
|
||||||
|
root: Value,
|
||||||
|
oas31: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||||
|
struct NodeKey {
|
||||||
|
digest: String,
|
||||||
|
pointer: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct TraversalLocation<'a> {
|
||||||
|
projection: &'a str,
|
||||||
|
origin: &'a str,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Resolver<'a> {
|
||||||
|
config: &'a NormalizationConfig,
|
||||||
|
documents: Vec<Document>,
|
||||||
|
by_uri: BTreeMap<String, usize>,
|
||||||
|
graph: ResolvedReferenceGraph,
|
||||||
|
findings: Vec<NormalizedFinding>,
|
||||||
|
references: usize,
|
||||||
|
expanded_nodes: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn resolve(
|
||||||
|
root: Value,
|
||||||
|
primary_digest: SourceDigest,
|
||||||
|
snapshots: &[ExternalDocumentSnapshot],
|
||||||
|
config: &NormalizationConfig,
|
||||||
|
) -> Result<ResolutionResult, ImportParseError> {
|
||||||
|
if snapshots.len() > config.max_reference_documents {
|
||||||
|
return Err(ImportParseError::LimitExceeded);
|
||||||
|
}
|
||||||
|
let mut documents = vec![Document {
|
||||||
|
uri: None,
|
||||||
|
digest: primary_digest,
|
||||||
|
oas31: is_oas31(&root),
|
||||||
|
root,
|
||||||
|
}];
|
||||||
|
let mut by_uri = BTreeMap::new();
|
||||||
|
let mut dependency_digests = BTreeSet::new();
|
||||||
|
let primary_oas31 = documents[0].oas31;
|
||||||
|
for snapshot in snapshots {
|
||||||
|
if snapshot.canonical_uri.is_empty()
|
||||||
|
|| by_uri
|
||||||
|
.insert(snapshot.canonical_uri.clone(), documents.len())
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
return Err(ImportParseError::InvalidDocument);
|
||||||
|
}
|
||||||
|
if super::normalize_limits::alias_count(&snapshot.document) > config.max_aliases {
|
||||||
|
return Err(ImportParseError::LimitExceeded);
|
||||||
|
}
|
||||||
|
let root = decode_snapshot(&snapshot.document)?;
|
||||||
|
super::normalize::validate_value_limits(&root, config)?;
|
||||||
|
dependency_digests.insert(snapshot.digest.clone());
|
||||||
|
documents.push(Document {
|
||||||
|
uri: Some(snapshot.canonical_uri.clone()),
|
||||||
|
digest: snapshot.digest.clone(),
|
||||||
|
// External documents are fragments of the primary contract. In a
|
||||||
|
// 3.1 bundle they therefore use 3.1 `$ref` sibling semantics even
|
||||||
|
// when the fragment itself omits an `openapi` declaration.
|
||||||
|
oas31: primary_oas31 || is_oas31(&root),
|
||||||
|
root,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let mut resolver = Resolver {
|
||||||
|
config,
|
||||||
|
documents,
|
||||||
|
by_uri,
|
||||||
|
graph: ResolvedReferenceGraph {
|
||||||
|
dependency_digests: dependency_digests.into_iter().collect(),
|
||||||
|
edges: Vec::new(),
|
||||||
|
},
|
||||||
|
findings: Vec::new(),
|
||||||
|
references: 0,
|
||||||
|
expanded_nodes: 0,
|
||||||
|
};
|
||||||
|
let root = resolver.documents[0].root.clone();
|
||||||
|
let root = resolver.expand_value(
|
||||||
|
0,
|
||||||
|
root,
|
||||||
|
TraversalLocation {
|
||||||
|
projection: "",
|
||||||
|
origin: "",
|
||||||
|
},
|
||||||
|
0,
|
||||||
|
&mut Vec::new(),
|
||||||
|
)?;
|
||||||
|
resolver
|
||||||
|
.graph
|
||||||
|
.edges
|
||||||
|
.sort_by(|left, right| edge_key(left).cmp(&edge_key(right)));
|
||||||
|
resolver.graph.edges.dedup();
|
||||||
|
resolver.findings.sort_by(|left, right| {
|
||||||
|
(
|
||||||
|
&left.operation_key,
|
||||||
|
&left.construct_id,
|
||||||
|
&left.code,
|
||||||
|
&left.location.pointer,
|
||||||
|
)
|
||||||
|
.cmp(&(
|
||||||
|
&right.operation_key,
|
||||||
|
&right.construct_id,
|
||||||
|
&right.code,
|
||||||
|
&right.location.pointer,
|
||||||
|
))
|
||||||
|
});
|
||||||
|
resolver.findings.dedup();
|
||||||
|
Ok(ResolutionResult {
|
||||||
|
root,
|
||||||
|
graph: resolver.graph,
|
||||||
|
findings: resolver.findings,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Resolver<'_> {
|
||||||
|
fn expand_value(
|
||||||
|
&mut self,
|
||||||
|
document_index: usize,
|
||||||
|
value: Value,
|
||||||
|
location: TraversalLocation<'_>,
|
||||||
|
depth: usize,
|
||||||
|
stack: &mut Vec<NodeKey>,
|
||||||
|
) -> Result<Value, ImportParseError> {
|
||||||
|
if depth > 0 {
|
||||||
|
self.expanded_nodes = self.expanded_nodes.saturating_add(1);
|
||||||
|
if self.expanded_nodes > self.config.max_expanded_nodes {
|
||||||
|
self.push_finding("reference_graph_limit", location.projection);
|
||||||
|
return Ok(Value::Object(Map::new()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match value {
|
||||||
|
Value::Object(mut object) => {
|
||||||
|
if let Some(reference) = object
|
||||||
|
.get("$ref")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::to_owned)
|
||||||
|
{
|
||||||
|
return self.expand_reference(
|
||||||
|
document_index,
|
||||||
|
object,
|
||||||
|
&reference,
|
||||||
|
location,
|
||||||
|
depth,
|
||||||
|
stack,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let keys = object.keys().cloned().collect::<Vec<_>>();
|
||||||
|
for key in keys {
|
||||||
|
if let Some(child) = object.remove(&key) {
|
||||||
|
if is_literal_payload_key(&key) {
|
||||||
|
object.insert(key, child);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let child_pointer =
|
||||||
|
format!("{}/{}", location.projection, escape_pointer(&key));
|
||||||
|
let child_origin_pointer =
|
||||||
|
format!("{}/{}", location.origin, escape_pointer(&key));
|
||||||
|
object.insert(
|
||||||
|
key,
|
||||||
|
self.expand_value(
|
||||||
|
document_index,
|
||||||
|
child,
|
||||||
|
TraversalLocation {
|
||||||
|
projection: &child_pointer,
|
||||||
|
origin: &child_origin_pointer,
|
||||||
|
},
|
||||||
|
depth,
|
||||||
|
stack,
|
||||||
|
)?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let composition_count = ["allOf", "oneOf", "anyOf"]
|
||||||
|
.into_iter()
|
||||||
|
.filter(|operator| object.contains_key(*operator))
|
||||||
|
.count();
|
||||||
|
if composition_count > 1 {
|
||||||
|
self.push_finding("unsupported_composition", location.projection);
|
||||||
|
}
|
||||||
|
if let Some(discriminator) = object.get("discriminator") {
|
||||||
|
let valid = discriminator
|
||||||
|
.get("propertyName")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.is_some_and(|value| !value.is_empty())
|
||||||
|
&& discriminator.get("mapping").is_none_or(|mapping| {
|
||||||
|
mapping
|
||||||
|
.as_object()
|
||||||
|
.is_some_and(|mapping| mapping.values().all(Value::is_string))
|
||||||
|
})
|
||||||
|
&& (object.contains_key("oneOf") || object.contains_key("anyOf"));
|
||||||
|
if !valid {
|
||||||
|
self.push_finding("unsupported_discriminator", location.projection);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.merge_all_of(object, location.projection)
|
||||||
|
}
|
||||||
|
Value::Array(items) => Ok(Value::Array(
|
||||||
|
items
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, child)| {
|
||||||
|
self.expand_value(
|
||||||
|
document_index,
|
||||||
|
child,
|
||||||
|
TraversalLocation {
|
||||||
|
projection: &format!("{}/{index}", location.projection),
|
||||||
|
origin: &format!("{}/{index}", location.origin),
|
||||||
|
},
|
||||||
|
depth,
|
||||||
|
stack,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Result<Vec<_>, _>>()?,
|
||||||
|
)),
|
||||||
|
other => Ok(other),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expand_reference(
|
||||||
|
&mut self,
|
||||||
|
document_index: usize,
|
||||||
|
mut source_object: Map<String, Value>,
|
||||||
|
reference: &str,
|
||||||
|
location: TraversalLocation<'_>,
|
||||||
|
depth: usize,
|
||||||
|
stack: &mut Vec<NodeKey>,
|
||||||
|
) -> Result<Value, ImportParseError> {
|
||||||
|
self.references = self.references.saturating_add(1);
|
||||||
|
if self.references > self.config.max_references || depth >= self.config.max_reference_depth
|
||||||
|
{
|
||||||
|
self.push_finding(
|
||||||
|
"reference_graph_limit",
|
||||||
|
&format!("{}/$ref", location.projection),
|
||||||
|
);
|
||||||
|
return Ok(Value::Object(source_object));
|
||||||
|
}
|
||||||
|
let target = self.target(document_index, reference);
|
||||||
|
if matches!(target, Err(TargetError::MalformedFragment)) {
|
||||||
|
self.push_finding(
|
||||||
|
"reference_uri_malformed",
|
||||||
|
&format!("{}/$ref", location.projection),
|
||||||
|
);
|
||||||
|
return Ok(Value::Object(source_object));
|
||||||
|
}
|
||||||
|
let Some((target_document, target_pointer)) = target.ok().flatten() else {
|
||||||
|
let external = reference.starts_with("http://") || reference.starts_with("https://");
|
||||||
|
let code = if external {
|
||||||
|
if self.config.external_references_enabled {
|
||||||
|
"external_reference_unavailable"
|
||||||
|
} else {
|
||||||
|
"external_reference_disabled"
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
"reference_target_missing"
|
||||||
|
};
|
||||||
|
self.push_finding(code, &format!("{}/$ref", location.projection));
|
||||||
|
return Ok(Value::Object(source_object));
|
||||||
|
};
|
||||||
|
let source = ResolvedReferenceNode {
|
||||||
|
snapshot_digest: self.documents[document_index].digest.clone(),
|
||||||
|
location: SourceLocation {
|
||||||
|
pointer: format!("{}/$ref", location.origin),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let target = ResolvedReferenceNode {
|
||||||
|
snapshot_digest: self.documents[target_document].digest.clone(),
|
||||||
|
location: SourceLocation {
|
||||||
|
pointer: target_pointer.clone(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let key = NodeKey {
|
||||||
|
digest: target.snapshot_digest.as_str().to_owned(),
|
||||||
|
pointer: target_pointer.clone(),
|
||||||
|
};
|
||||||
|
let recursive = stack.contains(&key);
|
||||||
|
self.graph.edges.push(ResolvedReferenceEdge {
|
||||||
|
source,
|
||||||
|
target,
|
||||||
|
recursive,
|
||||||
|
});
|
||||||
|
if recursive {
|
||||||
|
// The edge is the lossless representation. Expansion stops here,
|
||||||
|
// producing an opaque object for the finite preview projection.
|
||||||
|
return Ok(Value::Object(Map::new()));
|
||||||
|
}
|
||||||
|
let Some(target_value) = self.documents[target_document]
|
||||||
|
.root
|
||||||
|
.pointer(&target_pointer)
|
||||||
|
.cloned()
|
||||||
|
else {
|
||||||
|
self.push_finding(
|
||||||
|
"reference_target_missing",
|
||||||
|
&format!("{}/$ref", location.projection),
|
||||||
|
);
|
||||||
|
return Ok(Value::Object(source_object));
|
||||||
|
};
|
||||||
|
if !target_value.is_object() {
|
||||||
|
self.push_finding(
|
||||||
|
"reference_type_mismatch",
|
||||||
|
&format!("{}/$ref", location.projection),
|
||||||
|
);
|
||||||
|
return Ok(Value::Object(source_object));
|
||||||
|
}
|
||||||
|
stack.push(key);
|
||||||
|
let mut expanded = self.expand_value(
|
||||||
|
target_document,
|
||||||
|
target_value,
|
||||||
|
TraversalLocation {
|
||||||
|
projection: location.projection,
|
||||||
|
origin: &target_pointer,
|
||||||
|
},
|
||||||
|
depth + 1,
|
||||||
|
stack,
|
||||||
|
)?;
|
||||||
|
stack.pop();
|
||||||
|
|
||||||
|
// OAS 3.1 Schema Objects permit siblings next to `$ref`; OAS 3.0 and
|
||||||
|
// Swagger Reference Objects ignore them. The source document version
|
||||||
|
// controls the semantics at the reference site.
|
||||||
|
source_object.remove("$ref");
|
||||||
|
if self.documents[document_index].oas31 && !source_object.is_empty() {
|
||||||
|
let Value::Object(expanded_object) = &mut expanded else {
|
||||||
|
self.push_finding(
|
||||||
|
"reference_type_mismatch",
|
||||||
|
&format!("{}/$ref", location.projection),
|
||||||
|
);
|
||||||
|
return Ok(Value::Object(source_object));
|
||||||
|
};
|
||||||
|
for (key, sibling) in source_object {
|
||||||
|
let child_pointer = format!("{}/{}", location.projection, escape_pointer(&key));
|
||||||
|
let child_origin_pointer = format!("{}/{}", location.origin, escape_pointer(&key));
|
||||||
|
expanded_object.insert(
|
||||||
|
key,
|
||||||
|
self.expand_value(
|
||||||
|
document_index,
|
||||||
|
sibling,
|
||||||
|
TraversalLocation {
|
||||||
|
projection: &child_pointer,
|
||||||
|
origin: &child_origin_pointer,
|
||||||
|
},
|
||||||
|
depth,
|
||||||
|
stack,
|
||||||
|
)?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(expanded)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn target(
|
||||||
|
&self,
|
||||||
|
current: usize,
|
||||||
|
reference: &str,
|
||||||
|
) -> Result<Option<(usize, String)>, TargetError> {
|
||||||
|
let (document, fragment) = reference.split_once('#').unwrap_or((reference, ""));
|
||||||
|
let fragment = percent_decode(fragment)?;
|
||||||
|
let pointer = if fragment.is_empty() {
|
||||||
|
String::new()
|
||||||
|
} else if fragment.starts_with('/') {
|
||||||
|
fragment
|
||||||
|
} else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
if document.is_empty() {
|
||||||
|
return Ok(Some((current, pointer)));
|
||||||
|
}
|
||||||
|
let canonical = if has_uri_scheme(document) {
|
||||||
|
canonical_absolute_uri(document)
|
||||||
|
} else {
|
||||||
|
let Some(base) = self.documents[current].uri.as_deref() else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
join_relative(base, document)
|
||||||
|
};
|
||||||
|
Ok(canonical.and_then(|canonical| {
|
||||||
|
self.by_uri
|
||||||
|
.get(&canonical)
|
||||||
|
.copied()
|
||||||
|
.map(|index| (index, pointer))
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn merge_all_of(
|
||||||
|
&mut self,
|
||||||
|
mut object: Map<String, Value>,
|
||||||
|
pointer: &str,
|
||||||
|
) -> Result<Value, ImportParseError> {
|
||||||
|
let branches = match object.remove("allOf") {
|
||||||
|
None => return Ok(Value::Object(object)),
|
||||||
|
Some(Value::Array(branches)) => branches,
|
||||||
|
Some(value) => {
|
||||||
|
object.insert("allOf".to_owned(), value);
|
||||||
|
self.push_finding("unsupported_composition", &format!("{pointer}/allOf"));
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let original = branches.clone();
|
||||||
|
for branch in branches {
|
||||||
|
let Some(fields) = branch.as_object() else {
|
||||||
|
object.insert("allOf".to_owned(), Value::Array(original));
|
||||||
|
self.push_finding("all_of_conflict", &format!("{pointer}/allOf"));
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
};
|
||||||
|
if !merge_object(&mut object, fields) {
|
||||||
|
object.insert("allOf".to_owned(), Value::Array(original));
|
||||||
|
self.push_finding("all_of_conflict", &format!("{pointer}/allOf"));
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Value::Object(object))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_finding(&mut self, code: &str, pointer: &str) {
|
||||||
|
self.findings.push(NormalizedFinding {
|
||||||
|
code: code.to_owned(),
|
||||||
|
severity: ImportFindingSeverity::Error,
|
||||||
|
message: match code {
|
||||||
|
"external_reference_disabled" => "Внешняя ссылка отключена политикой импорта.",
|
||||||
|
"reference_target_missing" => {
|
||||||
|
"Цель ссылки отсутствует или имеет неверный JSON Pointer."
|
||||||
|
}
|
||||||
|
"external_reference_unavailable" => {
|
||||||
|
"Внешний snapshot недоступен или отклонён политикой импорта."
|
||||||
|
}
|
||||||
|
"reference_type_mismatch" => "Цель ссылки имеет неподдерживаемый тип.",
|
||||||
|
"reference_graph_limit" => "Граф ссылок превышает установленный предел.",
|
||||||
|
"all_of_conflict" => "Ветки allOf содержат несовместимые определения.",
|
||||||
|
"reference_uri_malformed" => "URI fragment ссылки содержит некорректное percent-кодирование.",
|
||||||
|
"unsupported_composition" => "Несколько операторов composition в одной schema не могут быть спроецированы без потерь.",
|
||||||
|
"unsupported_discriminator" => "Discriminator имеет неподдерживаемую или неполную структуру.",
|
||||||
|
_ => "Ссылка не может быть безопасно разрешена.",
|
||||||
|
}
|
||||||
|
.to_owned(),
|
||||||
|
construct_id: format!("source:{pointer}"),
|
||||||
|
location: SourceLocation {
|
||||||
|
pointer: pointer.to_owned(),
|
||||||
|
},
|
||||||
|
operation_key: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn merge_object(target: &mut Map<String, Value>, source: &Map<String, Value>) -> bool {
|
||||||
|
for (key, value) in source {
|
||||||
|
match key.as_str() {
|
||||||
|
"properties" => {
|
||||||
|
let Some(source_properties) = value.as_object() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let properties = target
|
||||||
|
.entry(key.clone())
|
||||||
|
.or_insert_with(|| Value::Object(Map::new()));
|
||||||
|
let Some(target_properties) = properties.as_object_mut() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
for (name, schema) in source_properties {
|
||||||
|
if let Some(existing) = target_properties.get(name) {
|
||||||
|
let (Some(existing), Some(schema)) =
|
||||||
|
(existing.as_object(), schema.as_object())
|
||||||
|
else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let mut merged = existing.clone();
|
||||||
|
if !merge_object(&mut merged, schema) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
target_properties.insert(name.clone(), Value::Object(merged));
|
||||||
|
} else {
|
||||||
|
target_properties.insert(name.clone(), schema.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"required" => {
|
||||||
|
let Some(source_required) = value.as_array() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let required = target
|
||||||
|
.entry(key.clone())
|
||||||
|
.or_insert_with(|| Value::Array(Vec::new()));
|
||||||
|
let Some(target_required) = required.as_array_mut() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
target_required.extend(source_required.iter().cloned());
|
||||||
|
target_required.sort_by(|left, right| left.as_str().cmp(&right.as_str()));
|
||||||
|
target_required.dedup();
|
||||||
|
}
|
||||||
|
"minimum" | "maximum" => {
|
||||||
|
let Some(source_value) = value.as_f64() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if target.contains_key(key) && target.get(key).and_then(Value::as_f64).is_none() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let merged = match (key.as_str(), target.get(key).and_then(Value::as_f64)) {
|
||||||
|
("minimum", Some(current)) => current.max(source_value),
|
||||||
|
("maximum", Some(current)) => current.min(source_value),
|
||||||
|
_ => source_value,
|
||||||
|
};
|
||||||
|
let Some(number) = serde_json::Number::from_f64(merged) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
target.insert(key.clone(), Value::Number(number));
|
||||||
|
if constraint_contradiction(target, "minimum", "maximum") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"minLength" | "maxLength" => {
|
||||||
|
let Some(source_value) = value.as_u64() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if target.contains_key(key) && target.get(key).and_then(Value::as_u64).is_none() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let merged = match (key.as_str(), target.get(key).and_then(Value::as_u64)) {
|
||||||
|
("minLength", Some(current)) => current.max(source_value),
|
||||||
|
("maxLength", Some(current)) => current.min(source_value),
|
||||||
|
_ => source_value,
|
||||||
|
};
|
||||||
|
target.insert(key.clone(), Value::Number(merged.into()));
|
||||||
|
if integer_constraint_contradiction(target, "minLength", "maxLength") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
if target.get(key).is_some_and(|existing| existing != value) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
target.insert(key.clone(), value.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn constraint_contradiction(target: &Map<String, Value>, minimum: &str, maximum: &str) -> bool {
|
||||||
|
match (
|
||||||
|
target.get(minimum).and_then(Value::as_f64),
|
||||||
|
target.get(maximum).and_then(Value::as_f64),
|
||||||
|
) {
|
||||||
|
(Some(minimum), Some(maximum)) => minimum > maximum,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn integer_constraint_contradiction(
|
||||||
|
target: &Map<String, Value>,
|
||||||
|
minimum: &str,
|
||||||
|
maximum: &str,
|
||||||
|
) -> bool {
|
||||||
|
match (
|
||||||
|
target.get(minimum).and_then(Value::as_u64),
|
||||||
|
target.get(maximum).and_then(Value::as_u64),
|
||||||
|
) {
|
||||||
|
(Some(minimum), Some(maximum)) => minimum > maximum,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn edge_key(edge: &ResolvedReferenceEdge) -> (&str, &str, &str, &str, bool) {
|
||||||
|
(
|
||||||
|
edge.source.snapshot_digest.as_str(),
|
||||||
|
&edge.source.location.pointer,
|
||||||
|
edge.target.snapshot_digest.as_str(),
|
||||||
|
&edge.target.location.pointer,
|
||||||
|
edge.recursive,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_snapshot(document: &str) -> Result<Value, ImportParseError> {
|
||||||
|
if let Ok(value) = serde_json::from_str(document) {
|
||||||
|
return Ok(value);
|
||||||
|
}
|
||||||
|
let yaml: serde_yaml::Value =
|
||||||
|
serde_yaml::from_str(document).map_err(|_| ImportParseError::InvalidDocument)?;
|
||||||
|
serde_json::to_value(yaml).map_err(|_| ImportParseError::InvalidDocument)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_oas31(root: &Value) -> bool {
|
||||||
|
root.get("openapi")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.is_some_and(|version| version.starts_with("3.1."))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn join_relative(base: &str, relative: &str) -> Option<String> {
|
||||||
|
let base = UriReference::parse(base)?;
|
||||||
|
let relative = UriReference::parse(relative)?;
|
||||||
|
let scheme = relative.scheme.or(base.scheme)?;
|
||||||
|
let authority = if relative.scheme.is_some() || relative.authority.is_some() {
|
||||||
|
relative.authority
|
||||||
|
} else {
|
||||||
|
base.authority
|
||||||
|
};
|
||||||
|
let (path, query) = if relative.scheme.is_some() || relative.authority.is_some() {
|
||||||
|
(remove_dot_segments(&relative.path), relative.query)
|
||||||
|
} else if relative.path.is_empty() {
|
||||||
|
(base.path, relative.query.or(base.query))
|
||||||
|
} else if relative.path.starts_with('/') {
|
||||||
|
(remove_dot_segments(&relative.path), relative.query)
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
remove_dot_segments(&merge_paths(
|
||||||
|
&base.path,
|
||||||
|
authority.is_some(),
|
||||||
|
&relative.path,
|
||||||
|
)),
|
||||||
|
relative.query,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
UriReference {
|
||||||
|
scheme: Some(scheme),
|
||||||
|
authority,
|
||||||
|
path,
|
||||||
|
query,
|
||||||
|
}
|
||||||
|
.render()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn canonical_absolute_uri(uri: &str) -> Option<String> {
|
||||||
|
let reference = UriReference::parse(uri)?;
|
||||||
|
let scheme = reference.scheme?;
|
||||||
|
UriReference {
|
||||||
|
scheme: Some(scheme),
|
||||||
|
authority: reference.authority,
|
||||||
|
path: remove_dot_segments(&reference.path),
|
||||||
|
query: reference.query,
|
||||||
|
}
|
||||||
|
.render()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct UriReference<'a> {
|
||||||
|
scheme: Option<&'a str>,
|
||||||
|
authority: Option<&'a str>,
|
||||||
|
path: String,
|
||||||
|
query: Option<&'a str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> UriReference<'a> {
|
||||||
|
fn parse(value: &'a str) -> Option<Self> {
|
||||||
|
if value.contains('\\') || value.contains('#') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let (without_query, query) = value
|
||||||
|
.split_once('?')
|
||||||
|
.map_or((value, None), |(path, query)| (path, Some(query)));
|
||||||
|
let (scheme, rest) = if let Some(index) = without_query.find(':') {
|
||||||
|
let candidate = &without_query[..index];
|
||||||
|
if is_uri_scheme(candidate) {
|
||||||
|
(Some(candidate), &without_query[index + 1..])
|
||||||
|
} else {
|
||||||
|
(None, without_query)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
(None, without_query)
|
||||||
|
};
|
||||||
|
let (authority, path) = if let Some(rest) = rest.strip_prefix("//") {
|
||||||
|
match rest.find('/') {
|
||||||
|
Some(index) => (Some(&rest[..index]), rest[index..].to_owned()),
|
||||||
|
None => (Some(rest), String::new()),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
(None, rest.to_owned())
|
||||||
|
};
|
||||||
|
Some(Self {
|
||||||
|
scheme,
|
||||||
|
authority,
|
||||||
|
path,
|
||||||
|
query,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn render(&self) -> Option<String> {
|
||||||
|
let scheme = self.scheme?;
|
||||||
|
let scheme = scheme.to_ascii_lowercase();
|
||||||
|
let mut value = format!("{scheme}:");
|
||||||
|
if let Some(authority) = self.authority {
|
||||||
|
value.push_str("//");
|
||||||
|
value.push_str(&canonical_authority(authority, &scheme));
|
||||||
|
}
|
||||||
|
if self.authority.is_some() && self.path.is_empty() {
|
||||||
|
value.push('/');
|
||||||
|
} else {
|
||||||
|
value.push_str(&self.path);
|
||||||
|
}
|
||||||
|
if let Some(query) = self.query {
|
||||||
|
value.push('?');
|
||||||
|
value.push_str(query);
|
||||||
|
}
|
||||||
|
Some(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn canonical_authority(authority: &str, scheme: &str) -> String {
|
||||||
|
let authority = authority.to_ascii_lowercase();
|
||||||
|
let default_port = match scheme {
|
||||||
|
"http" => Some("80"),
|
||||||
|
"https" => Some("443"),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
if let Some(default_port) = default_port
|
||||||
|
&& let Some((host, port)) = authority.rsplit_once(':')
|
||||||
|
&& port == default_port
|
||||||
|
&& (!host.contains(':') || host.ends_with(']'))
|
||||||
|
{
|
||||||
|
return host.to_owned();
|
||||||
|
}
|
||||||
|
authority
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_uri_scheme(value: &str) -> bool {
|
||||||
|
value
|
||||||
|
.split_once(':')
|
||||||
|
.is_some_and(|(candidate, _)| is_uri_scheme(candidate))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_uri_scheme(candidate: &str) -> bool {
|
||||||
|
let Some(first) = candidate.as_bytes().first() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
first.is_ascii_alphabetic()
|
||||||
|
&& candidate
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn merge_paths(base_path: &str, has_authority: bool, relative_path: &str) -> String {
|
||||||
|
match base_path.rfind('/') {
|
||||||
|
Some(index) => format!("{}{}", &base_path[..=index], relative_path),
|
||||||
|
None if has_authority => format!("/{relative_path}"),
|
||||||
|
None => relative_path.to_owned(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_dot_segments(path: &str) -> String {
|
||||||
|
let leading_slash = path.starts_with('/');
|
||||||
|
let trailing_slash = path.ends_with('/');
|
||||||
|
let mut output = Vec::new();
|
||||||
|
for segment in path.split('/') {
|
||||||
|
match segment {
|
||||||
|
"." => {}
|
||||||
|
".." => {
|
||||||
|
output.pop();
|
||||||
|
}
|
||||||
|
_ => output.push(segment),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut result = output.join("/");
|
||||||
|
if leading_slash && !result.starts_with('/') {
|
||||||
|
result.insert(0, '/');
|
||||||
|
}
|
||||||
|
if trailing_slash && !result.ends_with('/') {
|
||||||
|
result.push('/');
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
enum TargetError {
|
||||||
|
MalformedFragment,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn percent_decode(fragment: &str) -> Result<String, TargetError> {
|
||||||
|
let bytes = fragment.as_bytes();
|
||||||
|
let mut decoded = Vec::with_capacity(bytes.len());
|
||||||
|
let mut index = 0;
|
||||||
|
while index < bytes.len() {
|
||||||
|
if bytes[index] != b'%' {
|
||||||
|
decoded.push(bytes[index]);
|
||||||
|
index += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(high) = bytes.get(index + 1).and_then(|byte| hex_value(*byte)) else {
|
||||||
|
return Err(TargetError::MalformedFragment);
|
||||||
|
};
|
||||||
|
let Some(low) = bytes.get(index + 2).and_then(|byte| hex_value(*byte)) else {
|
||||||
|
return Err(TargetError::MalformedFragment);
|
||||||
|
};
|
||||||
|
decoded.push((high << 4) | low);
|
||||||
|
index += 3;
|
||||||
|
}
|
||||||
|
String::from_utf8(decoded).map_err(|_| TargetError::MalformedFragment)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex_value(byte: u8) -> Option<u8> {
|
||||||
|
match byte {
|
||||||
|
b'0'..=b'9' => Some(byte - b'0'),
|
||||||
|
b'a'..=b'f' => Some(byte - b'a' + 10),
|
||||||
|
b'A'..=b'F' => Some(byte - b'A' + 10),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_literal_payload_key(key: &str) -> bool {
|
||||||
|
matches!(key, "example" | "examples" | "default" | "enum" | "const") || key.starts_with("x-")
|
||||||
|
}
|
||||||
@@ -32,6 +32,33 @@ pub fn schema_from_openapi(
|
|||||||
let Some(value) = value else {
|
let Some(value) = value else {
|
||||||
return primitive(SchemaKind::String, required, description);
|
return primitive(SchemaKind::String, required, description);
|
||||||
};
|
};
|
||||||
|
// Only v3 NormalizedIR emits the private marker. Pending legacy-v1 jobs
|
||||||
|
// retain their historical first-branch behavior, while modern oneOf/anyOf
|
||||||
|
// is projected losslessly through crank-schema's existing Oneof shape.
|
||||||
|
if value
|
||||||
|
.get("x-crank-lossless-composition")
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
== Some(true)
|
||||||
|
&& let Some(items) = value
|
||||||
|
.get("oneOf")
|
||||||
|
.or_else(|| value.get("anyOf"))
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
{
|
||||||
|
return Schema {
|
||||||
|
kind: SchemaKind::Oneof,
|
||||||
|
description: description.or_else(|| text(value, "description")),
|
||||||
|
required,
|
||||||
|
nullable: nullable(value),
|
||||||
|
default_value: value.get("default").cloned(),
|
||||||
|
fields: BTreeMap::new(),
|
||||||
|
items: None,
|
||||||
|
enum_values: Vec::new(),
|
||||||
|
variants: items
|
||||||
|
.iter()
|
||||||
|
.map(|item| schema_from_openapi(Some(item), true, None))
|
||||||
|
.collect(),
|
||||||
|
};
|
||||||
|
}
|
||||||
let resolved = collapse_composition(value);
|
let resolved = collapse_composition(value);
|
||||||
|
|
||||||
if let Some(values) = resolved.get("enum").and_then(Value::as_array) {
|
if let Some(values) = resolved.get("enum").and_then(Value::as_array) {
|
||||||
@@ -100,6 +127,17 @@ pub fn schema_from_openapi(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn collapse_composition(value: &Value) -> Value {
|
||||||
|
for key in ["allOf", "oneOf", "anyOf"] {
|
||||||
|
if let Some(items) = value.get(key).and_then(Value::as_array)
|
||||||
|
&& let Some(first) = items.first()
|
||||||
|
{
|
||||||
|
return first.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
value.clone()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn object_with_fields(description: Option<String>, fields: BTreeMap<String, Schema>) -> Schema {
|
pub fn object_with_fields(description: Option<String>, fields: BTreeMap<String, Schema>) -> Schema {
|
||||||
Schema {
|
Schema {
|
||||||
kind: SchemaKind::Object,
|
kind: SchemaKind::Object,
|
||||||
@@ -190,14 +228,3 @@ fn text(value: &Value, key: &str) -> Option<String> {
|
|||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(ToOwned::to_owned)
|
.map(ToOwned::to_owned)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn collapse_composition(value: &Value) -> Value {
|
|
||||||
for key in ["allOf", "oneOf", "anyOf"] {
|
|
||||||
if let Some(items) = value.get(key).and_then(Value::as_array)
|
|
||||||
&& let Some(first) = items.first()
|
|
||||||
{
|
|
||||||
return first.clone();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
value.clone()
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,12 +2,19 @@ use crank_core::HttpMethod;
|
|||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::rest::{
|
use crate::rest::{
|
||||||
model::{RestImportDocument, RestImportOperation, RestImportParameter, RestParameterLocation},
|
model::{
|
||||||
normalize::{ImportParseError, resolve_local_ref},
|
ImportFinding, ImportFindingSeverity, RestImportDocument, RestImportOperation,
|
||||||
recommendations::{document_finding, operation_finding},
|
RestImportParameter, RestParameterLocation,
|
||||||
|
},
|
||||||
|
normalize::ImportParseError,
|
||||||
|
recommendations::{document_blocker, document_finding, operation_finding},
|
||||||
};
|
};
|
||||||
|
|
||||||
pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
||||||
|
parse_document_v2(root)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
||||||
let title = root
|
let title = root
|
||||||
.pointer("/info/title")
|
.pointer("/info/title")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
@@ -15,11 +22,15 @@ pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseErr
|
|||||||
.to_owned();
|
.to_owned();
|
||||||
let servers = swagger_servers(root);
|
let servers = swagger_servers(root);
|
||||||
let mut findings = Vec::new();
|
let mut findings = Vec::new();
|
||||||
|
let mut internal_finding_locations = Vec::new();
|
||||||
if servers.is_empty() {
|
if servers.is_empty() {
|
||||||
findings.push(document_finding(
|
findings.push(document_finding(
|
||||||
"missing_servers",
|
"missing_servers",
|
||||||
"В Swagger 2.0 документе не указаны host/schemes, base URL нужно будет выбрать вручную.",
|
"В Swagger 2.0 документе не указаны host/schemes, base URL нужно будет выбрать вручную.",
|
||||||
));
|
));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: String::new(),
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut operations = Vec::new();
|
let mut operations = Vec::new();
|
||||||
@@ -29,20 +40,96 @@ pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseErr
|
|||||||
.ok_or(ImportParseError::UnsupportedDocument)?;
|
.ok_or(ImportParseError::UnsupportedDocument)?;
|
||||||
|
|
||||||
for (path, path_item) in paths {
|
for (path, path_item) in paths {
|
||||||
let path_parameters = parameters(root, path_item.get("parameters"));
|
if !path_item.is_object() {
|
||||||
|
findings.push(path_item_blocker(path));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("/paths/{}", path.replace('~', "~0").replace('/', "~1")),
|
||||||
|
}));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for method_name in ["head", "options", "trace", "connect"] {
|
||||||
|
if path_item.get(method_name).is_some() {
|
||||||
|
findings.push(document_blocker(
|
||||||
|
"unsupported_http_method",
|
||||||
|
format!("Метод {method_name} для пути {path} пока не поддерживается."),
|
||||||
|
));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"/paths/{}/{}",
|
||||||
|
path.replace('~', "~0").replace('/', "~1"),
|
||||||
|
method_name
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let path_pointer = format!("/paths/{}", path.replace('~', "~0").replace('/', "~1"));
|
||||||
|
let path_parameters = parameters(
|
||||||
|
root,
|
||||||
|
path_item.get("parameters"),
|
||||||
|
&format!("{path_pointer}/parameters"),
|
||||||
|
);
|
||||||
|
append_parameter_findings(
|
||||||
|
&mut findings,
|
||||||
|
&mut internal_finding_locations,
|
||||||
|
path_parameters.findings,
|
||||||
|
);
|
||||||
|
let path_parameters = path_parameters.parameters;
|
||||||
for method_name in ["get", "post", "put", "patch", "delete"] {
|
for method_name in ["get", "post", "put", "patch", "delete"] {
|
||||||
let Some(operation_value) = path_item.get(method_name) else {
|
let Some(operation_value) = path_item.get(method_name) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
if !operation_value.is_object() {
|
||||||
|
findings.push(document_blocker(
|
||||||
|
"invalid_operation",
|
||||||
|
"Операция имеет неверную структуру и не была интерпретирована.",
|
||||||
|
));
|
||||||
|
internal_finding_locations.push(Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"/paths/{}/{}",
|
||||||
|
path.replace('~', "~0").replace('/', "~1"),
|
||||||
|
method_name
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let Some(method) = method_from_lower(method_name) else {
|
let Some(method) = method_from_lower(method_name) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let mut operation_parameters = path_parameters.clone();
|
let mut operation_parameters = path_parameters.clone();
|
||||||
operation_parameters.extend(parameters(root, operation_value.get("parameters")));
|
let parsed_parameters = parameters(
|
||||||
|
root,
|
||||||
|
operation_value.get("parameters"),
|
||||||
|
&format!("{path_pointer}/{method_name}/parameters"),
|
||||||
|
);
|
||||||
|
append_parameter_findings(
|
||||||
|
&mut findings,
|
||||||
|
&mut internal_finding_locations,
|
||||||
|
parsed_parameters.findings,
|
||||||
|
);
|
||||||
|
operation_parameters.extend(parsed_parameters.parameters);
|
||||||
|
deduplicate_parameters(&mut operation_parameters);
|
||||||
|
let tags = tags(
|
||||||
|
operation_value.get("tags"),
|
||||||
|
&format!("{path_pointer}/{method_name}/tags"),
|
||||||
|
);
|
||||||
|
append_parameter_findings(
|
||||||
|
&mut findings,
|
||||||
|
&mut internal_finding_locations,
|
||||||
|
tags.findings,
|
||||||
|
);
|
||||||
let request_body_schema = operation_parameters
|
let request_body_schema = operation_parameters
|
||||||
.iter()
|
.iter()
|
||||||
.find(|parameter| parameter.name == "body")
|
.find(|parameter| parameter.name == "body")
|
||||||
.and_then(|parameter| parameter.schema.clone());
|
.and_then(|parameter| {
|
||||||
|
parameter.schema.clone().map(|schema| {
|
||||||
|
(
|
||||||
|
schema,
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("{}/schema", parameter.source_location.pointer),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
});
|
||||||
let operation_parameters = operation_parameters
|
let operation_parameters = operation_parameters
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|parameter| parameter.name != "body")
|
.filter(|parameter| parameter.name != "body")
|
||||||
@@ -64,20 +151,22 @@ pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseErr
|
|||||||
.get("description")
|
.get("description")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(ToOwned::to_owned),
|
.map(ToOwned::to_owned),
|
||||||
tags: operation_value
|
tags: tags.tags,
|
||||||
.get("tags")
|
|
||||||
.and_then(Value::as_array)
|
|
||||||
.map(|items| {
|
|
||||||
items
|
|
||||||
.iter()
|
|
||||||
.filter_map(Value::as_str)
|
|
||||||
.map(ToOwned::to_owned)
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.unwrap_or_default(),
|
|
||||||
parameters: operation_parameters,
|
parameters: operation_parameters,
|
||||||
request_body_schema,
|
request_body_schema: request_body_schema
|
||||||
response_schema: response_schema(root, operation_value),
|
.as_ref()
|
||||||
|
.map(|(schema, _)| schema.clone()),
|
||||||
|
request_body_schema_location: request_body_schema.map(|(_, location)| location),
|
||||||
|
response_schema: response_schema(
|
||||||
|
operation_value,
|
||||||
|
&format!("{path_pointer}/{method_name}"),
|
||||||
|
)
|
||||||
|
.map(|(schema, _)| schema),
|
||||||
|
response_schema_location: response_schema(
|
||||||
|
operation_value,
|
||||||
|
&format!("{path_pointer}/{method_name}"),
|
||||||
|
)
|
||||||
|
.map(|(_, location)| location),
|
||||||
servers: Vec::new(),
|
servers: Vec::new(),
|
||||||
findings: swagger_operation_findings(path, operation_value),
|
findings: swagger_operation_findings(path, operation_value),
|
||||||
});
|
});
|
||||||
@@ -91,9 +180,208 @@ pub fn parse_document(root: &Value) -> Result<RestImportDocument, ImportParseErr
|
|||||||
servers,
|
servers,
|
||||||
operations,
|
operations,
|
||||||
findings,
|
findings,
|
||||||
|
internal_finding_locations,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn deduplicate_parameters(parameters: &mut Vec<RestImportParameter>) {
|
||||||
|
let mut seen = std::collections::BTreeSet::new();
|
||||||
|
parameters.reverse();
|
||||||
|
parameters.retain(|parameter| seen.insert((parameter.name.clone(), parameter.location)));
|
||||||
|
parameters.reverse();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse_document_legacy_v1(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
||||||
|
let title = root
|
||||||
|
.pointer("/info/title")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("Imported API")
|
||||||
|
.to_owned();
|
||||||
|
let servers = swagger_servers(root);
|
||||||
|
let mut findings = Vec::new();
|
||||||
|
if servers.is_empty() {
|
||||||
|
findings.push(document_finding(
|
||||||
|
"missing_servers",
|
||||||
|
"В Swagger 2.0 документе не указаны host/schemes, base URL нужно будет выбрать вручную.",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut operations = Vec::new();
|
||||||
|
let paths = root
|
||||||
|
.get("paths")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.ok_or(ImportParseError::UnsupportedDocument)?;
|
||||||
|
for (path, path_item) in paths {
|
||||||
|
let path_pointer = format!("/paths/{}", path.replace('~', "~0").replace('/', "~1"));
|
||||||
|
let path_parameters = legacy_parameters(
|
||||||
|
root,
|
||||||
|
path_item.get("parameters"),
|
||||||
|
&format!("{path_pointer}/parameters"),
|
||||||
|
);
|
||||||
|
for method_name in ["get", "post", "put", "patch", "delete"] {
|
||||||
|
let Some(operation) = path_item.get(method_name) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(method) = method_from_lower(method_name) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let operation_pointer = format!("{path_pointer}/{method_name}");
|
||||||
|
let mut parameters = path_parameters.clone();
|
||||||
|
parameters.extend(legacy_parameters(
|
||||||
|
root,
|
||||||
|
operation.get("parameters"),
|
||||||
|
&format!("{operation_pointer}/parameters"),
|
||||||
|
));
|
||||||
|
let request_body_schema = parameters
|
||||||
|
.iter()
|
||||||
|
.find(|parameter| parameter.name == "body")
|
||||||
|
.and_then(|parameter| parameter.schema.clone());
|
||||||
|
let parameters = parameters
|
||||||
|
.into_iter()
|
||||||
|
.filter(|parameter| parameter.name != "body")
|
||||||
|
.collect();
|
||||||
|
operations.push(RestImportOperation {
|
||||||
|
key: format!("{} {}", method_name.to_uppercase(), path),
|
||||||
|
method,
|
||||||
|
path: path.clone(),
|
||||||
|
operation_id: operation
|
||||||
|
.get("operationId")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
summary: operation
|
||||||
|
.get("summary")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
description: operation
|
||||||
|
.get("description")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
tags: legacy_tags(operation.get("tags")),
|
||||||
|
parameters,
|
||||||
|
request_body_schema,
|
||||||
|
request_body_schema_location: None,
|
||||||
|
response_schema: legacy_response_schema(root, operation),
|
||||||
|
response_schema_location: None,
|
||||||
|
servers: Vec::new(),
|
||||||
|
findings: swagger_operation_findings(path, operation),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(RestImportDocument {
|
||||||
|
format: "swagger".to_owned(),
|
||||||
|
version: Some("2.0".to_owned()),
|
||||||
|
title,
|
||||||
|
servers,
|
||||||
|
operations,
|
||||||
|
findings,
|
||||||
|
internal_finding_locations: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn legacy_tags(value: Option<&Value>) -> Vec<String> {
|
||||||
|
value
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.filter_map(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn legacy_parameters(
|
||||||
|
root: &Value,
|
||||||
|
value: Option<&Value>,
|
||||||
|
base_pointer: &str,
|
||||||
|
) -> Vec<RestImportParameter> {
|
||||||
|
value
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, item)| {
|
||||||
|
let item = crate::rest::normalize::resolve_local_ref(root, item, 0);
|
||||||
|
let raw_location = item.get("in").and_then(Value::as_str)?;
|
||||||
|
let pointer = format!("{base_pointer}/{index}");
|
||||||
|
if raw_location == "body" {
|
||||||
|
return Some(RestImportParameter {
|
||||||
|
name: "body".to_owned(),
|
||||||
|
location: RestParameterLocation::Query,
|
||||||
|
required: item
|
||||||
|
.get("required")
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
.unwrap_or(false),
|
||||||
|
description: item
|
||||||
|
.get("description")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
schema: item.get("schema").map(|schema| {
|
||||||
|
crate::rest::normalize::resolve_local_ref(root, schema, 0)
|
||||||
|
}),
|
||||||
|
source_location: crate::rest::model::SourceLocation { pointer },
|
||||||
|
schema_source_location: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let location = match raw_location {
|
||||||
|
"path" => RestParameterLocation::Path,
|
||||||
|
"query" => RestParameterLocation::Query,
|
||||||
|
"header" => RestParameterLocation::Header,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
Some(RestImportParameter {
|
||||||
|
name: item.get("name").and_then(Value::as_str)?.to_owned(),
|
||||||
|
location,
|
||||||
|
required: item
|
||||||
|
.get("required")
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
.unwrap_or(false)
|
||||||
|
|| location == RestParameterLocation::Path,
|
||||||
|
description: item
|
||||||
|
.get("description")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
|
schema: legacy_parameter_schema(root, &item),
|
||||||
|
source_location: crate::rest::model::SourceLocation { pointer },
|
||||||
|
schema_source_location: None,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn legacy_parameter_schema(root: &Value, parameter: &Value) -> Option<Value> {
|
||||||
|
if let Some(schema) = parameter.get("schema") {
|
||||||
|
return Some(crate::rest::normalize::resolve_local_ref(root, schema, 0));
|
||||||
|
}
|
||||||
|
let mut schema = serde_json::Map::new();
|
||||||
|
for key in ["type", "format", "items", "enum", "default", "description"] {
|
||||||
|
if let Some(value) = parameter.get(key) {
|
||||||
|
schema.insert(
|
||||||
|
key.to_owned(),
|
||||||
|
crate::rest::normalize::resolve_local_ref(root, value, 0),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(!schema.is_empty()).then_some(Value::Object(schema))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn legacy_response_schema(root: &Value, operation: &Value) -> Option<Value> {
|
||||||
|
let responses = operation.get("responses")?.as_object()?;
|
||||||
|
for code in ["200", "201", "202", "default"] {
|
||||||
|
let Some(response) = responses.get(code) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let response = crate::rest::normalize::resolve_local_ref(root, response, 0);
|
||||||
|
if let Some(schema) = response.get("schema") {
|
||||||
|
return Some(crate::rest::normalize::resolve_local_ref(root, schema, 0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
fn swagger_servers(root: &Value) -> Vec<String> {
|
fn swagger_servers(root: &Value) -> Vec<String> {
|
||||||
let Some(host) = root.get("host").and_then(Value::as_str) else {
|
let Some(host) = root.get("host").and_then(Value::as_str) else {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
@@ -116,14 +404,51 @@ fn swagger_servers(root: &Value) -> Vec<String> {
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parameters(root: &Value, value: Option<&Value>) -> Vec<RestImportParameter> {
|
fn parameters(_root: &Value, value: Option<&Value>, base_pointer: &str) -> ParsedParameters {
|
||||||
value
|
let mut findings = Vec::new();
|
||||||
|
let parameters = value
|
||||||
.and_then(Value::as_array)
|
.and_then(Value::as_array)
|
||||||
.map(|items| {
|
.map(|items| {
|
||||||
items
|
items
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|item| {
|
.enumerate()
|
||||||
let item = resolve_local_ref(root, item, 0);
|
.filter_map(|(index, item)| {
|
||||||
|
let pointer = format!("{base_pointer}/{index}");
|
||||||
|
{
|
||||||
|
let Some(object) = item.as_object() else {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"invalid_parameter",
|
||||||
|
"Параметр должен быть объектом и не был интерпретирован.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
if object.contains_key("$ref") {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"unresolved_parameter_reference",
|
||||||
|
"Параметр по $ref требует разрешения перед импортом.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if !object.get("name").is_some_and(Value::is_string) {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"missing_parameter_name",
|
||||||
|
"У параметра отсутствует строковое поле name.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if !object.get("in").is_some_and(Value::is_string) {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"missing_parameter_location",
|
||||||
|
"У параметра отсутствует строковое поле in.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let item = item.clone();
|
||||||
let raw_location = item.get("in").and_then(Value::as_str)?;
|
let raw_location = item.get("in").and_then(Value::as_str)?;
|
||||||
if raw_location == "body" {
|
if raw_location == "body" {
|
||||||
return Some(RestImportParameter {
|
return Some(RestImportParameter {
|
||||||
@@ -137,16 +462,37 @@ fn parameters(root: &Value, value: Option<&Value>) -> Vec<RestImportParameter> {
|
|||||||
.get("description")
|
.get("description")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(ToOwned::to_owned),
|
.map(ToOwned::to_owned),
|
||||||
schema: item
|
schema: item.get("schema").cloned(),
|
||||||
.get("schema")
|
schema_source_location: item.get("schema").map(|_| {
|
||||||
.map(|schema| resolve_local_ref(root, schema, 0)),
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("{base_pointer}/{index}/schema"),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
source_location: crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("{base_pointer}/{index}"),
|
||||||
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let location = match raw_location {
|
let location = match raw_location {
|
||||||
"path" => RestParameterLocation::Path,
|
"path" => RestParameterLocation::Path,
|
||||||
"query" => RestParameterLocation::Query,
|
"query" => RestParameterLocation::Query,
|
||||||
"header" => RestParameterLocation::Header,
|
"header" => RestParameterLocation::Header,
|
||||||
_ => return None,
|
"cookie" => {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"Cookie parameter пока не поддерживается и не был импортирован.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
findings.push(parameter_blocker(
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"Параметр использует неподдерживаемое значение in и не был импортирован.",
|
||||||
|
&pointer,
|
||||||
|
));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
Some(RestImportParameter {
|
Some(RestImportParameter {
|
||||||
name: item.get("name").and_then(Value::as_str)?.to_owned(),
|
name: item.get("name").and_then(Value::as_str)?.to_owned(),
|
||||||
@@ -160,22 +506,95 @@ fn parameters(root: &Value, value: Option<&Value>) -> Vec<RestImportParameter> {
|
|||||||
.get("description")
|
.get("description")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(ToOwned::to_owned),
|
.map(ToOwned::to_owned),
|
||||||
schema: swagger_parameter_schema(root, &item),
|
schema: swagger_parameter_schema(_root, &item),
|
||||||
|
schema_source_location: Some(crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("{base_pointer}/{index}"),
|
||||||
|
}),
|
||||||
|
source_location: crate::rest::model::SourceLocation {
|
||||||
|
pointer,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
})
|
})
|
||||||
.unwrap_or_default()
|
.unwrap_or_default();
|
||||||
|
ParsedParameters {
|
||||||
|
parameters,
|
||||||
|
findings,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn swagger_parameter_schema(root: &Value, parameter: &Value) -> Option<Value> {
|
struct ParsedParameters {
|
||||||
|
parameters: Vec<RestImportParameter>,
|
||||||
|
findings: Vec<(ImportFinding, crate::rest::model::SourceLocation)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ParsedTags {
|
||||||
|
tags: Vec<String>,
|
||||||
|
findings: Vec<(ImportFinding, crate::rest::model::SourceLocation)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tags(value: Option<&Value>, base_pointer: &str) -> ParsedTags {
|
||||||
|
let mut findings = Vec::new();
|
||||||
|
let tags = value
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, item)| match item.as_str() {
|
||||||
|
Some(tag) => Some(tag.to_owned()),
|
||||||
|
None => {
|
||||||
|
findings.push((
|
||||||
|
document_finding(
|
||||||
|
"invalid_tag",
|
||||||
|
"Тег должен быть строкой и не был импортирован.",
|
||||||
|
),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!("{base_pointer}/{index}"),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
ParsedTags { tags, findings }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parameter_blocker(
|
||||||
|
code: &str,
|
||||||
|
message: &str,
|
||||||
|
pointer: &str,
|
||||||
|
) -> (ImportFinding, crate::rest::model::SourceLocation) {
|
||||||
|
(
|
||||||
|
document_blocker(code, message),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: pointer.to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn append_parameter_findings(
|
||||||
|
findings: &mut Vec<ImportFinding>,
|
||||||
|
locations: &mut Vec<Option<crate::rest::model::SourceLocation>>,
|
||||||
|
parameter_findings: Vec<(ImportFinding, crate::rest::model::SourceLocation)>,
|
||||||
|
) {
|
||||||
|
for (finding, location) in parameter_findings {
|
||||||
|
findings.push(finding);
|
||||||
|
locations.push(Some(location));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn swagger_parameter_schema(_root: &Value, parameter: &Value) -> Option<Value> {
|
||||||
if let Some(schema) = parameter.get("schema") {
|
if let Some(schema) = parameter.get("schema") {
|
||||||
return Some(resolve_local_ref(root, schema, 0));
|
return Some(schema.clone());
|
||||||
}
|
}
|
||||||
let mut schema = serde_json::Map::new();
|
let mut schema = serde_json::Map::new();
|
||||||
for key in ["type", "format", "items", "enum", "default", "description"] {
|
for key in ["type", "format", "items", "enum", "default", "description"] {
|
||||||
if let Some(value) = parameter.get(key) {
|
if let Some(value) = parameter.get(key) {
|
||||||
schema.insert(key.to_owned(), resolve_local_ref(root, value, 0));
|
schema.insert(key.to_owned(), value.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if schema.is_empty() {
|
if schema.is_empty() {
|
||||||
@@ -185,18 +604,48 @@ fn swagger_parameter_schema(root: &Value, parameter: &Value) -> Option<Value> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn response_schema(root: &Value, operation: &Value) -> Option<Value> {
|
fn response_schema(
|
||||||
|
operation: &Value,
|
||||||
|
operation_pointer: &str,
|
||||||
|
) -> Option<(Value, crate::rest::model::SourceLocation)> {
|
||||||
let responses = operation.get("responses")?.as_object()?;
|
let responses = operation.get("responses")?.as_object()?;
|
||||||
for code in ["200", "201", "202", "default"] {
|
let mut numeric = responses
|
||||||
let Some(response) = responses.get(code) else {
|
.iter()
|
||||||
continue;
|
.filter_map(|(code, response)| {
|
||||||
};
|
(code.len() == 3)
|
||||||
let response = resolve_local_ref(root, response, 0);
|
.then(|| code.parse::<u16>().ok())
|
||||||
if let Some(schema) = response.get("schema") {
|
.flatten()
|
||||||
return Some(resolve_local_ref(root, schema, 0));
|
.filter(|status| (200..300).contains(status))
|
||||||
|
.map(|status| (status, code.as_str(), response))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
numeric.sort_by(|left, right| (left.0, left.1).cmp(&(right.0, right.1)));
|
||||||
|
for (_, code, response) in numeric {
|
||||||
|
if let Some(schema) = response_schema_at(response, operation_pointer, code) {
|
||||||
|
return Some(schema);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None
|
responses
|
||||||
|
.get("default")
|
||||||
|
.and_then(|response| response_schema_at(response, operation_pointer, "default"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn response_schema_at(
|
||||||
|
response: &Value,
|
||||||
|
operation_pointer: &str,
|
||||||
|
code: &str,
|
||||||
|
) -> Option<(Value, crate::rest::model::SourceLocation)> {
|
||||||
|
response.get("schema").map(|schema| {
|
||||||
|
(
|
||||||
|
schema.clone(),
|
||||||
|
crate::rest::model::SourceLocation {
|
||||||
|
pointer: format!(
|
||||||
|
"{operation_pointer}/responses/{}/schema",
|
||||||
|
code.replace('~', "~0").replace('/', "~1")
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn swagger_operation_findings(
|
fn swagger_operation_findings(
|
||||||
@@ -232,3 +681,12 @@ fn method_from_lower(value: &str) -> Option<HttpMethod> {
|
|||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn path_item_blocker(_path: &str) -> ImportFinding {
|
||||||
|
ImportFinding {
|
||||||
|
code: "invalid_path_item".to_owned(),
|
||||||
|
severity: ImportFindingSeverity::Error,
|
||||||
|
message: "Path Item имеет неверную структуру и не был интерпретирован.".to_owned(),
|
||||||
|
operation_key: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,275 @@
|
|||||||
|
use crank_import::rest::model::{CanonicalSourceNode, CanonicalSourceValue, CoverageDisposition};
|
||||||
|
use crank_import::rest::{
|
||||||
|
ImportFindingSeverity, ImportParseError, NormalizationConfig, NormalizedFinding, NormalizedIr,
|
||||||
|
SourceDigest, SourceLocation, normalize_verified_document, validate_normalized_ir,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn normalize(document: &str) -> NormalizedIr {
|
||||||
|
normalize_verified_document(
|
||||||
|
document,
|
||||||
|
SourceDigest::parse("c".repeat(64)).unwrap(),
|
||||||
|
&NormalizationConfig::default(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn valid_source() -> &'static str {
|
||||||
|
r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Coverage, version: v1 }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
tags: [items]
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { type: object, properties: { id: { type: string } } }
|
||||||
|
"#
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_invalid(ir: &NormalizedIr) {
|
||||||
|
assert_eq!(
|
||||||
|
validate_normalized_ir(ir),
|
||||||
|
Err(ImportParseError::InvalidDocument)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn source_digest_deserialization_preserves_the_validated_invariant() {
|
||||||
|
assert!(serde_json::from_str::<SourceDigest>(&format!("\"{}\"", "a".repeat(64))).is_ok());
|
||||||
|
assert!(serde_json::from_str::<SourceDigest>(&format!("\"{}\"", "A".repeat(64))).is_err());
|
||||||
|
assert!(serde_json::from_str::<SourceDigest>("\"not-a-digest\"").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_tampered_coverage_location_disposition_and_orphan_finding() {
|
||||||
|
let ir = normalize(valid_source());
|
||||||
|
let operation_id = ir.operations[0].stable_id.clone();
|
||||||
|
|
||||||
|
let mut wrong_location = ir.clone();
|
||||||
|
wrong_location
|
||||||
|
.coverage
|
||||||
|
.iter_mut()
|
||||||
|
.find(|entry| entry.construct_id == operation_id)
|
||||||
|
.unwrap()
|
||||||
|
.location
|
||||||
|
.pointer = "/paths/~1other/get".to_owned();
|
||||||
|
assert_invalid(&wrong_location);
|
||||||
|
|
||||||
|
let mut wrong_disposition = ir.clone();
|
||||||
|
wrong_disposition
|
||||||
|
.coverage
|
||||||
|
.iter_mut()
|
||||||
|
.find(|entry| entry.construct_id == operation_id)
|
||||||
|
.unwrap()
|
||||||
|
.disposition = CoverageDisposition::Finding;
|
||||||
|
assert_invalid(&wrong_disposition);
|
||||||
|
|
||||||
|
let mut duplicate = ir.clone();
|
||||||
|
duplicate.coverage.push(duplicate.coverage[0].clone());
|
||||||
|
assert_invalid(&duplicate);
|
||||||
|
|
||||||
|
let mut orphan = ir;
|
||||||
|
orphan.findings.push(NormalizedFinding {
|
||||||
|
code: "forged".to_owned(),
|
||||||
|
severity: ImportFindingSeverity::Warning,
|
||||||
|
message: "forged".to_owned(),
|
||||||
|
construct_id: "orphan".to_owned(),
|
||||||
|
location: SourceLocation {
|
||||||
|
pointer: String::new(),
|
||||||
|
},
|
||||||
|
operation_key: None,
|
||||||
|
});
|
||||||
|
assert_invalid(&orphan);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_tampered_contract_operation_identity_and_source_tree() {
|
||||||
|
let ir = normalize(valid_source());
|
||||||
|
|
||||||
|
let mut normalizer = ir.clone();
|
||||||
|
normalizer.normalizer_version = "future-normalizer".to_owned();
|
||||||
|
assert_invalid(&normalizer);
|
||||||
|
|
||||||
|
let mut projection = ir.clone();
|
||||||
|
projection.projection_version = "future-projection".to_owned();
|
||||||
|
assert_invalid(&projection);
|
||||||
|
|
||||||
|
let mut source_version = ir.clone();
|
||||||
|
source_version.source.version = Some("3.0.3".to_owned());
|
||||||
|
assert_invalid(&source_version);
|
||||||
|
|
||||||
|
let mut stable_id = ir.clone();
|
||||||
|
stable_id.operations[0].stable_id.push_str(":forged");
|
||||||
|
assert_invalid(&stable_id);
|
||||||
|
|
||||||
|
let mut operation_location = ir.clone();
|
||||||
|
operation_location.operations[0].location.pointer = "/paths/~1items/post".to_owned();
|
||||||
|
assert_invalid(&operation_location);
|
||||||
|
|
||||||
|
let mut operation_key = ir.clone();
|
||||||
|
operation_key.operations[0].key = "POST /items".to_owned();
|
||||||
|
assert_invalid(&operation_key);
|
||||||
|
|
||||||
|
let mut schema_description = ir.clone();
|
||||||
|
schema_description.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_mut()
|
||||||
|
.unwrap()
|
||||||
|
.description = Some("forged description".to_owned());
|
||||||
|
assert_invalid(&schema_description);
|
||||||
|
|
||||||
|
let mut source_tree = ir;
|
||||||
|
source_tree.source_tree.location.pointer = "/forged".to_owned();
|
||||||
|
assert_invalid(&source_tree);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validates_every_recursive_source_tree_pointer_and_construct_id() {
|
||||||
|
let mut ir = normalize(valid_source());
|
||||||
|
let CanonicalSourceValue::Object(root) = &mut ir.source_tree.value else {
|
||||||
|
panic!("source root must be an object");
|
||||||
|
};
|
||||||
|
let CanonicalSourceValue::Object(info) = &mut root.get_mut("info").unwrap().value else {
|
||||||
|
panic!("info must be an object");
|
||||||
|
};
|
||||||
|
let title = info.get_mut("title").unwrap();
|
||||||
|
title.construct_id = "source:/info/wrong".to_owned();
|
||||||
|
assert_invalid(&ir);
|
||||||
|
|
||||||
|
let mut ir = normalize(valid_source());
|
||||||
|
let title = source_node_mut(&mut ir.source_tree, "/info/title").unwrap();
|
||||||
|
title.location.pointer = "/info/wrong".to_owned();
|
||||||
|
assert_invalid(&ir);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn accepts_virtual_missing_operation_id_with_exact_finding_coverage() {
|
||||||
|
let ir = normalize(
|
||||||
|
"openapi: 3.1.0\ninfo: { title: Missing ID }\npaths: { /ok: { get: { responses: { '200': { description: ok } } } } }",
|
||||||
|
);
|
||||||
|
validate_normalized_ir(&ir).unwrap();
|
||||||
|
let operation = &ir.operations[0];
|
||||||
|
let id = format!("{}:operation_id", operation.stable_id);
|
||||||
|
let expected_pointer = format!("{}/operationId", operation.location.pointer);
|
||||||
|
assert!(source_node(&ir.source_tree, &expected_pointer).is_none());
|
||||||
|
assert!(operation.findings.iter().any(|finding| {
|
||||||
|
finding.code == "missing_operation_id"
|
||||||
|
&& finding.construct_id == id
|
||||||
|
&& finding.location.pointer == expected_pointer
|
||||||
|
}));
|
||||||
|
assert!(ir.coverage.iter().any(|entry| {
|
||||||
|
entry.construct_id == id
|
||||||
|
&& entry.location.pointer == expected_pointer
|
||||||
|
&& entry.disposition == CoverageDisposition::Finding
|
||||||
|
}));
|
||||||
|
|
||||||
|
let mut missing_finding = ir;
|
||||||
|
missing_finding.operations[0]
|
||||||
|
.findings
|
||||||
|
.retain(|finding| finding.code != "missing_operation_id");
|
||||||
|
assert_invalid(&missing_finding);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deduplicates_shared_path_parameter_and_schema_coverage_exactly() {
|
||||||
|
let ir = normalize(
|
||||||
|
r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Shared }
|
||||||
|
paths:
|
||||||
|
/items/{id}:
|
||||||
|
parameters:
|
||||||
|
- { name: id, in: path, required: true, schema: { type: string } }
|
||||||
|
get:
|
||||||
|
operationId: getItem
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
post:
|
||||||
|
operationId: updateItem
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
validate_normalized_ir(&ir).unwrap();
|
||||||
|
assert_eq!(ir.operations.len(), 2);
|
||||||
|
assert_eq!(ir.operations[0].parameters, ir.operations[1].parameters);
|
||||||
|
let parameter_id = &ir.operations[0].parameters[0].construct_id;
|
||||||
|
let schema_id = &ir.operations[0].parameters[0]
|
||||||
|
.schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.construct_id;
|
||||||
|
assert_eq!(
|
||||||
|
ir.coverage
|
||||||
|
.iter()
|
||||||
|
.filter(|entry| &entry.construct_id == parameter_id)
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ir.coverage
|
||||||
|
.iter()
|
||||||
|
.filter(|entry| &entry.construct_id == schema_id)
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unsupported_method_has_exact_finding_pointer_and_coverage() {
|
||||||
|
let ir = normalize(
|
||||||
|
"openapi: 3.1.0\ninfo: { title: Unsupported }\npaths: { /ok: { get: { responses: { '200': { description: ok } } }, head: { responses: {} } } }",
|
||||||
|
);
|
||||||
|
validate_normalized_ir(&ir).unwrap();
|
||||||
|
let pointer = "/paths/~1ok/head";
|
||||||
|
let construct_id = format!("3.1.0:{pointer}");
|
||||||
|
assert!(ir.findings.iter().any(|finding| {
|
||||||
|
finding.code == "unsupported_http_method"
|
||||||
|
&& finding.construct_id == construct_id
|
||||||
|
&& finding.location.pointer == pointer
|
||||||
|
}));
|
||||||
|
assert!(ir.coverage.iter().any(|entry| {
|
||||||
|
entry.construct_id == construct_id
|
||||||
|
&& entry.location.pointer == pointer
|
||||||
|
&& entry.disposition == CoverageDisposition::Finding
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source_node<'a>(
|
||||||
|
node: &'a CanonicalSourceNode,
|
||||||
|
pointer: &str,
|
||||||
|
) -> Option<&'a CanonicalSourceNode> {
|
||||||
|
if node.location.pointer == pointer {
|
||||||
|
return Some(node);
|
||||||
|
}
|
||||||
|
match &node.value {
|
||||||
|
CanonicalSourceValue::Array(items) => {
|
||||||
|
items.iter().find_map(|child| source_node(child, pointer))
|
||||||
|
}
|
||||||
|
CanonicalSourceValue::Object(items) => {
|
||||||
|
items.values().find_map(|child| source_node(child, pointer))
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source_node_mut<'a>(
|
||||||
|
node: &'a mut CanonicalSourceNode,
|
||||||
|
pointer: &str,
|
||||||
|
) -> Option<&'a mut CanonicalSourceNode> {
|
||||||
|
if node.location.pointer == pointer {
|
||||||
|
return Some(node);
|
||||||
|
}
|
||||||
|
match &mut node.value {
|
||||||
|
CanonicalSourceValue::Array(items) => items
|
||||||
|
.iter_mut()
|
||||||
|
.find_map(|child| source_node_mut(child, pointer)),
|
||||||
|
CanonicalSourceValue::Object(items) => items
|
||||||
|
.values_mut()
|
||||||
|
.find_map(|child| source_node_mut(child, pointer)),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
["not", "an", "openapi", "root"]
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
openapi: 3.2.0
|
||||||
|
info: { title: unsupported }
|
||||||
|
paths: {}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
openapi: [wrong-root-field]
|
||||||
|
info: { title: Invalid }
|
||||||
|
paths: {}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Fixture OpenAPI 3.0 }
|
||||||
|
paths:
|
||||||
|
/health:
|
||||||
|
get:
|
||||||
|
responses: { '200': { description: OK } }
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"openapi": "3.1.0",
|
||||||
|
"info": { "title": "Fixture OpenAPI 3.1" },
|
||||||
|
"paths": {
|
||||||
|
"/health": {
|
||||||
|
"get": { "responses": { "200": { "description": "OK" } } }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Fixture Swagger 2.0 }
|
||||||
|
paths:
|
||||||
|
/health:
|
||||||
|
get:
|
||||||
|
responses: { '200': { description: OK } }
|
||||||
@@ -0,0 +1,803 @@
|
|||||||
|
mod normalization_details {
|
||||||
|
use crank_import::rest::{
|
||||||
|
ImportFindingSeverity, ImportParseError, NormalizationConfig, normalize_verified_document,
|
||||||
|
preview_document, preview_document_legacy_v1,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn normalize_document(
|
||||||
|
document: &str,
|
||||||
|
config: &NormalizationConfig,
|
||||||
|
) -> Result<crank_import::rest::NormalizedIr, ImportParseError> {
|
||||||
|
normalize_verified_document(
|
||||||
|
document,
|
||||||
|
crank_import::rest::SourceDigest::parse("b".repeat(64)).unwrap(),
|
||||||
|
config,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn operation_identity_does_not_depend_on_operation_id() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Identity }
|
||||||
|
paths:
|
||||||
|
/same:
|
||||||
|
get:
|
||||||
|
operationId: duplicate
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
post:
|
||||||
|
operationId: duplicate
|
||||||
|
responses: { '201': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_ne!(ir.operations[0].stable_id, ir.operations[1].stable_id);
|
||||||
|
assert_eq!(ir.operations[0].key, "GET /same");
|
||||||
|
assert_eq!(ir.operations[1].key, "POST /same");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blank_and_duplicate_operation_ids_have_exact_findings_and_path_servers_win() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: IDs }
|
||||||
|
servers: [{ url: https://document.test }]
|
||||||
|
paths:
|
||||||
|
/one:
|
||||||
|
servers: [{ url: https://path.test }]
|
||||||
|
get:
|
||||||
|
operationId: duplicate
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
/two:
|
||||||
|
get:
|
||||||
|
operationId: duplicate
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
/blank:
|
||||||
|
get:
|
||||||
|
operationId: ' '
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
let duplicate = ir
|
||||||
|
.operations
|
||||||
|
.iter()
|
||||||
|
.filter(|operation| {
|
||||||
|
operation
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "duplicate_operation_id")
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(duplicate.len(), 2);
|
||||||
|
assert!(duplicate.iter().all(|operation| {
|
||||||
|
operation
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.location.pointer.ends_with("/operationId"))
|
||||||
|
}));
|
||||||
|
let blank = ir
|
||||||
|
.operations
|
||||||
|
.iter()
|
||||||
|
.find(|operation| operation.path == "/blank")
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
blank
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "missing_operation_id")
|
||||||
|
);
|
||||||
|
let preview = preview_document(document).unwrap();
|
||||||
|
let blank_preview = preview
|
||||||
|
.groups
|
||||||
|
.iter()
|
||||||
|
.flat_map(|group| &group.operations)
|
||||||
|
.find(|operation| operation.path == "/blank")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
blank_preview
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| finding.code == "missing_operation_id")
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
preview
|
||||||
|
.groups
|
||||||
|
.iter()
|
||||||
|
.flat_map(|group| &group.operations)
|
||||||
|
.find(|operation| operation.path == "/one")
|
||||||
|
.unwrap()
|
||||||
|
.server_urls,
|
||||||
|
vec!["https://path.test"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parameters_keep_exact_path_and_operation_source_pointers() {
|
||||||
|
let openapi = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Parameter locations }
|
||||||
|
paths:
|
||||||
|
/items/{id}:
|
||||||
|
parameters:
|
||||||
|
- { name: id, in: path, required: true, schema: { type: string } }
|
||||||
|
get:
|
||||||
|
operationId: getItem
|
||||||
|
parameters:
|
||||||
|
- { name: filter, in: query, schema: { type: string } }
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(openapi, &NormalizationConfig::default()).unwrap();
|
||||||
|
let parameters = &ir.operations[0].parameters;
|
||||||
|
assert_eq!(
|
||||||
|
parameters[0].source_location.pointer,
|
||||||
|
"/paths/~1items~1{id}/parameters/0"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parameters[1].source_location.pointer,
|
||||||
|
"/paths/~1items~1{id}/get/parameters/0"
|
||||||
|
);
|
||||||
|
assert_ne!(parameters[0].construct_id, parameters[1].construct_id);
|
||||||
|
|
||||||
|
let swagger = r#"
|
||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Swagger parameter locations }
|
||||||
|
paths:
|
||||||
|
/items/{id}:
|
||||||
|
parameters:
|
||||||
|
- { name: id, in: path, required: true, type: string }
|
||||||
|
post:
|
||||||
|
operationId: updateItem
|
||||||
|
parameters:
|
||||||
|
- { name: body, in: body, schema: { type: object } }
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(swagger, &NormalizationConfig::default()).unwrap();
|
||||||
|
let parameter = &ir.operations[0].parameters[0];
|
||||||
|
assert_eq!(
|
||||||
|
parameter.source_location.pointer,
|
||||||
|
"/paths/~1items~1{id}/parameters/0"
|
||||||
|
);
|
||||||
|
assert!(ir.operations[0].request_body_schema.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn operation_parameters_override_path_parameters_by_name_and_location() {
|
||||||
|
let openapi = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: OAS parameter overrides }
|
||||||
|
servers: [{ url: https://example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
parameters:
|
||||||
|
- { name: page, in: query, description: path, schema: { type: integer } }
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
parameters:
|
||||||
|
- { name: page, in: query, description: operation, schema: { type: string } }
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(openapi, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(ir.operations[0].parameters.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
ir.operations[0].parameters[0].description.as_deref(),
|
||||||
|
Some("operation")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ir.operations[0].parameters[0].source_location.pointer,
|
||||||
|
"/paths/~1items/get/parameters/0"
|
||||||
|
);
|
||||||
|
|
||||||
|
let swagger = r#"
|
||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Swagger parameter overrides }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
parameters:
|
||||||
|
- { name: page, in: query, description: path, type: integer }
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
parameters:
|
||||||
|
- { name: page, in: query, description: operation, type: string }
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(swagger, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(ir.operations[0].parameters.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
ir.operations[0].parameters[0].description.as_deref(),
|
||||||
|
Some("operation")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ir.operations[0].parameters[0].source_location.pointer,
|
||||||
|
"/paths/~1items/get/parameters/0"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openapi_parameter_omissions_are_errors_at_the_dropped_item_pointer() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Parameter findings }
|
||||||
|
servers: [{ url: https://example.test }]
|
||||||
|
paths:
|
||||||
|
/items/{id}:
|
||||||
|
parameters:
|
||||||
|
- { name: id, in: path, required: true, schema: { type: string } }
|
||||||
|
- { name: path_cookie, in: cookie }
|
||||||
|
- not-an-object
|
||||||
|
- { $ref: '#/components/parameters/Id' }
|
||||||
|
get:
|
||||||
|
operationId: getItem
|
||||||
|
parameters:
|
||||||
|
- { in: query }
|
||||||
|
- { name: missing_location }
|
||||||
|
- { name: form_value, in: formData }
|
||||||
|
- { name: operation_cookie, in: cookie }
|
||||||
|
- { name: query, in: query, schema: { type: string } }
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
components:
|
||||||
|
parameters:
|
||||||
|
Id: { name: id, in: path, required: true, schema: { type: string } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
ir.operations[0]
|
||||||
|
.parameters
|
||||||
|
.iter()
|
||||||
|
.map(|parameter| parameter.name.as_str())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["id", "query"]
|
||||||
|
);
|
||||||
|
let errors = ir
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| {
|
||||||
|
[
|
||||||
|
"invalid_parameter",
|
||||||
|
"missing_parameter_name",
|
||||||
|
"missing_parameter_location",
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"unresolved_parameter_reference",
|
||||||
|
]
|
||||||
|
.contains(&finding.code.as_str())
|
||||||
|
})
|
||||||
|
.map(|finding| (finding.code.as_str(), finding.location.pointer.as_str()))
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
errors,
|
||||||
|
std::collections::BTreeSet::from([
|
||||||
|
(
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"/paths/~1items~1{id}/parameters/1",
|
||||||
|
),
|
||||||
|
("invalid_parameter", "/paths/~1items~1{id}/parameters/2",),
|
||||||
|
(
|
||||||
|
"missing_parameter_name",
|
||||||
|
"/paths/~1items~1{id}/get/parameters/0",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"missing_parameter_location",
|
||||||
|
"/paths/~1items~1{id}/get/parameters/1",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"/paths/~1items~1{id}/get/parameters/2",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"/paths/~1items~1{id}/get/parameters/3",
|
||||||
|
),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ir.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| {
|
||||||
|
[
|
||||||
|
"invalid_parameter",
|
||||||
|
"missing_parameter_name",
|
||||||
|
"missing_parameter_location",
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"unresolved_parameter_reference",
|
||||||
|
]
|
||||||
|
.contains(&finding.code.as_str())
|
||||||
|
})
|
||||||
|
.all(|finding| finding.severity == ImportFindingSeverity::Error)
|
||||||
|
);
|
||||||
|
|
||||||
|
let legacy = preview_document_legacy_v1(document).unwrap();
|
||||||
|
assert!(!legacy.findings.iter().any(|finding| {
|
||||||
|
[
|
||||||
|
"invalid_parameter",
|
||||||
|
"missing_parameter_name",
|
||||||
|
"missing_parameter_location",
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"unresolved_parameter_reference",
|
||||||
|
]
|
||||||
|
.contains(&finding.code.as_str())
|
||||||
|
}));
|
||||||
|
assert!(
|
||||||
|
!legacy.groups[0].operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "unsupported_cookie_parameter")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn swagger_parameter_omissions_are_errors_at_the_dropped_item_pointer() {
|
||||||
|
let document = r#"
|
||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Swagger parameter findings }
|
||||||
|
host: example.test
|
||||||
|
paths:
|
||||||
|
/items/{id}:
|
||||||
|
parameters:
|
||||||
|
- { name: id, in: path, required: true, type: string }
|
||||||
|
- { name: path_cookie, in: cookie }
|
||||||
|
- not-an-object
|
||||||
|
- { $ref: '#/parameters/Id' }
|
||||||
|
get:
|
||||||
|
operationId: getItem
|
||||||
|
parameters:
|
||||||
|
- { in: query }
|
||||||
|
- { name: missing_location }
|
||||||
|
- { name: form_value, in: formData }
|
||||||
|
- { name: operation_cookie, in: cookie }
|
||||||
|
- { name: query, in: query, type: string }
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
parameters:
|
||||||
|
Id: { name: id, in: path, required: true, type: string }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
ir.operations[0]
|
||||||
|
.parameters
|
||||||
|
.iter()
|
||||||
|
.map(|parameter| parameter.name.as_str())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["id", "query"]
|
||||||
|
);
|
||||||
|
let errors = ir
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| {
|
||||||
|
[
|
||||||
|
"invalid_parameter",
|
||||||
|
"missing_parameter_name",
|
||||||
|
"missing_parameter_location",
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"unresolved_parameter_reference",
|
||||||
|
]
|
||||||
|
.contains(&finding.code.as_str())
|
||||||
|
})
|
||||||
|
.map(|finding| (finding.code.as_str(), finding.location.pointer.as_str()))
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
errors,
|
||||||
|
std::collections::BTreeSet::from([
|
||||||
|
(
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"/paths/~1items~1{id}/parameters/1",
|
||||||
|
),
|
||||||
|
("invalid_parameter", "/paths/~1items~1{id}/parameters/2",),
|
||||||
|
(
|
||||||
|
"missing_parameter_name",
|
||||||
|
"/paths/~1items~1{id}/get/parameters/0",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"missing_parameter_location",
|
||||||
|
"/paths/~1items~1{id}/get/parameters/1",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"/paths/~1items~1{id}/get/parameters/2",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"/paths/~1items~1{id}/get/parameters/3",
|
||||||
|
),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ir.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| {
|
||||||
|
[
|
||||||
|
"invalid_parameter",
|
||||||
|
"missing_parameter_name",
|
||||||
|
"missing_parameter_location",
|
||||||
|
"unsupported_parameter_location",
|
||||||
|
"unsupported_cookie_parameter",
|
||||||
|
"unresolved_parameter_reference",
|
||||||
|
]
|
||||||
|
.contains(&finding.code.as_str())
|
||||||
|
})
|
||||||
|
.all(|finding| finding.severity == ImportFindingSeverity::Error)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openapi_invalid_servers_and_tags_keep_valid_siblings_with_exact_findings() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Server and tag findings }
|
||||||
|
servers:
|
||||||
|
- { url: https://document.test/ }
|
||||||
|
- not-an-object
|
||||||
|
- {}
|
||||||
|
- { url: 42 }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
servers:
|
||||||
|
- { url: https://path.test/ }
|
||||||
|
- false
|
||||||
|
- {}
|
||||||
|
get:
|
||||||
|
operationId: getItems
|
||||||
|
servers:
|
||||||
|
- { url: https://operation.test/ }
|
||||||
|
- { url: false }
|
||||||
|
tags: [items, 42, {}, null]
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(ir.source.servers, vec!["https://document.test"]);
|
||||||
|
assert_eq!(ir.operations[0].servers, vec!["https://operation.test"]);
|
||||||
|
assert_eq!(ir.operations[0].tags, vec!["items"]);
|
||||||
|
|
||||||
|
let server_errors = ir
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| finding.code == "invalid_server")
|
||||||
|
.map(|finding| finding.location.pointer.as_str())
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
server_errors,
|
||||||
|
std::collections::BTreeSet::from([
|
||||||
|
"/servers/1",
|
||||||
|
"/servers/2",
|
||||||
|
"/servers/3",
|
||||||
|
"/paths/~1items/servers/1",
|
||||||
|
"/paths/~1items/servers/2",
|
||||||
|
"/paths/~1items/get/servers/1",
|
||||||
|
])
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ir.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| finding.code == "invalid_server")
|
||||||
|
.all(|finding| finding.severity == ImportFindingSeverity::Error)
|
||||||
|
);
|
||||||
|
let tag_warnings = ir
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| finding.code == "invalid_tag")
|
||||||
|
.map(|finding| finding.location.pointer.as_str())
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
tag_warnings,
|
||||||
|
std::collections::BTreeSet::from([
|
||||||
|
"/paths/~1items/get/tags/1",
|
||||||
|
"/paths/~1items/get/tags/2",
|
||||||
|
"/paths/~1items/get/tags/3",
|
||||||
|
])
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ir.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| finding.code == "invalid_tag")
|
||||||
|
.all(|finding| finding.severity == ImportFindingSeverity::Warning)
|
||||||
|
);
|
||||||
|
|
||||||
|
let legacy = preview_document_legacy_v1(document).unwrap();
|
||||||
|
assert_eq!(legacy.source.servers, vec!["https://document.test"]);
|
||||||
|
assert_eq!(
|
||||||
|
legacy.groups[0].operations[0].server_urls,
|
||||||
|
vec!["https://operation.test"]
|
||||||
|
);
|
||||||
|
assert_eq!(legacy.groups[0].operations[0].category, "items");
|
||||||
|
assert!(
|
||||||
|
!legacy.findings.iter().any(|finding| {
|
||||||
|
["invalid_server", "invalid_tag"].contains(&finding.code.as_str())
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn swagger_invalid_tags_keep_valid_siblings_with_exact_warning_pointers() {
|
||||||
|
let document = r#"
|
||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Swagger tag findings }
|
||||||
|
host: example.test
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: getItems
|
||||||
|
tags: [items, 42, {}, null]
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(ir.operations[0].tags, vec!["items"]);
|
||||||
|
let warnings = ir
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| finding.code == "invalid_tag")
|
||||||
|
.map(|finding| finding.location.pointer.as_str())
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
warnings,
|
||||||
|
std::collections::BTreeSet::from([
|
||||||
|
"/paths/~1items/get/tags/1",
|
||||||
|
"/paths/~1items/get/tags/2",
|
||||||
|
"/paths/~1items/get/tags/3",
|
||||||
|
])
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ir.findings
|
||||||
|
.iter()
|
||||||
|
.filter(|finding| finding.code == "invalid_tag")
|
||||||
|
.all(|finding| finding.severity == ImportFindingSeverity::Warning)
|
||||||
|
);
|
||||||
|
|
||||||
|
let legacy = preview_document_legacy_v1(document).unwrap();
|
||||||
|
assert_eq!(legacy.groups[0].operations[0].category, "items");
|
||||||
|
assert!(
|
||||||
|
!legacy
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "invalid_tag")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn v2_response_selection_uses_all_success_codes_then_openapi_wildcard_and_default() {
|
||||||
|
let openapi = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Response priority }
|
||||||
|
paths:
|
||||||
|
/numeric:
|
||||||
|
get:
|
||||||
|
operationId: numeric
|
||||||
|
responses:
|
||||||
|
'200': { description: no schema }
|
||||||
|
'203': { description: accepted, content: { application/json: { schema: { type: string } } } }
|
||||||
|
'206': { description: partial, content: { application/json: { schema: { type: integer } } } }
|
||||||
|
/wildcard:
|
||||||
|
get:
|
||||||
|
operationId: wildcard
|
||||||
|
responses:
|
||||||
|
'200': { description: unsupported, content: { text/plain: { schema: { type: string } } } }
|
||||||
|
'2xX': { description: wildcard, content: { application/json: { schema: { type: boolean } } } }
|
||||||
|
default: { description: fallback, content: { application/json: { schema: { type: string } } } }
|
||||||
|
/default:
|
||||||
|
get:
|
||||||
|
operationId: fallback
|
||||||
|
responses:
|
||||||
|
'203': { description: no schema, content: { application/json: {} } }
|
||||||
|
default: { description: fallback, content: { application/json: { schema: { type: number } } } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(openapi, &NormalizationConfig::default()).unwrap();
|
||||||
|
let response_pointer = |path: &str| {
|
||||||
|
ir.operations
|
||||||
|
.iter()
|
||||||
|
.find(|operation| operation.path == path)
|
||||||
|
.unwrap()
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.location
|
||||||
|
.pointer
|
||||||
|
.as_str()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
response_pointer("/numeric"),
|
||||||
|
"/paths/~1numeric/get/responses/203/content/application~1json/schema"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response_pointer("/wildcard"),
|
||||||
|
"/paths/~1wildcard/get/responses/2xX/content/application~1json/schema"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response_pointer("/default"),
|
||||||
|
"/paths/~1default/get/responses/default/content/application~1json/schema"
|
||||||
|
);
|
||||||
|
|
||||||
|
let swagger = r#"
|
||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Swagger response priority }
|
||||||
|
paths:
|
||||||
|
/numeric:
|
||||||
|
get:
|
||||||
|
operationId: numeric
|
||||||
|
responses:
|
||||||
|
'200': { description: no schema }
|
||||||
|
'206': { description: partial, schema: { type: integer } }
|
||||||
|
default: { description: fallback, schema: { type: string } }
|
||||||
|
/no-wildcard:
|
||||||
|
get:
|
||||||
|
operationId: noWildcard
|
||||||
|
responses:
|
||||||
|
'2XX': { description: ignored wildcard, schema: { type: string } }
|
||||||
|
default: { description: fallback, schema: { type: boolean } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(swagger, &NormalizationConfig::default()).unwrap();
|
||||||
|
let response_pointer = |path: &str| {
|
||||||
|
ir.operations
|
||||||
|
.iter()
|
||||||
|
.find(|operation| operation.path == path)
|
||||||
|
.unwrap()
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.location
|
||||||
|
.pointer
|
||||||
|
.as_str()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
response_pointer("/numeric"),
|
||||||
|
"/paths/~1numeric/get/responses/206/schema"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response_pointer("/no-wildcard"),
|
||||||
|
"/paths/~1no-wildcard/get/responses/default/schema"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schemas_keep_selected_source_pointers_and_pointer_derived_ids() {
|
||||||
|
let openapi = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Schema locations }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
post:
|
||||||
|
operationId: updateItems
|
||||||
|
requestBody:
|
||||||
|
content:
|
||||||
|
application/vnd.example+json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
a/b: { type: string }
|
||||||
|
a~1b: { type: string }
|
||||||
|
responses:
|
||||||
|
'201':
|
||||||
|
description: created
|
||||||
|
content:
|
||||||
|
application/vnd.example+json:
|
||||||
|
schema: { type: integer }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(openapi, &NormalizationConfig::default()).unwrap();
|
||||||
|
let operation = &ir.operations[0];
|
||||||
|
assert_eq!(
|
||||||
|
operation
|
||||||
|
.request_body_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.location
|
||||||
|
.pointer,
|
||||||
|
"/paths/~1items/post/requestBody/content/application~1vnd.example+json/schema"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
operation.response_schema.as_ref().unwrap().location.pointer,
|
||||||
|
"/paths/~1items/post/responses/201/content/application~1vnd.example+json/schema"
|
||||||
|
);
|
||||||
|
let crank_import::rest::NormalizedSchemaKind::Object { properties, .. } =
|
||||||
|
&operation.request_body_schema.as_ref().unwrap().kind
|
||||||
|
else {
|
||||||
|
panic!("request schema must be object");
|
||||||
|
};
|
||||||
|
assert_ne!(
|
||||||
|
properties["a/b"].construct_id,
|
||||||
|
properties["a~1b"].construct_id
|
||||||
|
);
|
||||||
|
|
||||||
|
let swagger = r#"
|
||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Swagger schema locations }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
post:
|
||||||
|
operationId: updateItems
|
||||||
|
parameters:
|
||||||
|
- in: body
|
||||||
|
name: body
|
||||||
|
schema: { type: object }
|
||||||
|
responses:
|
||||||
|
'201': { description: created, schema: { type: integer } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize_document(swagger, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
ir.operations[0]
|
||||||
|
.request_body_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.location
|
||||||
|
.pointer,
|
||||||
|
"/paths/~1items/post/parameters/0/schema"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ir.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.location
|
||||||
|
.pointer,
|
||||||
|
"/paths/~1items/post/responses/201/schema"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolves_local_reference_graph_and_preserves_external_blocker() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: References }
|
||||||
|
paths:
|
||||||
|
/ok:
|
||||||
|
parameters:
|
||||||
|
- { $ref: '#/components/parameters/Id' }
|
||||||
|
get:
|
||||||
|
operationId: getOk
|
||||||
|
requestBody: { $ref: '#/components/requestBodies/Body' }
|
||||||
|
responses:
|
||||||
|
'200': { $ref: '#/components/responses/Ok' }
|
||||||
|
/reused:
|
||||||
|
$ref: '#/components/pathItems/Reusable'
|
||||||
|
components:
|
||||||
|
schemas:
|
||||||
|
Loop: { $ref: '#/components/schemas/Loop' }
|
||||||
|
Remote: { $ref: 'https://example.test/schema.json#/Remote' }
|
||||||
|
parameters:
|
||||||
|
Id: { $ref: '#/components/parameters/Id' }
|
||||||
|
requestBodies:
|
||||||
|
Body: { $ref: '#/components/requestBodies/Body' }
|
||||||
|
responses:
|
||||||
|
Ok: { $ref: '#/components/responses/Ok' }
|
||||||
|
pathItems:
|
||||||
|
Reusable: { $ref: '#/components/pathItems/Reusable' }
|
||||||
|
"#;
|
||||||
|
let first = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
let second = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_vec(&first).unwrap(),
|
||||||
|
serde_json::to_vec(&second).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(first.unresolved_references.len(), 1);
|
||||||
|
let references = first
|
||||||
|
.unresolved_references
|
||||||
|
.iter()
|
||||||
|
.map(|reference| (reference.uri.as_str(), reference.location.pointer.as_str()))
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert!(references.contains(&(
|
||||||
|
"https://example.test/schema.json#/Remote",
|
||||||
|
"/components/schemas/Remote/$ref"
|
||||||
|
)));
|
||||||
|
assert!(first.reference_graph.edges.len() >= 5);
|
||||||
|
assert!(
|
||||||
|
first
|
||||||
|
.reference_graph
|
||||||
|
.edges
|
||||||
|
.iter()
|
||||||
|
.any(|edge| edge.recursive)
|
||||||
|
);
|
||||||
|
assert!(first.unresolved_references.iter().all(|reference| {
|
||||||
|
reference.construct_id
|
||||||
|
== format!(
|
||||||
|
"reference:{}",
|
||||||
|
reference
|
||||||
|
.location
|
||||||
|
.pointer
|
||||||
|
.replace('~', "~0")
|
||||||
|
.replace('/', "~1")
|
||||||
|
)
|
||||||
|
&& first.coverage.iter().any(|entry| {
|
||||||
|
entry.construct_id == reference.construct_id
|
||||||
|
&& entry.location == reference.location
|
||||||
|
})
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,615 @@
|
|||||||
|
mod unit {
|
||||||
|
use crank_core::HttpMethod;
|
||||||
|
use crank_import::rest::{ImportParseError, preview_document, preview_document_legacy_v1};
|
||||||
|
use crank_schema::SchemaKind;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
const OPENAPI3: &str = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info:
|
||||||
|
title: Frankfurter API
|
||||||
|
servers:
|
||||||
|
- url: https://api.frankfurter.dev
|
||||||
|
paths:
|
||||||
|
/v2/latest:
|
||||||
|
get:
|
||||||
|
operationId: getLatestRates
|
||||||
|
summary: Получить последние курсы
|
||||||
|
description: Возвращает последние курсы валют для базовой валюты.
|
||||||
|
tags: [currency]
|
||||||
|
parameters:
|
||||||
|
- name: base
|
||||||
|
in: query
|
||||||
|
required: true
|
||||||
|
schema:
|
||||||
|
type: string
|
||||||
|
- name: symbols
|
||||||
|
in: query
|
||||||
|
schema:
|
||||||
|
type: string
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: OK
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
required: [amount, base]
|
||||||
|
properties:
|
||||||
|
amount:
|
||||||
|
type: number
|
||||||
|
base:
|
||||||
|
type: string
|
||||||
|
"#;
|
||||||
|
|
||||||
|
const SWAGGER2: &str = r#"
|
||||||
|
swagger: "2.0"
|
||||||
|
info:
|
||||||
|
title: Pet API
|
||||||
|
host: petstore.example.com
|
||||||
|
basePath: /api
|
||||||
|
schemes: [https]
|
||||||
|
paths:
|
||||||
|
/pets/{id}:
|
||||||
|
get:
|
||||||
|
operationId: getPet
|
||||||
|
summary: Получить питомца
|
||||||
|
tags: [pets]
|
||||||
|
parameters:
|
||||||
|
- name: id
|
||||||
|
in: path
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: OK
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/Pet'
|
||||||
|
definitions:
|
||||||
|
Pet:
|
||||||
|
type: object
|
||||||
|
required: [id, name]
|
||||||
|
properties:
|
||||||
|
id:
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
type: string
|
||||||
|
"#;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn previews_openapi3_rest_operations_grouped_by_tag() {
|
||||||
|
let preview = preview_document(OPENAPI3).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(preview.source.format, "openapi");
|
||||||
|
assert_eq!(preview.source.servers, vec!["https://api.frankfurter.dev"]);
|
||||||
|
assert_eq!(preview.groups.len(), 1);
|
||||||
|
assert_eq!(preview.groups[0].key, "currency");
|
||||||
|
let operation = &preview.groups[0].operations[0];
|
||||||
|
assert_eq!(operation.method, HttpMethod::Get);
|
||||||
|
assert_eq!(operation.suggested_name, "get_latest_rates");
|
||||||
|
assert_eq!(operation.input_fields, 2);
|
||||||
|
assert_eq!(operation.output_fields, 2);
|
||||||
|
assert_eq!(operation.draft.target.path_template, "/v2/latest");
|
||||||
|
assert_eq!(operation.draft.input_mapping.rules.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn previews_swagger2_and_resolves_local_definitions() {
|
||||||
|
let preview = preview_document(SWAGGER2).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(preview.source.format, "swagger");
|
||||||
|
assert_eq!(
|
||||||
|
preview.source.servers,
|
||||||
|
vec!["https://petstore.example.com/api"]
|
||||||
|
);
|
||||||
|
let operation = &preview.groups[0].operations[0];
|
||||||
|
assert_eq!(operation.suggested_name, "get_pet");
|
||||||
|
assert_eq!(operation.input_fields, 1);
|
||||||
|
assert_eq!(operation.output_fields, 2);
|
||||||
|
assert_eq!(operation.draft.target.path_template, "/pets/{id}");
|
||||||
|
assert_eq!(
|
||||||
|
operation.draft.input_mapping.rules[0].target,
|
||||||
|
"$.request.path.id"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalized_schema_descriptions_round_trip_to_draft_fields_and_arrays_default_to_string_items()
|
||||||
|
{
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Schema descriptions }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
post:
|
||||||
|
operationId: createItem
|
||||||
|
parameters:
|
||||||
|
- name: filter
|
||||||
|
in: query
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
description: Filter input
|
||||||
|
properties:
|
||||||
|
nested:
|
||||||
|
type: object
|
||||||
|
description: Nested filter
|
||||||
|
properties:
|
||||||
|
labels:
|
||||||
|
type: array
|
||||||
|
description: Labels without item schema
|
||||||
|
requestBody:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
description: Request root
|
||||||
|
properties:
|
||||||
|
payload:
|
||||||
|
type: object
|
||||||
|
description: Request payload
|
||||||
|
properties:
|
||||||
|
enabled: { type: boolean, description: Enable the item }
|
||||||
|
responses:
|
||||||
|
'201':
|
||||||
|
description: created
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
description: Created item response
|
||||||
|
properties:
|
||||||
|
id: { type: string, description: New item identifier }
|
||||||
|
"#;
|
||||||
|
let preview = preview_document(document).unwrap();
|
||||||
|
let operation = &preview.groups[0].operations[0];
|
||||||
|
let filter = operation.draft.input_schema.field("filter").unwrap();
|
||||||
|
assert_eq!(filter.description.as_deref(), Some("Filter input"));
|
||||||
|
let nested = filter.field("nested").unwrap();
|
||||||
|
assert_eq!(nested.description.as_deref(), Some("Nested filter"));
|
||||||
|
let labels = nested.field("labels").unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
labels.description.as_deref(),
|
||||||
|
Some("Labels without item schema")
|
||||||
|
);
|
||||||
|
assert_eq!(labels.kind, SchemaKind::Array);
|
||||||
|
assert_eq!(labels.items.as_ref().unwrap().kind, SchemaKind::String);
|
||||||
|
|
||||||
|
let payload = operation.draft.input_schema.field("payload").unwrap();
|
||||||
|
assert_eq!(payload.description.as_deref(), Some("Request payload"));
|
||||||
|
assert_eq!(
|
||||||
|
payload.field("enabled").unwrap().description.as_deref(),
|
||||||
|
Some("Enable the item")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
operation.draft.output_schema.description.as_deref(),
|
||||||
|
Some("Ответ API")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
operation
|
||||||
|
.draft
|
||||||
|
.output_schema
|
||||||
|
.field("id")
|
||||||
|
.unwrap()
|
||||||
|
.description
|
||||||
|
.as_deref(),
|
||||||
|
Some("New item identifier")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_v1_keeps_frozen_response_priority_when_v2_selects_later_success_code() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Response replay priority }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200': { description: no schema }
|
||||||
|
'203': { description: selected by v2, content: { application/json: { schema: { type: string } } } }
|
||||||
|
'206': { description: later success, content: { application/json: { schema: { type: integer } } } }
|
||||||
|
default: { description: legacy fallback, content: { application/json: { schema: { type: boolean } } } }
|
||||||
|
"#;
|
||||||
|
let v2 = preview_document(document).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
v2.groups[0].operations[0].draft.output_schema.kind,
|
||||||
|
SchemaKind::String
|
||||||
|
);
|
||||||
|
|
||||||
|
let legacy = preview_document_legacy_v1(document).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
legacy.groups[0].operations[0].draft.output_schema.kind,
|
||||||
|
SchemaKind::Boolean
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_v1_projection_matches_baseline_local_ref_and_composition_behavior() {
|
||||||
|
let document = r#"
|
||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Legacy }
|
||||||
|
paths:
|
||||||
|
/pets:
|
||||||
|
get:
|
||||||
|
responses:
|
||||||
|
'200': { description: OK, schema: { $ref: '#/definitions/Pet' } }
|
||||||
|
definitions:
|
||||||
|
Pet:
|
||||||
|
allOf:
|
||||||
|
- type: object
|
||||||
|
properties: { id: { type: string }, name: { type: string } }
|
||||||
|
"#;
|
||||||
|
let preview = preview_document_legacy_v1(document).unwrap();
|
||||||
|
assert_eq!(preview.groups[0].operations[0].output_fields, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_v1_resolves_object_level_local_refs_before_projection() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Legacy object refs }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: getItem
|
||||||
|
parameters: [{ $ref: '#/components/parameters/Id' }]
|
||||||
|
requestBody: { $ref: '#/components/requestBodies/Body' }
|
||||||
|
responses:
|
||||||
|
'200': { $ref: '#/components/responses/Ok' }
|
||||||
|
components:
|
||||||
|
parameters:
|
||||||
|
Id:
|
||||||
|
name: id
|
||||||
|
in: query
|
||||||
|
schema: { type: string }
|
||||||
|
requestBodies:
|
||||||
|
Body:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/Input' }
|
||||||
|
responses:
|
||||||
|
Ok:
|
||||||
|
description: ok
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/Output' }
|
||||||
|
schemas:
|
||||||
|
Input:
|
||||||
|
type: object
|
||||||
|
properties: { name: { type: string } }
|
||||||
|
Output:
|
||||||
|
type: object
|
||||||
|
properties: { result: { type: string } }
|
||||||
|
"#;
|
||||||
|
let preview = preview_document_legacy_v1(document).unwrap();
|
||||||
|
let operation = &preview.groups[0].operations[0];
|
||||||
|
assert_eq!(operation.input_fields, 2);
|
||||||
|
assert_eq!(operation.output_fields, 1);
|
||||||
|
assert!(operation.draft.input_schema.fields.contains_key("id"));
|
||||||
|
assert!(operation.draft.input_schema.fields.contains_key("name"));
|
||||||
|
assert!(operation.draft.output_schema.fields.contains_key("result"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_v1_rejects_hostile_alias_input_before_decode() {
|
||||||
|
let document = format!(
|
||||||
|
"openapi: 3.0.3\ninfo: {{ title: aliases }}\npaths: {{}}\nitems: [{}]",
|
||||||
|
std::iter::repeat_n("*bomb", 129)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
preview_document_legacy_v1(&document),
|
||||||
|
Err(ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_v1_keeps_baseline_version_dispatch_and_does_not_expand_path_item_refs() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 9.9.0
|
||||||
|
info: { title: Legacy dispatch }
|
||||||
|
paths:
|
||||||
|
/valid:
|
||||||
|
get:
|
||||||
|
operationId: valid
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
/referenced:
|
||||||
|
$ref: '#/x-path-items/referenced'
|
||||||
|
x-path-items:
|
||||||
|
referenced:
|
||||||
|
post:
|
||||||
|
operationId: mustNotAppear
|
||||||
|
responses: { '201': { description: ok } }
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let preview = preview_document_legacy_v1(document).unwrap();
|
||||||
|
assert_eq!(preview.source.version.as_deref(), Some("9.9.0"));
|
||||||
|
let keys = preview
|
||||||
|
.groups
|
||||||
|
.iter()
|
||||||
|
.flat_map(|group| group.operations.iter())
|
||||||
|
.map(|operation| operation.key.as_str())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(keys, vec!["GET /valid"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_v1_preview_exactly_matches_baseline_for_path_refs_servers_and_malformed_operations() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Legacy parity }
|
||||||
|
servers: [{ url: https://document.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
parameters:
|
||||||
|
- { $ref: '#/components/parameters/Query' }
|
||||||
|
- { name: session, in: cookie }
|
||||||
|
servers: [{ url: https://path.test }]
|
||||||
|
head: { responses: {} }
|
||||||
|
get: false
|
||||||
|
components:
|
||||||
|
parameters:
|
||||||
|
Query: { name: q, in: query, schema: { type: string } }
|
||||||
|
"#;
|
||||||
|
let preview = preview_document_legacy_v1(document).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(preview).unwrap(),
|
||||||
|
json!({
|
||||||
|
"source": {
|
||||||
|
"format": "openapi",
|
||||||
|
"version": "3.0.3",
|
||||||
|
"title": "Legacy parity",
|
||||||
|
"servers": ["https://document.test"]
|
||||||
|
},
|
||||||
|
"groups": [{
|
||||||
|
"key": "imported_operation",
|
||||||
|
"title": "Без группы",
|
||||||
|
"operations": [{
|
||||||
|
"key": "GET /items",
|
||||||
|
"method": "GET",
|
||||||
|
"path": "/items",
|
||||||
|
"suggested_name": "g_e_t_items",
|
||||||
|
"suggested_display_name": "G E T Items",
|
||||||
|
"description": "Выполняет GET /items",
|
||||||
|
"category": "imported",
|
||||||
|
"input_fields": 1,
|
||||||
|
"output_fields": 0,
|
||||||
|
"server_urls": ["https://document.test"],
|
||||||
|
"findings": [
|
||||||
|
{
|
||||||
|
"code": "missing_operation_id",
|
||||||
|
"severity": "warning",
|
||||||
|
"message": "У метода нет operationId, имя инструмента будет сгенерировано из метода и пути.",
|
||||||
|
"operation_key": "GET /items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"code": "missing_summary",
|
||||||
|
"severity": "warning",
|
||||||
|
"message": "У метода нет summary, отображаемое имя будет сгенерировано автоматически.",
|
||||||
|
"operation_key": "GET /items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"code": "missing_description",
|
||||||
|
"severity": "warning",
|
||||||
|
"message": "У метода нет description. Перед публикацией лучше описать, когда агенту стоит вызывать этот инструмент.",
|
||||||
|
"operation_key": "GET /items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"code": "missing_response_schema",
|
||||||
|
"severity": "warning",
|
||||||
|
"message": "У метода не найдена схема успешного ответа, результат будет описан как общий объект.",
|
||||||
|
"operation_key": "GET /items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"code": "parameter_descriptions_missing",
|
||||||
|
"severity": "warning",
|
||||||
|
"message": "У 1 входных параметров нет описания. Модели будет сложнее понять, какие значения туда передавать.",
|
||||||
|
"operation_key": "GET /items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"code": "empty_output_schema",
|
||||||
|
"severity": "warning",
|
||||||
|
"message": "В ответе не найдено отдельных полей. Перед публикацией проверьте схему ответа и маппинг результата.",
|
||||||
|
"operation_key": "GET /items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"code": "weak_tool_description",
|
||||||
|
"severity": "warning",
|
||||||
|
"message": "Описание инструмента слишком короткое или техническое. Перед публикацией добавьте, когда агент должен вызывать инструмент и что будет в успешном ответе.",
|
||||||
|
"operation_key": "GET /items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"code": "weak_tool_name",
|
||||||
|
"severity": "warning",
|
||||||
|
"message": "Имя инструмента `g_e_t_items` выглядит слишком общим. Лучше использовать имя с конкретным действием и объектом.",
|
||||||
|
"operation_key": "GET /items"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"draft": {
|
||||||
|
"name": "g_e_t_items",
|
||||||
|
"display_name": "G E T Items",
|
||||||
|
"category": "imported",
|
||||||
|
"target": {
|
||||||
|
"base_url": "https://document.test",
|
||||||
|
"method": "GET",
|
||||||
|
"path_template": "/items"
|
||||||
|
},
|
||||||
|
"input_schema": {
|
||||||
|
"type": "object",
|
||||||
|
"description": "Входные параметры MCP-инструмента",
|
||||||
|
"required": true,
|
||||||
|
"nullable": false,
|
||||||
|
"fields": {
|
||||||
|
"q": {
|
||||||
|
"type": "string",
|
||||||
|
"required": false,
|
||||||
|
"nullable": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"output_schema": {
|
||||||
|
"type": "object",
|
||||||
|
"description": "Ответ API",
|
||||||
|
"required": true,
|
||||||
|
"nullable": false
|
||||||
|
},
|
||||||
|
"input_mapping": {
|
||||||
|
"rules": [{
|
||||||
|
"source": "$.mcp.q",
|
||||||
|
"target": "$.request.query.q",
|
||||||
|
"required": false
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
"output_mapping": {
|
||||||
|
"rules": [{
|
||||||
|
"source": "$.response.body",
|
||||||
|
"target": "$.output",
|
||||||
|
"required": true
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
"tool_description": {
|
||||||
|
"title": "G E T Items",
|
||||||
|
"description": "Выполняет GET /items",
|
||||||
|
"examples": [{ "input": {} }]
|
||||||
|
},
|
||||||
|
"wizard_state": {}
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}],
|
||||||
|
"findings": []
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reports_missing_descriptions_as_recommendations() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Minimal API }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
responses:
|
||||||
|
'204': { description: Empty }
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let preview = preview_document(document).unwrap();
|
||||||
|
let operation = &preview.groups[0].operations[0];
|
||||||
|
let codes = operation
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.map(|finding| finding.code.as_str())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert!(codes.contains(&"missing_operation_id"));
|
||||||
|
assert!(codes.contains(&"missing_summary"));
|
||||||
|
assert!(codes.contains(&"missing_description"));
|
||||||
|
assert!(codes.contains(&"missing_response_schema"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expands_json_request_body_object_into_tool_inputs() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: CRM API }
|
||||||
|
servers:
|
||||||
|
- url: https://crm.example.test
|
||||||
|
paths:
|
||||||
|
/leads:
|
||||||
|
post:
|
||||||
|
operationId: createLead
|
||||||
|
summary: Создать лид
|
||||||
|
description: Создает лид в CRM.
|
||||||
|
tags: [crm]
|
||||||
|
requestBody:
|
||||||
|
required: true
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
required: [email]
|
||||||
|
properties:
|
||||||
|
email: { type: string }
|
||||||
|
name: { type: string }
|
||||||
|
responses:
|
||||||
|
'201':
|
||||||
|
description: Created
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
id: { type: string }
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let preview = preview_document(document).unwrap();
|
||||||
|
let operation = &preview.groups[0].operations[0];
|
||||||
|
let input_fields = &operation.draft.input_schema.fields;
|
||||||
|
let targets = operation
|
||||||
|
.draft
|
||||||
|
.input_mapping
|
||||||
|
.rules
|
||||||
|
.iter()
|
||||||
|
.map(|rule| rule.target.as_str())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert!(input_fields.contains_key("email"));
|
||||||
|
assert!(input_fields.contains_key("name"));
|
||||||
|
assert!(targets.contains(&"$.request.body.email"));
|
||||||
|
assert!(targets.contains(&"$.request.body.name"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reports_tool_quality_recommendations_for_imported_operations() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Wide API }
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: getItems
|
||||||
|
summary: Get items
|
||||||
|
description: Get items.
|
||||||
|
parameters:
|
||||||
|
- name: page
|
||||||
|
in: query
|
||||||
|
schema: { type: integer }
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: OK
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
field01: { type: string }
|
||||||
|
field02: { type: string }
|
||||||
|
field03: { type: string }
|
||||||
|
field04: { type: string }
|
||||||
|
field05: { type: string }
|
||||||
|
field06: { type: string }
|
||||||
|
field07: { type: string }
|
||||||
|
field08: { type: string }
|
||||||
|
field09: { type: string }
|
||||||
|
field10: { type: string }
|
||||||
|
field11: { type: string }
|
||||||
|
field12: { type: string }
|
||||||
|
field13: { type: string }
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let preview = preview_document(document).unwrap();
|
||||||
|
let operation = &preview.groups[0].operations[0];
|
||||||
|
let codes = operation
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.map(|finding| finding.code.as_str())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert!(codes.contains(&"parameter_descriptions_missing"));
|
||||||
|
assert!(codes.contains(&"weak_tool_description"));
|
||||||
|
assert!(codes.contains(&"weak_tool_name"));
|
||||||
|
assert!(codes.contains(&"too_many_output_fields"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,793 @@
|
|||||||
|
use crank_import::rest::{
|
||||||
|
ExternalDocumentSnapshot, ImportFindingSeverity, NormalizationConfig, NormalizedSchemaKind,
|
||||||
|
SourceDigest, external_reference_uris, normalize_verified_bundle, normalize_verified_document,
|
||||||
|
preview_from_ir, reference_uris,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn digest(byte: char) -> SourceDigest {
|
||||||
|
SourceDigest::parse(byte.to_string().repeat(64)).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize(document: &str) -> crank_import::rest::NormalizedIr {
|
||||||
|
normalize_verified_document(document, digest('a'), &NormalizationConfig::default()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolves_local_schema_and_object_references_with_rfc6901_escaping() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Local refs }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items/{id}:
|
||||||
|
get:
|
||||||
|
operationId: getItem
|
||||||
|
parameters:
|
||||||
|
- { $ref: '#/components/parameters/Id' }
|
||||||
|
responses:
|
||||||
|
'200': { $ref: '#/components/responses/Ok' }
|
||||||
|
components:
|
||||||
|
parameters:
|
||||||
|
Id: { name: id, in: path, required: true, schema: { type: string } }
|
||||||
|
responses:
|
||||||
|
Ok:
|
||||||
|
description: ok
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/a~1b~0c' }
|
||||||
|
schemas:
|
||||||
|
a/b~c:
|
||||||
|
type: object
|
||||||
|
required: [id]
|
||||||
|
properties: { id: { type: string } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
let operation = &ir.operations[0];
|
||||||
|
assert_eq!(operation.parameters.len(), 1);
|
||||||
|
assert_eq!(operation.parameters[0].name, "id");
|
||||||
|
assert!(matches!(
|
||||||
|
operation.response_schema.as_ref().map(|schema| &schema.kind),
|
||||||
|
Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id")
|
||||||
|
));
|
||||||
|
assert!(ir.unresolved_references.is_empty());
|
||||||
|
assert_eq!(ir.reference_graph.edges.len(), 3);
|
||||||
|
assert!(
|
||||||
|
ir.findings
|
||||||
|
.iter()
|
||||||
|
.chain(operation.findings.iter())
|
||||||
|
.all(|finding| finding.code != "unresolved_reference")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn broken_reference_blocks_only_affected_candidate_and_full_preview_remains_visible() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Partial graph }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/broken:
|
||||||
|
get:
|
||||||
|
operationId: broken
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: nope
|
||||||
|
content: { application/json: { schema: { $ref: '#/components/schemas/Missing' } } }
|
||||||
|
/healthy:
|
||||||
|
get:
|
||||||
|
operationId: healthy
|
||||||
|
responses: { '204': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
let broken = ir
|
||||||
|
.operations
|
||||||
|
.iter()
|
||||||
|
.find(|operation| operation.path == "/broken")
|
||||||
|
.unwrap();
|
||||||
|
let healthy = ir
|
||||||
|
.operations
|
||||||
|
.iter()
|
||||||
|
.find(|operation| operation.path == "/healthy")
|
||||||
|
.unwrap();
|
||||||
|
assert!(broken.findings.iter().any(|finding| {
|
||||||
|
finding.code == "reference_target_missing"
|
||||||
|
&& finding.severity == ImportFindingSeverity::Error
|
||||||
|
}));
|
||||||
|
assert!(healthy.findings.is_empty());
|
||||||
|
let preview = preview_from_ir(&ir);
|
||||||
|
assert_eq!(
|
||||||
|
preview
|
||||||
|
.groups
|
||||||
|
.iter()
|
||||||
|
.map(|group| group.operations.len())
|
||||||
|
.sum::<usize>(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn external_references_are_default_deny_without_network_or_snapshot() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: External deny }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content: { application/json: { schema: { $ref: 'https://schemas.example.test/root.yaml#/Item' } } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
assert!(
|
||||||
|
ir.operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "external_reference_disabled")
|
||||||
|
);
|
||||||
|
assert!(ir.reference_graph.edges.is_empty());
|
||||||
|
assert!(ir.reference_graph.dependency_digests.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolves_supplied_external_snapshot_and_relative_chain_deterministically() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: External snapshots }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content: { application/json: { schema: { $ref: 'https://schemas.example.test/root.yaml#/Item' } } }
|
||||||
|
"#;
|
||||||
|
let snapshots = vec![
|
||||||
|
ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: "https://schemas.example.test/root.yaml".to_owned(),
|
||||||
|
digest: digest('b'),
|
||||||
|
document: "Item: { $ref: 'child.yaml#/Child' }".to_owned(),
|
||||||
|
},
|
||||||
|
ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: "https://schemas.example.test/child.yaml".to_owned(),
|
||||||
|
digest: digest('c'),
|
||||||
|
document: "Child: { type: object, properties: { value: { type: integer } } }"
|
||||||
|
.to_owned(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let first = normalize_verified_bundle(
|
||||||
|
document,
|
||||||
|
digest('a'),
|
||||||
|
&snapshots,
|
||||||
|
&NormalizationConfig::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let second = normalize_verified_bundle(
|
||||||
|
document,
|
||||||
|
digest('a'),
|
||||||
|
&snapshots,
|
||||||
|
&NormalizationConfig::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_vec(&first).unwrap(),
|
||||||
|
serde_json::to_vec(&second).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
first.reference_graph.dependency_digests,
|
||||||
|
vec![digest('b'), digest('c')]
|
||||||
|
);
|
||||||
|
assert_eq!(first.reference_graph.edges.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
first.reference_graph.edges[1].source.snapshot_digest,
|
||||||
|
digest('b')
|
||||||
|
);
|
||||||
|
assert!(first.unresolved_references.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recursion_is_a_stable_graph_edge_without_unbounded_expansion() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Recursive }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/nodes:
|
||||||
|
get:
|
||||||
|
operationId: getNode
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content: { application/json: { schema: { $ref: '#/components/schemas/Node' } } }
|
||||||
|
components:
|
||||||
|
schemas:
|
||||||
|
Node:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
child: { $ref: '#/components/schemas/Node' }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
assert!(ir.reference_graph.edges.iter().any(|edge| edge.recursive));
|
||||||
|
assert!(ir.unresolved_references.is_empty());
|
||||||
|
assert!(
|
||||||
|
ir.operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.all(|finding| finding.code != "reference_graph_limit")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn merges_compatible_all_of_and_blocks_conflicts_without_first_branch_loss() {
|
||||||
|
let compatible = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: AllOf }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
post:
|
||||||
|
operationId: createItem
|
||||||
|
requestBody:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
allOf:
|
||||||
|
- { type: object, required: [id], properties: { id: { type: string } } }
|
||||||
|
- { type: object, required: [name], properties: { name: { type: string } } }
|
||||||
|
responses: { '204': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(compatible);
|
||||||
|
assert!(matches!(
|
||||||
|
ir.operations[0].request_body_schema.as_ref().map(|schema| &schema.kind),
|
||||||
|
Some(NormalizedSchemaKind::Object { properties, required })
|
||||||
|
if properties.len() == 2 && required == &vec!["id".to_owned(), "name".to_owned()]
|
||||||
|
));
|
||||||
|
|
||||||
|
let conflict = compatible.replace("{ name: { type: string } }", "{ id: { type: integer } }");
|
||||||
|
let ir = normalize(&conflict);
|
||||||
|
assert!(
|
||||||
|
ir.operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "all_of_conflict")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn preserves_one_of_discriminator_and_projects_all_alternatives() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Alternatives }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/events:
|
||||||
|
post:
|
||||||
|
operationId: createEvent
|
||||||
|
requestBody:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
discriminator:
|
||||||
|
propertyName: kind
|
||||||
|
mapping: { text: '#/components/schemas/Text' }
|
||||||
|
oneOf:
|
||||||
|
- { type: object, properties: { text: { type: string } } }
|
||||||
|
- { type: object, properties: { count: { type: integer } } }
|
||||||
|
responses: { '204': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
let schema = ir.operations[0].request_body_schema.as_ref().unwrap();
|
||||||
|
assert_eq!(schema.discriminator.as_ref().unwrap().property_name, "kind");
|
||||||
|
assert!(matches!(
|
||||||
|
&schema.kind,
|
||||||
|
NormalizedSchemaKind::Composition { operator, variants }
|
||||||
|
if operator == "oneOf" && variants.len() == 2
|
||||||
|
));
|
||||||
|
let preview = preview_from_ir(&ir);
|
||||||
|
let candidate = &preview.groups[0].operations[0];
|
||||||
|
assert_eq!(
|
||||||
|
candidate
|
||||||
|
.draft
|
||||||
|
.input_schema
|
||||||
|
.fields
|
||||||
|
.get("body")
|
||||||
|
.unwrap()
|
||||||
|
.variants
|
||||||
|
.len(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
|
||||||
|
let any_of = document
|
||||||
|
.replace("discriminator:\n propertyName: kind\n mapping: { text: '#/components/schemas/Text' }\n oneOf:", "anyOf:");
|
||||||
|
let ir = normalize(&any_of);
|
||||||
|
assert!(matches!(
|
||||||
|
&ir.operations[0].request_body_schema.as_ref().unwrap().kind,
|
||||||
|
NormalizedSchemaKind::Composition { operator, variants }
|
||||||
|
if operator == "anyOf" && variants.len() == 2
|
||||||
|
));
|
||||||
|
let preview = preview_from_ir(&ir);
|
||||||
|
assert_eq!(
|
||||||
|
preview.groups[0].operations[0].draft.input_schema.fields["body"]
|
||||||
|
.variants
|
||||||
|
.len(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn oas_31_applies_ref_siblings_while_oas_30_ignores_them() {
|
||||||
|
let template = |version: &str| {
|
||||||
|
format!(
|
||||||
|
r#"
|
||||||
|
openapi: {version}
|
||||||
|
info: {{ title: Siblings }}
|
||||||
|
servers: [{{ url: https://api.example.test }}]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/Item'
|
||||||
|
description: sibling-description
|
||||||
|
components:
|
||||||
|
schemas:
|
||||||
|
Item: {{ type: string, description: target-description }}
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let v30 = normalize(&template("3.0.3"));
|
||||||
|
let v31 = normalize(&template("3.1.0"));
|
||||||
|
assert_eq!(
|
||||||
|
v30.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.description
|
||||||
|
.as_deref(),
|
||||||
|
Some("target-description")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
v31.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.description
|
||||||
|
.as_deref(),
|
||||||
|
Some("sibling-description")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn invalid_pointer_and_type_mismatch_are_exact_blockers_without_panic() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Invalid targets }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/missing:
|
||||||
|
get:
|
||||||
|
operationId: missing
|
||||||
|
responses:
|
||||||
|
'200': { description: ok, content: { application/json: { schema: { $ref: '#not-a-pointer' } } } }
|
||||||
|
/scalar:
|
||||||
|
get:
|
||||||
|
operationId: scalar
|
||||||
|
responses:
|
||||||
|
'200': { description: ok, content: { application/json: { schema: { $ref: '#/info/title' } } } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
let codes = ir
|
||||||
|
.operations
|
||||||
|
.iter()
|
||||||
|
.flat_map(|operation| {
|
||||||
|
operation
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.map(move |finding| (operation.path.as_str(), finding.code.as_str()))
|
||||||
|
})
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert!(codes.contains(&("/missing", "reference_target_missing")));
|
||||||
|
assert!(codes.contains(&("/scalar", "reference_type_mismatch")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reference_depth_and_expanded_node_limits_fail_closed() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Bounded }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200': { description: ok, content: { application/json: { schema: { $ref: '#/components/schemas/A' } } } }
|
||||||
|
components:
|
||||||
|
schemas:
|
||||||
|
A: { $ref: '#/components/schemas/B' }
|
||||||
|
B: { $ref: '#/components/schemas/C' }
|
||||||
|
C: { type: object, properties: { id: { type: string } } }
|
||||||
|
"#;
|
||||||
|
let config = NormalizationConfig {
|
||||||
|
max_reference_depth: 1,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
};
|
||||||
|
let ir = normalize_verified_document(document, digest('a'), &config).unwrap();
|
||||||
|
assert!(
|
||||||
|
ir.operations
|
||||||
|
.iter()
|
||||||
|
.flat_map(|operation| &operation.findings)
|
||||||
|
.any(|finding| finding.code == "reference_graph_limit")
|
||||||
|
);
|
||||||
|
|
||||||
|
let config = NormalizationConfig {
|
||||||
|
max_expanded_nodes: 8,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
};
|
||||||
|
let ir = normalize_verified_document(document, digest('a'), &config).unwrap();
|
||||||
|
assert!(
|
||||||
|
ir.findings
|
||||||
|
.iter()
|
||||||
|
.chain(
|
||||||
|
ir.operations
|
||||||
|
.iter()
|
||||||
|
.flat_map(|operation| &operation.findings)
|
||||||
|
)
|
||||||
|
.any(|finding| finding.code == "reference_graph_limit")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn invalid_discriminator_and_multiple_composition_operators_are_blockers() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Unsupported composition }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/events:
|
||||||
|
post:
|
||||||
|
operationId: createEvent
|
||||||
|
requestBody:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
discriminator: { mapping: { bad: 42 } }
|
||||||
|
oneOf: [{ type: string }, { type: integer }]
|
||||||
|
anyOf: [{ type: boolean }, { type: string }]
|
||||||
|
responses: { '204': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
let codes = ir.operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.map(|finding| finding.code.as_str())
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert!(codes.contains("unsupported_discriminator"));
|
||||||
|
assert!(codes.contains("unsupported_composition"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reference_uri_scan_applies_external_size_depth_and_alias_limits_before_traversal() {
|
||||||
|
let config = NormalizationConfig {
|
||||||
|
max_bytes: 8,
|
||||||
|
max_external_document_bytes: 8,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
reference_uris("external-document", &config),
|
||||||
|
Err(crank_import::rest::ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
|
||||||
|
let config = NormalizationConfig {
|
||||||
|
max_bytes: 8,
|
||||||
|
max_external_document_bytes: 32,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
external_reference_uris("external-document", &config),
|
||||||
|
Ok(Vec::new())
|
||||||
|
);
|
||||||
|
|
||||||
|
let config = NormalizationConfig {
|
||||||
|
max_depth: 1,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
reference_uris("a: { b: { $ref: '#/x' } }", &config),
|
||||||
|
Err(crank_import::rest::ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
|
||||||
|
let aliases = format!(
|
||||||
|
"items: [{}]",
|
||||||
|
std::iter::repeat_n("*a", 129)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
reference_uris(&aliases, &NormalizationConfig::default()),
|
||||||
|
Err(crank_import::rest::ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolves_relative_references_using_rfc3986_paths_and_decoded_fragments() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Relative refs }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content: { application/json: { schema: { $ref: 'HTTPS://SCHEMAS.EXAMPLE.TEST:443/a/b/root.yaml#/Item' } } }
|
||||||
|
"#;
|
||||||
|
let snapshots = vec![
|
||||||
|
ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: "https://schemas.example.test/a/b/root.yaml".to_owned(),
|
||||||
|
digest: digest('b'),
|
||||||
|
document: r#"
|
||||||
|
Item:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
dot: { $ref: './child.yaml#/Value' }
|
||||||
|
parent: { $ref: '../common.yaml#/Value' }
|
||||||
|
absolute: { $ref: '/shared.yaml#/Value' }
|
||||||
|
"#
|
||||||
|
.to_owned(),
|
||||||
|
},
|
||||||
|
ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: "https://schemas.example.test/a/b/child.yaml".to_owned(),
|
||||||
|
digest: digest('c'),
|
||||||
|
document: "Value: { type: string }".to_owned(),
|
||||||
|
},
|
||||||
|
ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: "https://schemas.example.test/a/common.yaml".to_owned(),
|
||||||
|
digest: digest('d'),
|
||||||
|
document: "Value: { type: integer }".to_owned(),
|
||||||
|
},
|
||||||
|
ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: "https://schemas.example.test/shared.yaml".to_owned(),
|
||||||
|
digest: digest('e'),
|
||||||
|
document: "Value: { $ref: '#/a%7E1b' }\na/b: { type: boolean }".to_owned(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let ir = normalize_verified_bundle(
|
||||||
|
document,
|
||||||
|
digest('a'),
|
||||||
|
&snapshots,
|
||||||
|
&NormalizationConfig::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let Some(NormalizedSchemaKind::Object { properties, .. }) = ir.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.map(|schema| &schema.kind)
|
||||||
|
else {
|
||||||
|
panic!("response should be an expanded object schema");
|
||||||
|
};
|
||||||
|
assert_eq!(properties.len(), 3);
|
||||||
|
assert_eq!(ir.reference_graph.edges.len(), 5);
|
||||||
|
assert!(ir.findings.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_percent_encoded_fragment_is_a_blocker() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Malformed reference }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200': { description: ok, content: { application/json: { schema: { $ref: '#/components/%ZZ' } } } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
assert!(
|
||||||
|
ir.operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "reference_uri_malformed")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn does_not_follow_references_inside_literal_payloads() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Literal payloads }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: ok
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
example: { $ref: '#/components/schemas/Missing' }
|
||||||
|
examples: { sample: { value: { $ref: '#/components/schemas/Missing' } } }
|
||||||
|
default: { $ref: '#/components/schemas/Missing' }
|
||||||
|
enum: [{ $ref: '#/components/schemas/Missing' }]
|
||||||
|
const: { $ref: '#/components/schemas/Missing' }
|
||||||
|
x-fixture: { $ref: '#/components/schemas/Missing' }
|
||||||
|
properties: { known: { $ref: '#/components/schemas/Known' } }
|
||||||
|
components:
|
||||||
|
schemas:
|
||||||
|
Known: { type: string }
|
||||||
|
"#;
|
||||||
|
assert_eq!(
|
||||||
|
reference_uris(document, &NormalizationConfig::default()),
|
||||||
|
Ok(vec!["#/components/schemas/Known".to_owned()])
|
||||||
|
);
|
||||||
|
let ir = normalize(document);
|
||||||
|
assert_eq!(ir.reference_graph.edges.len(), 1);
|
||||||
|
assert!(
|
||||||
|
ir.operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.all(|finding| finding.code != "reference_target_missing")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn primary_oas31_applies_ref_siblings_in_external_snapshots_without_openapi_field() {
|
||||||
|
let source = |version: &str| {
|
||||||
|
format!(
|
||||||
|
r#"
|
||||||
|
openapi: {version}
|
||||||
|
info: {{ title: External siblings }}
|
||||||
|
servers: [{{ url: https://api.example.test }}]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
get:
|
||||||
|
operationId: listItems
|
||||||
|
responses:
|
||||||
|
'200': {{ description: ok, content: {{ application/json: {{ schema: {{ $ref: 'https://schemas.example.test/root.yaml#/Item' }} }} }} }}
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let snapshots = vec![
|
||||||
|
ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: "https://schemas.example.test/root.yaml".to_owned(),
|
||||||
|
digest: digest('b'),
|
||||||
|
document: "Item: { $ref: 'child.yaml#/Base', description: sibling }".to_owned(),
|
||||||
|
},
|
||||||
|
ExternalDocumentSnapshot {
|
||||||
|
canonical_uri: "https://schemas.example.test/child.yaml".to_owned(),
|
||||||
|
digest: digest('c'),
|
||||||
|
document: "Base: { type: string, description: target }".to_owned(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let v31 = normalize_verified_bundle(
|
||||||
|
&source("3.1.0"),
|
||||||
|
digest('a'),
|
||||||
|
&snapshots,
|
||||||
|
&NormalizationConfig::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let v30 = normalize_verified_bundle(
|
||||||
|
&source("3.0.3"),
|
||||||
|
digest('a'),
|
||||||
|
&snapshots,
|
||||||
|
&NormalizationConfig::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
v31.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.description
|
||||||
|
.as_deref(),
|
||||||
|
Some("sibling")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
v30.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.description
|
||||||
|
.as_deref(),
|
||||||
|
Some("target")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn all_of_intersects_constraints_and_nested_properties_without_losing_conflicts() {
|
||||||
|
let source = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: allOf intersections }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
post:
|
||||||
|
operationId: createItem
|
||||||
|
requestBody:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
allOf:
|
||||||
|
- type: object
|
||||||
|
minimum: 1
|
||||||
|
maximum: 10
|
||||||
|
minLength: 2
|
||||||
|
maxLength: 12
|
||||||
|
properties: { nested: { type: object, properties: { left: { type: string } } } }
|
||||||
|
- type: object
|
||||||
|
minimum: 4
|
||||||
|
maximum: 8
|
||||||
|
minLength: 5
|
||||||
|
maxLength: 9
|
||||||
|
properties: { nested: { type: object, properties: { right: { type: integer } } } }
|
||||||
|
responses: { '204': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(source);
|
||||||
|
let schema = ir.operations[0].request_body_schema.as_ref().unwrap();
|
||||||
|
assert_eq!(schema.constraints.minimum, Some(4.0));
|
||||||
|
assert_eq!(schema.constraints.maximum, Some(8.0));
|
||||||
|
assert_eq!(schema.constraints.min_length, Some(5));
|
||||||
|
assert_eq!(schema.constraints.max_length, Some(9));
|
||||||
|
let NormalizedSchemaKind::Object { properties, .. } = &schema.kind else {
|
||||||
|
panic!("merged schema should remain an object");
|
||||||
|
};
|
||||||
|
let NormalizedSchemaKind::Object { properties, .. } = &properties["nested"].kind else {
|
||||||
|
panic!("nested property should remain an object");
|
||||||
|
};
|
||||||
|
assert!(properties.contains_key("left") && properties.contains_key("right"));
|
||||||
|
|
||||||
|
let conflicting = source.replace("maximum: 8", "maximum: 3");
|
||||||
|
let ir = normalize(&conflicting);
|
||||||
|
assert!(
|
||||||
|
ir.operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "all_of_conflict")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn non_array_all_of_is_preserved_and_reported() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Invalid allOf }
|
||||||
|
servers: [{ url: https://api.example.test }]
|
||||||
|
paths:
|
||||||
|
/items:
|
||||||
|
post:
|
||||||
|
operationId: createItem
|
||||||
|
requestBody:
|
||||||
|
content: { application/json: { schema: { allOf: { type: string } } } }
|
||||||
|
responses: { '204': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let ir = normalize(document);
|
||||||
|
assert!(
|
||||||
|
ir.operations[0]
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.any(|finding| finding.code == "unsupported_composition")
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
ir.operations[0]
|
||||||
|
.request_body_schema
|
||||||
|
.as_ref()
|
||||||
|
.map(|schema| &schema.kind),
|
||||||
|
Some(NormalizedSchemaKind::Unknown)
|
||||||
|
));
|
||||||
|
}
|
||||||
+506
-167
@@ -1,44 +1,8 @@
|
|||||||
mod unit {
|
mod unit {
|
||||||
use crank_core::HttpMethod;
|
use crank_import::rest::{
|
||||||
use crank_import::rest::preview_document;
|
ImportFindingSeverity, ImportParseError, NormalizationConfig, normalize_verified_document,
|
||||||
|
preview_document,
|
||||||
const OPENAPI3: &str = r#"
|
};
|
||||||
openapi: 3.0.3
|
|
||||||
info:
|
|
||||||
title: Frankfurter API
|
|
||||||
servers:
|
|
||||||
- url: https://api.frankfurter.dev
|
|
||||||
paths:
|
|
||||||
/v2/latest:
|
|
||||||
get:
|
|
||||||
operationId: getLatestRates
|
|
||||||
summary: Получить последние курсы
|
|
||||||
description: Возвращает последние курсы валют для базовой валюты.
|
|
||||||
tags: [currency]
|
|
||||||
parameters:
|
|
||||||
- name: base
|
|
||||||
in: query
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
- name: symbols
|
|
||||||
in: query
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: OK
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
type: object
|
|
||||||
required: [amount, base]
|
|
||||||
properties:
|
|
||||||
amount:
|
|
||||||
type: number
|
|
||||||
base:
|
|
||||||
type: string
|
|
||||||
"#;
|
|
||||||
|
|
||||||
const SWAGGER2: &str = r#"
|
const SWAGGER2: &str = r#"
|
||||||
swagger: "2.0"
|
swagger: "2.0"
|
||||||
@@ -74,170 +38,545 @@ definitions:
|
|||||||
type: string
|
type: string
|
||||||
"#;
|
"#;
|
||||||
|
|
||||||
#[test]
|
fn normalize_document(
|
||||||
fn previews_openapi3_rest_operations_grouped_by_tag() {
|
document: &str,
|
||||||
let preview = preview_document(OPENAPI3).unwrap();
|
config: &NormalizationConfig,
|
||||||
|
) -> Result<crank_import::rest::NormalizedIr, ImportParseError> {
|
||||||
|
normalize_verified_document(
|
||||||
|
document,
|
||||||
|
crank_import::rest::SourceDigest::parse("b".repeat(64)).unwrap(),
|
||||||
|
config,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
assert_eq!(preview.source.format, "openapi");
|
fn matrix_source() -> String {
|
||||||
assert_eq!(preview.source.servers, vec!["https://api.frankfurter.dev"]);
|
"openapi: 3.1.0\ninfo: { title: Matrix }\npaths:\n /ok:\n get:\n operationId: getOk\n tags: [matrix]\n parameters: [{ name: q, in: query, schema: { type: string } }]\n responses: { '200': { description: ok, content: { application/json: { schema: { type: object, properties: { value: { type: string } } } } } } }".to_owned()
|
||||||
assert_eq!(preview.groups.len(), 1);
|
|
||||||
assert_eq!(preview.groups[0].key, "currency");
|
|
||||||
let operation = &preview.groups[0].operations[0];
|
|
||||||
assert_eq!(operation.method, HttpMethod::Get);
|
|
||||||
assert_eq!(operation.suggested_name, "get_latest_rates");
|
|
||||||
assert_eq!(operation.input_fields, 2);
|
|
||||||
assert_eq!(operation.output_fields, 2);
|
|
||||||
assert_eq!(operation.draft.target.path_template, "/v2/latest");
|
|
||||||
assert_eq!(operation.draft.input_mapping.rules.len(), 2);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn previews_swagger2_and_resolves_definitions() {
|
fn normalizes_supported_versions_with_canonical_order_and_stable_ids() {
|
||||||
let preview = preview_document(SWAGGER2).unwrap();
|
let fixture_matrix = [
|
||||||
|
(include_str!("fixtures/openapi-3.0.yaml"), "3.0.3"),
|
||||||
|
(include_str!("fixtures/openapi-3.1.json"), "3.1.0"),
|
||||||
|
(include_str!("fixtures/swagger-2.0.yaml"), "2.0"),
|
||||||
|
];
|
||||||
|
for (fixture, version) in fixture_matrix {
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(fixture, &NormalizationConfig::default())
|
||||||
|
.unwrap()
|
||||||
|
.source
|
||||||
|
.version
|
||||||
|
.as_deref(),
|
||||||
|
Some(version)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let openapi_31_json = r#"{
|
||||||
|
"openapi":"3.1.0", "info":{"title":"Canonical"},
|
||||||
|
"paths": {
|
||||||
|
"/z":{"post":{"responses":{"201":{"description":"ok"}}}},
|
||||||
|
"/a":{"get":{"responses":{"200":{"description":"ok"}}}}
|
||||||
|
}
|
||||||
|
}"#;
|
||||||
|
let openapi_30_yaml = r#"
|
||||||
|
openapi: 3.0.3
|
||||||
|
info: { title: Canonical }
|
||||||
|
paths:
|
||||||
|
/a:
|
||||||
|
get:
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
let swagger = r#"
|
||||||
|
swagger: '2.0'
|
||||||
|
info: { title: Legacy }
|
||||||
|
paths:
|
||||||
|
/a:
|
||||||
|
get:
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
|
||||||
assert_eq!(preview.source.format, "swagger");
|
let first = normalize_document(openapi_31_json, &NormalizationConfig::default()).unwrap();
|
||||||
|
let second = normalize_document(openapi_31_json, &NormalizationConfig::default()).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
preview.source.servers,
|
serde_json::to_vec(&first).unwrap(),
|
||||||
vec!["https://petstore.example.com/api"]
|
serde_json::to_vec(&second).unwrap()
|
||||||
);
|
);
|
||||||
let operation = &preview.groups[0].operations[0];
|
assert_eq!(first.operations[0].path, "/a");
|
||||||
assert_eq!(operation.suggested_name, "get_pet");
|
assert_eq!(first.operations[0].stable_id, "3.1.0:get:/paths/~1a/get");
|
||||||
assert_eq!(operation.input_fields, 1);
|
assert!(first.coverage.len() >= 2);
|
||||||
assert_eq!(operation.output_fields, 2);
|
|
||||||
assert_eq!(operation.draft.target.path_template, "/pets/{id}");
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
operation.draft.input_mapping.rules[0].target,
|
normalize_document(openapi_30_yaml, &NormalizationConfig::default())
|
||||||
"$.request.path.id"
|
.unwrap()
|
||||||
|
.source
|
||||||
|
.version
|
||||||
|
.as_deref(),
|
||||||
|
Some("3.0.3")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(swagger, &NormalizationConfig::default())
|
||||||
|
.unwrap()
|
||||||
|
.source
|
||||||
|
.version
|
||||||
|
.as_deref(),
|
||||||
|
Some("2.0")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reports_missing_descriptions_as_recommendations() {
|
fn resolves_local_references_into_typed_graph() {
|
||||||
let document = r#"
|
let ir = normalize_document(SWAGGER2, &NormalizationConfig::default()).unwrap();
|
||||||
openapi: 3.0.3
|
let operation = &ir.operations[0];
|
||||||
info: { title: Minimal API }
|
assert!(matches!(
|
||||||
paths:
|
operation
|
||||||
/items:
|
.response_schema
|
||||||
get:
|
.as_ref()
|
||||||
responses:
|
.map(|schema| &schema.kind),
|
||||||
'204': { description: Empty }
|
Some(crank_import::rest::NormalizedSchemaKind::Object { .. })
|
||||||
"#;
|
));
|
||||||
|
assert!(ir.unresolved_references.is_empty());
|
||||||
let preview = preview_document(document).unwrap();
|
assert!(!ir.reference_graph.edges.is_empty());
|
||||||
let operation = &preview.groups[0].operations[0];
|
assert!(
|
||||||
let codes = operation
|
operation
|
||||||
.findings
|
.findings
|
||||||
.iter()
|
.iter()
|
||||||
.map(|finding| finding.code.as_str())
|
.all(|finding| finding.code != "unresolved_reference")
|
||||||
.collect::<Vec<_>>();
|
);
|
||||||
|
|
||||||
assert!(codes.contains(&"missing_operation_id"));
|
|
||||||
assert!(codes.contains(&"missing_summary"));
|
|
||||||
assert!(codes.contains(&"missing_description"));
|
|
||||||
assert!(codes.contains(&"missing_response_schema"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn expands_json_request_body_object_into_tool_inputs() {
|
fn preserves_valid_operations_when_another_path_item_is_malformed() {
|
||||||
let document = r#"
|
let document = r#"
|
||||||
openapi: 3.0.3
|
openapi: 3.0.3
|
||||||
info: { title: CRM API }
|
info: { title: Partial }
|
||||||
servers:
|
|
||||||
- url: https://crm.example.test
|
|
||||||
paths:
|
paths:
|
||||||
/leads:
|
/valid:
|
||||||
post:
|
get:
|
||||||
operationId: createLead
|
responses: { '200': { description: ok } }
|
||||||
summary: Создать лид
|
/broken: invalid
|
||||||
description: Создает лид в CRM.
|
|
||||||
tags: [crm]
|
|
||||||
requestBody:
|
|
||||||
required: true
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
type: object
|
|
||||||
required: [email]
|
|
||||||
properties:
|
|
||||||
email: { type: string }
|
|
||||||
name: { type: string }
|
|
||||||
responses:
|
|
||||||
'201':
|
|
||||||
description: Created
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
type: object
|
|
||||||
properties:
|
|
||||||
id: { type: string }
|
|
||||||
"#;
|
"#;
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
let preview = preview_document(document).unwrap();
|
assert_eq!(ir.operations.len(), 1);
|
||||||
let operation = &preview.groups[0].operations[0];
|
assert!(ir.findings.iter().any(|finding| {
|
||||||
let input_fields = &operation.draft.input_schema.fields;
|
finding.code == "invalid_path_item"
|
||||||
let targets = operation
|
&& finding.severity == crank_import::rest::ImportFindingSeverity::Error
|
||||||
.draft
|
}));
|
||||||
.input_mapping
|
assert!(ir.findings.iter().any(|finding| {
|
||||||
.rules
|
finding.code == "invalid_path_item"
|
||||||
.iter()
|
&& finding.location.pointer == "/paths/~1broken"
|
||||||
.map(|rule| rule.target.as_str())
|
&& finding.construct_id.contains("/paths/~1broken")
|
||||||
.collect::<Vec<_>>();
|
}));
|
||||||
|
|
||||||
assert!(input_fields.contains_key("email"));
|
|
||||||
assert!(input_fields.contains_key("name"));
|
|
||||||
assert!(targets.contains(&"$.request.body.email"));
|
|
||||||
assert!(targets.contains(&"$.request.body.name"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reports_tool_quality_recommendations_for_imported_operations() {
|
fn malformed_method_values_are_blockers_while_valid_siblings_survive() {
|
||||||
|
for (format, document) in [
|
||||||
|
(
|
||||||
|
"openapi",
|
||||||
|
"openapi: 3.0.3\ninfo: { title: malformed }\npaths: { /mixed: { get: { responses: { '200': { description: ok } } }, post: broken } }",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"swagger",
|
||||||
|
"swagger: '2.0'\ninfo: { title: malformed }\npaths: { /mixed: { get: { responses: { '200': { description: ok } } }, post: broken } }",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(ir.operations.len(), 1, "{format}");
|
||||||
|
assert_eq!(ir.operations[0].key, "GET /mixed", "{format}");
|
||||||
|
let finding = ir
|
||||||
|
.findings
|
||||||
|
.iter()
|
||||||
|
.find(|finding| finding.code == "invalid_operation")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(finding.severity, ImportFindingSeverity::Error, "{format}");
|
||||||
|
assert_eq!(finding.location.pointer, "/paths/~1mixed/post", "{format}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_hostile_input_at_configured_limits() {
|
||||||
let document = r#"
|
let document = r#"
|
||||||
openapi: 3.0.3
|
openapi: 3.0.3
|
||||||
info: { title: Wide API }
|
info: { title: Too deep }
|
||||||
|
paths: {}
|
||||||
|
"#;
|
||||||
|
let config = NormalizationConfig {
|
||||||
|
max_bytes: 10,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(document, &config),
|
||||||
|
Err(ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
|
||||||
|
let unsupported = "openapi: 3.2.0\ninfo: { title: Nope }\npaths: {}\n";
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(unsupported, &NormalizationConfig::default()),
|
||||||
|
Err(ImportParseError::UnsupportedDocument)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checked_in_hostile_regressions_are_bounded_and_redacted() {
|
||||||
|
// Minimal regression corpus distilled from malformed/fuzzed inputs.
|
||||||
|
// Assertions deliberately compare typed errors, never parser text.
|
||||||
|
for document in [
|
||||||
|
include_str!("fixtures/fuzz-regressions/invalid-root.json"),
|
||||||
|
include_str!("fixtures/fuzz-regressions/unsupported-version.yaml"),
|
||||||
|
include_str!("fixtures/fuzz-regressions/wrong-version-shape.yaml"),
|
||||||
|
] {
|
||||||
|
let error = normalize_document(document, &NormalizationConfig::default()).unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
ImportParseError::InvalidDocument | ImportParseError::UnsupportedDocument
|
||||||
|
));
|
||||||
|
assert!(!error.to_string().contains("wrong-root-field"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn verified_digest_stays_inside_ir_and_unknown_contract_fails_closed() {
|
||||||
|
let digest = crank_import::rest::SourceDigest::parse("a".repeat(64)).unwrap();
|
||||||
|
let ir = crank_import::rest::normalize_verified_document(
|
||||||
|
include_str!("fixtures/openapi-3.1.json"),
|
||||||
|
digest,
|
||||||
|
&NormalizationConfig::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(ir.source_identity.digest.as_str(), "a".repeat(64));
|
||||||
|
let preview = crank_import::rest::preview_from_ir(&ir);
|
||||||
|
assert!(
|
||||||
|
!serde_json::to_string(&preview)
|
||||||
|
.unwrap()
|
||||||
|
.contains("source_identity")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!serde_json::to_string(&preview)
|
||||||
|
.unwrap()
|
||||||
|
.contains(&"a".repeat(64))
|
||||||
|
);
|
||||||
|
let config = NormalizationConfig {
|
||||||
|
normalizer_version: "future".to_owned(),
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(include_str!("fixtures/openapi-3.1.json"), &config),
|
||||||
|
Err(ImportParseError::UnsupportedDocument)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn yaml_alias_preflight_counts_inline_aliases_but_not_quotes_or_comments() {
|
||||||
|
let mut document =
|
||||||
|
"openapi: 3.0.3\ninfo: { title: aliases }\npaths: {}\nitems: [".to_owned();
|
||||||
|
document.push_str(
|
||||||
|
&std::iter::repeat_n("*a", 129)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", "),
|
||||||
|
);
|
||||||
|
document.push(']');
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(&document, &NormalizationConfig::default()),
|
||||||
|
Err(ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
let harmless = "openapi: 3.0.3\ninfo: { title: '*not_alias' } # *also_not_alias\npaths: { /ok: { get: { responses: { '200': { description: ok } } } } }";
|
||||||
|
assert!(normalize_document(harmless, &NormalizationConfig::default()).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn yaml_alias_preflight_keeps_quote_state_after_escaped_and_doubled_quotes() {
|
||||||
|
let mut escaped_quote = String::from(
|
||||||
|
"openapi: 3.0.3\ninfo:\n title: \"escaped \\\" quote\"\npaths: {}\nitems: [",
|
||||||
|
);
|
||||||
|
escaped_quote.push_str(
|
||||||
|
&std::iter::repeat_n("*alias", 129)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", "),
|
||||||
|
);
|
||||||
|
escaped_quote.push(']');
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(&escaped_quote, &NormalizationConfig::default()),
|
||||||
|
Err(ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
|
||||||
|
let harmless = r#"openapi: 3.0.3
|
||||||
|
info:
|
||||||
|
title: 'it''s *not_an_alias'
|
||||||
|
description: "escaped \" *also_not_an_alias"
|
||||||
paths:
|
paths:
|
||||||
/items:
|
/ok:
|
||||||
|
get:
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
assert!(normalize_document(harmless, &NormalizationConfig::default()).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn yaml_alias_preflight_fails_closed_for_stars_in_block_scalars() {
|
||||||
|
let hidden_alias = r#"openapi: 3.0.3
|
||||||
|
info:
|
||||||
|
title: Block scalar
|
||||||
|
description: |
|
||||||
|
this text contains *an_alias_like_token
|
||||||
|
paths:
|
||||||
|
/ok:
|
||||||
|
get:
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(hidden_alias, &NormalizationConfig::default()),
|
||||||
|
Err(ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
|
||||||
|
let safe_block = r#"openapi: 3.0.3
|
||||||
|
info:
|
||||||
|
title: Block scalar
|
||||||
|
description: |
|
||||||
|
this text contains no alias-looking marker
|
||||||
|
paths:
|
||||||
|
/ok:
|
||||||
|
get:
|
||||||
|
responses: { '200': { description: ok } }
|
||||||
|
"#;
|
||||||
|
assert!(normalize_document(safe_block, &NormalizationConfig::default()).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn typed_scalar_literals_round_trip_without_stringification() {
|
||||||
|
let document = r#"
|
||||||
|
openapi: 3.1.0
|
||||||
|
info: { title: Scalars }
|
||||||
|
paths:
|
||||||
|
/value:
|
||||||
get:
|
get:
|
||||||
operationId: getItems
|
|
||||||
summary: Get items
|
|
||||||
description: Get items.
|
|
||||||
parameters:
|
|
||||||
- name: page
|
|
||||||
in: query
|
|
||||||
schema: { type: integer }
|
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
description: OK
|
description: ok
|
||||||
content:
|
content:
|
||||||
application/json:
|
application/json:
|
||||||
schema:
|
schema:
|
||||||
type: object
|
type: object
|
||||||
properties:
|
properties:
|
||||||
field01: { type: string }
|
count: { type: integer, default: 7, enum: [1, 2] }
|
||||||
field02: { type: string }
|
enabled: { type: boolean, default: true, enum: [true, false] }
|
||||||
field03: { type: string }
|
ratio: { type: number, default: 1.5, enum: [0.5, 1.5] }
|
||||||
field04: { type: string }
|
|
||||||
field05: { type: string }
|
|
||||||
field06: { type: string }
|
|
||||||
field07: { type: string }
|
|
||||||
field08: { type: string }
|
|
||||||
field09: { type: string }
|
|
||||||
field10: { type: string }
|
|
||||||
field11: { type: string }
|
|
||||||
field12: { type: string }
|
|
||||||
field13: { type: string }
|
|
||||||
"#;
|
"#;
|
||||||
|
let ir = normalize_document(document, &NormalizationConfig::default()).unwrap();
|
||||||
let preview = preview_document(document).unwrap();
|
let preview = preview_document(document).unwrap();
|
||||||
let operation = &preview.groups[0].operations[0];
|
let fields = &preview.groups[0].operations[0].draft.output_schema.fields;
|
||||||
let codes = operation
|
assert_eq!(fields["count"].kind, crank_schema::SchemaKind::Integer);
|
||||||
|
assert_eq!(fields["enabled"].kind, crank_schema::SchemaKind::Boolean);
|
||||||
|
let rendered = serde_json::to_value(&ir.operations[0].response_schema).unwrap();
|
||||||
|
assert!(rendered.to_string().contains("integer"));
|
||||||
|
assert!(rendered.to_string().contains("1.5"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn coverage_validator_rejects_nested_gap_and_duplicate_findings() {
|
||||||
|
let mut ir = normalize_document(&matrix_source(), &NormalizationConfig::default()).unwrap();
|
||||||
|
let nested = ir.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.construct_id
|
||||||
|
.clone();
|
||||||
|
ir.coverage.retain(|entry| entry.construct_id != nested);
|
||||||
|
assert_eq!(
|
||||||
|
crank_import::rest::validate_normalized_ir(&ir),
|
||||||
|
Err(ImportParseError::InvalidDocument)
|
||||||
|
);
|
||||||
|
let mut ir = normalize_document(&matrix_source(), &NormalizationConfig::default()).unwrap();
|
||||||
|
ir.coverage
|
||||||
|
.retain(|entry| entry.construct_id != "source:/paths/~1ok/get/responses/200");
|
||||||
|
assert_eq!(
|
||||||
|
crank_import::rest::validate_normalized_ir(&ir),
|
||||||
|
Err(ImportParseError::InvalidDocument)
|
||||||
|
);
|
||||||
|
let ir = normalize_document(&matrix_source(), &NormalizationConfig::default()).unwrap();
|
||||||
|
let findings = &ir.operations[0].findings;
|
||||||
|
assert_eq!(
|
||||||
|
findings.len(),
|
||||||
|
findings
|
||||||
|
.iter()
|
||||||
|
.map(|finding| (&finding.code, &finding.construct_id))
|
||||||
|
.collect::<std::collections::BTreeSet<_>>()
|
||||||
|
.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn two_malformed_paths_keep_distinct_internal_locations() {
|
||||||
|
let ir = normalize_document(
|
||||||
|
"openapi: 3.0.3\ninfo: { title: broken }\npaths: { /one: bad, /two: bad }",
|
||||||
|
&NormalizationConfig::default(),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(ir, ImportParseError::NoMethods);
|
||||||
|
// Structural parser findings require at least one valid operation to return IR.
|
||||||
|
let ir = normalize_document("openapi: 3.0.3\ninfo: { title: broken }\npaths: { /ok: { get: { responses: { '200': { description: ok } } } }, /one: bad, /two: bad }", &NormalizationConfig::default()).unwrap();
|
||||||
|
let pointers = ir
|
||||||
.findings
|
.findings
|
||||||
.iter()
|
.iter()
|
||||||
.map(|finding| finding.code.as_str())
|
.filter(|finding| finding.code == "invalid_path_item")
|
||||||
.collect::<Vec<_>>();
|
.map(|finding| finding.location.pointer.as_str())
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
pointers,
|
||||||
|
std::collections::BTreeSet::from(["/paths/~1one", "/paths/~1two"])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
assert!(codes.contains(&"parameter_descriptions_missing"));
|
#[test]
|
||||||
assert!(codes.contains(&"weak_tool_description"));
|
fn canonical_matrix_covers_all_supported_versions_syntaxes_and_concurrency() {
|
||||||
assert!(codes.contains(&"weak_tool_name"));
|
let matrix = [
|
||||||
assert!(codes.contains(&"too_many_output_fields"));
|
(
|
||||||
|
"oas30-yaml",
|
||||||
|
"openapi: 3.0.3\ninfo: { title: OAS30 }\npaths: { /ok: { get: { responses: { '200': { description: ok } } } } }",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"oas30-json",
|
||||||
|
r#"{"openapi":"3.0.3","info":{"title":"OAS30"},"paths":{"/ok":{"get":{"responses":{"200":{"description":"ok"}}}}}}"#,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"oas31-yaml",
|
||||||
|
"openapi: 3.1.0\ninfo: { title: OAS31 }\npaths: { /ok: { get: { responses: { '200': { description: ok } } } } }",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"oas31-json",
|
||||||
|
r#"{"openapi":"3.1.0","info":{"title":"OAS31"},"paths":{"/ok":{"get":{"responses":{"200":{"description":"ok"}}}}}}"#,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"swagger-yaml",
|
||||||
|
"swagger: '2.0'\ninfo: { title: Swagger }\npaths: { /ok: { get: { responses: { '200': { description: ok } } } } }",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"swagger-json",
|
||||||
|
r#"{"swagger":"2.0","info":{"title":"Swagger"},"paths":{"/ok":{"get":{"responses":{"200":{"description":"ok"}}}}}}"#,
|
||||||
|
),
|
||||||
|
];
|
||||||
|
for (name, source) in matrix {
|
||||||
|
let first = normalize_document(source, &NormalizationConfig::default()).unwrap();
|
||||||
|
let second = normalize_document(source, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_vec(&first).unwrap(),
|
||||||
|
serde_json::to_vec(&second).unwrap(),
|
||||||
|
"{name}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let source = std::sync::Arc::new(matrix_source());
|
||||||
|
let results = (0..8)
|
||||||
|
.map(|_| {
|
||||||
|
let source = std::sync::Arc::clone(&source);
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
serde_json::to_vec(
|
||||||
|
&normalize_document(&source, &NormalizationConfig::default()).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.map(|thread| thread.join().unwrap())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert!(results.windows(2).all(|pair| pair[0] == pair[1]));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_small_path_order_permutation_has_the_same_canonical_ir() {
|
||||||
|
let paths = [
|
||||||
|
("/a", "get", "200"),
|
||||||
|
("/b", "post", "201"),
|
||||||
|
("/c", "delete", "204"),
|
||||||
|
];
|
||||||
|
let permutations = [
|
||||||
|
[0, 1, 2],
|
||||||
|
[0, 2, 1],
|
||||||
|
[1, 0, 2],
|
||||||
|
[1, 2, 0],
|
||||||
|
[2, 0, 1],
|
||||||
|
[2, 1, 0],
|
||||||
|
];
|
||||||
|
let mut canonical = None;
|
||||||
|
for permutation in permutations {
|
||||||
|
let path_entries = permutation
|
||||||
|
.into_iter()
|
||||||
|
.map(|index| {
|
||||||
|
let (path, method, status) = paths[index];
|
||||||
|
format!(
|
||||||
|
"\"{path}\":{{\"{method}\":{{\"responses\":{{\"{status}\":{{\"description\":\"ok\"}}}}}}}}"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(",");
|
||||||
|
let source = format!(
|
||||||
|
"{{\"openapi\":\"3.1.0\",\"info\":{{\"title\":\"Permutation\"}},\"paths\":{{{path_entries}}}}}"
|
||||||
|
);
|
||||||
|
let bytes = serde_json::to_vec(
|
||||||
|
&normalize_document(&source, &NormalizationConfig::default()).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
if let Some(expected) = &canonical {
|
||||||
|
assert_eq!(&bytes, expected);
|
||||||
|
} else {
|
||||||
|
canonical = Some(bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn limit_and_reference_regression_matrix_is_bounded_and_typed() {
|
||||||
|
let source = matrix_source();
|
||||||
|
for config in [
|
||||||
|
NormalizationConfig {
|
||||||
|
max_bytes: 1,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
},
|
||||||
|
NormalizationConfig {
|
||||||
|
max_depth: 1,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
},
|
||||||
|
NormalizationConfig {
|
||||||
|
max_nodes: 1,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
},
|
||||||
|
NormalizationConfig {
|
||||||
|
max_collection_items: 1,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
},
|
||||||
|
NormalizationConfig {
|
||||||
|
max_scalar_bytes: 1,
|
||||||
|
..NormalizationConfig::default()
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(&source, &config),
|
||||||
|
Err(ImportParseError::LimitExceeded)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document(
|
||||||
|
"openapi: 3.0.3\ninfo: { title: none }\npaths: {}",
|
||||||
|
&NormalizationConfig::default()
|
||||||
|
),
|
||||||
|
Err(ImportParseError::NoMethods)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
normalize_document("[]", &NormalizationConfig::default()),
|
||||||
|
Err(ImportParseError::UnsupportedDocument)
|
||||||
|
);
|
||||||
|
|
||||||
|
for reference in [
|
||||||
|
"#/components/schemas/Loop",
|
||||||
|
"https://example.test/schema.json",
|
||||||
|
"#/components/schemas/Loop",
|
||||||
|
] {
|
||||||
|
let document = format!(
|
||||||
|
"openapi: 3.0.3\ninfo: {{ title: refs }}\npaths:\n /ok:\n get:\n responses:\n '200': {{ description: ok, content: {{ application/json: {{ schema: {{ $ref: '{reference}' }} }} }} }}"
|
||||||
|
);
|
||||||
|
let ir = normalize_document(&document, &NormalizationConfig::default()).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
ir.operations[0]
|
||||||
|
.response_schema
|
||||||
|
.as_ref()
|
||||||
|
.map(|schema| &schema.kind),
|
||||||
|
Some(crank_import::rest::NormalizedSchemaKind::Reference { .. })
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ mod onboarding_product_events_v11;
|
|||||||
mod owned_relations;
|
mod owned_relations;
|
||||||
mod platform_key_name_reuse_v6;
|
mod platform_key_name_reuse_v6;
|
||||||
mod schema_guard;
|
mod schema_guard;
|
||||||
|
mod schema_guard_legacy_v1;
|
||||||
mod schema_guard_v10;
|
mod schema_guard_v10;
|
||||||
mod schema_guard_v11;
|
mod schema_guard_v11;
|
||||||
mod schema_guard_v12;
|
mod schema_guard_v12;
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ use super::platform_key_name_reuse_v6;
|
|||||||
use super::schema_guard::{
|
use super::schema_guard::{
|
||||||
OWNED_RELATIONS, relation_exists, validate_required_relations, validate_schema_fingerprint,
|
OWNED_RELATIONS, relation_exists, validate_required_relations, validate_schema_fingerprint,
|
||||||
};
|
};
|
||||||
|
use super::schema_guard_legacy_v1::{
|
||||||
|
validate_ledgerless_baseline_fingerprint, validate_ledgerless_optional_fingerprints,
|
||||||
|
};
|
||||||
use super::{BASELINE_CHECKSUM, BASELINE_VERSION, apply_baseline};
|
use super::{BASELINE_CHECKSUM, BASELINE_VERSION, apply_baseline};
|
||||||
use crate::ext::ExtensionMigration;
|
use crate::ext::ExtensionMigration;
|
||||||
use sqlx::{PgConnection, PgPool, Row, Transaction, query};
|
use sqlx::{PgConnection, PgPool, Row, Transaction, query};
|
||||||
@@ -389,6 +392,19 @@ impl MigrationAuthority {
|
|||||||
if from < 13 {
|
if from < 13 {
|
||||||
artifact_cleanup_indexes_v13::apply(&mut transaction, &Self::sequence()[12]).await?;
|
artifact_cleanup_indexes_v13::apply(&mut transaction, &Self::sequence()[12]).await?;
|
||||||
}
|
}
|
||||||
|
match inspect(&mut transaction).await? {
|
||||||
|
MigrationPreflight::Current {
|
||||||
|
version: CURRENT_VERSION,
|
||||||
|
} => {}
|
||||||
|
_ => {
|
||||||
|
return Err(MigrationError::new(
|
||||||
|
"apply_failed",
|
||||||
|
"apply.postflight",
|
||||||
|
Some(CURRENT_VERSION),
|
||||||
|
"restore_known_good_backup",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
transaction
|
transaction
|
||||||
.commit()
|
.commit()
|
||||||
.await
|
.await
|
||||||
@@ -408,9 +424,15 @@ async fn inspect(connection: &mut PgConnection) -> Result<MigrationPreflight, Mi
|
|||||||
owned_exists |= relation_exists(connection, relation).await?;
|
owned_exists |= relation_exists(connection, relation).await?;
|
||||||
}
|
}
|
||||||
if owned_exists {
|
if owned_exists {
|
||||||
|
if is_ledgerless_legacy_baseline(connection, canonical_exists).await? {
|
||||||
|
return Ok(MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: CURRENT_VERSION,
|
||||||
|
});
|
||||||
|
}
|
||||||
return Err(MigrationError::new(
|
return Err(MigrationError::new(
|
||||||
"partial_sequence",
|
"partial_sequence",
|
||||||
"preflight.core",
|
"preflight.core_missing",
|
||||||
None,
|
None,
|
||||||
"restore_known_good_backup",
|
"restore_known_good_backup",
|
||||||
));
|
));
|
||||||
@@ -541,6 +563,47 @@ async fn inspect(connection: &mut PgConnection) -> Result<MigrationPreflight, Mi
|
|||||||
Ok(MigrationPreflight::Current { version: current })
|
Ok(MigrationPreflight::Current { version: current })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
async fn is_ledgerless_legacy_baseline(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
canonical_exists: bool,
|
||||||
|
) -> Result<bool, MigrationError> {
|
||||||
|
if canonical_exists {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
for relation in [
|
||||||
|
"__crank_migration_legacy_audit",
|
||||||
|
"master_key_identities",
|
||||||
|
"master_key_rotations",
|
||||||
|
"admin_bootstrap_contracts",
|
||||||
|
"admin_login_backoff",
|
||||||
|
"admin_security_audit_events",
|
||||||
|
"product_events",
|
||||||
|
"product_event_daily_rollups",
|
||||||
|
"onboarding_selections",
|
||||||
|
"artifact_blobs",
|
||||||
|
"artifact_sources",
|
||||||
|
] {
|
||||||
|
if relation_exists(connection, relation).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for relation in owned_relations::BASELINE {
|
||||||
|
if !relation_exists(connection, relation).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validate_required_relations(connection, owned_relations::BASELINE, 1).await?;
|
||||||
|
validate_ledgerless_baseline_fingerprint(connection).await?;
|
||||||
|
let mcp_ledger = relation_exists(connection, "__crank_mcp_migrations").await?;
|
||||||
|
if mcp_ledger {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
let mcp_sessions = relation_exists(connection, "mcp_transport_sessions").await?;
|
||||||
|
let extension_ledger = relation_exists(connection, "__crank_ext_migrations").await?;
|
||||||
|
validate_ledgerless_optional_fingerprints(connection, mcp_sessions, extension_ledger).await?;
|
||||||
|
inspect_optional_legacy_for_ledgerless_baseline(connection).await?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
async fn validate_core_ledger(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
async fn validate_core_ledger(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
||||||
let rows = query("select version, description, checksum from __crank_core_migrations order by version limit 2")
|
let rows = query("select version, description, checksum from __crank_core_migrations order by version limit 2")
|
||||||
.fetch_all(connection)
|
.fetch_all(connection)
|
||||||
@@ -549,7 +612,7 @@ async fn validate_core_ledger(connection: &mut PgConnection) -> Result<(), Migra
|
|||||||
if rows.len() != 1 {
|
if rows.len() != 1 {
|
||||||
return Err(MigrationError::new(
|
return Err(MigrationError::new(
|
||||||
"partial_sequence",
|
"partial_sequence",
|
||||||
"preflight.core",
|
"preflight.core_cardinality",
|
||||||
None,
|
None,
|
||||||
"restore_known_good_backup",
|
"restore_known_good_backup",
|
||||||
));
|
));
|
||||||
@@ -577,9 +640,21 @@ async fn validate_core_ledger(connection: &mut PgConnection) -> Result<(), Migra
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
async fn inspect_optional_legacy(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
async fn inspect_optional_legacy(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
||||||
|
inspect_optional_legacy_with_policy(connection, false).await
|
||||||
|
}
|
||||||
|
async fn inspect_optional_legacy_for_ledgerless_baseline(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
) -> Result<(), MigrationError> {
|
||||||
|
inspect_optional_legacy_with_policy(connection, true).await
|
||||||
|
}
|
||||||
|
async fn inspect_optional_legacy_with_policy(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
allow_sessions_without_ledger: bool,
|
||||||
|
) -> Result<(), MigrationError> {
|
||||||
let mcp_ledger = relation_exists(connection, "__crank_mcp_migrations").await?;
|
let mcp_ledger = relation_exists(connection, "__crank_mcp_migrations").await?;
|
||||||
let mcp_sessions = relation_exists(connection, "mcp_transport_sessions").await?;
|
let mcp_sessions = relation_exists(connection, "mcp_transport_sessions").await?;
|
||||||
if mcp_ledger != mcp_sessions {
|
if mcp_ledger != mcp_sessions && !(allow_sessions_without_ledger && !mcp_ledger && mcp_sessions)
|
||||||
|
{
|
||||||
return Err(MigrationError::new(
|
return Err(MigrationError::new(
|
||||||
"legacy_conflict",
|
"legacy_conflict",
|
||||||
"preflight.legacy_mcp",
|
"preflight.legacy_mcp",
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ fn sequence_is_deterministic_and_append_only() {
|
|||||||
MigrationAuthority::validate_sequence().unwrap();
|
MigrationAuthority::validate_sequence().unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
first.iter().map(|item| item.version).collect::<Vec<_>>(),
|
first.iter().map(|item| item.version).collect::<Vec<_>>(),
|
||||||
vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
|
vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
|
||||||
);
|
);
|
||||||
assert_eq!(first[0].checksum, "crank-community-baseline-v1");
|
assert_eq!(first[0].checksum, "crank-community-baseline-v1");
|
||||||
assert_eq!(first[0].source_digest, BASELINE_SOURCE_SHA256);
|
assert_eq!(first[0].source_digest, BASELINE_SOURCE_SHA256);
|
||||||
|
|||||||
@@ -325,6 +325,15 @@ pub(super) async fn validate_schema_fingerprint(
|
|||||||
.iter()
|
.iter()
|
||||||
.filter_map(|row| row.try_get::<String, _>("column_name").ok())
|
.filter_map(|row| row.try_get::<String, _>("column_name").ok())
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
let required = required
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|column| {
|
||||||
|
!(current_version == 1
|
||||||
|
&& *table == "__crank_ext_migrations"
|
||||||
|
&& *column == "checksum")
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
if actual.len() != required.len()
|
if actual.len() != required.len()
|
||||||
|| required
|
|| required
|
||||||
.iter()
|
.iter()
|
||||||
@@ -337,6 +346,9 @@ pub(super) async fn validate_schema_fingerprint(
|
|||||||
if !relation_exists(connection, table).await? {
|
if !relation_exists(connection, table).await? {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if current_version == 1 && *table == "__crank_ext_migrations" && *column == "checksum" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let row = query(
|
let row = query(
|
||||||
"select data_type, is_nullable from information_schema.columns
|
"select data_type, is_nullable from information_schema.columns
|
||||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||||
@@ -424,23 +436,25 @@ pub(super) async fn validate_schema_fingerprint(
|
|||||||
return Err(schema_error(current_version));
|
return Err(schema_error(current_version));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let required_indexes = [
|
if relation_exists(connection, "mcp_transport_sessions").await? {
|
||||||
"mcp_transport_sessions_workspace_agent_idx",
|
let required_indexes = [
|
||||||
"mcp_transport_sessions_expires_at_idx",
|
"mcp_transport_sessions_workspace_agent_idx",
|
||||||
];
|
"mcp_transport_sessions_expires_at_idx",
|
||||||
for index in required_indexes {
|
];
|
||||||
let present = query(
|
for index in required_indexes {
|
||||||
"select exists (select 1 from pg_catalog.pg_indexes
|
let present = query(
|
||||||
where schemaname = current_schema() and indexname = $1) as present",
|
"select exists (select 1 from pg_catalog.pg_indexes
|
||||||
)
|
where schemaname = current_schema() and indexname = $1) as present",
|
||||||
.bind(index)
|
)
|
||||||
.fetch_one(&mut *connection)
|
.bind(index)
|
||||||
.await
|
.fetch_one(&mut *connection)
|
||||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
.await
|
||||||
.try_get::<bool, _>("present")
|
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
.try_get::<bool, _>("present")
|
||||||
if !present {
|
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||||
return Err(schema_error(current_version));
|
if !present {
|
||||||
|
return Err(schema_error(current_version));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if current_version >= 3 {
|
if current_version >= 3 {
|
||||||
|
|||||||
@@ -0,0 +1,196 @@
|
|||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use sqlx::PgConnection;
|
||||||
|
|
||||||
|
use super::authority::MigrationError;
|
||||||
|
|
||||||
|
// Exact PostgreSQL 16 catalog contract produced by the last published
|
||||||
|
// pre-ledger Community schema (commit 8318e4b).
|
||||||
|
const LEDGERLESS_BASELINE_FINGERPRINT_SHA256: &str =
|
||||||
|
"36624ca42a28c1388f9c50e6d6c489a8e5ca1f73441a894d9af89e5e115b542c";
|
||||||
|
const LEDGERLESS_MCP_FINGERPRINTS_SHA256: &[&str] = &[
|
||||||
|
// Initial published session table.
|
||||||
|
"9240c3d85dbc9eeddb1cd99661ec8f7d8d6ece0e62ec486151dcc951a7f3c81c",
|
||||||
|
// Published session table after supports_elicitation was added.
|
||||||
|
"ac9f99a7e667552a07480d5d45379dd0a8b529b00ba98855ce6725535198249f",
|
||||||
|
];
|
||||||
|
const LEDGERLESS_EXTENSION_FINGERPRINT_SHA256: &str =
|
||||||
|
"0809a80c0bb6e80f68d0557006c4b62f2e63556f61f6fdedc5abc324950e2587";
|
||||||
|
|
||||||
|
const BASELINE_RELATIONS: &[&str] = &[
|
||||||
|
"workspaces",
|
||||||
|
"users",
|
||||||
|
"memberships",
|
||||||
|
"user_sessions",
|
||||||
|
"invitation_tokens",
|
||||||
|
"platform_api_keys",
|
||||||
|
"operations",
|
||||||
|
"operation_versions",
|
||||||
|
"published_operations",
|
||||||
|
"operation_samples",
|
||||||
|
"descriptors",
|
||||||
|
"agents",
|
||||||
|
"agent_versions",
|
||||||
|
"published_agents",
|
||||||
|
"agent_operation_bindings",
|
||||||
|
"secrets",
|
||||||
|
"secret_versions",
|
||||||
|
"auth_profiles",
|
||||||
|
"workspace_upstreams",
|
||||||
|
"yaml_import_jobs",
|
||||||
|
"import_jobs",
|
||||||
|
"approval_requests",
|
||||||
|
"invocation_logs",
|
||||||
|
"usage_rollups",
|
||||||
|
];
|
||||||
|
|
||||||
|
pub(super) async fn validate_ledgerless_baseline_fingerprint(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
) -> Result<(), MigrationError> {
|
||||||
|
let actual = catalog_fingerprint(connection, BASELINE_RELATIONS).await?;
|
||||||
|
if actual == LEDGERLESS_BASELINE_FINGERPRINT_SHA256 {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(fingerprint_error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn validate_ledgerless_optional_fingerprints(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
has_mcp_sessions: bool,
|
||||||
|
has_extension_ledger: bool,
|
||||||
|
) -> Result<(), MigrationError> {
|
||||||
|
if has_mcp_sessions {
|
||||||
|
let actual = catalog_fingerprint(connection, &["mcp_transport_sessions"]).await?;
|
||||||
|
if !LEDGERLESS_MCP_FINGERPRINTS_SHA256.contains(&actual.as_str()) {
|
||||||
|
return Err(fingerprint_error());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if has_extension_ledger {
|
||||||
|
let actual = catalog_fingerprint(connection, &["__crank_ext_migrations"]).await?;
|
||||||
|
if actual != LEDGERLESS_EXTENSION_FINGERPRINT_SHA256 {
|
||||||
|
return Err(fingerprint_error());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn catalog_fingerprint(
|
||||||
|
connection: &mut PgConnection,
|
||||||
|
relations: &[&str],
|
||||||
|
) -> Result<String, MigrationError> {
|
||||||
|
let relations = relations
|
||||||
|
.iter()
|
||||||
|
.map(|value| (*value).to_owned())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let unsafe_catalog_state = sqlx::query_scalar::<_, bool>(
|
||||||
|
"with selected(table_name) as (select unnest($1::text[]))
|
||||||
|
select
|
||||||
|
exists (
|
||||||
|
select 1 from pg_catalog.pg_class c
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = c.relnamespace
|
||||||
|
join selected s on s.table_name = c.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
and (c.relpersistence <> 'p' or c.relrowsecurity
|
||||||
|
or c.relforcerowsecurity or c.relreplident <> 'd')
|
||||||
|
)
|
||||||
|
or exists (
|
||||||
|
select 1 from pg_catalog.pg_index i
|
||||||
|
join pg_catalog.pg_class t on t.oid = i.indrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join selected s on s.table_name = t.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
and (not i.indisvalid or not i.indisready or not i.indislive)
|
||||||
|
)
|
||||||
|
or exists (
|
||||||
|
select 1 from pg_catalog.pg_policy p
|
||||||
|
join pg_catalog.pg_class t on t.oid = p.polrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join selected s on s.table_name = t.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
)
|
||||||
|
or exists (
|
||||||
|
select 1 from pg_catalog.pg_trigger tg
|
||||||
|
join pg_catalog.pg_class t on t.oid = tg.tgrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join selected s on s.table_name = t.relname
|
||||||
|
where n.nspname = current_schema() and tg.tgenabled <> 'O'
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.bind(relations.clone())
|
||||||
|
.fetch_one(&mut *connection)
|
||||||
|
.await
|
||||||
|
.map_err(|_| MigrationError::storage("preflight.legacy_fingerprint"))?;
|
||||||
|
if unsafe_catalog_state {
|
||||||
|
return Err(fingerprint_error());
|
||||||
|
}
|
||||||
|
let fingerprint = sqlx::query_scalar::<_, String>(
|
||||||
|
"with baseline(table_name) as (select unnest($1::text[])), relation_rows as (
|
||||||
|
select jsonb_build_array('relation', c.relname, c.relkind::text) item
|
||||||
|
from pg_catalog.pg_class c
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = c.relnamespace
|
||||||
|
join baseline b on b.table_name = c.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
), column_rows as (
|
||||||
|
select jsonb_build_array(
|
||||||
|
'column', c.table_name, c.column_name,
|
||||||
|
c.data_type, c.udt_name, c.is_nullable, coalesce(c.column_default, '')
|
||||||
|
) item
|
||||||
|
from information_schema.columns c
|
||||||
|
join baseline b using (table_name)
|
||||||
|
where c.table_schema = current_schema()
|
||||||
|
), constraint_rows as (
|
||||||
|
select jsonb_build_array(
|
||||||
|
'constraint', t.relname, c.conname, c.contype::text,
|
||||||
|
c.convalidated, pg_get_constraintdef(c.oid, true)
|
||||||
|
) item
|
||||||
|
from pg_catalog.pg_constraint c
|
||||||
|
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join baseline b on b.table_name = t.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
), index_rows as (
|
||||||
|
select jsonb_build_array(
|
||||||
|
'index', t.relname, idx.relname,
|
||||||
|
replace(pg_get_indexdef(i.indexrelid), format('%I.', current_schema()), '')
|
||||||
|
) item
|
||||||
|
from pg_catalog.pg_index i
|
||||||
|
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
|
||||||
|
join pg_catalog.pg_class t on t.oid = i.indrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join baseline b on b.table_name = t.relname
|
||||||
|
where n.nspname = current_schema()
|
||||||
|
), trigger_rows as (
|
||||||
|
select jsonb_build_array(
|
||||||
|
'trigger', t.relname, tg.tgname,
|
||||||
|
replace(pg_get_triggerdef(tg.oid, true), format('%I.', current_schema()), '')
|
||||||
|
) item
|
||||||
|
from pg_catalog.pg_trigger tg
|
||||||
|
join pg_catalog.pg_class t on t.oid = tg.tgrelid
|
||||||
|
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||||
|
join baseline b on b.table_name = t.relname
|
||||||
|
where n.nspname = current_schema() and not tg.tgisinternal
|
||||||
|
), all_rows as (
|
||||||
|
select item from relation_rows
|
||||||
|
union all select item from column_rows
|
||||||
|
union all select item from constraint_rows
|
||||||
|
union all select item from index_rows
|
||||||
|
union all select item from trigger_rows
|
||||||
|
)
|
||||||
|
select coalesce(jsonb_agg(item order by item::text), '[]'::jsonb)::text
|
||||||
|
from all_rows",
|
||||||
|
)
|
||||||
|
.bind(relations)
|
||||||
|
.fetch_one(connection)
|
||||||
|
.await
|
||||||
|
.map_err(|_| MigrationError::storage("preflight.legacy_fingerprint"))?;
|
||||||
|
Ok(format!("{:x}", Sha256::digest(fingerprint.as_bytes())))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fingerprint_error() -> MigrationError {
|
||||||
|
MigrationError::new(
|
||||||
|
"partial_sequence",
|
||||||
|
"preflight.legacy_fingerprint",
|
||||||
|
Some(1),
|
||||||
|
"restore_known_good_backup",
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ use crank_artifacts::ArtifactRef;
|
|||||||
|
|
||||||
const APPLICATION_RESULT_KEY: &str = "_crank_application_result";
|
const APPLICATION_RESULT_KEY: &str = "_crank_application_result";
|
||||||
const IMPORT_JOB_CLEANUP_BATCH: u32 = 128;
|
const IMPORT_JOB_CLEANUP_BATCH: u32 = 128;
|
||||||
|
const MAX_IMPORT_JOB_DOCUMENTS: usize = 32;
|
||||||
const DANGLING_OPENAPI_SOURCE_GRACE: time::Duration = time::Duration::minutes(5);
|
const DANGLING_OPENAPI_SOURCE_GRACE: time::Duration = time::Duration::minutes(5);
|
||||||
|
|
||||||
impl PostgresRegistry {
|
impl PostgresRegistry {
|
||||||
@@ -12,6 +13,7 @@ impl PostgresRegistry {
|
|||||||
request: CreateImportJobRequest<'_>,
|
request: CreateImportJobRequest<'_>,
|
||||||
) -> Result<(), RegistryError> {
|
) -> Result<(), RegistryError> {
|
||||||
validate_source_envelope(request.source, request.preview_payload)?;
|
validate_source_envelope(request.source, request.preview_payload)?;
|
||||||
|
validate_dependency_envelopes(request.preview_payload)?;
|
||||||
let mut transaction = self.pool.begin().await?;
|
let mut transaction = self.pool.begin().await?;
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"insert into import_jobs (
|
"insert into import_jobs (
|
||||||
@@ -81,6 +83,26 @@ impl PostgresRegistry {
|
|||||||
&self,
|
&self,
|
||||||
request: FinishImportJobRequest<'_>,
|
request: FinishImportJobRequest<'_>,
|
||||||
) -> Result<(), RegistryError> {
|
) -> Result<(), RegistryError> {
|
||||||
|
let mut transaction = self.pool.begin().await?;
|
||||||
|
let row = sqlx::query(
|
||||||
|
"select workspace_id, status, preview_payload
|
||||||
|
from import_jobs
|
||||||
|
where id = $1
|
||||||
|
for update",
|
||||||
|
)
|
||||||
|
.bind(request.id.as_str())
|
||||||
|
.fetch_optional(&mut *transaction)
|
||||||
|
.await?
|
||||||
|
.ok_or_else(|| RegistryError::ImportJobNotFound {
|
||||||
|
job_id: request.id.as_str().to_owned(),
|
||||||
|
})?;
|
||||||
|
let current_status =
|
||||||
|
deserialize_enum_text::<ImportJobStatus>(row.try_get("status")?, "status")?;
|
||||||
|
if request.status == ImportJobStatus::Failed && current_status == ImportJobStatus::Completed
|
||||||
|
{
|
||||||
|
transaction.commit().await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let result = sqlx::query(
|
let result = sqlx::query(
|
||||||
"update import_jobs
|
"update import_jobs
|
||||||
set status = $2,
|
set status = $2,
|
||||||
@@ -94,7 +116,7 @@ impl PostgresRegistry {
|
|||||||
.bind(request.created_operation_ids)
|
.bind(request.created_operation_ids)
|
||||||
.bind(request.error_text)
|
.bind(request.error_text)
|
||||||
.bind(request.finished_at)
|
.bind(request.finished_at)
|
||||||
.execute(&self.pool)
|
.execute(&mut *transaction)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if result.rows_affected() == 0 {
|
if result.rows_affected() == 0 {
|
||||||
@@ -103,6 +125,25 @@ impl PostgresRegistry {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if request.status == ImportJobStatus::Failed {
|
||||||
|
let workspace_id = WorkspaceId::new(row.try_get::<String, _>("workspace_id")?);
|
||||||
|
let payload = row.try_get::<Value, _>("preview_payload")?;
|
||||||
|
// Failure finalization must itself be fail-safe. The strict
|
||||||
|
// envelope parsers are used before Apply; cleanup only needs the
|
||||||
|
// bounded source identities and must not roll back the terminal
|
||||||
|
// state because some other payload field was corrupted.
|
||||||
|
for source_id in cleanup_source_ids(&payload) {
|
||||||
|
let _ = detach_source_in_transaction(
|
||||||
|
&mut transaction,
|
||||||
|
&workspace_id,
|
||||||
|
&source_id,
|
||||||
|
*request.finished_at,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
transaction.commit().await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,24 +161,18 @@ impl PostgresRegistry {
|
|||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
tx.rollback().await?;
|
tx.rollback().await?;
|
||||||
let error_text = error.to_string();
|
if !matches!(error, RegistryError::ImportJobAlreadyApplied { .. }) {
|
||||||
let _ = sqlx::query(
|
let empty = serde_json::json!([]);
|
||||||
"update import_jobs
|
let _ = self
|
||||||
set status = $3,
|
.finish_import_job(FinishImportJobRequest {
|
||||||
error_text = $4,
|
id: request.id,
|
||||||
finished_at = $5::timestamptz
|
status: ImportJobStatus::Failed,
|
||||||
where id = $1
|
created_operation_ids: &empty,
|
||||||
and workspace_id = $2
|
error_text: Some("import_apply_failed"),
|
||||||
and status <> $6",
|
finished_at: request.finished_at,
|
||||||
)
|
})
|
||||||
.bind(request.id.as_str())
|
.await;
|
||||||
.bind(request.workspace_id.as_str())
|
}
|
||||||
.bind(serialize_enum_text(&ImportJobStatus::Failed, "status")?)
|
|
||||||
.bind(error_text)
|
|
||||||
.bind(request.finished_at)
|
|
||||||
.bind(serialize_enum_text(&ImportJobStatus::Completed, "status")?)
|
|
||||||
.execute(&self.pool)
|
|
||||||
.await;
|
|
||||||
Err(error)
|
Err(error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -176,17 +211,13 @@ impl PostgresRegistry {
|
|||||||
let payload = row.try_get::<Value, _>("preview_payload")?;
|
let payload = row.try_get::<Value, _>("preview_payload")?;
|
||||||
// The job itself is expired regardless of whether a legacy or
|
// The job itself is expired regardless of whether a legacy or
|
||||||
// corrupt payload can be decoded. Do not let one bad row roll
|
// corrupt payload can be decoded. Do not let one bad row roll
|
||||||
// back cleanup for every tenant.
|
// back cleanup for every workspace.
|
||||||
if let Ok(Some(source)) = source_from_payload(&payload)
|
for source_id in cleanup_source_ids(&payload) {
|
||||||
&& detach_source_in_transaction(
|
if detach_source_in_transaction(&mut transaction, &workspace_id, &source_id, now)
|
||||||
&mut transaction,
|
.await?
|
||||||
&workspace_id,
|
{
|
||||||
&source.source_id,
|
detached_sources += 1;
|
||||||
now,
|
}
|
||||||
)
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
detached_sources += 1;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let deleted = if expired_ids.is_empty() {
|
let deleted = if expired_ids.is_empty() {
|
||||||
@@ -214,7 +245,14 @@ impl PostgresRegistry {
|
|||||||
select 1
|
select 1
|
||||||
from import_jobs j
|
from import_jobs j
|
||||||
where j.workspace_id = s.workspace_id
|
where j.workspace_id = s.workspace_id
|
||||||
and j.preview_payload -> 'source' ->> 'source_id' = s.source_id
|
and (
|
||||||
|
j.preview_payload -> 'source' ->> 'source_id' = s.source_id
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from jsonb_array_elements(coalesce(j.preview_payload -> 'dependencies', '[]'::jsonb)) d
|
||||||
|
where d ->> 'source_id' = s.source_id
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
order by s.created_at, s.workspace_id, s.source_id
|
order by s.created_at, s.workspace_id, s.source_id
|
||||||
limit $2
|
limit $2
|
||||||
@@ -283,6 +321,7 @@ async fn apply_import_job_transaction(
|
|||||||
source_from_payload(&preview_payload)?.ok_or(RegistryError::InvalidArtifactSource {
|
source_from_payload(&preview_payload)?.ok_or(RegistryError::InvalidArtifactSource {
|
||||||
field: "import_job_source",
|
field: "import_job_source",
|
||||||
})?;
|
})?;
|
||||||
|
let dependencies = dependencies_from_payload(&preview_payload)?;
|
||||||
|
|
||||||
if status == ImportJobStatus::Completed
|
if status == ImportJobStatus::Completed
|
||||||
&& let Some(result) = stored_application_result(&preview_payload)?
|
&& let Some(result) = stored_application_result(&preview_payload)?
|
||||||
@@ -323,6 +362,31 @@ async fn apply_import_job_transaction(
|
|||||||
if !source_is_current {
|
if !source_is_current {
|
||||||
return Err(RegistryError::SourceUnavailable);
|
return Err(RegistryError::SourceUnavailable);
|
||||||
}
|
}
|
||||||
|
for dependency in &dependencies {
|
||||||
|
let recorded = sqlx::query(
|
||||||
|
"select s.lifecycle, b.artifact_ref
|
||||||
|
from artifact_sources s
|
||||||
|
join artifact_blobs b on b.digest = s.blob_digest
|
||||||
|
where s.workspace_id = $1 and s.source_id = $2
|
||||||
|
for update",
|
||||||
|
)
|
||||||
|
.bind(request.workspace_id.as_str())
|
||||||
|
.bind(dependency.source_id.as_str())
|
||||||
|
.fetch_optional(&mut **tx)
|
||||||
|
.await?;
|
||||||
|
let current = recorded
|
||||||
|
.as_ref()
|
||||||
|
.map(|row| {
|
||||||
|
let lifecycle = row.try_get::<String, _>("lifecycle").ok();
|
||||||
|
let artifact_ref = row.try_get::<String, _>("artifact_ref").ok();
|
||||||
|
lifecycle.as_deref() == Some("active")
|
||||||
|
&& artifact_ref.as_deref() == Some(dependency.digest.as_str())
|
||||||
|
})
|
||||||
|
.unwrap_or(false);
|
||||||
|
if !current {
|
||||||
|
return Err(RegistryError::SourceUnavailable);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let mut result = ImportJobApplyResult {
|
let mut result = ImportJobApplyResult {
|
||||||
application_key: request.application_key.to_owned(),
|
application_key: request.application_key.to_owned(),
|
||||||
@@ -408,6 +472,15 @@ async fn apply_import_job_transaction(
|
|||||||
*request.finished_at,
|
*request.finished_at,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
for dependency in dependencies {
|
||||||
|
let _ = detach_source_in_transaction(
|
||||||
|
tx,
|
||||||
|
request.workspace_id,
|
||||||
|
&dependency.source_id,
|
||||||
|
*request.finished_at,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
@@ -433,6 +506,11 @@ fn validate_source_envelope(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn validate_dependency_envelopes(preview_payload: &Value) -> Result<(), RegistryError> {
|
||||||
|
dependencies_from_payload(preview_payload)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope>, RegistryError> {
|
fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope>, RegistryError> {
|
||||||
let Some(source) = payload.get("source") else {
|
let Some(source) = payload.get("source") else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -466,6 +544,73 @@ fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn dependencies_from_payload(
|
||||||
|
payload: &Value,
|
||||||
|
) -> Result<Vec<ImportJobSourceEnvelope>, RegistryError> {
|
||||||
|
let Some(value) = payload.get("dependencies") else {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
};
|
||||||
|
let items = value
|
||||||
|
.as_array()
|
||||||
|
.filter(|items| items.len() <= MAX_IMPORT_JOB_DOCUMENTS)
|
||||||
|
.ok_or(RegistryError::InvalidArtifactSource {
|
||||||
|
field: "import_job_dependencies",
|
||||||
|
})?;
|
||||||
|
let mut dependencies = Vec::with_capacity(items.len());
|
||||||
|
let mut identities = std::collections::BTreeSet::new();
|
||||||
|
for item in items {
|
||||||
|
let source_id = item
|
||||||
|
.get("source_id")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.filter(|value| value.len() <= 132)
|
||||||
|
.ok_or(RegistryError::InvalidArtifactSource {
|
||||||
|
field: "import_job_dependencies",
|
||||||
|
})?;
|
||||||
|
let digest = item.get("digest").and_then(Value::as_str).ok_or(
|
||||||
|
RegistryError::InvalidArtifactSource {
|
||||||
|
field: "import_job_dependencies",
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
let digest =
|
||||||
|
ArtifactRef::parse(digest).map_err(|_| RegistryError::InvalidArtifactSource {
|
||||||
|
field: "import_job_dependencies",
|
||||||
|
})?;
|
||||||
|
if !identities.insert((source_id.to_owned(), digest.as_str().to_owned())) {
|
||||||
|
return Err(RegistryError::InvalidArtifactSource {
|
||||||
|
field: "import_job_dependencies",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
dependencies.push(ImportJobSourceEnvelope {
|
||||||
|
source_id: ArtifactSourceId::new(source_id),
|
||||||
|
digest,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(dependencies)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cleanup_source_ids(payload: &Value) -> Vec<ArtifactSourceId> {
|
||||||
|
let mut identities = std::collections::BTreeSet::new();
|
||||||
|
if let Some(source_id) = payload
|
||||||
|
.pointer("/source/source_id")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.filter(|value| value.len() <= 132)
|
||||||
|
{
|
||||||
|
identities.insert(source_id.to_owned());
|
||||||
|
}
|
||||||
|
if let Some(dependencies) = payload.get("dependencies").and_then(Value::as_array) {
|
||||||
|
for source_id in dependencies
|
||||||
|
.iter()
|
||||||
|
.take(MAX_IMPORT_JOB_DOCUMENTS)
|
||||||
|
.filter_map(|item| item.get("source_id"))
|
||||||
|
.filter_map(Value::as_str)
|
||||||
|
.filter(|value| value.len() <= 132)
|
||||||
|
{
|
||||||
|
identities.insert(source_id.to_owned());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
identities.into_iter().map(ArtifactSourceId::new).collect()
|
||||||
|
}
|
||||||
|
|
||||||
async fn detach_source_in_transaction(
|
async fn detach_source_in_transaction(
|
||||||
transaction: &mut Transaction<'_, Postgres>,
|
transaction: &mut Transaction<'_, Postgres>,
|
||||||
workspace_id: &WorkspaceId,
|
workspace_id: &WorkspaceId,
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use crank_registry::{MigrationAuthority, MigrationPreflight, PostgresRegistry};
|
|||||||
use sqlx::Row;
|
use sqlx::Row;
|
||||||
|
|
||||||
mod artifact_metadata;
|
mod artifact_metadata;
|
||||||
|
mod legacy_adoption;
|
||||||
mod rollback;
|
mod rollback;
|
||||||
|
|
||||||
static EVENT_TRIGGER_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
static EVENT_TRIGGER_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||||
@@ -153,7 +154,7 @@ async fn changed_checksum_fails_closed_without_repair() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn legacy_core_baseline_is_consolidated_without_data_loss() {
|
async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
|
||||||
let database_url = crank_test_support::postgres_schema_url("test_legacy_core").await;
|
let database_url = crank_test_support::postgres_schema_url("test_legacy_core").await;
|
||||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
MigrationAuthority::apply(&pool).await.unwrap();
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
@@ -221,7 +222,8 @@ async fn legacy_core_baseline_is_consolidated_without_data_loss() {
|
|||||||
}
|
}
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"drop table __crank_migrations, __crank_migration_legacy_audit,
|
"drop table __crank_migrations, __crank_migration_legacy_audit,
|
||||||
__crank_mcp_migrations, mcp_transport_sessions, __crank_ext_migrations",
|
__crank_mcp_migrations, mcp_transport_sessions, __crank_ext_migrations,
|
||||||
|
__crank_core_migrations",
|
||||||
)
|
)
|
||||||
.execute(&pool)
|
.execute(&pool)
|
||||||
.await
|
.await
|
||||||
@@ -229,7 +231,7 @@ async fn legacy_core_baseline_is_consolidated_without_data_loss() {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
MigrationPreflight::MigrationRequired {
|
MigrationPreflight::MigrationRequired {
|
||||||
current: 1,
|
current: 0,
|
||||||
target: 13,
|
target: 13,
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@@ -937,6 +939,20 @@ async fn any_owned_relation_without_core_ledger_is_partial() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||||
assert_eq!(error.code(), "partial_sequence");
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.core_missing");
|
||||||
|
}
|
||||||
|
#[tokio::test]
|
||||||
|
async fn empty_core_ledger_is_reported_separately() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_empty_core_ledger").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
sqlx::query("delete from __crank_core_migrations")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.core_cardinality");
|
||||||
}
|
}
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn current_ledger_with_structural_drift_fails_closed() {
|
async fn current_ledger_with_structural_drift_fails_closed() {
|
||||||
|
|||||||
@@ -0,0 +1,439 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
async fn ledgerless_v1(pool: &sqlx::PgPool) {
|
||||||
|
MigrationAuthority::apply(pool).await.unwrap();
|
||||||
|
remove_v4_schema(pool).await;
|
||||||
|
remove_v3_schema(pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"drop table __crank_migrations, __crank_migration_legacy_audit;
|
||||||
|
drop table __crank_mcp_migrations;
|
||||||
|
drop index mcp_transport_sessions_expires_at_idx;
|
||||||
|
drop table __crank_ext_migrations;
|
||||||
|
create table __crank_ext_migrations (
|
||||||
|
extension_name text not null,
|
||||||
|
version integer not null,
|
||||||
|
applied_at timestamptz not null default now(),
|
||||||
|
primary key (extension_name, version)
|
||||||
|
);
|
||||||
|
drop table __crank_core_migrations;",
|
||||||
|
)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_optional_index_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_index").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"drop index mcp_transport_sessions_workspace_agent_idx;
|
||||||
|
create index mcp_transport_sessions_workspace_agent_idx
|
||||||
|
on mcp_transport_sessions(id);",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_disabled_integrity_triggers_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_triggers").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table memberships disable trigger all")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn published_ledgerless_baseline_upgrades_without_data_loss() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_baseline").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query(
|
||||||
|
"insert into operations
|
||||||
|
(id, workspace_id, name, display_name, protocol, status, created_at, updated_at)
|
||||||
|
values ('op_ledgerless', 'ws_default', 'ledgerless', 'Ledgerless', 'rest', 'draft', now(), now())",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: 13,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
|
||||||
|
let operation_count: i64 =
|
||||||
|
sqlx::query_scalar("select count(*) from operations where id = 'op_ledgerless'")
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(operation_count, 1);
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::Current { version: 13 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn published_ledgerless_historical_column_layout_upgrades_without_data_loss() {
|
||||||
|
let database_url =
|
||||||
|
crank_test_support::postgres_schema_url("test_ledgerless_historical_layout").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"alter table platform_api_keys
|
||||||
|
drop column key_kind,
|
||||||
|
drop column expires_at,
|
||||||
|
drop column allowed_origins_json;
|
||||||
|
alter table platform_api_keys
|
||||||
|
add column key_kind text not null default 'mcp_client',
|
||||||
|
add column expires_at timestamptz null,
|
||||||
|
add column allowed_origins_json jsonb not null default '[]'::jsonb;
|
||||||
|
|
||||||
|
drop index approval_requests_pending_fingerprint_idx;
|
||||||
|
alter table approval_requests
|
||||||
|
drop column execution_started_at,
|
||||||
|
drop column execution_attempts,
|
||||||
|
drop column request_fingerprint,
|
||||||
|
add column confirmation_title text not null default '',
|
||||||
|
add column confirmation_body text not null default '';
|
||||||
|
alter table approval_requests
|
||||||
|
alter column confirmation_title drop default,
|
||||||
|
alter column confirmation_body drop default,
|
||||||
|
drop column confirmation_title,
|
||||||
|
drop column confirmation_body,
|
||||||
|
add column execution_started_at timestamptz null,
|
||||||
|
add column execution_attempts integer not null default 0,
|
||||||
|
add column request_fingerprint text null;
|
||||||
|
create unique index approval_requests_pending_fingerprint_idx
|
||||||
|
on approval_requests(agent_id, operation_id, operation_version, request_fingerprint)
|
||||||
|
where status = 'pending' and request_fingerprint is not null;
|
||||||
|
|
||||||
|
insert into operations
|
||||||
|
(id, workspace_id, name, display_name, protocol, status, created_at, updated_at)
|
||||||
|
values ('op_historical_layout', 'ws_default', 'historical-layout', 'Historical layout', 'rest', 'draft', now(), now());
|
||||||
|
insert into agents
|
||||||
|
(id, workspace_id, slug, display_name, description, status, created_at, updated_at)
|
||||||
|
values ('agent_historical_layout', 'ws_default', 'historical-layout', 'Historical layout', '', 'draft', now(), now());
|
||||||
|
insert into platform_api_keys
|
||||||
|
(id, workspace_id, agent_id, name, prefix, secret_hash, key_kind, scopes_json, status,
|
||||||
|
created_at, expires_at, allowed_origins_json)
|
||||||
|
values ('key_historical_layout', 'ws_default', 'agent_historical_layout', 'Historical layout',
|
||||||
|
'cp_', 'hash', 'admin', '[]'::jsonb, 'active', now(),
|
||||||
|
'2030-01-02 03:04:05+00'::timestamptz, '[\"https://example.test\"]'::jsonb);
|
||||||
|
insert into approval_requests
|
||||||
|
(id, workspace_id, agent_id, operation_id, operation_version, status, risk_level,
|
||||||
|
request_payload_json, created_at, expires_at, execution_started_at,
|
||||||
|
execution_attempts, request_fingerprint)
|
||||||
|
values ('approval_historical_layout', 'ws_default', 'agent_historical_layout', 'op_historical_layout', 1,
|
||||||
|
'pending', 'high', '{\"layout\":\"historical\"}'::jsonb, now(), now() + interval '1 hour',
|
||||||
|
'2029-02-03 04:05:06+00'::timestamptz, 3, 'historical-fingerprint');",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let historical_ordinals: Vec<(String, i32)> = sqlx::query_as(
|
||||||
|
"select table_name || '.' || column_name, ordinal_position
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = current_schema()
|
||||||
|
and ((table_name = 'platform_api_keys'
|
||||||
|
and column_name in ('key_kind', 'expires_at', 'allowed_origins_json'))
|
||||||
|
or (table_name = 'approval_requests'
|
||||||
|
and column_name in ('execution_started_at', 'execution_attempts', 'request_fingerprint')))
|
||||||
|
order by table_name, ordinal_position",
|
||||||
|
)
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
historical_ordinals,
|
||||||
|
vec![
|
||||||
|
("approval_requests.execution_started_at".to_owned(), 22),
|
||||||
|
("approval_requests.execution_attempts".to_owned(), 23),
|
||||||
|
("approval_requests.request_fingerprint".to_owned(), 24),
|
||||||
|
("platform_api_keys.key_kind".to_owned(), 15),
|
||||||
|
("platform_api_keys.expires_at".to_owned(), 16),
|
||||||
|
("platform_api_keys.allowed_origins_json".to_owned(), 17),
|
||||||
|
],
|
||||||
|
"fixture must reproduce the published in-place column layout",
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: 13,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
|
||||||
|
let key_count: i64 = sqlx::query_scalar(
|
||||||
|
"select count(*) from platform_api_keys
|
||||||
|
where id = 'key_historical_layout'
|
||||||
|
and key_kind = 'admin'
|
||||||
|
and expires_at = '2030-01-02 03:04:05+00'::timestamptz
|
||||||
|
and allowed_origins_json = '[\"https://example.test\"]'::jsonb",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
key_count, 1,
|
||||||
|
"platform key row must survive the layout upgrade"
|
||||||
|
);
|
||||||
|
let approval_count: i64 = sqlx::query_scalar(
|
||||||
|
"select count(*) from approval_requests
|
||||||
|
where id = 'approval_historical_layout'
|
||||||
|
and request_payload_json = '{\"layout\":\"historical\"}'::jsonb
|
||||||
|
and execution_started_at = '2029-02-03 04:05:06+00'::timestamptz
|
||||||
|
and execution_attempts = 3
|
||||||
|
and request_fingerprint = 'historical-fingerprint'",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
approval_count, 1,
|
||||||
|
"approval row must survive the layout upgrade"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::Current { version: 13 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn published_initial_mcp_layout_is_accepted() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_initial_mcp").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"drop table mcp_transport_sessions;
|
||||||
|
create table mcp_transport_sessions (
|
||||||
|
id text primary key,
|
||||||
|
protocol_version text not null,
|
||||||
|
initialized boolean not null default false,
|
||||||
|
workspace_slug text not null,
|
||||||
|
agent_slug text not null,
|
||||||
|
created_at timestamptz not null,
|
||||||
|
updated_at timestamptz not null,
|
||||||
|
expires_at timestamptz null
|
||||||
|
);
|
||||||
|
create index mcp_transport_sessions_workspace_agent_idx
|
||||||
|
on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc);",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: 13,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
let supports_elicitation: bool = sqlx::query_scalar(
|
||||||
|
"select exists (
|
||||||
|
select 1 from information_schema.columns
|
||||||
|
where table_schema = current_schema()
|
||||||
|
and table_name = 'mcp_transport_sessions'
|
||||||
|
and column_name = 'supports_elicitation'
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(supports_elicitation);
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::Current { version: 13 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn published_in_place_mcp_upgrade_layout_is_accepted() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_mcp_upgrade").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::raw_sql(
|
||||||
|
"drop table mcp_transport_sessions;
|
||||||
|
create table mcp_transport_sessions (
|
||||||
|
id text primary key,
|
||||||
|
protocol_version text not null,
|
||||||
|
initialized boolean not null default false,
|
||||||
|
workspace_slug text not null,
|
||||||
|
agent_slug text not null,
|
||||||
|
created_at timestamptz not null,
|
||||||
|
updated_at timestamptz not null,
|
||||||
|
expires_at timestamptz null
|
||||||
|
);
|
||||||
|
create index mcp_transport_sessions_workspace_agent_idx
|
||||||
|
on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc);
|
||||||
|
alter table mcp_transport_sessions
|
||||||
|
add column supports_elicitation boolean not null default false;",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::MigrationRequired {
|
||||||
|
current: 0,
|
||||||
|
target: 13,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
MigrationAuthority::apply(&pool).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||||
|
MigrationPreflight::Current { version: 13 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_future_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_drift").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table invocation_logs add column trace_id text")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_default_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_default").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table workspaces alter column status set default 'active'")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_constraint_drift_is_rejected_without_writes() {
|
||||||
|
let database_url =
|
||||||
|
crank_test_support::postgres_schema_url("test_ledgerless_constraint_definition").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query(
|
||||||
|
"alter table workspaces
|
||||||
|
add constraint workspaces_status_nonempty check (status <> '')",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_rls_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_rls").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table workspaces enable row level security")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ledgerless_baseline_with_nullability_drift_is_rejected_without_writes() {
|
||||||
|
let database_url = crank_test_support::postgres_schema_url("test_ledgerless_constraint").await;
|
||||||
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||||
|
ledgerless_v1(&pool).await;
|
||||||
|
sqlx::query("alter table workspaces alter column status drop not null")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
|
||||||
|
assert_eq!(error.code(), "partial_sequence");
|
||||||
|
assert_eq!(error.stage(), "preflight.legacy_fingerprint");
|
||||||
|
let core_exists: bool = sqlx::query_scalar(
|
||||||
|
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
|
||||||
|
)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!core_exists, "rejected adoption must remain read-only");
|
||||||
|
}
|
||||||
@@ -25,7 +25,9 @@ pub use cache::{
|
|||||||
pub use cache_factory::{
|
pub use cache_factory::{
|
||||||
BuiltinCacheBackendFactory, CacheBackendFactory, SharedCacheBackendFactory,
|
BuiltinCacheBackendFactory, CacheBackendFactory, SharedCacheBackendFactory,
|
||||||
};
|
};
|
||||||
pub use crank_adapter_rest::OutboundHttpPolicy;
|
pub use crank_adapter_rest::{
|
||||||
|
ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy,
|
||||||
|
};
|
||||||
pub use error::RuntimeError;
|
pub use error::RuntimeError;
|
||||||
pub use execution_failure::{normalize_runtime_error, normalize_runtime_error_for_operation};
|
pub use execution_failure::{normalize_runtime_error, normalize_runtime_error_for_operation};
|
||||||
pub use execution_request::{
|
pub use execution_request::{
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS=
|
|||||||
CRANK_OUTBOUND_DENIED_HOSTS=
|
CRANK_OUTBOUND_DENIED_HOSTS=
|
||||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
||||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES=
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000
|
||||||
CRANK_ENVIRONMENT=production
|
CRANK_ENVIRONMENT=production
|
||||||
CRANK_LOG_LEVEL=
|
CRANK_LOG_LEVEL=
|
||||||
CRANK_SENTRY_DSN=
|
CRANK_SENTRY_DSN=
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS=
|
|||||||
CRANK_OUTBOUND_DENIED_HOSTS=
|
CRANK_OUTBOUND_DENIED_HOSTS=
|
||||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
||||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES=
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000
|
||||||
CRANK_ENVIRONMENT=production
|
CRANK_ENVIRONMENT=production
|
||||||
CRANK_LOG_LEVEL=
|
CRANK_LOG_LEVEL=
|
||||||
CRANK_SENTRY_DSN=
|
CRANK_SENTRY_DSN=
|
||||||
|
|||||||
@@ -127,12 +127,18 @@ services:
|
|||||||
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
||||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
||||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000}
|
||||||
volumes:
|
volumes:
|
||||||
- artifact_storage:${CRANK_STORAGE_ROOT:-/var/lib/crank/storage}
|
- artifact_storage:${CRANK_STORAGE_ROOT:-/var/lib/crank/storage}
|
||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_ADMIN_PUBLISH_PORT:-3001}:3001"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_ADMIN_PUBLISH_PORT:-3001}:3001"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3001", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
@@ -193,7 +199,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_MCP_PUBLISH_PORT:-3002}:3002"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_MCP_PUBLISH_PORT:-3002}:3002"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3002", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
required: false
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
|
POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
|
||||||
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
|
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
|
||||||
@@ -130,12 +131,18 @@ services:
|
|||||||
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
||||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
||||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000}
|
||||||
volumes:
|
volumes:
|
||||||
- artifact_storage:${CRANK_STORAGE_ROOT:-/var/lib/crank/storage}
|
- artifact_storage:${CRANK_STORAGE_ROOT:-/var/lib/crank/storage}
|
||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_ADMIN_PUBLISH_PORT:-3001}:3001"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_ADMIN_PUBLISH_PORT:-3001}:3001"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3001", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
@@ -196,7 +203,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_MCP_PUBLISH_PORT:-3002}:3002"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:${CRANK_MCP_PUBLISH_PORT:-3002}:3002"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3002", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|||||||
+8
-2
@@ -109,6 +109,12 @@ services:
|
|||||||
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
||||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
||||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000}
|
||||||
|
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000}
|
||||||
depends_on:
|
depends_on:
|
||||||
migrate:
|
migrate:
|
||||||
condition: service_completed_successfully
|
condition: service_completed_successfully
|
||||||
@@ -121,7 +127,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:3001:3001"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:3001:3001"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/ready"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3001", "/ready"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
@@ -184,7 +190,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:3002:3002"
|
- "${CRANK_PUBLISH_BIND:-127.0.0.1}:3002:3002"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/health"]
|
test: ["CMD", "/usr/local/bin/crank-http-healthcheck", "3002", "/health"]
|
||||||
interval: 15s
|
interval: 15s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|||||||
+4
-1
@@ -16,7 +16,7 @@ docker compose -f deploy/community/docker-compose.yml --env-file deploy/communit
|
|||||||
2. Создайте и проверьте согласованный backup PostgreSQL и artifact storage. Для первой пустой установки зафиксируйте, что восстанавливать нечего.
|
2. Создайте и проверьте согласованный backup PostgreSQL и artifact storage. Для первой пустой установки зафиксируйте, что восстанавливать нечего.
|
||||||
3. Проверьте immutable plan: `cargo run -p admin-api --bin crank-migrate -- plan --check` в source checkout либо `<compose> run --rm migrate crank-migrate plan` для образа.
|
3. Проверьте immutable plan: `cargo run -p admin-api --bin crank-migrate -- plan --check` в source checkout либо `<compose> run --rm migrate crank-migrate plan` для образа.
|
||||||
4. Примените sequence: `<compose> run --rm migrate crank-migrate apply`.
|
4. Примените sequence: `<compose> run --rm migrate crank-migrate apply`.
|
||||||
5. Повторите preflight и убедитесь в `{"status":"current","version":12}`.
|
5. Повторите preflight и убедитесь в `{"status":"current","version":13}`.
|
||||||
6. Только теперь запускайте long-running services: `<compose> up -d`.
|
6. Только теперь запускайте long-running services: `<compose> up -d`.
|
||||||
|
|
||||||
Обычный `up` также содержит обязательный migration job, но при upgrade он не заменяет предварительные preflight и backup. Migrator делает до десяти bounded попыток подключения с секундной паузой и затем безопасно завершается ошибкой.
|
Обычный `up` также содержит обязательный migration job, но при upgrade он не заменяет предварительные preflight и backup. Migrator делает до десяти bounded попыток подключения с секундной паузой и затем безопасно завершается ошибкой.
|
||||||
@@ -90,6 +90,9 @@ cargo run -p admin-api --bin crank-migrate -- plan --check
|
|||||||
sensitivity и source lifecycle принадлежат scoped relation. Claim token/expiry
|
sensitivity и source lifecycle принадлежат scoped relation. Claim token/expiry
|
||||||
зарезервированы bounded all-or-none contract для последующего reconciliation,
|
зарезервированы bounded all-or-none contract для последующего reconciliation,
|
||||||
но эта версия не запускает cleanup и не удаляет physical blobs.
|
но эта версия не запускает cleanup и не удаляет physical blobs.
|
||||||
|
- V13 — expand-only indexes, поддерживающие bounded artifact/import cleanup и
|
||||||
|
reconciliation ранее зарезервированных claim token/expiry; эта версия сама
|
||||||
|
не выполняет cleanup, не переписывает данные и не удаляет physical blobs.
|
||||||
- Каждая версия имеет contiguous `i64` version, стабильное имя, lowercase SHA-256, owner, phase, explicit readable schema min/max и backfill policy.
|
- Каждая версия имеет contiguous `i64` version, стабильное имя, lowercase SHA-256, owner, phase, explicit readable schema min/max и backfill policy.
|
||||||
- `migrate` требует bounded cursor/batch policy; `contract` дополнительно требует tracked compatibility evidence и закрытого окна.
|
- `migrate` требует bounded cursor/batch policy; `contract` дополнительно требует tracked compatibility evidence и закрытого окна.
|
||||||
- Добавление descriptor без executable implementation блокируется `invalid_contract` до DB I/O.
|
- Добавление descriptor без executable implementation блокируется `invalid_contract` до DB I/O.
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ Crank настраивается через переменные окружен
|
|||||||
| `CRANK_OUTBOUND_DENIED_HOSTS` | `outbound.denied_hosts` | `Shared` | `host_list/-` | `` | `-` | `Internal` | `Effective` |
|
| `CRANK_OUTBOUND_DENIED_HOSTS` | `outbound.denied_hosts` | `Shared` | `host_list/-` | `` | `-` | `Internal` | `Effective` |
|
||||||
| `CRANK_OUTBOUND_MAX_REQUEST_BYTES` | `outbound.max_request_bytes` | `Shared` | `u64/bytes` | `4194304` | `1..=67108864` | `Public` | `Effective` |
|
| `CRANK_OUTBOUND_MAX_REQUEST_BYTES` | `outbound.max_request_bytes` | `Shared` | `u64/bytes` | `4194304` | `1..=67108864` | `Public` | `Effective` |
|
||||||
| `CRANK_OUTBOUND_MAX_RESPONSE_BYTES` | `outbound.max_response_bytes` | `Shared` | `u64/bytes` | `4194304` | `1..=67108864` | `Public` | `Effective` |
|
| `CRANK_OUTBOUND_MAX_RESPONSE_BYTES` | `outbound.max_response_bytes` | `Shared` | `u64/bytes` | `4194304` | `1..=67108864` | `Public` | `Effective` |
|
||||||
|
| `CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES` | `import.external_references.allowed_url_prefixes` | `AdminApi` | `url_prefix_list/-` | `` | `-` | `Internal` | `Effective` |
|
||||||
|
| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH` | `import.external_references.max_depth` | `AdminApi` | `u32/edges` | `8` | `1..=32` | `Public` | `Effective` |
|
||||||
|
| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS` | `import.external_references.max_documents` | `AdminApi` | `u32/documents` | `32` | `1..=32` | `Public` | `Effective` |
|
||||||
|
| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES` | `import.external_references.max_fetch_bytes` | `AdminApi` | `u64/bytes` | `262144` | `1..=4194304` | `Public` | `Effective` |
|
||||||
|
| `CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS` | `import.external_references.fetch_timeout_ms` | `AdminApi` | `u64/milliseconds` | `10000` | `1..=300000` | `Public` | `Effective` |
|
||||||
|
| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES` | `import.external_references.max_expanded_nodes` | `AdminApi` | `u32/nodes` | `10000` | `1..=100000` | `Public` | `Effective` |
|
||||||
| `CRANK_ENVIRONMENT` | `observability.environment` | `Shared` | `label/-` | `development` | `-` | `Public` | `Effective` |
|
| `CRANK_ENVIRONMENT` | `observability.environment` | `Shared` | `label/-` | `development` | `-` | `Public` | `Effective` |
|
||||||
| `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` | `-` | `Public` | `Effective` |
|
| `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` | `-` | `Public` | `Effective` |
|
||||||
| `CRANK_SENTRY_DSN` | `observability.sentry_dsn` | `Shared` | `url/-` | `blank` | `-` | `Secret` | `Effective` |
|
| `CRANK_SENTRY_DSN` | `observability.sentry_dsn` | `Shared` | `url/-` | `blank` | `-` | `Secret` | `Effective` |
|
||||||
|
|||||||
@@ -192,6 +192,22 @@
|
|||||||
"source_digest": "052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe",
|
"source_digest": "052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe",
|
||||||
"transactional": true,
|
"transactional": true,
|
||||||
"version": 12
|
"version": 12
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"backfill": {
|
||||||
|
"kind": "none"
|
||||||
|
},
|
||||||
|
"checksum": "abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05",
|
||||||
|
"compatibility": "n-minus-one-readable",
|
||||||
|
"contract_evidence": null,
|
||||||
|
"name": "artifact-cleanup-indexes-v13",
|
||||||
|
"owner": "crank-registry",
|
||||||
|
"phase": "expand",
|
||||||
|
"readable_schema_max": 13,
|
||||||
|
"readable_schema_min": 12,
|
||||||
|
"source_digest": "abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05",
|
||||||
|
"transactional": true,
|
||||||
|
"version": 13
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -330,6 +330,99 @@
|
|||||||
"compatibility": null,
|
"compatibility": null,
|
||||||
"rules": []
|
"rules": []
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"semantic_path": "import.external_references.allowed_url_prefixes",
|
||||||
|
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES",
|
||||||
|
"process": "admin_api",
|
||||||
|
"value_type": "url_prefix_list",
|
||||||
|
"unit": null,
|
||||||
|
"default": "",
|
||||||
|
"required": false,
|
||||||
|
"minimum": null,
|
||||||
|
"maximum": null,
|
||||||
|
"sensitivity": "internal",
|
||||||
|
"mode": "effective",
|
||||||
|
"compatibility": null,
|
||||||
|
"rules": [
|
||||||
|
"empty list disables external OpenAPI reference fetching",
|
||||||
|
"each prefix must be canonical HTTP(S) without userinfo, query, or fragment"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"semantic_path": "import.external_references.max_depth",
|
||||||
|
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH",
|
||||||
|
"process": "admin_api",
|
||||||
|
"value_type": "u32",
|
||||||
|
"unit": "edges",
|
||||||
|
"default": "8",
|
||||||
|
"required": false,
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 32,
|
||||||
|
"sensitivity": "public",
|
||||||
|
"mode": "effective",
|
||||||
|
"compatibility": null,
|
||||||
|
"rules": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"semantic_path": "import.external_references.max_documents",
|
||||||
|
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS",
|
||||||
|
"process": "admin_api",
|
||||||
|
"value_type": "u32",
|
||||||
|
"unit": "documents",
|
||||||
|
"default": "32",
|
||||||
|
"required": false,
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 32,
|
||||||
|
"sensitivity": "public",
|
||||||
|
"mode": "effective",
|
||||||
|
"compatibility": null,
|
||||||
|
"rules": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"semantic_path": "import.external_references.max_fetch_bytes",
|
||||||
|
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES",
|
||||||
|
"process": "admin_api",
|
||||||
|
"value_type": "u64",
|
||||||
|
"unit": "bytes",
|
||||||
|
"default": "262144",
|
||||||
|
"required": false,
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 4194304,
|
||||||
|
"sensitivity": "public",
|
||||||
|
"mode": "effective",
|
||||||
|
"compatibility": null,
|
||||||
|
"rules": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"semantic_path": "import.external_references.fetch_timeout_ms",
|
||||||
|
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS",
|
||||||
|
"process": "admin_api",
|
||||||
|
"value_type": "u64",
|
||||||
|
"unit": "milliseconds",
|
||||||
|
"default": "10000",
|
||||||
|
"required": false,
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 300000,
|
||||||
|
"sensitivity": "public",
|
||||||
|
"mode": "effective",
|
||||||
|
"compatibility": null,
|
||||||
|
"rules": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"semantic_path": "import.external_references.max_expanded_nodes",
|
||||||
|
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES",
|
||||||
|
"process": "admin_api",
|
||||||
|
"value_type": "u32",
|
||||||
|
"unit": "nodes",
|
||||||
|
"default": "10000",
|
||||||
|
"required": false,
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 100000,
|
||||||
|
"sensitivity": "public",
|
||||||
|
"mode": "effective",
|
||||||
|
"compatibility": null,
|
||||||
|
"rules": []
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"semantic_path": "observability.environment",
|
"semantic_path": "observability.environment",
|
||||||
"env_name": "CRANK_ENVIRONMENT",
|
"env_name": "CRANK_ENVIRONMENT",
|
||||||
|
|||||||
@@ -21,10 +21,17 @@ class SmokeError(RuntimeError):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def safe_error(stage: str, code: str, status: int | None = None) -> SmokeError:
|
def safe_error(
|
||||||
|
stage: str,
|
||||||
|
code: str,
|
||||||
|
status: int | None = None,
|
||||||
|
trace_id: str | None = None,
|
||||||
|
) -> SmokeError:
|
||||||
message = f"stage={stage[:64]} code={code[:96]}"
|
message = f"stage={stage[:64]} code={code[:96]}"
|
||||||
if status is not None:
|
if status is not None:
|
||||||
message += f" status={status}"
|
message += f" status={status}"
|
||||||
|
if trace_id is not None:
|
||||||
|
message += f" trace_id={trace_id}"
|
||||||
return SmokeError(message[:256])
|
return SmokeError(message[:256])
|
||||||
|
|
||||||
|
|
||||||
@@ -134,6 +141,7 @@ class Client:
|
|||||||
def __init__(self, base_url: str, timeout_seconds: int) -> None:
|
def __init__(self, base_url: str, timeout_seconds: int) -> None:
|
||||||
self.base_url = base_url.rstrip("/")
|
self.base_url = base_url.rstrip("/")
|
||||||
self.timeout_seconds = timeout_seconds
|
self.timeout_seconds = timeout_seconds
|
||||||
|
self.csrf_token: str | None = None
|
||||||
cookie_jar = http.cookiejar.CookieJar()
|
cookie_jar = http.cookiejar.CookieJar()
|
||||||
self.opener = urllib.request.build_opener(
|
self.opener = urllib.request.build_opener(
|
||||||
urllib.request.HTTPCookieProcessor(cookie_jar)
|
urllib.request.HTTPCookieProcessor(cookie_jar)
|
||||||
@@ -156,6 +164,21 @@ class Client:
|
|||||||
request_headers = {"Accept": "application/json"}
|
request_headers = {"Accept": "application/json"}
|
||||||
if headers:
|
if headers:
|
||||||
request_headers.update(headers)
|
request_headers.update(headers)
|
||||||
|
if (
|
||||||
|
self.csrf_token
|
||||||
|
and method not in ("GET", "HEAD", "OPTIONS")
|
||||||
|
and (
|
||||||
|
path_or_url.startswith("/api/admin/")
|
||||||
|
or path_or_url.startswith("/api/auth/")
|
||||||
|
)
|
||||||
|
and path_or_url
|
||||||
|
not in (
|
||||||
|
"/api/auth/login",
|
||||||
|
"/api/auth/bootstrap/complete",
|
||||||
|
"/api/auth/session/csrf",
|
||||||
|
)
|
||||||
|
):
|
||||||
|
request_headers.setdefault("x-csrf-token", self.csrf_token)
|
||||||
if payload is not None:
|
if payload is not None:
|
||||||
data = json.dumps(payload).encode("utf-8")
|
data = json.dumps(payload).encode("utf-8")
|
||||||
request_headers["Content-Type"] = "application/json"
|
request_headers["Content-Type"] = "application/json"
|
||||||
@@ -203,12 +226,21 @@ def admin_path(workspace_id: str, suffix: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def login(client: Client, email: str, password: str) -> None:
|
def login(client: Client, email: str, password: str) -> None:
|
||||||
client.request_json(
|
response = client.request_json(
|
||||||
"POST",
|
"POST",
|
||||||
"/api/auth/login",
|
"/api/auth/login",
|
||||||
{"email": email, "password": password},
|
{"email": email, "password": password},
|
||||||
expected=(200,),
|
expected=(200,),
|
||||||
)
|
)
|
||||||
|
session = require_object(response.body, "login")
|
||||||
|
csrf_token = session.get("csrf_token")
|
||||||
|
if (
|
||||||
|
not isinstance(csrf_token, str)
|
||||||
|
or not 32 <= len(csrf_token) <= 256
|
||||||
|
or not all(character.isalnum() or character in "-_." for character in csrf_token)
|
||||||
|
):
|
||||||
|
raise safe_error("login", "invalid_csrf_token")
|
||||||
|
client.csrf_token = csrf_token
|
||||||
|
|
||||||
|
|
||||||
def resolve_workspace(
|
def resolve_workspace(
|
||||||
@@ -261,6 +293,29 @@ def create_operation(
|
|||||||
raise safe_error("operation_create", "invalid_response") from error
|
raise safe_error("operation_create", "invalid_response") from error
|
||||||
|
|
||||||
|
|
||||||
|
def is_safe_diagnostic_identifier(value: Any) -> bool:
|
||||||
|
return (
|
||||||
|
isinstance(value, str)
|
||||||
|
and 1 <= len(value) <= 64
|
||||||
|
and value[0].islower()
|
||||||
|
and value[0].isascii()
|
||||||
|
and all(
|
||||||
|
character.isascii()
|
||||||
|
and (character.islower() or character.isdigit() or character == "_")
|
||||||
|
for character in value
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_safe_trace_id(value: Any) -> bool:
|
||||||
|
return (
|
||||||
|
isinstance(value, str)
|
||||||
|
and len(value) == 32
|
||||||
|
and value != "0" * 32
|
||||||
|
and all(character in "0123456789abcdef" for character in value)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def run_operation_test(
|
def run_operation_test(
|
||||||
client: Client,
|
client: Client,
|
||||||
workspace_id: str,
|
workspace_id: str,
|
||||||
@@ -272,9 +327,24 @@ def run_operation_test(
|
|||||||
admin_path(workspace_id, f"/operations/{operation_id}/test-runs"),
|
admin_path(workspace_id, f"/operations/{operation_id}/test-runs"),
|
||||||
{"version": operation_version, "input": {"probe": "ok"}},
|
{"version": operation_version, "input": {"probe": "ok"}},
|
||||||
).body
|
).body
|
||||||
if not isinstance(result, dict) or result.get("ok") is not True:
|
if isinstance(result, dict) and result.get("ok") is True:
|
||||||
|
return
|
||||||
|
if not isinstance(result, dict) or result.get("ok") is not False:
|
||||||
raise safe_error("operation_test", "outcome_not_ok")
|
raise safe_error("operation_test", "outcome_not_ok")
|
||||||
|
|
||||||
|
errors = result.get("errors")
|
||||||
|
failure = errors[0] if isinstance(errors, list) and errors else None
|
||||||
|
code = failure.get("code") if isinstance(failure, dict) else None
|
||||||
|
stage = failure.get("stage") if isinstance(failure, dict) else None
|
||||||
|
trace_id = result.get("trace_id")
|
||||||
|
if (
|
||||||
|
is_safe_diagnostic_identifier(code)
|
||||||
|
and is_safe_diagnostic_identifier(stage)
|
||||||
|
and is_safe_trace_id(trace_id)
|
||||||
|
):
|
||||||
|
raise safe_error(stage, code, trace_id=trace_id)
|
||||||
|
raise safe_error("operation_test", "outcome_not_ok")
|
||||||
|
|
||||||
|
|
||||||
def operation_etag(client: Client, workspace_id: str, operation_id: str) -> str:
|
def operation_etag(client: Client, workspace_id: str, operation_id: str) -> str:
|
||||||
response = client.request_json(
|
response = client.request_json(
|
||||||
@@ -326,6 +396,17 @@ def create_agent(client: Client, workspace_id: str, agent_slug: str) -> tuple[st
|
|||||||
raise safe_error("agent_create", "invalid_response") from error
|
raise safe_error("agent_create", "invalid_response") from error
|
||||||
|
|
||||||
|
|
||||||
|
def agent_etag(client: Client, workspace_id: str, agent_id: str) -> str:
|
||||||
|
response = client.request_json(
|
||||||
|
"GET",
|
||||||
|
admin_path(workspace_id, f"/agents/{agent_id}"),
|
||||||
|
)
|
||||||
|
etag = response.headers.get("ETag") if response.headers is not None else None
|
||||||
|
if not isinstance(etag, str) or len(etag) > 128 or not etag.startswith('"') or not etag.endswith('"'):
|
||||||
|
raise safe_error("agent_precondition", "invalid_response")
|
||||||
|
return etag
|
||||||
|
|
||||||
|
|
||||||
def edit_and_archive_operation(
|
def edit_and_archive_operation(
|
||||||
client: Client,
|
client: Client,
|
||||||
workspace_id: str,
|
workspace_id: str,
|
||||||
@@ -377,11 +458,13 @@ def bind_and_publish_agent(
|
|||||||
"enabled": True,
|
"enabled": True,
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
headers={"If-Match": agent_etag(client, workspace_id, agent_id)},
|
||||||
)
|
)
|
||||||
published = client.request_json(
|
published = client.request_json(
|
||||||
"POST",
|
"POST",
|
||||||
admin_path(workspace_id, f"/agents/{agent_id}/publish"),
|
admin_path(workspace_id, f"/agents/{agent_id}/publish"),
|
||||||
{"version": agent_version},
|
{"version": agent_version},
|
||||||
|
headers={"If-Match": agent_etag(client, workspace_id, agent_id)},
|
||||||
).body
|
).body
|
||||||
try:
|
try:
|
||||||
published_version = int(published["published_version"])
|
published_version = int(published["published_version"])
|
||||||
@@ -436,8 +519,15 @@ def cleanup_smoke_assets(
|
|||||||
if agent_id:
|
if agent_id:
|
||||||
try:
|
try:
|
||||||
client.request_json(
|
client.request_json(
|
||||||
"DELETE",
|
"POST",
|
||||||
admin_path(workspace_id, f"/agents/{agent_id}"),
|
admin_path(workspace_id, f"/agents/{agent_id}/unpublish"),
|
||||||
|
headers={"If-Match": agent_etag(client, workspace_id, agent_id)},
|
||||||
|
expected=(200, 404),
|
||||||
|
)
|
||||||
|
client.request_json(
|
||||||
|
"POST",
|
||||||
|
admin_path(workspace_id, f"/agents/{agent_id}/archive"),
|
||||||
|
headers={"If-Match": agent_etag(client, workspace_id, agent_id)},
|
||||||
expected=(200, 404),
|
expected=(200, 404),
|
||||||
)
|
)
|
||||||
except SmokeError as error:
|
except SmokeError as error:
|
||||||
|
|||||||
@@ -90,26 +90,52 @@ def playwright_verdict(report: dict[str, Any], required_titles: list[str]) -> tu
|
|||||||
return "fail", counts
|
return "fail", counts
|
||||||
tests = list(iter_tests(report))
|
tests = list(iter_tests(report))
|
||||||
if not tests:
|
if not tests:
|
||||||
|
if required_titles:
|
||||||
|
counts["failed"] = len(set(required_titles))
|
||||||
|
return "fail", counts
|
||||||
counts["not_run"] = 1
|
counts["not_run"] = 1
|
||||||
return "not_run", counts
|
return "not_run", counts
|
||||||
required = {title: False for title in required_titles}
|
required = {title: False for title in required_titles}
|
||||||
for test, title in tests:
|
for test, title in tests:
|
||||||
|
if required and title not in required:
|
||||||
|
continue
|
||||||
status = test.get("status")
|
status = test.get("status")
|
||||||
results = test.get("results") if isinstance(test.get("results"), list) else []
|
results = test.get("results")
|
||||||
result_statuses = [result.get("status") for result in results if isinstance(result, dict)]
|
if not isinstance(results, list):
|
||||||
retries = [result.get("retry", 0) for result in results if isinstance(result, dict)]
|
|
||||||
if len(result_statuses) != len(results):
|
|
||||||
counts["not_run"] += 1
|
counts["not_run"] += 1
|
||||||
continue
|
continue
|
||||||
if status == "flaky" or any(isinstance(retry, int) and retry > 0 for retry in retries):
|
result_statuses: list[Any] = []
|
||||||
counts["flaky"] += 1
|
retries: list[int] = []
|
||||||
|
malformed = False
|
||||||
|
for result in results:
|
||||||
|
if not isinstance(result, dict):
|
||||||
|
malformed = True
|
||||||
|
break
|
||||||
|
result_status = result.get("status")
|
||||||
|
retry = result.get("retry", 0)
|
||||||
|
if result_status not in ("passed", "failed", "timedOut", "skipped", "interrupted") \
|
||||||
|
or type(retry) is not int or retry < 0:
|
||||||
|
malformed = True
|
||||||
|
break
|
||||||
|
result_statuses.append(result_status)
|
||||||
|
retries.append(retry)
|
||||||
|
if malformed:
|
||||||
|
counts["not_run"] += 1
|
||||||
|
continue
|
||||||
|
final_status = result_statuses[-1] if result_statuses else None
|
||||||
|
if status in ("unexpected", "failed", "timedOut", "interrupted") \
|
||||||
|
or final_status in ("failed", "timedOut", "interrupted"):
|
||||||
|
counts["failed"] += 1
|
||||||
|
elif status == "flaky":
|
||||||
|
if final_status == "passed" and any(retry > 0 for retry in retries):
|
||||||
|
counts["flaky"] += 1
|
||||||
|
else:
|
||||||
|
counts["not_run"] += 1
|
||||||
elif status == "skipped" or (not results and status in ("skipped", "expected")):
|
elif status == "skipped" or (not results and status in ("skipped", "expected")):
|
||||||
counts["skipped"] += 1
|
counts["skipped"] += 1
|
||||||
elif status in ("unexpected", "failed", "timedOut", "interrupted") or any(
|
elif any(retry > 0 for retry in retries):
|
||||||
result_status in ("failed", "timedOut", "interrupted") for result_status in result_statuses
|
counts["flaky"] += 1
|
||||||
):
|
elif status == "expected" and results and all(result_status == "passed" for result_status in result_statuses):
|
||||||
counts["failed"] += 1
|
|
||||||
elif results and all(result_status == "passed" for result_status in result_statuses):
|
|
||||||
counts["passed"] += 1
|
counts["passed"] += 1
|
||||||
if title in required:
|
if title in required:
|
||||||
required[title] = True
|
required[title] = True
|
||||||
|
|||||||
+121
-8
@@ -26,13 +26,28 @@ compose_profiles=""
|
|||||||
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
||||||
compose_profiles="--profile cache"
|
compose_profiles="--profile cache"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
compose() {
|
compose() {
|
||||||
# Intentional word splitting: compose_profiles is either empty or two arguments.
|
# Intentional word splitting: compose_profiles is either empty or two arguments.
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
docker compose $compose_profiles "$@"
|
docker compose $compose_profiles "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
compose_up() {
|
||||||
|
postgres_scale=""
|
||||||
|
if [ "$(env_value POSTGRES_HOST postgres)" != "postgres" ] \
|
||||||
|
&& compose config --services | grep -Fxq postgres; then
|
||||||
|
postgres_scale="--scale postgres=0"
|
||||||
|
fi
|
||||||
|
# Intentional word splitting: postgres_scale is either empty or two arguments.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
compose up -d --remove-orphans $postgres_scale
|
||||||
|
}
|
||||||
|
|
||||||
|
show_failure_diagnostics() {
|
||||||
|
compose ps >&2 || true
|
||||||
|
compose logs --no-color migrate >&2 || true
|
||||||
|
}
|
||||||
|
|
||||||
wait_for_stack() {
|
wait_for_stack() {
|
||||||
readiness_path="$1"
|
readiness_path="$1"
|
||||||
attempt=1
|
attempt=1
|
||||||
@@ -48,6 +63,34 @@ wait_for_stack() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
find_artifact_container() {
|
||||||
|
expected_root="$1"
|
||||||
|
expected_mount="$(printf 'volume\t%s' "$expected_root")"
|
||||||
|
|
||||||
|
for service in admin-api artifact-storage-init; do
|
||||||
|
candidates="$(compose ps -aq "$service" 2>/dev/null || true)"
|
||||||
|
# Intentional word splitting: Docker container IDs cannot contain whitespace.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
set -- $candidates
|
||||||
|
if [ "$#" -gt 1 ]; then
|
||||||
|
echo "Artifact backup found multiple $service containers; refusing an ambiguous volume source" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [ "$#" -eq 1 ]; then
|
||||||
|
candidate="$1"
|
||||||
|
oneoff="$(docker inspect --format '{{index .Config.Labels "com.docker.compose.oneoff"}}' "$candidate" 2>/dev/null || true)"
|
||||||
|
mounts="$(docker inspect --format '{{range .Mounts}}{{printf "%s\t%s\n" .Type .Destination}}{{end}}' "$candidate" 2>/dev/null || true)"
|
||||||
|
if [ "$oneoff" != "True" ] && [ "$oneoff" != "true" ] \
|
||||||
|
&& printf '%s\n' "$mounts" | grep -Fxq "$expected_mount"; then
|
||||||
|
printf '%s' "$candidate"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
create_backup() {
|
create_backup() {
|
||||||
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
backup_dir="$(pwd)/backups/${timestamp}"
|
backup_dir="$(pwd)/backups/${timestamp}"
|
||||||
@@ -86,10 +129,9 @@ create_backup() {
|
|||||||
--username "$postgres_user" --dbname "$postgres_db" \
|
--username "$postgres_user" --dbname "$postgres_db" \
|
||||||
--format custom --file /backup/postgres.dump
|
--format custom --file /backup/postgres.dump
|
||||||
|
|
||||||
admin_container="$(compose ps -q admin-api 2>/dev/null || true)"
|
storage_root="$(env_value_from "$backup_env_file" CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
||||||
if [ -n "$admin_container" ]; then
|
if artifact_container="$(find_artifact_container "$storage_root")"; then
|
||||||
storage_root="$(env_value_from "$backup_env_file" CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
docker run --rm --volumes-from "$artifact_container:ro" \
|
||||||
docker run --rm --volumes-from "$admin_container" \
|
|
||||||
-v "$backup_dir:/backup" alpine:3.21 \
|
-v "$backup_dir:/backup" alpine:3.21 \
|
||||||
tar -C "$storage_root" -czf /backup/artifacts.tar.gz .
|
tar -C "$storage_root" -czf /backup/artifacts.tar.gz .
|
||||||
elif [ "$previous_deployment" = true ]; then
|
elif [ "$previous_deployment" = true ]; then
|
||||||
@@ -104,6 +146,76 @@ create_backup() {
|
|||||||
| sort -nr | awk 'NR > 5 { print $2 }' | xargs -r rm -rf
|
| sort -nr | awk 'NR > 5 { print $2 }' | xargs -r rm -rf
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validate_backup_migration() {
|
||||||
|
admin_image="$(env_value CRANK_ADMIN_API_IMAGE)"
|
||||||
|
if [ -z "$admin_image" ]; then
|
||||||
|
echo "CRANK_ADMIN_API_IMAGE is required for shadow migration validation" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
shadow_id="crank-migration-check-$$"
|
||||||
|
shadow_network="${shadow_id}-network"
|
||||||
|
shadow_postgres="${shadow_id}-postgres"
|
||||||
|
shadow_password="crank-shadow-migration-password"
|
||||||
|
shadow_admin="crank_shadow_admin"
|
||||||
|
shadow_user="crank_shadow_owner"
|
||||||
|
shadow_cleanup() {
|
||||||
|
docker rm -fv "$shadow_postgres" >/dev/null 2>&1 || true
|
||||||
|
docker network rm "$shadow_network" >/dev/null 2>&1 || true
|
||||||
|
}
|
||||||
|
trap shadow_cleanup EXIT
|
||||||
|
trap 'exit 130' HUP INT TERM
|
||||||
|
|
||||||
|
docker network create "$shadow_network" >/dev/null
|
||||||
|
docker run -d --name "$shadow_postgres" --network "$shadow_network" \
|
||||||
|
-e POSTGRES_USER="$shadow_admin" \
|
||||||
|
-e POSTGRES_PASSWORD="$shadow_password" \
|
||||||
|
-e POSTGRES_DB=crank \
|
||||||
|
postgres:16-alpine >/dev/null
|
||||||
|
|
||||||
|
shadow_ready=false
|
||||||
|
attempt=1
|
||||||
|
while [ "$attempt" -le 30 ]; do
|
||||||
|
if docker logs "$shadow_postgres" 2>&1 | grep -q 'PostgreSQL init process complete' \
|
||||||
|
&& docker exec "$shadow_postgres" pg_isready --username "$shadow_admin" --dbname crank >/dev/null 2>&1; then
|
||||||
|
shadow_ready=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
attempt=$((attempt + 1))
|
||||||
|
done
|
||||||
|
if [ "$shadow_ready" != true ]; then
|
||||||
|
echo "Shadow PostgreSQL did not become ready" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker exec "$shadow_postgres" psql --username "$shadow_admin" --dbname crank \
|
||||||
|
--set ON_ERROR_STOP=1 \
|
||||||
|
--command "create role ${shadow_user} login password '${shadow_password}' nosuperuser nocreatedb nocreaterole noreplication" \
|
||||||
|
--command "alter database crank owner to ${shadow_user}"
|
||||||
|
|
||||||
|
timeout 10m docker run --rm --network "$shadow_network" \
|
||||||
|
-e PGPASSWORD="$shadow_password" \
|
||||||
|
-v "$backup_dir:/backup:ro" \
|
||||||
|
postgres:16-alpine \
|
||||||
|
pg_restore --host "$shadow_postgres" --username "$shadow_user" --dbname crank \
|
||||||
|
--no-owner --no-privileges --single-transaction --exit-on-error \
|
||||||
|
/backup/postgres.dump
|
||||||
|
|
||||||
|
shadow_database_url="postgres://${shadow_user}:${shadow_password}@${shadow_postgres}:5432/crank"
|
||||||
|
timeout 10m docker run --rm --network "$shadow_network" \
|
||||||
|
-e CRANK_DATABASE_URL="$shadow_database_url" \
|
||||||
|
"$admin_image" crank-migrate apply
|
||||||
|
shadow_preflight="$(timeout 2m docker run --rm --network "$shadow_network" \
|
||||||
|
-e CRANK_DATABASE_URL="$shadow_database_url" \
|
||||||
|
"$admin_image" crank-migrate preflight)"
|
||||||
|
printf '%s\n' "$shadow_preflight" | grep -Fxq '{"status":"current","version":13}'
|
||||||
|
|
||||||
|
shadow_cleanup
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
echo "Shadow migration validation passed"
|
||||||
|
}
|
||||||
|
|
||||||
rollback() {
|
rollback() {
|
||||||
echo "New release failed readiness; restoring previous deployment" >&2
|
echo "New release failed readiness; restoring previous deployment" >&2
|
||||||
if [ ! -f .env.previous ] || [ ! -f docker-compose.previous.yml ]; then
|
if [ ! -f .env.previous ] || [ ! -f docker-compose.previous.yml ]; then
|
||||||
@@ -119,16 +231,17 @@ rollback() {
|
|||||||
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
||||||
compose_profiles="--profile cache"
|
compose_profiles="--profile cache"
|
||||||
fi
|
fi
|
||||||
compose up -d --remove-orphans
|
compose_up
|
||||||
wait_for_stack health
|
wait_for_stack health
|
||||||
}
|
}
|
||||||
|
|
||||||
compose config -q
|
compose config -q
|
||||||
create_backup
|
create_backup
|
||||||
compose pull
|
compose pull
|
||||||
|
validate_backup_migration
|
||||||
|
|
||||||
if ! compose up -d --remove-orphans || ! wait_for_stack ready; then
|
if ! compose_up || ! wait_for_stack ready; then
|
||||||
compose ps >&2 || true
|
show_failure_diagnostics
|
||||||
rollback
|
rollback
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
port="${1:?port is required}"
|
||||||
|
path="${2:?path is required}"
|
||||||
|
|
||||||
|
if [[ ! "$port" =~ ^[0-9]{1,5}$ ]] || (( port < 1 || port > 65535 )); then
|
||||||
|
exit 64
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$path" != /* || "$path" == *$'\r'* || "$path" == *$'\n'* ]]; then
|
||||||
|
exit 64
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Bash provides /dev/tcp without adding a network client package to the runtime
|
||||||
|
# image. Compose still applies its own five-second timeout to the whole probe.
|
||||||
|
exec 3<>"/dev/tcp/127.0.0.1/${port}"
|
||||||
|
printf 'GET %s HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n' "$path" >&3
|
||||||
|
|
||||||
|
IFS=$'\r' read -r -t 2 status <&3
|
||||||
|
case "$status" in
|
||||||
|
'HTTP/1.0 200 '*|'HTTP/1.1 200 '*) ;;
|
||||||
|
*) exit 1 ;;
|
||||||
|
esac
|
||||||
@@ -32,6 +32,15 @@ compose() {
|
|||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
docker compose $compose_profiles "$@"
|
docker compose $compose_profiles "$@"
|
||||||
}
|
}
|
||||||
|
compose_up() {
|
||||||
|
postgres_scale=""
|
||||||
|
if [ "$(env_value POSTGRES_HOST postgres)" != "postgres" ] \
|
||||||
|
&& compose config --services | grep -Fxq postgres; then
|
||||||
|
postgres_scale="--scale postgres=0"
|
||||||
|
fi
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
compose up -d --remove-orphans $postgres_scale
|
||||||
|
}
|
||||||
|
|
||||||
compose stop admin-api mcp-server ui
|
compose stop admin-api mcp-server ui
|
||||||
|
|
||||||
@@ -40,13 +49,18 @@ postgres_port="$(env_value POSTGRES_PORT 5432)"
|
|||||||
postgres_db="$(env_value POSTGRES_DB crank)"
|
postgres_db="$(env_value POSTGRES_DB crank)"
|
||||||
postgres_user="$(env_value POSTGRES_USER crank)"
|
postgres_user="$(env_value POSTGRES_USER crank)"
|
||||||
postgres_password="$(env_value POSTGRES_PASSWORD)"
|
postgres_password="$(env_value POSTGRES_PASSWORD)"
|
||||||
|
docker run --rm \
|
||||||
|
-v "$backup_dir:/backup:ro" \
|
||||||
|
postgres:16-alpine \
|
||||||
|
pg_restore --list /backup/postgres.dump >/dev/null
|
||||||
docker run --rm --network host \
|
docker run --rm --network host \
|
||||||
-e PGPASSWORD="$postgres_password" \
|
-e PGPASSWORD="$postgres_password" \
|
||||||
-v "$backup_dir:/backup:ro" \
|
-v "$backup_dir:/backup:ro" \
|
||||||
postgres:16-alpine \
|
postgres:16-alpine \
|
||||||
pg_restore --host "$postgres_host" --port "$postgres_port" \
|
pg_restore --host "$postgres_host" --port "$postgres_port" \
|
||||||
--username "$postgres_user" --dbname "$postgres_db" \
|
--username "$postgres_user" --dbname "$postgres_db" \
|
||||||
--clean --if-exists --no-owner --no-privileges /backup/postgres.dump
|
--clean --if-exists --no-owner --no-privileges \
|
||||||
|
--single-transaction --exit-on-error /backup/postgres.dump
|
||||||
|
|
||||||
admin_container="$(compose ps -aq admin-api)"
|
admin_container="$(compose ps -aq admin-api)"
|
||||||
storage_root="$(env_value CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
storage_root="$(env_value CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
||||||
@@ -54,7 +68,7 @@ docker run --rm --volumes-from "$admin_container" \
|
|||||||
-v "$backup_dir:/backup:ro" alpine:3.21 sh -eu -c \
|
-v "$backup_dir:/backup:ro" alpine:3.21 sh -eu -c \
|
||||||
"find '$storage_root' -mindepth 1 -delete; tar -C '$storage_root' -xzf /backup/artifacts.tar.gz"
|
"find '$storage_root' -mindepth 1 -delete; tar -C '$storage_root' -xzf /backup/artifacts.tar.gz"
|
||||||
|
|
||||||
compose up -d --remove-orphans
|
compose_up
|
||||||
attempt=1
|
attempt=1
|
||||||
while [ "$attempt" -le 45 ]; do
|
while [ "$attempt" -le 45 ]; do
|
||||||
if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \
|
if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \
|
||||||
|
|||||||
@@ -17,6 +17,48 @@ def load_smoke_module():
|
|||||||
|
|
||||||
|
|
||||||
class AuthenticatedProductSmokeTests(unittest.TestCase):
|
class AuthenticatedProductSmokeTests(unittest.TestCase):
|
||||||
|
def test_client_attaches_csrf_only_to_browser_api_mutations(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
|
||||||
|
class Response:
|
||||||
|
status = 200
|
||||||
|
headers = {}
|
||||||
|
|
||||||
|
def read(self, _limit):
|
||||||
|
return b"{}"
|
||||||
|
|
||||||
|
class Opener:
|
||||||
|
def __init__(self):
|
||||||
|
self.requests = []
|
||||||
|
|
||||||
|
def open(self, request, timeout):
|
||||||
|
self.requests.append((request, timeout))
|
||||||
|
return Response()
|
||||||
|
|
||||||
|
client = smoke.Client("http://crank.test", 5)
|
||||||
|
opener = Opener()
|
||||||
|
client.opener = opener
|
||||||
|
client.csrf_token = "a" * 32
|
||||||
|
|
||||||
|
client.request_json("POST", "/api/admin/workspaces/ws/operations", {})
|
||||||
|
client.request_json("POST", "http://mcp.test/v1/ws/agent", {})
|
||||||
|
|
||||||
|
self.assertEqual(opener.requests[0][0].get_header("X-csrf-token"), "a" * 32)
|
||||||
|
self.assertIsNone(opener.requests[1][0].get_header("X-csrf-token"))
|
||||||
|
|
||||||
|
def test_login_keeps_server_issued_csrf_token(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
csrf_token = None
|
||||||
|
|
||||||
|
def request_json(self, *args, **kwargs):
|
||||||
|
return smoke.JsonResponse(200, {}, {"csrf_token": "b" * 32})
|
||||||
|
|
||||||
|
client = FakeClient()
|
||||||
|
smoke.login(client, "owner@crank.test", "safe-password")
|
||||||
|
self.assertEqual(client.csrf_token, "b" * 32)
|
||||||
|
|
||||||
def test_operation_payload_uses_internal_upstream(self) -> None:
|
def test_operation_payload_uses_internal_upstream(self) -> None:
|
||||||
smoke = load_smoke_module()
|
smoke = load_smoke_module()
|
||||||
|
|
||||||
@@ -136,6 +178,8 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
|||||||
return smoke.JsonResponse(200, {}, {"operation_id": "op_safe", "version": 7})
|
return smoke.JsonResponse(200, {}, {"operation_id": "op_safe", "version": 7})
|
||||||
if path.endswith("/operations/op_safe"):
|
if path.endswith("/operations/op_safe"):
|
||||||
return smoke.JsonResponse(200, {"ETag": '"safe-etag"'}, {"id": "op_safe"})
|
return smoke.JsonResponse(200, {"ETag": '"safe-etag"'}, {"id": "op_safe"})
|
||||||
|
if path.endswith("/agents/agent_safe"):
|
||||||
|
return smoke.JsonResponse(200, {"ETag": '"safe-agent-etag"'}, {"id": "agent_safe"})
|
||||||
if path.endswith("/publish") and "/operations/" in path:
|
if path.endswith("/publish") and "/operations/" in path:
|
||||||
return smoke.JsonResponse(200, {}, {"published_version": 7})
|
return smoke.JsonResponse(200, {}, {"published_version": 7})
|
||||||
if path.endswith("/agents"):
|
if path.endswith("/agents"):
|
||||||
@@ -158,11 +202,14 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
|||||||
self.assertEqual((agent_id, agent_version, published_agent_version), ("agent_safe", 3, 3))
|
self.assertEqual((agent_id, agent_version, published_agent_version), ("agent_safe", 3, 3))
|
||||||
binding = next(payload for _, path, payload, _ in client.requests if path.endswith("/bindings"))[0]
|
binding = next(payload for _, path, payload, _ in client.requests if path.endswith("/bindings"))[0]
|
||||||
self.assertEqual(binding["operation_version"], 7)
|
self.assertEqual(binding["operation_version"], 7)
|
||||||
|
binding_request = next(request for request in client.requests if request[1].endswith("/bindings"))
|
||||||
|
self.assertEqual(binding_request[3]["headers"], {"If-Match": '"safe-agent-etag"'})
|
||||||
publish = next(request for request in client.requests if request[1].endswith("/operations/op_safe/publish"))
|
publish = next(request for request in client.requests if request[1].endswith("/operations/op_safe/publish"))
|
||||||
self.assertEqual(publish[3]["headers"], {"If-Match": '"safe-etag"'})
|
self.assertEqual(publish[3]["headers"], {"If-Match": '"safe-etag"'})
|
||||||
|
|
||||||
def test_admin_test_run_uses_created_version_and_rejects_failed_outcome(self) -> None:
|
def test_admin_test_run_uses_created_version_and_reports_safe_typed_failure(self) -> None:
|
||||||
smoke = load_smoke_module()
|
smoke = load_smoke_module()
|
||||||
|
trace_id = "0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
class FakeClient:
|
class FakeClient:
|
||||||
def __init__(self, ok):
|
def __init__(self, ok):
|
||||||
@@ -171,7 +218,22 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
|||||||
|
|
||||||
def request_json(self, method, path, payload=None, **kwargs):
|
def request_json(self, method, path, payload=None, **kwargs):
|
||||||
self.payload = payload
|
self.payload = payload
|
||||||
return smoke.JsonResponse(200, {}, {"ok": self.ok, "errors": [{"message": "secret-canary"}]})
|
return smoke.JsonResponse(
|
||||||
|
200,
|
||||||
|
{},
|
||||||
|
{
|
||||||
|
"ok": self.ok,
|
||||||
|
"trace_id": trace_id,
|
||||||
|
"errors": [
|
||||||
|
{
|
||||||
|
"code": "outbound_target_rejected",
|
||||||
|
"stage": "adapter",
|
||||||
|
"message": "secret-canary",
|
||||||
|
"context": {"url": "https://private.invalid/token-canary"},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
passing = FakeClient(True)
|
passing = FakeClient(True)
|
||||||
smoke.run_operation_test(passing, "ws", "op", 9)
|
smoke.run_operation_test(passing, "ws", "op", 9)
|
||||||
@@ -179,7 +241,188 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
|||||||
|
|
||||||
with self.assertRaises(smoke.SmokeError) as raised:
|
with self.assertRaises(smoke.SmokeError) as raised:
|
||||||
smoke.run_operation_test(FakeClient(False), "ws", "op", 9)
|
smoke.run_operation_test(FakeClient(False), "ws", "op", 9)
|
||||||
self.assertNotIn("secret-canary", str(raised.exception))
|
self.assertEqual(
|
||||||
|
str(raised.exception),
|
||||||
|
"stage=adapter code=outbound_target_rejected trace_id=0123456789abcdef0123456789abcdef",
|
||||||
|
)
|
||||||
|
for forbidden in ("secret-canary", "private.invalid", "token-canary", "context"):
|
||||||
|
self.assertNotIn(forbidden, str(raised.exception))
|
||||||
|
|
||||||
|
def test_admin_test_run_fails_closed_for_malformed_diagnostics(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
valid_trace_id = "0123456789abcdef0123456789abcdef"
|
||||||
|
valid_failure = {"code": "upstream_timeout", "stage": "upstream"}
|
||||||
|
malformed_results = (
|
||||||
|
None,
|
||||||
|
[],
|
||||||
|
"secret-canary",
|
||||||
|
{"trace_id": valid_trace_id, "errors": [valid_failure]},
|
||||||
|
{"ok": None, "trace_id": valid_trace_id, "errors": [valid_failure]},
|
||||||
|
{"ok": 0, "trace_id": valid_trace_id, "errors": [valid_failure]},
|
||||||
|
{"ok": "false", "trace_id": valid_trace_id, "errors": [valid_failure]},
|
||||||
|
{"ok": False, "errors": []},
|
||||||
|
{"ok": False, "errors": {}},
|
||||||
|
{"ok": False, "trace_id": valid_trace_id, "errors": [None]},
|
||||||
|
{"ok": False, "errors": ["secret-canary"]},
|
||||||
|
{"ok": False, "errors": [valid_failure]},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id[:-1],
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id + "0",
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": "0" * 32,
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id.upper(),
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": "a" * 31 + "١",
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": "a" * 31 + "\n",
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": 42,
|
||||||
|
"errors": [valid_failure],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "a" * 65, "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "Upstream_timeout", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_таймаут", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": 42, "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": ""}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "a" * 65}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "Upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "вверх"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": 42}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "unsafe-value://secret-canary", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": valid_trace_id,
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "unsafe stage secret-canary"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": "not-a-trace-id-secret-canary",
|
||||||
|
"errors": [{"code": "upstream_timeout", "stage": "upstream"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def __init__(self, result):
|
||||||
|
self.result = result
|
||||||
|
|
||||||
|
def request_json(self, *args, **kwargs):
|
||||||
|
return smoke.JsonResponse(200, {}, self.result)
|
||||||
|
|
||||||
|
for result in malformed_results:
|
||||||
|
with self.subTest(result=result), self.assertRaises(smoke.SmokeError) as raised:
|
||||||
|
smoke.run_operation_test(FakeClient(result), "ws", "op", 9)
|
||||||
|
self.assertEqual(str(raised.exception), "stage=operation_test code=outcome_not_ok")
|
||||||
|
self.assertNotIn("secret-canary", str(raised.exception))
|
||||||
|
|
||||||
|
def test_admin_test_run_accepts_maximum_length_safe_diagnostics(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
identifier = "a" * 64
|
||||||
|
trace_id = "0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def request_json(self, *args, **kwargs):
|
||||||
|
return smoke.JsonResponse(
|
||||||
|
200,
|
||||||
|
{},
|
||||||
|
{
|
||||||
|
"ok": False,
|
||||||
|
"trace_id": trace_id,
|
||||||
|
"request_preview": {"credential": "secret-canary"},
|
||||||
|
"response_preview": {"token": "secret-canary"},
|
||||||
|
"errors": [{"code": identifier, "stage": identifier}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaises(smoke.SmokeError) as raised:
|
||||||
|
smoke.run_operation_test(FakeClient(), "ws", "op", 9)
|
||||||
|
rendered = str(raised.exception)
|
||||||
|
self.assertEqual(rendered, f"stage={identifier} code={identifier} trace_id={trace_id}")
|
||||||
|
self.assertLessEqual(len(rendered.encode("ascii")), 256)
|
||||||
|
self.assertNotIn("secret-canary", rendered)
|
||||||
|
|
||||||
|
def test_admin_test_run_success_ignores_malformed_diagnostics(self) -> None:
|
||||||
|
smoke = load_smoke_module()
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def request_json(self, *args, **kwargs):
|
||||||
|
return smoke.JsonResponse(
|
||||||
|
200,
|
||||||
|
{},
|
||||||
|
{
|
||||||
|
"ok": True,
|
||||||
|
"trace_id": "secret-canary",
|
||||||
|
"errors": "secret-canary",
|
||||||
|
"request_preview": {"token": "secret-canary"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
smoke.run_operation_test(FakeClient(), "ws", "op", 9)
|
||||||
|
|
||||||
def test_edit_and_archive_use_fresh_operation_preconditions(self) -> None:
|
def test_edit_and_archive_use_fresh_operation_preconditions(self) -> None:
|
||||||
smoke = load_smoke_module()
|
smoke = load_smoke_module()
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
|
|||||||
def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None:
|
def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None:
|
||||||
required = ["OpenAPI required scenario"]
|
required = ["OpenAPI required scenario"]
|
||||||
reports = [
|
reports = [
|
||||||
|
{"suites": []},
|
||||||
{"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]},
|
{"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]},
|
||||||
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]},
|
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]},
|
||||||
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]},
|
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]},
|
||||||
@@ -70,6 +71,73 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
|
|||||||
self.assertEqual(candidate["execution_verdict"], "fail")
|
self.assertEqual(candidate["execution_verdict"], "fail")
|
||||||
self.assertFalse(candidate["accepted"])
|
self.assertFalse(candidate["accepted"])
|
||||||
|
|
||||||
|
def test_required_scope_ignores_unrelated_flaky_tests(self) -> None:
|
||||||
|
required = "OpenAPI required scenario"
|
||||||
|
report = {
|
||||||
|
"suites": [
|
||||||
|
{
|
||||||
|
"specs": [
|
||||||
|
{
|
||||||
|
"title": required,
|
||||||
|
"tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Unrelated wizard scenario",
|
||||||
|
"tests": [
|
||||||
|
{
|
||||||
|
"status": "flaky",
|
||||||
|
"results": [
|
||||||
|
{"status": "failed", "retry": 0},
|
||||||
|
{"status": "passed", "retry": 1},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{"title": "Unrelated failure", "tests": [{"status": "unexpected", "results": [{"status": "failed", "retry": 0}]}]},
|
||||||
|
{"title": "Unrelated skip", "tests": [{"status": "skipped", "results": []}]},
|
||||||
|
{"title": "Unrelated malformed", "tests": [{"status": "expected", "results": None}]},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
result, output, temporary = self.run_playwright(report, [required])
|
||||||
|
self.addCleanup(temporary.cleanup)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
||||||
|
self.assertEqual(candidate["execution_verdict"], "pass")
|
||||||
|
self.assertTrue(candidate["accepted"])
|
||||||
|
self.assertEqual(candidate["summary"], {"passed": 1, "failed": 0, "flaky": 0, "skipped": 0, "not_run": 0})
|
||||||
|
|
||||||
|
def test_final_failure_after_retry_is_failed_not_flaky(self) -> None:
|
||||||
|
for status in ("unexpected", "flaky"):
|
||||||
|
with self.subTest(status=status):
|
||||||
|
report = {
|
||||||
|
"suites": [
|
||||||
|
{
|
||||||
|
"specs": [
|
||||||
|
{
|
||||||
|
"tests": [
|
||||||
|
{
|
||||||
|
"status": status,
|
||||||
|
"results": [
|
||||||
|
{"status": "failed", "retry": 0},
|
||||||
|
{"status": "failed", "retry": 1},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
result, output, temporary = self.run_playwright(report)
|
||||||
|
self.addCleanup(temporary.cleanup)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
||||||
|
self.assertEqual(candidate["execution_verdict"], "fail")
|
||||||
|
self.assertEqual(candidate["summary"]["failed"], 1)
|
||||||
|
self.assertEqual(candidate["summary"]["flaky"], 0)
|
||||||
|
|
||||||
def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
|
def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
|
||||||
canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
|
canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
|
||||||
report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
|
report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
|
||||||
@@ -84,6 +152,8 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
|
|||||||
reports = [
|
reports = [
|
||||||
{"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
|
{"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
|
||||||
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
|
||||||
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": None}]}]}]},
|
||||||
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": True}]}]}]}]},
|
||||||
]
|
]
|
||||||
for report in reports:
|
for report in reports:
|
||||||
with self.subTest(report=report):
|
with self.subTest(report=report):
|
||||||
|
|||||||
Reference in New Issue
Block a user