feat: harden community production foundation through story 1.5
This commit is contained in:
@@ -47,6 +47,11 @@ CRANK_STAGING_ADMIN_PASSWORD=secret \
|
||||
CRANK_PRODUCT_SMOKE_KEEP_ASSETS=1 ./scripts/authenticated-product-smoke.sh https://crank.example.com
|
||||
```
|
||||
|
||||
Smoke выполняет Admin test-run до publish и фиксирует точные Operation version
|
||||
и Agent revision. Для split-port local stack можно передать `--mcp-base-url` и
|
||||
пустой `--mcp-path-prefix`; `--summary-output` пишет bounded safe summary без
|
||||
ключей, cookies и payloads.
|
||||
|
||||
## `check-rust-code-health.sh`
|
||||
|
||||
Проверяет базовые правила сопровождаемости Rust-кода:
|
||||
@@ -73,6 +78,28 @@ CRANK_PRODUCT_SMOKE_KEEP_ASSETS=1 ./scripts/authenticated-product-smoke.sh https
|
||||
./scripts/check-rust-boundaries.sh
|
||||
```
|
||||
|
||||
## Typed runtime configuration contract
|
||||
|
||||
Единый Rust registry генерирует machine contract, sections `.env.example` и
|
||||
таблицу параметров. Проверка drift не изменяет файлы:
|
||||
|
||||
```bash
|
||||
cargo run -p crank-config --bin crank-config-contract -- --check
|
||||
python3 scripts/check-runtime-config.py --root .
|
||||
python3 scripts/check-config-boundaries.py --root .
|
||||
```
|
||||
|
||||
После намеренного изменения registry artifacts обновляются через
|
||||
`crank-config-contract --write`, просматриваются в diff и проверяются командой
|
||||
`just config-contract-check`. Explicit `--files` у boundary checker-а
|
||||
позволяет проверять новые/untracked Rust-файлы.
|
||||
|
||||
## Versioned migration contract
|
||||
|
||||
`just migration-contract-check` сравнивает canonical Rust migration sequence с
|
||||
committed machine contract. PostgreSQL DDL вне `crank-registry::migrations`
|
||||
блокируется Rust module boundary check.
|
||||
|
||||
## `check-community-scope.sh`
|
||||
|
||||
Проверяет, что в community-репозиторий не попали функции и тексты за пределами
|
||||
@@ -87,3 +114,51 @@ Unit-тесты checker-а лежат в `tests/unit`:
|
||||
```bash
|
||||
python3 -m unittest discover -s tests/unit
|
||||
```
|
||||
|
||||
Новые или ещё не tracked файлы по умолчанию не видны wrapper-у. Перед handoff
|
||||
передайте только файлы текущего изменения явно:
|
||||
|
||||
```bash
|
||||
python3 scripts/check-community-scope.py --root . --files <paths...>
|
||||
```
|
||||
|
||||
## `validate-capability-inventory.py`
|
||||
|
||||
Проверяет canonical `docs/capability-inventory.json` по versioned schema,
|
||||
обязательным FR, Community boundary и существующим evidence-ссылкам:
|
||||
|
||||
```bash
|
||||
python3 scripts/validate-capability-inventory.py \
|
||||
--root . \
|
||||
--inventory docs/capability-inventory.json \
|
||||
--schema docs/schemas/capability-inventory.schema.json \
|
||||
$(for n in $(seq 1 54); do printf -- '--required-fr FR-%s ' "$n"; done)
|
||||
```
|
||||
|
||||
Статусы `planned`, `gap` и `blocked` валидны, но не считаются pass.
|
||||
|
||||
## Capability baseline tools
|
||||
|
||||
`collect-capability-baseline.py` принимает только allowlisted command reports
|
||||
или bounded Playwright JSON, вычисляет verdict и удаляет raw output, paths и
|
||||
attachments. Retry-pass становится `flaky`, а skipped/missing/non-zero result
|
||||
не становится pass.
|
||||
|
||||
```bash
|
||||
python3 scripts/collect-capability-baseline.py playwright \
|
||||
--report <temporary-report.json> \
|
||||
--output <candidate.json> \
|
||||
--source-revision <git-sha> \
|
||||
--environment-class community-test \
|
||||
--flow-id ui-operation-lifecycle
|
||||
```
|
||||
|
||||
После review candidate и обновления canonical artifacts пересчитайте exact-byte
|
||||
SHA-256 в manifest и запустите:
|
||||
|
||||
```bash
|
||||
python3 scripts/validate-capability-baseline.py \
|
||||
--root . \
|
||||
--manifest docs/capability-baseline/manifest.json \
|
||||
--schema docs/schemas/capability-baseline.schema.json
|
||||
```
|
||||
|
||||
@@ -7,6 +7,7 @@ import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -20,6 +21,19 @@ class SmokeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def safe_error(stage: str, code: str, status: int | None = None) -> SmokeError:
|
||||
message = f"stage={stage[:64]} code={code[:96]}"
|
||||
if status is not None:
|
||||
message += f" status={status}"
|
||||
return SmokeError(message[:256])
|
||||
|
||||
|
||||
def require_object(value: Any, stage: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise safe_error(stage, "invalid_response")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass
|
||||
class JsonResponse:
|
||||
status: int
|
||||
@@ -94,8 +108,14 @@ def build_operation_payload(name: str, upstream_base_url: str) -> dict[str, Any]
|
||||
}
|
||||
|
||||
|
||||
def agent_mcp_url(base_url: str, workspace_slug: str, agent_slug: str) -> str:
|
||||
return f"{base_url.rstrip('/')}/mcp/v1/{workspace_slug}/{agent_slug}"
|
||||
def agent_mcp_url(
|
||||
base_url: str,
|
||||
workspace_slug: str,
|
||||
agent_slug: str,
|
||||
path_prefix: str = "/mcp",
|
||||
) -> str:
|
||||
normalized_prefix = "/" + path_prefix.strip("/") if path_prefix.strip("/") else ""
|
||||
return f"{base_url.rstrip('/')}{normalized_prefix}/v1/{workspace_slug}/{agent_slug}"
|
||||
|
||||
|
||||
def tools_call_payload(tool_name: str, arguments: dict[str, Any]) -> dict[str, Any]:
|
||||
@@ -150,17 +170,30 @@ class Client:
|
||||
try:
|
||||
response = self.opener.open(request, timeout=self.timeout_seconds)
|
||||
status = response.status
|
||||
raw = response.read().decode("utf-8")
|
||||
body = json.loads(raw) if raw else None
|
||||
raw_bytes = response.read(1_048_577)
|
||||
if len(raw_bytes) > 1_048_576:
|
||||
raise safe_error("http", "response_too_large", status=status)
|
||||
try:
|
||||
raw = raw_bytes.decode("utf-8")
|
||||
body = json.loads(raw) if raw else None
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, ValueError, RecursionError) as error:
|
||||
raise safe_error("http", "invalid_json", status=status) from error
|
||||
headers_obj = response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
status = error.code
|
||||
raw = error.read().decode("utf-8")
|
||||
body = json.loads(raw) if raw else None
|
||||
raw = error.read(1_048_577)
|
||||
if len(raw) > 1_048_576:
|
||||
raise safe_error("http", "response_too_large", status=status)
|
||||
try:
|
||||
body = json.loads(raw.decode("utf-8")) if raw else None
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, ValueError, RecursionError) as parse_error:
|
||||
raise safe_error("http", "invalid_json", status=status) from parse_error
|
||||
headers_obj = error.headers
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as error:
|
||||
raise safe_error("http", "request_failed") from error
|
||||
|
||||
if status not in expected:
|
||||
raise SmokeError(f"{method} {url} returned {status}: {body}")
|
||||
raise safe_error("http", "unexpected_status", status=status)
|
||||
|
||||
return JsonResponse(status=status, headers=headers_obj, body=body)
|
||||
|
||||
@@ -183,14 +216,16 @@ def resolve_workspace(
|
||||
fallback_workspace_id: str,
|
||||
fallback_workspace_slug: str,
|
||||
) -> tuple[str, str]:
|
||||
session = client.request_json("GET", "/api/auth/session").body
|
||||
session = require_object(client.request_json("GET", "/api/auth/session").body, "workspace")
|
||||
memberships = session.get("memberships") or []
|
||||
if not isinstance(memberships, list) or any(not isinstance(item, dict) for item in memberships):
|
||||
raise safe_error("workspace", "invalid_response")
|
||||
current_workspace_id = session.get("current_workspace_id") or fallback_workspace_id
|
||||
membership = next(
|
||||
(
|
||||
item
|
||||
for item in memberships
|
||||
if item.get("workspace", {}).get("id") == current_workspace_id
|
||||
if isinstance(item.get("workspace"), dict) and item["workspace"].get("id") == current_workspace_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -198,12 +233,14 @@ def resolve_workspace(
|
||||
membership = memberships[0]
|
||||
|
||||
workspace = membership.get("workspace", {}) if membership else {}
|
||||
if not isinstance(workspace, dict):
|
||||
raise safe_error("workspace", "invalid_response")
|
||||
workspace_id = workspace.get("id") or current_workspace_id
|
||||
workspace_slug = workspace.get("slug") or fallback_workspace_slug
|
||||
if not workspace_id:
|
||||
raise SmokeError("authenticated session does not include a workspace id")
|
||||
raise safe_error("workspace", "missing_id")
|
||||
if not workspace_slug:
|
||||
raise SmokeError("authenticated session does not include a workspace slug")
|
||||
raise safe_error("workspace", "missing_slug")
|
||||
return workspace_id, workspace_slug
|
||||
|
||||
|
||||
@@ -212,24 +249,54 @@ def create_operation(
|
||||
workspace_id: str,
|
||||
operation_name: str,
|
||||
internal_upstream: str,
|
||||
) -> str:
|
||||
) -> tuple[str, int]:
|
||||
created = client.request_json(
|
||||
"POST",
|
||||
admin_path(workspace_id, "/operations"),
|
||||
build_operation_payload(operation_name, internal_upstream),
|
||||
).body
|
||||
return created["operation_id"]
|
||||
try:
|
||||
return str(created["operation_id"]), int(created["version"])
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise safe_error("operation_create", "invalid_response") from error
|
||||
|
||||
|
||||
def publish_operation(client: Client, workspace_id: str, operation_id: str) -> None:
|
||||
client.request_json(
|
||||
def run_operation_test(
|
||||
client: Client,
|
||||
workspace_id: str,
|
||||
operation_id: str,
|
||||
operation_version: int,
|
||||
) -> None:
|
||||
result = client.request_json(
|
||||
"POST",
|
||||
admin_path(workspace_id, f"/operations/{operation_id}/test-runs"),
|
||||
{"version": operation_version, "input": {"probe": "ok"}},
|
||||
).body
|
||||
if not isinstance(result, dict) or result.get("ok") is not True:
|
||||
raise safe_error("operation_test", "outcome_not_ok")
|
||||
|
||||
|
||||
def publish_operation(
|
||||
client: Client,
|
||||
workspace_id: str,
|
||||
operation_id: str,
|
||||
operation_version: int,
|
||||
) -> int:
|
||||
published = client.request_json(
|
||||
"POST",
|
||||
admin_path(workspace_id, f"/operations/{operation_id}/publish"),
|
||||
{"version": 1},
|
||||
)
|
||||
{"version": operation_version},
|
||||
).body
|
||||
try:
|
||||
published_version = int(published["published_version"])
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise safe_error("operation_publish", "invalid_response") from error
|
||||
if published_version != operation_version:
|
||||
raise safe_error("operation_publish", "version_mismatch")
|
||||
return published_version
|
||||
|
||||
|
||||
def create_agent(client: Client, workspace_id: str, agent_slug: str) -> str:
|
||||
def create_agent(client: Client, workspace_id: str, agent_slug: str) -> tuple[str, int]:
|
||||
created = client.request_json(
|
||||
"POST",
|
||||
admin_path(workspace_id, "/agents"),
|
||||
@@ -241,23 +308,28 @@ def create_agent(client: Client, workspace_id: str, agent_slug: str) -> str:
|
||||
"tool_selection_policy": {},
|
||||
},
|
||||
).body
|
||||
return created["agent_id"]
|
||||
try:
|
||||
return str(created["agent_id"]), int(created["version"])
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise safe_error("agent_create", "invalid_response") from error
|
||||
|
||||
|
||||
def bind_and_publish_agent(
|
||||
client: Client,
|
||||
workspace_id: str,
|
||||
agent_id: str,
|
||||
agent_version: int,
|
||||
operation_id: str,
|
||||
operation_version: int,
|
||||
tool_name: str,
|
||||
) -> None:
|
||||
) -> int:
|
||||
client.request_json(
|
||||
"POST",
|
||||
admin_path(workspace_id, f"/agents/{agent_id}/bindings"),
|
||||
[
|
||||
{
|
||||
"operation_id": operation_id,
|
||||
"operation_version": 1,
|
||||
"operation_version": operation_version,
|
||||
"tool_name": tool_name,
|
||||
"tool_title": "Check internal service health",
|
||||
"tool_description_override": None,
|
||||
@@ -265,11 +337,18 @@ def bind_and_publish_agent(
|
||||
}
|
||||
],
|
||||
)
|
||||
client.request_json(
|
||||
published = client.request_json(
|
||||
"POST",
|
||||
admin_path(workspace_id, f"/agents/{agent_id}/publish"),
|
||||
{"version": 1},
|
||||
)
|
||||
{"version": agent_version},
|
||||
).body
|
||||
try:
|
||||
published_version = int(published["published_version"])
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise safe_error("agent_publish", "invalid_response") from error
|
||||
if published_version != agent_version:
|
||||
raise safe_error("agent_publish", "version_mismatch")
|
||||
return published_version
|
||||
|
||||
|
||||
def create_agent_key(client: Client, workspace_id: str, agent_id: str) -> tuple[str, str]:
|
||||
@@ -278,7 +357,18 @@ def create_agent_key(client: Client, workspace_id: str, agent_id: str) -> tuple[
|
||||
admin_path(workspace_id, f"/agents/{agent_id}/platform-api-keys"),
|
||||
{"name": f"smoke-key-{int(time.time())}", "scopes": ["read", "write"]},
|
||||
).body
|
||||
return created["secret"], created["api_key"]["api_key"]["id"]
|
||||
try:
|
||||
secret = created["secret"]
|
||||
key_id = created["api_key"]["api_key"]["id"]
|
||||
except (KeyError, TypeError) as error:
|
||||
raise safe_error("key_create", "invalid_response") from error
|
||||
if not nonempty_string(secret) or not nonempty_string(key_id):
|
||||
raise safe_error("key_create", "invalid_response")
|
||||
return secret, key_id
|
||||
|
||||
|
||||
def nonempty_string(value: Any) -> bool:
|
||||
return isinstance(value, str) and bool(value.strip())
|
||||
|
||||
|
||||
def cleanup_smoke_assets(
|
||||
@@ -347,7 +437,7 @@ def initialize_mcp_session(client: Client, mcp_url: str, api_key: str) -> str:
|
||||
)
|
||||
session_id = initialized.headers.get("MCP-Session-Id")
|
||||
if not session_id:
|
||||
raise SmokeError("MCP initialize response did not include MCP-Session-Id")
|
||||
raise safe_error("mcp_initialize", "missing_session_id")
|
||||
|
||||
client.request_json(
|
||||
"POST",
|
||||
@@ -368,6 +458,20 @@ def initialize_mcp_session(client: Client, mcp_url: str, api_key: str) -> str:
|
||||
return session_id
|
||||
|
||||
|
||||
def terminate_mcp_session(client: Client, mcp_url: str, api_key: str, session_id: str) -> None:
|
||||
client.request_json(
|
||||
"DELETE",
|
||||
mcp_url,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"MCP-Session-Id": session_id,
|
||||
"MCP-Protocol-Version": MCP_PROTOCOL_VERSION,
|
||||
},
|
||||
expected=(204, 404),
|
||||
)
|
||||
|
||||
|
||||
def call_mcp(
|
||||
client: Client,
|
||||
mcp_url: str,
|
||||
@@ -388,13 +492,50 @@ def call_mcp(
|
||||
).body
|
||||
|
||||
|
||||
def validate_tools_list(value: Any, expected_name: str) -> None:
|
||||
payload = require_object(value, "mcp_tools_list")
|
||||
result = payload.get("result")
|
||||
tools = result.get("tools") if isinstance(result, dict) else None
|
||||
if not isinstance(tools, list) or any(not isinstance(tool, dict) or not nonempty_string(tool.get("name")) for tool in tools):
|
||||
raise safe_error("mcp_tools_list", "invalid_response")
|
||||
if expected_name not in [tool["name"] for tool in tools]:
|
||||
raise safe_error("mcp_tools_list", "published_tool_missing")
|
||||
|
||||
|
||||
def validate_tool_call_result(value: Any) -> None:
|
||||
payload = require_object(value, "mcp_tools_call")
|
||||
result = payload.get("result")
|
||||
if "error" in payload or not isinstance(result, dict) or result.get("isError") is not False:
|
||||
raise safe_error("mcp_tools_call", "outcome_not_ok")
|
||||
structured = result.get("structuredContent")
|
||||
if not isinstance(structured, dict) or structured.get("status") != "ok":
|
||||
raise safe_error("mcp_tools_call", "unexpected_output")
|
||||
|
||||
|
||||
def build_safe_summary(
|
||||
operation_id: str,
|
||||
operation_version: int,
|
||||
agent_id: str,
|
||||
agent_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"agent_id": agent_id[:128],
|
||||
"agent_revision": agent_revision,
|
||||
"command_id": "authenticated-product-smoke",
|
||||
"operation_id": operation_id[:128],
|
||||
"operation_version": operation_version,
|
||||
"stages": ["admin_test", "operation_publish", "agent_publish", "mcp_list", "mcp_call"],
|
||||
"verdict": "pass",
|
||||
}
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> None:
|
||||
admin_email = os.environ.get("CRANK_STAGING_ADMIN_EMAIL")
|
||||
admin_password = os.environ.get("CRANK_STAGING_ADMIN_PASSWORD")
|
||||
if not admin_email:
|
||||
raise SmokeError("CRANK_STAGING_ADMIN_EMAIL is required")
|
||||
raise safe_error("configuration", "missing_admin_email")
|
||||
if not admin_password:
|
||||
raise SmokeError("CRANK_STAGING_ADMIN_PASSWORD is required")
|
||||
raise safe_error("configuration", "missing_admin_password")
|
||||
|
||||
timestamp = int(time.time())
|
||||
operation_name = f"internal_health_smoke_{timestamp}"
|
||||
@@ -403,8 +544,11 @@ def run(args: argparse.Namespace) -> None:
|
||||
operation_id = None
|
||||
agent_id = None
|
||||
key_id = None
|
||||
api_key = None
|
||||
mcp_url = None
|
||||
session_id = None
|
||||
|
||||
print(f"authenticated product smoke: {args.base_url.rstrip('/')}")
|
||||
print("authenticated product smoke: started")
|
||||
login(client, admin_email, admin_password)
|
||||
print("login: ok")
|
||||
workspace_id, workspace_slug = resolve_workspace(
|
||||
@@ -414,49 +558,80 @@ def run(args: argparse.Namespace) -> None:
|
||||
)
|
||||
print(f"workspace: {workspace_id} / {workspace_slug}")
|
||||
|
||||
operation_id = create_operation(
|
||||
client,
|
||||
workspace_id,
|
||||
operation_name,
|
||||
args.internal_upstream,
|
||||
)
|
||||
print(f"operation created: {operation_id}")
|
||||
publish_operation(client, workspace_id, operation_id)
|
||||
print("operation published: v1")
|
||||
try:
|
||||
operation_id, operation_version = create_operation(
|
||||
client,
|
||||
workspace_id,
|
||||
operation_name,
|
||||
args.internal_upstream,
|
||||
)
|
||||
print(f"operation created: {operation_id} version={operation_version}")
|
||||
run_operation_test(client, workspace_id, operation_id, operation_version)
|
||||
print("operation test: ok")
|
||||
published_operation_version = publish_operation(
|
||||
client, workspace_id, operation_id, operation_version
|
||||
)
|
||||
print(f"operation published: version={published_operation_version}")
|
||||
|
||||
agent_id = create_agent(client, workspace_id, agent_slug)
|
||||
bind_and_publish_agent(client, workspace_id, agent_id, operation_id, operation_name)
|
||||
print(f"agent published: {agent_id}")
|
||||
agent_id, agent_version = create_agent(client, workspace_id, agent_slug)
|
||||
published_agent_version = bind_and_publish_agent(
|
||||
client,
|
||||
workspace_id,
|
||||
agent_id,
|
||||
agent_version,
|
||||
operation_id,
|
||||
published_operation_version,
|
||||
operation_name,
|
||||
)
|
||||
print(f"agent published: {agent_id} revision={published_agent_version}")
|
||||
|
||||
api_key, key_id = create_agent_key(client, workspace_id, agent_id)
|
||||
mcp_url = agent_mcp_url(args.base_url, workspace_slug, agent_slug)
|
||||
session_id = initialize_mcp_session(client, mcp_url, api_key)
|
||||
print("mcp initialized: ok")
|
||||
api_key, key_id = create_agent_key(client, workspace_id, agent_id)
|
||||
mcp_url = agent_mcp_url(
|
||||
args.mcp_base_url or args.base_url,
|
||||
workspace_slug,
|
||||
agent_slug,
|
||||
args.mcp_path_prefix,
|
||||
)
|
||||
session_id = initialize_mcp_session(client, mcp_url, api_key)
|
||||
print("mcp initialized: ok")
|
||||
|
||||
tools = call_mcp(
|
||||
client,
|
||||
mcp_url,
|
||||
api_key,
|
||||
session_id,
|
||||
{"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}},
|
||||
)
|
||||
tool_names = [tool["name"] for tool in tools.get("result", {}).get("tools", [])]
|
||||
if operation_name not in tool_names:
|
||||
raise SmokeError(f"published tool {operation_name} not found in tools/list: {tool_names}")
|
||||
print("tools/list: ok")
|
||||
tools = call_mcp(
|
||||
client,
|
||||
mcp_url,
|
||||
api_key,
|
||||
session_id,
|
||||
{"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}},
|
||||
)
|
||||
validate_tools_list(tools, operation_name)
|
||||
print("tools/list: ok")
|
||||
|
||||
result = call_mcp(
|
||||
client,
|
||||
mcp_url,
|
||||
api_key,
|
||||
session_id,
|
||||
tools_call_payload(operation_name, {"probe": "ok"}),
|
||||
)
|
||||
if "error" in result:
|
||||
raise SmokeError(f"tools/call returned error: {result['error']}")
|
||||
print("tools/call: ok")
|
||||
cleanup_smoke_assets(client, workspace_id, operation_id, agent_id, key_id)
|
||||
print("authenticated product smoke completed")
|
||||
result = call_mcp(
|
||||
client,
|
||||
mcp_url,
|
||||
api_key,
|
||||
session_id,
|
||||
tools_call_payload(operation_name, {"probe": "ok"}),
|
||||
)
|
||||
validate_tool_call_result(result)
|
||||
print("tools/call: ok")
|
||||
summary = build_safe_summary(
|
||||
operation_id, published_operation_version, agent_id, published_agent_version
|
||||
)
|
||||
if args.summary_output:
|
||||
try:
|
||||
Path(args.summary_output).write_text(
|
||||
json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
except OSError as error:
|
||||
raise safe_error("summary", "write_failed") from error
|
||||
print("authenticated product smoke completed")
|
||||
finally:
|
||||
if mcp_url and api_key and session_id:
|
||||
try:
|
||||
terminate_mcp_session(client, mcp_url, api_key, session_id)
|
||||
except SmokeError as error:
|
||||
print(f"cleanup warning: session termination failed: {error}")
|
||||
cleanup_smoke_assets(client, workspace_id, operation_id, agent_id, key_id)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -467,6 +642,16 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--workspace-id", default=DEFAULT_WORKSPACE_ID)
|
||||
parser.add_argument("--workspace-slug", default=DEFAULT_WORKSPACE_SLUG)
|
||||
parser.add_argument("--internal-upstream", default=DEFAULT_INTERNAL_UPSTREAM)
|
||||
parser.add_argument(
|
||||
"--mcp-base-url",
|
||||
help="Optional direct MCP base URL for a local split-port stack.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mcp-path-prefix",
|
||||
default="/mcp",
|
||||
help="MCP proxy prefix; use an empty value for a direct MCP listener.",
|
||||
)
|
||||
parser.add_argument("--summary-output")
|
||||
parser.add_argument(
|
||||
"--timeout-seconds",
|
||||
type=int,
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
DEFAULT_EXCLUDED_PATHS = {
|
||||
@@ -26,7 +28,14 @@ ALLOW_DIRECTIVE = re.compile(
|
||||
)
|
||||
|
||||
FORBIDDEN_PATTERNS = [
|
||||
("enterprise", re.compile(r"\benterprise\b", re.IGNORECASE)),
|
||||
(
|
||||
"multi-workspace",
|
||||
re.compile(r"\bmulti[-_ ]workspace\b", re.IGNORECASE),
|
||||
),
|
||||
(
|
||||
"enterprise",
|
||||
re.compile(r"\benterprise\b|\benterprise_rbac\b", re.IGNORECASE),
|
||||
),
|
||||
("cloud", re.compile(r"\bcloud\b", re.IGNORECASE)),
|
||||
("commercial", re.compile(r"\bcommercial\b|коммер", re.IGNORECASE)),
|
||||
("cloud-russian", re.compile(r"облач", re.IGNORECASE)),
|
||||
@@ -47,8 +56,26 @@ FORBIDDEN_PATTERNS = [
|
||||
("machine-token", re.compile(r"machine token", re.IGNORECASE)),
|
||||
("token-issuer", re.compile(r"token issuer", re.IGNORECASE)),
|
||||
("request-variables", re.compile(r"request\.variables", re.IGNORECASE)),
|
||||
(
|
||||
"non-rest-upstream",
|
||||
re.compile(r"\bnon[-_ ]rest[-_ ]upstream\b", re.IGNORECASE),
|
||||
),
|
||||
(
|
||||
"distributed-load-targets",
|
||||
re.compile(r"\barbitrary[-_ ]distributed[-_ ]load[-_ ]targets\b", re.IGNORECASE),
|
||||
),
|
||||
]
|
||||
|
||||
MAX_PATH_LENGTH = 1024
|
||||
MAX_FINDINGS = 1000
|
||||
MAX_REPORT_BYTES = 65_536
|
||||
BINARY_PROBE_BYTES = 4096
|
||||
MAX_LINE_CHARACTERS = 65_536
|
||||
MAX_TEXT_FILE_BYTES = 16 * 1024 * 1024
|
||||
MAX_TOTAL_TEXT_BYTES = 256 * 1024 * 1024
|
||||
MAX_FILES = 100_000
|
||||
MAX_FILE_LIST_BYTES = 16 * 1024 * 1024
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -69,15 +96,33 @@ def parse_args() -> argparse.Namespace:
|
||||
|
||||
|
||||
def git_tracked_files(root: Path) -> list[str]:
|
||||
result = subprocess.run(
|
||||
["git", "ls-files"],
|
||||
process = subprocess.Popen(
|
||||
["git", "ls-files", "--cached", "-z"],
|
||||
cwd=root,
|
||||
check=True,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
return [line for line in result.stdout.splitlines() if line]
|
||||
if process.stdout is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
raise OSError("git file discovery has no output stream")
|
||||
output = process.stdout.read(MAX_FILE_LIST_BYTES + 1)
|
||||
if len(output) > MAX_FILE_LIST_BYTES:
|
||||
process.kill()
|
||||
process.wait()
|
||||
raise ValueError("tracked file list exceeds limit")
|
||||
return_code = process.wait()
|
||||
if return_code != 0:
|
||||
raise subprocess.CalledProcessError(return_code, process.args)
|
||||
discovered = [path.decode("utf-8") for path in output.split(b"\0") if path]
|
||||
# A tracked deletion has no content to inspect and is a valid worktree state.
|
||||
# Existing and broken symlink entries remain in the list so path validation
|
||||
# can reject them fail-closed.
|
||||
return [
|
||||
path
|
||||
for path in discovered
|
||||
if (root / path).exists() or (root / path).is_symlink()
|
||||
]
|
||||
|
||||
|
||||
def should_skip_path(path: str) -> bool:
|
||||
@@ -87,54 +132,173 @@ def should_skip_path(path: str) -> bool:
|
||||
|
||||
|
||||
def is_binary(data: bytes) -> bool:
|
||||
return b"\0" in data[:4096]
|
||||
return b"\0" in data[:BINARY_PROBE_BYTES]
|
||||
|
||||
|
||||
def scan_file(root: Path, relative_path: str) -> list[str]:
|
||||
def path_has_symlink(root: Path, logical: PurePosixPath) -> bool:
|
||||
current = root
|
||||
for part in logical.parts:
|
||||
current = current / part
|
||||
if current.is_symlink():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def resolve_scannable_file(root: Path, relative_path: str) -> Path | None:
|
||||
if (
|
||||
not relative_path
|
||||
or len(relative_path) > MAX_PATH_LENGTH
|
||||
or "\\" in relative_path
|
||||
or any(ord(character) < 32 or ord(character) == 127 for character in relative_path)
|
||||
):
|
||||
return None
|
||||
logical = PurePosixPath(relative_path)
|
||||
if logical.is_absolute() or any(part in {"", ".", ".."} for part in logical.parts):
|
||||
return None
|
||||
if str(logical) != relative_path or path_has_symlink(root, logical):
|
||||
return None
|
||||
candidate = root.joinpath(*logical.parts)
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
resolved.relative_to(root)
|
||||
except (FileNotFoundError, OSError, RuntimeError, ValueError):
|
||||
return None
|
||||
return resolved if resolved.is_file() else None
|
||||
|
||||
|
||||
def normalized_identifier_text(line: str) -> str:
|
||||
with_word_boundaries = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", line)
|
||||
return re.sub(r"(?<=[A-Z])(?=[A-Z][a-z])", "_", with_word_boundaries)
|
||||
|
||||
|
||||
def scan_file(
|
||||
root: Path,
|
||||
relative_path: str,
|
||||
remaining_text_bytes: int = MAX_TOTAL_TEXT_BYTES,
|
||||
) -> tuple[list[str], bool, bool, int]:
|
||||
path = resolve_scannable_file(root, relative_path)
|
||||
if path is None:
|
||||
return [], False, False, 0
|
||||
|
||||
if should_skip_path(relative_path):
|
||||
return []
|
||||
return [], True, False, 0
|
||||
|
||||
path = root / relative_path
|
||||
if not path.is_file():
|
||||
return []
|
||||
|
||||
data = path.read_bytes()
|
||||
if is_binary(data):
|
||||
return []
|
||||
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
findings: list[str] = []
|
||||
for line_no, line in enumerate(text.splitlines(), start=1):
|
||||
directive = ALLOW_DIRECTIVE.search(line)
|
||||
allowed_markers = (
|
||||
{label.lower() for label in directive.group(1).split(",")}
|
||||
if directive
|
||||
else set()
|
||||
)
|
||||
for label, pattern in FORBIDDEN_PATTERNS:
|
||||
if label in allowed_markers:
|
||||
continue
|
||||
if pattern.search(line):
|
||||
findings.append(f"{relative_path}:{line_no}: forbidden marker `{label}`")
|
||||
return findings
|
||||
scanned_bytes = 0
|
||||
try:
|
||||
with path.open("rb") as binary_file:
|
||||
if is_binary(binary_file.read(BINARY_PROBE_BYTES)):
|
||||
return [], True, False, 0
|
||||
file_size = os.fstat(binary_file.fileno()).st_size
|
||||
if file_size > MAX_TEXT_FILE_BYTES or file_size > remaining_text_bytes:
|
||||
return [], False, False, 0
|
||||
binary_file.seek(0)
|
||||
with io.TextIOWrapper(binary_file, encoding="utf-8", errors="strict") as text_file:
|
||||
line_no = 0
|
||||
while True:
|
||||
line = text_file.readline(MAX_LINE_CHARACTERS + 1)
|
||||
if not line:
|
||||
break
|
||||
line_no += 1
|
||||
if len(line) > MAX_LINE_CHARACTERS:
|
||||
return findings, False, False, scanned_bytes
|
||||
if len(line) == MAX_LINE_CHARACTERS and not line.endswith("\n"):
|
||||
if text_file.read(1):
|
||||
return findings, False, False, scanned_bytes
|
||||
|
||||
scanned_bytes += len(line.encode("utf-8"))
|
||||
if (
|
||||
scanned_bytes > MAX_TEXT_FILE_BYTES
|
||||
or scanned_bytes > remaining_text_bytes
|
||||
):
|
||||
return findings, False, False, scanned_bytes
|
||||
|
||||
directive = ALLOW_DIRECTIVE.search(line)
|
||||
allowed_markers = (
|
||||
{label.lower() for label in directive.group(1).split(",")}
|
||||
if directive
|
||||
else set()
|
||||
)
|
||||
normalized_line = normalized_identifier_text(line)
|
||||
for label, pattern in FORBIDDEN_PATTERNS:
|
||||
if label in allowed_markers:
|
||||
continue
|
||||
if pattern.search(line) or pattern.search(normalized_line):
|
||||
if len(findings) >= MAX_FINDINGS:
|
||||
return findings, True, True, scanned_bytes
|
||||
findings.append(
|
||||
f"{relative_path}:{line_no}: forbidden marker `{label}`"
|
||||
)
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return findings, False, False, scanned_bytes
|
||||
return findings, True, False, scanned_bytes
|
||||
|
||||
|
||||
def render_failure(findings: list[str], invalid_count: int, input_truncated: bool) -> str:
|
||||
lines = ["Community scope check failed:"]
|
||||
selected_findings = findings[:MAX_FINDINGS]
|
||||
remaining = MAX_FINDINGS - len(selected_findings)
|
||||
selected_invalid = min(invalid_count, remaining)
|
||||
lines.extend(f"error: {finding}" for finding in selected_findings)
|
||||
lines.extend("error: invalid explicit file path" for _ in range(selected_invalid))
|
||||
if len(findings) + invalid_count > MAX_FINDINGS or input_truncated:
|
||||
lines.append("error: findings truncated")
|
||||
marker = "error: report truncated\n"
|
||||
report = "\n".join(lines) + "\n"
|
||||
if len(report.encode("utf-8")) <= MAX_REPORT_BYTES:
|
||||
return report
|
||||
budget = MAX_REPORT_BYTES - len(marker.encode("utf-8"))
|
||||
kept: list[str] = []
|
||||
used = 0
|
||||
for line in lines:
|
||||
encoded = (line + "\n").encode("utf-8")
|
||||
if used + len(encoded) > budget:
|
||||
break
|
||||
kept.append(line)
|
||||
used += len(encoded)
|
||||
return "\n".join(kept) + ("\n" if kept else "") + marker
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
root = args.root.resolve()
|
||||
files = args.files if args.files is not None else git_tracked_files(root)
|
||||
try:
|
||||
files = args.files if args.files is not None else git_tracked_files(root)
|
||||
except (OSError, subprocess.SubprocessError, UnicodeDecodeError, ValueError):
|
||||
sys.stderr.write(render_failure([], 1, True))
|
||||
return 1
|
||||
findings: list[str] = []
|
||||
invalid_count = 0
|
||||
input_truncated = False
|
||||
scanned_text_bytes = 0
|
||||
|
||||
for relative_path in sorted(files):
|
||||
findings.extend(scan_file(root, relative_path))
|
||||
|
||||
if findings:
|
||||
print("Community scope check failed:", file=sys.stderr)
|
||||
for finding in findings:
|
||||
print(f"error: {finding}", file=sys.stderr)
|
||||
if len(files) > MAX_FILES:
|
||||
sys.stderr.write(render_failure([], 1, True))
|
||||
return 1
|
||||
|
||||
print(f"Community scope check passed ({len(files)} tracked files scanned)")
|
||||
sorted_files = sorted(files)
|
||||
for index, relative_path in enumerate(sorted_files):
|
||||
file_findings, valid, file_truncated, file_bytes = scan_file(
|
||||
root,
|
||||
relative_path,
|
||||
MAX_TOTAL_TEXT_BYTES - scanned_text_bytes,
|
||||
)
|
||||
scanned_text_bytes += file_bytes
|
||||
if not valid:
|
||||
invalid_count += 1
|
||||
findings.extend(file_findings)
|
||||
if file_truncated:
|
||||
input_truncated = True
|
||||
break
|
||||
if len(findings) + invalid_count >= MAX_FINDINGS:
|
||||
input_truncated = index + 1 < len(sorted_files)
|
||||
break
|
||||
|
||||
if findings or invalid_count:
|
||||
sys.stderr.write(render_failure(findings, invalid_count, input_truncated))
|
||||
return 1
|
||||
|
||||
print(f"Community scope check passed ({len(files)} files scanned)")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reject production environment readers outside the crank-config leaf crate."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAX_FILE_BYTES = 1024 * 1024
|
||||
IDENTIFIER = re.compile(r"(?:r#)?[A-Za-z_][A-Za-z0-9_]*")
|
||||
ALLOWED_ENV_MACRO = ("env", "!", "(", '"CARGO_PKG_VERSION"', ")")
|
||||
|
||||
|
||||
class BoundaryError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def resolve_file(root: Path, supplied: str) -> Path:
|
||||
logical = Path(supplied)
|
||||
if logical.is_absolute() or ".." in logical.parts:
|
||||
raise BoundaryError("explicit path must be repository-relative")
|
||||
path = root / logical
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise BoundaryError("explicit path must be a regular non-symlink file")
|
||||
try:
|
||||
path.resolve().relative_to(root)
|
||||
except ValueError as error:
|
||||
raise BoundaryError("explicit path escapes repository") from error
|
||||
return path
|
||||
|
||||
|
||||
def default_files(root: Path) -> list[Path]:
|
||||
files: list[Path] = []
|
||||
for base in (root / "apps", root / "crates"):
|
||||
if not base.exists():
|
||||
continue
|
||||
for path in base.rglob("*.rs"):
|
||||
relative_parts = path.relative_to(root).parts
|
||||
if "crank-config" in relative_parts or "tests" in relative_parts:
|
||||
continue
|
||||
if path.is_symlink():
|
||||
raise BoundaryError("production Rust source must not be a symlink")
|
||||
if not path.is_file():
|
||||
raise BoundaryError("production Rust source must be a regular file")
|
||||
files.append(path)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[tuple[str, int]]:
|
||||
"""Return Rust-like tokens while discarding comments and preserving literals."""
|
||||
tokens: list[tuple[str, int]] = []
|
||||
index = 0
|
||||
line = 1
|
||||
block_depth = 0
|
||||
while index < len(text):
|
||||
if block_depth:
|
||||
if text.startswith("/*", index):
|
||||
block_depth += 1
|
||||
index += 2
|
||||
elif text.startswith("*/", index):
|
||||
block_depth -= 1
|
||||
index += 2
|
||||
else:
|
||||
line += text[index] == "\n"
|
||||
index += 1
|
||||
continue
|
||||
if text.startswith("//", index):
|
||||
newline = text.find("\n", index + 2)
|
||||
if newline < 0:
|
||||
break
|
||||
index = newline
|
||||
continue
|
||||
if text.startswith("/*", index):
|
||||
block_depth = 1
|
||||
index += 2
|
||||
continue
|
||||
character = text[index]
|
||||
if character.isspace():
|
||||
line += character == "\n"
|
||||
index += 1
|
||||
continue
|
||||
token_line = line
|
||||
char_match = re.match(r"'(?:\\.|[^\\'\n])'", text[index:])
|
||||
if char_match:
|
||||
tokens.append((char_match.group(), token_line))
|
||||
index += char_match.end()
|
||||
continue
|
||||
raw_match = re.match(r'r(#{0,255})"', text[index:])
|
||||
if raw_match:
|
||||
terminator = '"' + raw_match.group(1)
|
||||
end = text.find(terminator, index + raw_match.end())
|
||||
if end < 0:
|
||||
raise BoundaryError("unterminated raw string literal")
|
||||
end += len(terminator)
|
||||
token = text[index:end]
|
||||
tokens.append((token, token_line))
|
||||
line += token.count("\n")
|
||||
index = end
|
||||
continue
|
||||
if character == '"':
|
||||
end = index + 1
|
||||
escaped = False
|
||||
while end < len(text):
|
||||
current = text[end]
|
||||
if current == '"' and not escaped:
|
||||
end += 1
|
||||
break
|
||||
escaped = current == "\\" and not escaped
|
||||
if current != "\\":
|
||||
escaped = False
|
||||
line += current == "\n"
|
||||
end += 1
|
||||
else:
|
||||
raise BoundaryError("unterminated string literal")
|
||||
tokens.append((text[index:end], token_line))
|
||||
index = end
|
||||
continue
|
||||
match = IDENTIFIER.match(text, index)
|
||||
if match:
|
||||
tokens.append((match.group(), token_line))
|
||||
index = match.end()
|
||||
continue
|
||||
if text.startswith("::", index):
|
||||
tokens.append(("::", token_line))
|
||||
index += 2
|
||||
continue
|
||||
tokens.append((character, token_line))
|
||||
index += 1
|
||||
if block_depth:
|
||||
raise BoundaryError("unterminated block comment")
|
||||
return tokens
|
||||
|
||||
|
||||
def scan(path: Path) -> list[tuple[int, int]]:
|
||||
data = path.read_bytes()
|
||||
if len(data) > MAX_FILE_BYTES:
|
||||
raise BoundaryError("Rust source exceeds boundary scanner size limit")
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError as error:
|
||||
raise BoundaryError("Rust source is not valid UTF-8") from error
|
||||
tokens = tokenize(text)
|
||||
values = [token for token, _ in tokens]
|
||||
findings: set[tuple[int, int]] = set()
|
||||
std_aliases: set[str] = set()
|
||||
for index in range(len(tokens)):
|
||||
window = values[index : index + 5]
|
||||
line = tokens[index][1]
|
||||
if (
|
||||
len(window) >= 5
|
||||
and window[:4] == ["std", "::", "env", "::"]
|
||||
and window[4] in {"var", "vars", "var_os", "vars_os"}
|
||||
):
|
||||
findings.add((line, 1))
|
||||
if len(window) >= 3 and window[0] == "env" and window[1] == "::" and window[2] in {
|
||||
"var",
|
||||
"vars",
|
||||
"var_os",
|
||||
"vars_os",
|
||||
}:
|
||||
findings.add((line, 2))
|
||||
if values[index].removeprefix("r#").endswith("_from_env"):
|
||||
findings.add((line, 3))
|
||||
if len(window) >= 2 and window[0] in {"dotenv", "dotenvy"} and window[1] == "::":
|
||||
findings.add((line, 4))
|
||||
if len(window) >= 5 and window[0] in {"env", "option_env"} and window[1] == "!":
|
||||
if tuple(window) != ALLOWED_ENV_MACRO:
|
||||
findings.add((line, 5))
|
||||
if (
|
||||
len(window) >= 4
|
||||
and window[0] in {"use", "crate"}
|
||||
and window[1] == "std"
|
||||
and window[2] == "as"
|
||||
):
|
||||
std_aliases.add(window[3])
|
||||
grouped = values[index : index + 8]
|
||||
if (
|
||||
len(grouped) == 8
|
||||
and grouped[:6] == ["use", "std", "::", "{", "self", "as"]
|
||||
and IDENTIFIER.fullmatch(grouped[6])
|
||||
and grouped[7] in {"}", ","}
|
||||
):
|
||||
std_aliases.add(grouped[6])
|
||||
if len(window) >= 5 and window[:3] == ["extern", "crate", "std"] and window[3] == "as":
|
||||
std_aliases.add(window[4])
|
||||
if window[:3] == ["use", "std", "::"]:
|
||||
end = index + 3
|
||||
while end < len(tokens) and values[end] != ";":
|
||||
if values[end] == "env":
|
||||
findings.add((line, 1))
|
||||
break
|
||||
end += 1
|
||||
for index, (token, line) in enumerate(tokens):
|
||||
if token in std_aliases and values[index : index + 3] == [token, "::", "env"]:
|
||||
findings.add((line, 6))
|
||||
return sorted(findings)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, default=Path("."))
|
||||
parser.add_argument("--files", nargs="*")
|
||||
args = parser.parse_args()
|
||||
root = args.root.resolve()
|
||||
try:
|
||||
files = (
|
||||
[resolve_file(root, item) for item in args.files]
|
||||
if args.files is not None
|
||||
else default_files(root)
|
||||
)
|
||||
violations: list[tuple[str, int, int]] = []
|
||||
for path in files:
|
||||
if "crank-config" in path.relative_to(root).parts:
|
||||
continue
|
||||
for line, pattern in scan(path):
|
||||
violations.append((path.relative_to(root).as_posix(), line, pattern))
|
||||
except (BoundaryError, OSError) as error:
|
||||
print(f"config boundary check failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
if violations:
|
||||
for path, line, pattern in sorted(violations):
|
||||
print(
|
||||
f"config boundary violation: {path}:{line} pattern={pattern}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print(f"config boundary check passed: files={len(files)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail closed when PostgreSQL migration authority escapes its canonical module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAX_FILE_BYTES = 2 * 1024 * 1024
|
||||
DDL = re.compile(
|
||||
r"(?is)\b(?:create|alter|drop|truncate|comment\s+on|grant|revoke)\s+"
|
||||
r"(?:(?:or\s+replace|unique|temporary|temp|unlogged)\s+)*"
|
||||
r"(?:table|index|schema|view|materialized\s+view|type|sequence|function|procedure|"
|
||||
r"trigger|extension|domain|policy|role)\b"
|
||||
)
|
||||
RUNNER = re.compile(
|
||||
r"(?is)pg_advisory_(?:xact_)?lock|sqlx\s*::\s*migrate|_sqlx_migrations|"
|
||||
r"include_(?:str|bytes)!\s*\([^)]*\.sql|"
|
||||
r"(?:insert\s+into|update|delete\s+from)\s+__crank_(?:core_|mcp_|ext_)?migrations"
|
||||
)
|
||||
|
||||
|
||||
def canonical(path: Path, root: Path) -> bool:
|
||||
relative = path.relative_to(root).as_posix()
|
||||
return relative == "crates/crank-registry/src/migrations.rs" or relative.startswith(
|
||||
"crates/crank-registry/src/migrations/"
|
||||
)
|
||||
|
||||
|
||||
def candidates(root: Path) -> list[Path]:
|
||||
result: list[Path] = []
|
||||
for base_name in ("apps", "crates"):
|
||||
base = root / base_name
|
||||
if not base.exists():
|
||||
continue
|
||||
for path in base.rglob("*"):
|
||||
if path.suffix not in {".rs", ".sql"}:
|
||||
continue
|
||||
if "crank-test-support" in path.relative_to(root).parts:
|
||||
continue
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("production Rust/SQL path must be a regular non-symlink file")
|
||||
if "/tests/" in f"/{path.relative_to(root).as_posix()}/":
|
||||
continue
|
||||
result.append(path)
|
||||
return sorted(result)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, default=Path("."))
|
||||
args = parser.parse_args()
|
||||
root = args.root.resolve()
|
||||
try:
|
||||
files = candidates(root)
|
||||
violations: list[str] = []
|
||||
for path in files:
|
||||
if canonical(path, root):
|
||||
continue
|
||||
data = path.read_bytes()
|
||||
if len(data) > MAX_FILE_BYTES:
|
||||
raise ValueError("production Rust/SQL file exceeds scanner limit")
|
||||
text = data.decode("utf-8")
|
||||
if DDL.search(text) or RUNNER.search(text):
|
||||
violations.append(path.relative_to(root).as_posix())
|
||||
except (OSError, UnicodeDecodeError, ValueError) as error:
|
||||
print(f"migration boundary check failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
if violations:
|
||||
for path in violations:
|
||||
print(f"migration boundary violation: {path}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"migration boundary check passed: files={len(files)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed drift check for the generated runtime configuration contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAX_INPUT_BYTES = 4 * 1024 * 1024
|
||||
OWNED_PREFIXES = ("CRANK_", "POSTGRES_", "OTEL_")
|
||||
ENV_FILES = (
|
||||
".env.example",
|
||||
"deploy/community/.env.example",
|
||||
"deploy/community/.env.images.example",
|
||||
)
|
||||
COMPOSE_FILES = (
|
||||
"docker-compose.yml",
|
||||
"deploy/community/docker-compose.yml",
|
||||
"deploy/community/docker-compose.images.yml",
|
||||
)
|
||||
BEGIN = "# BEGIN GENERATED CRANK RUNTIME CONFIG"
|
||||
END = "# END GENERATED CRANK RUNTIME CONFIG"
|
||||
ENV_NAME = re.compile(r"^[A-Z_][A-Z0-9_]*$")
|
||||
INTERPOLATION_OPERATORS = (":-", ":?", ":+", "-", "?", "+")
|
||||
|
||||
|
||||
class ContractError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def reject_duplicates(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
||||
result: dict[str, object] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ContractError("duplicate JSON key")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
if not path.is_file() or path.is_symlink():
|
||||
raise ContractError(f"missing regular file: {path.name}")
|
||||
data = path.read_bytes()
|
||||
if len(data) > MAX_INPUT_BYTES:
|
||||
raise ContractError(f"input too large: {path.name}")
|
||||
try:
|
||||
return data.decode("utf-8")
|
||||
except UnicodeDecodeError as error:
|
||||
raise ContractError(f"invalid UTF-8: {path.name}") from error
|
||||
|
||||
|
||||
def load_contract(
|
||||
root: Path,
|
||||
) -> tuple[set[str], dict[str, dict[str, object]], set[str]]:
|
||||
path = root / "docs/schemas/runtime-config.schema.json"
|
||||
try:
|
||||
payload = json.loads(read_text(path), object_pairs_hook=reject_duplicates)
|
||||
except (json.JSONDecodeError, ValueError, RecursionError) as error:
|
||||
raise ContractError("invalid runtime configuration contract JSON") from error
|
||||
fields = payload.get("fields")
|
||||
if payload.get("schema_version") != 1 or not isinstance(fields, list):
|
||||
raise ContractError("invalid runtime configuration contract shape")
|
||||
effective: set[str] = set()
|
||||
specifications: dict[str, dict[str, object]] = {}
|
||||
for field in fields:
|
||||
if not isinstance(field, dict):
|
||||
raise ContractError("invalid field entry")
|
||||
name = field.get("env_name")
|
||||
scope = field.get("process")
|
||||
mode = field.get("mode")
|
||||
if not isinstance(name, str) or scope not in {"shared", "admin_api", "mcp_server"}:
|
||||
raise ContractError("invalid field identity")
|
||||
if name in specifications:
|
||||
raise ContractError("duplicate runtime field")
|
||||
required = field.get("required")
|
||||
sensitivity = field.get("sensitivity")
|
||||
default = field.get("default")
|
||||
if not isinstance(required, bool) or sensitivity not in {
|
||||
"public",
|
||||
"internal",
|
||||
"secret",
|
||||
}:
|
||||
raise ContractError("invalid field semantics")
|
||||
if default is not None and not isinstance(default, str):
|
||||
raise ContractError("invalid field default")
|
||||
specifications[name] = field
|
||||
if mode == "effective":
|
||||
effective.add(name)
|
||||
elif mode != "deprecated_no_effect":
|
||||
raise ContractError("unknown runtime field mode")
|
||||
deployment = payload.get("deployment_only_fields")
|
||||
if (
|
||||
not isinstance(deployment, list)
|
||||
or not all(isinstance(name, str) and ENV_NAME.fullmatch(name) for name in deployment)
|
||||
or len(set(deployment)) != len(deployment)
|
||||
):
|
||||
raise ContractError("invalid deployment-only fields")
|
||||
return effective, specifications, set(deployment)
|
||||
|
||||
|
||||
def env_entries(content: str) -> dict[str, str]:
|
||||
if content.count(BEGIN) != 1 or content.count(END) != 1:
|
||||
raise ContractError("generated environment markers are missing or duplicated")
|
||||
section = content.split(BEGIN, 1)[1].split(END, 1)[0]
|
||||
entries: dict[str, str] = {}
|
||||
for line in section.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
name, separator, _ = line.partition("=")
|
||||
if not separator or not ENV_NAME.fullmatch(name) or name in entries:
|
||||
raise ContractError("invalid or duplicate generated environment entry")
|
||||
entries[name] = line.partition("=")[2]
|
||||
return entries
|
||||
|
||||
|
||||
def interpolations(content: str) -> list[tuple[str, str, str]]:
|
||||
"""Parse bounded, non-nested Compose ${NAME<operator>payload} expressions."""
|
||||
results: list[tuple[str, str, str]] = []
|
||||
cursor = 0
|
||||
while True:
|
||||
start = content.find("${", cursor)
|
||||
if start < 0:
|
||||
return results
|
||||
end = content.find("}", start + 2)
|
||||
if end < 0:
|
||||
raise ContractError("unterminated Compose interpolation")
|
||||
expression = content[start + 2 : end]
|
||||
if "${" in expression or len(expression) > 8192:
|
||||
raise ContractError("invalid nested or oversized Compose interpolation")
|
||||
name_end = 0
|
||||
while name_end < len(expression) and (
|
||||
expression[name_end].isalnum() or expression[name_end] == "_"
|
||||
):
|
||||
name_end += 1
|
||||
name = expression[:name_end]
|
||||
remainder = expression[name_end:]
|
||||
if not ENV_NAME.fullmatch(name):
|
||||
raise ContractError("invalid Compose interpolation name")
|
||||
operator = ""
|
||||
payload = ""
|
||||
if remainder:
|
||||
operator = next(
|
||||
(candidate for candidate in INTERPOLATION_OPERATORS if remainder.startswith(candidate)),
|
||||
"",
|
||||
)
|
||||
if not operator:
|
||||
raise ContractError("invalid Compose interpolation operator")
|
||||
payload = remainder[len(operator) :]
|
||||
results.append((name, operator, payload))
|
||||
cursor = end + 1
|
||||
|
||||
|
||||
def parse_runtime_reference(value: str) -> tuple[str, str, str]:
|
||||
references = interpolations(value)
|
||||
if len(references) != 1 or value.strip() != "${" + "".join(references[0]) + "}":
|
||||
raise ContractError("runtime Compose value must be one direct interpolation")
|
||||
return references[0]
|
||||
|
||||
|
||||
def compose_environment(content: str, service: str) -> dict[str, str]:
|
||||
lines = content.splitlines()
|
||||
in_service = False
|
||||
in_environment = False
|
||||
entries: dict[str, str] = {}
|
||||
for line in lines:
|
||||
if line == f" {service}:":
|
||||
in_service = True
|
||||
in_environment = False
|
||||
continue
|
||||
if in_service and line.startswith(" ") and not line.startswith(" "):
|
||||
break
|
||||
if in_service and line == " environment:":
|
||||
in_environment = True
|
||||
continue
|
||||
if in_environment:
|
||||
if not line.startswith(" "):
|
||||
break
|
||||
stripped = line.strip()
|
||||
name, separator, _ = stripped.partition(":")
|
||||
if separator and name.startswith(OWNED_PREFIXES):
|
||||
if name in entries:
|
||||
raise ContractError("duplicate Compose runtime field")
|
||||
entries[name] = stripped.partition(":")[2].strip()
|
||||
return entries
|
||||
|
||||
|
||||
def validate_runtime_reference(
|
||||
name: str,
|
||||
value: str,
|
||||
specification: dict[str, object],
|
||||
declared_values: set[str],
|
||||
) -> None:
|
||||
reference, operator, payload = parse_runtime_reference(value)
|
||||
if reference != name:
|
||||
raise ContractError(f"wrong Compose interpolation reference: {name}")
|
||||
required = specification["required"]
|
||||
sensitivity = specification["sensitivity"]
|
||||
if required:
|
||||
if operator not in {"", ":?"} or (operator == ":?" and not payload):
|
||||
raise ContractError(f"required runtime field has fallback: {name}")
|
||||
return
|
||||
if operator != ":-":
|
||||
raise ContractError(f"optional runtime field must use empty-aware default: {name}")
|
||||
if sensitivity != "secret" and payload not in declared_values:
|
||||
raise ContractError(f"divergent Compose inline default: {name}")
|
||||
|
||||
|
||||
def validate(root: Path) -> None:
|
||||
effective, specifications, deployment = load_contract(root)
|
||||
declared_values: dict[str, set[str]] = {name: set() for name in effective}
|
||||
for relative in ENV_FILES:
|
||||
entries = env_entries(read_text(root / relative))
|
||||
if set(entries) != effective:
|
||||
raise ContractError(f"generated environment drift: {Path(relative).name}")
|
||||
for name, value in entries.items():
|
||||
declared_values[name].add(value)
|
||||
for name in effective:
|
||||
default = specifications[name].get("default")
|
||||
if isinstance(default, str):
|
||||
declared_values[name].add(default)
|
||||
|
||||
for relative in COMPOSE_FILES:
|
||||
content = read_text(root / relative)
|
||||
known = effective | deployment
|
||||
for name, _, _ in interpolations(content):
|
||||
if name not in known:
|
||||
raise ContractError(f"unknown Compose interpolation: {name}")
|
||||
admin = compose_environment(content, "admin-api")
|
||||
mcp = compose_environment(content, "mcp-server")
|
||||
required_admin = {
|
||||
name
|
||||
for name in effective
|
||||
if specifications[name]["process"] in {"shared", "admin_api"}
|
||||
}
|
||||
required_mcp = {
|
||||
name
|
||||
for name in effective
|
||||
if specifications[name]["process"] in {"shared", "mcp_server"}
|
||||
}
|
||||
if set(admin) != required_admin:
|
||||
raise ContractError(f"admin Compose runtime drift: {Path(relative).name}")
|
||||
if set(mcp) != required_mcp:
|
||||
raise ContractError(f"MCP Compose runtime drift: {Path(relative).name}")
|
||||
for entries in (admin, mcp):
|
||||
for name, value in entries.items():
|
||||
validate_runtime_reference(
|
||||
name,
|
||||
value,
|
||||
specifications[name],
|
||||
declared_values[name],
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, default=Path("."))
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
validate(args.root.resolve())
|
||||
except (ContractError, OSError) as error:
|
||||
print(f"runtime configuration contract check failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
print("runtime configuration contract check passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -51,6 +51,8 @@ def package_category(name: str, manifest_path: Path, workspace_root: Path) -> st
|
||||
return "core"
|
||||
if name == "crank-metrics":
|
||||
return "metrics"
|
||||
if name == "crank-config":
|
||||
return "config"
|
||||
if name == "crank-observability":
|
||||
return "observability"
|
||||
if name == "crank-registry":
|
||||
@@ -123,6 +125,9 @@ def boundary_reason(source: Package, dependency: Package) -> str | None:
|
||||
if source.category == "metrics":
|
||||
return "crank-metrics must not depend on other workspace crates"
|
||||
|
||||
if source.category == "config":
|
||||
return "crank-config must not depend on other workspace crates"
|
||||
|
||||
if source.category == "observability" and dependency.category != "metrics":
|
||||
return "crank-observability must not depend on other workspace crates"
|
||||
|
||||
|
||||
@@ -20,6 +20,9 @@ check_no_match() {
|
||||
|
||||
echo "Rust module boundary check: checking module-level imports"
|
||||
|
||||
python3 "$ROOT_DIR/scripts/check-config-boundaries.py" --root "$ROOT_DIR" || status=1
|
||||
python3 "$ROOT_DIR/scripts/check-migration-boundaries.py" --root "$ROOT_DIR" || status=1
|
||||
|
||||
check_no_match \
|
||||
"admin-api service modules must not depend on axum HTTP types" \
|
||||
'^\s*use\s+axum(::|[;\{])' \
|
||||
@@ -66,6 +69,8 @@ Rules:
|
||||
- registry remains storage-only and HTTP-client agnostic;
|
||||
- runtime remains execution-only and storage/framework agnostic.
|
||||
- names and labels of metrics remain inside the typed crank-metrics contract.
|
||||
- process environment is read only by the leaf crank-config adapter.
|
||||
- PostgreSQL DDL and migration advisory locks are owned only by crank-registry migrations.
|
||||
|
||||
EOF
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create bounded, sanitized capability-baseline evidence candidates."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
MAX_REPORT_BYTES = 4 * 1024 * 1024
|
||||
REVISION_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
FLOW_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
ENVIRONMENT_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
COMMAND_IDS = {
|
||||
"python-tooling-tests",
|
||||
"rust-admin-integration",
|
||||
"rust-mcp-integration",
|
||||
"ui-build",
|
||||
"ui-playwright",
|
||||
"just-verify",
|
||||
"authenticated-product-smoke",
|
||||
}
|
||||
|
||||
|
||||
class CollectionError(Exception):
|
||||
def __init__(self, code: str, pointer: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.pointer = pointer
|
||||
|
||||
|
||||
def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise CollectionError("INVALID_JSON", "/")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def load_report(path: Path) -> tuple[dict[str, Any], str]:
|
||||
try:
|
||||
with path.open("rb") as file:
|
||||
raw = file.read(MAX_REPORT_BYTES + 1)
|
||||
except OSError as error:
|
||||
raise CollectionError("BROKEN_REPORT_LINK", "/report") from error
|
||||
if len(raw) > MAX_REPORT_BYTES:
|
||||
raise CollectionError("INPUT_TOO_LARGE", "/report")
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
try:
|
||||
value = json.loads(raw.decode("utf-8"), object_pairs_hook=reject_duplicates)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, ValueError, RecursionError) as error:
|
||||
raise CollectionError("INVALID_JSON", "/report") from error
|
||||
if not isinstance(value, dict):
|
||||
raise CollectionError("INVALID_REPORT", "/report")
|
||||
return value, digest
|
||||
|
||||
|
||||
def iter_tests(value: Any):
|
||||
if isinstance(value, dict):
|
||||
tests = value.get("tests")
|
||||
if isinstance(tests, list):
|
||||
for test in tests:
|
||||
if isinstance(test, dict):
|
||||
yield test
|
||||
for key in ("suites", "specs"):
|
||||
children = value.get(key)
|
||||
if isinstance(children, list):
|
||||
for child in children:
|
||||
yield from iter_tests(child)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
yield from iter_tests(item)
|
||||
|
||||
|
||||
def playwright_verdict(report: dict[str, Any]) -> tuple[str, dict[str, int]]:
|
||||
counts = {"passed": 0, "failed": 0, "flaky": 0, "skipped": 0, "not_run": 0}
|
||||
report_errors = report.get("errors", [])
|
||||
if not isinstance(report_errors, list):
|
||||
counts["failed"] = 1
|
||||
return "fail", counts
|
||||
if report_errors:
|
||||
counts["failed"] = min(len(report_errors), 10_000)
|
||||
return "fail", counts
|
||||
tests = list(iter_tests(report))
|
||||
if not tests:
|
||||
counts["not_run"] = 1
|
||||
return "not_run", counts
|
||||
for test in tests:
|
||||
status = test.get("status")
|
||||
results = test.get("results") if isinstance(test.get("results"), list) else []
|
||||
result_statuses = [result.get("status") for result in results if isinstance(result, dict)]
|
||||
retries = [result.get("retry", 0) for result in results if isinstance(result, dict)]
|
||||
if len(result_statuses) != len(results):
|
||||
counts["not_run"] += 1
|
||||
continue
|
||||
if status == "flaky" or any(isinstance(retry, int) and retry > 0 for retry in retries):
|
||||
counts["flaky"] += 1
|
||||
elif status == "skipped" or (not results and status in ("skipped", "expected")):
|
||||
counts["skipped"] += 1
|
||||
elif status in ("unexpected", "failed", "timedOut", "interrupted") or any(
|
||||
result_status in ("failed", "timedOut", "interrupted") for result_status in result_statuses
|
||||
):
|
||||
counts["failed"] += 1
|
||||
elif results and all(result_status == "passed" for result_status in result_statuses):
|
||||
counts["passed"] += 1
|
||||
else:
|
||||
counts["not_run"] += 1
|
||||
if counts["failed"]:
|
||||
return "fail", counts
|
||||
if counts["flaky"]:
|
||||
return "flaky", counts
|
||||
if counts["skipped"]:
|
||||
return "skipped", counts
|
||||
if counts["not_run"]:
|
||||
return "not_run", counts
|
||||
return "pass", counts
|
||||
|
||||
|
||||
def validate_labels(args: argparse.Namespace) -> None:
|
||||
if not REVISION_RE.fullmatch(args.source_revision):
|
||||
raise CollectionError("INVALID_ARGUMENT", "/source_revision")
|
||||
if not ENVIRONMENT_RE.fullmatch(args.environment_class):
|
||||
raise CollectionError("INVALID_ARGUMENT", "/environment_class")
|
||||
if not args.flow_id or any(not FLOW_RE.fullmatch(flow_id) for flow_id in args.flow_id):
|
||||
raise CollectionError("INVALID_ARGUMENT", "/flow_ids")
|
||||
|
||||
|
||||
def collect_playwright(args: argparse.Namespace) -> dict[str, Any]:
|
||||
validate_labels(args)
|
||||
report, digest = load_report(Path(args.report))
|
||||
verdict, counts = playwright_verdict(report)
|
||||
return {
|
||||
"accepted": verdict == "pass",
|
||||
"collector": "capability-baseline-collector-v1",
|
||||
"command_id": "ui-playwright",
|
||||
"environment_class": args.environment_class,
|
||||
"evidence_mode": "automated",
|
||||
"execution_verdict": verdict,
|
||||
"flow_ids": sorted(set(args.flow_id)),
|
||||
"id": f"run-ui-playwright-{digest[:12]}",
|
||||
"source_report_sha256": digest,
|
||||
"source_revision": args.source_revision,
|
||||
"summary": counts,
|
||||
}
|
||||
|
||||
|
||||
def collect_command_report(args: argparse.Namespace) -> dict[str, Any]:
|
||||
validate_labels(args)
|
||||
report, digest = load_report(Path(args.report))
|
||||
command_id = report.get("command_id")
|
||||
if command_id not in COMMAND_IDS:
|
||||
raise CollectionError("UNKNOWN_COMMAND", "/report/command_id")
|
||||
exit_code = report.get("exit_code")
|
||||
timed_out = report.get("timed_out")
|
||||
skipped = report.get("skipped")
|
||||
if type(exit_code) is not int or not isinstance(timed_out, bool) or type(skipped) is not int or skipped < 0:
|
||||
raise CollectionError("INVALID_REPORT", "/report")
|
||||
if timed_out:
|
||||
verdict = "blocked"
|
||||
elif exit_code != 0:
|
||||
verdict = "fail"
|
||||
elif skipped:
|
||||
verdict = "skipped"
|
||||
else:
|
||||
verdict = "pass"
|
||||
return {
|
||||
"accepted": verdict == "pass",
|
||||
"collector": "capability-baseline-collector-v1",
|
||||
"command_id": command_id,
|
||||
"environment_class": args.environment_class,
|
||||
"evidence_mode": "automated",
|
||||
"execution_verdict": verdict,
|
||||
"flow_ids": sorted(set(args.flow_id)),
|
||||
"id": f"run-{command_id}-{digest[:12]}",
|
||||
"source_report_sha256": digest,
|
||||
"source_revision": args.source_revision,
|
||||
"summary": {"exit_code": exit_code, "skipped": skipped, "timed_out": timed_out},
|
||||
}
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = root.add_subparsers(dest="mode", required=True)
|
||||
playwright = subparsers.add_parser("playwright")
|
||||
playwright.add_argument("--report", required=True)
|
||||
playwright.add_argument("--output", required=True)
|
||||
playwright.add_argument("--source-revision", required=True)
|
||||
playwright.add_argument("--environment-class", required=True)
|
||||
playwright.add_argument("--flow-id", action="append", required=True)
|
||||
command = subparsers.add_parser("command-report")
|
||||
command.add_argument("--report", required=True)
|
||||
command.add_argument("--output", required=True)
|
||||
command.add_argument("--source-revision", required=True)
|
||||
command.add_argument("--environment-class", required=True)
|
||||
command.add_argument("--flow-id", action="append", required=True)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parser().parse_args()
|
||||
try:
|
||||
candidate = collect_playwright(args) if args.mode == "playwright" else collect_command_report(args)
|
||||
encoded = (json.dumps(candidate, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
||||
if len(encoded) > 65_536:
|
||||
raise CollectionError("OUTPUT_TOO_LARGE", "/output")
|
||||
Path(args.output).write_bytes(encoded)
|
||||
except CollectionError as error:
|
||||
print(f"{error.code} pointer={error.pointer}", file=sys.stderr)
|
||||
return 1
|
||||
except OSError:
|
||||
print("OUTPUT_WRITE_FAILED pointer=/output", file=sys.stderr)
|
||||
return 1
|
||||
print(f"capability baseline candidate: verdict={candidate['execution_verdict']}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,542 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the versioned Crank Community capability-baseline snapshot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
MAX_JSON_BYTES = 4 * 1024 * 1024
|
||||
MAX_CHECKLIST_BYTES = 1024 * 1024
|
||||
MAX_DIAGNOSTICS = 1000
|
||||
MAX_REPORT_BYTES = 65_536
|
||||
VERSION_RE = re.compile(r"^[0-9]{4}\.[0-9]{2}\.[0-9]{2}\.[1-9][0-9]*$")
|
||||
HEX_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
FLOW_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
EXPECTED_KINDS = {"inventory", "required_surfaces", "taxonomy", "checklist", "results"}
|
||||
IMPLEMENTATION_STATUSES = {"implemented", "planned", "gap", "blocked"}
|
||||
EXECUTION_VERDICTS = {"pass", "fail", "blocked", "skipped", "flaky", "not_run"}
|
||||
EVIDENCE_MODES = {"automated", "manual_only"}
|
||||
SEVERE = {"Critical", "High"}
|
||||
COMMAND_IDS = {
|
||||
"python-tooling-tests", "rust-admin-integration", "rust-mcp-integration",
|
||||
"ui-build", "ui-playwright", "just-verify", "authenticated-product-smoke",
|
||||
}
|
||||
CHECKLIST_VERDICTS = {"pass", "fail", "blocked", "not_run", "gap", "n/a"}
|
||||
CHECKLIST_STATES = {"happy", "loading", "empty", "error", "recovery", "stale", "ru-en", "safe-output"}
|
||||
UNSAFE_RE = re.compile(r"(?i)(bearer\s+\S+|cookie\s*[:=]|authorization\s*[:=]|https?://|/(?:home|users|root)/)")
|
||||
UNSUPPORTED_SCHEMA_KEYWORDS = {
|
||||
"allOf", "anyOf", "oneOf", "not", "if", "then", "else", "contains",
|
||||
"dependentSchemas", "patternProperties", "propertyNames", "unevaluatedItems",
|
||||
"unevaluatedProperties", "prefixItems",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=True)
|
||||
class Diagnostic:
|
||||
code: str
|
||||
pointer: str
|
||||
|
||||
|
||||
class DuplicateKey(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise DuplicateKey(key)
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def logical_path(root: Path, value: Any) -> Path | None:
|
||||
if not isinstance(value, str) or not value or len(value) > 1024 or any(ord(char) < 32 for char in value):
|
||||
return None
|
||||
candidate = Path(value)
|
||||
if candidate.is_absolute() or "\\" in value or any(part in ("", ".", "..") for part in candidate.parts):
|
||||
return None
|
||||
try:
|
||||
root_resolved = root.resolve(strict=True)
|
||||
joined = root / candidate
|
||||
if joined.is_symlink():
|
||||
return None
|
||||
resolved = joined.resolve(strict=True)
|
||||
resolved.relative_to(root_resolved)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not resolved.is_file():
|
||||
return None
|
||||
return resolved
|
||||
|
||||
|
||||
def tracked_path(root: Path, value: Any) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "ls-files", "--error-unmatch", "--", value],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
timeout=5,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return False
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def read_bytes(path: Path, limit: int) -> bytes:
|
||||
try:
|
||||
with path.open("rb") as file:
|
||||
raw = file.read(limit + 1)
|
||||
except OSError as error:
|
||||
raise ValueError("BROKEN") from error
|
||||
if len(raw) > limit:
|
||||
raise OverflowError
|
||||
return raw
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
raw = read_bytes(path, MAX_JSON_BYTES)
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"), object_pairs_hook=reject_duplicates)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, DuplicateKey, ValueError, RecursionError) as error:
|
||||
raise TypeError from error
|
||||
|
||||
|
||||
def schema_contract_valid(schema: Any) -> bool:
|
||||
if not isinstance(schema, dict):
|
||||
return False
|
||||
allowed_root = {"$schema", "$id", "title", "description", "type", "additionalProperties", "required", "properties", "$defs"}
|
||||
if set(schema) - allowed_root:
|
||||
return False
|
||||
pending = [schema]
|
||||
while pending:
|
||||
node = pending.pop()
|
||||
if isinstance(node, dict):
|
||||
if set(node) & UNSUPPORTED_SCHEMA_KEYWORDS:
|
||||
return False
|
||||
pending.extend(node.values())
|
||||
elif isinstance(node, list):
|
||||
pending.extend(node)
|
||||
if schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema":
|
||||
return False
|
||||
if schema.get("type") != "object" or schema.get("additionalProperties") is not False:
|
||||
return False
|
||||
if schema.get("required") != ["schema_version", "baseline_version", "artifacts"]:
|
||||
return False
|
||||
properties = schema.get("properties")
|
||||
definitions = schema.get("$defs")
|
||||
if not isinstance(properties, dict):
|
||||
return False
|
||||
if not isinstance(definitions, dict) or set(definitions) != {"artifact", "taxonomy", "run", "manual_result", "defect", "results"}:
|
||||
return False
|
||||
for name in ("artifact", "taxonomy", "run", "defect", "results"):
|
||||
definition = definitions.get(name)
|
||||
if not isinstance(definition, dict) or definition.get("type") != "object" or definition.get("additionalProperties") is not False:
|
||||
return False
|
||||
if definitions["run"].get("required") != ["id", "command_id", "flow_ids", "execution_verdict", "evidence_mode", "accepted", "source_report_sha256", "source_revision", "environment_class", "collector"]:
|
||||
return False
|
||||
if definitions["defect"].get("required") != ["id", "severity", "steps", "contract", "owner", "flow_ids", "next_action"]:
|
||||
return False
|
||||
taxonomy_properties = definitions["taxonomy"].get("properties")
|
||||
full_pass = taxonomy_properties.get("full_pass") if isinstance(taxonomy_properties, dict) else None
|
||||
if not isinstance(full_pass, dict) or set(full_pass.get("properties", {})) != {"implementation_status", "execution_verdict", "evidence_mode"}:
|
||||
return False
|
||||
manual_result = definitions.get("manual_result")
|
||||
if not isinstance(manual_result, dict) or manual_result.get("type") != "object" or manual_result.get("additionalProperties") is not False:
|
||||
return False
|
||||
version = properties.get("schema_version")
|
||||
artifacts = properties.get("artifacts")
|
||||
return (
|
||||
isinstance(version, dict)
|
||||
and type(version.get("const")) is int
|
||||
and version.get("const") == 1
|
||||
and isinstance(artifacts, dict)
|
||||
and artifacts.get("minItems") == 5
|
||||
and artifacts.get("maxItems") == 5
|
||||
)
|
||||
|
||||
|
||||
def add(diagnostics: list[Diagnostic], code: str, pointer: str) -> None:
|
||||
if len(diagnostics) < MAX_DIAGNOSTICS:
|
||||
diagnostics.append(Diagnostic(code, pointer[:1024]))
|
||||
|
||||
|
||||
def exact_keys(value: Any, required: set[str], optional: set[str] = set()) -> bool:
|
||||
return isinstance(value, dict) and required <= set(value) and not (set(value) - required - optional)
|
||||
|
||||
|
||||
def nonempty_text(value: Any, limit: int) -> bool:
|
||||
return isinstance(value, str) and bool(value.strip()) and len(value) <= limit
|
||||
|
||||
|
||||
def parse_checklist(text: Any, diagnostics: list[Diagnostic]) -> dict[str, dict[str, Any]]:
|
||||
if not isinstance(text, str):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/checklist")
|
||||
return {}
|
||||
matches = list(re.finditer(r"(?m)^## (UI-[0-9]{2})\s+[^\n]+\n", text))
|
||||
if not 1 <= len(matches) <= 16:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/checklist/checks")
|
||||
checks: dict[str, dict[str, Any]] = {}
|
||||
for index, match in enumerate(matches):
|
||||
check_id = match.group(1)
|
||||
block = text[match.end(): matches[index + 1].start() if index + 1 < len(matches) else len(text)]
|
||||
fields: dict[str, str] = {}
|
||||
for key in ("flow_id", "states", "verdict", "reason"):
|
||||
field = re.search(rf"(?m)^- {key}:\s*(.+?)\s*$", block)
|
||||
if field:
|
||||
fields[key] = field.group(1)
|
||||
if check_id in checks or set(fields) != {"flow_id", "states", "verdict", "reason"}:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/checklist/{check_id}")
|
||||
continue
|
||||
states = [item.strip() for item in fields["states"].split(",") if item.strip()]
|
||||
if not states or len(states) != len(set(states)) or set(states) - CHECKLIST_STATES:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/checklist/{check_id}/states")
|
||||
if fields["verdict"] not in CHECKLIST_VERDICTS or not nonempty_text(fields["reason"], 2048):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/checklist/{check_id}/verdict")
|
||||
checks[check_id] = {"flow_id": fields["flow_id"], "verdict": fields["verdict"]}
|
||||
return checks
|
||||
|
||||
|
||||
def validate(root: Path, manifest_path: str, schema_path: str) -> tuple[list[Diagnostic], str | None, int]:
|
||||
diagnostics: list[Diagnostic] = []
|
||||
manifest_file = logical_path(root, manifest_path)
|
||||
schema_file = logical_path(root, schema_path)
|
||||
if manifest_file is None:
|
||||
add(diagnostics, "BROKEN_ARTIFACT_LINK", "/manifest")
|
||||
if schema_file is None:
|
||||
add(diagnostics, "BROKEN_ARTIFACT_LINK", "/schema")
|
||||
if diagnostics:
|
||||
return diagnostics, None, 0
|
||||
try:
|
||||
manifest = read_json(manifest_file) # type: ignore[arg-type]
|
||||
schema = read_json(schema_file) # type: ignore[arg-type]
|
||||
except OverflowError:
|
||||
add(diagnostics, "INPUT_TOO_LARGE", "/")
|
||||
return diagnostics, None, 0
|
||||
except TypeError:
|
||||
add(diagnostics, "INVALID_JSON", "/")
|
||||
return diagnostics, None, 0
|
||||
except ValueError:
|
||||
add(diagnostics, "BROKEN_ARTIFACT_LINK", "/")
|
||||
return diagnostics, None, 0
|
||||
if not schema_contract_valid(schema):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/schema")
|
||||
if not isinstance(manifest, dict):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/manifest")
|
||||
return diagnostics, None, 0
|
||||
if set(manifest) != {"schema_version", "baseline_version", "artifacts"}:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/manifest")
|
||||
version = manifest.get("baseline_version")
|
||||
if not isinstance(version, str) or not VERSION_RE.fullmatch(version):
|
||||
add(diagnostics, "BASELINE_VERSION_MISMATCH", "/baseline_version")
|
||||
version = None
|
||||
if type(manifest.get("schema_version")) is not int or manifest.get("schema_version") != 1:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/schema_version")
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, list) or len(artifacts) != 5:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/artifacts")
|
||||
return diagnostics, version, 0
|
||||
loaded: dict[str, Any] = {}
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(artifacts):
|
||||
pointer = f"/artifacts/{index}"
|
||||
if not isinstance(item, dict):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", pointer)
|
||||
continue
|
||||
if set(item) != {"kind", "path", "sha256"}:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", pointer)
|
||||
kind = item.get("kind")
|
||||
if kind not in EXPECTED_KINDS or kind in seen:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"{pointer}/kind")
|
||||
continue
|
||||
seen.add(kind)
|
||||
artifact_file = logical_path(root, item.get("path"))
|
||||
if artifact_file is None:
|
||||
add(diagnostics, "BROKEN_ARTIFACT_LINK", f"{pointer}/path")
|
||||
continue
|
||||
if not tracked_path(root, item.get("path")):
|
||||
add(diagnostics, "UNTRACKED_ARTIFACT", f"{pointer}/path")
|
||||
expected_hash = item.get("sha256")
|
||||
if not isinstance(expected_hash, str) or not HEX_RE.fullmatch(expected_hash):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"{pointer}/sha256")
|
||||
continue
|
||||
try:
|
||||
raw = read_bytes(artifact_file, MAX_CHECKLIST_BYTES if kind == "checklist" else MAX_JSON_BYTES)
|
||||
except OverflowError:
|
||||
add(diagnostics, "INPUT_TOO_LARGE", pointer)
|
||||
continue
|
||||
except ValueError:
|
||||
add(diagnostics, "BROKEN_ARTIFACT_LINK", pointer)
|
||||
continue
|
||||
if hashlib.sha256(raw).hexdigest() != expected_hash:
|
||||
add(diagnostics, "CHECKSUM_MISMATCH", f"{pointer}/sha256")
|
||||
if kind == "checklist":
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
add(diagnostics, "INVALID_JSON", pointer)
|
||||
continue
|
||||
match = re.search(r"(?m)^baseline_version:\s*([^\s]+)\s*$", text)
|
||||
if not match or match.group(1) != version:
|
||||
add(diagnostics, "BASELINE_VERSION_MISMATCH", pointer)
|
||||
loaded[kind] = text
|
||||
else:
|
||||
try:
|
||||
loaded[kind] = json.loads(raw.decode("utf-8"), object_pairs_hook=reject_duplicates)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, DuplicateKey, ValueError, RecursionError):
|
||||
add(diagnostics, "INVALID_JSON", pointer)
|
||||
if seen != EXPECTED_KINDS:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/artifacts")
|
||||
checklist_checks = parse_checklist(loaded.get("checklist"), diagnostics)
|
||||
validate_loaded(loaded, version, checklist_checks, diagnostics)
|
||||
return diagnostics, version, len(loaded.get("inventory", {}).get("flows", [])) if isinstance(loaded.get("inventory"), dict) else 0
|
||||
|
||||
|
||||
def validate_loaded(
|
||||
loaded: dict[str, Any],
|
||||
version: str | None,
|
||||
checklist_checks: dict[str, dict[str, Any]],
|
||||
diagnostics: list[Diagnostic],
|
||||
) -> None:
|
||||
for kind in ("required_surfaces", "taxonomy", "results"):
|
||||
value = loaded.get(kind)
|
||||
if not isinstance(value, dict) or value.get("baseline_version") != version:
|
||||
add(diagnostics, "BASELINE_VERSION_MISMATCH", f"/{kind}/baseline_version")
|
||||
inventory = loaded.get("inventory")
|
||||
flows = inventory.get("flows") if isinstance(inventory, dict) else None
|
||||
if not isinstance(flows, list):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/inventory/flows")
|
||||
return
|
||||
statuses: dict[str, str] = {}
|
||||
for index, flow in enumerate(flows):
|
||||
if not isinstance(flow, dict) or not FLOW_RE.fullmatch(str(flow.get("id", ""))):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/inventory/flows/{index}")
|
||||
continue
|
||||
flow_id = flow["id"]
|
||||
status = flow.get("status")
|
||||
if flow_id in statuses or status not in IMPLEMENTATION_STATUSES:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/inventory/flows/{index}")
|
||||
continue
|
||||
statuses[flow_id] = status
|
||||
required = loaded.get("required_surfaces")
|
||||
required_ids = required.get("required_flow_ids") if exact_keys(required, {"baseline_version", "required_flow_ids", "surface_groups"}) else None
|
||||
current_ids = {flow_id for flow_id, status in statuses.items() if status != "planned"}
|
||||
if (
|
||||
not isinstance(required_ids, list)
|
||||
or any(not isinstance(flow_id, str) for flow_id in required_ids)
|
||||
or len(required_ids) != len(set(required_ids))
|
||||
or set(required_ids) != current_ids
|
||||
):
|
||||
add(diagnostics, "INVALID_REQUIRED_SURFACES", "/required_surfaces/required_flow_ids")
|
||||
required_ids = []
|
||||
groups = required.get("surface_groups") if isinstance(required, dict) else None
|
||||
grouped: list[str] = []
|
||||
group_ids: set[str] = set()
|
||||
if not isinstance(groups, list) or not 1 <= len(groups) <= 64:
|
||||
add(diagnostics, "INVALID_REQUIRED_SURFACES", "/required_surfaces/surface_groups")
|
||||
groups = []
|
||||
for index, group in enumerate(groups):
|
||||
pointer = f"/required_surfaces/surface_groups/{index}"
|
||||
if not exact_keys(group, {"id", "flow_ids"}) or not FLOW_RE.fullmatch(str(group.get("id", ""))):
|
||||
add(diagnostics, "INVALID_REQUIRED_SURFACES", pointer)
|
||||
continue
|
||||
group_flow_ids = group.get("flow_ids")
|
||||
if group["id"] in group_ids or not isinstance(group_flow_ids, list) or not group_flow_ids or any(not isinstance(item, str) for item in group_flow_ids):
|
||||
add(diagnostics, "INVALID_REQUIRED_SURFACES", pointer)
|
||||
continue
|
||||
group_ids.add(group["id"])
|
||||
grouped.extend(group_flow_ids)
|
||||
if len(grouped) != len(set(grouped)) or set(grouped) != set(required_ids):
|
||||
add(diagnostics, "INVALID_REQUIRED_SURFACES", "/required_surfaces/surface_groups")
|
||||
taxonomy = loaded.get("taxonomy")
|
||||
if isinstance(taxonomy, dict):
|
||||
taxonomy_required = {"baseline_version", "implementation_statuses", "execution_verdicts", "evidence_modes", "full_pass"}
|
||||
taxonomy_optional = {"manual_only_rule", "non_pass_rule"}
|
||||
if not exact_keys(taxonomy, taxonomy_required, taxonomy_optional):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/taxonomy")
|
||||
for rule in taxonomy_optional & set(taxonomy):
|
||||
if not nonempty_text(taxonomy.get(rule), 2048):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/taxonomy/{rule}")
|
||||
if set(taxonomy.get("implementation_statuses", [])) != IMPLEMENTATION_STATUSES:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/taxonomy/implementation_statuses")
|
||||
if set(taxonomy.get("execution_verdicts", [])) != EXECUTION_VERDICTS:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/taxonomy/execution_verdicts")
|
||||
if set(taxonomy.get("evidence_modes", [])) != EVIDENCE_MODES:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/taxonomy/evidence_modes")
|
||||
if taxonomy.get("full_pass") != {"implementation_status": "implemented", "execution_verdict": "pass", "evidence_mode": "automated"}:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/taxonomy/full_pass")
|
||||
results = loaded.get("results")
|
||||
if not isinstance(results, dict):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/results")
|
||||
return
|
||||
if set(results) != {"baseline_version", "source_revision", "environment_class", "runs", "manual_results", "defects"}:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/results")
|
||||
root_revision = results.get("source_revision")
|
||||
root_environment = results.get("environment_class")
|
||||
if not isinstance(root_revision, str) or not re.fullmatch(r"[0-9a-f]{40}", root_revision):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/results/source_revision")
|
||||
if not isinstance(root_environment, str) or not re.fullmatch(r"[a-z0-9-]{1,64}", root_environment):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/results/environment_class")
|
||||
safe_serialized = json.dumps(results, sort_keys=True)
|
||||
if UNSAFE_RE.search(safe_serialized):
|
||||
add(diagnostics, "UNSAFE_EVIDENCE", "/results")
|
||||
evidenced: set[str] = set()
|
||||
seen_run_ids: set[str] = set()
|
||||
seen_hashes: set[str] = set()
|
||||
runs = results.get("runs")
|
||||
if not isinstance(runs, list):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/results/runs")
|
||||
runs = []
|
||||
for index, run in enumerate(runs):
|
||||
pointer = f"/results/runs/{index}"
|
||||
if not isinstance(run, dict):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", pointer)
|
||||
continue
|
||||
required_run_keys = {"id", "command_id", "flow_ids", "execution_verdict", "evidence_mode", "accepted", "source_report_sha256", "source_revision", "environment_class", "collector"}
|
||||
if not exact_keys(run, required_run_keys, {"summary", "safe_outcome"}):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", pointer)
|
||||
verdict = run.get("execution_verdict")
|
||||
mode = run.get("evidence_mode")
|
||||
accepted = run.get("accepted")
|
||||
source_revision = run.get("source_revision")
|
||||
environment_class = run.get("environment_class")
|
||||
command_id = run.get("command_id")
|
||||
source_hash = run.get("source_report_sha256")
|
||||
run_id = run.get("id")
|
||||
if not nonempty_text(run_id, 128) or run_id in seen_run_ids:
|
||||
add(diagnostics, "DUPLICATE_RUN_ID", f"{pointer}/id")
|
||||
elif isinstance(run_id, str):
|
||||
seen_run_ids.add(run_id)
|
||||
if command_id not in COMMAND_IDS:
|
||||
add(diagnostics, "UNKNOWN_COMMAND", f"{pointer}/command_id")
|
||||
if not isinstance(source_revision, str) or not re.fullmatch(r"[0-9a-f]{40}", source_revision):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"{pointer}/source_revision")
|
||||
elif source_revision != root_revision:
|
||||
add(diagnostics, "PROVENANCE_MISMATCH", f"{pointer}/source_revision")
|
||||
if not isinstance(environment_class, str) or not re.fullmatch(r"[a-z0-9-]{1,64}", environment_class):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"{pointer}/environment_class")
|
||||
elif environment_class != root_environment:
|
||||
add(diagnostics, "PROVENANCE_MISMATCH", f"{pointer}/environment_class")
|
||||
if not isinstance(source_hash, str) or not HEX_RE.fullmatch(source_hash):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"{pointer}/source_report_sha256")
|
||||
elif source_hash in seen_hashes:
|
||||
add(diagnostics, "DUPLICATE_REPORT_HASH", f"{pointer}/source_report_sha256")
|
||||
else:
|
||||
seen_hashes.add(source_hash)
|
||||
if verdict not in EXECUTION_VERDICTS or mode not in EVIDENCE_MODES:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", pointer)
|
||||
expected_accepted = verdict == "pass" and mode == "automated" and run.get("collector") == "capability-baseline-collector-v1"
|
||||
if type(accepted) is not bool or accepted is not expected_accepted:
|
||||
add(diagnostics, "NON_PASS_RECORDED_AS_PASS", pointer)
|
||||
flow_ids = run.get("flow_ids")
|
||||
if not isinstance(flow_ids, list) or not flow_ids or any(not isinstance(flow_id, str) for flow_id in flow_ids) or len(flow_ids) != len(set(flow_ids)):
|
||||
add(diagnostics, "MISSING_FLOW_EVIDENCE", f"{pointer}/flow_ids")
|
||||
continue
|
||||
for flow_id in flow_ids:
|
||||
if flow_id not in statuses:
|
||||
add(diagnostics, "UNKNOWN_FLOW_ID", f"{pointer}/flow_ids")
|
||||
elif accepted is True and statuses[flow_id] != "implemented":
|
||||
add(diagnostics, "NON_PASS_RECORDED_AS_PASS", f"{pointer}/flow_ids")
|
||||
elif accepted is True:
|
||||
evidenced.add(flow_id)
|
||||
for flow_id, status in statuses.items():
|
||||
if status == "implemented" and flow_id not in evidenced:
|
||||
add(diagnostics, "MISSING_FLOW_EVIDENCE", f"/inventory/{flow_id}")
|
||||
manual_results = results.get("manual_results")
|
||||
if not isinstance(manual_results, list) or len(manual_results) > 1000:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/results/manual_results")
|
||||
manual_results = []
|
||||
seen_checks: set[str] = set()
|
||||
for index, manual in enumerate(manual_results):
|
||||
pointer = f"/results/manual_results/{index}"
|
||||
required_manual_keys = {"check_id", "evidence_mode", "execution_verdict", "flow_ids", "next_evidence"}
|
||||
if not exact_keys(manual, required_manual_keys):
|
||||
add(diagnostics, "INVALID_MANUAL_EVIDENCE", pointer)
|
||||
continue
|
||||
check_id = manual.get("check_id")
|
||||
flow_ids = manual.get("flow_ids")
|
||||
if check_id in seen_checks or check_id not in checklist_checks:
|
||||
add(diagnostics, "INVALID_MANUAL_EVIDENCE", f"{pointer}/check_id")
|
||||
elif isinstance(check_id, str):
|
||||
seen_checks.add(check_id)
|
||||
if manual.get("evidence_mode") != "manual_only" or manual.get("execution_verdict") not in EXECUTION_VERDICTS:
|
||||
add(diagnostics, "INVALID_MANUAL_EVIDENCE", pointer)
|
||||
if not nonempty_text(manual.get("next_evidence"), 2048):
|
||||
add(diagnostics, "INVALID_MANUAL_EVIDENCE", f"{pointer}/next_evidence")
|
||||
if not isinstance(flow_ids, list) or not flow_ids or any(not isinstance(flow_id, str) or flow_id not in statuses for flow_id in flow_ids):
|
||||
add(diagnostics, "INVALID_MANUAL_EVIDENCE", f"{pointer}/flow_ids")
|
||||
elif isinstance(check_id, str) and check_id in checklist_checks:
|
||||
if checklist_checks[check_id]["flow_id"] not in flow_ids or checklist_checks[check_id]["verdict"] != manual.get("execution_verdict"):
|
||||
add(diagnostics, "INVALID_MANUAL_EVIDENCE", pointer)
|
||||
if seen_checks != set(checklist_checks):
|
||||
add(diagnostics, "INVALID_MANUAL_EVIDENCE", "/results/manual_results")
|
||||
defects = results.get("defects")
|
||||
if not isinstance(defects, list):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", "/results/defects")
|
||||
defects = []
|
||||
for index, defect in enumerate(defects):
|
||||
if not isinstance(defect, dict):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/results/defects/{index}")
|
||||
continue
|
||||
required_fields = ("id", "severity", "contract", "owner", "steps", "next_action", "flow_ids")
|
||||
if set(defect) != set(required_fields):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/results/defects/{index}")
|
||||
continue
|
||||
if defect.get("severity") not in {"Critical", "High", "Medium", "Low"}:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/results/defects/{index}/severity")
|
||||
if not isinstance(defect.get("steps"), list) or not 1 <= len(defect["steps"]) <= 20:
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/results/defects/{index}/steps")
|
||||
defect_flow_ids = defect.get("flow_ids")
|
||||
if not isinstance(defect_flow_ids, list) or not defect_flow_ids or any(not isinstance(flow_id, str) for flow_id in defect_flow_ids):
|
||||
add(diagnostics, "INVALID_SCHEMA_CONTRACT", f"/results/defects/{index}/flow_ids")
|
||||
defect_flow_ids = []
|
||||
for flow_id in defect_flow_ids:
|
||||
if flow_id not in statuses:
|
||||
add(diagnostics, "UNKNOWN_FLOW_ID", f"/results/defects/{index}/flow_ids")
|
||||
elif defect.get("severity") in SEVERE and statuses[flow_id] != "blocked":
|
||||
add(diagnostics, "SEVERE_DEFECT_FLOW_NOT_BLOCKED", f"/results/defects/{index}")
|
||||
|
||||
|
||||
def render(diagnostics: list[Diagnostic]) -> str:
|
||||
lines = [f"{item.code} pointer={item.pointer}" for item in sorted(set(diagnostics))]
|
||||
encoded = "\n".join(lines) + ("\n" if lines else "")
|
||||
raw = encoded.encode("utf-8")
|
||||
if len(raw) <= MAX_REPORT_BYTES:
|
||||
return encoded
|
||||
marker = "REPORT_TRUNCATED pointer=/\n"
|
||||
budget = MAX_REPORT_BYTES - len(marker.encode("utf-8"))
|
||||
return raw[:budget].decode("utf-8", "ignore") + marker
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", default=".")
|
||||
parser.add_argument("--manifest", required=True)
|
||||
parser.add_argument("--schema", required=True)
|
||||
args = parser.parse_args()
|
||||
root = Path(args.root)
|
||||
try:
|
||||
diagnostics, version, flow_count = validate(root, args.manifest, args.schema)
|
||||
except (OSError, ValueError, RecursionError):
|
||||
diagnostics, version, flow_count = [Diagnostic("VALIDATION_FAILED", "/")], None, 0
|
||||
if diagnostics:
|
||||
sys.stderr.write(render(diagnostics))
|
||||
return 1
|
||||
print(f"Capability baseline validation passed: baseline_version={version} flows={flow_count}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,715 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the versioned Crank Community Capability Inventory contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import heapq
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Iterable, Iterator
|
||||
|
||||
|
||||
SCHEMA_DIALECT = "https://json-schema.org/draft/2020-12/schema"
|
||||
MAX_DOCUMENT_BYTES = 4_194_304
|
||||
MAX_FLOWS = 10_000
|
||||
MAX_REQUIREMENTS_PER_FLOW = 64
|
||||
MAX_CAPABILITIES_PER_FLOW = 16
|
||||
MAX_EVIDENCE_LINKS_PER_KIND = 64
|
||||
MAX_ID_LENGTH = 128
|
||||
MAX_REQUIREMENT_LENGTH = 128
|
||||
MAX_CAPABILITY_LENGTH = 128
|
||||
MAX_OWNER_LENGTH = 256
|
||||
MAX_OUTCOME_LENGTH = 4_096
|
||||
MAX_NOTES_LENGTH = 4_096
|
||||
MAX_EVIDENCE_PATH_LENGTH = 1_024
|
||||
MAX_DIAGNOSTICS = 1_000
|
||||
MAX_REPORT_BYTES = 65_536
|
||||
|
||||
FLOW_TYPES = ("ui", "api", "mcp")
|
||||
FLOW_STATUSES = ("implemented", "planned", "gap", "blocked")
|
||||
ALLOWED_CAPABILITIES = ("tools", "resources", "prompts", "tasks", "load_runs")
|
||||
FORBIDDEN_CAPABILITIES = (
|
||||
"multi_workspace", # community-scope: allow=multi-workspace
|
||||
"enterprise_rbac", # community-scope: allow=enterprise
|
||||
"sso", # community-scope: allow=sso
|
||||
"non_rest_upstream", # community-scope: allow=non-rest-upstream
|
||||
"arbitrary_distributed_load_targets", # community-scope: allow=distributed-load-targets
|
||||
)
|
||||
|
||||
TOP_LEVEL_FIELDS = frozenset({"schema_version", "product", "flows"})
|
||||
FLOW_FIELDS = frozenset(
|
||||
{
|
||||
"id",
|
||||
"type",
|
||||
"requirements",
|
||||
"user_outcome",
|
||||
"owner",
|
||||
"status",
|
||||
"capabilities",
|
||||
"evidence",
|
||||
"notes",
|
||||
}
|
||||
)
|
||||
EVIDENCE_FIELDS = frozenset({"automated", "manual"})
|
||||
|
||||
SCHEMA_ALLOWED_KEYS: dict[tuple[str, ...], frozenset[str]] = {
|
||||
(): frozenset(
|
||||
{
|
||||
"$schema",
|
||||
"$id",
|
||||
"title",
|
||||
"type",
|
||||
"additionalProperties",
|
||||
"required",
|
||||
"properties",
|
||||
"$defs",
|
||||
}
|
||||
),
|
||||
("properties",): frozenset({"schema_version", "product", "flows"}),
|
||||
("properties", "schema_version"): frozenset({"const"}),
|
||||
("properties", "product"): frozenset({"const"}),
|
||||
("properties", "flows"): frozenset({"type", "minItems", "maxItems", "items"}),
|
||||
("properties", "flows", "items"): frozenset({"$ref"}),
|
||||
("$defs",): frozenset({"flow", "evidence", "evidencePaths"}),
|
||||
("$defs", "flow"): frozenset(
|
||||
{"type", "additionalProperties", "required", "properties"}
|
||||
),
|
||||
("$defs", "flow", "properties"): FLOW_FIELDS,
|
||||
("$defs", "flow", "properties", "id"): frozenset(
|
||||
{"type", "minLength", "maxLength", "pattern"}
|
||||
),
|
||||
("$defs", "flow", "properties", "type"): frozenset({"enum"}),
|
||||
("$defs", "flow", "properties", "requirements"): frozenset(
|
||||
{"type", "minItems", "maxItems", "uniqueItems", "items"}
|
||||
),
|
||||
("$defs", "flow", "properties", "requirements", "items"): frozenset(
|
||||
{"type", "minLength", "maxLength", "pattern"}
|
||||
),
|
||||
("$defs", "flow", "properties", "user_outcome"): frozenset(
|
||||
{"type", "minLength", "maxLength", "pattern"}
|
||||
),
|
||||
("$defs", "flow", "properties", "owner"): frozenset(
|
||||
{"type", "minLength", "maxLength", "pattern"}
|
||||
),
|
||||
("$defs", "flow", "properties", "status"): frozenset({"enum"}),
|
||||
("$defs", "flow", "properties", "capabilities"): frozenset(
|
||||
{"type", "minItems", "maxItems", "uniqueItems", "items"}
|
||||
),
|
||||
("$defs", "flow", "properties", "capabilities", "items"): frozenset({"enum"}),
|
||||
("$defs", "flow", "properties", "evidence"): frozenset({"$ref"}),
|
||||
("$defs", "flow", "properties", "notes"): frozenset({"type", "maxLength"}),
|
||||
("$defs", "evidence"): frozenset(
|
||||
{"type", "additionalProperties", "required", "properties"}
|
||||
),
|
||||
("$defs", "evidence", "properties"): EVIDENCE_FIELDS,
|
||||
("$defs", "evidence", "properties", "automated"): frozenset({"$ref"}),
|
||||
("$defs", "evidence", "properties", "manual"): frozenset({"$ref"}),
|
||||
("$defs", "evidencePaths"): frozenset(
|
||||
{"type", "minItems", "maxItems", "uniqueItems", "items"}
|
||||
),
|
||||
("$defs", "evidencePaths", "items"): frozenset(
|
||||
{"type", "minLength", "maxLength", "pattern"}
|
||||
),
|
||||
}
|
||||
|
||||
FLOW_ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
REQUIREMENT_ID = re.compile(r"^FR-[1-9][0-9]*$")
|
||||
NON_WHITESPACE_PATTERN = r"[\s\S]*\S[\s\S]*"
|
||||
EVIDENCE_PATH_PATTERN = r"^(?!/)(?!.*//)(?!.*(?:^|/)\.\.?($|/))(?!.*\\).+$"
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=True)
|
||||
class Diagnostic:
|
||||
code: str
|
||||
pointer: str
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ReverseDiagnostic:
|
||||
diagnostic: Diagnostic
|
||||
|
||||
def __lt__(self, other: "_ReverseDiagnostic") -> bool:
|
||||
return self.diagnostic > other.diagnostic
|
||||
|
||||
|
||||
class DiagnosticCollector:
|
||||
"""Retain only the lexicographically first diagnostics with an exact omitted count."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._heap: list[_ReverseDiagnostic] = []
|
||||
self.total = 0
|
||||
|
||||
def append(self, diagnostic: Diagnostic) -> None:
|
||||
self.total += 1
|
||||
wrapped = _ReverseDiagnostic(diagnostic)
|
||||
if len(self._heap) < MAX_DIAGNOSTICS:
|
||||
heapq.heappush(self._heap, wrapped)
|
||||
elif diagnostic < self._heap[0].diagnostic:
|
||||
heapq.heapreplace(self._heap, wrapped)
|
||||
|
||||
def extend(self, diagnostics: Iterable[Diagnostic] | "DiagnosticCollector") -> None:
|
||||
if isinstance(diagnostics, DiagnosticCollector):
|
||||
for diagnostic in diagnostics.ordered():
|
||||
self.append(diagnostic)
|
||||
self.total += diagnostics.omitted
|
||||
return
|
||||
for diagnostic in diagnostics:
|
||||
self.append(diagnostic)
|
||||
|
||||
@property
|
||||
def omitted(self) -> int:
|
||||
return self.total - len(self._heap)
|
||||
|
||||
def ordered(self) -> list[Diagnostic]:
|
||||
return sorted(item.diagnostic for item in self._heap)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return self.total > 0
|
||||
|
||||
|
||||
class DuplicateJsonMemberError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def reject_duplicate_json_members(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
value: dict[str, Any] = {}
|
||||
for key, item in pairs:
|
||||
if key in value:
|
||||
raise DuplicateJsonMemberError("duplicate JSON object member")
|
||||
value[key] = item
|
||||
return value
|
||||
|
||||
|
||||
def reject_nonstandard_json_constant(_: str) -> None:
|
||||
raise ValueError("non-standard JSON numeric constant")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--inventory", required=True)
|
||||
parser.add_argument("--schema", required=True)
|
||||
parser.add_argument("--required-fr", action="append", default=[])
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def path_has_symlink(root: Path, logical: PurePosixPath) -> bool:
|
||||
current = root
|
||||
for part in logical.parts:
|
||||
current = current / part
|
||||
if current.is_symlink():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def resolve_regular_file(root: Path, raw: str) -> Path | None:
|
||||
if not raw or len(raw) > MAX_EVIDENCE_PATH_LENGTH or "\\" in raw:
|
||||
return None
|
||||
logical = PurePosixPath(raw)
|
||||
if logical.is_absolute() or any(part in {"", ".", ".."} for part in logical.parts):
|
||||
return None
|
||||
if str(logical) != raw or path_has_symlink(root, logical):
|
||||
return None
|
||||
candidate = root.joinpath(*logical.parts)
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
resolved.relative_to(root)
|
||||
except (FileNotFoundError, OSError, RuntimeError, ValueError):
|
||||
return None
|
||||
if not resolved.is_file():
|
||||
return None
|
||||
return resolved
|
||||
|
||||
|
||||
def read_json_document(
|
||||
root: Path,
|
||||
raw_path: str,
|
||||
invalid_code: str,
|
||||
) -> tuple[Any | None, Diagnostic | None]:
|
||||
path = resolve_regular_file(root, raw_path)
|
||||
pointer = "/schema" if invalid_code == "INVALID_SCHEMA" else "/inventory"
|
||||
if path is None:
|
||||
return None, Diagnostic(invalid_code, pointer, "document path is unavailable")
|
||||
try:
|
||||
if path.stat().st_size > MAX_DOCUMENT_BYTES:
|
||||
return None, Diagnostic("INPUT_TOO_LARGE", pointer, "document exceeds limit")
|
||||
data = path.read_bytes()
|
||||
if len(data) > MAX_DOCUMENT_BYTES:
|
||||
return None, Diagnostic("INPUT_TOO_LARGE", pointer, "document exceeds limit")
|
||||
text = data.decode("utf-8")
|
||||
return (
|
||||
json.loads(
|
||||
text,
|
||||
object_pairs_hook=reject_duplicate_json_members,
|
||||
parse_constant=reject_nonstandard_json_constant,
|
||||
),
|
||||
None,
|
||||
)
|
||||
except (OSError, UnicodeDecodeError, ValueError, RecursionError, json.JSONDecodeError):
|
||||
return None, Diagnostic(invalid_code, pointer, "document is not valid UTF-8 JSON")
|
||||
|
||||
|
||||
def nested(document: Any, *parts: str) -> Any:
|
||||
current = document
|
||||
for part in parts:
|
||||
if not isinstance(current, dict) or part not in current:
|
||||
return None
|
||||
current = current[part]
|
||||
return current
|
||||
|
||||
|
||||
def contract_values_equal(actual: Any, expected: Any) -> bool:
|
||||
if isinstance(expected, list):
|
||||
return (
|
||||
isinstance(actual, list)
|
||||
and all(isinstance(item, str) for item in actual)
|
||||
and len(actual) == len(set(actual))
|
||||
and set(actual) == set(expected)
|
||||
)
|
||||
return type(actual) is type(expected) and actual == expected
|
||||
|
||||
|
||||
def validate_required_keyword(
|
||||
schema: dict[str, Any], parts: tuple[str, ...], expected: frozenset[str]
|
||||
) -> list[Diagnostic]:
|
||||
raw = nested(schema, *parts)
|
||||
valid = (
|
||||
isinstance(raw, list)
|
||||
and all(isinstance(item, str) for item in raw)
|
||||
and len(raw) == len(set(raw))
|
||||
and set(raw) == expected
|
||||
)
|
||||
if valid:
|
||||
return []
|
||||
return [
|
||||
Diagnostic(
|
||||
"INVALID_SCHEMA_CONTRACT",
|
||||
"/schema/" + "/".join(parts),
|
||||
"required fields differ from validator",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def validate_schema_contract(schema: Any) -> list[Diagnostic]:
|
||||
expected: list[tuple[tuple[str, ...], Any]] = [
|
||||
(("$schema",), SCHEMA_DIALECT),
|
||||
(("$id",), "https://crank.local/schemas/capability-inventory.schema.json"),
|
||||
(("title",), "Crank Community Capability Inventory"),
|
||||
(("type",), "object"),
|
||||
(("additionalProperties",), False),
|
||||
(("properties", "schema_version", "const"), 1),
|
||||
(("properties", "product", "const"), "crank-community"),
|
||||
(("properties", "flows", "type"), "array"),
|
||||
(("properties", "flows", "maxItems"), MAX_FLOWS),
|
||||
(("properties", "flows", "minItems"), 1),
|
||||
(("properties", "flows", "items", "$ref"), "#/$defs/flow"),
|
||||
(("$defs", "flow", "type"), "object"),
|
||||
(("$defs", "flow", "additionalProperties"), False),
|
||||
(("$defs", "flow", "properties", "id", "type"), "string"),
|
||||
(("$defs", "flow", "properties", "id", "minLength"), 1),
|
||||
(("$defs", "flow", "properties", "id", "maxLength"), MAX_ID_LENGTH),
|
||||
(("$defs", "flow", "properties", "id", "pattern"), FLOW_ID.pattern),
|
||||
(("$defs", "flow", "properties", "type", "enum"), list(FLOW_TYPES)),
|
||||
(("$defs", "flow", "properties", "status", "enum"), list(FLOW_STATUSES)),
|
||||
(
|
||||
("$defs", "flow", "properties", "requirements", "maxItems"),
|
||||
MAX_REQUIREMENTS_PER_FLOW,
|
||||
),
|
||||
(("$defs", "flow", "properties", "requirements", "type"), "array"),
|
||||
(("$defs", "flow", "properties", "requirements", "minItems"), 1),
|
||||
(("$defs", "flow", "properties", "requirements", "uniqueItems"), True),
|
||||
(
|
||||
("$defs", "flow", "properties", "requirements", "items", "type"),
|
||||
"string",
|
||||
),
|
||||
(("$defs", "flow", "properties", "requirements", "items", "minLength"), 1),
|
||||
(
|
||||
("$defs", "flow", "properties", "requirements", "items", "maxLength"),
|
||||
MAX_REQUIREMENT_LENGTH,
|
||||
),
|
||||
(
|
||||
("$defs", "flow", "properties", "requirements", "items", "pattern"),
|
||||
REQUIREMENT_ID.pattern,
|
||||
),
|
||||
(
|
||||
("$defs", "flow", "properties", "capabilities", "maxItems"),
|
||||
MAX_CAPABILITIES_PER_FLOW,
|
||||
),
|
||||
(("$defs", "flow", "properties", "capabilities", "type"), "array"),
|
||||
(("$defs", "flow", "properties", "capabilities", "minItems"), 1),
|
||||
(("$defs", "flow", "properties", "capabilities", "uniqueItems"), True),
|
||||
(
|
||||
("$defs", "flow", "properties", "capabilities", "items", "enum"),
|
||||
list(ALLOWED_CAPABILITIES),
|
||||
),
|
||||
(("$defs", "flow", "properties", "owner", "type"), "string"),
|
||||
(("$defs", "flow", "properties", "owner", "minLength"), 1),
|
||||
(("$defs", "flow", "properties", "owner", "maxLength"), MAX_OWNER_LENGTH),
|
||||
(("$defs", "flow", "properties", "owner", "pattern"), NON_WHITESPACE_PATTERN),
|
||||
(("$defs", "flow", "properties", "user_outcome", "type"), "string"),
|
||||
(("$defs", "flow", "properties", "user_outcome", "minLength"), 1),
|
||||
(
|
||||
("$defs", "flow", "properties", "user_outcome", "maxLength"),
|
||||
MAX_OUTCOME_LENGTH,
|
||||
),
|
||||
(
|
||||
("$defs", "flow", "properties", "user_outcome", "pattern"),
|
||||
NON_WHITESPACE_PATTERN,
|
||||
),
|
||||
(("$defs", "flow", "properties", "evidence", "$ref"), "#/$defs/evidence"),
|
||||
(("$defs", "flow", "properties", "notes", "type"), "string"),
|
||||
(("$defs", "flow", "properties", "notes", "maxLength"), MAX_NOTES_LENGTH),
|
||||
(("$defs", "evidence", "type"), "object"),
|
||||
(("$defs", "evidence", "additionalProperties"), False),
|
||||
(
|
||||
("$defs", "evidence", "properties", "automated", "$ref"),
|
||||
"#/$defs/evidencePaths",
|
||||
),
|
||||
(
|
||||
("$defs", "evidence", "properties", "manual", "$ref"),
|
||||
"#/$defs/evidencePaths",
|
||||
),
|
||||
(("$defs", "evidencePaths", "type"), "array"),
|
||||
(("$defs", "evidencePaths", "minItems"), 1),
|
||||
(("$defs", "evidencePaths", "maxItems"), MAX_EVIDENCE_LINKS_PER_KIND),
|
||||
(("$defs", "evidencePaths", "uniqueItems"), True),
|
||||
(("$defs", "evidencePaths", "items", "type"), "string"),
|
||||
(("$defs", "evidencePaths", "items", "minLength"), 1),
|
||||
(("$defs", "evidencePaths", "items", "maxLength"), MAX_EVIDENCE_PATH_LENGTH),
|
||||
(("$defs", "evidencePaths", "items", "pattern"), EVIDENCE_PATH_PATTERN),
|
||||
]
|
||||
if not isinstance(schema, dict):
|
||||
return [Diagnostic("INVALID_SCHEMA_CONTRACT", "/schema", "schema must be an object")]
|
||||
diagnostics = []
|
||||
for parts, allowed_keys in SCHEMA_ALLOWED_KEYS.items():
|
||||
node = schema if not parts else nested(schema, *parts)
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if any(key not in allowed_keys for key in node):
|
||||
pointer = "/schema" + ("/" + "/".join(parts) if parts else "")
|
||||
diagnostics.append(
|
||||
Diagnostic(
|
||||
"INVALID_SCHEMA_CONTRACT",
|
||||
f"{pointer}/<unsupported>",
|
||||
"schema contains an unsupported keyword",
|
||||
)
|
||||
)
|
||||
for parts, value in expected:
|
||||
if not contract_values_equal(nested(schema, *parts), value):
|
||||
diagnostics.append(
|
||||
Diagnostic(
|
||||
"INVALID_SCHEMA_CONTRACT",
|
||||
"/schema/" + "/".join(parts),
|
||||
"schema contract differs from validator",
|
||||
)
|
||||
)
|
||||
diagnostics.extend(
|
||||
validate_required_keyword(
|
||||
schema,
|
||||
("required",),
|
||||
frozenset({"schema_version", "product", "flows"}),
|
||||
)
|
||||
)
|
||||
diagnostics.extend(
|
||||
validate_required_keyword(
|
||||
schema,
|
||||
("$defs", "flow", "required"),
|
||||
frozenset(FLOW_FIELDS - {"notes"}),
|
||||
)
|
||||
)
|
||||
diagnostics.extend(
|
||||
validate_required_keyword(
|
||||
schema,
|
||||
("$defs", "evidence", "required"),
|
||||
EVIDENCE_FIELDS,
|
||||
)
|
||||
)
|
||||
return diagnostics
|
||||
|
||||
|
||||
def is_nonempty_string(value: Any, maximum: int) -> bool:
|
||||
return isinstance(value, str) and bool(value.strip()) and len(value) <= maximum
|
||||
|
||||
|
||||
def validate_string_list(
|
||||
value: Any,
|
||||
pointer: str,
|
||||
maximum_items: int,
|
||||
maximum_length: int,
|
||||
pattern: re.Pattern[str] | None = None,
|
||||
) -> list[Diagnostic]:
|
||||
if not isinstance(value, list) or not value:
|
||||
return [Diagnostic("MISSING_REQUIRED_FIELD", pointer, "non-empty list is required")]
|
||||
diagnostics: list[Diagnostic] = []
|
||||
if len(value) > maximum_items:
|
||||
diagnostics.append(Diagnostic("LIMIT_EXCEEDED", pointer, "collection exceeds limit"))
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(value[: maximum_items + 1]):
|
||||
if not is_nonempty_string(item, maximum_length):
|
||||
diagnostics.append(
|
||||
Diagnostic(
|
||||
"LIMIT_EXCEEDED",
|
||||
f"{pointer}/{index}",
|
||||
"string is empty, invalid, or exceeds limit",
|
||||
)
|
||||
)
|
||||
continue
|
||||
if pattern is not None and pattern.fullmatch(item) is None:
|
||||
diagnostics.append(
|
||||
Diagnostic("INVALID_FORMAT", f"{pointer}/{index}", "string format is invalid")
|
||||
)
|
||||
if item in seen:
|
||||
diagnostics.append(
|
||||
Diagnostic(
|
||||
"DUPLICATE_LIST_ITEM",
|
||||
f"{pointer}/{index}",
|
||||
"collection item is duplicated",
|
||||
)
|
||||
)
|
||||
seen.add(item)
|
||||
return diagnostics
|
||||
|
||||
|
||||
def reject_unknown_fields(
|
||||
value: dict[str, Any], allowed: frozenset[str], pointer: str
|
||||
) -> Iterator[Diagnostic]:
|
||||
for field in value:
|
||||
if field not in allowed:
|
||||
yield Diagnostic("UNKNOWN_FIELD", f"{pointer}/<unknown>", "field is not allowed")
|
||||
|
||||
|
||||
def validate_evidence(root: Path, value: Any, pointer: str) -> Iterator[Diagnostic]:
|
||||
if not isinstance(value, dict):
|
||||
yield Diagnostic("MISSING_REQUIRED_FIELD", pointer, "evidence object is required")
|
||||
return
|
||||
yield from reject_unknown_fields(value, EVIDENCE_FIELDS, pointer)
|
||||
for kind in ("automated", "manual"):
|
||||
paths = value.get(kind)
|
||||
path_pointer = f"{pointer}/{kind}"
|
||||
yield from validate_string_list(
|
||||
paths,
|
||||
path_pointer,
|
||||
MAX_EVIDENCE_LINKS_PER_KIND,
|
||||
MAX_EVIDENCE_PATH_LENGTH,
|
||||
)
|
||||
if not isinstance(paths, list):
|
||||
continue
|
||||
for index, raw_path in enumerate(paths[: MAX_EVIDENCE_LINKS_PER_KIND + 1]):
|
||||
if not isinstance(raw_path, str) or resolve_regular_file(root, raw_path) is None:
|
||||
yield Diagnostic(
|
||||
"BROKEN_EVIDENCE_LINK",
|
||||
f"{path_pointer}/{index}",
|
||||
"evidence path is unavailable",
|
||||
)
|
||||
|
||||
|
||||
def validate_inventory(
|
||||
root: Path,
|
||||
inventory: Any,
|
||||
required_frs: list[str],
|
||||
) -> tuple[DiagnosticCollector, dict[str, int]]:
|
||||
counts = {status: 0 for status in FLOW_STATUSES}
|
||||
diagnostics = DiagnosticCollector()
|
||||
if not isinstance(inventory, dict):
|
||||
diagnostics.append(Diagnostic("MISSING_REQUIRED_FIELD", "/", "inventory must be an object"))
|
||||
return diagnostics, counts
|
||||
|
||||
diagnostics.extend(reject_unknown_fields(inventory, TOP_LEVEL_FIELDS, ""))
|
||||
schema_version = inventory.get("schema_version")
|
||||
if type(schema_version) is not int or schema_version != 1:
|
||||
diagnostics.append(
|
||||
Diagnostic("MISSING_REQUIRED_FIELD", "/schema_version", "schema_version 1 is required")
|
||||
)
|
||||
if inventory.get("product") != "crank-community":
|
||||
diagnostics.append(
|
||||
Diagnostic("MISSING_REQUIRED_FIELD", "/product", "product identity is required")
|
||||
)
|
||||
flows = inventory.get("flows")
|
||||
if not isinstance(flows, list) or not flows:
|
||||
diagnostics.append(Diagnostic("MISSING_REQUIRED_FIELD", "/flows", "non-empty flows are required"))
|
||||
return diagnostics, counts
|
||||
if len(flows) > MAX_FLOWS:
|
||||
diagnostics.append(Diagnostic("LIMIT_EXCEEDED", "/flows", "flow collection exceeds limit"))
|
||||
|
||||
seen_ids: set[str] = set()
|
||||
observed_frs: set[str] = set()
|
||||
required_fields = (
|
||||
"id",
|
||||
"type",
|
||||
"requirements",
|
||||
"user_outcome",
|
||||
"owner",
|
||||
"status",
|
||||
"capabilities",
|
||||
"evidence",
|
||||
)
|
||||
for index, flow in enumerate(flows[: MAX_FLOWS + 1]):
|
||||
pointer = f"/flows/{index}"
|
||||
if not isinstance(flow, dict):
|
||||
diagnostics.append(Diagnostic("MISSING_REQUIRED_FIELD", pointer, "flow must be an object"))
|
||||
continue
|
||||
diagnostics.extend(reject_unknown_fields(flow, FLOW_FIELDS, pointer))
|
||||
for field in required_fields:
|
||||
if field not in flow:
|
||||
code = "MISSING_OWNER" if field == "owner" else "MISSING_REQUIRED_FIELD"
|
||||
diagnostics.append(Diagnostic(code, f"{pointer}/{field}", "field is required"))
|
||||
|
||||
flow_id = flow.get("id")
|
||||
if not is_nonempty_string(flow_id, MAX_ID_LENGTH) or not FLOW_ID.fullmatch(flow_id):
|
||||
diagnostics.append(Diagnostic("LIMIT_EXCEEDED", f"{pointer}/id", "flow id is invalid"))
|
||||
elif flow_id in seen_ids:
|
||||
diagnostics.append(Diagnostic("DUPLICATE_FLOW_ID", f"{pointer}/id", "flow id is duplicated"))
|
||||
else:
|
||||
seen_ids.add(flow_id)
|
||||
|
||||
flow_type = flow.get("type")
|
||||
if flow_type not in FLOW_TYPES:
|
||||
diagnostics.append(Diagnostic("UNKNOWN_FLOW_TYPE", f"{pointer}/type", "flow type is unknown"))
|
||||
|
||||
owner = flow.get("owner")
|
||||
if "owner" in flow:
|
||||
if not isinstance(owner, str) or not owner.strip():
|
||||
diagnostics.append(
|
||||
Diagnostic("MISSING_OWNER", f"{pointer}/owner", "owner is required")
|
||||
)
|
||||
elif len(owner) > MAX_OWNER_LENGTH:
|
||||
diagnostics.append(
|
||||
Diagnostic("LIMIT_EXCEEDED", f"{pointer}/owner", "owner exceeds limit")
|
||||
)
|
||||
|
||||
outcome = flow.get("user_outcome")
|
||||
if "user_outcome" in flow and not is_nonempty_string(outcome, MAX_OUTCOME_LENGTH):
|
||||
diagnostics.append(
|
||||
Diagnostic("LIMIT_EXCEEDED", f"{pointer}/user_outcome", "user outcome is invalid")
|
||||
)
|
||||
|
||||
notes = flow.get("notes")
|
||||
if notes is not None and (not isinstance(notes, str) or len(notes) > MAX_NOTES_LENGTH):
|
||||
diagnostics.append(Diagnostic("LIMIT_EXCEEDED", f"{pointer}/notes", "notes exceed limit"))
|
||||
|
||||
status = flow.get("status")
|
||||
if status not in FLOW_STATUSES:
|
||||
diagnostics.append(Diagnostic("UNKNOWN_STATUS", f"{pointer}/status", "status is unknown"))
|
||||
else:
|
||||
counts[status] += 1
|
||||
|
||||
requirements = flow.get("requirements")
|
||||
diagnostics.extend(
|
||||
validate_string_list(
|
||||
requirements,
|
||||
f"{pointer}/requirements",
|
||||
MAX_REQUIREMENTS_PER_FLOW,
|
||||
MAX_REQUIREMENT_LENGTH,
|
||||
REQUIREMENT_ID,
|
||||
)
|
||||
)
|
||||
if isinstance(requirements, list):
|
||||
for requirement in requirements[: MAX_REQUIREMENTS_PER_FLOW + 1]:
|
||||
if isinstance(requirement, str) and REQUIREMENT_ID.fullmatch(requirement):
|
||||
observed_frs.add(requirement)
|
||||
|
||||
capabilities = flow.get("capabilities")
|
||||
diagnostics.extend(
|
||||
validate_string_list(
|
||||
capabilities,
|
||||
f"{pointer}/capabilities",
|
||||
MAX_CAPABILITIES_PER_FLOW,
|
||||
MAX_CAPABILITY_LENGTH,
|
||||
)
|
||||
)
|
||||
if isinstance(capabilities, list):
|
||||
for cap_index, capability in enumerate(capabilities[: MAX_CAPABILITIES_PER_FLOW + 1]):
|
||||
cap_pointer = f"{pointer}/capabilities/{cap_index}"
|
||||
if capability in FORBIDDEN_CAPABILITIES:
|
||||
diagnostics.append(
|
||||
Diagnostic(
|
||||
"FORBIDDEN_COMMUNITY_CAPABILITY",
|
||||
cap_pointer,
|
||||
f"forbidden capability: {capability}",
|
||||
)
|
||||
)
|
||||
elif isinstance(capability, str) and capability not in ALLOWED_CAPABILITIES:
|
||||
diagnostics.append(
|
||||
Diagnostic("LIMIT_EXCEEDED", cap_pointer, "capability is not allowed")
|
||||
)
|
||||
|
||||
diagnostics.extend(validate_evidence(root, flow.get("evidence"), f"{pointer}/evidence"))
|
||||
|
||||
for required_fr in sorted(set(required_frs)):
|
||||
if not REQUIREMENT_ID.fullmatch(required_fr) or len(required_fr) > MAX_REQUIREMENT_LENGTH:
|
||||
diagnostics.append(
|
||||
Diagnostic("LIMIT_EXCEEDED", "/required-fr", "required FR identifier is invalid")
|
||||
)
|
||||
elif required_fr not in observed_frs:
|
||||
diagnostics.append(
|
||||
Diagnostic(
|
||||
"MISSING_REQUIRED_FR",
|
||||
"/requirements",
|
||||
f"mandatory requirement is missing: {required_fr}",
|
||||
)
|
||||
)
|
||||
return diagnostics, counts
|
||||
|
||||
|
||||
def render_report(diagnostics: DiagnosticCollector) -> str:
|
||||
selected = diagnostics.ordered()
|
||||
lines = [f"error[{item.code}] {item.pointer}: {item.message}" for item in selected]
|
||||
if diagnostics.omitted:
|
||||
lines.append(f"error[TRUNCATED] /: diagnostics omitted={diagnostics.omitted}")
|
||||
|
||||
report = "\n".join(lines) + "\n"
|
||||
encoded = report.encode("utf-8")
|
||||
if len(encoded) <= MAX_REPORT_BYTES:
|
||||
return report
|
||||
|
||||
marker = "error[TRUNCATED] /: report byte limit reached\n"
|
||||
marker_bytes = marker.encode("utf-8")
|
||||
budget = MAX_REPORT_BYTES - len(marker_bytes)
|
||||
kept: list[str] = []
|
||||
used = 0
|
||||
for line in lines:
|
||||
line_bytes = (line + "\n").encode("utf-8")
|
||||
if used + len(line_bytes) > budget:
|
||||
break
|
||||
kept.append(line)
|
||||
used += len(line_bytes)
|
||||
return "\n".join(kept) + ("\n" if kept else "") + marker
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
root = args.root.resolve()
|
||||
|
||||
schema, schema_error = read_json_document(root, args.schema, "INVALID_SCHEMA")
|
||||
inventory, inventory_error = read_json_document(root, args.inventory, "INVALID_JSON")
|
||||
diagnostics = DiagnosticCollector()
|
||||
diagnostics.extend(error for error in (schema_error, inventory_error) if error is not None)
|
||||
if schema_error is None:
|
||||
diagnostics.extend(validate_schema_contract(schema))
|
||||
|
||||
counts = {status: 0 for status in FLOW_STATUSES}
|
||||
if inventory_error is None:
|
||||
inventory_diagnostics, counts = validate_inventory(root, inventory, args.required_fr)
|
||||
diagnostics.extend(inventory_diagnostics)
|
||||
|
||||
if diagnostics:
|
||||
sys.stderr.write(render_report(diagnostics))
|
||||
return 1
|
||||
|
||||
total = sum(counts.values())
|
||||
print(
|
||||
"Capability inventory validation passed "
|
||||
f"total={total} implemented={counts['implemented']} "
|
||||
f"planned={counts['planned']} gap={counts['gap']} "
|
||||
f"blocked={counts['blocked']} pass={counts['implemented']}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user