bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

docs: freeze capability evidence for story 1.5

Browse Source
This commit is contained in:
bsodfather
2026-08-14 00:39:12 +03:00
parent f6fc2e5c9b
commit 17c8e3a8f0
5 changed files with 107 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/capability-baseline/results.json
+99 -95
View File
@@ -1,9 +1,11 @@
{
"baseline_version": "2026.08.10.1",
"baseline_version": "2026.08.14.1",
"defects": [
{
"contract": "Local Playwright stack starts only after the configured database exists and is stable.",
"flow_ids": ["ui-auth-workspace"],
"flow_ids": [
"ui-auth-workspace"
],
"id": "DEF-BL-001",
"next_action": "Make readiness wait for the configured database after PostgreSQL bootstrap completes.",
"owner": "quality-community",
@@ -16,7 +18,9 @@
},
{
"contract": "Stopping the local Playwright stack terminates its loop and all child services.",
"flow_ids": ["ui-auth-workspace"],
"flow_ids": [
"ui-auth-workspace"
],
"id": "DEF-BL-002",
"next_action": "Exit after the signal cleanup trap and terminate the actual cargo child processes.",
"owner": "quality-community",
@@ -49,7 +53,9 @@
},
{
"contract": "OpenAPI import user-visible text follows the selected RU or EN locale.",
"flow_ids": ["ui-operation-import-export"],
"flow_ids": [
"ui-operation-import-export"
],
"id": "DEF-UI-002",
"next_action": "Move OpenAPI import strings into the existing RU and EN i18n contract.",
"owner": "import-ui-community",
@@ -86,132 +92,117 @@
"check_id": "UI-01",
"evidence_mode": "manual_only",
"execution_verdict": "pass",
"flow_ids": ["ui-auth-workspace"],
"flow_ids": [
"ui-auth-workspace"
],
"next_evidence": "Automate the observed login error, recovery and locale transitions."
},
{
"check_id": "UI-02",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-operation-lifecycle"],
"flow_ids": [
"ui-operation-lifecycle"
],
"next_evidence": "Retest reordered responses and safe error rendering after DEF-UI-001 and DEF-UI-003 are fixed."
},
{
"check_id": "UI-03",
"evidence_mode": "manual_only",
"execution_verdict": "pass",
"flow_ids": ["ui-operation-test"],
"flow_ids": [
"ui-operation-test"
],
"next_evidence": "Automate the observed wizard validation and recovery states."
},
{
"check_id": "UI-04",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-operation-import-export"],
"flow_ids": [
"ui-operation-import-export"
],
"next_evidence": "Retest RU/EN after DEF-UI-002 and exercise malformed import recovery."
},
{
"check_id": "UI-05",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-agent-management"],
"flow_ids": [
"ui-agent-management"
],
"next_evidence": "Retest reordered responses and safe error rendering after DEF-UI-001 and DEF-UI-003 are fixed."
},
{
"check_id": "UI-06",
"evidence_mode": "manual_only",
"execution_verdict": "pass",
"flow_ids": ["ui-mcp-approval-keys"],
"flow_ids": [
"ui-mcp-approval-keys"
],
"next_evidence": "Automate the observed empty/key-guidance state and a bounded create-and-cleanup reveal pass."
},
{
"check_id": "UI-07",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-auth-profiles-upstreams-secrets"],
"flow_ids": [
"ui-auth-profiles-upstreams-secrets"
],
"next_evidence": "Retest safe error rendering after DEF-UI-003 is fixed."
},
{
"check_id": "UI-08",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-logs-approvals"],
"flow_ids": [
"ui-logs-approvals"
],
"next_evidence": "Retest safe error rendering after DEF-UI-003 is fixed."
},
{
"check_id": "UI-09",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-usage"],
"flow_ids": [
"ui-usage"
],
"next_evidence": "Retest safe errors and reordered responses after DEF-UI-001 and DEF-UI-003 are fixed."
},
{
"check_id": "UI-10",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-auth-workspace"],
"flow_ids": [
"ui-auth-workspace"
],
"next_evidence": "Add deterministic reordered-response tests after DEF-UI-001 is addressed."
},
{
"check_id": "UI-11",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-operation-lifecycle"],
"flow_ids": [
"ui-operation-lifecycle"
],
"next_evidence": "Repeat the EN locale pass after DEF-UI-002 is addressed."
},
{
"check_id": "UI-12",
"evidence_mode": "manual_only",
"execution_verdict": "fail",
"flow_ids": ["ui-logs-approvals"],
"flow_ids": [
"ui-logs-approvals"
],
"next_evidence": "Repeat the bounded canary pass after DEF-UI-003 is fixed."
}
],
"runs": [
{
"accepted": true,
"collector": "capability-baseline-collector-v1",
"command_id": "rust-admin-integration",
"evidence_mode": "automated",
"execution_verdict": "pass",
"flow_ids": [
"api-agent-catalog",
"api-approvals",
"api-auth-profiles-upstreams-secrets",
"api-auth-workspace",
"api-logs-usage",
"api-mcp-approval-keys",
"api-operation-import-export",
"api-operation-lifecycle",
"api-operation-test-run"
],
"id": "run-rust-admin-integration-fa07285c8e58",
"environment_class": "community-test",
"source_revision": "c30461cc92491ebe3d527e125a1657b73fa67006",
"source_report_sha256": "fa07285c8e58dfad3834ffd420d80c91cdf0f0b5d0c54ef3e0eeaf35df3ab3f8"
},
{
"accepted": true,
"collector": "capability-baseline-collector-v1",
"command_id": "rust-mcp-integration",
"evidence_mode": "automated",
"execution_verdict": "pass",
"flow_ids": [
"mcp-approval-lifecycle",
"mcp-published-tool-call",
"mcp-published-tool-list",
"mcp-scoped-tool-search",
"mcp-transport-session"
],
"id": "run-rust-mcp-integration-ccff7c0029c1",
"environment_class": "community-test",
"source_revision": "c30461cc92491ebe3d527e125a1657b73fa67006",
"source_report_sha256": "ccff7c0029c1347459a76b8ecd63a308a7162ddb5bc272441e79ffbf2ccdfb83"
},
{
"accepted": true,
"collector": "capability-baseline-collector-v1",
"command_id": "ui-build",
"environment_class": "community-test",
"evidence_mode": "automated",
"execution_verdict": "pass",
"flow_ids": [
@@ -220,15 +211,20 @@
"ui-operation-import-export",
"ui-operation-test"
],
"id": "run-ui-build-755e91f2200b",
"environment_class": "community-test",
"source_revision": "c30461cc92491ebe3d527e125a1657b73fa67006",
"source_report_sha256": "755e91f2200b4ac43783820afde4f5d6b336ff7a6ebd5e71b055c43d5c2e4986"
"id": "run-ui-build-4e1216d7a69e",
"source_report_sha256": "4e1216d7a69e30c773c4e58799bbc2e41ebc247d2d400511d3bd8c32084b7a74",
"source_revision": "f6fc2e5c9bb748ad8db4bdc52ffdbdb6ac24d6d9",
"summary": {
"exit_code": 0,
"skipped": 0,
"timed_out": false
}
},
{
"accepted": true,
"collector": "capability-baseline-collector-v1",
"command_id": "ui-playwright",
"environment_class": "community-test",
"evidence_mode": "automated",
"execution_verdict": "pass",
"flow_ids": [
@@ -237,16 +233,22 @@
"ui-operation-import-export",
"ui-operation-test"
],
"id": "run-ui-playwright-a1236d5f45ab",
"environment_class": "community-test",
"source_revision": "c30461cc92491ebe3d527e125a1657b73fa67006",
"summary": {"passed": 21, "failed": 0, "flaky": 0, "skipped": 0},
"source_report_sha256": "a1236d5f45ab6273625bdb61634b1ea6c6d3ac876757b85a0a5ed8b5855fea6b"
"id": "run-ui-playwright-6040821b7473",
"source_report_sha256": "6040821b74732ade95dd604b091967d45edf9eb04633016d536ecaf1585973d9",
"source_revision": "f6fc2e5c9bb748ad8db4bdc52ffdbdb6ac24d6d9",
"summary": {
"failed": 0,
"flaky": 0,
"not_run": 0,
"passed": 26,
"skipped": 0
}
},
{
"accepted": true,
"collector": "capability-baseline-collector-v1",
"command_id": "authenticated-product-smoke",
"environment_class": "community-test",
"evidence_mode": "automated",
"execution_verdict": "pass",
"flow_ids": [
@@ -258,17 +260,29 @@
"mcp-published-tool-list",
"mcp-transport-session"
],
"id": "run-authenticated-product-smoke-471a29ea400f",
"environment_class": "community-test",
"source_revision": "c30461cc92491ebe3d527e125a1657b73fa67006",
"safe_outcome": {
"agent_id": "agent_019fe3a5468d75039020e367f1096136",
"agent_revision": 1,
"operation_id": "op_019fe3a546177533a2df26a36c79d358",
"operation_version": 1,
"stages": ["admin_test", "operation_publish", "agent_publish", "mcp_list", "mcp_call"]
"id": "run-authenticated-product-smoke-bf9a835d877f",
"source_report_sha256": "bf9a835d877f0631051413a758f98b4cd7be55c3666acb22dcf012b22416d517",
"source_revision": "f6fc2e5c9bb748ad8db4bdc52ffdbdb6ac24d6d9",
"summary": {
"exit_code": 0,
"skipped": 0,
"timed_out": false
},
"source_report_sha256": "471a29ea400fe1cb3eceb68d073c15ecba4e95ee5dff487fec0bb969f09c65e0"
"safe_outcome": {
"agent_id": "agent_019ffd03085174338d660c8a7f19718a",
"agent_revision": 1,
"command_id": "authenticated-product-smoke",
"operation_id": "op_019ffd0307d37ae3ba38dd2986a22fb1",
"operation_version": 1,
"stages": [
"admin_test",
"operation_publish",
"agent_publish",
"mcp_list",
"mcp_call"
],
"verdict": "pass"
}
},
{
"accepted": true,
@@ -282,6 +296,7 @@
"api-approvals",
"api-auth-profiles-upstreams-secrets",
"api-auth-workspace",
"api-canonical-request-trace-identity",
"api-logs-usage",
"api-mcp-approval-keys",
"api-operation-import-export",
@@ -293,26 +308,15 @@
"mcp-scoped-tool-search",
"mcp-transport-session"
],
"id": "run-just-verify-c6c16ce3c4b2",
"source_report_sha256": "c6c16ce3c4b2fa8355ae73c6ad6e87f3c829e86d720cd153133c161d073330b2",
"source_revision": "c30461cc92491ebe3d527e125a1657b73fa67006",
"summary": {"exit_code": 0, "skipped": 0, "timed_out": false}
},
{
"accepted": true,
"collector": "capability-baseline-collector-v1",
"command_id": "just-verify",
"environment_class": "community-test",
"evidence_mode": "automated",
"execution_verdict": "pass",
"flow_ids": [
"api-canonical-request-trace-identity"
],
"id": "run-just-verify-a3e1538374ee",
"source_report_sha256": "a3e1538374eea02b3140ea20a7966c01225a78ac510296d0bb08eaf87a3c476c",
"source_revision": "c30461cc92491ebe3d527e125a1657b73fa67006",
"summary": {"exit_code": 0, "skipped": 0, "timed_out": false}
"id": "run-just-verify-fa53a3c600d7",
"source_report_sha256": "fa53a3c600d774dbed79b6029ffc8ffa20f210610388ac6810df328c36c6685a",
"source_revision": "f6fc2e5c9bb748ad8db4bdc52ffdbdb6ac24d6d9",
"summary": {
"exit_code": 0,
"skipped": 0,
"timed_out": false
}
}
],
"source_revision": "c30461cc92491ebe3d527e125a1657b73fa67006"
"source_revision": "f6fc2e5c9bb748ad8db4bdc52ffdbdb6ac24d6d9"
}