391 lines
13 KiB
JSON
391 lines
13 KiB
JSON
{
|
|
"baseline_version": "2026.08.24.1",
|
|
"defects": [
|
|
{
|
|
"contract": "Local Playwright stack starts only after the configured database exists and is stable.",
|
|
"flow_ids": [
|
|
"ui-auth-workspace"
|
|
],
|
|
"id": "DEF-BL-001",
|
|
"next_action": "Make readiness wait for the configured database after PostgreSQL bootstrap completes.",
|
|
"owner": "quality-community",
|
|
"severity": "Medium",
|
|
"steps": [
|
|
"Start the default Playwright stack with a new local PostgreSQL container.",
|
|
"Observe pg_isready succeed against the temporary bootstrap server.",
|
|
"Observe Admin API lose its connection when PostgreSQL restarts after database creation."
|
|
]
|
|
},
|
|
{
|
|
"contract": "Stopping the local Playwright stack terminates its loop and all child services.",
|
|
"flow_ids": [
|
|
"ui-auth-workspace"
|
|
],
|
|
"id": "DEF-BL-002",
|
|
"next_action": "Exit after the signal cleanup trap and terminate the actual cargo child processes.",
|
|
"owner": "quality-community",
|
|
"severity": "Medium",
|
|
"steps": [
|
|
"Start the Playwright stack and wait for readiness.",
|
|
"Send an interrupt to the stack process.",
|
|
"Observe the loop or re-parented Admin API and MCP processes remain alive."
|
|
]
|
|
},
|
|
{
|
|
"contract": "Workspace-dependent asynchronous UI loads reject stale responses.",
|
|
"flow_ids": [
|
|
"ui-agent-management",
|
|
"ui-auth-profiles-upstreams-secrets",
|
|
"ui-logs-approvals",
|
|
"ui-mcp-approval-keys",
|
|
"ui-operation-lifecycle",
|
|
"ui-usage"
|
|
],
|
|
"id": "DEF-UI-001",
|
|
"next_action": "Add request-generation identity guards in the owning UI hardening story and automate reordered responses.",
|
|
"owner": "admin-ui-community",
|
|
"severity": "Medium",
|
|
"steps": [
|
|
"Inspect workspace-change loaders for Operations, Agents, keys, secrets, logs, and usage.",
|
|
"Confirm requests are restarted on workspace change.",
|
|
"Confirm no shared request-generation or abort guard prevents an older response from overwriting newer state."
|
|
]
|
|
},
|
|
{
|
|
"contract": "OpenAPI import user-visible text follows the selected RU or EN locale.",
|
|
"flow_ids": [
|
|
"ui-operation-import-export"
|
|
],
|
|
"id": "DEF-UI-002",
|
|
"next_action": "Move OpenAPI import strings into the existing RU and EN i18n contract.",
|
|
"owner": "import-ui-community",
|
|
"severity": "Medium",
|
|
"steps": [
|
|
"Select the EN locale.",
|
|
"Open the OpenAPI import flow.",
|
|
"Observe Russian-only labels and status messages in the import component."
|
|
]
|
|
},
|
|
{
|
|
"contract": "Browser-visible error states never render credential-shaped fields received from an untrusted API response.",
|
|
"flow_ids": [
|
|
"ui-agent-management",
|
|
"ui-auth-profiles-upstreams-secrets",
|
|
"ui-logs-approvals",
|
|
"ui-operation-lifecycle",
|
|
"ui-usage"
|
|
],
|
|
"id": "DEF-UI-003",
|
|
"next_action": "Normalize browser API errors through a safe allowlisted message before rendering and add canary regressions for every async screen.",
|
|
"owner": "admin-ui-community",
|
|
"severity": "High",
|
|
"steps": [
|
|
"Authenticate to the local Community UI with seeded test data.",
|
|
"Return a bounded credential-shaped canary in a 500 response for each affected list endpoint.",
|
|
"Observe the untrusted response message rendered in the corresponding browser error state."
|
|
]
|
|
}
|
|
],
|
|
"environment_class": "community-test",
|
|
"manual_results": [
|
|
{
|
|
"check_id": "UI-01",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"ui-auth-workspace"
|
|
],
|
|
"next_evidence": "Automate the observed login error, recovery and locale transitions."
|
|
},
|
|
{
|
|
"check_id": "UI-02",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-operation-lifecycle"
|
|
],
|
|
"next_evidence": "Retest reordered responses and safe error rendering after DEF-UI-001 and DEF-UI-003 are fixed."
|
|
},
|
|
{
|
|
"check_id": "UI-03",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"ui-operation-test"
|
|
],
|
|
"next_evidence": "Automate the observed wizard validation and recovery states."
|
|
},
|
|
{
|
|
"check_id": "UI-04",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-operation-import-export"
|
|
],
|
|
"next_evidence": "Retest RU/EN after DEF-UI-002 and exercise malformed import recovery."
|
|
},
|
|
{
|
|
"check_id": "UI-05",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-agent-management"
|
|
],
|
|
"next_evidence": "Retest reordered responses and safe error rendering after DEF-UI-001 and DEF-UI-003 are fixed."
|
|
},
|
|
{
|
|
"check_id": "UI-06",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"ui-mcp-approval-keys"
|
|
],
|
|
"next_evidence": "Automate the observed empty/key-guidance state and a bounded create-and-cleanup reveal pass."
|
|
},
|
|
{
|
|
"check_id": "UI-07",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-auth-profiles-upstreams-secrets"
|
|
],
|
|
"next_evidence": "Retest safe error rendering after DEF-UI-003 is fixed."
|
|
},
|
|
{
|
|
"check_id": "UI-08",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-logs-approvals"
|
|
],
|
|
"next_evidence": "Retest safe error rendering after DEF-UI-003 is fixed."
|
|
},
|
|
{
|
|
"check_id": "UI-09",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-usage"
|
|
],
|
|
"next_evidence": "Retest safe errors and reordered responses after DEF-UI-001 and DEF-UI-003 are fixed."
|
|
},
|
|
{
|
|
"check_id": "UI-10",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-auth-workspace"
|
|
],
|
|
"next_evidence": "Add deterministic reordered-response tests after DEF-UI-001 is addressed."
|
|
},
|
|
{
|
|
"check_id": "UI-11",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-operation-lifecycle"
|
|
],
|
|
"next_evidence": "Repeat the EN locale pass after DEF-UI-002 is addressed."
|
|
},
|
|
{
|
|
"check_id": "UI-12",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "fail",
|
|
"flow_ids": [
|
|
"ui-logs-approvals"
|
|
],
|
|
"next_evidence": "Repeat the bounded canary pass after DEF-UI-003 is fixed."
|
|
},
|
|
{
|
|
"check_id": "UI-13",
|
|
"evidence_mode": "manual_only",
|
|
"execution_verdict": "not_run",
|
|
"flow_ids": [
|
|
"ui-onboarding-first-call"
|
|
],
|
|
"next_evidence": "Run the five-participant fresh-install protocol in docs/manual-regression-checklist.md after a clean Story 1.16 revision; record all censored non-completers and separate eligible denominator."
|
|
}
|
|
],
|
|
"runs": [
|
|
{
|
|
"accepted": true,
|
|
"collector": "capability-baseline-collector-v1",
|
|
"command_id": "ui-build",
|
|
"environment_class": "community-test",
|
|
"evidence_mode": "automated",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"ui-auth-workspace",
|
|
"ui-mcp-approval-keys",
|
|
"ui-operation-import-export",
|
|
"ui-operation-test"
|
|
],
|
|
"id": "run-ui-build-7a7f63c86e0e",
|
|
"source_report_sha256": "7a7f63c86e0ef376bee32d4ed202a086b1f16710b1d3b0cc70a21c3475367e2d",
|
|
"source_revision": "767428436d3a54e60809a727044dfcec48e1f1dd",
|
|
"summary": {
|
|
"exit_code": 0,
|
|
"skipped": 0,
|
|
"timed_out": false
|
|
}
|
|
},
|
|
{
|
|
"accepted": true,
|
|
"collector": "capability-baseline-collector-v1",
|
|
"command_id": "ui-playwright",
|
|
"environment_class": "community-test",
|
|
"evidence_mode": "automated",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"ui-auth-workspace",
|
|
"ui-mcp-approval-keys",
|
|
"ui-operation-import-export",
|
|
"ui-operation-test"
|
|
],
|
|
"id": "run-ui-playwright-60a3cd095202",
|
|
"source_report_sha256": "60a3cd095202739e8d317cd38464d3dace4d02fadb57fbc13f2aaef25d0359cf",
|
|
"source_revision": "767428436d3a54e60809a727044dfcec48e1f1dd",
|
|
"summary": {
|
|
"failed": 0,
|
|
"flaky": 0,
|
|
"not_run": 0,
|
|
"passed": 26,
|
|
"skipped": 0
|
|
}
|
|
},
|
|
{
|
|
"accepted": true,
|
|
"collector": "capability-baseline-collector-v1",
|
|
"command_id": "authenticated-product-smoke",
|
|
"environment_class": "community-test",
|
|
"evidence_mode": "automated",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"api-agent-catalog",
|
|
"api-mcp-approval-keys",
|
|
"api-operation-lifecycle",
|
|
"api-operation-test-run",
|
|
"mcp-published-tool-call",
|
|
"mcp-published-tool-list",
|
|
"mcp-transport-session"
|
|
],
|
|
"id": "run-authenticated-product-smoke-20a893677c88",
|
|
"safe_outcome": {
|
|
"agent_id": "agent_019fffea9a327832bf3c30d0496b9426",
|
|
"agent_revision": 1,
|
|
"command_id": "authenticated-product-smoke",
|
|
"operation_id": "op_019fffea99b1736299743b2ecd93ba0c",
|
|
"operation_version": 1,
|
|
"stages": [
|
|
"admin_test",
|
|
"operation_publish",
|
|
"agent_publish",
|
|
"mcp_list",
|
|
"mcp_call"
|
|
],
|
|
"verdict": "pass"
|
|
},
|
|
"source_report_sha256": "20a893677c8849d1e0ab7255a1ee54cd591e6b87e9678ebf64bc551f87abbd4b",
|
|
"source_revision": "767428436d3a54e60809a727044dfcec48e1f1dd",
|
|
"summary": {
|
|
"exit_code": 0,
|
|
"skipped": 0,
|
|
"timed_out": false
|
|
}
|
|
},
|
|
{
|
|
"accepted": true,
|
|
"collector": "capability-baseline-collector-v1",
|
|
"command_id": "just-verify",
|
|
"environment_class": "community-test",
|
|
"evidence_mode": "automated",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"api-agent-catalog",
|
|
"api-approvals",
|
|
"api-auth-profiles-upstreams-secrets",
|
|
"api-auth-workspace",
|
|
"api-canonical-request-trace-identity",
|
|
"api-logs-usage",
|
|
"api-mcp-approval-keys",
|
|
"api-operation-import-export",
|
|
"api-operation-lifecycle",
|
|
"api-operation-test-run",
|
|
"api-typed-metrics-foundation",
|
|
"mcp-approval-lifecycle",
|
|
"mcp-published-tool-call",
|
|
"mcp-published-tool-list",
|
|
"mcp-scoped-tool-search",
|
|
"mcp-transport-session"
|
|
],
|
|
"id": "run-just-verify-4f6cd6c62692",
|
|
"source_report_sha256": "4f6cd6c62692b95de705bf1c382a412d001f3e98470e4f1b6b4c16d96b625995",
|
|
"source_revision": "767428436d3a54e60809a727044dfcec48e1f1dd",
|
|
"summary": {
|
|
"exit_code": 0,
|
|
"skipped": 0,
|
|
"timed_out": false
|
|
}
|
|
},
|
|
{
|
|
"accepted": true,
|
|
"collector": "capability-baseline-collector-v1",
|
|
"command_id": "rust-admin-integration",
|
|
"environment_class": "community-test",
|
|
"evidence_mode": "automated",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"api-auth-profiles-upstreams-secrets",
|
|
"api-operation-lifecycle",
|
|
"api-operation-test-run"
|
|
],
|
|
"id": "run-credential-lifecycle-59bcb7a400b9",
|
|
"safe_outcome": {
|
|
"command": "cargo test -p admin-api --test integration integration::credential_lifecycle -- --test-threads=1",
|
|
"credential_lifecycle": "pass",
|
|
"secret_material_exposed": false,
|
|
"verdict": "pass"
|
|
},
|
|
"source_report_sha256": "59bcb7a400b91025fdb90fdf5a8d623dfa3f1bd54581522b640ae3d6532c554e",
|
|
"source_revision": "767428436d3a54e60809a727044dfcec48e1f1dd",
|
|
"summary": {
|
|
"exit_code": 0,
|
|
"failed": 0,
|
|
"passed": 8,
|
|
"skipped": 0,
|
|
"timed_out": false
|
|
}
|
|
},
|
|
{
|
|
"accepted": true,
|
|
"collector": "capability-baseline-collector-v1",
|
|
"command_id": "rust-admin-integration",
|
|
"environment_class": "community-test",
|
|
"evidence_mode": "automated",
|
|
"execution_verdict": "pass",
|
|
"flow_ids": [
|
|
"api-master-key-identity-rotation"
|
|
],
|
|
"id": "run-master-key-rotation-94df00120348",
|
|
"safe_outcome": {
|
|
"admin_startup_mismatch": "pass",
|
|
"cli_preflight_resume_verify_promote": "pass",
|
|
"mcp_startup_mismatch": "pass",
|
|
"registry_rotation_abort_resume_promote": "pass",
|
|
"secret_material_exposed": false,
|
|
"verdict": "pass"
|
|
},
|
|
"source_report_sha256": "94df0012034871bc5a6c65e7fa6a5e0f9eb14e2d651916cecff2b764fba5cefa",
|
|
"source_revision": "767428436d3a54e60809a727044dfcec48e1f1dd",
|
|
"summary": {
|
|
"exit_code": 0,
|
|
"failed": 0,
|
|
"passed": 28,
|
|
"skipped": 0,
|
|
"timed_out": false
|
|
}
|
|
}
|
|
],
|
|
"source_revision": "767428436d3a54e60809a727044dfcec48e1f1dd"
|
|
}
|