// Operator-run AC-11 evidence against a Reference Deployment. This spec is // intentionally separate from the local/mocked Playwright matrix. const fs = require('node:fs'); const { test, expect } = require('@playwright/test'); const { localized, login } = require('./helpers'); // Remote operator credentials must not enter retry artifacts or failure captures. test.use({ trace: 'off', video: 'off', screenshot: 'off' }); test.describe.configure({ retries: 0 }); const referenceBaseUrl = process.env.PLAYWRIGHT_BASE_URL; const SAMPLE_COUNT = 20; const OPERATION_COUNT = 2000; const MAX_UPLOAD_BYTES = 256 * 1024; const MAX_P95_MS = 2000; function openApiDocument(operationCount) { const lines = [ 'openapi: 3.0.3', 'info:', ' title: Crank Reference Preview Benchmark', ' version: "1.0.0"', 'servers:', ' - url: https://api.example.test', 'paths:', ]; for (let index = 0; index < operationCount; index += 1) { lines.push(` /op-${index}:`); lines.push(' get:'); lines.push(' responses:'); lines.push(' "200":'); lines.push(' description: OK'); } return Buffer.from(lines.join('\n') + '\n', 'utf8'); } function percentile95(samples) { const sorted = [...samples].sort((left, right) => left - right); return sorted[Math.ceil(sorted.length * 0.95) - 1]; } function safeReferenceOrigin(rawUrl) { const url = new URL(rawUrl); if (!['http:', 'https:'].includes(url.protocol)) { throw new Error('PLAYWRIGHT_BASE_URL must use http or https'); } if (url.username || url.password || url.search || url.hash) { throw new Error('PLAYWRIGHT_BASE_URL must not contain credentials, query parameters, or fragments'); } if (['localhost', '127.0.0.1', '::1'].includes(url.hostname)) { throw new Error('AC-11 Reference benchmark requires a remote PLAYWRIGHT_BASE_URL'); } return url.origin; } function readReferenceManifest(manifestPath) { if (!manifestPath) { throw new Error('CRANK_AC11_REFERENCE_ENV_JSON is required when PLAYWRIGHT_BASE_URL is set'); } let manifest; try { manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); } catch (_error) { throw new Error('CRANK_AC11_REFERENCE_ENV_JSON must point to a readable valid JSON manifest'); } const deployment = manifest && manifest.deployment; const loadGenerator = manifest && manifest.load_generator; const browser = manifest && manifest.browser; const nonempty = (value) => typeof value === 'string' && value.trim().length > 0; const finiteNumber = (value) => typeof value === 'number' && Number.isFinite(value); const versionHasMajor = (value, major) => nonempty(value) && new RegExp(`^${major}(?:\\.|$)`).test(value.trim()); if (!deployment || !loadGenerator || !browser) { throw new Error('Reference manifest requires deployment, load_generator, and browser objects'); } for (const [name, host] of [['deployment', deployment], ['load_generator', loadGenerator], ['browser', browser]]) { if (!nonempty(host.host_id)) throw new Error(`Reference manifest ${name}.host_id is required`); } if (deployment.host_id.trim() === loadGenerator.host_id.trim() || deployment.host_id.trim() === browser.host_id.trim()) { throw new Error('Reference manifest requires deployment and load generator/browser on separate hosts'); } if (!nonempty(deployment.cpu_model) || !nonempty(deployment.storage_type) || deployment.storage_type.trim().toLowerCase() !== 'ssd' || !finiteNumber(deployment.cpu_cores) || !finiteNumber(deployment.ram_gib) || !finiteNumber(deployment.ssd_iops) || !finiteNumber(deployment.ssd_throughput_mb_s) || !finiteNumber(deployment.ssd_capacity_gb) || !finiteNumber(deployment.lan_mbps) || !versionHasMajor(deployment.postgres_version, 16) || !versionHasMajor(deployment.valkey_version, 8)) { throw new Error('Reference manifest is missing valid CPU, RAM, database, SSD, or LAN fields'); } if (deployment.cpu_cores !== 4 || deployment.ram_gib !== 8 || deployment.ssd_capacity_gb < 100 || deployment.ssd_iops < 3000 || deployment.ssd_throughput_mb_s < 100 || deployment.lan_mbps < 1000) { throw new Error('Reference deployment must meet 4 CPU / 8 GiB RAM, SSD 3000 IOPS / 100 MB/s, and 1 Gbit LAN'); } if (!finiteNumber(loadGenerator.cpu_cores) || !finiteNumber(loadGenerator.ram_gib) || loadGenerator.cpu_cores !== 4 || loadGenerator.ram_gib !== 8) { throw new Error('Reference load_generator must have 4 CPU and 8 GiB RAM'); } // Emit only the fields needed for benchmark interpretation. Do not copy // arbitrary manifest keys (which could contain credentials) into reports. return { provenance: 'operator_manifest', deployment: { cpu_model: deployment.cpu_model.trim(), cpu_cores: deployment.cpu_cores, ram_gib: deployment.ram_gib, postgres_version: deployment.postgres_version.trim(), valkey_version: deployment.valkey_version.trim(), storage_type: 'SSD', ssd_capacity_gb: deployment.ssd_capacity_gb, ssd_iops: deployment.ssd_iops, ssd_throughput_mb_s: deployment.ssd_throughput_mb_s, lan_mbps: deployment.lan_mbps, }, load_generator: { cpu_cores: loadGenerator.cpu_cores, ram_gib: loadGenerator.ram_gib }, browser_host_separate_from_deployment: true, }; } test('[P1][reference] AC-11 measures 20 real 2000-operation preview round trips', async ({ page }, testInfo) => { test.skip(!referenceBaseUrl, 'Set PLAYWRIGHT_BASE_URL to a Reference Deployment to run AC-11 evidence'); test.setTimeout(300_000); expect(process.env.PLAYWRIGHT_SKIP_WEB_SERVER).toBe('1'); const referenceOrigin = safeReferenceOrigin(referenceBaseUrl); const referenceManifest = readReferenceManifest(process.env.CRANK_AC11_REFERENCE_ENV_JSON); const yaml = openApiDocument(OPERATION_COUNT); expect(yaml.byteLength).toBeLessThan(MAX_UPLOAD_BYTES); await login(page); await page.waitForFunction(() => Boolean(document.querySelector('[data-testid="onboarding-dismiss"]'))); await page.locator('[data-testid="onboarding-dismiss"]').click(); await expect(page.locator('#crank-onboarding-panel')).toBeHidden(); await page.evaluate(() => window.setLang('en')); await page.getByRole('button', { name: localized('Import OpenAPI', 'Импорт OpenAPI') }).click(); await expect(page.locator('#openapi-import-modal')).toBeVisible(); await page.locator('#openapi-import-file').setInputFiles({ name: 'reference-preview-2000.yaml', mimeType: 'application/yaml', buffer: yaml, }); const previewButton = page.locator('#openapi-import-preview'); const createButton = page.locator('#openapi-import-create'); const rows = page.locator('.openapi-import-operation'); const samplesMs = []; let browserEnvironment; // Warm the preview once before timing. Warm-up still uses the real remote // preview API. const warmupResponsePromise = page.waitForResponse((response) => response.request().method() === 'POST' && response.url().includes('/imports/openapi/preview'), ); await previewButton.click(); const warmupResponse = await warmupResponsePromise; expect(warmupResponse.status(), 'warm-up preview status').toBe(200); expect(new URL(warmupResponse.url()).origin).toBe(referenceOrigin); await expect(page.locator('[data-testid="openapi-import-page-status"]')).toContainText('2000'); await expect(createButton).toBeEnabled(); const previewResponses = []; page.on('response', (response) => { if (response.request().method() === 'POST' && response.url().includes('/imports/openapi/preview')) { previewResponses.push({ status: response.status(), url: response.url() }); } }); for (let sampleIndex = 0; sampleIndex < SAMPLE_COUNT; sampleIndex += 1) { await expect(previewButton).toBeEnabled(); await page.evaluate((index) => { const measurements = window.__ac11Measurements || (window.__ac11Measurements = []); const sample = { start: null, elapsed: null }; measurements[index] = sample; const ready = () => { const panel = document.querySelector('#openapi-import-preview-panel'); const button = document.querySelector('#openapi-import-preview'); const status = document.querySelector('[data-testid="openapi-import-page-status"]'); const create = document.querySelector('#openapi-import-create'); const rowCount = document.querySelectorAll('.openapi-import-operation').length; if (sample.start === null || !button || button.disabled || !panel || panel.hidden || !status || !status.textContent.includes('2000') || !create || create.disabled || rowCount < 1 || rowCount > 100) return; sample.elapsed = Number((performance.now() - sample.start).toFixed(3)); observer.disconnect(); }; const observer = new MutationObserver(ready); observer.observe(document.body, { childList: true, subtree: true, attributes: true, characterData: true }); document.addEventListener('click', (event) => { if (event.target.closest('#openapi-import-preview')) sample.start = performance.now(); }, { capture: true, once: true }); }, sampleIndex); const responsePromise = page.waitForResponse((response) => response.request().method() === 'POST' && response.url().includes('/imports/openapi/preview'), ); await previewButton.click(); const previewResponse = await responsePromise; expect(previewResponse.status(), `preview sample ${sampleIndex + 1} status`).toBe(200); expect(new URL(previewResponse.url()).origin).toBe(referenceOrigin); await page.waitForFunction((index) => { const sample = window.__ac11Measurements && window.__ac11Measurements[index]; return sample && sample.elapsed !== null; }, sampleIndex, { polling: 'raf' }); const elapsed = await page.evaluate((index) => window.__ac11Measurements[index].elapsed, sampleIndex); await expect(page.locator('#openapi-import-preview-panel')).toBeVisible(); await expect(page.locator('[data-testid="openapi-import-page-status"]')).toContainText('2000'); await expect(rows.first()).toBeVisible(); expect(await rows.count()).toBeGreaterThan(0); expect(await rows.count()).toBeLessThanOrEqual(100); await expect(createButton).toBeEnabled(); samplesMs.push(elapsed); browserEnvironment = await page.evaluate(() => ({ user_agent: navigator.userAgent, platform: navigator.platform, language: navigator.language, hardware_concurrency: navigator.hardwareConcurrency || null, viewport: { width: innerWidth, height: innerHeight }, timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, })); } expect(previewResponses).toHaveLength(SAMPLE_COUNT); expect(previewResponses.every((item) => item.status === 200)).toBe(true); const p95Ms = percentile95(samplesMs); const report = { benchmark: 'story-2.12-ac-11-reference-preview', generated_at: new Date().toISOString(), result: { sample_count: samplesMs.length, p95_ms: p95Ms, threshold_ms: MAX_P95_MS }, workload: { operations: OPERATION_COUNT, yaml_bytes: yaml.byteLength, max_yaml_bytes: MAX_UPLOAD_BYTES, preview_http_statuses: previewResponses.map((item) => item.status), rendered_operation_rows: await rows.count(), max_rendered_operation_rows: 100, enabled_interaction: 'Create button enabled after preview', interaction_enabled: true, measurement_scope: 'click through real HTTP request, parsing, populated first page, and enabled Create button', }, samples_ms: samplesMs, environment: { reference_origin: referenceOrigin, playwright_base_url_configured: true, playwright_skip_web_server: process.env.PLAYWRIGHT_SKIP_WEB_SERVER === '1', node_version: process.version, os_platform: process.platform, os_arch: process.arch, browser: browserEnvironment, reference_manifest: referenceManifest, }, }; const rawReport = JSON.stringify(report, null, 2); await testInfo.attach('ac11-reference-preview-report.json', { body: Buffer.from(rawReport), contentType: 'application/json', }); console.log('AC11_REFERENCE_PREVIEW_REPORT ' + JSON.stringify(report)); expect(p95Ms, 'Reference preview p95 must be at most 2 seconds').toBeLessThanOrEqual(MAX_P95_MS); });