274 lines
12 KiB
JavaScript
274 lines
12 KiB
JavaScript
// Operator-run AC-11 evidence against a Reference Deployment. This spec is
|
|
// intentionally separate from the local/mocked Playwright matrix.
|
|
const fs = require('node:fs');
|
|
const { test, expect } = require('@playwright/test');
|
|
const { localized, login } = require('./helpers');
|
|
|
|
// Remote operator credentials must not enter retry artifacts or failure captures.
|
|
test.use({ trace: 'off', video: 'off', screenshot: 'off' });
|
|
test.describe.configure({ retries: 0 });
|
|
|
|
const referenceBaseUrl = process.env.PLAYWRIGHT_BASE_URL;
|
|
const SAMPLE_COUNT = 20;
|
|
const OPERATION_COUNT = 2000;
|
|
const MAX_UPLOAD_BYTES = 256 * 1024;
|
|
const MAX_P95_MS = 2000;
|
|
|
|
function openApiDocument(operationCount) {
|
|
const lines = [
|
|
'openapi: 3.0.3',
|
|
'info:',
|
|
' title: Crank Reference Preview Benchmark',
|
|
' version: "1.0.0"',
|
|
'servers:',
|
|
' - url: https://api.example.test',
|
|
'paths:',
|
|
];
|
|
|
|
for (let index = 0; index < operationCount; index += 1) {
|
|
lines.push(` /op-${index}:`);
|
|
lines.push(' get:');
|
|
lines.push(' responses:');
|
|
lines.push(' "200":');
|
|
lines.push(' description: OK');
|
|
}
|
|
|
|
return Buffer.from(lines.join('\n') + '\n', 'utf8');
|
|
}
|
|
|
|
function percentile95(samples) {
|
|
const sorted = [...samples].sort((left, right) => left - right);
|
|
return sorted[Math.ceil(sorted.length * 0.95) - 1];
|
|
}
|
|
|
|
function safeReferenceOrigin(rawUrl) {
|
|
const url = new URL(rawUrl);
|
|
if (!['http:', 'https:'].includes(url.protocol)) {
|
|
throw new Error('PLAYWRIGHT_BASE_URL must use http or https');
|
|
}
|
|
if (url.username || url.password || url.search || url.hash) {
|
|
throw new Error('PLAYWRIGHT_BASE_URL must not contain credentials, query parameters, or fragments');
|
|
}
|
|
if (['localhost', '127.0.0.1', '::1'].includes(url.hostname)) {
|
|
throw new Error('AC-11 Reference benchmark requires a remote PLAYWRIGHT_BASE_URL');
|
|
}
|
|
return url.origin;
|
|
}
|
|
|
|
function readReferenceManifest(manifestPath) {
|
|
if (!manifestPath) {
|
|
throw new Error('CRANK_AC11_REFERENCE_ENV_JSON is required when PLAYWRIGHT_BASE_URL is set');
|
|
}
|
|
|
|
let manifest;
|
|
try {
|
|
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
} catch (_error) {
|
|
throw new Error('CRANK_AC11_REFERENCE_ENV_JSON must point to a readable valid JSON manifest');
|
|
}
|
|
|
|
const deployment = manifest && manifest.deployment;
|
|
const loadGenerator = manifest && manifest.load_generator;
|
|
const browser = manifest && manifest.browser;
|
|
const nonempty = (value) => typeof value === 'string' && value.trim().length > 0;
|
|
const finiteNumber = (value) => typeof value === 'number' && Number.isFinite(value);
|
|
const versionHasMajor = (value, major) => nonempty(value) && new RegExp(`^${major}(?:\\.|$)`).test(value.trim());
|
|
|
|
if (!deployment || !loadGenerator || !browser) {
|
|
throw new Error('Reference manifest requires deployment, load_generator, and browser objects');
|
|
}
|
|
for (const [name, host] of [['deployment', deployment], ['load_generator', loadGenerator], ['browser', browser]]) {
|
|
if (!nonempty(host.host_id)) throw new Error(`Reference manifest ${name}.host_id is required`);
|
|
}
|
|
if (deployment.host_id.trim() === loadGenerator.host_id.trim()
|
|
|| deployment.host_id.trim() === browser.host_id.trim()) {
|
|
throw new Error('Reference manifest requires deployment and load generator/browser on separate hosts');
|
|
}
|
|
if (!nonempty(deployment.cpu_model) || !nonempty(deployment.storage_type)
|
|
|| deployment.storage_type.trim().toLowerCase() !== 'ssd'
|
|
|| !finiteNumber(deployment.cpu_cores)
|
|
|| !finiteNumber(deployment.ram_gib) || !finiteNumber(deployment.ssd_iops)
|
|
|| !finiteNumber(deployment.ssd_throughput_mb_s) || !finiteNumber(deployment.ssd_capacity_gb)
|
|
|| !finiteNumber(deployment.lan_mbps)
|
|
|| !versionHasMajor(deployment.postgres_version, 16)
|
|
|| !versionHasMajor(deployment.valkey_version, 8)) {
|
|
throw new Error('Reference manifest is missing valid CPU, RAM, database, SSD, or LAN fields');
|
|
}
|
|
if (deployment.cpu_cores !== 4 || deployment.ram_gib !== 8
|
|
|| deployment.ssd_capacity_gb < 100 || deployment.ssd_iops < 3000
|
|
|| deployment.ssd_throughput_mb_s < 100
|
|
|| deployment.lan_mbps < 1000) {
|
|
throw new Error('Reference deployment must meet 4 CPU / 8 GiB RAM, SSD 3000 IOPS / 100 MB/s, and 1 Gbit LAN');
|
|
}
|
|
if (!finiteNumber(loadGenerator.cpu_cores) || !finiteNumber(loadGenerator.ram_gib)
|
|
|| loadGenerator.cpu_cores !== 4 || loadGenerator.ram_gib !== 8) {
|
|
throw new Error('Reference load_generator must have 4 CPU and 8 GiB RAM');
|
|
}
|
|
|
|
// Emit only the fields needed for benchmark interpretation. Do not copy
|
|
// arbitrary manifest keys (which could contain credentials) into reports.
|
|
return {
|
|
provenance: 'operator_manifest',
|
|
deployment: {
|
|
cpu_model: deployment.cpu_model.trim(),
|
|
cpu_cores: deployment.cpu_cores,
|
|
ram_gib: deployment.ram_gib,
|
|
postgres_version: deployment.postgres_version.trim(),
|
|
valkey_version: deployment.valkey_version.trim(),
|
|
storage_type: 'SSD',
|
|
ssd_capacity_gb: deployment.ssd_capacity_gb,
|
|
ssd_iops: deployment.ssd_iops,
|
|
ssd_throughput_mb_s: deployment.ssd_throughput_mb_s,
|
|
lan_mbps: deployment.lan_mbps,
|
|
},
|
|
load_generator: { cpu_cores: loadGenerator.cpu_cores, ram_gib: loadGenerator.ram_gib },
|
|
browser_host_separate_from_deployment: true,
|
|
};
|
|
}
|
|
|
|
test('[P1][reference] AC-11 measures 20 real 2000-operation preview round trips', async ({ page }, testInfo) => {
|
|
test.skip(!referenceBaseUrl, 'Set PLAYWRIGHT_BASE_URL to a Reference Deployment to run AC-11 evidence');
|
|
test.setTimeout(300_000);
|
|
|
|
expect(process.env.PLAYWRIGHT_SKIP_WEB_SERVER).toBe('1');
|
|
const referenceOrigin = safeReferenceOrigin(referenceBaseUrl);
|
|
const referenceManifest = readReferenceManifest(process.env.CRANK_AC11_REFERENCE_ENV_JSON);
|
|
const yaml = openApiDocument(OPERATION_COUNT);
|
|
expect(yaml.byteLength).toBeLessThan(MAX_UPLOAD_BYTES);
|
|
|
|
await login(page);
|
|
await page.waitForFunction(() => Boolean(document.querySelector('[data-testid="onboarding-dismiss"]')));
|
|
await page.locator('[data-testid="onboarding-dismiss"]').click();
|
|
await expect(page.locator('#crank-onboarding-panel')).toBeHidden();
|
|
await page.evaluate(() => window.setLang('en'));
|
|
await page.getByRole('button', { name: localized('Import OpenAPI', 'Импорт OpenAPI') }).click();
|
|
await expect(page.locator('#openapi-import-modal')).toBeVisible();
|
|
await page.locator('#openapi-import-file').setInputFiles({
|
|
name: 'reference-preview-2000.yaml',
|
|
mimeType: 'application/yaml',
|
|
buffer: yaml,
|
|
});
|
|
|
|
const previewButton = page.locator('#openapi-import-preview');
|
|
const createButton = page.locator('#openapi-import-create');
|
|
const rows = page.locator('.openapi-import-operation');
|
|
const samplesMs = [];
|
|
let browserEnvironment;
|
|
|
|
// Warm the preview once before timing. Warm-up still uses the real remote
|
|
// preview API.
|
|
const warmupResponsePromise = page.waitForResponse((response) =>
|
|
response.request().method() === 'POST'
|
|
&& response.url().includes('/imports/openapi/preview'),
|
|
);
|
|
await previewButton.click();
|
|
const warmupResponse = await warmupResponsePromise;
|
|
expect(warmupResponse.status(), 'warm-up preview status').toBe(200);
|
|
expect(new URL(warmupResponse.url()).origin).toBe(referenceOrigin);
|
|
await expect(page.locator('[data-testid="openapi-import-page-status"]')).toContainText('2000');
|
|
await expect(createButton).toBeEnabled();
|
|
|
|
const previewResponses = [];
|
|
page.on('response', (response) => {
|
|
if (response.request().method() === 'POST' && response.url().includes('/imports/openapi/preview')) {
|
|
previewResponses.push({ status: response.status(), url: response.url() });
|
|
}
|
|
});
|
|
|
|
for (let sampleIndex = 0; sampleIndex < SAMPLE_COUNT; sampleIndex += 1) {
|
|
await expect(previewButton).toBeEnabled();
|
|
await page.evaluate((index) => {
|
|
const measurements = window.__ac11Measurements || (window.__ac11Measurements = []);
|
|
const sample = { start: null, elapsed: null };
|
|
measurements[index] = sample;
|
|
const ready = () => {
|
|
const panel = document.querySelector('#openapi-import-preview-panel');
|
|
const button = document.querySelector('#openapi-import-preview');
|
|
const status = document.querySelector('[data-testid="openapi-import-page-status"]');
|
|
const create = document.querySelector('#openapi-import-create');
|
|
const rowCount = document.querySelectorAll('.openapi-import-operation').length;
|
|
if (sample.start === null || !button || button.disabled || !panel || panel.hidden
|
|
|| !status || !status.textContent.includes('2000') || !create || create.disabled
|
|
|| rowCount < 1 || rowCount > 100) return;
|
|
sample.elapsed = Number((performance.now() - sample.start).toFixed(3));
|
|
observer.disconnect();
|
|
};
|
|
const observer = new MutationObserver(ready);
|
|
observer.observe(document.body, { childList: true, subtree: true, attributes: true, characterData: true });
|
|
document.addEventListener('click', (event) => {
|
|
if (event.target.closest('#openapi-import-preview')) sample.start = performance.now();
|
|
}, { capture: true, once: true });
|
|
}, sampleIndex);
|
|
const responsePromise = page.waitForResponse((response) =>
|
|
response.request().method() === 'POST'
|
|
&& response.url().includes('/imports/openapi/preview'),
|
|
);
|
|
await previewButton.click();
|
|
const previewResponse = await responsePromise;
|
|
expect(previewResponse.status(), `preview sample ${sampleIndex + 1} status`).toBe(200);
|
|
expect(new URL(previewResponse.url()).origin).toBe(referenceOrigin);
|
|
|
|
await page.waitForFunction((index) => {
|
|
const sample = window.__ac11Measurements && window.__ac11Measurements[index];
|
|
return sample && sample.elapsed !== null;
|
|
}, sampleIndex, { polling: 'raf' });
|
|
const elapsed = await page.evaluate((index) => window.__ac11Measurements[index].elapsed, sampleIndex);
|
|
|
|
await expect(page.locator('#openapi-import-preview-panel')).toBeVisible();
|
|
await expect(page.locator('[data-testid="openapi-import-page-status"]')).toContainText('2000');
|
|
await expect(rows.first()).toBeVisible();
|
|
expect(await rows.count()).toBeGreaterThan(0);
|
|
expect(await rows.count()).toBeLessThanOrEqual(100);
|
|
await expect(createButton).toBeEnabled();
|
|
|
|
samplesMs.push(elapsed);
|
|
browserEnvironment = await page.evaluate(() => ({
|
|
user_agent: navigator.userAgent,
|
|
platform: navigator.platform,
|
|
language: navigator.language,
|
|
hardware_concurrency: navigator.hardwareConcurrency || null,
|
|
viewport: { width: innerWidth, height: innerHeight },
|
|
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
}));
|
|
}
|
|
|
|
expect(previewResponses).toHaveLength(SAMPLE_COUNT);
|
|
expect(previewResponses.every((item) => item.status === 200)).toBe(true);
|
|
const p95Ms = percentile95(samplesMs);
|
|
const report = {
|
|
benchmark: 'story-2.12-ac-11-reference-preview',
|
|
generated_at: new Date().toISOString(),
|
|
result: { sample_count: samplesMs.length, p95_ms: p95Ms, threshold_ms: MAX_P95_MS },
|
|
workload: {
|
|
operations: OPERATION_COUNT,
|
|
yaml_bytes: yaml.byteLength,
|
|
max_yaml_bytes: MAX_UPLOAD_BYTES,
|
|
preview_http_statuses: previewResponses.map((item) => item.status),
|
|
rendered_operation_rows: await rows.count(),
|
|
max_rendered_operation_rows: 100,
|
|
enabled_interaction: 'Create button enabled after preview',
|
|
interaction_enabled: true,
|
|
measurement_scope: 'click through real HTTP request, parsing, populated first page, and enabled Create button',
|
|
},
|
|
samples_ms: samplesMs,
|
|
environment: {
|
|
reference_origin: referenceOrigin,
|
|
playwright_base_url_configured: true,
|
|
playwright_skip_web_server: process.env.PLAYWRIGHT_SKIP_WEB_SERVER === '1',
|
|
node_version: process.version,
|
|
os_platform: process.platform,
|
|
os_arch: process.arch,
|
|
browser: browserEnvironment,
|
|
reference_manifest: referenceManifest,
|
|
},
|
|
};
|
|
|
|
const rawReport = JSON.stringify(report, null, 2);
|
|
await testInfo.attach('ac11-reference-preview-report.json', {
|
|
body: Buffer.from(rawReport),
|
|
contentType: 'application/json',
|
|
});
|
|
console.log('AC11_REFERENCE_PREVIEW_REPORT ' + JSON.stringify(report));
|
|
expect(p95Ms, 'Reference preview p95 must be at most 2 seconds').toBeLessThanOrEqual(MAX_P95_MS);
|
|
});
|