Files
crank/apps/ui/tests/e2e/openapi-preview-reference.spec.js
T
bsodfather e93de7015a
CI / Rust Checks (pull_request) Failing after 1s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
feat(import): complete OpenAPI onboarding and verify delivery
2026-10-09 17:50:38 +03:00

274 lines
12 KiB
JavaScript

// Operator-run AC-11 evidence against a Reference Deployment. This spec is
// intentionally separate from the local/mocked Playwright matrix.
const fs = require('node:fs');
const { test, expect } = require('@playwright/test');
const { localized, login } = require('./helpers');
// Remote operator credentials must not enter retry artifacts or failure captures.
test.use({ trace: 'off', video: 'off', screenshot: 'off' });
test.describe.configure({ retries: 0 });
const referenceBaseUrl = process.env.PLAYWRIGHT_BASE_URL;
const SAMPLE_COUNT = 20;
const OPERATION_COUNT = 2000;
const MAX_UPLOAD_BYTES = 256 * 1024;
const MAX_P95_MS = 2000;
function openApiDocument(operationCount) {
const lines = [
'openapi: 3.0.3',
'info:',
' title: Crank Reference Preview Benchmark',
' version: "1.0.0"',
'servers:',
' - url: https://api.example.test',
'paths:',
];
for (let index = 0; index < operationCount; index += 1) {
lines.push(` /op-${index}:`);
lines.push(' get:');
lines.push(' responses:');
lines.push(' "200":');
lines.push(' description: OK');
}
return Buffer.from(lines.join('\n') + '\n', 'utf8');
}
function percentile95(samples) {
const sorted = [...samples].sort((left, right) => left - right);
return sorted[Math.ceil(sorted.length * 0.95) - 1];
}
function safeReferenceOrigin(rawUrl) {
const url = new URL(rawUrl);
if (!['http:', 'https:'].includes(url.protocol)) {
throw new Error('PLAYWRIGHT_BASE_URL must use http or https');
}
if (url.username || url.password || url.search || url.hash) {
throw new Error('PLAYWRIGHT_BASE_URL must not contain credentials, query parameters, or fragments');
}
if (['localhost', '127.0.0.1', '::1'].includes(url.hostname)) {
throw new Error('AC-11 Reference benchmark requires a remote PLAYWRIGHT_BASE_URL');
}
return url.origin;
}
function readReferenceManifest(manifestPath) {
if (!manifestPath) {
throw new Error('CRANK_AC11_REFERENCE_ENV_JSON is required when PLAYWRIGHT_BASE_URL is set');
}
let manifest;
try {
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
} catch (_error) {
throw new Error('CRANK_AC11_REFERENCE_ENV_JSON must point to a readable valid JSON manifest');
}
const deployment = manifest && manifest.deployment;
const loadGenerator = manifest && manifest.load_generator;
const browser = manifest && manifest.browser;
const nonempty = (value) => typeof value === 'string' && value.trim().length > 0;
const finiteNumber = (value) => typeof value === 'number' && Number.isFinite(value);
const versionHasMajor = (value, major) => nonempty(value) && new RegExp(`^${major}(?:\\.|$)`).test(value.trim());
if (!deployment || !loadGenerator || !browser) {
throw new Error('Reference manifest requires deployment, load_generator, and browser objects');
}
for (const [name, host] of [['deployment', deployment], ['load_generator', loadGenerator], ['browser', browser]]) {
if (!nonempty(host.host_id)) throw new Error(`Reference manifest ${name}.host_id is required`);
}
if (deployment.host_id.trim() === loadGenerator.host_id.trim()
|| deployment.host_id.trim() === browser.host_id.trim()) {
throw new Error('Reference manifest requires deployment and load generator/browser on separate hosts');
}
if (!nonempty(deployment.cpu_model) || !nonempty(deployment.storage_type)
|| deployment.storage_type.trim().toLowerCase() !== 'ssd'
|| !finiteNumber(deployment.cpu_cores)
|| !finiteNumber(deployment.ram_gib) || !finiteNumber(deployment.ssd_iops)
|| !finiteNumber(deployment.ssd_throughput_mb_s) || !finiteNumber(deployment.ssd_capacity_gb)
|| !finiteNumber(deployment.lan_mbps)
|| !versionHasMajor(deployment.postgres_version, 16)
|| !versionHasMajor(deployment.valkey_version, 8)) {
throw new Error('Reference manifest is missing valid CPU, RAM, database, SSD, or LAN fields');
}
if (deployment.cpu_cores !== 4 || deployment.ram_gib !== 8
|| deployment.ssd_capacity_gb < 100 || deployment.ssd_iops < 3000
|| deployment.ssd_throughput_mb_s < 100
|| deployment.lan_mbps < 1000) {
throw new Error('Reference deployment must meet 4 CPU / 8 GiB RAM, SSD 3000 IOPS / 100 MB/s, and 1 Gbit LAN');
}
if (!finiteNumber(loadGenerator.cpu_cores) || !finiteNumber(loadGenerator.ram_gib)
|| loadGenerator.cpu_cores !== 4 || loadGenerator.ram_gib !== 8) {
throw new Error('Reference load_generator must have 4 CPU and 8 GiB RAM');
}
// Emit only the fields needed for benchmark interpretation. Do not copy
// arbitrary manifest keys (which could contain credentials) into reports.
return {
provenance: 'operator_manifest',
deployment: {
cpu_model: deployment.cpu_model.trim(),
cpu_cores: deployment.cpu_cores,
ram_gib: deployment.ram_gib,
postgres_version: deployment.postgres_version.trim(),
valkey_version: deployment.valkey_version.trim(),
storage_type: 'SSD',
ssd_capacity_gb: deployment.ssd_capacity_gb,
ssd_iops: deployment.ssd_iops,
ssd_throughput_mb_s: deployment.ssd_throughput_mb_s,
lan_mbps: deployment.lan_mbps,
},
load_generator: { cpu_cores: loadGenerator.cpu_cores, ram_gib: loadGenerator.ram_gib },
browser_host_separate_from_deployment: true,
};
}
test('[P1][reference] AC-11 measures 20 real 2000-operation preview round trips', async ({ page }, testInfo) => {
test.skip(!referenceBaseUrl, 'Set PLAYWRIGHT_BASE_URL to a Reference Deployment to run AC-11 evidence');
test.setTimeout(300_000);
expect(process.env.PLAYWRIGHT_SKIP_WEB_SERVER).toBe('1');
const referenceOrigin = safeReferenceOrigin(referenceBaseUrl);
const referenceManifest = readReferenceManifest(process.env.CRANK_AC11_REFERENCE_ENV_JSON);
const yaml = openApiDocument(OPERATION_COUNT);
expect(yaml.byteLength).toBeLessThan(MAX_UPLOAD_BYTES);
await login(page);
await page.waitForFunction(() => Boolean(document.querySelector('[data-testid="onboarding-dismiss"]')));
await page.locator('[data-testid="onboarding-dismiss"]').click();
await expect(page.locator('#crank-onboarding-panel')).toBeHidden();
await page.evaluate(() => window.setLang('en'));
await page.getByRole('button', { name: localized('Import OpenAPI', 'Импорт OpenAPI') }).click();
await expect(page.locator('#openapi-import-modal')).toBeVisible();
await page.locator('#openapi-import-file').setInputFiles({
name: 'reference-preview-2000.yaml',
mimeType: 'application/yaml',
buffer: yaml,
});
const previewButton = page.locator('#openapi-import-preview');
const createButton = page.locator('#openapi-import-create');
const rows = page.locator('.openapi-import-operation');
const samplesMs = [];
let browserEnvironment;
// Warm the preview once before timing. Warm-up still uses the real remote
// preview API.
const warmupResponsePromise = page.waitForResponse((response) =>
response.request().method() === 'POST'
&& response.url().includes('/imports/openapi/preview'),
);
await previewButton.click();
const warmupResponse = await warmupResponsePromise;
expect(warmupResponse.status(), 'warm-up preview status').toBe(200);
expect(new URL(warmupResponse.url()).origin).toBe(referenceOrigin);
await expect(page.locator('[data-testid="openapi-import-page-status"]')).toContainText('2000');
await expect(createButton).toBeEnabled();
const previewResponses = [];
page.on('response', (response) => {
if (response.request().method() === 'POST' && response.url().includes('/imports/openapi/preview')) {
previewResponses.push({ status: response.status(), url: response.url() });
}
});
for (let sampleIndex = 0; sampleIndex < SAMPLE_COUNT; sampleIndex += 1) {
await expect(previewButton).toBeEnabled();
await page.evaluate((index) => {
const measurements = window.__ac11Measurements || (window.__ac11Measurements = []);
const sample = { start: null, elapsed: null };
measurements[index] = sample;
const ready = () => {
const panel = document.querySelector('#openapi-import-preview-panel');
const button = document.querySelector('#openapi-import-preview');
const status = document.querySelector('[data-testid="openapi-import-page-status"]');
const create = document.querySelector('#openapi-import-create');
const rowCount = document.querySelectorAll('.openapi-import-operation').length;
if (sample.start === null || !button || button.disabled || !panel || panel.hidden
|| !status || !status.textContent.includes('2000') || !create || create.disabled
|| rowCount < 1 || rowCount > 100) return;
sample.elapsed = Number((performance.now() - sample.start).toFixed(3));
observer.disconnect();
};
const observer = new MutationObserver(ready);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, characterData: true });
document.addEventListener('click', (event) => {
if (event.target.closest('#openapi-import-preview')) sample.start = performance.now();
}, { capture: true, once: true });
}, sampleIndex);
const responsePromise = page.waitForResponse((response) =>
response.request().method() === 'POST'
&& response.url().includes('/imports/openapi/preview'),
);
await previewButton.click();
const previewResponse = await responsePromise;
expect(previewResponse.status(), `preview sample ${sampleIndex + 1} status`).toBe(200);
expect(new URL(previewResponse.url()).origin).toBe(referenceOrigin);
await page.waitForFunction((index) => {
const sample = window.__ac11Measurements && window.__ac11Measurements[index];
return sample && sample.elapsed !== null;
}, sampleIndex, { polling: 'raf' });
const elapsed = await page.evaluate((index) => window.__ac11Measurements[index].elapsed, sampleIndex);
await expect(page.locator('#openapi-import-preview-panel')).toBeVisible();
await expect(page.locator('[data-testid="openapi-import-page-status"]')).toContainText('2000');
await expect(rows.first()).toBeVisible();
expect(await rows.count()).toBeGreaterThan(0);
expect(await rows.count()).toBeLessThanOrEqual(100);
await expect(createButton).toBeEnabled();
samplesMs.push(elapsed);
browserEnvironment = await page.evaluate(() => ({
user_agent: navigator.userAgent,
platform: navigator.platform,
language: navigator.language,
hardware_concurrency: navigator.hardwareConcurrency || null,
viewport: { width: innerWidth, height: innerHeight },
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
}));
}
expect(previewResponses).toHaveLength(SAMPLE_COUNT);
expect(previewResponses.every((item) => item.status === 200)).toBe(true);
const p95Ms = percentile95(samplesMs);
const report = {
benchmark: 'story-2.12-ac-11-reference-preview',
generated_at: new Date().toISOString(),
result: { sample_count: samplesMs.length, p95_ms: p95Ms, threshold_ms: MAX_P95_MS },
workload: {
operations: OPERATION_COUNT,
yaml_bytes: yaml.byteLength,
max_yaml_bytes: MAX_UPLOAD_BYTES,
preview_http_statuses: previewResponses.map((item) => item.status),
rendered_operation_rows: await rows.count(),
max_rendered_operation_rows: 100,
enabled_interaction: 'Create button enabled after preview',
interaction_enabled: true,
measurement_scope: 'click through real HTTP request, parsing, populated first page, and enabled Create button',
},
samples_ms: samplesMs,
environment: {
reference_origin: referenceOrigin,
playwright_base_url_configured: true,
playwright_skip_web_server: process.env.PLAYWRIGHT_SKIP_WEB_SERVER === '1',
node_version: process.version,
os_platform: process.platform,
os_arch: process.arch,
browser: browserEnvironment,
reference_manifest: referenceManifest,
},
};
const rawReport = JSON.stringify(report, null, 2);
await testInfo.attach('ac11-reference-preview-report.json', {
body: Buffer.from(rawReport),
contentType: 'application/json',
});
console.log('AC11_REFERENCE_PREVIEW_REPORT ' + JSON.stringify(report));
expect(p95Ms, 'Reference preview p95 must be at most 2 seconds').toBeLessThanOrEqual(MAX_P95_MS);
});