feat(import): complete OpenAPI onboarding and verify delivery
This commit is contained in:
1 parent
bb3e376767
commit
e93de7015a
244 files changed
+57020
-5139
No files matched your search
@@ -0,0 +1,273 @@
|
||||
// Operator-run AC-11 evidence against a Reference Deployment. This spec is
|
||||
// intentionally separate from the local/mocked Playwright matrix.
|
||||
const fs = require('node:fs');
|
||||
const { test, expect } = require('@playwright/test');
|
||||
const { localized, login } = require('./helpers');
|
||||
|
||||
// Remote operator credentials must not enter retry artifacts or failure captures.
|
||||
test.use({ trace: 'off', video: 'off', screenshot: 'off' });
|
||||
test.describe.configure({ retries: 0 });
|
||||
|
||||
const referenceBaseUrl = process.env.PLAYWRIGHT_BASE_URL;
|
||||
const SAMPLE_COUNT = 20;
|
||||
const OPERATION_COUNT = 2000;
|
||||
const MAX_UPLOAD_BYTES = 256 * 1024;
|
||||
const MAX_P95_MS = 2000;
|
||||
|
||||
function openApiDocument(operationCount) {
|
||||
const lines = [
|
||||
'openapi: 3.0.3',
|
||||
'info:',
|
||||
' title: Crank Reference Preview Benchmark',
|
||||
' version: "1.0.0"',
|
||||
'servers:',
|
||||
' - url: https://api.example.test',
|
||||
'paths:',
|
||||
];
|
||||
|
||||
for (let index = 0; index < operationCount; index += 1) {
|
||||
lines.push(` /op-${index}:`);
|
||||
lines.push(' get:');
|
||||
lines.push(' responses:');
|
||||
lines.push(' "200":');
|
||||
lines.push(' description: OK');
|
||||
}
|
||||
|
||||
return Buffer.from(lines.join('\n') + '\n', 'utf8');
|
||||
}
|
||||
|
||||
function percentile95(samples) {
|
||||
const sorted = [...samples].sort((left, right) => left - right);
|
||||
return sorted[Math.ceil(sorted.length * 0.95) - 1];
|
||||
}
|
||||
|
||||
function safeReferenceOrigin(rawUrl) {
|
||||
const url = new URL(rawUrl);
|
||||
if (!['http:', 'https:'].includes(url.protocol)) {
|
||||
throw new Error('PLAYWRIGHT_BASE_URL must use http or https');
|
||||
}
|
||||
if (url.username || url.password || url.search || url.hash) {
|
||||
throw new Error('PLAYWRIGHT_BASE_URL must not contain credentials, query parameters, or fragments');
|
||||
}
|
||||
if (['localhost', '127.0.0.1', '::1'].includes(url.hostname)) {
|
||||
throw new Error('AC-11 Reference benchmark requires a remote PLAYWRIGHT_BASE_URL');
|
||||
}
|
||||
return url.origin;
|
||||
}
|
||||
|
||||
function readReferenceManifest(manifestPath) {
|
||||
if (!manifestPath) {
|
||||
throw new Error('CRANK_AC11_REFERENCE_ENV_JSON is required when PLAYWRIGHT_BASE_URL is set');
|
||||
}
|
||||
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||
} catch (_error) {
|
||||
throw new Error('CRANK_AC11_REFERENCE_ENV_JSON must point to a readable valid JSON manifest');
|
||||
}
|
||||
|
||||
const deployment = manifest && manifest.deployment;
|
||||
const loadGenerator = manifest && manifest.load_generator;
|
||||
const browser = manifest && manifest.browser;
|
||||
const nonempty = (value) => typeof value === 'string' && value.trim().length > 0;
|
||||
const finiteNumber = (value) => typeof value === 'number' && Number.isFinite(value);
|
||||
const versionHasMajor = (value, major) => nonempty(value) && new RegExp(`^${major}(?:\\.|$)`).test(value.trim());
|
||||
|
||||
if (!deployment || !loadGenerator || !browser) {
|
||||
throw new Error('Reference manifest requires deployment, load_generator, and browser objects');
|
||||
}
|
||||
for (const [name, host] of [['deployment', deployment], ['load_generator', loadGenerator], ['browser', browser]]) {
|
||||
if (!nonempty(host.host_id)) throw new Error(`Reference manifest ${name}.host_id is required`);
|
||||
}
|
||||
if (deployment.host_id.trim() === loadGenerator.host_id.trim()
|
||||
|| deployment.host_id.trim() === browser.host_id.trim()) {
|
||||
throw new Error('Reference manifest requires deployment and load generator/browser on separate hosts');
|
||||
}
|
||||
if (!nonempty(deployment.cpu_model) || !nonempty(deployment.storage_type)
|
||||
|| deployment.storage_type.trim().toLowerCase() !== 'ssd'
|
||||
|| !finiteNumber(deployment.cpu_cores)
|
||||
|| !finiteNumber(deployment.ram_gib) || !finiteNumber(deployment.ssd_iops)
|
||||
|| !finiteNumber(deployment.ssd_throughput_mb_s) || !finiteNumber(deployment.ssd_capacity_gb)
|
||||
|| !finiteNumber(deployment.lan_mbps)
|
||||
|| !versionHasMajor(deployment.postgres_version, 16)
|
||||
|| !versionHasMajor(deployment.valkey_version, 8)) {
|
||||
throw new Error('Reference manifest is missing valid CPU, RAM, database, SSD, or LAN fields');
|
||||
}
|
||||
if (deployment.cpu_cores !== 4 || deployment.ram_gib !== 8
|
||||
|| deployment.ssd_capacity_gb < 100 || deployment.ssd_iops < 3000
|
||||
|| deployment.ssd_throughput_mb_s < 100
|
||||
|| deployment.lan_mbps < 1000) {
|
||||
throw new Error('Reference deployment must meet 4 CPU / 8 GiB RAM, SSD 3000 IOPS / 100 MB/s, and 1 Gbit LAN');
|
||||
}
|
||||
if (!finiteNumber(loadGenerator.cpu_cores) || !finiteNumber(loadGenerator.ram_gib)
|
||||
|| loadGenerator.cpu_cores !== 4 || loadGenerator.ram_gib !== 8) {
|
||||
throw new Error('Reference load_generator must have 4 CPU and 8 GiB RAM');
|
||||
}
|
||||
|
||||
// Emit only the fields needed for benchmark interpretation. Do not copy
|
||||
// arbitrary manifest keys (which could contain credentials) into reports.
|
||||
return {
|
||||
provenance: 'operator_manifest',
|
||||
deployment: {
|
||||
cpu_model: deployment.cpu_model.trim(),
|
||||
cpu_cores: deployment.cpu_cores,
|
||||
ram_gib: deployment.ram_gib,
|
||||
postgres_version: deployment.postgres_version.trim(),
|
||||
valkey_version: deployment.valkey_version.trim(),
|
||||
storage_type: 'SSD',
|
||||
ssd_capacity_gb: deployment.ssd_capacity_gb,
|
||||
ssd_iops: deployment.ssd_iops,
|
||||
ssd_throughput_mb_s: deployment.ssd_throughput_mb_s,
|
||||
lan_mbps: deployment.lan_mbps,
|
||||
},
|
||||
load_generator: { cpu_cores: loadGenerator.cpu_cores, ram_gib: loadGenerator.ram_gib },
|
||||
browser_host_separate_from_deployment: true,
|
||||
};
|
||||
}
|
||||
|
||||
test('[P1][reference] AC-11 measures 20 real 2000-operation preview round trips', async ({ page }, testInfo) => {
|
||||
test.skip(!referenceBaseUrl, 'Set PLAYWRIGHT_BASE_URL to a Reference Deployment to run AC-11 evidence');
|
||||
test.setTimeout(300_000);
|
||||
|
||||
expect(process.env.PLAYWRIGHT_SKIP_WEB_SERVER).toBe('1');
|
||||
const referenceOrigin = safeReferenceOrigin(referenceBaseUrl);
|
||||
const referenceManifest = readReferenceManifest(process.env.CRANK_AC11_REFERENCE_ENV_JSON);
|
||||
const yaml = openApiDocument(OPERATION_COUNT);
|
||||
expect(yaml.byteLength).toBeLessThan(MAX_UPLOAD_BYTES);
|
||||
|
||||
await login(page);
|
||||
await page.waitForFunction(() => Boolean(document.querySelector('[data-testid="onboarding-dismiss"]')));
|
||||
await page.locator('[data-testid="onboarding-dismiss"]').click();
|
||||
await expect(page.locator('#crank-onboarding-panel')).toBeHidden();
|
||||
await page.evaluate(() => window.setLang('en'));
|
||||
await page.getByRole('button', { name: localized('Import OpenAPI', 'Импорт OpenAPI') }).click();
|
||||
await expect(page.locator('#openapi-import-modal')).toBeVisible();
|
||||
await page.locator('#openapi-import-file').setInputFiles({
|
||||
name: 'reference-preview-2000.yaml',
|
||||
mimeType: 'application/yaml',
|
||||
buffer: yaml,
|
||||
});
|
||||
|
||||
const previewButton = page.locator('#openapi-import-preview');
|
||||
const createButton = page.locator('#openapi-import-create');
|
||||
const rows = page.locator('.openapi-import-operation');
|
||||
const samplesMs = [];
|
||||
let browserEnvironment;
|
||||
|
||||
// Warm the preview once before timing. Warm-up still uses the real remote
|
||||
// preview API.
|
||||
const warmupResponsePromise = page.waitForResponse((response) =>
|
||||
response.request().method() === 'POST'
|
||||
&& response.url().includes('/imports/openapi/preview'),
|
||||
);
|
||||
await previewButton.click();
|
||||
const warmupResponse = await warmupResponsePromise;
|
||||
expect(warmupResponse.status(), 'warm-up preview status').toBe(200);
|
||||
expect(new URL(warmupResponse.url()).origin).toBe(referenceOrigin);
|
||||
await expect(page.locator('[data-testid="openapi-import-page-status"]')).toContainText('2000');
|
||||
await expect(createButton).toBeEnabled();
|
||||
|
||||
const previewResponses = [];
|
||||
page.on('response', (response) => {
|
||||
if (response.request().method() === 'POST' && response.url().includes('/imports/openapi/preview')) {
|
||||
previewResponses.push({ status: response.status(), url: response.url() });
|
||||
}
|
||||
});
|
||||
|
||||
for (let sampleIndex = 0; sampleIndex < SAMPLE_COUNT; sampleIndex += 1) {
|
||||
await expect(previewButton).toBeEnabled();
|
||||
await page.evaluate((index) => {
|
||||
const measurements = window.__ac11Measurements || (window.__ac11Measurements = []);
|
||||
const sample = { start: null, elapsed: null };
|
||||
measurements[index] = sample;
|
||||
const ready = () => {
|
||||
const panel = document.querySelector('#openapi-import-preview-panel');
|
||||
const button = document.querySelector('#openapi-import-preview');
|
||||
const status = document.querySelector('[data-testid="openapi-import-page-status"]');
|
||||
const create = document.querySelector('#openapi-import-create');
|
||||
const rowCount = document.querySelectorAll('.openapi-import-operation').length;
|
||||
if (sample.start === null || !button || button.disabled || !panel || panel.hidden
|
||||
|| !status || !status.textContent.includes('2000') || !create || create.disabled
|
||||
|| rowCount < 1 || rowCount > 100) return;
|
||||
sample.elapsed = Number((performance.now() - sample.start).toFixed(3));
|
||||
observer.disconnect();
|
||||
};
|
||||
const observer = new MutationObserver(ready);
|
||||
observer.observe(document.body, { childList: true, subtree: true, attributes: true, characterData: true });
|
||||
document.addEventListener('click', (event) => {
|
||||
if (event.target.closest('#openapi-import-preview')) sample.start = performance.now();
|
||||
}, { capture: true, once: true });
|
||||
}, sampleIndex);
|
||||
const responsePromise = page.waitForResponse((response) =>
|
||||
response.request().method() === 'POST'
|
||||
&& response.url().includes('/imports/openapi/preview'),
|
||||
);
|
||||
await previewButton.click();
|
||||
const previewResponse = await responsePromise;
|
||||
expect(previewResponse.status(), `preview sample ${sampleIndex + 1} status`).toBe(200);
|
||||
expect(new URL(previewResponse.url()).origin).toBe(referenceOrigin);
|
||||
|
||||
await page.waitForFunction((index) => {
|
||||
const sample = window.__ac11Measurements && window.__ac11Measurements[index];
|
||||
return sample && sample.elapsed !== null;
|
||||
}, sampleIndex, { polling: 'raf' });
|
||||
const elapsed = await page.evaluate((index) => window.__ac11Measurements[index].elapsed, sampleIndex);
|
||||
|
||||
await expect(page.locator('#openapi-import-preview-panel')).toBeVisible();
|
||||
await expect(page.locator('[data-testid="openapi-import-page-status"]')).toContainText('2000');
|
||||
await expect(rows.first()).toBeVisible();
|
||||
expect(await rows.count()).toBeGreaterThan(0);
|
||||
expect(await rows.count()).toBeLessThanOrEqual(100);
|
||||
await expect(createButton).toBeEnabled();
|
||||
|
||||
samplesMs.push(elapsed);
|
||||
browserEnvironment = await page.evaluate(() => ({
|
||||
user_agent: navigator.userAgent,
|
||||
platform: navigator.platform,
|
||||
language: navigator.language,
|
||||
hardware_concurrency: navigator.hardwareConcurrency || null,
|
||||
viewport: { width: innerWidth, height: innerHeight },
|
||||
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
||||
}));
|
||||
}
|
||||
|
||||
expect(previewResponses).toHaveLength(SAMPLE_COUNT);
|
||||
expect(previewResponses.every((item) => item.status === 200)).toBe(true);
|
||||
const p95Ms = percentile95(samplesMs);
|
||||
const report = {
|
||||
benchmark: 'story-2.12-ac-11-reference-preview',
|
||||
generated_at: new Date().toISOString(),
|
||||
result: { sample_count: samplesMs.length, p95_ms: p95Ms, threshold_ms: MAX_P95_MS },
|
||||
workload: {
|
||||
operations: OPERATION_COUNT,
|
||||
yaml_bytes: yaml.byteLength,
|
||||
max_yaml_bytes: MAX_UPLOAD_BYTES,
|
||||
preview_http_statuses: previewResponses.map((item) => item.status),
|
||||
rendered_operation_rows: await rows.count(),
|
||||
max_rendered_operation_rows: 100,
|
||||
enabled_interaction: 'Create button enabled after preview',
|
||||
interaction_enabled: true,
|
||||
measurement_scope: 'click through real HTTP request, parsing, populated first page, and enabled Create button',
|
||||
},
|
||||
samples_ms: samplesMs,
|
||||
environment: {
|
||||
reference_origin: referenceOrigin,
|
||||
playwright_base_url_configured: true,
|
||||
playwright_skip_web_server: process.env.PLAYWRIGHT_SKIP_WEB_SERVER === '1',
|
||||
node_version: process.version,
|
||||
os_platform: process.platform,
|
||||
os_arch: process.arch,
|
||||
browser: browserEnvironment,
|
||||
reference_manifest: referenceManifest,
|
||||
},
|
||||
};
|
||||
|
||||
const rawReport = JSON.stringify(report, null, 2);
|
||||
await testInfo.attach('ac11-reference-preview-report.json', {
|
||||
body: Buffer.from(rawReport),
|
||||
contentType: 'application/json',
|
||||
});
|
||||
console.log('AC11_REFERENCE_PREVIEW_REPORT ' + JSON.stringify(report));
|
||||
expect(p95Ms, 'Reference preview p95 must be at most 2 seconds').toBeLessThanOrEqual(MAX_P95_MS);
|
||||
});
|
||||
Reference in new issue
Block a user