diff --git a/.agents/skills/testing-workflow/SKILL.md b/.agents/skills/testing-workflow/SKILL.md index 402c289b1..b37177ab4 100644 --- a/.agents/skills/testing-workflow/SKILL.md +++ b/.agents/skills/testing-workflow/SKILL.md @@ -69,6 +69,16 @@ does not replace source validation. The release command commits matching root/TU The release workflow rejects version mismatches and runs the full profile before package installation checks. +## Performance coverage + +Use [CONTRIBUTING.md](../../../CONTRIBUTING.md#performance-checks) to classify the +change. Add `perf:full` for performance optimizations, changes to history/token +processing, storage, streaming or tool execution, and large runtime refactors. +The label automatically selects the full suite and keeps it selected for later +commits. Before merge, link a passing `performance` run with `Suite: full` for the +latest head and intended base. Read [Performance CI](../../../docs/performance-ci.md) +for evidence, limits and manual dispatch. + ## Manual evidence and reporting For CLI behavior, exercise the built `dist/cli.js` and inspect stdout, stderr and diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 868a4f99f..c2c5eccdc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,6 +7,7 @@ Describe the user-visible problem and resulting behavior. Link a public issue wh ## Validation - Checks run and results (include the revision/profile where relevant): +- Performance: basic / `perf:full` (see [requirements](https://github.com/MiniMax-AI/minimax-code/blob/main/CONTRIBUTING.md#performance-checks)); for full coverage, link a passing run for the latest PR head and intended base: - NOT RUN, platform limitations and live-service boundaries: ## Publication and contribution checks diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000..c24f2c25e --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,86 @@ +name: Performance +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + workflow_dispatch: + inputs: + base: + description: Baseline commit or branch + required: true + default: main + type: string + suite: + description: Performance suite (full includes startup and long history) + required: true + default: full + type: choice + options: [full, basic] +permissions: + contents: read +jobs: + performance: + # Unrelated label events must not replace the actual performance check. + name: ${{ contains(fromJSON('["labeled","unlabeled"]'), github.event.action) && github.event.label.name != 'perf:full' && 'performance (label ignored)' || 'performance' }} + if: ${{ !contains(fromJSON('["labeled","unlabeled"]'), github.event.action) || github.event.label.name == 'perf:full' }} + strategy: + matrix: + suite: ${{ fromJSON((inputs.suite == 'full' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'perf:full'))) && '["full"]' || '["basic"]') }} + concurrency: + group: performance-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && matrix.suite || 'auto' }} + cancel-in-progress: true + runs-on: macos-15 + timeout-minutes: ${{ matrix.suite == 'full' && 45 || 15 }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + path: candidate + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + path: baseline + ref: ${{ github.event.pull_request.base.sha || inputs.base }} + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: KonghaYao/harness-perf-benchmark + ref: 8ad1e09f7653d7e97b83b4f5b4d347ca2fa129c7 + path: benchmark + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 9.12.0 + package_json_file: candidate/package.json + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.23.2 + package-manager-cache: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.2 + - name: Build baseline + working-directory: baseline + run: | + pnpm install --frozen-lockfile + pnpm build + - name: Build candidate + working-directory: candidate + run: | + pnpm install --frozen-lockfile + pnpm build + - name: Install pinned benchmark + working-directory: benchmark + run: bun install --frozen-lockfile + - name: Compare on this runner + env: + PERF_SUITE: ${{ matrix.suite }} + run: node candidate/scripts/perf/run.mjs --base baseline --head candidate --benchmark benchmark --suite "$PERF_SUITE" --out "$RUNNER_TEMP/performance" + - name: Upload performance evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: performance-${{ matrix.suite }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/performance/ + include-hidden-files: true + retention-days: 14 + if-no-files-found: warn diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e3531287f..2505d6e43 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,6 +55,33 @@ Test files are declared in `test/vitest-suites.json`, grouped by the gate that r Review added or removed files before running `node scripts/source-inventory.mjs --write`. Updating the inventory must not bypass checks for private protocols, internal addresses, credentials, or third-party licensing. Before a source release, also scan the complete Git history with Gitleaks; see the [release process](docs/releasing.md). +## Performance checks + +Every PR runs the pinned 100-round upstream benchmark. Add the `perf:full` label +for the changes below; it automatically selects startup, 100-round and 300-round +long-history tests instead. Keep the label until merge so new commits run the +full suite too. + +| Change | Required performance coverage | +| --- | --- | +| Performance optimization, including CPU, memory, startup or throughput fixes | `perf:full` | +| History construction/serialization, token estimation/caching, session storage, streaming or the tool execution loop | `perf:full` | +| Large runtime refactors spanning multiple components, or runtime dependency changes affecting these paths | `perf:full` | +| Other changes, including documentation-only edits and mechanical changes outside runtime paths | Automatic 100-round basic check | + +The author adds the label; reviewers verify the classification. File count alone +does not make a change a large runtime refactor. Before merging a labeled PR, +require a successful `performance` check with `Suite: full` for the latest PR +head and intended base, and link the run in the PR's Validation section. Failed, +cancelled or inconclusive runs do not satisfy this requirement. If the base +changes after measurement, rerun against the updated base. + +Removing `perf:full` restores the basic suite; do not remove it to bypass a +failure. Unrelated label changes do not run benchmarks. This is a review rule; +repository administrators must configure required checks separately. See +[Performance CI](docs/performance-ci.md) for manual dispatch, runner configuration +and raw evidence. + ## Capability boundaries Preserve MiniMax OAuth, Token Plan, BYOK, mcode-tools, search, plugins, connectors, updates, and feedback. Do not solve standalone-build problems by removing capabilities. Managed services use public clients; internal HTTP services, generated IDL, and cloud executor implementations are outside this repository. diff --git a/docs/performance-ci.md b/docs/performance-ci.md new file mode 100644 index 000000000..224d316a1 --- /dev/null +++ b/docs/performance-ci.md @@ -0,0 +1,82 @@ +# Performance CI + +Every PR runs the `performance` check: the original 100-round scenario by default. +Adding `perf:full` automatically runs the full suite. While the label remains, +new commits and reopened PRs run the full suite too. Removing it starts a basic +run and cancels any previous automatic run for that PR. Unrelated label changes +skip benchmarking and do not replace the active `performance` check. Both +revisions build separately, then run serially on one `macos-15` runner with Node +22.23.2 and Bun 1.4.2. Build and installation time are excluded. + +| Mode | Trigger | Scenarios | Job timeout | +| --- | --- | --- | --- | +| `basic` | PR without `perf:full`; optional manual run | `upstream-100` | 15 minutes | +| `full` | Add `perf:full`, then new commits/reopens while labeled; optional manual run | Startup, `upstream-100`, `history-300` | 45 minutes | + +| Scenario | Input | Purpose | +| --- | --- | --- | +| `startup` | 1 tool round, minimal body | Startup and shutdown | +| `upstream-100` | 100 rounds × approximately 4 KiB, 64-character chunks, zero delay | Original [harness-perf-benchmark](https://github.com/KonghaYao/harness-perf-benchmark) workload | +| `history-300` | 300 rounds × approximately 8 KiB | History growth beyond the 1,048,576 UTF-16-unit token cache budget | + +The upstream commit is pinned in `scripts/perf/config.json` and the workflow. +Its generator, mock, sampler and CPU accounting run without source edits. +An observing loopback proxy forwards the request body and response stream to +the mock, validating the complete assistant/tool history on every request. +It records request bytes and validation duration in `requests.jsonl`, without +headers. This adds observer latency to wall time; it is identical on both sides +and outside the sampled CLI process tree. Absolute timings differ from running +the upstream demo without the observer. +The adapter uses synthetic workspaces and the repository's offline network +fixture. No live-model key is needed. The long scenario advertises a +2,000,000-token mock context to prevent compaction from shortening the history; +this is a stress fixture, not production model capacity. The standard scenario +keeps upstream provider defaults. The suite does not measure live-model quality, +latency, compaction acceptance or other products' rankings. + +| Rule | Initial setting | +| --- | --- | +| Repetitions | One excluded warmup per revision/scenario; three measured pairs, alternating base/head order | +| Duration regression | Median increases by more than both 25% and 1 second | +| CPU regression | Median increases by more than both 20% and 0.5 core-seconds | +| Sampled peak RSS regression | Median increases by more than both 20% and 32 MiB | +| Pair confirmation | Candidate is worse in all three matched pairs | +| Noise | Range exceeds 30% of median on either revision: `INCONCLUSIVE`, exit 2 | +| Correctness | Exit 0; N+1 requests with complete wire history; N matched bash commands/results; expected echo, pwd and listing output; complete stored bodies; expected final response | +| Invalid evidence | Missing samples, wrong sampler, bad metrics, incomplete runs or changed fixture: fail | + +These are initial regression budgets, not service-level targets or statistical +confidence intervals. `REGRESSION` exits 1; `PASS` exits 0. An inconclusive run +needs another clean measurement; it is neither a confirmed regression nor a pass. +Tune budgets from repeated same-revision measurements. Review changes to +scenarios, comparison logic or limits as changes to the performance contract. +The workflow does not change branch protection; maintainers can add +`performance` as a required check after runner calibration; its name stays the +same for basic and full runs. The summary and artifact identify the selected +suite. Follow the [label and review requirements](../CONTRIBUTING.md#performance-checks) +when deciding which PRs need full coverage. + +The Job Summary contains machine/method and comparison tables. The 14-day +artifact includes JSON results, configuration, commit and fixture hashes, +warmup/measured samples, CLI/mock logs, and synthetic histories. Confirmed +regressions get a separate candidate CPU-profile run, excluded from comparison. +Reports checkpoint after each run; setup failures before the runner starts +remain in job logs. CPU and sampled RSS are compared separately; the combined +CU score is not a gate. Sampling can miss brief memory peaks. + +Install and build both checkouts with identical Node/pnpm versions and install +the pinned benchmark with `bun install --frozen-lockfile`, then run: + +```bash +node scripts/perf/run.mjs --base /path/to/base --head /path/to/head \ + --benchmark /path/to/pinned-benchmark --out /tmp/new-performance-output +``` + +The output directory must be new. The CLI defaults to `--suite basic`; pass +`--suite full` explicitly for the complete suite or `--scenario startup` for a +focused local check. Once the workflow is merged into the default branch, open +**Actions → Performance → Run workflow**, choose the PR branch, set the baseline +ref, and select `full` for long-history testing. Manual runs use a separate +concurrency group and do not cancel automatic PR checks. The workflow uses read-only +permissions and `pull_request`, without comments or secrets. See GitHub's +[event security guidance](https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target). diff --git a/release/public-source.json b/release/public-source.json index 6331c59ad..0045632f6 100644 --- a/release/public-source.json +++ b/release/public-source.json @@ -21,6 +21,7 @@ ".github/workflows/cli-release.yml", ".github/workflows/compatibility.yml", ".github/workflows/label-issue-product.yml", + ".github/workflows/performance.yml", ".github/workflows/security.yml", ".github/workflows/source-candidate.yml", ".github/workflows/sync-issue-to-feishu.yml", @@ -53,6 +54,7 @@ "docs/installation.md", "docs/maintainers.md", "docs/open-source-status.md", + "docs/performance-ci.md", "docs/publication-authorization.md", "docs/release-audit.md", "docs/releasing.md", @@ -3291,6 +3293,10 @@ "scripts/lib/tui-package-privacy.mjs", "scripts/lib/vitest-suites.mjs", "scripts/package-cli-release.mjs", + "scripts/perf/config.json", + "scripts/perf/driver.ts", + "scripts/perf/report.mjs", + "scripts/perf/run.mjs", "scripts/prepare-source-sync.mjs", "scripts/publish-cli-release.mjs", "scripts/release-cli.mjs", diff --git a/scripts/perf/config.json b/scripts/perf/config.json new file mode 100644 index 000000000..7fa2192e7 --- /dev/null +++ b/scripts/perf/config.json @@ -0,0 +1,51 @@ +{ + "benchmarkRepository": "https://github.com/KonghaYao/harness-perf-benchmark", + "benchmarkRevision": "8ad1e09f7653d7e97b83b4f5b4d347ca2fa129c7", + "warmups": 1, + "repetitions": 3, + "intervalMs": 100, + "maxSpread": 0.3, + "suites": { + "basic": ["upstream-100"], + "full": ["startup", "upstream-100", "history-300"] + }, + "thresholds": { + "durationMs": { + "relative": 0.25, + "absolute": 1000 + }, + "cpuSeconds": { + "relative": 0.2, + "absolute": 0.5 + }, + "rssBytes": { + "relative": 0.2, + "absolute": 33554432 + } + }, + "scenarios": [ + { + "id": "startup", + "turns": 1, + "bodyKb": 0, + "chunkSize": 64, + "timeoutMs": 60000 + }, + { + "id": "upstream-100", + "turns": 100, + "bodyKb": 4, + "chunkSize": 64, + "timeoutMs": 180000 + }, + { + "id": "history-300", + "turns": 300, + "bodyKb": 8, + "chunkSize": 64, + "timeoutMs": 360000, + "contextWindow": 2000000, + "minimumTextUnits": 1048577 + } + ] +} diff --git a/scripts/perf/driver.ts b/scripts/perf/driver.ts new file mode 100644 index 000000000..06846f62f --- /dev/null +++ b/scripts/perf/driver.ts @@ -0,0 +1,81 @@ +// Run under Bun. Reuse the pinned upstream generator, mock and sampler unchanged. +import { readFileSync, mkdirSync, writeFileSync, readdirSync, appendFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { validateRun, validateRequest } from './report.mjs'; + +const input = JSON.parse(readFileSync(process.argv[2]!, 'utf8')); +const { runPerf } = await import(pathToFileURL(join(input.benchmark, 'scripts/perf/run.ts')).href); +const { loadPerfConfig } = await import(pathToFileURL(join(input.benchmark, 'scripts/perf/config.ts')).href); +const workspace = join(input.directory, 'workspace'); +const data = join(input.directory, 'data'); +mkdirSync(workspace, { recursive: true }); +mkdirSync(data, { recursive: true }); +writeFileSync(join(workspace, 'README.txt'), 'Synthetic performance fixture.\n'); +const server = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: () => new Response('') }); +const port = server.port; +server.stop(true); +const scenario = JSON.parse(readFileSync(input.script, 'utf8')); +let requests = 0; +let auditFailure: string | undefined; +const proxy = Bun.serve({ port: 0, hostname: '127.0.0.1', async fetch(request) { + const url = new URL(request.url); + const bytes = await request.arrayBuffer(); + if (url.pathname === '/v1/chat/completions' && request.method === 'POST') { + requests++; + const start = performance.now(); + let error: string | undefined; + try { validateRequest(JSON.parse(new TextDecoder().decode(bytes)), scenario.responses, requests, workspace, cfg.prompt); } + catch (failure) { error = String(failure); auditFailure ??= error; } + appendFileSync(join(input.directory, 'requests.jsonl'), JSON.stringify({ request: requests, + bytes: bytes.byteLength, validationMs: performance.now() - start, ok: !error, ...(error ? { error } : {}) }) + '\n'); + } + return fetch(`http://127.0.0.1:${port}${url.pathname}${url.search}`, { + method: request.method, headers: { 'content-type': request.headers.get('content-type') ?? 'application/json' }, + ...(request.method === 'GET' || request.method === 'HEAD' ? {} : { body: bytes }), + }); +} }); +writeFileSync(join(data, 'config.yaml'), [ + 'logLevel: info', 'custom_provider:', ' llm-mock:', ' name: llm-mock', + ' kind: custom', ' enabled: true', ' api: openai-completions', + ' options:', ' apiKey: mock-key', ` baseURL: http://127.0.0.1:${proxy.port}/v1`, + ' authMode: api-key', ' models:', ' llm-mock:', ' reasoning: true', + ' thinking_config:', ' mode: switchable', " default_value: 'true'", + ...(input.scenario.contextWindow ? [' limit:', ` context: ${input.scenario.contextWindow}`, ' output: 16384'] : []), '', +].join('\n')); +const cfg = loadPerfConfig([ + '--script', input.script, '--exhausted', 'stop', '--work-dir', workspace, + '--out-dir', join(input.directory, 'raw'), '--harness', 'minimax-code', + '--port', String(port), '--interval-ms', String(input.intervalMs), + '--timeout-ms', String(input.scenario.timeoutMs), '--sampler', 'rusage', + '--prompt', '压测：请持续用只读命令检查当前目录状态', +]); +let code: number; +try { code = await runPerf(cfg, { + harnessCommand: () => [input.node, + ...(input.profile ? ['--cpu-prof', `--cpu-prof-dir=${input.directory}`] : []), + '--import', pathToFileURL(join(input.control, 'test/network-deny.mjs')).href, + join(input.build, 'dist/cli.js'), 'exec', '--permission', 'off', '--cwd', workspace, + '--model', 'custom_provider:llm-mock/llm-mock', cfg.prompt], + harnessEnv: () => ({ MINIMAX_DATA_DIR: data, PI_TELEMETRY: '0', + MCODE_TEST_ALLOWED_ORIGIN: `http://127.0.0.1:${proxy.port}`, + MCODE_TEST_NETWORK_AUDIT: join(input.directory, 'network.log'), MCODE_TEST_MANAGED_OFFLINE: '1', + }), +}); } finally { proxy.stop(true); } +if (code !== 0) throw new Error(`Upstream runner failed: ${code}`); +if (auditFailure) throw new Error(`Request history validation failed: ${auditFailure}`); +if (requests !== input.scenario.turns + 1) throw new Error('Request audit count mismatch'); +function files(directory: string): string[] { + return readdirSync(directory, { withFileTypes: true }).flatMap(entry => { + const name = join(directory, entry.name); + return entry.isDirectory() ? files(name) : [name]; + }); +} +const metas = files(join(input.directory, 'raw')).filter(p => p.endsWith('/run.json')); +if (metas.length !== 1) throw new Error('Expected exactly one upstream run'); +const histories = files(data).filter(p => p.endsWith('/messages.jsonl')); +if (histories.length !== 1) throw new Error('Expected exactly one canonical session history'); +const meta = JSON.parse(readFileSync(metas[0]!, 'utf8')); +const messages = readFileSync(histories[0]!, 'utf8').trim().split('\n').map(line => JSON.parse(line).message); +const metrics = validateRun(meta, messages, input.scenario, scenario.responses[input.scenario.turns].message.content, scenario.responses, workspace); +writeFileSync(join(input.directory, 'result.json'), JSON.stringify({ metrics, meta: resolve(metas[0]!), history: resolve(histories[0]!) }, null, 2)); diff --git a/scripts/perf/report.mjs b/scripts/perf/report.mjs new file mode 100644 index 000000000..9c3a43510 --- /dev/null +++ b/scripts/perf/report.mjs @@ -0,0 +1,145 @@ +import assert from 'node:assert/strict'; +import { realpathSync } from 'node:fs'; + +const textContent = content => typeof content === 'string' ? content : Array.isArray(content) + ? content.filter(b => b.type === 'text').map(b => b.text).join('') : ''; + +export function selectScenarios(config, { suite = 'basic', scenario } = {}) { + assert.ok(Object.hasOwn(config.suites, suite), 'Unknown performance suite'); + const ids = scenario ? [scenario] : config.suites[suite]; + assert.ok(ids.length > 0 && new Set(ids).size === ids.length, 'Invalid scenario selection'); + return ids.map(id => { + const selected = config.scenarios.find(item => item.id === id); + assert.ok(selected, `Unknown scenario: ${id}`); + return selected; + }); +} + +export function validateToolOutput(content, command, workspace) { + const text = textContent(content).trim(); + if (command.startsWith('echo ')) assert.equal(text, command.slice(5), 'Echo result changed'); + else if (command === 'pwd') assert.equal(text, realpathSync(workspace), 'Working directory result changed'); + else if (command === 'ls' || command === 'ls -la') assert.match(text, /(?:^|\s)README\.txt(?:\s|$)/, 'Directory listing lost fixture'); + else throw new Error('Unknown benchmark command'); +} + +export function validateRequest(body, responses, requestNumber, workspace, prompt) { + const rounds = requestNumber - 1; + assert.ok(rounds >= 0 && rounds < responses.length - 1, 'Unexpected request'); + assert.ok(Array.isArray(body.messages), 'Request messages missing'); + assert.ok(typeof prompt === 'string' && prompt.length > 0, 'Expected task missing'); + const conversation = body.messages.filter(m => m.role !== 'system' && m.role !== 'developer'); + assert.equal(conversation.length, 1 + rounds * 2, 'Request conversation length changed'); + assert.equal(conversation[0]?.role, 'user', 'Initial user task missing'); + assert.ok(textContent(conversation[0].content).includes(prompt), 'Initial user task changed'); + for (let i = 0; i < rounds; i++) { + assert.equal(conversation[i * 2 + 1].role, 'assistant', 'Request conversation order changed'); + assert.equal(conversation[i * 2 + 2].role, 'tool', 'Request conversation order changed'); + } + const assistant = body.messages.filter(m => m.role === 'assistant'); + const tools = body.messages.filter(m => m.role === 'tool'); + assert.equal(assistant.length, rounds, 'Request lost assistant history'); + assert.equal(tools.length, rounds, 'Request lost tool history'); + for (let i = 0; i < rounds; i++) { + const expected = responses[i].message; + assert.equal(textContent(assistant[i].content), expected.content, 'Request history body changed'); + assert.equal(assistant[i].tool_calls?.length, 1, 'Request tool calls changed'); + const call = assistant[i].tool_calls[0]; + assert.equal(call.id, expected.tool_calls[0].id, 'Request call identity changed'); + assert.equal(call.function.name, 'bash', 'Request tool changed'); + const args = typeof call.function.arguments === 'string' ? JSON.parse(call.function.arguments) : call.function.arguments; + assert.deepEqual(args, expected.tool_calls[0].function.arguments, 'Request command changed'); + assert.equal(tools[i].tool_call_id, call.id, 'Request tool result identity changed'); + validateToolOutput(tools[i].content, args.command, workspace); + } +} + +export function median(values) { + assert.ok(values.length > 0 && values.every(Number.isFinite), 'Nonempty finite samples required'); + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +} + +export function validateRun(meta, messages, scenario, expectedFinal, expectedResponses, workspace) { + assert.equal(meta.schemaVersion, 1, 'Unknown upstream run schema'); + assert.equal(meta.status, 'ok', 'Benchmark did not complete'); + assert.equal(meta.exit?.code, 0, 'CLI failed'); + assert.equal(meta.mock?.requests, scenario.turns + 1, 'Unexpected model request count'); + assert.equal(meta.sampling?.backend, 'rusage', 'macOS rusage sampler is required'); + assert.equal(meta.sampling?.withTree, true, 'Process-tree sampling is required'); + assert.ok(meta.summary?.count >= 3, 'Insufficient samples'); + const calls = messages.flatMap(m => m.role === 'assistant' ? m.content.filter(b => b.type === 'toolCall') : []); + const assistant = messages.filter(m => m.role === 'assistant'); + const results = messages.filter(m => m.role === 'toolResult'); + assert.equal(calls.length, scenario.turns, 'Missing or extra tool calls'); + assert.equal(results.length, scenario.turns, 'Missing or extra tool results'); + const ids = new Set(); + for (let i = 0; i < scenario.turns; i++) { + assert.equal(calls[i].id, `call_long_run_${i + 1}`, 'Unexpected tool sequence'); + assert.equal(calls[i].name, 'bash', 'Unexpected tool'); + if (expectedResponses) { + assert.deepEqual(calls[i].arguments, expectedResponses[i].message.tool_calls[0].function.arguments, 'Tool command changed'); + assert.equal(assistant[i].content.filter(b => b.type === 'text').map(b => b.text).join(''), expectedResponses[i].message.content, 'History body changed'); + } + assert.equal(results[i].toolCallId, calls[i].id, 'Tool result does not match call'); + assert.equal(results[i].isError, false, 'Tool execution failed'); + if (expectedResponses && workspace) validateToolOutput(results[i].content, calls[i].arguments.command, workspace); + assert.ok(!ids.has(results[i].toolCallId), 'Duplicate tool result'); + ids.add(results[i].toolCallId); + } + const last = messages.filter(m => m.role === 'assistant').at(-1); + const final = last?.content.filter(b => b.type === 'text').map(b => b.text).join(''); + assert.equal(final, expectedFinal, 'Final response changed'); + const metrics = { + durationMs: meta.duration?.endToEndMs, + cpuSeconds: meta.cost?.cpuSeconds, + rssBytes: meta.peaks?.treeRssBytes, + }; + for (const [name, value] of Object.entries(metrics)) { + assert.ok(Number.isFinite(value) && value > 0, `Invalid ${name}`); + } + return metrics; +} + +export function compareRuns(base, head, config) { + assert.equal(base.length, config.repetitions, 'Incomplete baseline repetitions'); + assert.equal(head.length, config.repetitions, 'Incomplete candidate repetitions'); + return Object.entries(config.thresholds).map(([metric, threshold]) => { + const before = base.map(r => r[metric]); + const after = head.map(r => r[metric]); + assert.ok([...before, ...after].every(v => Number.isFinite(v) && v > 0), `Invalid ${metric}`); + const baseline = median(before), candidate = median(after); + const spread = values => (Math.max(...values) - Math.min(...values)) / median(values); + const unstable = spread(before) > config.maxSpread || spread(after) > config.maxSpread; + const delta = candidate - baseline; + const regression = delta > Math.max(baseline * threshold.relative, threshold.absolute) + && after.every((v, i) => v > before[i]); + return { metric, baseline, candidate, change: delta / baseline, + baseSpread: spread(before), headSpread: spread(after), + status: unstable ? 'INCONCLUSIVE' : regression ? 'REGRESSION' : 'PASS' }; + }); +} + +export function renderReport(report) { + const lines = ['# Performance comparison', '', + `Status: **${report.status}**`, '', + '| Item | Value |', '| --- | --- |', + `| Base | \`${report.baseRevision}\` |`, `| Candidate | \`${report.headRevision}\` |`, + `| Benchmark | [${report.benchmarkRevision}](${report.benchmarkRepository}/tree/${report.benchmarkRevision}) |`, + `| Machine | ${report.host.cpuModel}; ${report.host.cpus} CPUs; ${Math.round(report.host.totalMemBytes / 2 ** 30)} GiB; ${report.host.platform} ${report.host.osRelease}; ${report.host.arch} |`, + `| Runtime | Node ${report.nodeVersion}; Bun ${report.bunVersion} |`, + `| Suite | ${report.suite}; ${report.selectedScenarios.join(', ')} |`, + `| Method | One warmup per revision/scenario; three alternating serial pairs; 100 ms process-tree rusage sampling |`, + '', '| Scenario | Metric | Base median | Candidate median | Change | Result |', + '| --- | --- | ---: | ---: | ---: | --- |']; + for (const scenario of report.results) for (const row of scenario.comparison ?? []) { + const scale = row.metric === 'rssBytes' ? 2 ** 20 : row.metric === 'durationMs' ? 1000 : 1; + const unit = row.metric === 'rssBytes' ? 'MiB' : row.metric === 'durationMs' ? 's' : 'core-s'; + lines.push(`| ${scenario.id} | ${row.metric} | ${(row.baseline / scale).toFixed(2)} ${unit} | ${(row.candidate / scale).toFixed(2)} ${unit} | ${(row.change * 100).toFixed(1)}% | ${row.status} |`); + } + if (report.error) lines.push('', `Failure: ${String(report.error).replaceAll('\n', ' ').replaceAll('`', "'")}`); + lines.push('', 'Raw run.json, samples.csv, CLI/mock logs, synthetic histories and any diagnostic CPU profile are attached. Warmups and diagnostic runs are excluded from comparisons.', + '', 'This is a mock throughput check, not live-model acceptance. RSS is sampled. INCONCLUSIVE means excessive spread and fails the check; do not treat it as evidence of a regression or a pass.'); + return lines.join('\n') + '\n'; +} diff --git a/scripts/perf/run.mjs b/scripts/perf/run.mjs new file mode 100644 index 000000000..72d1529bd --- /dev/null +++ b/scripts/perf/run.mjs @@ -0,0 +1,110 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, appendFileSync, existsSync } from 'node:fs'; +import { cpus, totalmem, platform, release, arch } from 'node:os'; +import { resolve, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; +import { createHash } from 'node:crypto'; +import { compareRuns, renderReport, selectScenarios } from './report.mjs'; + +const control = fileURLToPath(new URL('../../', import.meta.url)); +const config = JSON.parse(readFileSync(new URL('./config.json', import.meta.url), 'utf8')); +const { values } = parseArgs({ options: { + base: { type: 'string' }, head: { type: 'string' }, benchmark: { type: 'string' }, + out: { type: 'string' }, scenario: { type: 'string' }, + suite: { type: 'string', default: 'basic' }, +}, strict: true }); +for (const name of ['base', 'head', 'benchmark', 'out']) { + if (!values[name]) throw new Error(`Missing --${name}`); +} +const base = resolve(values.base), head = resolve(values.head); +const benchmark = resolve(values.benchmark), out = resolve(values.out); +if (existsSync(out)) throw new Error('Output directory must be new; old results cannot be reused'); +if (platform() !== 'darwin') throw new Error('This benchmark requires macOS process-tree rusage'); +const git = (directory, ...args) => execFileSync('git', ['-C', directory, ...args], { encoding: 'utf8' }).trim(); +if (git(benchmark, 'rev-parse', 'HEAD') !== config.benchmarkRevision) throw new Error('Wrong benchmark revision'); +if (git(benchmark, 'status', '--porcelain', '--untracked-files=no')) throw new Error('Benchmark has tracked edits'); +const bun = execFileSync('which', ['bun'], { encoding: 'utf8' }).trim(); +const scenarios = selectScenarios(config, values); +mkdirSync(out, { recursive: true }); +const report = { + status: 'RUNNING', baseRevision: git(base, 'rev-parse', 'HEAD'), headRevision: git(head, 'rev-parse', 'HEAD'), + benchmarkRepository: config.benchmarkRepository, benchmarkRevision: config.benchmarkRevision, + nodeVersion: process.version, bunVersion: execFileSync(bun, ['--version'], { encoding: 'utf8' }).trim(), + host: { cpuModel: cpus()[0]?.model, cpus: cpus().length, totalMemBytes: totalmem(), platform: platform(), osRelease: release(), arch: arch() }, + config, suite: values.scenario ? 'focused' : values.suite, selectedScenarios: scenarios.map(s => s.id), results: [], + builds: Object.fromEntries([['base', base], ['head', head]].map(([name, directory]) => [name, { + path: directory, entrySha256: createHash('sha256').update(readFileSync(join(directory, 'dist/cli.js'))).digest('hex'), + }])), +}; +function save() { + writeFileSync(join(out, 'comparison.json'), JSON.stringify(report, null, 2) + '\n'); + writeFileSync(join(out, 'comparison.md'), renderReport(report)); +} +// Children receive synthetic state and an explicit environment, never local provider credentials. +const environment = { PATH: process.env.PATH, HOME: join(out, 'home'), TMPDIR: process.env.TMPDIR ?? '/tmp', + LANG: 'en_US.UTF-8', TERM: 'dumb', CI: '1', NO_COLOR: '1' }; +mkdirSync(environment.HOME, { recursive: true }); +function run(build, scenario, script, name, profile = false) { + const directory = join(out, scenario.id, name); + mkdirSync(directory, { recursive: true }); + const input = { benchmark, control, build, script, scenario, directory, profile, + intervalMs: config.intervalMs, node: process.execPath }; + const inputPath = join(directory, 'input.json'); + writeFileSync(inputPath, JSON.stringify(input)); + const result = spawnSync(bun, [join(control, 'scripts/perf/driver.ts'), inputPath], { + cwd: benchmark, env: environment, encoding: 'utf8', timeout: scenario.timeoutMs + 45000, maxBuffer: 16 * 1024 * 1024, + }); + writeFileSync(join(directory, 'driver.log'), (result.stdout ?? '') + (result.stderr ?? '')); + if (result.error || result.status !== 0) throw new Error(`${scenario.id}/${name} failed; see driver.log (${result.error?.message ?? result.status})`); + return { ...JSON.parse(readFileSync(join(directory, 'result.json'), 'utf8')), directory }; +} +save(); +try { + for (const scenario of scenarios) { + const script = join(out, `${scenario.id}.json`); + execFileSync(bun, [join(benchmark, 'scripts/perf/gen-long-run.ts'), '--turns', String(scenario.turns), + '--body-kb', String(scenario.bodyKb), '--tool', 'bash', '--chunk-size', String(scenario.chunkSize), + '--chunk-delay-ms', '0', '--out', script], { cwd: benchmark, env: environment }); + const scriptData = JSON.parse(readFileSync(script, 'utf8')); + const textUnits = scriptData.responses.slice(0, scenario.turns).reduce((sum, entry) => sum + entry.message.content.length, 0); + if (scenario.minimumTextUnits && textUnits < scenario.minimumTextUnits) throw new Error('Long-history fixture no longer exceeds token cache capacity'); + const result = { id: scenario.id, textUnits, scenarioSha256: createHash('sha256').update(readFileSync(script)).digest('hex'), warmups: [], pairs: [] }; + report.results.push(result); + save(); + for (let i = 0; i < config.warmups; i++) for (const [side, directory] of [['base', base], ['head', head]]) { + console.log(`[perf] ${scenario.id}: ${side} warmup`); + result.warmups.push({ side, ...run(directory, scenario, script, `${side}-warmup-${i}`) }); + save(); + } + for (let pair = 0; pair < config.repetitions; pair++) { + const item = { pair, order: pair % 2 ? ['head', 'base'] : ['base', 'head'] }; + result.pairs.push(item); + for (const side of item.order) { + console.log(`[perf] ${scenario.id}: pair ${pair + 1}, ${side}`); + item[side] = run(side === 'base' ? base : head, scenario, script, `${side}-${pair}`); + save(); + } + } + result.comparison = compareRuns(result.pairs.map(p => p.base.metrics), result.pairs.map(p => p.head.metrics), config); + save(); + if (result.comparison.some(row => row.status === 'REGRESSION')) { + console.log(`[perf] ${scenario.id}: separate diagnostic CPU profile`); + try { result.diagnostic = run(head, scenario, script, 'head-profile', true); } + catch (error) { result.diagnosticError = error.message; } + save(); + } + } + const statuses = report.results.flatMap(r => r.comparison.map(row => row.status)); + report.status = statuses.includes('REGRESSION') ? 'REGRESSION' : statuses.includes('INCONCLUSIVE') ? 'INCONCLUSIVE' : 'PASS'; + process.exitCode = report.status === 'PASS' ? 0 : report.status === 'REGRESSION' ? 1 : 2; +} catch (error) { + report.status = 'ERROR'; + report.error = error.message; + process.exitCode = 1; +} finally { + save(); + const markdown = readFileSync(join(out, 'comparison.md'), 'utf8'); + console.log(markdown); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); +} diff --git a/test/source-sync.test.mjs b/test/source-sync.test.mjs index 584f464ec..8a92c52f8 100644 --- a/test/source-sync.test.mjs +++ b/test/source-sync.test.mjs @@ -18,6 +18,121 @@ import { cliBuildVersion, cliExternalModules, cliReleaseTargets, versionFromTag import { releaseManifest } from '../scripts/package-cli-release.mjs'; import { validateReleaseReports } from '../scripts/publish-cli-release.mjs'; import { compareVersions, releaseCli } from '../scripts/release-cli.mjs'; +import { compareRuns, validateRun, validateRequest, validateToolOutput, median, selectScenarios } from '../scripts/perf/report.mjs'; + +test('performance defaults to the 100-round suite; long history requires explicit selection', () => { + const config = JSON.parse(readFileSync(new URL('../scripts/perf/config.json', import.meta.url), 'utf8')); + assert.deepEqual(selectScenarios(config).map(s => s.id), ['upstream-100']); + assert.deepEqual(selectScenarios(config, { suite: 'full' }).map(s => s.id), ['startup', 'upstream-100', 'history-300']); + assert.deepEqual(selectScenarios(config, { scenario: 'startup' }).map(s => s.id), ['startup']); + assert.throws(() => selectScenarios(config, { suite: 'typo' })); + assert.throws(() => selectScenarios(config, { scenario: 'typo' })); + const workflow = parseYaml(readFileSync(new URL('../.github/workflows/performance.yml', import.meta.url), 'utf8')); + assert.equal(workflow.on.workflow_dispatch.inputs.suite.default, 'full'); + assert.deepEqual(workflow.on.workflow_dispatch.inputs.suite.options, ['full', 'basic']); + const compare = workflow.jobs.performance.steps.find(s => s.name === 'Compare on this runner'); + assert.equal(compare.env.PERF_SUITE, "${{ matrix.suite }}"); + assert.match(compare.run, /--suite "\$PERF_SUITE"/); +}); + +test('performance labels select full coverage without interrupting checks for unrelated labels', () => { + const workflow = parseYaml(readFileSync(new URL('../.github/workflows/performance.yml', import.meta.url), 'utf8')); + const job = workflow.jobs.performance; + assert.deepEqual(workflow.on.pull_request.types, ['opened', 'synchronize', 'reopened', 'labeled', 'unlabeled']); + assert.equal(job.strategy.matrix.suite, `\${{ fromJSON((inputs.suite == 'full' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'perf:full'))) && '["full"]' || '["basic"]') }}`); + assert.equal(job.if, `\${{ !contains(fromJSON('["labeled","unlabeled"]'), github.event.action) || github.event.label.name == 'perf:full' }}`); + assert.equal(job.name, `\${{ contains(fromJSON('["labeled","unlabeled"]'), github.event.action) && github.event.label.name != 'perf:full' && 'performance (label ignored)' || 'performance' }}`); + assert.equal(workflow.concurrency, undefined); + assert.equal(job.concurrency.group, `performance-\${{ github.event_name }}-\${{ github.ref }}-\${{ github.event_name == 'workflow_dispatch' && matrix.suite || 'auto' }}`); + assert.equal(job.concurrency['cancel-in-progress'], true); + assert.equal(job['timeout-minutes'], `\${{ matrix.suite == 'full' && 45 || 15 }}`); +}); + +test('performance request audit rejects truncated wire history and empty tool results', () => { + const responses = [ + { message: { content: 'full history', tool_calls: [{ id: 'call_long_run_1', function: { name: 'bash', arguments: { command: 'ls' } } }] } }, + { message: { content: 'done' } }, { message: { content: ' ' } }, + ]; + const messages = [{ role: 'user', content: 'task' }, + { role: 'assistant', content: 'full history', tool_calls: [{ id: 'call_long_run_1', function: { name: 'bash', arguments: '{"command":"ls"}' } }] }, + { role: 'tool', tool_call_id: 'call_long_run_1', content: 'README.txt\n' }]; + assert.doesNotThrow(() => validateRequest({ messages }, responses, 2, process.cwd(), 'task')); + assert.throws(() => validateRequest({ messages: messages.slice(0, 1) }, responses, 2, process.cwd(), 'task'), /conversation length/); + assert.throws(() => validateRequest({ messages: [] }, responses, 1, process.cwd(), 'task'), /conversation length/); + assert.throws(() => validateRequest({ messages: [messages[0], messages[2], messages[1]] }, responses, 2, process.cwd(), 'task'), /order changed/); + assert.throws(() => validateRequest({ messages: [{ role: 'user', content: 'wrong' }] }, responses, 1, process.cwd(), 'task'), /task changed/); + assert.doesNotThrow(() => validateRequest({ messages: [messages[0]] }, responses, 1, process.cwd(), 'task')); + const changed = structuredClone(messages); + changed[1].content = 'shortened'; + assert.throws(() => validateRequest({ messages: changed }, responses, 2, process.cwd(), 'task'), /body changed/); + changed[1].content = 'full history'; changed[2].content = ''; + assert.throws(() => validateRequest({ messages: changed }, responses, 2, process.cwd(), 'task'), /lost fixture/); + assert.throws(() => validateToolOutput([], 'ls', process.cwd()), /lost fixture/); + assert.throws(() => validateToolOutput('wrong', 'echo expected', process.cwd()), /Echo/); + assert.throws(() => validateToolOutput('/wrong', 'pwd', process.cwd()), /directory/); + assert.doesNotThrow(() => validateToolOutput(realpathSync(process.cwd()) + '\n', 'pwd', process.cwd())); +}); + +test('performance comparison rejects incomplete, invalid and unstable samples', () => { + const config = { repetitions: 3, maxSpread: 0.3, thresholds: { cpuSeconds: { relative: 0.2, absolute: 0.5 } } }; + const runs = values => values.map(cpuSeconds => ({ cpuSeconds })); + assert.equal(median([3, 1, 2]), 2); + assert.equal(compareRuns(runs([10, 10, 10]), runs([10, 10, 10]), config)[0].status, 'PASS'); + assert.equal(compareRuns(runs([10, 10, 10]), runs([13, 13, 13]), config)[0].status, 'REGRESSION'); + assert.equal(compareRuns(runs([1, 1, 1]), runs([1.4, 1.4, 1.4]), config)[0].status, 'PASS'); + assert.equal(compareRuns(runs([10, 10, 10]), runs([10, 10, 20]), config)[0].status, 'INCONCLUSIVE'); + assert.throws(() => compareRuns(runs([10, 10]), runs([10, 10, 10]), config)); + assert.throws(() => compareRuns(runs([10, NaN, 10]), runs([10, 10, 10]), config)); + assert.throws(() => compareRuns(runs([0, 0, 0]), runs([10, 10, 10]), config)); +}); + +test('performance measurements require complete successful tool execution', () => { + const meta = { schemaVersion: 1, status: 'ok', exit: { code: 0 }, mock: { requests: 2 }, + sampling: { backend: 'rusage', withTree: true }, summary: { count: 10 }, + duration: { endToEndMs: 1000 }, cost: { cpuSeconds: 0.5 }, peaks: { treeRssBytes: 1024 } }; + const messages = [ + { role: 'assistant', content: [{ type: 'toolCall', id: 'call_long_run_1', name: 'bash' }] }, + { role: 'toolResult', toolCallId: 'call_long_run_1', isError: false }, + { role: 'assistant', content: [{ type: 'text', text: 'done' }] }, + ]; + assert.equal(validateRun(meta, messages, { turns: 1 }, 'done').rssBytes, 1024); + const expected = [{ message: { content: '', tool_calls: [{ function: { arguments: { command: 'pwd' } } }] } }]; + const withArguments = structuredClone(messages); + withArguments[0].content[0].arguments = { command: 'pwd' }; + assert.doesNotThrow(() => validateRun(meta, withArguments, { turns: 1 }, 'done', expected)); + withArguments[0].content[0].arguments.command = 'true'; + assert.throws(() => validateRun(meta, withArguments, { turns: 1 }, 'done', expected), /command changed/); + withArguments[0].content[0].arguments.command = 'pwd'; + expected[0].message.content = 'body that must be retained'; + assert.throws(() => validateRun(meta, withArguments, { turns: 1 }, 'done', expected), /body changed/); + for (const mutate of [ + (m, _) => { m.status = 'timeout'; }, (m, _) => { m.mock.requests = 1; }, + (m, _) => { m.sampling.backend = 'ps'; }, (m, _) => { m.summary.count = 0; }, + (m, _) => { m.cost.cpuSeconds = null; }, (_, rows) => { rows[1].isError = true; }, + (_, rows) => { rows[1].toolCallId = 'wrong'; }, (_, rows) => { rows.pop(); }, + (_, rows) => { rows.push(rows[1]); }, + ]) { + const copiedMeta = structuredClone(meta), copiedMessages = structuredClone(messages); + mutate(copiedMeta, copiedMessages); + assert.throws(() => validateRun(copiedMeta, copiedMessages, { turns: 1 }, 'done')); + } +}); + +test('performance workflow uses pinned mock input and an unprivileged PR job', () => { + const workflow = parseYaml(readFileSync(new URL('../.github/workflows/performance.yml', import.meta.url), 'utf8')); + const config = JSON.parse(readFileSync(new URL('../scripts/perf/config.json', import.meta.url), 'utf8')); + assert.ok(Object.hasOwn(workflow.on, 'pull_request')); + assert.ok(!Object.hasOwn(workflow.on, 'pull_request_target')); + assert.deepEqual(workflow.permissions, { contents: 'read' }); + const steps = workflow.jobs.performance.steps; + const benchmark = steps.find(s => s.with?.repository === 'KonghaYao/harness-perf-benchmark'); + assert.equal(benchmark.with.ref, config.benchmarkRevision); + assert.match(config.benchmarkRevision, /^[a-f0-9]{40}$/); + for (const step of steps.filter(s => s.uses)) assert.match(step.uses, /@[a-f0-9]{40}$/); + const standard = config.scenarios.find(s => s.id === 'upstream-100'); + assert.deepEqual([standard.turns, standard.bodyKb, standard.chunkSize], [100, 4, 64]); + assert.ok(config.scenarios.some(s => s.minimumTextUnits > 1048576)); +}); test('release tags are canonical and must match both source versions without overriding them', t => { for (const tag of ['v0.4.13', 'v1.0.0-rc.1', 'v0.0.0', 'v2.3.4-beta-test.0'])