diff --git a/.claude/skills/cut-release/SKILL.md b/.claude/skills/cut-release/SKILL.md index 0d627cdac..b5501beba 100644 --- a/.claude/skills/cut-release/SKILL.md +++ b/.claude/skills/cut-release/SKILL.md @@ -113,7 +113,7 @@ commit, treat it as blocking anyway. (Run this same query after tagging and `Release Artifacts` joins the list: the tag points at the bump commit, so it shares the SHA.) -`helm repo add bitnami` (seven workflow sites: `ci.yml` x2, `helm-release.yml` x3, `npm-publish.yml`, +`helm repo add bitnami` (eight workflow sites: `ci.yml` x3, `helm-release.yml` x3, `npm-publish.yml`, `operator-release.yml`) fetches a 27 MB index with no retry and flakes with `connection reset by peer`. Verify the repo really is reachable, then re-run only the failed job: @@ -122,7 +122,7 @@ curl -sSL -o /dev/null -w '%{http_code}\n' https://charts.bitnami.com/bitnami/in gh run rerun --failed ``` -Those seven sites do **not** all pin the same Helm CLI. Six run Helm 4.1.3; `helm-release.yml`'s +Those eight sites do **not** all pin the same Helm CLI. Seven run Helm 4.1.3; `helm-release.yml`'s `lint-test` job stays on Helm 3.16 on purpose, because its two `ct install` runs are the only place the chart is installed into a cluster and our users install with Helm 3. The split is enforced by `tests/unit/helm-pin-matrix.test.ts` in the required test lane - do not unify the odd one out. @@ -247,7 +247,7 @@ true on a tag ref. | `gh release create --target ` | Rejected ("target_commitish is invalid") - use `--target main` | | A `release-artifacts` run reporting `failure` | The release may still have published fine; check `Verify assets and publish release` before assuming otherwise | | Reusing a failed release's version after Snap published | Snap store revisions are immutable per version; bump the patch instead | -| Renaming or removing the `test:ci` script | `npm-publish.yml` validates with `bun run test:ci` (per-file process isolation via `tests/run-core.sh`), NOT `bun run test`. Losing that script breaks every release and every re-dispatch | +| Renaming or removing the `test` script | `npm-publish.yml` validates with `bun run test`, which is `bun tests/run-tests.ts` (one bun process per test file). Losing that script breaks every release and every re-dispatch | | Recreating a draft after a failed run | The hand-written notes are gone with it. Keep the notes file in the scratchpad and re-apply with `gh release edit --notes-file ` | | A release that touches `packaging/`, the Dockerfile or the payload scripts | The chain builds channels you cannot see locally. Validate them locally first (tarball/npx/docker build+run, deb/rpm with the CI-pinned nfpm) - that local pass is what separated the clean one-attempt releases from the four-attempt one | diff --git a/.devin/wiki.json b/.devin/wiki.json index 03c404248..c50453ad0 100644 --- a/.devin/wiki.json +++ b/.devin/wiki.json @@ -21,7 +21,7 @@ "author": "Maintainer" }, { - "content": "Testing is multi-layered: unit tests (tests/unit/), API route handler tests (tests/api/), integration tests with mocked database drivers (tests/integration/), React hook tests (tests/hooks/), component tests with happy-dom (tests/components/), and Playwright E2E tests (e2e/). Component tests use isolated execution groups via tests/run-components.sh to prevent mock.module() cross-contamination. CI pipeline runs lint, typecheck, test, and build.", + "content": "Testing is multi-layered: unit tests (tests/unit/), API route handler tests (tests/api/), integration tests with mocked database drivers (tests/integration/), React hook tests (tests/hooks/), component tests with happy-dom (tests/components/), and Playwright E2E tests (e2e/). Every test file runs in its own bun process (tests/run-tests.ts, which `bun run test` invokes) to prevent mock.module() cross-contamination. CI pipeline runs lint, typecheck, test, and build.", "author": "Maintainer" }, { @@ -158,7 +158,7 @@ }, { "title": "Testing Strategy", - "purpose": "Document the comprehensive multi-layer testing strategy. Cover: test setup files (tests/setup.ts for env/localStorage mocks, tests/setup-dom.ts for happy-dom), test helpers (tests/helpers/ with mock-monaco, mock-next, mock-navigation, mock-provider, mock-fetch, mock-sonner, render-with-providers), test fixtures (tests/fixtures/ with connections, schemas, query-results, masking-configs), and the component test isolation script (tests/run-components.sh). Explain why component tests need isolation (bun:test mock.module cross-contamination)." + "purpose": "Document the comprehensive multi-layer testing strategy. Cover: test setup files (tests/setup.ts for env/localStorage mocks, tests/setup-dom.ts for happy-dom), test helpers (tests/helpers/ with mock-monaco, mock-next, mock-navigation, mock-provider, mock-fetch, mock-sonner, render-with-providers), test fixtures (tests/fixtures/ with connections, schemas, query-results, masking-configs), and the cross-platform test runner (tests/run-tests.ts plus tests/runner/). Explain why every test file gets its own bun process (bun:test mock.module is process-wide, with no undo)." }, { "title": "Unit & API Tests", diff --git a/.github/curated-issue-footer.md b/.github/curated-issue-footer.md index 9b06eeebe..9dd432462 100644 --- a/.github/curated-issue-footer.md +++ b/.github/curated-issue-footer.md @@ -1,8 +1,9 @@ Curated for Hacktoberfest 2026. Comment to claim the issue before you start so two people do not work on the same change. A PR must reference the issue and include tests for executable changes; see [CONTRIBUTING.md](https://github.com/libredb/libredb-studio/blob/main/CONTRIBUTING.md). -Run `bun run test`, never bare `bun test`, so component -tests use their isolated execution groups. The 100% line-coverage gate must stay green. +Run `bun run test`, never bare `bun test` over a directory: the runner gives each test file its own +process, and `bun test tests/api` shares one, where a mock set up by one file leaks into the next. +The 100% line-coverage gate must stay green. **CI is the merge gate.** If you cannot run a command locally, list that command and the reason under a `Testing` heading in your PR body; submit the PR, and a maintainer will approve the fork's @@ -10,5 +11,5 @@ workflow run so CI can verify it. You do not need to withdraw correct work becau is unavailable. If your sandbox can reach the npm registry, `npm install -g bun` is another way to install Bun. -Helm is only needed for the chart tests in the test suite. The repository's devcontainer provides -Bun and Helm and installs the JavaScript and chart dependencies automatically. +Helm is only needed to run the chart tests: without it `bun run test` leaves those files out and names them, and CI runs them. +The repository's devcontainer provides Bun and Helm and installs the JavaScript and chart dependencies automatically. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a237da8c..9e4bc1ab1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,9 +152,17 @@ jobs: - name: Build chart dependencies run: helm dependency build charts/libredb-studio --skip-refresh - - name: Run merged test coverage (core + components) + - name: Run the test suite with coverage + # tests/run-tests.ts runs every test file in its own bun process, several + # at a time, and merges the per-file lcov reports. It is the same command + # a contributor runs (`bun run test`) with coverage turned on, which is + # the point: the gate and the documented command cannot drift apart. run: bun run test:coverage env: + # Optional on a contributor's machine, mandatory here: this job installs Helm and + # builds the chart dependency, so a chart test that did not run is a defect, and the + # runner refuses the run instead of listing it as not run (tests/runner/requirements.ts). + LIBREDB_REQUIRE_HELM: "1" JWT_SECRET: test-secret-for-ci-build-only-32ch ADMIN_EMAIL: admin@libredb.org ADMIN_PASSWORD: test-admin @@ -196,6 +204,86 @@ jobs: disable_search: true fail_ci_if_error: false + # The same suite, on the two platforms the project has never measured. Linux is + # covered by the `test` job above, which runs the identical runner with coverage. + # + # A SEPARATE job rather than a matrix on `test`: GitHub appends the matrix values + # to a matrix job's name, so matrixing `test` would rename the required check + # "Unit & Integration Tests" out of existence and every pull request would wait + # forever for a check that can no longer report. This job is deliberately not a + # required check yet; promote it once it has a history of being green. + # + # It uses no secrets, so it runs on fork pull requests like any other job. + test-cross-platform: + name: Cross-platform Tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Setup Node + # Both images ship Node 22 by default, below the product's floor, and the + # tests that spawn bin/studio.js get whatever `node` is on PATH (#709). + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.4.2" + + - name: Install dependencies + uses: ./.github/actions/bun-install + + - name: Set up Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.1.3 + + - name: Add Bitnami repo + run: helm repo add bitnami https://charts.bitnami.com/bitnami --timeout 5m + + - name: Build chart dependencies + run: helm dependency build charts/libredb-studio --skip-refresh + + - name: Report the toolchain + # A red leg on a platform nobody here runs is only actionable if the run + # says what it had: a missing 7z or an older bash changes what "red" means. + shell: bash + run: | + echo "platform: $(uname -s 2>/dev/null || echo windows) $(uname -m 2>/dev/null || echo unknown)" + echo "bun: $(bun --version)" + echo "node: $(node --version)" + echo "bash: ${BASH_VERSION:-unknown}" + echo "helm: $(helm version --short)" + for tool in sh bash tar unzip 7z git; do + printf '%-9s %s\n' "$tool:" "$(command -v "$tool" || echo "not on PATH")" + done + + - name: Run the test suite + # Deliberately NOT `shell: bash`. On Windows that would be Git Bash, whose + # PATH carries the whole unix toolset, and the contributor this job stands + # in for runs the command from PowerShell. Running it the way they do is + # what makes this job evidence. + run: bun run test + env: + # Optional on a contributor's machine, mandatory here: this job installs Helm and + # builds the chart dependency, so a chart test that did not run is a defect, and the + # runner refuses the run instead of listing it as not run (tests/runner/requirements.ts). + LIBREDB_REQUIRE_HELM: "1" + JWT_SECRET: test-secret-for-ci-build-only-32ch + ADMIN_EMAIL: admin@libredb.org + ADMIN_PASSWORD: test-admin + USER_EMAIL: user@libredb.org + USER_PASSWORD: test-user + e2e: name: E2E Tests needs: [lint-and-build] diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index fab751763..b0fd5b4a2 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -34,6 +34,16 @@ jobs: - name: Checkout code uses: actions/checkout@v7 + - name: Setup Node + # The suite spawns `node bin/studio.js`, and the launcher refuses anything below + # its own Node 24 floor, so the test that asserts its startup URL fails on the + # runner image's default Node. ci.yml's test job was pinned for exactly this in + # #709 and #737; this job runs the same suite before every npm release and was + # not, and the tests that need it arrived after the last release it validated. + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: @@ -61,10 +71,16 @@ jobs: run: bun run typecheck - name: Test - # Per-file process isolation (run-core.sh) — bun's mock.module() is - # process-wide, so the single-process `bun run test` is load-order flaky - # here. This mirrors the reliable path ci.yml uses via test:coverage. - run: bun run test:ci + # `bun run test` is tests/run-tests.ts: one bun process per test file, so + # the process-wide mock.module() cannot leak between files. There is no + # separate test:ci script any more, because there is no longer a + # shared-process form of the suite for it to be the alternative to. + run: bun run test + env: + # Optional on a contributor's machine, mandatory here: this job installs Helm and + # builds the chart dependency, so a chart test that did not run is a defect, and the + # runner refuses the run instead of listing it as not run (tests/runner/requirements.ts). + LIBREDB_REQUIRE_HELM: "1" - name: Build run: bun run build diff --git a/CLAUDE.md b/CLAUDE.md index e7a6444ae..229652c05 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,8 @@ bun run format # Biome formatter check (format:fix to write); CSS/JSON bun run lint # oxlint (fast, syntactic) then ESLint 9 bun run lint:oxc # oxlint only bun run typecheck # TypeScript strict -bun run test # all layers: unit + api + integration + hooks + security + evals + components +bun run test # every test file under tests/ (except tests/live/), one bun process per file +bun run test:unit # one layer; also test:api, test:integration, test:hooks, test:security, test:evals, test:components bun run test:e2e # Playwright (builds and starts its own servers; see playwright.config.ts) bun run test:coverage # coverage report (merged lcov) bun run coverage:check # enforce 100% line coverage on the merged lcov @@ -49,9 +50,9 @@ bun run security:check # security posture drift guard > **Run `build:lib` after changing anything reachable from `src/exports/`** (workspace, providers, components, security, …) — `bun run build` (Next.js) does NOT update the package dist. -> **Tests — always `bun run test`, never bare `bun test`.** Component tests need isolated execution groups (`tests/run-components.sh`) to avoid `mock.module()` cross-contamination. +> **Tests, always `bun run test`, never bare `bun test` over a directory.** The runner ([`tests/run-tests.ts`](tests/run-tests.ts)) discovers every test file and runs each one in its own bun process, several at a time (`--jobs=N`, `--list`). `bun test tests/api` puts all of them in one process instead, where one file's `mock.module()` becomes every file's, because bun's module mocks are process-wide with no undo. To run one file, name it: `bun tests/run-tests.ts tests/unit/x.test.ts`. -> **Coverage isolation:** `bun`'s `mock.module()` is process-wide, so `test:coverage:core` runs each core test file in its own process (`tests/run-core.sh`) and `test:coverage` merges the per-file lcov. Do NOT collapse it into one `bun test` invocation. Rationale: [`docs/TOOLCHAIN.md`](docs/TOOLCHAIN.md). +> **Coverage:** `bun run test:coverage` is the same runner with `--coverage --merge-into=coverage/lcov.info`: one lcov per test file, merged by `scripts/merge-lcov.mjs`. Two files run without coverage on purpose; they are `COVERAGE_EXEMPT_FILES` in [`tests/runner/discover.ts`](tests/runner/discover.ts), with the reason in its docblock. Rationale: [`docs/TOOLCHAIN.md`](docs/TOOLCHAIN.md). ## Pre-Commit Verification (MANDATORY) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f4a4cf11..113c7f57a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,12 +59,17 @@ Feature suggestions are welcome! Please provide: is unavailable. If your sandbox can reach the npm registry, `npm install -g bun` is another way to install Bun. - Helm is only needed for the chart tests in the test suite, not for editing the app or running - typecheck. The [devcontainer setup](#devcontainer--codespaces) below provides both tools. - - Always `bun run test`, never bare `bun test`: component tests need the isolated execution groups - the script sets up. `bun run test:coverage && bun run coverage:check` prints the exact uncovered - `file:line` ranges. + Helm is only needed to run the chart tests: without it `bun run test` leaves those files out and names them, and CI runs them. + The [devcontainer setup](#devcontainer--codespaces) below provides both tools. + + Always `bun run test`, never bare `bun test` over a directory: the runner gives each test file its + own bun process, and `bun test tests/api` puts them all in one, where one file's `mock.module()` + becomes every file's. To run a single file, name it: `bun tests/run-tests.ts tests/unit/x.test.ts`. + `bun run test:coverage && bun run coverage:check` prints the exact uncovered `file:line` ranges. + `bun tests/run-tests.ts --jobs=N` lowers the concurrency, which by default is one job per available CPU: that follows CPU affinity and a cgroup CPU limit, but no memory limit, and a job peaks at roughly 60 to 340 MiB. + Use it in a memory-limited container that has no CPU limit, and when a file comes back as killed by SIGKILL from outside the runner, which on Linux is usually the OOM killer. + A flag meant for `bun test` goes past a `--` the runner can see, which means invoking the runner directly with a selector first: `bun tests/run-tests.ts tests/unit -- --bail`. + `bun run test -- --bail` does not work, because `bun run` removes the first `--` before the script sees it, and bun removes one that sits straight after the script path too. 5. **Keep the provider triad in lockstep.** Anything under `src/lib/db/providers/**` has a matching `docs/providers/.md` and `tests/integration/db/-provider.test.ts`; a change to one moves the other two in the same PR. @@ -211,16 +216,41 @@ started by this setup. ### Prerequisites -- [Bun](https://bun.sh/) (recommended) or Node.js 24+ -- Git -- [Helm](https://helm.sh/) 4.1.3 (the version CI runs). Ten of the eleven `helm-chart-*.test.ts` files under `tests/unit/` spawn the `helm` binary - all but `helm-chart-readme-recipes.test.ts`, which is a static lint over the chart README. Without `helm` on `PATH`, `bun run test` fails with 166 `error: Executable not found in $PATH: "helm"` errors. The PostgreSQL subchart tarball is gitignored (`*.tgz`), so a fresh clone also needs: +The suite runs on Linux, macOS and Windows, from whichever shell the platform gives you. +`bun run test` is `bun tests/run-tests.ts`, a TypeScript runner rather than a shell script, and CI runs it on ubuntu-latest, macos-latest and windows-latest. + +| Tool | Version | Needed for | +| --- | --- | --- | +| [Bun](https://bun.sh/) | 1.4.2, the `packageManager` pin | Installing, the dev server, the build, and the test runner itself | +| [Node.js](https://nodejs.org/) | 24+, the `engines` floor | The `scripts/*.mjs` gates, including `merge-lcov.mjs` and `check-coverage.mjs` | +| Git | any | Cloning, and on Windows it is also where the POSIX tools below come from | +| [Helm](https://helm.sh/) | 4.1.3, the version CI runs | Optional locally: the chart tests, see below | +| A POSIX shell plus `tar`, `unzip` and `7z` | any | The packaging tests, which run the `packaging/` shell scripts and unpack what they produce | + +Twelve of the thirteen `helm-chart-*.test.ts` files under `tests/unit/` spawn the `helm` binary; the exception is `helm-chart-readme-recipes.test.ts`, a static lint over the chart README. +Each of the twelve opens with `// @requires helm`, and the runner reads that before it starts a file. +Where `helm` is not on `PATH`, or the chart's PostgreSQL subchart is not built, `bun run test` does not start those files: it runs the rest, and its summary lists them under "Files not run on this machine" with the reason and the command that fixes it. +Selecting only chart tests on such a machine is an error rather than an empty green run. +CI sets `LIBREDB_REQUIRE_HELM=1`, which makes the same condition stop the run before anything starts, so the chart tests are never left out of a gate. +If you change the chart, install Helm and run them before you push. +A new test file that runs `helm` needs the marker too, and `tests/unit/test-runner-requirements.test.ts` fails until it has it. +To run the chart tests, install Helm; the PostgreSQL subchart tarball is gitignored (`*.tgz`), so a fresh clone also needs: + +```bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm dependency build charts/libredb-studio --skip-refresh +``` + +Trap: a stale `docker login` can make that build fail with `401 Unauthorized` from `registry-1.docker.io` even though the chart is anonymously pullable. `docker logout` fixes it. - ```bash - helm repo add bitnami https://charts.bitnami.com/bitnami - helm dependency build charts/libredb-studio --skip-refresh - ``` +On Windows the POSIX tools come from the Git for Windows installation the clone already needed, and the tests locate them through git itself rather than through `PATH`. +PowerShell's `PATH` carries `git.exe` but not the `bin` and `usr\bin` directories beside it that hold `bash.exe`, `grep.exe` and `unzip.exe`, and where WSL is installed a bare `bash` does resolve, to `C:\Windows\System32\bash.exe`, a Linux shell that cannot read the Windows temp paths the fixtures hand it. +So `tests/helpers/posix-tools.ts` asks the git binary for its exec path, derives the installation root from it, falls back to `%LOCALAPPDATA%\Programs\Git` and the two `Program Files` defaults, and spawns each tool by absolute path; 7-Zip is looked for at `C:\Program Files\7-Zip\7z.exe` as well as on `PATH`. +A tool it cannot find turns the tests that need it into skips whose titles carry the reason, instead of a spawn that throws and takes the rest of the file with it. +Assertions about POSIX file modes skip on Windows in every case: NTFS has no exec bit, and Windows cannot exec an extension-less `#!` script. - Trap: a stale `docker login` can make that build fail with `401 Unauthorized` from `registry-1.docker.io` even though the chart is anonymously pullable. `docker logout` fixes it. +You do not need a `.env` file or a `data/` directory to run the tests. +`tests/setup.ts`, which `bunfig.toml` preloads into every test process, pins the credentials and settings the suite runs under, so a local `.env` cannot decide a test's outcome. ### Getting Started @@ -306,8 +336,8 @@ bun run format # Biome formatter check (format:fix to write) bun run lint # oxlint, then ESLint 9 bun run typecheck # TypeScript strict bun run knip # unused files, exports and dependencies -bun run test # every test layer; never bare `bun test`. Needs Helm and the built subchart, see Prerequisites -bun run test:ci # the same layers with one process per file, which is what CI runs; use it to verify +bun run test # every test file, one bun process each; never bare `bun test`. Without Helm the chart tests are listed as not run, see Prerequisites +bun run test:unit # one layer; also test:api, test:integration, test:hooks, test:security, test:evals, test:components bun run test:coverage # coverage report (merged lcov) bun run coverage:check # enforce 100% line coverage on the merged lcov bun run readme:check # localized README drift guard diff --git a/README.md b/README.md index 14538be66..2be78832c 100644 --- a/README.md +++ b/README.md @@ -540,23 +540,30 @@ Sample tables: `app.customers`, `app.products`, `app.orders`, `app.order_items`, ## Testing -LibreDB Studio has a comprehensive test suite with **3,000+ unit/integration tests** and **32 E2E tests** across 6 layers, with **100% line coverage** enforced by CI (`bun run coverage:check`). +LibreDB Studio has a comprehensive test suite: 549 test files and 17,692 tests across seven layers, plus 79 browser tests, with **100% line coverage** enforced by CI (`bun run coverage:check`). ### Quick Commands ```bash -# Run all tests (unit + API + integration + hooks + components) +# Every test file, each in its own bun process bun run test # Run by layer -bun run test:unit # Pure function tests (1,600+ cases) -bun run test:api # API route handler tests (270+ cases) -bun run test:integration # Database provider tests (340+ cases) -bun run test:hooks # React hook tests (250+ cases) -bun run test:components # Component tests with mock isolation (570+ cases) +bun run test:unit # Pure function tests (328 files) +bun run test:api # API route handler tests (35 files) +bun run test:integration # Database provider tests (24 files) +bun run test:hooks # React hook tests (21 files) +bun run test:security # Security posture tests (21 files) +bun run test:evals # LLM prompt evaluation tests (13 files) +bun run test:components # Component tests (107 files: tests/components and tests/isolated) + +# Any subset, and what the runner would run +bun tests/run-tests.ts tests/integration/db/duckdb-provider.test.ts +bun tests/run-tests.ts --list +bun tests/run-tests.ts --jobs=4 # bound the concurrency # E2E tests (requires build) -bun run test:e2e # Playwright browser tests (32 cases) +bun run test:e2e # Playwright browser tests (79 cases across chromium and webkit) # Coverage report (lcov) bun run test:coverage @@ -564,24 +571,31 @@ bun run test:coverage ### Test Architecture -| Layer | Directory | Runner | Tests | What it covers | -|-------|-----------|--------|-------|----------------| -| **Unit** | `tests/unit/` | `bun:test` | ~1,609 | Pure functions: SQL parser, connection strings, data masking, query limiter, schema diff, error classes, DB icons, showcase queries | -| **API** | `tests/api/` | `bun:test` | ~279 | Route handlers: auth, query, transaction, maintenance, AI endpoints, middleware | -| **Integration** | `tests/integration/` | `bun:test` | ~346 | Database providers: PG, MySQL, SQLite, MongoDB, Couchbase, Redis, Oracle, MSSQL, ClickHouse, Druid, Elasticsearch, OpenSearch, Trino | -| **Hooks** | `tests/hooks/` | `bun:test` | ~251 | React hooks: auth, connections, tabs, query execution, transactions, inline editing, monitoring | -| **Components** | `tests/components/` | `bun:test` + happy-dom | ~570 | UI components: Studio, Sidebar, QueryEditor, ResultsGrid, Admin Dashboard, Charts, ERD | -| **E2E** | `e2e/` | Playwright | ~32 | Full browser flows: login, connections, query execution, tabs, export, admin | +| Layer | Directory | Files | Tests | What it covers | +|-------|-----------|-------|-------|----------------| +| **Unit** | `tests/unit/` | 328 | 9,645 | Pure functions: SQL parser, connection strings, data masking, query limiter, schema diff, error classes, DB icons, showcase queries, and the packaging and chart manifests | +| **API** | `tests/api/` | 35 | 602 | Route handlers: auth, query, transaction, maintenance, AI endpoints, middleware | +| **Integration** | `tests/integration/` | 24 | 2,768 | Database providers: PG, MySQL, SQLite, MongoDB, Couchbase, Redis, Oracle, MSSQL, ClickHouse, Druid, Elasticsearch, OpenSearch, Trino | +| **Hooks** | `tests/hooks/` | 21 | 566 | React hooks: auth, connections, tabs, query execution, transactions, inline editing, monitoring | +| **Security** | `tests/security/` | 21 | 322 | The posture `docs/SECURITY.md` claims: route exposure, headers, audit channels, credential handling | +| **Evals** | `tests/evals/` | 13 | 198 | LLM prompt behaviour against recorded models | +| **Components** | `tests/components/`, `tests/isolated/` | 107 | 3,376 | UI components with `happy-dom`: Studio, Sidebar, QueryEditor, ResultsGrid, Admin Dashboard, Charts, ERD | +| **E2E** | `e2e/` | 18 | 79 | Full browser flows: login, connections, query execution, tabs, export, admin | + +The Files column was counted on 2026-09-15 with `bun tests/run-tests.ts --list` for the first seven rows and `playwright test --list` for the last. +The Tests column comes from an earlier full run the same day, over the 542 files the tree held then, so the per-layer numbers are a little below the 17,692 above: they do not yet count the seven test files this branch and the merge from main add under `tests/unit/`, nor the cases this branch adds to the runner's own test files. +The nineteenth spec in `e2e/`, `base-path.spec.ts`, is not in that 18: it needs its own server configuration and runs as `bun run test:e2e:base-path`. ### Key Details -- **Test runner**: `bun:test` (built-in, Jest-compatible API) with `happy-dom` for DOM environment -- **Component isolation**: Component tests run in 6 isolated groups via `tests/run-components.sh` to prevent `mock.module()` cross-contamination +- **Test runner**: [`tests/run-tests.ts`](tests/run-tests.ts) over `bun:test`. It discovers every `*.test.ts` and `*.test.tsx` file under `tests/` except `tests/live/`, so a new test file runs the moment it is added, and it runs each file in its own bun process, several at a time (one per CPU by default, `--jobs=N` to change it). +- **Why a process per file**: bun's `mock.module()` is process-wide with no undo, and whole-module mocks are the standard pattern in `tests/api/`, so files that share a process contaminate each other. On Linux with 20 cores and bun 1.4.2, the suite took 211 seconds one file at a time, 61 seconds 4 at a time and 36 seconds 20 at a time, measured on 2026-09-15 over the 538 files the tree held then; `docs/BACKLOG.md` D86 carries the same three timings and the same basis. +- **One command everywhere**: the runner is TypeScript rather than shell so that the command a contributor is told to run works on Linux, macOS and Windows from the platform's own shell. The bash scripts it replaced did not: one used `mapfile`, a bash 4 builtin that macOS's bash 3.2 does not have. - **E2E**: Playwright runs the full suite on Chromium and the `security-headers` spec on WebKit (`webkit-security`), against a production build (`bun run build && bun start`) -- **CI**: GitHub Actions runs lint + typecheck + build, unit/integration tests with coverage, E2E tests, and SonarCloud analysis -- **Coverage**: `bun test --coverage` generates lcov reports for SonarCloud integration +- **CI**: GitHub Actions runs lint + typecheck + build, the required `Unit & Integration Tests` job (`bun run test:coverage` then `bun run coverage:check`) on ubuntu, a non-required `Cross-platform Tests` job running `bun run test` on windows-latest and macos-latest, E2E tests, and SonarCloud analysis +- **Coverage**: `bun run test:coverage` is the same runner with `--coverage`, which writes one lcov per test file; `scripts/merge-lcov.mjs` merges them into `coverage/lcov.info` for the gate and for SonarCloud -> **Important**: Always use `bun run test` instead of bare `bun test`. The test script handles proper isolation between test groups. +> **Important**: Always use `bun run test`, never bare `bun test` over a directory. `bun test tests/api` puts every file in one process, where one file's module mock becomes every file's. To run a single file, name it to the runner: `bun tests/run-tests.ts tests/api/proxy.test.ts`. --- diff --git a/README_es.md b/README_es.md index 69edea24e..543875650 100644 --- a/README_es.md +++ b/README_es.md @@ -206,7 +206,7 @@ Studio es MIT porque tiene que poder ir a cualquier parte. Lo que se cobra es li ## Pruebas y calidad -- Seis capas de pruebas: unitarias, de API, de integración, de hooks, de componentes y end-to-end +- Siete capas de pruebas: unitarias, de API, de integración, de hooks, de seguridad, de evaluaciones y de componentes, más las end-to-end - **Cobertura de líneas del 100%**, y es una barrera dura en CI. Si la cobertura baja, el merge se bloquea - Quality gate de SonarCloud - Pruebas de humo en Node 24 y 26 en cada release diff --git a/README_ja.md b/README_ja.md index 46505ea47..e10c0aeb0 100644 --- a/README_ja.md +++ b/README_ja.md @@ -265,7 +265,7 @@ StudioがMITなのは、あらゆる場所に置ける必要があるからで ## テストと品質 -- ユニット、API、統合、hooks、コンポーネント、E2Eの6層 +- ユニット、API、統合、hooks、security、evals、コンポーネントの7層、さらにE2E - **行カバレッジ100%**、しかもCIの必須ゲート。下がればマージできません - SonarCloud品質ゲート - リリースごとにNode 24 / 26でスモークテスト diff --git a/README_ur.md b/README_ur.md index 0457bef84..13916b385 100644 --- a/README_ur.md +++ b/README_ur.md @@ -244,7 +244,7 @@ npm i @libredb/studio ## Tests اور quality
    -
  • Tests کی چھ layers: unit، API، integration، hooks، components اور end-to-end
    • +
  • Tests کی سات layers: unit، API، integration، hooks، security، evals اور components، اس کے علاوہ end-to-end
  • 100% line coverage، اور CI میں یہ سخت شرط ہے۔ coverage کم ہوئی تو merge رک جاتا ہے
  • SonarCloud quality gate
  • ہر release میں Node 24 اور 26 پر smoke tests
    • diff --git a/README_zh.md b/README_zh.md index 56d8988cc..e41c5cbb9 100644 --- a/README_zh.md +++ b/README_zh.md @@ -278,7 +278,7 @@ Studio 是 MIT，因为它必须能去任何地方。付费的是 libredb-platfo ## 测试与质量 -- 单元、API、集成、hooks、组件、E2E 六层测试 +- 单元、API、集成、hooks、security、evals、组件七层测试，外加 E2E - **行覆盖率 100%**，并且是 CI 的硬性门禁。覆盖率掉下来，合并就被拦住 - SonarCloud 质量门禁 - 每次发布跨 Node 24 / 26 做冒烟测试 diff --git a/docs/AGENT.md b/docs/AGENT.md index 4315aef2d..f2855b475 100644 --- a/docs/AGENT.md +++ b/docs/AGENT.md @@ -2618,6 +2618,12 @@ the role's own grants are the whole boundary (A3). and never what it said, so an empty completion reaches it too and a model's recorded `retryEmptyTurn: false` decides nothing. Pinned as it behaves rather than narrowed, because the narrowing would move behaviour five passing runs were measured under. +- **B81** — the seed list starts out claiming to be loaded and empty, so a non-OK the server did not + attribute to its seed configuration leaves the rail saying of a connection this application seeds + itself that its settings live in this browser. That is the same false sentence the seed-config + entry was filed about, reached through a proxy rather than through a malformed seed file. Not + fixed here because separating "unasked" from "measured empty" changes a type every consumer + reads, and two tests currently pin the wrong half as intended. **Settled as limits rather than as work.** The seven below have no entry in `docs/BACKLOG.md`, and that is the point: each is how the product behaves, stated where a reader of this document will meet diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index cc590af29..1063dff2d 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -28,7 +28,7 @@ None of it is a GitHub issue. **Sections** - [SQL statement reading](#sql-statement-reading) — S2–S6 · 4 -- [Drivers and connections](#drivers-and-connections) — D1–D94, U17 · 38 +- [Drivers and connections](#drivers-and-connections) — D1–D97, U17 · 43 - [Value interpolation](#value-interpolation) — V1 - [Row editing](#row-editing) — R1 - [Studio UI and query execution](#studio-ui-and-query-execution) — X2–X19, U2–U21 · 12 @@ -40,7 +40,7 @@ None of it is a GitHub issue. - [Security Phase 2 deferrals](#security-phase-2-deferrals) — C3–C11 · 7 - [Security Phase 3 deferrals](#security-phase-3-deferrals) — K4 - [Agent M1 deferrals (#328)](#agent-m1-deferrals-328) — A1–A5 · 4 -- [Agent M2 deferrals (#329)](#agent-m2-deferrals-329) — B2–B80 · 24 +- [Agent M2 deferrals (#329)](#agent-m2-deferrals-329) — B2–B81 · 25 --- @@ -876,41 +876,6 @@ collides with every one of them. **Done when:** one definition of each replaces the copies, with the sqlite and libsql source read sharing its statement. -### D68. `bun run test` is red on a shared process, and only CI's per-file isolation hides it - -`bun run test` is the pre-commit command CLAUDE.md documents, and it runs -`bun test tests/unit tests/api tests/integration` in ONE bun process. `mock.module()` is -process-wide, so a mock one layer needs reaches every file in that process. CI runs -`tests/run-core.sh` instead, one process per file, and is blind to the whole class by -construction. - -Measured 2026-09-13, and the same numbers at `acf50738` and on the #789 branch, so it predates -that epic: every file under `tests/api/` mocks `@/lib/auth` with stubbed `signJWT`, `verifyJWT`, -`getSession`, `login` and `logout`, which is that layer's standard pattern. Run -`tests/unit/lib/auth.test.ts`, `tests/unit/lib/auth-jwt-config.test.ts` and -`tests/unit/seed/resolve-connection.test.ts` beside `tests/api/db-objects.test.ts` and the four -files together are 31 fail; each of them alone is 0 fail. - -**It is not one module.** RE-MEASURED 2026-09-14 (#789 Phase 3): `tests/api/admin/audit.test.ts` -mocks `@/lib/audit` the same way, and `tests/api/db/objects/edit-apply.test.ts` reads the audit ring -to assert what an apply logs. Run those two files together and it is 20 pass 11 fail; each alone is -0 fail, and `bun run test:ci` runs all 424 core files and exits 0. So the pattern is a LAYER mocking -a module a sibling in the same layer legitimately needs, and `@/lib/auth` against three unit files is -one instance of it rather than the whole of it. The full `bun run test` at that commit is 42 fail, -all of them in these two groups. - -The cost is not a red gate, because no gate runs that shape. It is that a contributor following -CLAUDE.md sees dozens of failures on a clean checkout and cannot tell them from their own. - -#789 removed its own three instances by moving the files with the unshareable assumption into -`tests/isolated/`, where `tests/run-components.sh` gives each a process and -`tests/unit/component-runner-coverage.test.ts` makes an unregistered one a red test. The same -remedy does not fit here: it is not three files but a whole layer's mocking pattern against three -unit files that legitimately want the real module. `docs/TOOLCHAIN.md` carries the diagnosis. - -**Done when:** `bun run test` on a clean checkout is green, either because the auth mocking pattern -stops reaching `tests/unit`, or because the documented command runs the same isolation CI does. - ### D69. Six type-ids still open `readObjectSource` with their own entry guard, and one of its sentences is less true `requireSourceKind` in `src/lib/db/object-kinds.ts` is the one entry guard for `readObjectSource`: @@ -1244,6 +1209,146 @@ Two halves, and the second is what stops it recurring: and the citations present at that commit all resolve. The test needs one case per shape it must accept, a single line, a range and a comma pair, and one negative that fails when an anchor moves. +### D85. The `@/lib/auth` mock is hand-copied across a layer, untyped, and already misses two exports + +`grep -rl 'mock.module("@/lib/auth"' tests/` returns exactly 36 hits, measured 2026-09-15. Five of +them spread the real module and replace one function (`{ ...realAuth, getSession: mockGetSession }`, +the agent routes' pattern). Twenty-nine write out the same five-key object - `getSession`, `signJWT`, +`verifyJWT`, `login`, `logout` - down to the same `mock(async () => "mock-token")` for a token +nothing reads, and one of those twenty-nine is `tests/helpers/object-edit-route-harness.ts`, a shared +harness that could have been the factory and copied the stub instead. The remaining two write a +shorter stub of their own, one with two keys and one with a single `getSession`. + +`src/lib/auth.ts` exports seven names. The two no hand-written stub carries are +`shouldMarkCookieSecure` and `resetCookieSecurityWarning`: +`grep -rn 'shouldMarkCookieSecure' tests/` returns exactly one hit, and it is a sentence in a comment +rather than a stub key, while `resetCookieSecurityWarning` appears only in +`tests/unit/lib/auth.test.ts`, which imports the real module. +`src/app/api/auth/oidc/login/route.ts` imports `shouldMarkCookieSecure` and awaits it to decide the +auth cookie's `secure` flag, so every one of those stubs is already an export short of the module it +replaces. Nothing has hit that yet only because `tests/api/auth/oidc-login.test.ts` is one of the +route tests that does NOT mock `@/lib/auth`. + +Nothing can catch it either. `mock.module` is declared `module(id: string, factory: () => any)` in +`node_modules/bun-types/test.d.ts`, so a stub that has drifted from the module it stands in for is +invisible to `bun run typecheck`, and the drift can only show up as a `TypeError` in whichever route +reaches the missing export first. + +Per-file process isolation does nothing about this and was never meant to. The runner gives each +file its own process, so a stub can no longer reach a sibling that wants the real module. What a +process boundary cannot do is make the stub the right SHAPE. + +**Done when:** one factory in `tests/helpers/`, typed `(): typeof import("@/lib/auth")`, replaces the +hand-written stubs, so adding an export to `src/lib/auth.ts` fails `typecheck` in every file that +mocks it instead of at run time in one of them. The same shape then covers the other layer-wide +mocks, `@/lib/db` in fifteen files and `@/lib/audit` in four. + +### D86. `bun test --isolate` has not been re-probed, and the runner pays a process per test file + +`tests/run-tests.ts` spawns one bun process per test file, 549 of them on 2026-09-15, because +`mock.module()` is process-wide with no undo and whole-module mocks are a whole layer's standard +pattern. That is what it costs, measured on Linux with 20 cores and bun 1.4.2 earlier the same day, +over the 538 files the tree held then: 211 seconds one file at a time, 61 seconds 4 at a time, 36 +seconds 20 at a time, and about 60 seconds at 8 with coverage on. `README.md` carries the same three +timings against the same 538 files. + +bun 1.4.2 has `--isolate`, which resets the module registry per file inside ONE process and does +contain `mock.module`. If it were reliable here, the runner could start a handful of processes +rather than one per file. It is not adopted because of oven-sh/bun#41655, a NAPI finalizer SIGSEGV +that reproduces serially on 1.4.2, and this suite loads three NAPI addons: `better-sqlite3`, +`oracledb` and `@duckdb/node-api`. `docs/TOOLCHAIN.md` records the same refusal, beside the one for +`--parallel`. + +**Done when:** #41655 is closed and a probe has run the whole suite under `--isolate` twenty +consecutive times on each of Linux, macOS and Windows with no crash, no leaked subprocess and the +same per-file pass counts as the process-per-file runner, after which the runner may take it - or +the probe reproduced a failure and this entry is replaced by what it reproduced. A mode that is +flaky at this size is worse than a slow one, because its failures arrive wearing the tests' own +clothes. + +### D87. Two packaging tests cannot run on Windows because the scripts they drive shell out + +Measured 2026-09-15, while making `bun run test` green on all three platforms. Two tests now declare +a platform or tool requirement and say so in their own title, and in both cases the requirement comes +from the script under test rather than from the test: + +- `scripts/build-azure-package.mjs:273` builds the marketplace archive with `execFileSync("zip", ...)`. + A stock Windows 11 machine has neither `zip` nor `unzip`, so `tests/unit/build-azure-package.test.ts` + gates its build cases on both binaries. Writing the two-file archive with a pure JavaScript zip + writer would make the Azure package reproducible everywhere and let the test read the archive back + in process, which is what it already does for the standalone zip since this change. +- `scripts/ci-install.sh` is the bun install retry policy used by every workflow, and + `tests/unit/ci-install.test.ts` drives it with a fixture PATH holding two `chmod 0755` stubs. + Windows has no exec bit and no shebang dispatch, so the whole file is skipped there. The policy is + twenty lines of arithmetic and `bun install`; as `scripts/ci-install.mjs` it would run under the + same `shell: bash` steps and be testable on every platform. + +Neither is a correctness defect today: CI runs both on Linux, and the skips are declared rather than +silent. What they cost is that a Windows contributor cannot verify a change to either script. + +**Done when:** the Azure package is written without an external archiver, `ci-install` is a script bun +or node can run, and both test files run unconditionally on all three platforms. + +### D95. The runner's default concurrency reads the CPUs and never the memory limit + +Measured 2026-09-15 on Linux x64 with 20 cores and bun 1.4.2, while reviewing #837. +`tests/run-tests.ts` passes `availableParallelism()` to `parseRunnerArgs`, which makes the default one job per available CPU. +That much already behaves: `availableParallelism()` in bun 1.4.2 follows CPU affinity and a cgroup v2 CPU quota, measured as 2 under `taskset -c 0-1` and 2 under `systemd-run --property=CPUQuota=200%`, so a container with a CPU limit is sized by it. +A container with a MEMORY limit and no CPU limit on a many-core host is not, and that is the case that fails: `docker run --memory=2g` on a 64-core host starts 64 jobs. + +What a job costs, measured over a 32-file sample run one at a time under `/usr/bin/time`, in peak RSS: minimum 58 MiB, median 100 MiB, p90 199 MiB, maximum 341 MiB. +Under real concurrency the files do not peak together, so the marginal cost is lower: the runner over `tests/components` peaked at 599 MB with 4 jobs, 998 MB with 8 and 1599 MB with 16, a slope of about 80 to 90 MiB per extra job over a fixed 300 MB. +The largest run actually made was 16 jobs, so 64 jobs is 5 to 6 GiB extrapolated from that slope rather than measured, and 12.4 GiB if every file peaked at the p90 bound at once, against a 2 GiB limit. +Under Kubernetes or systemd the whole run is then killed rather than one file: measured before this branch handled SIGTERM, a run stopped by systemd's default `OOMPolicy` ended at exit 143 with no summary and its scratch directory left behind, and it now ends at the same 143 with `Interrupted (SIGTERM).`; under Kubernetes's `memory.oom.group` the kernel SIGKILLs the runner too, so nothing is printed at all (reasoned, not measured). +Neither shape names the file that ran the container out of memory, which is what a memory-aware default would prevent rather than explain. + +Which API can carry the limit was measured too, and only one of the three can. +`process.constrainedMemory()` follows a cgroup v2 `memory.max` (2147483648 under `MemoryMax=2G`) and equals `os.totalmem()` when there is no limit, which makes it usable with no fallback branch. +`process.availableMemory()` does NOT: inside the same 2 GiB scope it returned the host's 34 GB, unlike node 24, which follows the cgroup there. +`os.freemem()` is not a budget at all, since it moves with unrelated load and leaves out reclaimable page cache. +Not measured: what `constrainedMemory()` returns on macOS and on Windows, where there is no cgroup for bun to read; reasoned, it should be total RAM, and that is what the entry rests on. + +A fixed cap is the wrong shape and was rejected: `Math.min(cpuCount, 16)` still needs 1.6 to 3.1 GiB inside a 1 GiB container, and it caps a workstation on a constant nobody measured against memory. +What this PR did instead is make the failure readable: a file killed by SIGKILL from outside the runner now names the OOM killer and `--jobs=N` in its reason, `CONTRIBUTING.md` says when to pass it, and `docs/TOOLCHAIN.md` carries these numbers. + +The shape it would take: `parseRunnerArgs`'s injected context grows from `{ cpuCount }` to `{ cpuCount, memoryBytes }`, `tests/run-tests.ts` passes `process.constrainedMemory()`, and the default becomes +`Math.max(1, Math.min(cpuCount, Math.floor(memoryBytes / JOB_MEMORY_BUDGET_BYTES)))`. +A budget of 256 MiB is the one this measurement supports: above the p90 per-file peak of 199 MiB and about three times the concurrent slope. +A 2 GiB limit would then give 8 jobs, where 8 measured 940 MiB of anonymous memory, and the measured host, whose `constrainedMemory()` is 67,118,133,248 bytes (64 GB, 62.5 GiB), would give 250, so the CPUs stay the binding constraint everywhere else. +An explicit `--jobs=N` must still win over it, and the budget is a constant that drifts as the suite grows, so its docblock has to carry the basis above. + +**Done when:** `parseRunnerArgs` takes a memory budget beside the CPU count, `tests/unit/test-runner-options.test.ts` pins the four cases (64 CPUs with 2 GiB gives 8, 8 CPUs with 64 GiB gives 8, 4 CPUs with 100 MiB gives 1 and never 0, and an explicit `--jobs=32` wins over all of it), and a CI run on macos-latest and windows-latest has printed `process.constrainedMemory()` against `os.totalmem()` so the unmeasured half of the premise is measured rather than reasoned. + +### D96. bun 1.4.2 drops part of a child's own console output when the child exits under load + +Measured 2026-09-15 on Linux x64 with 20 cores and bun 1.4.2, while reviewing #837. +A test file that prints a megabyte and then fails does not always get that megabyte to whoever is reading the run: the bytes are lost by the child `bun test` process at its own exit, before anything the runner can drain. +A fixture printing 1024 lines of 1023 bytes, run 20 times with the machine deliberately loaded, lost output in 15 of the 20 runs and delivered as few as 182 of the 1024 lines; unloaded, 10 of 10 runs were whole. +The loss is not the runner's pipe: with the runner's own stdout redirected to a FILE, 3 of 10 loaded runs still lost 15 to 40 per cent of the file's output, and with no runner in the picture at all, `bun test ./fixture.test.ts 2>/dev/null | cat > out` under load delivered 126 of 1024 lines in 1 of 10 runs. + +What the runner does guarantee is its own last lines: the summary, the `Failed files:` block and the re-run hint are written through a drain that waits for the bytes to leave the process, and those survived every one of those runs. +So the cost is a contributor reading a red CI log from a busy machine and getting a truncated failure diff under an accurate verdict, not a wrong verdict. +`tests/unit/test-runner-cli.test.ts` states this where it would otherwise be tempting to assert the whole output back: its megabyte case asserts the verdict, the summary and that the file's output reached stdout at all, and says in a comment why it cannot assert the line count. + +There is nothing to fix inside this repository: the queue that is dropped belongs to the child process. +What can be done is to re-probe, and to stop the claim drifting back to "whole output" in the meantime. + +**Done when:** the focused repro has been run against a bun newer than 1.4.2 under the same load, and either it is whole 10 times out of 10 and this entry closes, or the entry names the newest version it still reproduces on and is reported upstream. + +### D97. A committed `.only` makes a file report PASS with the rest of its tests never run + +Measured 2026-09-15 on bun 1.4.2, while reviewing #837. +bun honours `.only` by default, and nothing in the runner, the lint configuration or the required checks refuses one that reaches `main`. +A fixture holding `it.only`, a failing `it`, a `describe.todo` and a `describe.concurrent` with two more tests wrote a junit report of `tests="1" failures="0"`, exited 0, and the runner printed `PASS 0.0s tests/unit/only.test.ts 1 pass`; the same file without the `.only` registers five tests. +So four registered tests, one of them failing, are absent from the report, from the run's totals and from CI's verdict, and the run is green. + +The runner cannot close this from the report it reads, which is why `toOutcome`'s docblock now names `.only` as the shape the report cannot see. +bun's report is honest about the one test it ran; the file that should have been refused is the one on disk. +It has to be refused before the run, and there are two cheap shapes: an `eslint-plugin-no-only-tests` rule (or oxlint's `jest/no-focused-tests`) scoped to `tests/**` and `e2e/**`, or a grep over the same paths inside the required `Lint, Typecheck and Build` check, which costs one command and no new dependency. +The coverage gate is not a reliable second line of defence either: whether it goes red depends on which lines the unrun tests were the only cover for, which is a property of the file rather than of the `.only` (reasoned, not measured). + +**Done when:** a file carrying `it.only`, `test.only` or `describe.only` under `tests/` or `e2e/` fails a required check, and a test pins that gate by driving it over a fixture that carries one, with a control fixture that does not and passes. + ## Value interpolation @@ -2746,3 +2851,43 @@ capability object cost nothing, and now the request SHAPE is derived from it. **Done when:** no read is issued for a connection whose declaration has not arrived, proven by a test that switches between two engines of different depth and asserts what was posted. + +### B81. A failure nobody attributed leaves the browser asserting that the server serves no seeds + +B37 landed and left this file; its id survives in the comments on `src/hooks/use-connection-payload.ts` +and `src/hooks/use-connection-manager.ts`, which is where the reasoning below can be read against the +code. It gave `ServedSeeds` a way to say "I do not have the seed list", and then gave the state an +initial value of `{loaded: true, seeds: []}` under the name `NO_SERVED_SEEDS`, commented as +"loaded, and genuinely empty". Before the first answer arrives nothing has been measured, so that +value is a claim the browser is not entitled to, and it is the claim B37 was filed about. + +Measured on 2026-09-15 by driving the whole path with a gateway error page, the shape +`tests/hooks/use-connection-manager.test.ts` already pins as intended: +`GET /api/connections/managed` answers 502 with `text/html`, no `reason` is read from the body, so +`setServedSeeds` is never called and the state is still the module constant by identity. +`initializeConnections` then falls back to `storage.getConnections()`, the user's editable seed copy +renders, and `resolveAgentRunConnectionId` answers `{id: null, reason: "browser-only"}`. The rail +says, of a connection this application seeds itself: + +> Sample (Employees) cannot be rebuilt on the server: its settings live in this browser. + +That is B37's sentence, false in both halves, reached through a proxy instead of a malformed +`seed-connections.yaml`. The hook's own comment says such a failure "says nothing" about the seed +configuration; leaving the state at `{loaded: true, seeds: []}` is not saying nothing, it is saying +the list is empty. + +The 404 arm is right by accident rather than by design: where the route does not exist at all, as in +the platform embed, there is no seed service and no seeds, so "loaded, empty" is the true answer. +Only a third state can hold both that and "asked, and the answer told me nothing". + +Two tests on the same subject cannot see this, and one of them is the reason it reads as deliberate. +`tests/hooks/use-connection-manager.test.ts` waits on `connections` reaching `[]` and then asserts +`servedSeeds` equals `{loaded: true, seeds: []}`, for the 404 arm and for the 502 arm. Both are the +initial values, and with an empty `localStorage` neither moves on either path, so both tests pass +against a hook that never issues the request. They pin the initial state under the name of a +measured one. There is no honest barrier to wait on there while the settled value and the unasked +value are the same object shape, which is the same defect one level up. + +**Done when:** an unasked seed list is distinguishable from a measured empty one, a non-OK the +server did not attribute leaves the browser in the unasked state rather than the empty one, and the +two tests above wait on a fact that a hook which never fetched cannot satisfy. diff --git a/docs/TOOLCHAIN.md b/docs/TOOLCHAIN.md index ee5f1a92a..0938e4647 100644 --- a/docs/TOOLCHAIN.md +++ b/docs/TOOLCHAIN.md @@ -29,7 +29,7 @@ license, etc.): | Linting today | oxlint + type-aware-only ESLint | `eslint-config-next` (core-web-vitals + typescript + react-hooks) | | Formatter today | Biome (present) | None (no prettier) | | knip | present | present (in CI gate) | -| Tests | single `bun test` | process-isolated (`run-core.sh` / `run-components.sh`) to avoid `mock.module()` cross-contamination | +| Tests | single `bun test` | `tests/run-tests.ts`: one bun process per test file, several at a time, to avoid `mock.module()` cross-contamination | Consequences: @@ -227,18 +227,35 @@ and the type-aware layer via `bun run lint`. ### `bun run test` and the process-wide `mock.module()`, and where isolation has to sit -`bun test` runs many files in ONE process and `mock.module()` is process-wide, which is why -`tests/run-core.sh` gives every core file its own process and `tests/run-components.sh` groups the -component files. CI runs both, so CI is structurally blind to a file that only passes when it loads -a module first. `bun run test` is not: it is the command CLAUDE.md documents for pre-commit, it runs -`bun test tests/unit tests/api tests/integration` in one process, and a contributor reads its -failures as their own. - -That is not a reason to accept a red developer command. A test file that breaks it is a defect -whether or not a gate notices, and the repair is to move the file whose assumption is unshareable -rather than to bend the files around it. - -Two instances measured in #789, and the rule they establish. +`mock.module()` is process-wide with no undo, so a mock one layer installs reaches every file that +shares its process. `bun run test` is `bun tests/run-tests.ts`, which discovers every test file under +`tests/` except `tests/live/` and runs EACH ONE in its own bun process, several at a time, so no +file is ever reached by another file's mock. That is the whole reason the runner spawns a process +per file instead of handing a directory to `bun test`; the decision is argued in +`tests/runner/execute.ts`. + +It used to be the other way round, and the cost was paid by contributors rather than by a gate. +`bun run test` ran `bun test tests/unit tests/api tests/integration` in ONE process while CI ran +the now-deleted `tests/run-core.sh`, one process per file, so the command CLAUDE.md documents for +pre-commit was red on a clean checkout and no gate could see it: 12849 pass, 42 fail, exit 1 on the +tree this change was written against, all of it one layer's mocks reaching a sibling that wanted the +real module. Those 42 are gone because the documented command and the gate now run the same way, not +because any mock was repaired. What the mocks are still wrong ABOUT is `docs/BACKLOG.md` D85. + +Two bun options were measured on this suite and are NOT used. `bun test --isolate` (1.4.2) resets +the module registry per file inside one process, which would let the runner start far fewer +processes, and it does contain `mock.module` - but it is the subject of +[oven-sh/bun#41655](https://github.com/oven-sh/bun/issues/41655), a NAPI finalizer SIGSEGV that +reproduces serially on 1.4.2, and this suite loads three NAPI addons (`better-sqlite3`, `oracledb`, +`@duckdb/node-api`). `bun test --parallel` gave 0 fail in seven runs, then one run that hung for 17 +minutes inside a synchronous helm spawn and one that failed 7 tests of +`tests/unit/docker-bind-address.test.ts` on 5000 ms timeouts; and +[oven-sh/bun PR 41467](https://github.com/oven-sh/bun/pull/41467), which stops a crashed worker +leaking its subprocesses, is merged but in no release, so 1.4.2 leaks them. A process boundary needs +no upstream fix. Re-probing `--isolate` when 41655 closes is `docs/BACKLOG.md` D86. + +The process boundary is now structural, but the two cases it was first built by hand for are still +the clearest statement of what it buys, so both measurements stay here. The first is a file that must load a module before anything else does: @@ -256,13 +273,9 @@ The first is a file that must load a module before anything else does: empty probe reproduces nothing. Both CLI orders give the same 56, because bun does not run test files in the order they are listed. - It used to hold by accident: nothing else under `tests/unit` imported the factory. - `tests/isolated/exports-shim.test.ts` had already been moved out for the same reason, and its - group comment names this file by name. #789 added two `tests/unit` files that construct all - seventeen providers through the real factory, and a fleet census cannot do its job without - importing it, so the accident ran out. -- The file therefore moved from `tests/unit/db/factory.test.ts` to `tests/isolated/factory.test.ts` - with its own group in `tests/run-components.sh`. `tests/unit/component-runner-coverage.test.ts` - makes an unregistered file in `tests/isolated/` a red test, so the isolation cannot be forgotten. + `tests/isolated/exports-shim.test.ts` had already been moved out for the same reason. #789 added + two `tests/unit` files that construct all seventeen providers through the real factory, and a + fleet census cannot do its job without importing it, so the accident ran out. The second is the mirror image: a file that must read a module the rest of a layer replaces. @@ -276,17 +289,88 @@ The second is the mirror image: a file that must read a module the rest of a lay `@/lib/db/factory` through the index re-export. Measured 2026-09-13: the census beside `tests/api/db-objects.test.ts` is 3 fail, the language guard beside it is 1 fail, and each of them alone is 0 fail. -- Nothing either file can do prevents that, so both moved to `tests/isolated/` with a shared group. -A pre-existing instance of the same class is NOT fixed and is filed as `docs/BACKLOG.md` D68: the -same `tests/api/` mocks of `@/lib/auth` take `tests/unit/lib/auth.test.ts`, -`tests/unit/lib/auth-jwt-config.test.ts` and `tests/unit/seed/resolve-connection.test.ts` from 0 to -31 failures in a shared process. Measured identical at `acf50738` and on the #789 branch, so it -predates the epic. - -The rule: when a test file can only pass while it is the first to load some module, it belongs in -`tests/isolated/` with a group of its own and a docblock saying which module and what the failure -looks like. Do not push the constraint outward onto every file that might legitimately import it. +Moving those files into `tests/isolated/` was the only way to get each a process of its own while a +runner named its groups by hand. It is no longer what protects them, and no new file needs +that treatment: the directory keeps its name and its files because `docs/SECURITY.md`, +`sonar-project.properties` and several source comments cite the paths, not because the runner treats +it specially. + +The rule the two cases leave: a test file whose assumption is unshareable - it has to be the first to +evaluate some module, or it has to read a module a whole layer replaces - says so in its own +docblock, naming the module and what the failure looks like. It needs no directory and no +registration, because the runner already gives it a process. What is still not allowed is the +reverse repair: pushing one file's constraint outward onto every file that might legitimately import +the module. + +A test file that needs a tool a contributor may not have says so on its first line, and today there is one such tool. +The twelve chart tests that drive the real `helm` binary open with `// @requires helm`, and `tests/runner/requirements.ts` reads the marker before it starts a file. +Where `helm` is missing, or the chart's PostgreSQL subchart is not built, those files are not started, and the summary names them once under the reason and the command that fixes it; a selection made only of such files is an error, not an empty green run. +Every CI job that runs the suite sets `LIBREDB_REQUIRE_HELM=1`, which makes the same condition stop the run before anything starts, and `tests/unit/helm-pin-matrix.test.ts` fails if one of those jobs loses the variable. +The decision is per file rather than per test because each of those files needs Helm for everything it does, so skipping inside them would print about 180 test titles where twelve file names say the same. +Leaving them out costs no line coverage, because what they exercise is the chart's templates, which no lcov measures: measured on 2026-09-15 with `helm` hidden from `PATH`, 531 of the 543 files the tree held then ran, and the merged report was still 100% of its lines. +`tests/unit/test-runner-requirements.test.ts` holds the marker true of the tree in both directions: a file that spawns helm carries it, and a file that carries it spawns helm. + +### What a file's verdict is read from, and how a run ends + +A file's counts come from the junit report bun writes for it, never from its console output. +Every child is spawned with `--reporter=junit --reporter-outfile=/file-N.xml`, and those two options are appended AFTER any argument forwarded past `--`, because bun takes the last of a repeated option: a forwarded `--reporter-outfile` would otherwise redirect the report and leave every file looking as though it wrote none. +Reading the console is what the runner did before, and free-form text mixed with whatever the tests printed can be made to say anything: measured on bun 1.4.2, a file that registered no test and printed the line ` 1 pass` was reported PASS with exit 0, and so was a test that printed a whole summary block on stderr and then called `process.exit(0)` so the tests after it never ran. +Both shapes defeat exactly the two guards that keep a red tree from turning green, "printed no summary" and "registered nothing". +`--bail` is the same defect from the other side: it prints no count line at all, while the report still carries the failure. +A report that is absent and one the parser cannot read are told apart, and neither ever becomes zero counts: the file fails, its line reads "no test report" or "unreadable test report", and the summary says how many files left no readable report and that their tests are not in the totals above it. +There is one shape no report can show, and the runner says so rather than pretending otherwise: bun honours a committed `.only`, so such a file writes an honest report naming that one test and exits 0, and the tests it never ran are absent from the report, the totals and the verdict alike (measured on 1.4.2). +That has to be refused before the run rather than read out of what the run wrote, and nothing refuses it today: `docs/BACKLOG.md` D97. + +Forwarding a flag to `bun test` works only when the runner is invoked directly and a selector comes first. +Measured on 1.4.2: `bun run test -- --bail` reaches the script as `["--bail"]`, because `bun run` removes the first `--`, and bun removes one that sits straight after the script path too, so `bun tests/run-tests.ts -- --bail` loses it as well. +`bun tests/run-tests.ts tests/unit -- --bail` is the form that arrives whole, and it is what the runner's unknown-option error names when it refuses a flag it does not own. + +The titles of the tests a file skipped come from the same report, and their describe path from its nested `` elements rather than from the `classname` attribute. +classname lists those titles too, but bun joins them with " > " and writes a literal ">" inside a title as ">" as well, so no split rule can tell the separator from the character: measured on 1.4.2, splitting classname turned a describe titled "rows where count > 100" into "100 > rows where count". +The titles are worth printing at all because a skip in this repository states its reason in its title, and bun prints that title nowhere: piped, with `FORCE_COLOR` set, and under a real pty, the output carries the count and nothing else. +A todo is in neither the count nor the list, although bun writes one as `` as well: a todo has no reason to state and is already its own column in the totals line, and listing one under an "(N skipped)" header that does not count it would print two different numbers for one block. +A file whose report could not be read still prints the titles the parser reached before it stopped, under "unreadable report; it named N skipped tests" instead of a count it does not have. + +Each child's stdout and stderr are captured rather than inherited, because several files run at once and interleaved output belongs to nobody, and each stream is bounded at BOTH ends by `tests/runner/capture.ts`: the first megabyte, the last megabyte, and one line naming how many bytes fell between them. +Both ends are kept because both are read, the head for bun's file header and the first failure diff, the tail for the rest of the diffs and bun's own per-file summary. +The megabyte is measured against this tree rather than guessed: across the 543 files the tree held when the measurement was taken on 2026-09-15, the largest prints 154,526 bytes on stdout, the largest stderr is 48,369 bytes and the median is 104 bytes, so nothing that runs here today is ever cut, and the worst case per running child is 4 MB. +Nothing is decided from that text, so a cut can never change a verdict. +A passing file's output is dropped once its line has been printed, which is what stops a whole run's output adding up: held to the end, four passing files printing 100 MB each peaked at 406 MB of RSS against 249 MB for one. + +Every exit path writes its last line and waits for the bytes to leave the process before it calls `process.exit`. +bun writes to a pipe asynchronously and `process.exit` throws away whatever is still queued: measured on 1.4.2 through a piped stdout, a run whose failing file printed a megabyte lost about a third of that output and the whole summary with it, "Failed files:" and the re-run hint included, while the exit code stayed 1. +The wait has to be a real write whose callback resolves, because an empty write's callback does not wait for the queue and a `drain` event never arrives: `write()` returned false while `writableLength` was 0 and `writableNeedDrain` was false. +A non-empty write's callback does wait for everything queued before it, measured at 1 MB and at 10 MB and against a reader that started 1.5 seconds late, so one such write also drains the lines printed as the files landed. +What that covers is the lines this process writes: the per-file lines, the summary, the failed-file block and the re-run hint. +It does not cover a child's own console output, and nothing here can: measured under CPU load on 1.4.2, a failing file's `bun test` process drops part of its queued stdout as it exits, between 20 and 90 per cent of a megabyte, and it does so with no runner in the picture at all, so a failure diff read from a busy CI machine can still be truncated under an accurate verdict (`docs/BACKLOG.md` D96). +A write that fails because the reader has gone rather than because this process could not write is not the runner's problem and does not become its exit code: an `EPIPE` from `| head -1` or a closed terminal leaves the run's own 0, 1 or 128 plus signal in place, and exit 2 stays for a write the runner really could not make, a full disk for instance (measured against `/dev/full`, which reports `ENOSPC` and does exit 2). + +SIGINT, SIGTERM, SIGHUP and SIGBREAK all end a run the same way: no further file is started, the files still running are killed, the scratch directory is removed, `Interrupted (SIGNAL).` is written, and the runner exits 128 plus the signal's number, so 130, 143 and 129, measured end to end for those three, and 149 for SIGBREAK, which only Windows can deliver and which is therefore pinned over the mapping rather than over a run. +The number is taken from the platform's own `os.constants.signals` where the platform names the signal, because that is the number its shell will report, and from the table written out in `tests/runner/signals.ts` where it does not: measured on 1.4.2, that table has no SIGBREAK on Linux or macOS, and `128 + undefined` is NaN, which `process.exit` refuses with a RangeError thrown from inside the listener. +SIGTERM is what `timeout(1)`, `docker stop`, Kubernetes and systemd send, and what `bun run test` forwards; SIGBREAK is Ctrl+Break, which GitHub Actions on Windows sends 7.5 seconds after Ctrl+C, and naming it is valid on every platform. +Handling only SIGINT, as this did, meant a run stopped any other way printed nothing at all and left its junit scratch directory behind in the temporary directory. +A scratch directory that cannot be removed is named on stderr and exits 2, neither swallowed nor thrown: measured on 1.4.2, a throw from inside a signal listener left the process RUNNING and the queue started the next file. +The default disposition is put back before the handler awaits anything, so a second signal kills the process outright instead of starting a second cleanup over the first, and the last write is raced against a three-second grace. +That grace is there because a reader that has stopped reading blocks the write behind a full pipe: measured on 1.4.2, the process then sat out the whole stall and survived a second SIGINT, a SIGTERM and a SIGHUP. +When the grace runs out the run still ends with its own exit code and its scratch directory gone, and the only thing lost is the `Interrupted` line, which the reader that was not reading would not have seen anyway. + +Discovery refuses a symbolic link or junction under `tests/` by name instead of walking through it. +It used to skip one in silence, because a `readdirSync` Dirent for a link reports neither `isDirectory()` nor `isFile()`, so a linked directory and a file link named `*.test.ts` both fell out of the selection with nothing printed; entries are classified with `lstat` now, which also reports a Windows junction as a link. +Refusing rather than following is the other half of that decision: following a link can run files from outside the repository, loop on a link to a parent, or list one file twice under two names, while skipping it drops its tests without a word. + +The default concurrency is one job per available CPU, and it is not memory-aware. +`availableParallelism()` in bun 1.4.2 follows CPU affinity and a cgroup v2 CPU quota, measured on a 20-core host: 2 under `taskset -c 0-1`, and 2 under `CPUQuota=200%`. +So a container with a CPU limit already gets as many jobs as it has CPU, and a laptop, a Codespace and a GitHub-hosted runner are all sized by that. +What the default does not read is a memory limit. +A test file's peak RSS over a 32-file sample runs from 58 MiB to 341 MiB with a median of 100 MiB, and under real concurrency the heaviest layer adds about 80 to 90 MiB per extra job: `tests/components` measured 599 MB at 4 jobs, 998 MB at 8 and 1599 MB at 16. +In a container with a memory limit and no CPU limit on a many-core host, pass `--jobs=N`. +A run that did not is readable rather than mysterious in the one case where the kernel kills a single child: that file's line names the OOM killer and `--jobs=N` in its reason, because the runner sends SIGKILL itself only to a child that outran its budget, and that child is reported as timed out instead. +Where the cgroup kills the whole group the runner dies with its children, so that reason is never printed and no file is named at all. +Under Kubernetes's default `memory.oom.group` the kernel SIGKILLs every process in the cgroup, and SIGKILL cannot be handled, so the output simply stops after the files that had already landed and the exit code is the only signal: reasoned from the kernel's semantics, not measured here. +Under systemd's default `OOMPolicy` the unit is stopped with SIGTERM instead, which this runner now takes: measured before that handling existed, such a run ended at exit 143 with no summary and its scratch directory left behind, so it now ends at the same 143 with `Interrupted (SIGTERM).` and nothing left in the temporary directory. +Neither shape names the file that exhausted the memory, which is the second reason a memory-aware default is worth having. +That default is `docs/BACKLOG.md` D95. ### Dependency installation in CI @@ -365,6 +449,15 @@ The merged `coverage/lcov.info` sits at 100% lines (#192/#195/#196) and CI enfor `scripts/check-coverage.mjs` fails the `Unit & Integration Tests` job on any zero-hit DA record, printing the uncovered file:line ranges. Local check: `bun run test:coverage && bun run coverage:check`. +`bun run test:coverage` is the same runner as `bun run test` with `--coverage +--merge-into=coverage/lcov.info`: one lcov per TEST FILE under `coverage/raw/`, merged by +`scripts/merge-lcov.mjs` at the end. Measured 2026-09-15 on Linux, 8 files at a time: about 60 +seconds, and the merged report is 100% of 57157 lines. Two mechanics of that merge exist for +Windows: the report list is handed over as a manifest (`--inputs-from=`) because 500-odd paths +do not fit in a Windows command line, and `merge-lcov.mjs` normalises a backslash `SF:` path, so +coverage produced on Windows merges as the same file as coverage produced on Linux instead of as a +second file the `src/` filter then drops. + Holding 100% honestly requires knowing how bun measures: 1. **Per-function granularity (V8 semantics).** Functions that executed in a process get a precise @@ -374,17 +467,25 @@ Holding 100% honestly requires knowing how bun measures: process does. 2. **Authority-universe merge** (`scripts/merge-lcov.mjs`, tested in `tests/unit/merge-lcov.test.ts`): per file, the record with the most executed lines decides which lines are coverable; per-line hit - counts still take the max across all records, so secondary groups (e.g. the mobile-drawer group of - a desktop-rendered component) keep contributing. Without this rule, load-only records surface - phantom uncovered lines that no test can ever close. -3. **`run_group --nocov`** (`tests/run-components.sh`): groups that import without exercising (the - exports CJS shim pulls the whole component chain) run without coverage collection entirely. + counts still take the max across all records, so a secondary record (e.g. the mobile-drawer test + of a desktop-rendered component) keeps contributing. Without this rule, load-only records surface + phantom uncovered lines that no test can ever close. Per-file processes made this rule carry MORE + than it used to: a run merges one report per test file rather than one per hand-written group, so + a source file is now described by every test file that so much as imports it, and the load-only + records among them outnumber what a grouped run produced. The max-hit record is what keeps those + extra descriptions from adding uncoverable lines. +3. **`COVERAGE_EXEMPT_FILES`** (`tests/runner/discover.ts`): the two files that pull in a module + chain without exercising it - the exports CJS shim loads every component, and the Monaco loader + file imports the editor to observe a call it makes at module scope - run without coverage + collection entirely. The docblock beside the list is where the reason lives, with what the shim + alone would otherwise contribute (31 phantom uncovered lines in `src/lib/llm/factory.ts`). 4. **Non-executable-line strip** (`merge-lcov.mjs`): bun emits DA records for blanks, comments, and bare punctuation; these are removed against the actual source before SonarCloud reads the report. 5. **Diagnosis recipe:** when a file shows stubborn uncovered lines, compare its records across - `coverage/components/group-*/lcov.info` and `coverage/core/file-*/lcov.info`. If the zero lines - are absent from the record with the most hits, they are measurement phantoms (fix the merge - inputs), not test gaps. + `coverage/raw/file-*/lcov.info` - one directory per test file, numbered by that file's position + in the sorted selection, so line N of `bun tests/run-tests.ts --list` is what wrote `file-N`. If + the zero lines are absent from the record with the most hits, they are measurement phantoms (fix + the merge inputs), not test gaps. 6. **Mock fidelity over stubs:** hover/portal-dependent branches are closed by making test mocks honor the real library contract (recharts `Tooltip` renders its `content` element with an active payload; the select mock drives `onValueChange` through clickable items; dropdown items honor diff --git a/docs/providers/duckdb.md b/docs/providers/duckdb.md index 1b6ae3d0d..dddc553a2 100644 --- a/docs/providers/duckdb.md +++ b/docs/providers/duckdb.md @@ -305,9 +305,9 @@ rather than a gap. | Scenario | Measured | |---|---| -| Second read-write `DuckDBInstance.create` on the same file, **same process** | ALLOWED | +| Second read-write `DuckDBInstance.create` on the same file, **same process** | ALLOWED on Linux and macOS, REFUSED on Windows (see below) | | `DuckDBInstance.fromCache` on the same file, same process | ALLOWED | -| Second `access_mode: 'READ_ONLY'` instance, same process, while a writer is open | ALLOWED, and genuinely read-only — `current_setting('access_mode')` is `read_only`, `duckdb_databases().readonly` is true, `INSERT` is refused | +| Second `access_mode: 'READ_ONLY'` instance, same process, while a writer is open | ALLOWED on Linux and macOS, and genuinely read-only: `current_setting('access_mode')` is `read_only`, `duckdb_databases().readonly` is true, `INSERT` is refused | | Second read-write **process** while a writer holds the file | `IO Error: Could not set lock on file …: Conflicting lock is held in … (PID nnn)` | | Second **READ_ONLY process** while a writer holds the file | **ALSO refused**, with the same lock error | | `READ_ONLY` open of a file that does not exist | `IO Error: Cannot open database … in read-only mode: database does not exist` — the engine does not create it | @@ -317,8 +317,24 @@ is stricter than the usual one-writer-many-readers summary, so `singleWriterFile conservative default here — it is the measurement. Two Studio replicas sharing a file is not a supported deployment. -The same table is why the agent's read-only handle works at all: it is a second handle **in the same -process** as the writer, and same-process handles are permitted. +The same-process rows are why the agent's read-only handle can sit beside an editor handle at all: +it is a second handle **in the same process** as the writer. + +**That allowance is POSIX's, not the engine's.** Measured on windows-latest (2026-09, the same +DuckDB v1.5.5 / `@duckdb/node-api` 1.5.5-r.4): a second handle on a file this process already holds +is refused at the operating system, `IO Error: Cannot open file "…": The process cannot access the +file because it is being used by another process`, with DuckDB naming this very process as the +holder. Windows arbitrates the share mode per HANDLE, so "the lock is per process" simply does not +apply there. `tests/integration/db/duckdb-provider.test.ts` asserts both answers, each positively, +rather than the POSIX one twice. + +The consequence for the product is bounded but real, and it is not fixed here: the editor borrows +the open handle rather than opening a second one (`findOpenSingleWriterProvider`), so ordinary +browsing is unaffected on every platform. The one path that really does want two handles at once is +an agent run reaching a connection the editor already has open, `acquireExecutionProfileProvider` +opens the file under the profiled key while the writable handle is live. On Windows that open is +refused, and the run fails with the engine's sentence instead of reading. Not yet measured against a +running Studio on Windows, only against the engine. ### 3.9 `interrupt()` exists, so `cancelQuery` is real diff --git a/docs/providers/mongodb.md b/docs/providers/mongodb.md index 2487275d6..065881365 100644 --- a/docs/providers/mongodb.md +++ b/docs/providers/mongodb.md @@ -1005,11 +1005,9 @@ mock collection/cursor/admin returns canned documents and stats, exercising ever serialization, schema inference, monitoring, and maintenance. > ⚠️ **Mock isolation:** `bun`'s `mock.module()` is process-wide; files mocking different drivers -> cross-contaminate in a shared process. CI runs the full suite via **`bun run test:ci`** (per-file -> process isolation via `tests/run-core.sh`) and **`bun run test:coverage`** for determinism. The -> `bun run test` pre-commit gate (per [`CLAUDE.md`](../../CLAUDE.md)) also works — it isolates the -> component group — but runs the core group in a single process, so prefer `test:ci` when isolation -> matters. Running a single file alone is always safe. +> would cross-contaminate if they shared one. They never do: `bun run test` gives every test file its +> own bun process, so a single file is safe and so is the whole suite, which is the same command CI +> runs. `bun run test:coverage` is that runner with coverage on. See [`CLAUDE.md`](../../CLAUDE.md). ### Coverage @@ -1053,7 +1051,7 @@ the engine re-measurable; [§6](#the-object-surface-789) says which claim each o ```bash bun test tests/integration/db/mongodb-provider.test.ts # just this file -bun run test:ci # CI publish gate +bun run test # the whole suite, one process per file bun run test:coverage # CI coverage workflow ``` diff --git a/docs/providers/mssql.md b/docs/providers/mssql.md index 4b2603f8b..b03fb9370 100644 --- a/docs/providers/mssql.md +++ b/docs/providers/mssql.md @@ -1341,10 +1341,9 @@ provider is imported — there is no live SQL Server in the suite. The mock's po canned `{ recordset, rowsAffected }` results, exercising the same code paths as the real driver. > ⚠️ **Mock isolation:** `bun`'s `mock.module()` is process-wide; files mocking different drivers -> cross-contaminate in a shared process. A **single file** is safe (one file = one process). The -> full `bun run test` script runs the core group in **one** process and is load-order flaky, so -> **CI does not use it** — the deterministic runner is **`bun run test:ci`** (per-file isolation via -> `tests/run-core.sh`); the coverage workflow uses `bun run test:coverage`. See [`CLAUDE.md`](../../CLAUDE.md). +> would cross-contaminate if they shared one. They never do: `bun run test` gives every test file its +> own bun process, so a single file is safe and so is the whole suite, which is the same command CI +> runs. `bun run test:coverage` is that runner with coverage on. See [`CLAUDE.md`](../../CLAUDE.md). ### 12.2 Coverage @@ -1365,8 +1364,8 @@ a mock that filtered on the bind kept passing for a listing that had lost its `W ```bash bun test tests/integration/db/mssql-provider.test.ts # just this file (single process — safe) -bun run test:ci # CI publish gate — per-file isolation (tests/run-core.sh) -bun run test:coverage # CI coverage workflow — per-file core + components +bun run test # the whole suite, one process per file, what CI runs +bun run test:coverage # CI coverage workflow: the same runner, with coverage ``` ### 12.4 Optional: verifying against a live SQL Server diff --git a/docs/providers/mysql.md b/docs/providers/mysql.md index 4e4474e9f..4dfa3c43d 100644 --- a/docs/providers/mysql.md +++ b/docs/providers/mysql.md @@ -716,11 +716,11 @@ exclusion cannot be added without saying why. `tests/live/mysql-object-vocabular real server for its own `SELECT DISTINCT TABLE_TYPE` and `SELECT DISTINCT ROUTINE_TYPE` and exits non-zero NAMING any value outside modelled-plus-excluded. -**Where it runs.** It is a live check, so it is not in `bun run test` or `bun run test:ci`: -`tests/run-core.sh` globs `tests/unit tests/api tests/integration tests/hooks tests/security -tests/evals`, and nothing under `tests/live/` is collected, the same arrangement -`tests/live/schema-diff-dialects.ts` has. It runs by hand against a disposable server, and belongs -permanently in #789's live acceptance run: +**Where it runs.** It is a live check, so it is not in `bun run test`: the runner collects every +`*.test.ts` / `*.test.tsx` file under `tests/` except the ones in `tests/live/`, which it excludes by +name (`EXCLUDED` in `tests/runner/discover.ts`). This file is outside that set twice over, by its +directory and by its name, the same arrangement `tests/live/schema-diff-dialects.ts` has. It runs +by hand against a disposable server, and belongs permanently in #789's live acceptance run: ```bash LIBREDB_LIVE_MYSQL_URLS="mysql://root:root@127.0.0.1:3306/app,mysql://root:root@127.0.0.1:3307/app" \ @@ -1607,10 +1607,9 @@ pins the method for `getHealth`, `getOverview`, `getPerformanceMetrics`, the obj parameters), the Explain statement `mysqlJsonStrategy` builds, and the transaction path. > ⚠️ **Mock isolation:** `bun`'s `mock.module()` is process-wide, so files mocking different drivers -> cross-contaminate when they share a process. A **single file** is safe (one file = one process). -> The full `bun run test` script runs the core group in **one** process and is load-order flaky, so -> **CI does not use it** — the deterministic runner is **`bun run test:ci`** (per-file isolation via -> `tests/run-core.sh`); the coverage workflow uses `bun run test:coverage`. See [`CLAUDE.md`](../../CLAUDE.md). +> would cross-contaminate if they shared one. They never do: `bun run test` gives every test file its +> own bun process, so a single file is safe and so is the whole suite, which is the same command CI +> runs. `bun run test:coverage` is that runner with coverage on. See [`CLAUDE.md`](../../CLAUDE.md). ### 12.2 Coverage @@ -1641,8 +1640,8 @@ declarations were then re-measured end to end against live containers, `mysql:la ```bash bun test tests/integration/db/mysql-provider.test.ts # just this file (single process — safe) -bun run test:ci # CI publish gate — per-file isolation (tests/run-core.sh) -bun run test:coverage # CI coverage workflow — per-file core + components +bun run test # the whole suite, one process per file, what CI runs +bun run test:coverage # CI coverage workflow: the same runner, with coverage ``` ### 12.4 Optional: verifying against a live MySQL, and a live MariaDB diff --git a/docs/providers/oracle.md b/docs/providers/oracle.md index e57a91053..dd6e77f8b 100644 --- a/docs/providers/oracle.md +++ b/docs/providers/oracle.md @@ -1880,10 +1880,9 @@ package, and no `@types/oracledb` dependency here), so a driver upgrade that cha caught by a live probe, not by `tsc`. > ⚠️ **Mock isolation:** `bun`'s `mock.module()` is process-wide; files mocking different drivers -> cross-contaminate in a shared process. A **single file** is safe (one file = one process). The -> full `bun run test` script runs the core group in **one** process and is load-order flaky, so -> **CI does not use it** — the deterministic runner is **`bun run test:ci`** (per-file isolation via -> `tests/run-core.sh`); the coverage workflow uses `bun run test:coverage`. See [`CLAUDE.md`](../../CLAUDE.md). +> would cross-contaminate if they shared one. They never do: `bun run test` gives every test file its +> own bun process, so a single file is safe and so is the whole suite, which is the same command CI +> runs. `bun run test:coverage` is that runner with coverage on. See [`CLAUDE.md`](../../CLAUDE.md). ### 12.2 Coverage @@ -1911,8 +1910,8 @@ go red, which caught two assertions that were passing vacuously. ```bash bun test tests/integration/db/oracle-provider.test.ts # just this file (single process — safe) -bun run test:ci # CI publish gate — per-file isolation (tests/run-core.sh) -bun run test:coverage # CI coverage workflow — per-file core + components +bun run test # the whole suite, one process per file, what CI runs +bun run test:coverage # CI coverage workflow: the same runner, with coverage ``` ### 12.4 Optional: verifying against a live Oracle diff --git a/docs/providers/postgres.md b/docs/providers/postgres.md index d3502c73e..f08229f9a 100644 --- a/docs/providers/postgres.md +++ b/docs/providers/postgres.md @@ -1931,12 +1931,10 @@ canned result sets keyed by query shape, which exercises the same provider code server. > **Mock isolation:** `bun`'s `mock.module()` is process-wide, so test files that mock different -> drivers (here `pg`, elsewhere `ioredis`, etc.) cross-contaminate when they share a process. Running -> a **single file** is safe (one file = one process). The full `bun run test` script runs the core -> group (`tests/unit tests/api tests/integration`) in **one process** and is therefore load-order -> flaky — so **CI does not use it**. The deterministic runner is **`bun run test:ci`** (per-file -> process isolation via `tests/run-core.sh`); the coverage workflow uses `bun run test:coverage` -> (also per-file). See [`CLAUDE.md`](../../CLAUDE.md). +> drivers (here `pg`, elsewhere `ioredis`, etc.) would cross-contaminate if they shared a process. +> They never do: `bun run test` gives every test file its own bun process, so a single file is safe +> and so is the whole suite, which is the same command CI runs. `bun run test:coverage` is that +> runner with coverage on. See [`CLAUDE.md`](../../CLAUDE.md). ### 13.2 Coverage @@ -1954,8 +1952,8 @@ table/index/storage stats, pool stats, capabilities, and `pg_stat_activity` pass ```bash bun test tests/integration/db/postgres-provider.test.ts # just this file (single process — safe) -bun run test:ci # CI publish gate — per-file isolation (tests/run-core.sh) -bun run test:coverage # CI coverage workflow — per-file core + components +bun run test # the whole suite, one process per file, what CI runs +bun run test:coverage # CI coverage workflow: the same runner, with coverage ``` ### 13.4 Optional: verifying against a live PostgreSQL diff --git a/docs/providers/redis.md b/docs/providers/redis.md index 152ad0a6f..314a99e5c 100644 --- a/docs/providers/redis.md +++ b/docs/providers/redis.md @@ -1403,9 +1403,9 @@ container in the suite. The mock simulates a Redis 7.2.x server (`redis_version: Redis 6.0+ instance. > ⚠️ **Mock isolation:** `bun`'s `mock.module()` is process-wide. Run the suite with -> `bun run test` (which isolates execution groups), **never** bare `bun test` across multiple -> files — see the note in [`CLAUDE.md`](../../CLAUDE.md). The Redis file mocks `ioredis`, which -> would otherwise leak into any other test sharing the process. +> `bun run test`, which gives every test file its own bun process, never bare `bun test` across +> multiple files - see the note in [`CLAUDE.md`](../../CLAUDE.md). The Redis file mocks `ioredis`, +> which would otherwise leak into any other test sharing the process. ### 11.2 Coverage diff --git a/docs/providers/sqlite.md b/docs/providers/sqlite.md index fd76532c9..ad3ba5240 100644 --- a/docs/providers/sqlite.md +++ b/docs/providers/sqlite.md @@ -76,8 +76,9 @@ SQLite driver by runtime: sqlite connection is actually used. - **Identical behaviour:** the adapter exposes the exact `bun:sqlite`-shaped surface the provider uses (`exec` / `prepare().all/get/run` / `close`) and bridges the small `node:sqlite` deltas - (`get()` miss returns `null` not `undefined`; `run().changes` normalized to `number`), so results - and error mapping are the same under both runtimes. + (`get()` miss returns `null` not `undefined`; `run().changes` normalized to `number`; + `close(throwOnError)` is bun's flag for "release the file now" and node:sqlite needs none), so + results and error mapping are the same under both runtimes. - **Why not `better-sqlite3`?** Bun refuses to load it outright, and its native binding must match the installing runtime's ABI (a bun-installed binding fails under Node). The built-in drivers need no native dependency at all. (`better-sqlite3` remains the *storage-layer* driver.) @@ -170,6 +171,31 @@ concurrency, NORMAL sync for a speed/durability balance. The agent read-only pro different open sequence entirely — `journal_mode = WAL` is itself a write and fails on a read-only handle ([§12.1](#121-where-the-boundary-is)). +`disconnect()` closes with `close(true)`, and the argument is load-bearing. Bare `close()` on +`bun:sqlite` is `sqlite3_close_v2`: with any statement still unfinalized the connection becomes a +zombie and the database, its `-wal` and its `-shm` stay **open** until the last statement is +finalized or garbage collected. This provider prepares a statement per query and drops the +reference, so that used to be whenever the collector got to it, measured through `/proc/self/fd`, +three descriptors survived a `disconnect()` that reported `isConnected() === false`. POSIX hides +that, because it unlinks a file that is still open; Windows does not, and a user could not delete or +move a database Studio had disconnected from. `close(true)` finalizes and closes for real, and +raises if SQLite cannot. `node:sqlite` needs no flag: its own `close()` finalizes the statements it +tracks (measured on Node 24.14.0). + +The sidecars are NOT the portable reading of this, although they look like it: probed on all three +runners, `close(true)` removes `-wal` and `-shm` on Linux and Windows and leaves both in place on +macOS, where bun:sqlite links Apple's system libsqlite3. That is the library keeping the WAL rather +than a handle keeping the file, because opening the same database with node:sqlite and closing it +removed them on that same macOS run, which takes the exclusive lock a surviving handle would deny. +So `tests/integration/db/sqlite-provider.test.ts` asks each platform what it can answer: everywhere, +the file can be renamed after `disconnect()`, which is what Windows refuses for a live handle; on +Linux, no descriptor of the process still points into the directory. The node adapter's harness does +assert the sidecars, because its own SQLite removes them everywhere. + +The same close runs on the failure path of `connect()`: an open that succeeds and then fails its +pragmas (a connection pointed at a file that is not a database, the ordinary wrong-file mistake) +used to leave the handle held, so the user could not delete or move the file they had just picked. + ### 3.3 Read vs write dispatch `query()` ([`sqlite.ts`](../../src/lib/db/providers/sql/sqlite.ts)) branches on @@ -934,9 +960,9 @@ SQLite is the **only** provider whose integration tests run against a **real eng Embedded + in-memory/tempfile means there is no server to provision, so the tests exercise actual SQL execution, schema PRAGMAs, maintenance, and monitoring end-to-end. -> Mock-isolation still applies to the *suite* (other files mock their drivers process-wide), so run -> with `bun run test:ci` / `bun run test:coverage`, not the single-process `bun run test`. See -> [`CLAUDE.md`](../../CLAUDE.md). +> Other files in the suite mock their drivers process-wide, and `bun run test` keeps them apart by +> giving every test file its own bun process, so neither this file nor the whole suite is exposed to +> another file's mocks. See [`CLAUDE.md`](../../CLAUDE.md). ### 11.2 Coverage @@ -958,7 +984,7 @@ since bun and node report read-only violations differently. ```bash bun test tests/integration/db/sqlite-provider.test.ts # real :memory: engine -bun run test:ci # CI publish gate (per-file isolation) +bun run test # the whole suite, one process per file bun run test:coverage # CI coverage workflow ``` diff --git a/loop/LOOP-ENGINEERING.md b/loop/LOOP-ENGINEERING.md index 9b99b29bc..8439f8f0c 100644 --- a/loop/LOOP-ENGINEERING.md +++ b/loop/LOOP-ENGINEERING.md @@ -157,7 +157,7 @@ Nothing commits unless every gate is green. Define one command in `loop/scripts/ ```bash # This repo's concrete gate — see loop/scripts/gate.sh for the authoritative list -format && lint && typecheck && knip && test && test:coverage + coverage:check && build +format && lint && typecheck && knip && test:coverage + coverage:check && build ``` Tests are the primary gate — derived from acceptance criteria, written before implementation. diff --git a/loop/README.md b/loop/README.md index eeabe5685..bd1228260 100644 --- a/loop/README.md +++ b/loop/README.md @@ -93,7 +93,7 @@ its `.loop/PROGRESS.md` entry and its commit; a trace line is not evidence that 1. **Mechanical** — `scripts/gate.sh`: the exact pre-commit verification from the root `CLAUDE.md` plus the required CI coverage check - (format · lint · typecheck · knip · test · test:coverage + coverage:check · build), per task, + (format · lint · typecheck · knip · test:coverage + coverage:check · build), per task, before every commit. 2. **Functional** — `scripts/functional-smoke.sh`: boot the built app, create a real PostgreSQL connection through the UI, run a SQL query, assert the rows render. Mandatory diff --git a/loop/scripts/gate.sh b/loop/scripts/gate.sh index e3c658742..17b3b2fd0 100755 --- a/loop/scripts/gate.sh +++ b/loop/scripts/gate.sh @@ -22,13 +22,13 @@ bun run typecheck echo "=== gate: knip ===" bun run knip -echo "=== gate: test ===" -bun run test - -# The required "Unit & Integration Tests" job runs these two, not `bun run test`: -# coverage goes through tests/run-core.sh per-file process isolation, and -# scripts/check-coverage.mjs enforces 100% lines on the merged lcov. -echo "=== gate: coverage ===" +# One run of the suite, not two. `bun run test:coverage` is `bun run test` with +# --coverage: the same runner over the same files, so a separate `bun run test` step +# would run all of them twice and catch nothing the coverage run does not. It used to +# be a different command, which is why both steps existed. The required "Unit & +# Integration Tests" job runs exactly these two; scripts/check-coverage.mjs enforces +# 100% of lines on the merged lcov. +echo "=== gate: test (with coverage) ===" bun run test:coverage bun run coverage:check diff --git a/package.json b/package.json index f96a77c3a..a09eef749 100644 --- a/package.json +++ b/package.json @@ -145,19 +145,17 @@ "lint": "oxlint && eslint .", "lint:oxc": "oxlint", "typecheck": "tsc --noEmit", - "test": "bun test tests/unit tests/api tests/integration && bun test tests/hooks && bun test tests/security && bun test tests/evals && bun run test:components", - "test:ci": "bash tests/run-core.sh && bun run test:components", + "test": "bun tests/run-tests.ts", "agent:eval": "bun tests/evals/real-model.ts", - "test:evals": "bun test tests/evals", - "test:unit": "bun test tests/unit", - "test:integration": "bun test tests/integration", - "test:hooks": "bun test tests/hooks", - "test:api": "bun test tests/api", - "test:components": "bash tests/run-components.sh", - "test:components:coverage": "bash tests/run-components.sh --coverage --coverage-reporter=lcov --coverage-dir=coverage/components", + "test:evals": "bun tests/run-tests.ts tests/evals", + "test:unit": "bun tests/run-tests.ts tests/unit", + "test:integration": "bun tests/run-tests.ts tests/integration", + "test:hooks": "bun tests/run-tests.ts tests/hooks", + "test:api": "bun tests/run-tests.ts tests/api", + "test:security": "bun tests/run-tests.ts tests/security", + "test:components": "bun tests/run-tests.ts tests/components tests/isolated", "test:e2e": "bunx playwright test", - "test:coverage:core": "bash tests/run-core.sh --coverage --coverage-reporter=lcov --coverage-dir=coverage/core", - "test:coverage": "rm -rf coverage && bun run test:coverage:core && bun run test:components:coverage && node scripts/merge-lcov.mjs coverage/core/file-*/lcov.info coverage/components/lcov.info coverage/lcov.info", + "test:coverage": "bun tests/run-tests.ts --coverage --merge-into=coverage/lcov.info", "coverage:check": "node scripts/check-coverage.mjs coverage/lcov.info", "test:coverage-html": "bun run test:coverage && genhtml coverage/lcov.info --output-directory coverage/html && echo '\n Open coverage/html/index.html in your browser'", "knip": "knip", diff --git a/scripts/lib/pack-standalone-tarball.sh b/scripts/lib/pack-standalone-tarball.sh index 27cbe9bd0..d72079ecb 100755 --- a/scripts/lib/pack-standalone-tarball.sh +++ b/scripts/lib/pack-standalone-tarball.sh @@ -22,6 +22,18 @@ PAYLOAD_DIR=$1 VERSION=$2 OUT_TARBALL=$3 +# Resolve the output path before tar sees it, the way pack-standalone-zip.sh +# resolves its own. GNU tar reads a -f argument whose first colon comes before +# any slash as `host:file` and tries to reach that host: measured with tar 1.35, +# `-f out:1.tar.gz` answers "Cannot connect to out: resolve failed" and exits 2 +# having written nothing. Every absolute path a Windows caller has is that shape +# (`C:\...`), so a bash script driven from node, Bun or PowerShell there dials a +# host called C instead of writing a file. An absolute path cannot be misread. +# scripts/build-standalone-payload.sh, the one production caller, already passes +# an absolute POSIX path, so this leaves the release build untouched. +OUT_PARENT=$(cd "$(dirname "$OUT_TARBALL")" && pwd) +OUT_TARBALL="$OUT_PARENT/$(basename "$OUT_TARBALL")" + ROOT_NAME="libredb-studio-${VERSION}" PARENT_DIR=$(cd "$(dirname "$PAYLOAD_DIR")" && pwd) ROOT_DIR="$PARENT_DIR/$ROOT_NAME" diff --git a/scripts/merge-lcov.mjs b/scripts/merge-lcov.mjs index bcde04e66..1ec91d852 100644 --- a/scripts/merge-lcov.mjs +++ b/scripts/merge-lcov.mjs @@ -31,7 +31,13 @@ function parseLcov(content) { for (const line of lines) { if (line.startsWith("SF:")) { - record.sf = line.slice(3); + // bun writes the path with the host separator, so the same file is + // `src/lib/db/factory.ts` on Linux and macOS and `src\lib\db\factory.ts` + // on Windows. Normalising here is what lets a Windows contributor's + // `bun run test:coverage` produce the same report as CI's: unnormalised, + // the two spellings merge as two files and the `src/` filter below drops + // both, leaving an empty report. + record.sf = line.slice(3).replaceAll("\\", "/"); continue; } @@ -285,15 +291,46 @@ function serializeRecords(records) { return chunks.join("\n"); } +/** + * The inputs, either spelled out in argv or listed one per line in a manifest. + * + * The manifest exists because the test runner merges one report per test file: + * over 500 paths, which is most of the 32767-character command line Windows + * allows. An empty manifest raises rather than producing an empty report. + */ +function resolveInputs(args) { + const manifestArg = args.find((arg) => arg.startsWith("--inputs-from=")); + if (!manifestArg) return { inputPaths: args.slice(0, -1), outputPath: args[args.length - 1] }; + + const manifestPath = manifestArg.slice("--inputs-from=".length); + const inputPaths = fs + .readFileSync(manifestPath, "utf8") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + if (inputPaths.length === 0) { + console.error(`No input reports listed in ${manifestPath}`); + process.exit(1); + } + + const rest = args.filter((arg) => arg !== manifestArg); + if (rest.length !== 1) { + console.error("Usage: node scripts/merge-lcov.mjs --inputs-from= "); + process.exit(1); + } + return { inputPaths, outputPath: rest[0] }; +} + function main() { const [, , ...args] = process.argv; - if (args.length < 3) { + const usesManifest = args.some((arg) => arg.startsWith("--inputs-from=")); + if (args.length < (usesManifest ? 2 : 3)) { console.error("Usage: node scripts/merge-lcov.mjs [moreInputs...] "); + console.error(" or: node scripts/merge-lcov.mjs --inputs-from= "); process.exit(1); } - const outputPath = args[args.length - 1]; - const inputPaths = args.slice(0, -1); + const { inputPaths, outputPath } = resolveInputs(args); const allRecords = []; for (const inputPath of inputPaths) { diff --git a/scripts/operator-catalog-submission.mjs b/scripts/operator-catalog-submission.mjs index 0951f9d96..55278661e 100644 --- a/scripts/operator-catalog-submission.mjs +++ b/scripts/operator-catalog-submission.mjs @@ -27,6 +27,7 @@ import fs from "node:fs"; import path from "node:path"; +import { fileURLToPath } from "node:url"; /** Bare `x.y.z` only. Our operator versions are app versions, which carry no prerelease suffix. */ const SEMVER = /^(\d+)\.(\d+)\.(\d+)$/; @@ -403,7 +404,15 @@ async function decide(argv) { return 0; } -if (import.meta.url === `file://${process.argv[1]}`) { +// CLI entry only when executed directly (the unit test imports this module). +// Compared as PATHS, the way every other script here does it: `file://` glued +// to argv[1] is a URL only by accident, and it stops matching as soon as the +// path needs percent-encoding or is not separated by forward slashes. Both +// happen in practice - a directory with a space, and every Windows invocation, +// where argv[1] is `D:\a\...\scripts\operator-catalog-submission.mjs` and the +// module URL is `file:///D:/a/...`. The mismatch is silent: the module loads, +// nothing runs, the workflow step exits 0 with an empty GITHUB_OUTPUT. +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { main(process.argv.slice(2)) .then((code) => process.exit(code)) .catch((error) => { diff --git a/scripts/security-check.mjs b/scripts/security-check.mjs index 362c490b0..215afecea 100644 --- a/scripts/security-check.mjs +++ b/scripts/security-check.mjs @@ -25,12 +25,13 @@ * Pure functions below are unit tested in tests/unit/security-check.test.ts. */ +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const POSTURE = "docs/SECURITY.md"; -const COMPONENTS_RUNNER = "tests/run-components.sh"; +const TEST_RUNNER = "tests/run-tests.ts"; const PLAYWRIGHT_CONFIG = "playwright.config.ts"; const SECURITY_TEST_DIR = "tests/security"; @@ -67,8 +68,27 @@ export const PROGRAMME_CONTROL_IDS = [ export const STATUSES = new Set(["Implemented", "Partial", "Not implemented"]); -/** Directories tests/run-core.sh enumerates with `find ... -name '*.test.ts' -o -name '*.test.tsx'`. */ -const CORE_TEST_DIRS = ["tests/unit/", "tests/api/", "tests/integration/", "tests/hooks/", "tests/security/"]; +/** + * Every test file `bun run test` runs, asked of the runner itself. + * + * This used to be a hardcoded directory list plus a grep of the component runner + * script, which drifted twice: the list omitted `tests/evals`, and a file could sit + * in `tests/isolated/` without being named by any group. The runner's own discovery + * rule is the repository's definition of "this test runs", so the gate asks it. + */ +function discoveredTestFiles(root) { + const listed = spawnSync("bun", [TEST_RUNNER, "--list"], { cwd: root, encoding: "utf8" }); + if (listed.status !== 0) { + console.error(`ERROR: could not ask ${TEST_RUNNER} what it runs: ${listed.stderr || listed.error}`); + process.exit(1); + } + return new Set( + listed.stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean), + ); +} /** Splits a markdown row into trimmed cells, dropping the leading and trailing empties. */ function cells(line) { @@ -140,7 +160,7 @@ export function linkTargets(cell) { * document itself as its own verifier (0.4, 0.5) is not linking a test, and existence is not the * claim a "Verified by" cell makes. */ -export function isExecuted(target, { componentsRunner, playwrightConfig, requireTest = false }) { +export function isExecuted(target, { discoveredTests, playwrightConfig, requireTest = false }) { if (!target.startsWith("tests/") && !target.startsWith("e2e/")) { return requireTest ? { executed: false, reason: "not a test" } : { executed: true, reason: "not a test path" }; } @@ -149,12 +169,8 @@ export function isExecuted(target, { componentsRunner, playwrightConfig, require if (inTestDir && target.endsWith(".spec.ts")) return { executed: true, reason: "playwright testDir" }; return { executed: false, reason: "not collected by playwright.config.ts" }; } - const isCoreName = target.endsWith(".test.ts") || target.endsWith(".test.tsx"); - if (isCoreName && CORE_TEST_DIRS.some((dir) => target.startsWith(dir))) { - return { executed: true, reason: "tests/run-core.sh" }; - } - if (componentsRunner.includes(target)) return { executed: true, reason: COMPONENTS_RUNNER }; - return { executed: false, reason: `named by neither tests/run-core.sh nor ${COMPONENTS_RUNNER}` }; + if (discoveredTests.has(target)) return { executed: true, reason: TEST_RUNNER }; + return { executed: false, reason: `not collected by ${TEST_RUNNER}` }; } /** @@ -163,7 +179,7 @@ export function isExecuted(target, { componentsRunner, playwrightConfig, require * `exists` is injected rather than read here so the whole rule set is testable without a * filesystem, following checkReadmes in scripts/readme-check.mjs. */ -export function checkPosture({ posture, componentsRunner, playwrightConfig, exists, securityTestFiles }) { +export function checkPosture({ posture, discoveredTests, playwrightConfig, exists, securityTestFiles }) { const table = findControlTable(parseTables(posture)); if (!table) { return [`${POSTURE}: no control table found (expected a header of exactly: ${CONTROL_HEADER.join(" | ")})`]; @@ -193,7 +209,7 @@ export function checkPosture({ posture, componentsRunner, playwrightConfig, exis violations.push(`${POSTURE}: control ${id} links ${target}, which does not exist`); continue; } - const { executed, reason } = isExecuted(target, { componentsRunner, playwrightConfig }); + const { executed, reason } = isExecuted(target, { discoveredTests, playwrightConfig }); if (!executed) { violations.push(`${POSTURE}: control ${id} links ${target}, which is never executed (${reason})`); } @@ -211,7 +227,7 @@ export function checkPosture({ posture, componentsRunner, playwrightConfig, exis // requireTest: true - a "Verified by" cell is a claim that a test verifies the control, and // isExecuted's normal existence-is-enough-for-a-source-file allowance would otherwise let a // checker script or a policy document stand in for a test that does not exist (0.4, 0.5). - const { executed, reason } = isExecuted(target, { componentsRunner, playwrightConfig, requireTest: true }); + const { executed, reason } = isExecuted(target, { discoveredTests, playwrightConfig, requireTest: true }); if (!executed) { violations.push(`${POSTURE}: control ${id} links ${target}, which is never executed (${reason})`); } @@ -261,7 +277,7 @@ function main(argv) { const violations = checkPosture({ posture: fs.readFileSync(posturePath, "utf8"), - componentsRunner: fs.readFileSync(path.join(root, COMPONENTS_RUNNER), "utf8"), + discoveredTests: discoveredTestFiles(root), playwrightConfig: fs.readFileSync(path.join(root, PLAYWRIGHT_CONFIG), "utf8"), exists: (target) => fs.existsSync(path.join(root, target)), securityTestFiles, diff --git a/sonar-project.properties b/sonar-project.properties index b5deffffb..780569e21 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -30,7 +30,10 @@ sonar.cpd.exclusions=tests/**,e2e/**,src/lib/db/compatibility.ts # for bundlers that cannot resolve .ts entry points. It has no testable logic; it is # smoke-tested by tests/isolated/exports-shim.test.ts, which runs WITHOUT coverage # because importing the shim loads the entire component chain unexercised and would -# pollute the merged lcov with load-only zero-hit records. +# pollute the merged lcov with load-only zero-hit records. What enforces that is +# COVERAGE_EXEMPT_FILES in tests/runner/discover.ts, which names that file (and the +# Monaco loader wiring test) as the ones the runner spawns without --coverage; its +# docblock carries the measurement. sonar.coverage.exclusions=src/exports/index.js sonar.javascript.lcov.reportPaths=coverage/lcov.info diff --git a/src/lib/agent/model-adapter.ts b/src/lib/agent/model-adapter.ts index 789d28db3..7d7e6a87b 100644 --- a/src/lib/agent/model-adapter.ts +++ b/src/lib/agent/model-adapter.ts @@ -38,8 +38,8 @@ export interface AgentModelOptions { readonly config?: Partial; /** * The SDK providers document `fetch` as the interception point for tests and - * proxies; injecting it keeps the suite off a global stub, which `bun test` - * would leak across files sharing the process. + * proxies; injecting it keeps the suite off a global stub, which would leak + * into every other test in the same process and has to be restored by hand. */ readonly fetch?: AgentFetch; } diff --git a/src/lib/api/object-route.ts b/src/lib/api/object-route.ts index 59ab813fc..3a86cf75d 100644 --- a/src/lib/api/object-route.ts +++ b/src/lib/api/object-route.ts @@ -254,11 +254,14 @@ async function readDefaultBody(req: NextRequest): Promise> { // `req.body` is null for a request that carried no body at all, which is what a GET or a bodiless diff --git a/src/lib/api/rate-limit.ts b/src/lib/api/rate-limit.ts index 0216f38f9..53481a2f1 100644 --- a/src/lib/api/rate-limit.ts +++ b/src/lib/api/rate-limit.ts @@ -352,9 +352,9 @@ export function resetRateLimit(bucket: RateLimitBucket, key: string): void { } /** - * Test seam. `bun run test` runs tests/unit, tests/api, tests/integration and tests/security in a - * single process, so this module's state is shared across every file in that run; any test file - * that exercises a rate-limited route calls this in beforeEach. + * Test seam. These counters are module state, and `bun run test` gives each test FILE its own + * process but not each test, so every test in a file that exercises a rate-limited route shares + * them, and one such file calls this in beforeEach. */ export function clearRateLimitState(): void { for (const store of Object.values(bucketStores)) store.clear(); diff --git a/src/lib/auth-compare.ts b/src/lib/auth-compare.ts index 60f3fd988..514c1e525 100644 --- a/src/lib/auth-compare.ts +++ b/src/lib/auth-compare.ts @@ -38,10 +38,11 @@ let comparisons = 0; * Test seam: how many constant-time comparisons this process has performed. * * The enumeration test asserts that exactly one comparison happens per login attempt whether or - * not the submitted email matched. It cannot use mock.module to count them: `bun run test` runs - * tests/unit, tests/api, tests/integration and tests/security in one process, and a module mock - * there is process-wide. A monotonic counter read as a before/after delta is deterministic and - * leaks nothing. This follows resetCookieSecurityWarning() in src/lib/auth.ts:73. + * not the submitted email matched. It cannot use mock.module to count them: counting the calls + * that way means replacing the comparison, so the test would no longer observe the constant-time + * path it exists to pin, and a module mock has no undo, so it would outlive the one test that + * wanted it. A monotonic counter read as a before/after delta is deterministic and leaks + * nothing. This follows resetCookieSecurityWarning() in src/lib/auth.ts:73. */ export function comparisonCount(): number { return comparisons; diff --git a/src/lib/db/providers/sql/sqlite-driver.ts b/src/lib/db/providers/sql/sqlite-driver.ts index 586d5cd76..2812b4d51 100644 --- a/src/lib/db/providers/sql/sqlite-driver.ts +++ b/src/lib/db/providers/sql/sqlite-driver.ts @@ -30,7 +30,26 @@ export type SQLiteStatement = { export type SQLiteDatabase = { exec(sql: string): void; prepare(sql: string): SQLiteStatement; - close(): void; + /** + * Close the handle. `throwOnError: true` means "release the file NOW", and the + * provider always asks for it. + * + * Measured 2026-09-15 on bun:sqlite (Bun 1.4.2), through /proc/self/fd: bare + * `close()` is `sqlite3_close_v2`, so a connection with any statement still + * unfinalized becomes a zombie and the database, its `-wal` and its `-shm` stay + * OPEN until the last statement is finalized or garbage collected. The provider + * prepares a statement per query and drops the reference, so that is whenever the + * collector gets to it. `close(true)` finalizes them and closes for real, and + * raises if SQLite cannot. + * + * POSIX hides the difference, because it unlinks a file that is still open; + * Windows does not, and a deferred close is a database the user cannot delete or + * move (and, on windows-latest, a test temp directory whose teardown fails with + * EBUSY). node:sqlite needs no flag - measured on Node 24.14.0, its `close()` + * finalizes the statements it tracks and releases every descriptor - so the node + * adapter below declares no parameter at all. + */ + close(throwOnError?: boolean): void; /** * Whether this handle currently has a transaction open, as SQLite itself reports it * (`sqlite3_get_autocommit`), in the bun:sqlite spelling. Both drivers publish it and @@ -115,6 +134,9 @@ async function loadBunDriver(): Promise { * - `inTransaction` is node:sqlite's `isTransaction` under bun:sqlite's name. The * provider reads it to tell whether a statement left a transaction open on the handle * (D71), and a handle whose answer never changed would report every script as clean. + * - `close(throwOnError)` is bun's spelling of "release the file now". node:sqlite has + * no such flag and needs none, so this is the one delta with nothing to bridge; see + * the measurement on `SQLiteDatabase.close` above. * - `get()` returns `undefined` on a miss where bun:sqlite returns `null`. * - `run()` reports `changes` as `number | bigint`; normalize to `number`. * @@ -146,6 +168,10 @@ export function createNodeSQLiteDriver(DatabaseSyncCtor: NodeSQLiteModule["Datab }; } + // Takes no `throwOnError`, and needs none: node:sqlite's own close already finalizes + // the statements it tracks and releases every descriptor (measured on Node 24.14.0), + // which is exactly what the flag asks bun:sqlite for. A method that declares fewer + // parameters still satisfies the surface, so `close(true)` reaches here unchanged. close(): void { this.db.close(); } diff --git a/src/lib/db/providers/sql/sqlite.ts b/src/lib/db/providers/sql/sqlite.ts index f2caee0b0..cf081d299 100644 --- a/src/lib/db/providers/sql/sqlite.ts +++ b/src/lib/db/providers/sql/sqlite.ts @@ -1119,6 +1119,22 @@ export class SQLiteProvider extends SQLBaseProvider { this.setConnected(true); } catch (error) { + // The handle is opened before the pragmas run, so a failure past that line + // leaves this holding the user's file: the ordinary case is a connection that + // points at something which is not a SQLite database at all, where the open + // succeeds and `PRAGMA journal_mode` raises "file is not a database". Closing + // here for the reason disconnect() does: POSIX hides an unreleased handle, + // Windows does not, and a file the user picked by mistake would stay + // undeletable. Nulling it also keeps `connect()`'s own `if (this.db) return` + // from turning a retry into a silent no-op on a provider that is not connected. + // + // `finally`, because `close(true)` raises when SQLite cannot close, and a + // reference kept past that throw is exactly the silent no-op described above. + try { + this.db?.close(true); + } finally { + this.db = null; + } this.setError(error instanceof Error ? error : new Error(String(error))); // Typed refusals keep their own identity: wrapping them would strip the @@ -1162,8 +1178,15 @@ export class SQLiteProvider extends SQLBaseProvider { try { this.enforceQueryOnly(); } catch (error) { - this.db.close(); - this.db = null; + // Released now, for the same reason disconnect() does it: a refused profile that + // left the file held open would be a lock on a database nobody is using. The + // reference goes in a `finally`, because `close(true)` raises when it cannot + // close, and the refusal the caller needs to see is still `error`. + try { + this.db.close(true); + } finally { + this.db = null; + } throw error; } @@ -1178,7 +1201,11 @@ export class SQLiteProvider extends SQLBaseProvider { public async disconnect(): Promise { if (this.db) { - this.db.close(); + // `true` means "release the file now" rather than "once the last statement is + // collected" - see the measurement on `SQLiteDatabase.close`. A caller that has + // disconnected is entitled to delete, move or reopen the database, and on Windows + // a deferred close makes all three impossible. + this.db.close(true); this.db = null; this.setConnected(false); } diff --git a/tests/components/ConnectionSignature.test.tsx b/tests/components/ConnectionSignature.test.tsx index e8723fe1c..5ad9651fe 100644 --- a/tests/components/ConnectionSignature.test.tsx +++ b/tests/components/ConnectionSignature.test.tsx @@ -4,7 +4,7 @@ import { ConnectionSignature, SIGNATURE_URIS } from "@/components/login/connecti import { ENGINE_URI_SCHEMES, parseConnectionString } from "@/lib/connection-string-parser"; import { afterAll, afterEach, describe, expect, test } from "bun:test"; -import { cleanup, render, waitFor } from "@testing-library/react"; +import { act, cleanup, render, waitFor } from "@testing-library/react"; /** The real `matchMedia` happy-dom installed, kept so the stub below can be handed back. */ const realMatchMedia = window.matchMedia; @@ -15,12 +15,13 @@ const realMatchMedia = window.matchMedia; * must answer it before the effect runs, hence the assignment before `render`. * * The returned object is a whole `MediaQueryList` shape, not just `{ matches }`, and the - * real implementation goes back on in `afterAll`. Both halves matter: `window` is shared by - * every file in this component group (`tests/run-components.sh` Group 11), and `next-themes` - * - which `RootLayout.test.tsx` renders in the same process - subscribes with the legacy - * `addListener`. A stub missing that method, or left in place after this file finishes, - * fails those tests instead of these, which is exactly the process-wide contamination the - * grouping exists to prevent. + * real implementation goes back on in `afterAll`. Both halves matter because `window` is + * shared by every test in this file: a subscriber that reaches for the legacy `addListener` + * gets `undefined is not a function` from a `{ matches }`-only stub, and a stub left standing + * decides the outcome of whatever runs after it. The blast radius stops at the file, and only + * because of how the suite runs: the runner gives every test file its own bun process, so a + * file states an assumption like this one in its own docblock and needs no directory and no + * registration anywhere. */ function stubReducedMotion(reduce: boolean) { Object.defineProperty(window, "matchMedia", { @@ -39,6 +40,29 @@ function stubReducedMotion(reduce: boolean) { }); } +/** + * Records what the component hands `setInterval`, and keeps each handler so a test can fire it + * instead of waiting for it. The swap follows the one in tests/components/admin/OverviewTab.test.tsx. + * + * The real interval is still started, so nothing about the component's lifetime changes and the + * unmount still clears it. Restoring is the caller's job, in a `finally`: `globalThis` is shared + * by every test in this file. + */ +function captureIntervals() { + const realSetInterval = globalThis.setInterval; + const scheduled: { delay: number | undefined; handler: () => void }[] = []; + globalThis.setInterval = ((handler: () => void, delay?: number) => { + scheduled.push({ delay, handler }); + return realSetInterval(handler, delay); + }) as unknown as typeof setInterval; + return { + scheduled, + restore: () => { + globalThis.setInterval = realSetInterval; + }, + }; +} + describe("ConnectionSignature", () => { afterEach(() => { cleanup(); @@ -69,22 +93,62 @@ describe("ConnectionSignature", () => { expect(announced).toEqual(SIGNATURE_URIS.map((uri) => uri.scheme)); }); - test("does not start the cycle when the viewer asked for reduced motion", async () => { - stubReducedMotion(true); - const { getByTestId } = render(); - const first = getByTestId("connection-signature").textContent; + test("does not start the cycle when the viewer asked for reduced motion", () => { + /* + * Watched at the component's own timer rather than on the wall clock, because the wall clock + * cannot see this. The cycle's first change is CYCLE_MS away, 2.6s, so the 200ms sleep this + * test used to take proved nothing: it passed just as well with the `prefers-reduced-motion` + * guard deleted, since nothing had moved yet either way. What the guard decides is whether + * the interval is scheduled at all, so that is what is read. + * + * The second half is the control, and it is what stops the first from being vacuous: a + * capture watching a surface the component does not use would record an empty list under + * both preferences and the negative would pass for the wrong reason. Motion off must record + * NO interval, motion on must record one, and firing that one must advance the line, which + * is what proves the thing recorded is the cycle and not some other timer. + */ + const cycle = captureIntervals(); + try { + stubReducedMotion(true); + const reduced = render(); + expect(reduced.getByTestId("connection-signature").textContent).toContain(SIGNATURE_URIS[0].scheme); + expect(cycle.scheduled).toHaveLength(0); + cleanup(); - await new Promise((resolve) => setTimeout(resolve, 200)); - expect(getByTestId("connection-signature").textContent).toBe(first); + stubReducedMotion(false); + const moving = render(); + expect(cycle.scheduled).toHaveLength(1); + act(() => { + cycle.scheduled[0].handler(); + }); + expect(moving.getByTestId("connection-signature").textContent).toContain(SIGNATURE_URIS[1].scheme); + } finally { + cycle.restore(); + } }); test("advances to the next URI on its own", async () => { + /* + * The wait asks for the URI it wants. It used to end on "the text changed at all" and + * then demand index 1 on the line after, which is a component-bug report waiting for a + * busy machine: the 6000ms window spans more than two 2600ms cycles, so a process that + * stalls long enough to miss the index-1 plateau satisfies "not the first text" with + * index 2 already on screen, and even a wait that ended on index 1 can have the interval + * fire again before the next statement reads the node. Asked for index 1 by name, a poll + * that finds index 2 is a failing poll rather than the end of the wait. + * + * The read before the wait is the control: it fixes the opening frame at index 0, so a + * component that painted index 1 from the start could not pass this. Both reads compare + * the whole URI rather than the scheme alone, because one scheme can be a prefix of + * another and `toContain` would then answer for the wrong frame. + */ stubReducedMotion(false); const { getByTestId } = render(); - const first = getByTestId("connection-signature").textContent; - expect(first).toContain(SIGNATURE_URIS[0].scheme); + const uriText = (uri: (typeof SIGNATURE_URIS)[number]) => `${uri.scheme}${uri.rest}`; + expect(getByTestId("connection-signature").textContent).toBe(uriText(SIGNATURE_URIS[0])); - await waitFor(() => expect(getByTestId("connection-signature").textContent).not.toBe(first), { timeout: 6000 }); - expect(getByTestId("connection-signature").textContent).toContain(SIGNATURE_URIS[1].scheme); + await waitFor(() => expect(getByTestId("connection-signature").textContent).toBe(uriText(SIGNATURE_URIS[1])), { + timeout: 6000, + }); }); }); diff --git a/tests/components/DataCharts.test.tsx b/tests/components/DataCharts.test.tsx index f268f9663..fbe6950a0 100644 --- a/tests/components/DataCharts.test.tsx +++ b/tests/components/DataCharts.test.tsx @@ -986,13 +986,19 @@ describe("DataCharts", () => { // ----------------------------------------------------------------------- // Aggregation / date grouping hidden for certain chart types + // + // The four polls below compare with `=== null` instead of asserting `toBeNull()` on the node. + // A poll that FAILS hands bun a live happy-dom element to pretty-print, and bun walks the whole + // node's object graph for the diff: 301 ms for a 260-node subtree, measured. waitFor's 5 s + // budget is gone in a few polls, so a machine that is briefly busy reds a healthy test. The + // boolean costs 0 ms and asserts the same absence. // ----------------------------------------------------------------------- test("aggregation hidden for scatter chart", async () => { const { queryByText } = render(React.createElement(DataCharts, { result: mockNumericResult })); fireEvent.click(queryByText("Scatter")!); await waitFor(() => { - expect(queryByText("Agg")).toBeNull(); + expect(queryByText("Agg") === null).toBe(true); }); }); @@ -1000,7 +1006,7 @@ describe("DataCharts", () => { const { queryByText } = render(React.createElement(DataCharts, { result: mockNumericResult })); fireEvent.click(queryByText("Histogram")!); await waitFor(() => { - expect(queryByText("Agg")).toBeNull(); + expect(queryByText("Agg") === null).toBe(true); }); }); @@ -1008,7 +1014,7 @@ describe("DataCharts", () => { const { queryByText } = render(React.createElement(DataCharts, { result: mockNumericResult })); fireEvent.click(queryByText("Scatter")!); await waitFor(() => { - expect(queryByText("Group")).toBeNull(); + expect(queryByText("Group") === null).toBe(true); }); }); @@ -1016,7 +1022,7 @@ describe("DataCharts", () => { const { queryByText } = render(React.createElement(DataCharts, { result: mockNumericResult })); fireEvent.click(queryByText("Histogram")!); await waitFor(() => { - expect(queryByText("Group")).toBeNull(); + expect(queryByText("Group") === null).toBe(true); }); }); diff --git a/tests/components/LoginPage.test.tsx b/tests/components/LoginPage.test.tsx index 6953cca33..01a9150bc 100644 --- a/tests/components/LoginPage.test.tsx +++ b/tests/components/LoginPage.test.tsx @@ -365,8 +365,12 @@ describe("LoginPage showcase (issue #425)", () => { // yet (or no longer does), and docs/CHANNELS.md is explicit that a deprecated channel // renders nothing at all. Parsed from the YAML rather than listed here, so a newly // promoted or newly retired row is covered without touching this test. + // Resolved against this file rather than against the cwd, the way RootLayout.test.tsx:43 + // reads its screenshot. A bare "distribution/channels.yaml" is only found when the process + // happens to start in the repo root, and one bun process per test file is exactly the setup + // in which that stops being something a test may assume. const { container } = renderShowcase(); - const inventory = parseYaml(readFileSync("distribution/channels.yaml", "utf8")) as { + const inventory = parseYaml(readFileSync(new URL("../../distribution/channels.yaml", import.meta.url), "utf8")) as { channels: { status: string; name: string; short_name?: string }[]; }; const unlisted = inventory.channels.filter((channel) => channel.status !== "live"); @@ -610,7 +614,12 @@ describe("LoginPage TOTP step", () => { // A code minted for the previous account would fail and cost that account a slot in the // per-account limiter, so the step resets with the credentials it was issued against. - await waitFor(() => expect(codeInput(result.container)).toBeNull()); + // + // `=== null` and not `toBeNull()` on the input, here and in the test below. On a FAILING poll + // bun pretty-prints the received value, and for a happy-dom node that means walking its whole + // object graph: 301 ms for a 260-node subtree, measured. waitFor's 5 s budget goes in a few + // polls, so a machine that is briefly busy reds a healthy test. The boolean costs 0 ms. + await waitFor(() => expect(codeInput(result.container) === null).toBe(true)); expect(result.getByText("Sign In")).not.toBeNull(); }); @@ -623,6 +632,6 @@ describe("LoginPage TOTP step", () => { await result.user.type(result.passwordInput, "x"); - await waitFor(() => expect(codeInput(result.container)).toBeNull()); + await waitFor(() => expect(codeInput(result.container) === null).toBe(true)); }); }); diff --git a/tests/components/QuerySafetyDialog.test.tsx b/tests/components/QuerySafetyDialog.test.tsx index 489c377a7..caf36514a 100644 --- a/tests/components/QuerySafetyDialog.test.tsx +++ b/tests/components/QuerySafetyDialog.test.tsx @@ -1302,16 +1302,54 @@ describe("isDangerousQuery", () => { * The statement below leads with `SELECT` on purpose: a leading `UPDATE` is * answered by the vocabulary test without the probe ever running, so it would * guard nothing. + * + * WHAT IS ASSERTED IS THE SHAPE, NOT A CPU BUDGET, and the defect is what decides + * that. Quadratic is a statement about GROWTH, so the same predicate is timed at + * 35 KB and at ten times that, and the question is what the tenfold text costs: + * linear answers about ten, and the quadratic pattern this replaced answers about a + * hundred, which is what its own numbers above say (2.5x the text, 6.25x the time). + * The absolute `< 200ms` this replaces measured the machine as much as the code - + * it is one sample on a shared box, and a test process descheduled for 200ms of + * its own turn reports a performance regression that is not there. + * + * Best of nine, interleaved: preemption only ever ADDS time, so the minimum of a + * run is its least-polluted sample, and alternating the two sizes keeps a slow + * patch of the machine from landing on one of them alone. Measured here on 20 cores: + * 10.3x idle, worst of 40 concurrent processes 17.8x, worst of 64 25.1x, so the + * ceiling of 40 sits above every load this suite is run under and well below the + * defect. + * + * Both answers are asserted on every sample, so a predicate that returned early + * cannot pass this by being fast for a reason the ratio cannot see. */ - test("answers in bounded time on a statement holding many UPDATE words and no SET", () => { - const query = `SELECT ${"UPDATE ".repeat(20000)}(`; + test("answers in time that grows with the statement, not with its square", () => { + const manyUpdates = (repeats: number) => `SELECT ${"UPDATE ".repeat(repeats)}(`; + const small = manyUpdates(5_000); // 35 KB + const large = manyUpdates(50_000); // 350 KB, ten times the text + + const timeOne = (query: string) => { + const started = performance.now(); + const dangerous = isDangerousQuery(query); + const elapsed = performance.now() - started; + expect(dangerous).toBe(false); + return elapsed; + }; - const started = performance.now(); - const dangerous = isDangerousQuery(query); - const elapsed = performance.now() - started; + let smallBest = Number.POSITIVE_INFINITY; + let largeBest = Number.POSITIVE_INFINITY; + for (let round = 0; round < 9; round += 1) { + smallBest = Math.min(smallBest, timeOne(small)); + largeBest = Math.min(largeBest, timeOne(large)); + } - expect(dangerous).toBe(false); - expect(elapsed, `took ${elapsed.toFixed(1)}ms`).toBeLessThan(200); + const ratio = largeBest / smallBest; + const measured = `${smallBest.toFixed(1)}ms at 35KB, ${largeBest.toFixed(1)}ms at 350KB, ${ratio.toFixed(1)}x`; + expect(ratio, measured).toBeLessThan(40); + // And a ceiling the ratio cannot see: a rewrite that is uniformly slow keeps its + // shape while costing a second per execute. 17.4ms measured here, 6406ms for the + // pattern this replaced, so 2000ms separates the two without measuring the + // machine the way the old absolute budget did. + expect(largeBest, measured).toBeLessThan(2000); }); // ── The gate reads what the RUNNER will run (S1) ───────────────────────── diff --git a/tests/components/SchemaDiagram.test.tsx b/tests/components/SchemaDiagram.test.tsx index 91b2eeecb..92f624cf8 100644 --- a/tests/components/SchemaDiagram.test.tsx +++ b/tests/components/SchemaDiagram.test.tsx @@ -162,7 +162,7 @@ mock.module("@zumer/snapdom", () => ({ })); import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { render, fireEvent, within, cleanup, act } from "@testing-library/react"; +import { render, fireEvent, within, cleanup, act, waitFor, waitForElementToBeRemoved } from "@testing-library/react"; import React from "react"; import { renderToStaticMarkup } from "react-dom/server"; @@ -383,6 +383,36 @@ function createDefaultProps(overrides: Partial[ }; } +/** + * Clicks an export button and returns when the export has actually finished. + * + * WHY THERE IS A HELPER AT ALL. Ten tests used to click and then sleep for a fixed 20ms or + * 40ms, which is not a wait for the export but a bet on how long one takes. The chain is + * `setExporting(format)`, two `yieldToPaint` hops (`requestAnimationFrame` then `setTimeout`), + * `getNodesBounds`, snapdom, a blob, a download; on an idle box that is a couple of + * milliseconds and on a box running one bun process per test file it is not. Every assertion + * after such a sleep - the capture happened, the toast was raised, culling went back on - + * then reads state the export may not have reached, and the test reports a product defect. + * + * WHAT IS WAITED ON. `exporting` is the component's own name for "an export is in flight": + * it is set before the first yield and cleared in the `finally`, so it spans the whole chain + * including the error path, and `disabled={exporting !== null}` puts it on the button where a + * test can read it. Going idle again is therefore co-extensive with the export being over. + * + * THE CONTROL. `expect(button.disabled).toBe(true)` right after the click is what stops the + * wait from being vacuous: `fireEvent` is act-wrapped and `setExporting` runs before the first + * await, so a click that started an export is disabled by then. Without that line, a click + * that started nothing at all - the early return when there are no nodes, say - would satisfy + * "not disabled" immediately and every assertion after it would be about an export that never + * ran. + */ +async function exportAndSettle(view: ReturnType, format: "PNG" | "SVG"): Promise { + const button = view.getByText(format).closest("button") as HTMLButtonElement; + fireEvent.click(button); + expect(button.disabled).toBe(true); + await waitFor(() => expect(button.disabled).toBe(false)); +} + // ============================================================================= // SchemaDiagram Tests // ============================================================================= @@ -519,13 +549,9 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - const pngButton = view.getByText("PNG").closest("button")!; // Let the async export flow finish inside this test so it cannot bleed // into later tests (the mocks are shared module-level state). - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 40)); - }); + await exportAndSettle(view, "PNG"); // Should not throw }); @@ -534,11 +560,7 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - const svgButton = view.getByText("SVG").closest("button")!; - await act(async () => { - fireEvent.click(svgButton); - await new Promise((r) => setTimeout(r, 40)); - }); + await exportAndSettle(view, "SVG"); // Should not throw }); @@ -1200,11 +1222,7 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - const pngButton = view.getByText("PNG").closest("button")!; - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 20)); - }); + await exportAndSettle(view, "PNG"); expect(mockSnapdom).toHaveBeenCalledTimes(1); const [capturedEl, options] = mockSnapdom.mock.calls[0] as unknown as [HTMLElement, Record]; @@ -1256,11 +1274,7 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - const svgButton = view.getByText("SVG").closest("button")!; - await act(async () => { - fireEvent.click(svgButton); - await new Promise((r) => setTimeout(r, 20)); - }); + await exportAndSettle(view, "SVG"); expect(mockSnapdom).toHaveBeenCalledTimes(1); const [capturedEl] = mockSnapdom.mock.calls[0] as unknown as [HTMLElement]; @@ -1287,11 +1301,7 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - const pngButton = view.getByText("PNG").closest("button")!; - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 20)); - }); + await exportAndSettle(view, "PNG"); const [, options] = mockSnapdom.mock.calls[0] as unknown as [HTMLElement, Record]; expect(options.backgroundColor).toBe("#050505"); @@ -1319,11 +1329,7 @@ describe("SchemaDiagram", () => { await Promise.resolve(); }); - const pngButton = view.getByText("PNG").closest("button")!; - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 20)); - }); + await exportAndSettle(view, "PNG"); const [, options] = mockSnapdom.mock.calls[0] as unknown as [HTMLElement, Record]; expect(options.backgroundColor).toBe("#050505"); @@ -1352,22 +1358,23 @@ describe("SchemaDiagram", () => { const props = createDefaultProps({ schema: schemaNoFK }); const { container } = render(); const view = within(container); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); + // No wait here: the grid fallback is the FIRST paint's own node set, and the ELK result + // cannot land before the `resolveLayout()` below, which this test holds. Nothing is in + // flight, so a sleep would only be a window in which nothing could happen anyway. // Still on the grid fallback: 320px apart, not the ELK 100px. expect((lastReactFlowProps.nodes as Array<{ position: { x: number } }>)[1].position.x).toBe(320); - const pngButton = view.getByText("PNG").closest("button")!; + // The one export in this file that cannot use `exportAndSettle`: the halves have to + // stay apart, because the ELK result is committed in the MIDDLE of the chain. + const pngButton = view.getByText("PNG").closest("button") as HTMLButtonElement; fireEvent.click(pngButton); + expect(pngButton.disabled).toBe(true); // Commit the ELK result in its own act, so it lands between the click // and the macrotasks the two paint yields wait on. await act(async () => { resolveLayout(); }); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); + await waitFor(() => expect(pngButton.disabled).toBe(false)); expect(mockGetNodesBounds).toHaveBeenCalledTimes(1); const measured = mockGetNodesBounds.mock.calls[0][0] as Array<{ position: { x: number } }>; @@ -1394,11 +1401,9 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - const pngButton = view.getByText("PNG").closest("button")!; - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 20)); - }); + // `exporting` is cleared in the `finally`, so the failure path settles the same way a + // successful one does and the toast is on screen when the wait ends. + await exportAndSettle(view, "PNG"); expect(mockToastError).toHaveBeenCalled(); }); @@ -1411,11 +1416,15 @@ describe("SchemaDiagram", () => { fireEvent.change(view.getByPlaceholderText("Filter tables..."), { target: { value: "no-such-table" } }); expect(view.queryByText("0 tables")).not.toBeNull(); - const pngButton = view.getByText("PNG").closest("button")!; - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 20)); - }); + // Asserted synchronously, and that is the point rather than an economy: the refusal + // runs BEFORE `exportDiagram`'s first await - no nodes, so it toasts and returns + // without ever setting `exporting` - so there is nothing in flight to wait for. + // `disabled` still being false is the control that keeps the negative below honest: it + // says the export was refused, not that it had merely not started yet, which is exactly + // what a fixed sleep cannot tell apart. + const pngButton = view.getByText("PNG").closest("button") as HTMLButtonElement; + fireEvent.click(pngButton); + expect(pngButton.disabled).toBe(false); expect(mockSnapdom).not.toHaveBeenCalled(); expect(mockToastError).toHaveBeenCalled(); @@ -1430,11 +1439,7 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - const pngButton = view.getByText("PNG").closest("button")!; - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 20)); - }); + await exportAndSettle(view, "PNG"); expect(mockToastError).toHaveBeenCalled(); }); @@ -1462,11 +1467,7 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - const pngButton = view.getByText("PNG").closest("button")!; - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 40)); - }); + await exportAndSettle(view, "PNG"); expect(mockSnapdom).toHaveBeenCalledTimes(1); expect(mockToBlob).toHaveBeenCalledTimes(1); @@ -1594,11 +1595,7 @@ describe("SchemaDiagram", () => { expect(lastReactFlowProps.onlyRenderVisibleElements).toBe(true); - const pngButton = view.getByText("PNG").closest("button")!; - await act(async () => { - fireEvent.click(pngButton); - await new Promise((r) => setTimeout(r, 40)); - }); + await exportAndSettle(view, "PNG"); // Culled (unmounted) nodes cannot be captured - the snapshot must run // with culling off so every table is in the DOM. @@ -1638,30 +1635,35 @@ describe("SchemaDiagram", () => { foreignKeys: [{ columnName: "user_id", referencedTable: "users", referencedColumn: "id" }], }, ]; - await act(async () => { - rerender(); - await new Promise((r) => setTimeout(r, 20)); - }); - - // posts is now a neighbor of the still-selected users -> highlighted - expect( - container.querySelector('[data-node-id="posts"]')!.querySelector(".border-brand-tint\\/60"), - ).not.toBeNull(); + rerender(); + + // posts is now a neighbor of the still-selected users -> highlighted. + // The wait is on that class arriving, which is the fact this test is about; the 20ms + // sleep it replaces asserted only that 20ms had passed, and the neighbour set is + // recomputed off the new FK data through the highlight store rather than in the render + // that `rerender` flushed. + await waitFor(() => + expect( + container.querySelector('[data-node-id="posts"]')!.querySelector(".border-brand-tint\\/60"), + ).not.toBeNull(), + ); }); test("node internals re-measure when FK anchors appear on existing tables", async () => { const onClose = mock(() => {}); const { rerender } = render(); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); + // The baseline has to be taken once the mount has stopped moving, and `fitView` is the + // signal that it has: it is called after the layout completes, which is the last thing + // that rebuilds the graph and so the last thing that can re-measure a node. + await waitFor(() => expect(mockFitView).toHaveBeenCalled()); const callsBefore = mockUpdateNodeInternals.mock.calls.length; - // Identity-only rebuild (same schema content) must NOT re-measure - await act(async () => { - rerender(); - await new Promise((r) => setTimeout(r, 20)); - }); + // Identity-only rebuild (same schema content) must NOT re-measure. + // No wait after the rerender: `TableNode`'s re-measure is a passive effect keyed on the + // handle signature, and RTL act-wraps `rerender`, so any effect this rebuild was going + // to run has already run when the line below reads the counter. A sleep here would add + // a window in which nothing new can happen and call it evidence. + rerender(); expect(mockUpdateNodeInternals.mock.calls.length).toBe(callsBefore); // FK arrival adds handles -> React Flow must be told to re-measure, @@ -1674,11 +1676,8 @@ describe("SchemaDiagram", () => { foreignKeys: [{ columnName: "user_id", referencedTable: "users", referencedColumn: "id" }], }, ]; - await act(async () => { - rerender(); - await new Promise((r) => setTimeout(r, 20)); - }); - expect(mockUpdateNodeInternals.mock.calls.length).toBeGreaterThan(callsBefore); + rerender(); + await waitFor(() => expect(mockUpdateNodeInternals.mock.calls.length).toBeGreaterThan(callsBefore)); }); }); @@ -1717,17 +1716,18 @@ describe("SchemaDiagram", () => { const props = createDefaultProps({ schema: wideTable }); const { container } = render(); const view = within(container); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); + // The spinner is derived from `signature !== layoutedSignature`, so it is on screen + // from the first paint and clears when the layout settles - including this one, which + // settles by answering null. Waiting for it to go is waiting for the fact; the 20ms + // sleep it replaces was a guess at how long a promise takes to come back. + await waitForElementToBeRemoved(() => view.queryByText("layout")); expect(layoutCalls).toBe(1); // Expanding a table changes graph identity but not structure - the - // known-failed layout must not rerun (no spinner churn). + // known-failed layout must not rerun (no spinner churn). No wait after the click: + // the layout effect calls the engine synchronously when it runs, and `fireEvent` is + // act-wrapped, so a rerun would already be counted below. fireEvent.click(view.getByText(/\+\d+ more/)); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); expect(layoutCalls).toBe(1); expect(view.queryByText("col_29")).not.toBeNull(); }); @@ -1745,18 +1745,15 @@ describe("SchemaDiagram", () => { const props = createDefaultProps({ schema: wideTable }); const { container } = render(); const view = within(container); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); + // The catch handler must clear the layouting spinner, and its removal is both the + // wait and the assertion: `waitForElementToBeRemoved` refuses to run at all unless the + // spinner was there to begin with, so it carries its own control. + await waitForElementToBeRemoved(() => view.queryByText("layout")); expect(layoutCalls).toBe(1); - // The catch handler must clear the layouting spinner... - expect(view.queryByText("layout")).toBeNull(); - // ...and record the signature so cosmetic rebuilds do not retry. + // ...and record the signature so cosmetic rebuilds do not retry. No wait after the + // click, for the reason the null-layout test above gives. fireEvent.click(view.getByText(/\+\d+ more/)); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); expect(layoutCalls).toBe(1); expect(view.queryByText("col_29")).not.toBeNull(); }); @@ -1771,14 +1768,13 @@ describe("SchemaDiagram", () => { const props = createDefaultProps(); render(); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); - // fitView is scheduled behind a paint yield, so any bookkeeping write // that re-runs the layout effect would fire its cleanup, set // `cancelled` and swallow this call — leaving the diagram unfitted. - expect(mockFitView).toHaveBeenCalledWith({ padding: 0.15 }); + // Waited on directly rather than behind a sleep: the call IS the assertion, so a wait + // for it cannot end early, and a machine slower than the sleep no longer reports a + // swallowed fit-view that was merely late. + await waitFor(() => expect(mockFitView).toHaveBeenCalledWith({ padding: 0.15 })); }); test("the layout spinner shows while ELK is in flight and clears when it resolves", async () => { @@ -1795,17 +1791,16 @@ describe("SchemaDiagram", () => { const { container } = render(); const view = within(container); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); + // No wait for the spinner to appear: it is derived during render rather than set by an + // effect, so it is on screen from the first paint and stays there for as long as this + // test holds the layout promise. Nothing can take it away in the meantime. expect(view.queryByText("layout")).not.toBeNull(); - await act(async () => { + act(() => { // null keeps the grid fallback but still completes the layout. resolveLayout(null); - await new Promise((r) => setTimeout(r, 20)); }); - expect(view.queryByText("layout")).toBeNull(); + await waitForElementToBeRemoved(() => view.queryByText("layout")); }); }); @@ -1903,9 +1898,11 @@ describe("SchemaDiagram", () => { const props = createDefaultProps({ schema: wideTable }); const { container } = render(); const view = within(container); - await act(async () => { - await new Promise((r) => setTimeout(r, 20)); - }); + // The drag has to happen AFTER the layout has landed, or the ELK positions arrive on + // top of the dragged one and this test fails for a reason that is not the subject. + // `fitView` is called once the layout completes, so it is that moment by name rather + // than 20ms of hoping. + await waitFor(() => expect(mockFitView).toHaveBeenCalled()); const onNodesChange = lastReactFlowProps.onNodesChange as (changes: unknown[]) => void; act(() => { diff --git a/tests/components/Studio.test.tsx b/tests/components/Studio.test.tsx index a81c8b63d..87bf81b8b 100644 --- a/tests/components/Studio.test.tsx +++ b/tests/components/Studio.test.tsx @@ -452,8 +452,8 @@ mock.module("@/components/agent/AgentRail", () => ({ * always holds an ask, and what the rail is handed has to BE it. The hook's own * behaviour — that nothing is asked for until a shortcut asks, and what an ask * contains — is covered in tests/hooks/use-agent-prefill.test.ts, which runs in a - * different process: `mock.module` is process-wide, and Studio.test.tsx is its own - * isolation group (tests/run-components.sh Group 1), so no suite shares this stub. + * different process: `mock.module` is process-wide, and the runner gives every test + * file a bun process of its own, so no other suite ever sees this stub. */ const PREFILL_SENTINEL = { id: 7, diff --git a/tests/components/VisualExplain.test.tsx b/tests/components/VisualExplain.test.tsx index a44264aa4..c7ecef3c4 100644 --- a/tests/components/VisualExplain.test.tsx +++ b/tests/components/VisualExplain.test.tsx @@ -1067,8 +1067,13 @@ describe("tree render model (sqlite-queryplan)", () => { rerender(); await waitFor(() => { - // the previous plan's response is gone and the tab is back to its initial state - expect(queryByText("Old Analysis")).toBeNull(); + // the previous plan's response is gone and the tab is back to its initial state. + // `=== null` and not `toBeNull()` on the node, in this poll and in the one the next test + // runs: a FAILING poll would make bun pretty-print the element, which walks the whole + // happy-dom node's object graph. 301 ms for a 260-node subtree, measured, so a few polls + // eat waitFor's 5 s budget and a briefly busy machine reds a healthy test. The boolean is + // 0 ms and asserts the same absence. + expect(queryByText("Old Analysis") === null).toBe(true); expect(queryByText("AI Query Analysis")).not.toBeNull(); // the previous request's controller was aborted expect(abortSpy).toHaveBeenCalled(); @@ -1099,7 +1104,7 @@ describe("tree render model (sqlite-queryplan)", () => { rerender(); await waitFor(() => { - expect(queryByText("Old Analysis")).toBeNull(); + expect(queryByText("Old Analysis") === null).toBe(true); expect(queryByText("AI Query Analysis")).not.toBeNull(); }); }); diff --git a/tests/components/admin/AdminDashboard.test.tsx b/tests/components/admin/AdminDashboard.test.tsx index 33ad2d93d..abc5fdcca 100644 --- a/tests/components/admin/AdminDashboard.test.tsx +++ b/tests/components/admin/AdminDashboard.test.tsx @@ -34,9 +34,9 @@ describe("AdminDashboard", () => { afterEach(() => { cleanup(); resetMockPathname(); - // Restore via the shared helper: this file shares a process with the other - // admin tests (run-components.sh Group 4), and an un-restored global fetch - // leaks into whichever file runs next. + // Restore via the shared helper: every test in this file shares one process + // and one global `fetch`, so an un-restored stub decides the outcome of + // whichever test runs after it. restoreGlobalFetch(); }); diff --git a/tests/components/admin/AuditTab.test.tsx b/tests/components/admin/AuditTab.test.tsx index 82e2f8b7a..8331c68c3 100644 --- a/tests/components/admin/AuditTab.test.tsx +++ b/tests/components/admin/AuditTab.test.tsx @@ -360,10 +360,16 @@ describe("AuditTab", () => { await user.clear(searchInput); await user.type(searchInput, "VACUUM"); - // VACUUM should still be visible, KILL should be filtered out + // VACUUM should still be visible, KILL should be filtered out. + // The filtered-out half is written `=== null` rather than `toBeNull()` on the node, here and + // in the two other filter tests below: a FAILING poll hands bun a live happy-dom element and + // bun walks its whole object graph to build the diff, 301 ms for a 260-node subtree measured. + // A few of those and waitFor's 5 s budget is spent, so a briefly busy machine reds a healthy + // test. The boolean costs 0 ms. The present half stays as it is: it fails on `null`, which is + // cheap to print. await waitFor(() => { expect(queryByText("VACUUM")).not.toBeNull(); - expect(queryByText("KILL")).toBeNull(); + expect(queryByText("KILL") === null).toBe(true); }); }); @@ -420,7 +426,7 @@ describe("AuditTab", () => { await waitFor(() => { expect(queryByText("SELECT 1")).not.toBeNull(); - expect(queryByText("DROP TABLE x")).toBeNull(); + expect(queryByText("DROP TABLE x") === null).toBe(true); }); }); @@ -460,7 +466,7 @@ describe("AuditTab", () => { // Only the error-status history item remains await waitFor(() => { expect(queryByText("DROP TABLE x")).not.toBeNull(); - expect(queryByText("SELECT 1")).toBeNull(); + expect(queryByText("SELECT 1") === null).toBe(true); }); }); test("changing the type filter refetches with the type param", async () => { diff --git a/tests/components/agent/AgentRail.test.tsx b/tests/components/agent/AgentRail.test.tsx index 13edb5b4d..35aa7e1cc 100644 --- a/tests/components/agent/AgentRail.test.tsx +++ b/tests/components/agent/AgentRail.test.tsx @@ -8,6 +8,7 @@ import { cleanup, render, renderHook, fireEvent, waitFor, act, type RenderResult import { AgentRail } from "@/components/agent/AgentRail"; import { useConnectionManager } from "@/hooks/use-connection-manager"; import { + NO_SERVED_SEEDS, resolveAgentRunConnectionId, SEED_CONFIG_UNREADABLE_REASON, type ManagedConnectionPayload, @@ -5106,8 +5107,12 @@ describe("AgentRail", () => { // The same answer every other classification failure reaches, and the rail is // idle again rather than stuck behind a request nobody will answer. expect(openRequests(fetchMock)[0]).toMatchObject({ workflowType: "investigation" }); + // `=== null` rather than `toBeNull()` on the node: a FAILING poll would hand bun the live + // happy-dom element, and bun walks its whole object graph to build the diff, measured at + // 301 ms for a 260-node subtree. A few of those spend waitFor's 5 s budget and a briefly + // busy machine reds a healthy test. The boolean costs 0 ms and asserts the same absence. await waitFor(() => { - expect(view.queryByTestId("agent-classifying")).toBeNull(); + expect(view.queryByTestId("agent-classifying") === null).toBe(true); }); } finally { AbortSignal.timeout = realTimeout; @@ -6627,8 +6632,22 @@ describe("a seed configuration the server could not read (B37)", () => { "/api/agent/config": { json: { enabled: true } }, }); const hook = renderHook(() => useConnectionManager(true)); + // Wait for the answer to have been APPLIED, not for the field to exist. `servedSeeds` + // starts life holding `NO_SERVED_SEEDS`, so `toBeDefined()` was already true on the + // first render and this helper read the state the hook held before it had asked the + // server anything. That is green on an idle machine and a coin toss on a busy one: + // measured 2026-09-15 at 24 concurrent `bun test` processes on a 20-core box, the read + // landed on the untouched constant in 19 runs out of 24 (by identity, so nothing had + // written it), and the file itself failed 6 of those 24 runs on this test alone. + // + // The wait has to compare by identity, because the value cannot carry the difference: + // the control arm below settles on `{loaded: true, seeds: []}`, which is deep-equal to + // the constant it started from. "The server answered, with no seeds" and "nobody has + // answered yet" are the same value, so `!== NO_SERVED_SEEDS` is the only thing that + // separates them, and it is what makes the control arm a control at all rather than a + // test of the initial state. That conflation is B37's own, one level up: see B81. await waitFor(() => { - expect(hook.result.current.servedSeeds).toBeDefined(); + expect(hook.result.current.servedSeeds).not.toBe(NO_SERVED_SEEDS); }); const seeds = hook.result.current.servedSeeds; hook.unmount(); diff --git a/tests/components/monitoring/TablesTab.test.tsx b/tests/components/monitoring/TablesTab.test.tsx index e17a96e92..972b249c4 100644 --- a/tests/components/monitoring/TablesTab.test.tsx +++ b/tests/components/monitoring/TablesTab.test.tsx @@ -144,28 +144,50 @@ describe("TablesTab", () => { />, ); - const analyzeButton = container.querySelector('button[title="Analyze"]'); - const vacuumButton = container.querySelector('button[title="Vacuum"]'); - const reindexButton = container.querySelector('button[title="Reindex"]'); + /* + * THE WAIT THAT MATTERS HERE IS NOT THE ONE ON THE CALL. `handleMaintenance` invokes + * `onRunMaintenance` before its first await, and `fireEvent` is act-wrapped, so the call + * has already happened when the line after the click reads it: a `waitFor` around that + * assertion was satisfied on its first synchronous check and waited for nothing. + * + * What the next click actually needs is the button being clickable, and it is not: + * `handleMaintenance` sets `actionLoading` first, and `disabled={!!actionLoading}` takes + * EVERY maintenance button on the panel with it. React delivers no click to a disabled + * button, so a click issued while the previous action is still in flight is dropped in + * silence and the assertion after it burns its whole timeout on a call that will never + * come. Measured with a probe on this tree: `analyze` called synchronously, `vacuum` + * disabled immediately after, so this suite only ever got through because `waitFor` + * happens to drain a macrotask on its way out - a coincidence, and a busy machine is + * under no obligation to repeat it. + * + * So each step asserts the call synchronously, asserts the NEXT button went disabled (the + * control, which is what stops the wait below from being vacuous), and waits for it to + * come back before clicking it. + */ + const analyzeButton = container.querySelector('button[title="Analyze"]'); + const vacuumButton = container.querySelector('button[title="Vacuum"]'); + const reindexButton = container.querySelector('button[title="Reindex"]'); expect(analyzeButton).not.toBeNull(); expect(vacuumButton).not.toBeNull(); expect(reindexButton).not.toBeNull(); fireEvent.click(analyzeButton!); + expect(onRunMaintenance).toHaveBeenCalledWith("analyze", "users"); + expect(vacuumButton!.disabled).toBe(true); await waitFor(() => { - expect(onRunMaintenance).toHaveBeenCalledWith("analyze", "users"); + expect(vacuumButton!.disabled).toBe(false); }); fireEvent.click(vacuumButton!); + expect(onRunMaintenance).toHaveBeenCalledWith("vacuum", "users"); + expect(reindexButton!.disabled).toBe(true); await waitFor(() => { - expect(onRunMaintenance).toHaveBeenCalledWith("vacuum", "users"); + expect(reindexButton!.disabled).toBe(false); }); fireEvent.click(reindexButton!); - await waitFor(() => { - expect(onRunMaintenance).toHaveBeenCalledWith("reindex", "users"); - }); + expect(onRunMaintenance).toHaveBeenCalledWith("reindex", "users"); }); test("shows non-admin placeholder for actions", () => { diff --git a/tests/components/object-source/ObjectSourceView.test.tsx b/tests/components/object-source/ObjectSourceView.test.tsx index 90b620072..a11a305c7 100644 --- a/tests/components/object-source/ObjectSourceView.test.tsx +++ b/tests/components/object-source/ObjectSourceView.test.tsx @@ -632,7 +632,15 @@ describe("ObjectSourceView", () => { render(); await waitFor(() => expect(screen.getByTestId("source-editor")).toBeTruthy()); - expect(definedThemes).toContain(STUDIO_THEME_DARK); + /* + * The registration is polled, the theme in use is not, and the difference is where each one + * happens: `theme` is a render prop, so it is on the element the poll above already found, + * while `beforeMount` runs from the editor double's passive effect, which React schedules + * AFTER the commit that put that element on screen. Reading `definedThemes` straight after + * the element therefore races the effect, and it lost on 5 of 24 concurrent runs of this file + * with "Expected to contain: db-dark / Received: []" before this poll was here. + */ + await waitFor(() => expect(definedThemes).toContain(STUDIO_THEME_DARK)); expect(definedThemes).toContain(STUDIO_THEME_LIGHT); expect(screen.getByTestId("source-editor").getAttribute("data-theme")).toBe(STUDIO_THEME_LIGHT); }); @@ -703,7 +711,18 @@ describe("ObjectSourceView", () => { expect(clear).toBeTruthy(); expect(Object.hasOwn(clear!, "failure")).toBe(true); expect(Object.hasOwn(clear!, "readAtToken")).toBe(true); - await waitFor(() => expect(screen.queryByTestId("object-source-stale")).toBeNull()); + /* + * `=== null` and not `expect(node).toBeNull()`, here and at every other absence poll in this + * file. A poll that FAILS hands bun the live happy-dom node to pretty-print, and bun walks the + * whole node's object graph to build the diff: measured at 301 ms for a 260-node subtree, so a + * handful of failing polls eats waitFor's entire 5 s budget and a momentarily slow machine + * turns a healthy test red. The boolean costs 0 ms and asserts exactly the same removal. + * + * `waitForElementToBeRemoved` is not the alternative: it demands the element still be present + * when it is called, and at this site, and at all five others below, it is already gone by + * then. That was measured, by logging the element on the line above each poll. + */ + await waitFor(() => expect(screen.queryByTestId("object-source-stale") === null).toBe(true)); }); test("keeps the part the reader was on across a re-read, so the stale control does not move them", async () => { @@ -728,7 +747,7 @@ describe("ObjectSourceView", () => { rerender(); await userEvent.click(screen.getByTestId("object-source-stale-reread")); await waitFor(() => expect(reader.calls).toBe(2)); - await waitFor(() => expect(screen.queryByTestId("object-source-stale")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-stale") === null).toBe(true)); expect(screen.getByTestId("source-editor").getAttribute("data-path")?.endsWith("/body")).toBe(true); expect(screen.getAllByRole("tab").map((tab) => tab.getAttribute("aria-selected"))).toEqual(["false", "true"]); @@ -2338,7 +2357,7 @@ describe("ObjectSourceView edit mode", () => { await click("object-source-apply-cancel"); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); expect(editor().readOnly).toBe(false); expect((screen.getByTestId("object-source-preview") as HTMLButtonElement).disabled).toBe(false); }); @@ -2366,7 +2385,7 @@ describe("ObjectSourceView edit mode", () => { await click("object-source-apply-confirm"); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); expect(readDraft(window.localStorage, draftKey("definition"))).toBeUndefined(); expect(patches).toContainEqual(expect.objectContaining({ editingPartId: undefined, dirty: undefined })); expect(applied).toHaveBeenCalledTimes(1); @@ -2626,7 +2645,7 @@ describe("ObjectSourceView edit mode", () => { await Promise.resolve(); }); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); expect(readDraft(window.localStorage, draftKey("definition"))).toBeUndefined(); }); @@ -3023,7 +3042,7 @@ describe("ObjectSourceView across two Source tabs", () => { await waitFor(() => expect(screen.getByTestId("object-source-apply-building")).toBeTruthy()); await click("object-source-apply-cancel"); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); await act(async () => { land(BUILT); await Promise.resolve(); diff --git a/tests/components/object-tree.test.tsx b/tests/components/object-tree.test.tsx index 73d3c5b49..38cf2640d 100644 --- a/tests/components/object-tree.test.tsx +++ b/tests/components/object-tree.test.tsx @@ -106,9 +106,23 @@ function row(name: string | RegExp): HTMLElement { return screen.getByRole("treeitem", { name }); } +/** + * Opens the `app` container and returns once its folders carry the engine's counts. + * + * WAITING ON THE FOLDER ROW IS NOT ENOUGH, and the difference is one network round trip. A + * folder is drawn from the DECLARED kinds, which the capabilities already carry, so `Tables` + * is on screen the moment the container opens and before the counts read has answered. Every + * caller then reads something the counts supply - a badge, a refusal sentence, an aria-busy + * flag - with a synchronous `getByTestId`, which on a machine that has not answered yet throws + * rather than retries. Returning on the badge makes the counts answer the postcondition. + * + * All fifteen callers install a `table` count, `{ count: 0 }` included, which still draws a + * badge reading "0", so this is a wait the caller always satisfies and never a new premise. + */ async function expandApp(): Promise { await userEvent.click(await screen.findByRole("treeitem", { name: /app/ })); - await waitFor(() => expect(screen.getByRole("treeitem", { name: /Tables/ })).toBeTruthy()); + await screen.findByRole("treeitem", { name: /Tables/ }); + await within(row(/Tables/)).findByTestId("tree-row-badge"); } afterEach(() => { @@ -476,7 +490,9 @@ describe("ObjectTree container shapes", () => { const tables = await screen.findByRole("treeitem", { name: /Tables/ }); expect(tables.getAttribute("aria-level")).toBe("1"); - expect(within(tables).getByTestId("tree-row-badge").textContent).toBe("3"); + // `find`, not `get`: the folder row is drawn from the declared kinds, so it is on screen + // one round trip before the counts that badge it. + expect((await within(tables).findByTestId("tree-row-badge")).textContent).toBe("3"); expect(calls.map((call) => call.route)).toEqual(["counts"]); expect(calls[0]?.body.container).toEqual([]); }); @@ -501,7 +517,7 @@ describe("ObjectTree container shapes", () => { await userEvent.click(app); const tables = await screen.findByRole("treeitem", { name: /Tables/ }); expect(tables.getAttribute("aria-level")).toBe("3"); - expect(within(tables).getByTestId("tree-row-badge").textContent).toBe("7"); + expect((await within(tables).findByTestId("tree-row-badge")).textContent).toBe("7"); expect(calls.map((call) => `${call.route}:${JSON.stringify(call.body.parent ?? call.body.container)}`)).toEqual([ "containers:undefined", 'containers:["prod"]', @@ -526,7 +542,9 @@ describe("ObjectTree container shapes", () => { render(); const tables = await screen.findByRole("treeitem", { name: /Tables/ }); - expect(within(tables).getByTestId("tree-row-badge").textContent).toBe("5"); + // The confirmed flake of this file, red in 4 of 24 concurrent runs: the folder row answers + // `findByRole` as soon as the container opens, and the badge only arrives with the counts. + expect((await within(tables).findByTestId("tree-row-badge")).textContent).toBe("5"); expect(screen.getByRole("treeitem", { name: /a\/b/ }).getAttribute("aria-expanded")).toBe("true"); }); }); @@ -962,8 +980,14 @@ describe("useTreeNodes", () => { generation = 1; act(() => result.current.refresh()); - await waitFor(() => expect(result.current.rows.map((r) => r.label)).toContain("t1")); - expect(result.current.rows.find((r) => r.kindId === "table" && r.kind === "folder")?.badge).toBe("1"); + // The listing and the counts are two reads that settle independently, and neither implies + // the other. Waiting on the listing alone and asserting the badge on the line after left + // the badge read at whatever the counts read happened to have reached; both facts are + // asked for in one wait instead. + await waitFor(() => { + expect(result.current.rows.map((r) => r.label)).toContain("t1"); + expect(result.current.rows.find((r) => r.kindId === "table" && r.kind === "folder")?.badge).toBe("1"); + }); expect(calls.filter((call) => call.route === "counts")).toHaveLength(2); expect(calls.filter((call) => call.route === "list")).toHaveLength(2); expect(calls.filter((call) => call.route === "containers")).toHaveLength(2); @@ -1063,10 +1087,19 @@ describe("useTreeNodes", () => { act(() => result.current.toggle("app")); await waitFor(() => expect(result.current.failureFor(result.current.rows[0] as never)).toBeDefined()); + // The premise that made the old wait below vacuous, pinned so it cannot go quiet: the + // folders come from the DECLARED kinds, so the tree is already three rows while the counts + // read is still refusing. + expect(result.current.rows).toHaveLength(3); + failing = false; act(() => result.current.refresh()); - await waitFor(() => expect(result.current.rows).toHaveLength(3)); - expect(result.current.failureFor(result.current.rows[0] as never)).toBeUndefined(); + // So `waitFor(rows).toHaveLength(3)` ended on its first check, before `refresh` had issued + // anything, and the assertion after it read the failure that was still there. The wait is + // on the failure clearing, which is the fact this test is about. + await waitFor(() => expect(result.current.failureFor(result.current.rows[0] as never)).toBeUndefined()); + // And it was REPLACED by the engine's answer rather than merely forgotten. + expect(result.current.rows.find((r) => r.kindId === "table" && r.kind === "folder")?.badge).toBe("2"); }); test("changing the connection throws the whole cache away rather than showing the last one's tree", async () => { diff --git a/tests/components/object-tree/first-paint.test.tsx b/tests/components/object-tree/first-paint.test.tsx index a47ab7a34..09e63ab3d 100644 --- a/tests/components/object-tree/first-paint.test.tsx +++ b/tests/components/object-tree/first-paint.test.tsx @@ -2,7 +2,7 @@ import "../../setup-dom"; import "../../helpers/mock-navigation"; import { afterEach, describe, expect, mock, test } from "bun:test"; -import { cleanup, render, renderHook, screen, waitFor } from "@testing-library/react"; +import { cleanup, render, renderHook, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { ObjectTree } from "@/components/object-tree"; import { useTreeNodes } from "@/components/object-tree/use-tree-nodes"; @@ -108,6 +108,26 @@ const schemaContainers: Container[] = [ { path: ["public"], name: "public", level: 0 }, ]; +/** + * Returns once first paint has actually SETTLED, which the Tables row alone does not say. + * + * Every assertion in this file counts the reads first paint issued, and the folder rows are + * not evidence that the last of them went out: a folder is drawn from the DECLARED kinds the + * moment the session-default container opens, so `Tables` is on screen while the counts + * request is still being issued from the effect after that paint. A test that stopped there + * read `calls` in the window between the two and reported a missing read as a product + * decision - `a two-level engine descends to the session default at each level` is the one + * that actually did it, red in 24-way concurrent runs. + * + * The badge is the counts ANSWER: `TreeRow` draws it only where `row.badge` is defined, so + * waiting on its text waits on the read this file is counting, and asserting the text rather + * than the node's presence keeps the wait tied to the fixture's own number. + */ +async function firstPaintSettled(badge: string): Promise { + const tables = await screen.findByRole("treeitem", { name: /Tables/ }); + expect((await within(tables).findByTestId("tree-row-badge")).textContent).toBe(badge); +} + afterEach(() => { cleanup(); globalThis.fetch = realFetch; @@ -122,7 +142,7 @@ describe("opening a connection", () => { render(); - await waitFor(() => expect(screen.getByRole("treeitem", { name: /Tables/ })).toBeTruthy()); + await firstPaintSettled("2"); expect(catalogPaths(calls)).toEqual(["/api/db/objects/containers", "/api/db/objects/counts"]); // The counts are read for the container the ENGINE named, not for the first one @@ -197,7 +217,7 @@ describe("opening a connection", () => { render(); - await waitFor(() => expect(screen.getByRole("treeitem", { name: /Tables/ })).toBeTruthy()); + await firstPaintSettled("7"); expect(catalogPaths(calls)).toEqual([ "/api/db/objects/containers", @@ -266,7 +286,7 @@ describe("the no-scan escape hatch", () => { render(); - await waitFor(() => expect(screen.getByRole("treeitem", { name: /Tables/ })).toBeTruthy()); + await firstPaintSettled("2"); expect(catalogPaths(calls)).toEqual(["/api/db/objects/containers", "/api/db/objects/counts"]); expect(screen.queryByTestId("tree-deferred")).toBeNull(); }); @@ -298,7 +318,7 @@ describe("the no-scan escape hatch", () => { // What the owner does with it, which is the state the reader then sees. rerender(); - await waitFor(() => expect(screen.getByRole("treeitem", { name: /Tables/ })).toBeTruthy()); + await firstPaintSettled("2"); expect(catalogPaths(calls)).toEqual(["/api/db/objects/containers", "/api/db/objects/counts"]); }); @@ -329,7 +349,10 @@ describe("the no-scan escape hatch", () => { // loading state the assertion above denies. rerender({ deferred: false }); expect(result.current.rootLoading).toBe(true); - await waitFor(() => expect(result.current.rows.length).toBeGreaterThan(1)); + // The badge, for the reason `firstPaintSettled` gives above: `rows.length > 1` was already + // true with the two containers alone, before the counts read had been issued at all, so the + // read count below was asserted against a paint that was still one request short. + await waitFor(() => expect(result.current.rows.find((r) => r.kind === "folder")?.badge).toBe("2")); expect(catalogPaths(calls)).toEqual(["/api/db/objects/containers", "/api/db/objects/counts"]); }); diff --git a/tests/components/object-tree/row-menu.test.tsx b/tests/components/object-tree/row-menu.test.tsx index c4dd3447d..f1356a1c8 100644 --- a/tests/components/object-tree/row-menu.test.tsx +++ b/tests/components/object-tree/row-menu.test.tsx @@ -316,7 +316,13 @@ describe("the row menu is reachable without a pointer", () => { await userEvent.keyboard("{ContextMenu}"); await userEvent.tab(); - await waitFor(() => expect(screen.queryByRole("menu")).toBeNull()); + /* + * `=== null` rather than `toBeNull()` on the node, here and at the two other absence polls in + * this file. A poll that FAILS makes bun pretty-print the menu, which means walking the whole + * happy-dom node's object graph: 301 ms for a 260-node subtree, measured. waitFor's 5 s budget + * is then gone in a few polls and a busy machine reds a healthy test. The boolean costs 0 ms. + */ + await waitFor(() => expect(screen.queryByRole("menu") === null).toBe(true)); }); test("a click outside the menu closes it", async () => { @@ -327,7 +333,7 @@ describe("the row menu is reachable without a pointer", () => { expect(screen.queryByRole("menu")).not.toBeNull(); await userEvent.click(document.body); - await waitFor(() => expect(screen.queryByRole("menu")).toBeNull()); + await waitFor(() => expect(screen.queryByRole("menu") === null).toBe(true)); }); }); @@ -490,7 +496,7 @@ describe("the row menu and the rows under it", () => { expect(screen.queryByRole("menu")).not.toBeNull(); await userEvent.click(row(/Tables/)); - await waitFor(() => expect(screen.queryByText("orders")).toBeNull()); + await waitFor(() => expect(screen.queryByText("orders") === null).toBe(true)); expect(screen.queryByRole("menu")).toBeNull(); }); }); diff --git a/tests/components/schema-explorer/TableItem.test.tsx b/tests/components/schema-explorer/TableItem.test.tsx index 56709723c..21595d573 100644 --- a/tests/components/schema-explorer/TableItem.test.tsx +++ b/tests/components/schema-explorer/TableItem.test.tsx @@ -6,9 +6,9 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import { cleanup, fireEvent, render, waitFor, within } from "@testing-library/react"; // The SHARED sonner mock rather than a local `mock.module("sonner", ...)`: mock.module is -// process-wide and the last call wins, so a second declaration here would hand -// StudioMobileHeader.test.tsx — same group in tests/run-components.sh — a `toast` whose -// error mock it holds no reference to. +// process-wide and the last call wins, so a second declaration here would replace the one +// bunfig preloads for this process and hand the tests below a `toast` whose error mock +// nothing in this file holds a reference to. import { mockToastError, mockToastSuccess } from "../../helpers/mock-sonner"; // The insecure-context harness, as in tests/components/copy-button.test.tsx: an absent diff --git a/tests/components/studio-agent-ask.test.tsx b/tests/components/studio-agent-ask.test.tsx index 149e6292f..ec820cafb 100644 --- a/tests/components/studio-agent-ask.test.tsx +++ b/tests/components/studio-agent-ask.test.tsx @@ -17,7 +17,7 @@ import React from "react"; * stubs `@/components/CommandPalette`, and it overrides `use-tab-manager`. `mock.module` * is process-wide and those stubs are registered at module scope, so a test in that * file cannot see the real hook, the real palette item, or the real tab. Hence a second - * file, in its own isolation group (tests/run-components.sh), which mocks LESS: + * file, which the runner gives a process of its own like every other, and which mocks LESS: * * real: the command palette and its item, `use-tab-manager`, `use-agent-prefill` * (so the id minting and the objective clamp actually run), and Studio's own diff --git a/tests/components/studio/StudioMobileHeader.test.tsx b/tests/components/studio/StudioMobileHeader.test.tsx index 01df3df5a..37b8b3008 100644 --- a/tests/components/studio/StudioMobileHeader.test.tsx +++ b/tests/components/studio/StudioMobileHeader.test.tsx @@ -38,9 +38,9 @@ mock.module("@/components/ui/dropdown-menu", () => ({ })); // The real provider writes to and localStorage; what matters here is only that a -// provider EXISTS, since `ThemeToggle` renders nothing when `themes` is empty. Group 9 in -// tests/run-components.sh holds no suite that reaches the real next-themes, so this -// process-wide mock costs its neighbours nothing. +// provider EXISTS, since `ThemeToggle` renders nothing when `themes` is empty. `mock.module` +// is process-wide and this file is the whole of that process, so a suite that does need the +// real next-themes is never reached by this. mock.module("next-themes", () => ({ useTheme: () => ({ theme: "dark", themes: ["dark", "light"], setTheme: () => {} }), })); diff --git a/tests/components/studio/embedded-source.test.tsx b/tests/components/studio/embedded-source.test.tsx index 317af720e..bdb5e194a 100644 --- a/tests/components/studio/embedded-source.test.tsx +++ b/tests/components/studio/embedded-source.test.tsx @@ -1165,12 +1165,20 @@ async function previewThrough(objectEditor: HostEditor, executed?: string[]): Pr await click("object-source-preview"); } -/** The whole reader-visible round trip: Edit, Preview, Confirm, and the dialog gone. */ +/** + * The whole reader-visible round trip: Edit, Preview, Confirm, and the dialog gone. + * + * The closing poll compares with `=== null` rather than asserting `toBeNull()` on the node, and + * so does every other absence poll in this file. A FAILING poll would hand bun the live happy-dom + * node, and bun walks that node's whole object graph to build the diff: 301 ms for a 260-node + * subtree, measured, which burns waitFor's 5 s budget in a few polls and turns a healthy test red + * on a machine that is briefly busy. The boolean costs 0 ms and asserts the same removal. + */ async function applySuccessfullyThroughTheHost(objectEditor: HostEditor): Promise { await previewThrough(objectEditor); await waitFor(() => expect(screen.getByTestId("object-source-apply-confirm")).toBeTruthy()); await click("object-source-apply-confirm"); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); } function refreshTokenPassedToTheViewer(): unknown { @@ -1229,7 +1237,7 @@ describe("the embedded workspace applies an object edit through the host", () => await click("object-source-preview"); await waitFor(() => expect(screen.getByTestId("object-source-apply-dialog")).toBeTruthy()); await click("object-source-apply-confirm"); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); // The connection ID, the address, the kind and the part: the whole surface, on both methods. expect(asked).toEqual([ @@ -1417,7 +1425,7 @@ describe("the embedded workspace applies an object edit through the host", () => seen.push(refreshTokenPassedToTheViewer()); await waitFor(() => expect(screen.getByTestId("object-source-apply-confirm")).toBeTruthy()); await click("object-source-apply-confirm"); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); seen.push(refreshTokenPassedToTheViewer()); expect(seen).toEqual([0, 1]); @@ -1797,7 +1805,7 @@ describe("the embedded workspace applies an object edit through the host", () => editorProbe.change?.(DEFINITION); await Promise.resolve(); }); - await waitFor(() => expect(screen.queryByTestId("tab-dirty-dot")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("tab-dirty-dot") === null).toBe(true)); }); /** * WHAT THE READER IS LOOKING AT ONE RENDER AFTER A SUCCESSFUL APPLY (X25, #789). @@ -1838,7 +1846,7 @@ describe("the embedded workspace applies an object edit through the host", () => await click("object-source-preview"); await waitFor(() => expect(screen.getByTestId("object-source-apply-confirm")).toBeTruthy()); await click("object-source-apply-confirm"); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); // The pane asked the host again, which is the only way new text can reach the screen here. await waitFor(() => expect(read).toBe(2)); diff --git a/tests/components/studio/source-tab.test.tsx b/tests/components/studio/source-tab.test.tsx index e2faa4fc7..7a9ef3def 100644 --- a/tests/components/studio/source-tab.test.tsx +++ b/tests/components/studio/source-tab.test.tsx @@ -1098,14 +1098,22 @@ async function openFunctionTab(): Promise { await waitFor(() => expect(screen.getByTestId("source-editor")).toBeTruthy()); } -/** Edit, Preview, confirm. Every step is the gesture a reader makes, in that order. */ +/** + * Edit, Preview, confirm. Every step is the gesture a reader makes, in that order. + * + * The last poll asks `=== null` instead of asserting `toBeNull()` on the node, and so does every + * other absence poll in this file. On a FAILING poll bun pretty-prints the received value, and + * for a happy-dom node that means walking its whole object graph: 301 ms for a 260-node subtree, + * measured. Four such polls and waitFor's 5 s budget is gone, so a briefly busy machine fails a + * test whose subject is fine. The boolean costs 0 ms and asserts the same removal. + */ async function applySuccessfully(): Promise { await click("object-source-edit"); await waitFor(() => expect(screen.getByTestId("object-source-preview")).toBeTruthy()); await click("object-source-preview"); await waitFor(() => expect(screen.getByTestId("object-source-apply-confirm")).toBeTruthy()); await click("object-source-apply-confirm"); - await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("object-source-apply-dialog") === null).toBe(true)); } describe("a successful apply in the standalone shell", () => { @@ -1330,7 +1338,7 @@ describe("the tab strip's dirty mark survives a remount and still clears", () => act(() => { screen.getAllByRole("tab")[0].click(); }); - await waitFor(() => expect(screen.queryByTestId("source-editor")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("source-editor") === null).toBe(true)); act(() => { screen.getAllByRole("tab")[1].click(); }); @@ -1342,7 +1350,7 @@ describe("the tab strip's dirty mark survives a remount and still clears", () => editorProbe.change?.(DEFINITION); await Promise.resolve(); }); - await waitFor(() => expect(screen.queryByTestId("tab-dirty-dot")).toBeNull()); + await waitFor(() => expect(screen.queryByTestId("tab-dirty-dot") === null).toBe(true)); }); }); diff --git a/tests/helpers/census-connection.ts b/tests/helpers/census-connection.ts new file mode 100644 index 000000000..c72a3a5d9 --- /dev/null +++ b/tests/helpers/census-connection.ts @@ -0,0 +1,73 @@ +import type { DatabaseConnection } from "@/lib/db/types"; +import type { DatabaseType } from "@/lib/types"; + +/** + * One unconnected connection per shipped type-id, shared by the provider censuses. + * + * WHY IT LIVES HERE. `tests/isolated/object-edit-declarations.test.ts` used to import this from + * `tests/isolated/object-source-declarations.test.ts`, which works but makes a TEST file + * importable by another one: loading the importer registers the census's own suite a second + * time. Under one bun process per test file that is nine extra tests, each building all + * seventeen providers through the real `createDatabaseProvider`, run twice and counted twice, so + * the runner's totals stop matching the suite. Moving the fixture into `tests/helpers/` keeps + * the single source both censuses need and takes the import out of a test file. + * + * Copying it instead was the alternative and it is the worse one: a second seventeen-row + * `Record` goes stale the first time an engine's port moves in + * only one of them, and the record exists so that a new member of the union is a COMPILE error + * rather than a missing row. Two records defeat exactly that. + */ + +/** + * The fields every provider's `validate()` demands, none of which is ever dialled. + * + * Nothing here connects: `createDatabaseProvider` is a switch over dynamic imports and a + * constructor, and the constructors validate their configuration without opening a socket or a + * file. The host is the loopback address and the port is 1 so that a provider which ever did + * try to dial would fail loudly rather than reach something real. + */ +const UNCONNECTED = { + id: "census", + name: "census", + host: "127.0.0.1", + port: 1, + database: "census", + user: "census", + password: "census", + filePath: ":memory:", + url: "http://127.0.0.1:1", + connectionString: "mongodb://127.0.0.1:1/census", + // Cassandra's driver refuses to build a client without one, so the census cannot reach that + // provider's declarations at all without it. A stock single-node install reports datacenter1. + localDataCenter: "datacenter1", + createdAt: new Date(0), +} as const; + +const unconnected = (type: DatabaseType): DatabaseConnection => ({ ...UNCONNECTED, type }) as DatabaseConnection; + +/** + * One connection per shipped type-id, as a Record so the compiler owns exhaustiveness. + * + * A new member of `DatabaseType` fails to compile here, which is a stronger failure than the + * runtime one the driven populations in the censuses also give: a census cannot be extended to a + * new engine by accident, and it cannot skip one either. + */ +export const CENSUS_CONNECTION: Readonly> = Object.freeze({ + postgres: unconnected("postgres"), + mysql: unconnected("mysql"), + sqlite: unconnected("sqlite"), + libsql: unconnected("libsql"), + duckdb: unconnected("duckdb"), + oracle: unconnected("oracle"), + mssql: unconnected("mssql"), + clickhouse: unconnected("clickhouse"), + druid: unconnected("druid"), + trino: unconnected("trino"), + cassandra: unconnected("cassandra"), + elasticsearch: unconnected("elasticsearch"), + opensearch: unconnected("opensearch"), + mongodb: unconnected("mongodb"), + redis: unconnected("redis"), + couchbase: unconnected("couchbase"), + libredb: unconnected("libredb"), +}); diff --git a/tests/helpers/mock-monaco.ts b/tests/helpers/mock-monaco.ts index 62b841cfd..01577844f 100644 --- a/tests/helpers/mock-monaco.ts +++ b/tests/helpers/mock-monaco.ts @@ -38,7 +38,7 @@ export function setupMonacoMock() { // `SyntaxError: Export named 'DiffEditor' not found` and fails the WHOLE FILE, so a suite that // never renders a diff still dies the moment one lands anywhere in its module graph. Measured // 2026-09-14: mounting ApplyPreviewDialog from ObjectSourceView put this import into the pane's - // graph and took Group 1 and Group 6 of run-components down without either suite touching it. + // graph and took two component suites down without either of them rendering a diff. DiffEditor: function MockDiffEditor(props: { original?: string; modified?: string; language?: string }) { return React.createElement("div", { "data-testid": "mock-monaco-diff-editor", diff --git a/tests/helpers/posix-tools.ts b/tests/helpers/posix-tools.ts new file mode 100644 index 000000000..59f3f537b --- /dev/null +++ b/tests/helpers/posix-tools.ts @@ -0,0 +1,163 @@ +/** + * Explicit resolution for the POSIX shell and the unix tools the packaging tests drive, plus the + * skip helpers that make an unavailable one visible instead of silent. + * + * Spawning `bash`, `sh`, `grep`, `tar`, `unzip` or `7z` by bare name is a bet on PATH, and on + * Windows it is a bet that loses twice. A contributor runs `bun run test` from PowerShell, where + * Git for Windows puts only `C:\Program Files\Git\cmd` (git.exe) on PATH - `usr\bin` and `bin`, + * which hold bash.exe, sh.exe, grep.exe and unzip.exe, are not on it. `Bun.spawnSync` THROWS + * ("Executable not found in $PATH") rather than returning a non-zero exit code, so one bare name + * takes the whole file down at its first spawn. And when WSL is installed `bash` DOES resolve, to + * `C:\Windows\System32\bash.exe` - a Linux shell that cannot stat the Win32 temp path every fixture + * hands it, so the script under test looks broken when nothing is. + * + * So the tool is resolved to an absolute path, from the Git for Windows installation the clone + * already needed, and when there is none the test says so by name rather than passing quietly. + * + * The lookup is injected rather than read from `process` directly, so the win32 branch is driven + * from Linux in tests/unit/posix-tools.test.ts: a platform branch nothing can execute is a branch + * nothing checks. + */ +import { describe, test } from "bun:test"; +import { existsSync } from "node:fs"; + +/** Everything the resolver touches outside itself. */ +export interface ToolLookup { + /** `process.platform` of the machine being resolved for. */ + readonly platform: string; + /** PATH lookup; null when the command is not on PATH. */ + which(command: string): string | null; + /** True when the path names something that exists. */ + exists(candidate: string): boolean; + /** `git --exec-path` for this git binary, or null when it does not answer. */ + gitExecPath(gitBinary: string): string | null; + /** `%LOCALAPPDATA%`, where a per-user Git for Windows install lands. */ + readonly localAppData: string | null; +} + +/** The real machine. */ +export const systemLookup: ToolLookup = { + platform: process.platform, + which: (command) => Bun.which(command), + exists: (candidate) => existsSync(candidate), + gitExecPath: (gitBinary) => { + const run = Bun.spawnSync([gitBinary, "--exec-path"], { stdout: "pipe", stderr: "pipe" }); + return run.exitCode === 0 ? run.stdout.toString().trim() : null; + }, + localAppData: process.env.LOCALAPPDATA ?? null, +}; + +/** + * Where Git for Windows keeps the tools: `bin` first because the bash.exe and sh.exe there are the + * wrappers that put `usr/bin` on the shell's own PATH, while `usr/bin` is where everything else + * (grep, tar, unzip) actually lives. Looking in that order gets both right with one list. + */ +const GIT_TOOL_DIRS = ["bin", "usr/bin"]; + +/** System-wide Git for Windows installs, for a machine whose PATH does not carry git at all. */ +const DEFAULT_GIT_ROOTS = ["C:/Program Files/Git", "C:/Program Files (x86)/Git"]; + +/** The shells WSL shadows. tar.exe and curl.exe in System32 are the genuine articles; bash.exe is not. */ +const WSL_SHADOWED_SHELLS = new Set(["sh", "bash"]); + +const forwardSlashes = (candidate: string): string => candidate.replaceAll("\\", "/"); + +/** WSL's bash.exe lives in %SystemRoot%\System32, and it is a Linux shell, not a Windows one. */ +const isSystem32 = (candidate: string): boolean => /\/windows\/system32\//i.test(forwardSlashes(candidate)); + +/** Installation roots to search, most specific first. */ +function gitForWindowsRoots(lookup: ToolLookup): string[] { + const roots: string[] = []; + const git = lookup.which("git"); + if (git !== null) { + const execPath = lookup.gitExecPath(git); + // `git --exec-path` prints /mingw64/libexec/git-core, so the installation root is three + // levels up. Asking git itself beats guessing: it finds a portable or D:-drive install too. + if (execPath !== null) { + const segments = forwardSlashes(execPath).split("/"); + if (segments.length > 3) roots.push(segments.slice(0, -3).join("/")); + } + } + if (lookup.localAppData !== null) roots.push(`${forwardSlashes(lookup.localAppData)}/Programs/Git`); + roots.push(...DEFAULT_GIT_ROOTS); + return roots; +} + +/** The absolute path of `name`, or null when this machine has no such tool. */ +export function resolveUnixTool(name: string, lookup: ToolLookup = systemLookup): string | null { + if (lookup.platform !== "win32") return lookup.which(name); + const onPath = lookup.which(name); + if (onPath !== null && !(WSL_SHADOWED_SHELLS.has(name) && isSystem32(onPath))) return onPath; + for (const root of gitForWindowsRoots(lookup)) { + for (const dir of GIT_TOOL_DIRS) { + const candidate = `${root}/${dir}/${name}.exe`; + if (lookup.exists(candidate)) return candidate; + } + } + return null; +} + +/** The POSIX shell to spawn, e.g. `Bun.spawnSync([posixShell(), SCRIPT, ...])`. */ +export function posixShell(name: "sh" | "bash" = "bash", lookup: ToolLookup = systemLookup): string | null { + return resolveUnixTool(name, lookup); +} + +/** Null when the shell is there, else the sentence that goes in the skipped title. */ +export function missingPosixShell(name: "sh" | "bash" = "bash", lookup: ToolLookup = systemLookup): string | null { + return posixShell(name, lookup) === null + ? `no POSIX ${name}: none on PATH and no Git for Windows installation carries one` + : null; +} + +/** Null when the tool is there, else the sentence that goes in the skipped title. */ +export function missingUnixTool(name: string, lookup: ToolLookup = systemLookup): string | null { + return resolveUnixTool(name, lookup) === null + ? `no ${name}: not on PATH and not in a Git for Windows installation` + : null; +} + +/** + * Null on Linux and macOS, a reason on Windows. NTFS carries no POSIX mode bits (chmod there only + * toggles the read-only flag, and stat reads back 0o666 or 0o444), and CreateProcess cannot exec an + * extension-less `#!` script, so a fixture that stands a `chmod 0755` stub on PATH has nothing to + * stand. Platform rather than a probe on purpose: a probe that answers "no" on Linux would turn a + * real regression into a skip. + */ +export const MISSING_POSIX_FILE_MODES: string | null = + process.platform === "win32" + ? "POSIX file modes: NTFS has no exec bit and Windows cannot exec an extension-less #! stub" + : null; + +/** `describe` when `missing` is null, else a skipped describe whose title carries the reason. */ +export function describeIf(missing: string | null, title: string, body: () => void): void { + if (missing === null) { + describe(title, body); + return; + } + // bun prints a skipped test's title NOWHERE: measured on 1.4.2 piped, with FORCE_COLOR, and under + // a real pty, its output carries the count and nothing else. Two readers need the reason anyway. + // `bun run test` gets it from bun's junit report, which the runner asks every child for and prints + // under the file in its summary; somebody running this one file directly gets it from this line. + console.warn(`posix-tools: skipping "${title}" - ${missing}`); + describe.skip(`${title} [skipped: ${missing}]`, body); +} + +/** + * `test` when `missing` is null, else a skipped test whose title carries the reason. + * + * The body may be async: bun awaits what it returns, so an assertion after an `await` still fails + * the test (measured - an async body whose post-await expect fails reports as a failure here too). + */ +export function testIf(missing: string | null, title: string, body: () => void | Promise): void { + if (missing === null) { + test(title, body); + return; + } + console.warn(`posix-tools: skipping "${title}" - ${missing}`); + test.skip(`${title} [skipped: ${missing}]`, body); +} + +/** The common case: a describe that needs a POSIX shell and nothing else. */ +export function describeIfPosixShell(shell: "sh" | "bash", title: string, body: () => void): void { + describeIf(missingPosixShell(shell), title, body); +} diff --git a/tests/hooks/use-monitoring-data.test.ts b/tests/hooks/use-monitoring-data.test.ts index 7cc34f75f..9185a4a5f 100644 --- a/tests/hooks/use-monitoring-data.test.ts +++ b/tests/hooks/use-monitoring-data.test.ts @@ -896,17 +896,18 @@ describe("useMonitoringData", () => { result.current.setAutoRefresh(true); }); - // Wait for interval to fire - await act(async () => { - await new Promise((resolve) => setTimeout(resolve, 200)); + // Waits on the fact this test asserts - the call count rising - instead of on 200ms of + // wall clock. The interval is armed at 100ms, so the sleep this replaces was betting that + // two interval periods fit inside one 200ms sleep, which is a bet a machine running every + // other test file at the same time loses. The wait cannot pass early: nothing but the + // interval issues a monitoring call after `callsBefore` is read. + await waitFor(() => { + const callsAfter = fetchMock.mock.calls.filter( + (call) => typeof call[0] === "string" && call[0].includes("/api/db/monitoring"), + ).length; + expect(callsAfter).toBeGreaterThan(callsBefore); }); - const callsAfter = fetchMock.mock.calls.filter( - (call) => typeof call[0] === "string" && call[0].includes("/api/db/monitoring"), - ).length; - - expect(callsAfter).toBeGreaterThan(callsBefore); - // Clean up act(() => { result.current.setAutoRefresh(false); @@ -940,7 +941,13 @@ describe("useMonitoringData", () => { (call) => typeof call[0] === "string" && call[0].includes("/api/db/monitoring"), ).length; - // Wait and verify no more calls + // THE SLEEP STAYS HERE, and it is the one shape of sleep that survives a busy machine. + // This is a negative: it asserts that nothing fires. There is no fact to wait on, and a + // `waitFor` would only prove the count had not risen YET. What makes it sound under load + // is deadline ordering rather than elapsed time: the interval was armed at 100ms and this + // sleep is due at 200ms, so however far behind the machine falls, an interval that was + // never cleared is overdue before this timer and runs first. Slowness delays both timers + // together and cannot reorder them, so a stall makes this negative stricter, never vacuous. await act(async () => { await new Promise((resolve) => setTimeout(resolve, 200)); }); @@ -981,15 +988,24 @@ describe("useMonitoringData", () => { const { result } = renderHook(() => useMonitoringData(mockConnection)); - // Give the initial fetch time to reject - await act(async () => { - await new Promise((resolve) => setTimeout(resolve, 50)); + // The control, and it is deterministic rather than lucky: `fetchData` calls `setLoading(true)` + // before its first await, and `renderHook` is act-wrapped, so the mount effect has run and the + // state is committed by the time this line reads it. Reading `true` here is what proves the + // fetch was issued at all, so the three silence assertions below cannot pass because nothing + // ever happened. + expect(result.current.loading).toBe(true); + + // Then wait on the `finally`, which is the moment the rejection has been handled. The 50ms + // sleep this replaces was a guess at how long a rejected promise takes to come back, and a + // machine that missed the guess read the three facts below while the fetch was still in + // flight - where they are all true for the wrong reason. + await waitFor(() => { + expect(result.current.loading).toBe(false); }); // Cancellation must be silent: no error, no data expect(result.current.error).toBeNull(); expect(result.current.data).toBeNull(); - expect(result.current.loading).toBe(false); globalThis.fetch = originalFetch; }); diff --git a/tests/integration/db/duckdb-provider.test.ts b/tests/integration/db/duckdb-provider.test.ts index 742792475..1f91cc6ab 100644 --- a/tests/integration/db/duckdb-provider.test.ts +++ b/tests/integration/db/duckdb-provider.test.ts @@ -22,7 +22,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, spyOn, test } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, relative, resolve } from "node:path"; import { DuckDBProvider, assertReadOnlyStatementIsBounded } from "@/lib/db/providers/sql/duckdb"; import type { DatabaseConnection } from "@/lib/types"; import type { ObjectKindSpec, ObjectSourceForm, ProviderCapabilities, ReadOnlyStatementBudget } from "@/lib/db/types"; @@ -71,6 +71,20 @@ const PINNED_VERSION = "v1.5.5"; */ const workDir = mkdtempSync(join(tmpdir(), "libredb-duckdb-test-")); +/** + * A second scratch directory, for the one test that needs a RELATIVE database path. + * + * It has to sit under the process directory: `path.relative` can only answer a relative path + * when both sides share a root, and on Windows a clone on D: with %TEMP% on C: has no relative + * spelling of `workDir` at all. It used to be a fixed `tests-tmp-duckdb/` in the repository + * working tree, which two concurrent test processes would share (DuckDB's single-writer file + * lock then fails the second one) and which a drift guard running beside the suite would see. + * `node_modules/.cache` is ignored by the VCS, and `mkdtempSync` makes the name per-process. + */ +const RELATIVE_WORK_PARENT = resolve(import.meta.dir, "../../../node_modules/.cache"); +mkdirSync(RELATIVE_WORK_PARENT, { recursive: true }); +const relativeWorkDir = mkdtempSync(join(RELATIVE_WORK_PARENT, "libredb-duckdb-rel-")); + /** * A CSV outside every database, for the bare-path form: DuckDB's replacement scan turns * `FROM '.csv'` into a `read_csv_auto`, so the statement carries no forbidden word @@ -134,7 +148,8 @@ beforeAll(() => { afterAll(() => { // One removal, not two: the CSV lives inside the scratch directory now. - rmSync(workDir, { recursive: true, force: true }); + rmSync(workDir, { recursive: true }); + rmSync(relativeWorkDir, { recursive: true }); }); // ============================================================================ @@ -267,18 +282,20 @@ describe("connect / disconnect", () => { test("a relative path is resolved against the process directory, matching factory.ts's fileIdentity", async () => { // `findOpenSingleWriterProvider` keys off `path.resolve(connection.database)`, so a // provider that resolved differently would silently stop matching its own handle. - const relative = `./${join("tests-tmp-duckdb", "relative.duckdb")}`; - provider = new DuckDBProvider(makeConfig({ database: relative })); + const absolute = join(relativeWorkDir, "relative.duckdb"); + const relPath = `./${relative(process.cwd(), absolute)}`; + provider = new DuckDBProvider(makeConfig({ database: relPath })); await provider.connect(); await provider.query("CREATE TABLE t (a INTEGER)"); await provider.query("CHECKPOINT"); const [storage] = await provider.getStorageStats(); - expect(storage.location).toBe(join(process.cwd(), "tests-tmp-duckdb", "relative.duckdb")); + expect(storage.location).toBe(absolute); + // The file itself is removed by the module afterAll with the rest of relativeWorkDir, so a + // failed assertion above no longer leaves a database behind. await provider.disconnect(); - rmSync(join(process.cwd(), "tests-tmp-duckdb"), { recursive: true, force: true }); }); test("a path carrying a NUL byte is refused as a configuration error", async () => { @@ -288,20 +305,55 @@ describe("connect / disconnect", () => { expect(provider.isConnected()).toBe(false); }); - test("two handles on the same file inside ONE process are both allowed", async () => { - // This is the measurement that makes the agent's read-only handle possible: the - // lock is per operating-system process, so a second in-process handle is fine. + test("a second handle on the same file inside ONE process is admitted on POSIX and refused on Windows", async () => { + // DuckDB takes an exclusive hold on the database file at open, and how far that hold + // reaches is a property of the operating system rather than of the engine. + // + // On POSIX it is a per-PROCESS lock, so a second handle from the same process is + // admitted. On Windows there is no such allowance: measured on windows-latest + // (2026-09, DuckDB v1.5.5 through @duckdb/node-api 1.5.5-r.4), the second open fails + // with "The process cannot access the file because it is being used by another + // process" and DuckDB names THIS process as the holder. + // + // Both arms are asserted positively, and the Windows arm ends by opening the second + // handle once the first has let go: a refusal that survived closing the first handle + // would be about something other than the hold, and the assertion would mean nothing. + // + // Browsing does not lean on the allowance either way - the editor is handed the open + // handle rather than a second one (`findOpenSingleWriterProvider`, BACKLOG D3). The one + // path that really wants two at once is an agent run reaching a connection the editor + // already holds; what Windows does to that is recorded in docs/providers/duckdb.md + // §3.8 and is not fixed here. const dbPath = await seededFile("shared.duckdb"); const first = new DuckDBProvider(makeConfig({ database: dbPath })); const second = new DuckDBProvider(makeConfig({ id: "second", database: dbPath })); + const rowCount = async (provider: DuckDBProvider): Promise => + (await provider.query("SELECT count(*) AS n FROM users")).rows[0].n; await first.connect(); - await second.connect(); - - expect((await second.query("SELECT count(*) AS n FROM users")).rows[0].n).toBe("2"); + try { + if (process.platform === "win32") { + const refusal = await second.connect().then( + () => null, + (error: unknown) => error, + ); + expect(refusal).toBeInstanceOf(ConnectionError); + expect((refusal as Error).message).toContain(dbPath); + expect(await rowCount(first)).toBe("2"); + + await first.disconnect(); + await second.connect(); + } else { + await second.connect(); + } - await first.disconnect(); - await second.disconnect(); + expect(await rowCount(second)).toBe("2"); + } finally { + // Whatever happened above: a handle left open is a file the module teardown cannot + // remove on Windows, which is how this test used to poison the rest of the file. + if (second.isConnected()) await second.disconnect(); + if (first.isConnected()) await first.disconnect(); + } }); }); @@ -683,13 +735,19 @@ describe("monitoring", () => { const [main] = await provider.getStorageStats(); expect(main.name).toBe("Main Database"); - // Both sides through `realpathSync`, because the location is DUCKDB's answer and DuckDB - // canonicalises it. A no-op wherever the temp directory is a real directory, which is why - // this read as portable: on macOS `os.tmpdir()` is `/var/folders/...`, a symlink to - // `/private/var/folders/...`, so the engine returns a path that names the same file by a - // different route and a string comparison fails on a correct answer. + // Both sides through `realpathSync.native`, because the location is DUCKDB's answer and + // DuckDB canonicalises it, so the two sides can name the same file by different routes: + // on macOS `os.tmpdir()` is `/var/folders/...`, a symlink to `/private/var/folders/...`, + // and on Windows it is the 8.3 SHORT form of whatever %TEMP% holds - on windows-latest + // `os.tmpdir()` answers `C:\Users\RUNNER~1\AppData\Local\Temp` while DuckDB reports + // `C:\Users\runneradmin\...` for the same file (measured 2026-09). + // + // `.native` and not the JS `realpathSync`: the JS one resolves symlinks, which covers + // macOS, but it leaves a short 8.3 component exactly as it found it. The native one is + // libuv's `uv_fs_realpath`, which on Windows goes through `GetFinalPathNameByHandle` and + // answers the long form, and on POSIX is plain `realpath(3)`. expect(main.location).toBeDefined(); - expect(realpathSync(main.location!)).toBe(realpathSync(dbPath)); + expect(realpathSync.native(main.location!)).toBe(realpathSync.native(dbPath)); expect(main.sizeBytes).toBeGreaterThan(0); }); @@ -1002,6 +1060,14 @@ describe("queryReadOnly()", () => { // The control that makes the assertion above mean something: the ordinary editor // handle on the SAME file keeps its filesystem reach, because COPY and read_csv are // features there rather than escapes. + // + // One at a time, and the read-only handle goes first: Windows admits a single handle + // per DuckDB file per process (see the connect/disconnect block), so holding both + // open would make the control unopenable there. Sequential proves the same thing - + // the setting belongs to the HANDLE, not to the file, or the same path could not + // answer both ways. + await provider.disconnect(); + const writable = new DuckDBProvider(makeConfig({ database: dbPath })); await writable.connect(); try { diff --git a/tests/integration/db/sqlite-node-harness.ts b/tests/integration/db/sqlite-node-harness.ts index dfd32bd62..c5dc6f4b7 100644 --- a/tests/integration/db/sqlite-node-harness.ts +++ b/tests/integration/db/sqlite-node-harness.ts @@ -120,6 +120,12 @@ async function main(): Promise { await provider.disconnect(); report.disconnected = !provider.isConnected(); + // disconnect() has to RELEASE the file on this adapter too, and the sidecars are the + // portable reading of it: SQLite checkpoints the WAL and removes `-wal` and `-shm` when + // the connection really closes, and leaves both when the close was only scheduled. The + // flag the bun adapter needs for that is meaningless to node:sqlite, which is exactly + // the kind of claim an adapter test cannot make for the real driver. + report.sidecarsAfterDisconnect = [`${dbPath}-wal`, `${dbPath}-shm`].filter((sidecar) => existsSync(sidecar)); await runAgentReadOnlyProfile(dbPath, report); diff --git a/tests/integration/db/sqlite-provider.test.ts b/tests/integration/db/sqlite-provider.test.ts index 61e7a7571..d64ce4df3 100644 --- a/tests/integration/db/sqlite-provider.test.ts +++ b/tests/integration/db/sqlite-provider.test.ts @@ -9,9 +9,23 @@ import { describe, test, expect, afterEach, beforeAll, afterAll, spyOn } from "bun:test"; import { spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, rmSync, statSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + readlinkSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { isAbsolute, join, relative } from "node:path"; +import { basename, isAbsolute, join, relative, resolve } from "node:path"; + +/** The repository root, anchored to this file so nothing here depends on the launcher's cwd. */ +const REPO_ROOT = resolve(import.meta.dir, "../../.."); import { SQLiteProvider, assertQueryOnlyEnabled, @@ -143,13 +157,23 @@ describe("SQLiteProvider", () => { describe("getDatabasePath() via connect()", () => { let pathTmpDir: string; + let sameVolumeTmpDir: string; beforeAll(() => { pathTmpDir = mkdtempSync(join(tmpdir(), "libredb-sqlite-path-")); + // A second fixture directory, deliberately on the same volume as the process cwd: the + // relative-path test below needs path.relative(cwd, target) to BE relative, and on Windows a + // clone on D: with %TEMP% on C: makes that impossible to express, so path.relative hands + // back the absolute target and the test fails on that machine only. node_modules/.cache is + // ignored by the VCS, so nothing here is visible to a working-tree drift guard. + const cache = join(REPO_ROOT, "node_modules", ".cache"); + mkdirSync(cache, { recursive: true }); + sameVolumeTmpDir = mkdtempSync(join(cache, "libredb-sqlite-path-")); }); afterAll(() => { - rmSync(pathTmpDir, { recursive: true, force: true }); + rmSync(pathTmpDir, { recursive: true }); + rmSync(sameVolumeTmpDir, { recursive: true }); }); test("a path containing a NUL byte throws DatabaseConfigError without claiming traversal protection", async () => { @@ -169,15 +193,24 @@ describe("SQLiteProvider", () => { test("a relative path with '..' segments is accepted and resolves to an absolute location", async () => { // Pins intended behavior: sqlite paths are trusted server-side paths, so // ".." segments are legal and simply resolve against the process cwd. - const relPath = relative(process.cwd(), join(pathTmpDir, "dotdot-ok.db")); + // The ".." is built explicitly, by leaving the working directory and coming straight back + // into it, rather than by pointing at a directory outside the tree and letting + // path.relative produce the hops. That used to be a path under the system temp directory: + // on Windows a clone on D: with %TEMP% on C: has no relative spelling at all, so + // path.relative returns the absolute target and the assertion below fails on that machine + // only. This form carries the same ".." segments on every platform. + const target = join(sameVolumeTmpDir, "dotdot-ok.db"); + const cwd = process.cwd(); + const relPath = join("..", basename(cwd), relative(cwd, target)); expect(isAbsolute(relPath)).toBe(false); expect(relPath).toContain(".."); + expect(resolve(relPath)).toBe(target); provider = new SQLiteProvider(makeSQLiteConfig({ database: relPath })); await provider.connect(); expect(provider.isConnected()).toBe(true); // The database file materializes at the resolved absolute location. - expect(existsSync(join(pathTmpDir, "dotdot-ok.db"))).toBe(true); + expect(existsSync(join(sameVolumeTmpDir, "dotdot-ok.db"))).toBe(true); }); test("a connectionString with a file: prefix is accepted and the prefix is stripped", async () => { @@ -848,7 +881,7 @@ describe("SQLiteProvider", () => { }); afterAll(() => { - rmSync(fileTmpDir, { recursive: true, force: true }); + rmSync(fileTmpDir, { recursive: true }); }); test("getHealth reports the on-disk file size and passes the integrity check", async () => { @@ -896,6 +929,111 @@ describe("SQLiteProvider", () => { expect(wal.location).toBe("storage.db-wal"); expect(typeof wal.walSizeBytes).toBe("number"); }); + + // ── disconnect() has to RELEASE the file, not schedule its release ──────── + // + // bun:sqlite's `close()` is `sqlite3_close_v2`: the connection becomes a zombie + // and the operating-system handle is released only once the last statement + // prepared from it is finalized or garbage collected. The provider prepares a + // statement per query and drops the reference, so on a collector's schedule that + // is "eventually", and `disconnect()` used to resolve with the database, its WAL + // and its shared-memory file still open (measured on Linux through + // /proc/self/fd: three descriptors survived a disconnect that reported + // isConnected() === false). + // + // Nothing on POSIX notices, because POSIX unlinks a file that is still open. On + // Windows it is the whole difference: every one of these directories failed its + // own teardown with `EBUSY: resource busy or locked` on windows-latest + // (2026-09), and a user could not delete or move a database Studio had + // disconnected from. + // + // Each platform is asked the strongest question it can answer. The WAL sidecars + // are NOT that question, though they look like it: measured on 2026-09-15 with + // the same probe on all three runners, `close(true)` removes `-wal` and `-shm` + // on Linux and on Windows, and leaves both in place on macOS, where bun:sqlite + // links Apple's system libsqlite3. That is the library keeping the WAL, not a + // handle keeping the file: opening the same database with node:sqlite and + // closing it removed both sidecars on that same macOS run, and removing a WAL + // takes the exclusive lock a surviving handle would have denied. + test("disconnect releases the file rather than scheduling it", async () => { + const dbPath = join(fileTmpDir, "release.db"); + provider = new SQLiteProvider(makeSQLiteConfig({ database: dbPath })); + await provider.connect(); + await provider.query("CREATE TABLE r (id INTEGER PRIMARY KEY, v TEXT)"); + await provider.query("INSERT INTO r VALUES (1, 'held')"); + expect(existsSync(`${dbPath}-wal`)).toBe(true); + + await provider.disconnect(); + + // Windows answers by refusing: a file with a live handle cannot be renamed, + // and renaming is exactly what a user does to a database they think they have + // closed. POSIX renames an open file, so this cannot fail there. + const moved = `${dbPath}.moved`; + renameSync(dbPath, moved); + renameSync(moved, dbPath); + + // Linux answers precisely: this is the measurement the defect was found with. + // /proc/self/fd is the process's own open files, so a scheduled close shows up + // as a descriptor still pointing into this directory. + if (existsSync("/proc/self/fd")) { + const held = readdirSync("/proc/self/fd").flatMap((fd) => { + try { + return [readlinkSync(join("/proc/self/fd", fd))]; + } catch { + // The descriptor closed between the listing and the read, which is this + // process's own bookkeeping rather than anything about the database. + return []; + } + }); + expect(held.filter((target) => target.startsWith(fileTmpDir))).toEqual([]); + } + + // And the data survived whatever the close had to checkpoint. + const reader = new SQLiteProvider(makeSQLiteConfig({ database: dbPath })); + await reader.connect(); + try { + expect((await reader.query("SELECT v FROM r")).rows).toEqual([{ v: "held" }]); + } finally { + await reader.disconnect(); + } + }); + + // The same claim on the path nobody plans for: the connection points at a file + // that is not a SQLite database, which is the ordinary "wrong file in the + // dialog" mistake. `connect()` opens the handle before it fails, so a catch that + // only records the error leaves the user's own file held open: on Windows they + // then cannot delete or move the file they just picked by accident. Measured + // 2026-09-15 through /proc/self/fd, before the fix: one descriptor on notes.txt + // survived a connect() that had already thrown and reported isConnected() false. + test("a connect that fails releases the file it had already opened", async () => { + const notADatabase = join(fileTmpDir, "notes.txt"); + writeFileSync(notADatabase, "these are notes, not a database\n"); + provider = new SQLiteProvider(makeSQLiteConfig({ database: notADatabase })); + + await expect(provider.connect()).rejects.toThrow(); + expect(provider.isConnected()).toBe(false); + // A retry has to ask the file again rather than answer from a handle that is + // not there: `connect()` returns early when it still holds one, so a catch that + // released the file but kept the reference would make this second call resolve, + // silently, on a provider that is not connected. + await expect(provider.connect()).rejects.toThrow(); + expect(provider.isConnected()).toBe(false); + + const moved = `${notADatabase}.moved`; + renameSync(notADatabase, moved); + renameSync(moved, notADatabase); + + if (existsSync("/proc/self/fd")) { + const held = readdirSync("/proc/self/fd").flatMap((fd) => { + try { + return [readlinkSync(join("/proc/self/fd", fd))]; + } catch { + return []; + } + }); + expect(held.filter((target) => target === notADatabase)).toEqual([]); + } + }); }); // -------------------------------------------------------------------------- @@ -1131,7 +1269,7 @@ function interceptReads(provider: SQLiteProvider, match: string, intercept: (sql const real = holder.db; holder.db = { exec: (sql: string) => real.exec(sql), - close: () => real.close(), + close: (throwOnError?: boolean) => real.close(throwOnError), get inTransaction() { return real.inTransaction; }, @@ -2220,7 +2358,7 @@ describe("SQLiteProvider bulk column read (#789)", () => { const seen: string[] = []; holder.db = { exec: (sql: string) => real.exec(sql), - close: () => real.close(), + close: (throwOnError?: boolean) => real.close(throwOnError), get inTransaction() { return real.inTransaction; }, @@ -2527,7 +2665,7 @@ describe("SQLiteProvider agent read-only execution profile (#328)", () => { }); afterAll(() => { - rmSync(agentTmpDir, { recursive: true, force: true }); + rmSync(agentTmpDir, { recursive: true }); }); afterEach(async () => { @@ -2775,7 +2913,9 @@ describe("SQLiteProvider agent read-only execution profile (#328)", () => { live arm above — it proves the template still reads that way, not that a running engine produces it — and still red on a reword, which is the property that matters. */ - const postgresSource = await Bun.file("src/lib/db/providers/sql/postgres.ts").text(); + // Anchored to this file, like every other source read in the suite: a cwd-relative read + // would fail here naming a path rather than the rule it is checking. + const postgresSource = readFileSync(join(REPO_ROOT, "src/lib/db/providers/sql/postgres.ts"), "utf8"); expect(postgresSource).toContain( "Read-only execution exceeded the row budget: ${result.rows.length} rows > ${budget.maxResultRows} allowed", ); @@ -2941,7 +3081,7 @@ describe.skipIf(!nodeDriverTestable)("SQLiteProvider with LIBREDB_SQLITE_DRIVER= }); afterAll(() => { - rmSync(tmpDir, { recursive: true, force: true }); + rmSync(tmpDir, { recursive: true }); }); test("core CRUD, schema, maintenance, and error mapping work under Node", () => { @@ -2955,6 +3095,13 @@ describe.skipIf(!nodeDriverTestable)("SQLiteProvider with LIBREDB_SQLITE_DRIVER= ["build", harnessEntry, "--target=node", "--format=esm", "--external", "bun:sqlite", "--outfile", bundlePath], { timeout: 60_000 }, ); + // `build.error` first: on a timeout spawnSync returns status null with error set, and + // `status !== 0` is true for null, so checking status alone raises "bun build failed:" with an + // empty stderr, a message that names nothing. Under a concurrent runner a timeout is the + // likely failure, so it has to say so. + if (build.error) { + throw new Error(`bun build could not run: ${build.error.message}`); + } if (build.status !== 0) { throw new Error(`bun build failed: ${build.stderr?.toString()}`); } @@ -2964,6 +3111,9 @@ describe.skipIf(!nodeDriverTestable)("SQLiteProvider with LIBREDB_SQLITE_DRIVER= env: { ...process.env, LIBREDB_SQLITE_DRIVER: "node" }, timeout: 60_000, }); + if (run.error) { + throw new Error(`node harness could not run: ${run.error.message}`); + } if (run.status !== 0) { throw new Error(`node harness failed: ${run.stderr?.toString()}`); } @@ -2975,6 +3125,9 @@ describe.skipIf(!nodeDriverTestable)("SQLiteProvider with LIBREDB_SQLITE_DRIVER= expect(report.driverEnv).toBe("node"); expect(report.connected).toBe(true); expect(report.disconnected).toBe(true); + // And the disconnect released the file, as the bun adapter's does: no WAL sidecar + // survived it. See "disconnect releases the file rather than scheduling it" above. + expect(report.sidecarsAfterDisconnect).toEqual([]); expect(existsSync(dbPath)).toBe(true); // real file-backed database // CRUD (same results as the bun driver) diff --git a/tests/integration/storage/sqlite-credential-encryption.test.ts b/tests/integration/storage/sqlite-credential-encryption.test.ts index 05c5a0e9a..04a20496e 100644 --- a/tests/integration/storage/sqlite-credential-encryption.test.ts +++ b/tests/integration/storage/sqlite-credential-encryption.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test, beforeAll, afterAll } from "bun:test"; import { spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -24,23 +24,32 @@ describe.skipIf(!nodeBetterSqliteTestable)( "credential encryption at rest against a real STORAGE_PROVIDER=sqlite file (posture control 3.1)", () => { let tmpDir: string; + let bundleDir: string; beforeAll(() => { tmpDir = mkdtempSync(join(tmpdir(), "libredb-storage-sqlite-enc-")); // The bundle is `--external better-sqlite3` (a native addon; a bundler cannot inline it), // so plain Node module resolution needs a node_modules it can find by walking up from the - // bundle's own directory. tmpDir sits outside the project tree, so nothing is found without - // this: a symlink is cheaper and more honest than moving the bundle output into the repo. - symlinkSync(resolve(import.meta.dir, "../../../node_modules"), join(tmpDir, "node_modules"), "dir"); + // bundle's own directory. This used to be a symlink from tmpDir to the project's + // node_modules, which needs a privilege Windows does not grant a normal shell (EPERM in + // beforeAll, failing the describe rather than skipping it). Put the bundle where resolution + // already works instead: a directory under node_modules/.cache, from which node's upward + // walk reaches the real node_modules with nothing to link. NODE_PATH is not an option here, + // it does not apply to the ESM resolution this bundle uses. The database still lives in + // tmpDir, outside the repository. + const cache = resolve(import.meta.dir, "../../../node_modules/.cache"); + mkdirSync(cache, { recursive: true }); + bundleDir = mkdtempSync(join(cache, "libredb-storage-sqlite-enc-")); }); afterAll(() => { rmSync(tmpDir, { recursive: true, force: true }); + rmSync(bundleDir, { recursive: true, force: true }); }); test("a canary password never reaches the file on disk, and a rotated key omits it on read instead of exposing it or crashing", () => { const harnessEntry = join(import.meta.dir, "sqlite-credential-encryption-node-harness.ts"); - const bundlePath = join(tmpDir, "sqlite-credential-encryption-node-harness.mjs"); + const bundlePath = join(bundleDir, "sqlite-credential-encryption-node-harness.mjs"); const dbPath = join(tmpDir, "storage.db"); const build = spawnSync( diff --git a/tests/isolated/agent-model-adapter.test.ts b/tests/isolated/agent-model-adapter.test.ts index 9beeaf728..756eddc53 100644 --- a/tests/isolated/agent-model-adapter.test.ts +++ b/tests/isolated/agent-model-adapter.test.ts @@ -22,12 +22,13 @@ import { * sentinel value and asserts the sentinel never leaves the process: what goes on * the wire is what `resolveConfig` resolved from `LLM_*`, or nothing. * - * This file lives in `tests/isolated/` — its own group in - * `tests/run-components.sh` — because every `tests/api/ai/*.test.ts` replaces - * `@/lib/llm/types` with stub error classes whose constructors take a message - * only. `mock.module` is process-wide, so in a shared process the mapper's - * provider tag silently vanishes while the class identity still matches, and - * the assertions below would fail against perfectly correct code. + * This file may not share a process with `tests/api/ai/*.test.ts`, every one of + * which replaces `@/lib/llm/types` with stub error classes whose constructors + * take a message only. `mock.module` is process-wide, so in a shared process the + * mapper's provider tag silently vanishes while the class identity still + * matches, and the assertions below would fail against perfectly correct code. + * The runner gives every test file a process of its own, so that is already the + * case and this paragraph, not a directory or a registration, records why. */ // ─── environment isolation ────────────────────────────────────────────────── diff --git a/tests/isolated/exports-shim.test.ts b/tests/isolated/exports-shim.test.ts index c91875347..ae2ec0d6e 100644 --- a/tests/isolated/exports-shim.test.ts +++ b/tests/isolated/exports-shim.test.ts @@ -2,7 +2,8 @@ import { describe, expect, test } from "bun:test"; // Load the shim the way a CJS consumer/bundler would: require(), not an ESM // import whose CJS interop would mask resolution differences. Coverage note: -// this group runs with --nocov (see tests/run-components.sh) and the shim is +// this file runs without --coverage (it is named in COVERAGE_EXEMPT_FILES in +// tests/runner/discover.ts, which carries the measurement) and the shim is // excluded from Sonar coverage — this test guards the npm entry point // functionally, not for lcov. // eslint-disable-next-line @typescript-eslint/no-require-imports diff --git a/tests/isolated/factory.test.ts b/tests/isolated/factory.test.ts index f3f8717b6..d68b70af6 100644 --- a/tests/isolated/factory.test.ts +++ b/tests/isolated/factory.test.ts @@ -29,11 +29,12 @@ * listed, measured by tracing the console output, so which file wins is not something the * other file can arrange. * - * `tests/isolated/exports-shim.test.ts`'s group comment in `tests/run-components.sh` already - * named this hazard from the other side, and the fix there was to move the OTHER file out. - * That stopped working when #789 added two `tests/unit` files that construct every provider - * through the real factory: a fleet census cannot do its job without importing it. So the - * isolation now sits on the file that needs it, and `bun test tests/unit` is clean again. + * The old component runner named this hazard from the other side and fixed it by moving the + * OTHER file out of the group. That stopped working when #789 added two `tests/unit` files that + * construct every provider through the real factory: a fleet census cannot do its job without + * importing it. So the requirement sits on the file that needs it, which is this paragraph, and + * the runner is what enforces it: one bun process per test file, no directory and no + * registration, and `bun test tests/unit` is clean again. */ import { describe, test, expect, mock, beforeEach, beforeAll, afterAll } from "bun:test"; import { open as libreOpen, kv as libreKv } from "@libredb/libredb"; @@ -63,6 +64,16 @@ const AGENT_BUDGET: ReadOnlyStatementBudget = { maxResultBytes: 64 * 1024, }; +/** + * The `database` the two construction censuses hand the libredb provider. + * + * It is a path that is never opened, so what matters about it is only that it is a legal one: + * the "/tmp/test.libredb" it replaces names a directory that does not exist on Windows, and a + * hardcoded absolute path shared by every process is a collision waiting for the day something + * does open it. `tmpdir()` answers the platform's own scratch directory on all three. + */ +const CENSUS_LIBREDB_FILE = join(tmpdir(), "factory-census.libredb"); + // ============================================================================ // Helper: build a minimal DatabaseConnection for a given type // ============================================================================ @@ -520,7 +531,11 @@ describe("createDatabaseProvider", () => { }); test('creates provider for type "libredb"', async () => { - const conn = makeConnection("libredb", { database: "/tmp/test.libredb" }); + // A path the platform owns rather than a hardcoded "/tmp/...", which is not a directory + // on Windows at all. Nothing opens this file: `createDatabaseProvider` constructs and + // validates without touching the disk, so this is the spelling of a path and not a + // fixture. It is named all the same, so a provider that ever did open it says where. + const conn = makeConnection("libredb", { database: CENSUS_LIBREDB_FILE }); const provider = await createDatabaseProvider(conn); expect(provider).toBeDefined(); expect(provider.type).toBe("libredb"); @@ -547,7 +562,7 @@ describe("createDatabaseProvider", () => { druid: { port: 8888 }, trino: { port: 8080, database: "tpch" }, cassandra: { port: 9042, database: "probe", localDataCenter: "datacenter1" } as Partial, - libredb: { database: "/tmp/test.libredb" }, + libredb: { database: CENSUS_LIBREDB_FILE }, }; const declaringTypes: string[] = []; @@ -1399,7 +1414,16 @@ describe("acquireExecutionProfileProvider", () => { sqliteTmpDir = mkdtempSync(join(tmpdir(), "libredb-factory-sqlite-")); }); - afterAll(() => { + afterAll(async () => { + /* + * The cache is emptied before the directory goes, and the await is the point. + * Nothing else clears it after the last test of this group, so a provider that + * connected here still holds an open handle on a file inside `sqliteTmpDir`. POSIX + * unlinks an open file and never complains, so the old spelling looked correct on + * Linux and macOS; Windows refuses to remove a file that is open and answers EBUSY, + * and `force: true` only swallows ENOENT. + */ + await clearProviderCache(); rmSync(sqliteTmpDir, { recursive: true, force: true }); }); @@ -1473,7 +1497,16 @@ describe("single-writer file reuse", () => { dir = mkdtempSync(join(tmpdir(), "libredb-factory-single-writer-")); }); - afterAll(() => { + afterAll(async () => { + /* + * The cache is emptied before the directory goes, and the await is the point. + * Nothing else clears it after the last test of this group, so a provider that connected + * here still holds an open handle on a file inside `dir`. POSIX unlinks an open file + * and never complains, so the old spelling looked correct on Linux and macOS; Windows + * refuses to remove a file that is open and answers EBUSY, and `force: true` only + * swallows ENOENT. + */ + await clearProviderCache(); rmSync(dir, { recursive: true, force: true }); }); @@ -1529,10 +1562,16 @@ describe("single-writer file reuse", () => { // Deliberately not built with path.join, which would normalise it before the // factory ever saw it: the lock is per inode, so the lookup has to resolve. + // The file name comes from basename rather than from splitting on "/": on + // Windows `dir` is a backslash path, so the split returned the whole path and + // the spelling became `C:\...\dir/./C:\...\held.libredb`, which resolves to + // nothing and matched nothing. Measured on windows-latest, 2026-09-15. A + // forward slash inside the spelling is fine there: Win32 accepts it, and + // path.resolve, which is what the factory uses, normalises it away. const spelled: DatabaseConnection = { ...held, id: "spelled", - database: `${dir}/./${held.database!.split("/").pop()!}`, + database: `${dir}/./${basename(held.database!)}`, }; expect(findOpenSingleWriterProvider(spelled)).toBe(writable); @@ -1567,6 +1606,17 @@ describe("single-writer file reuse", () => { id: "duck-file-b", database: join(dir, "..", basename(dir), "borrowed.duckdb"), }); + // Relative TO THE CWD, deliberately, and not to the file's own directory. `fileIdentity` + // normalises with `path.resolve` (src/lib/db/factory.ts:301), which resolves against + // `process.cwd()`, so a spelling relative to anything else would name a different file and + // this assertion would fail on every platform rather than exercise the borrow. + // + // ON WINDOWS THIS LINE CAN LOSE ITS POINT WITHOUT LOSING ITS TRUTH, which is why it is + // written down here. `path.relative` cannot express a path across volumes, so a machine + // whose TMP sits on a different drive from the checkout gets an ABSOLUTE spelling back and + // the case below repeats the `dotted` one instead of adding the relative one. That is a + // weaker test on that machine shape, never a false one, and no assertion is added to make + // the premise hard: a legitimate Windows layout should not be reported as a defect. const relative = makeConnection("duckdb", { id: "duck-file-c", database: relativePath(process.cwd(), file) }); expect(findOpenSingleWriterProvider(dotted)).toBe(writable); @@ -1791,7 +1841,16 @@ describe("grounding a plan run while the writable provider holds the file (B49)" dir = mkdtempSync(join(tmpdir(), "libredb-factory-grounding-")); }); - afterAll(() => { + afterAll(async () => { + /* + * The cache is emptied before the directory goes, and the await is the point. + * Nothing else clears it after the last test of this group, so a provider that connected + * here still holds an open handle on a file inside `dir`. POSIX unlinks an open file + * and never complains, so the old spelling looked correct on Linux and macOS; Windows + * refuses to remove a file that is open and answers EBUSY, and `force: true` only + * swallows ENOENT. + */ + await clearProviderCache(); rmSync(dir, { recursive: true, force: true }); }); diff --git a/tests/isolated/monaco-language-ids.test.ts b/tests/isolated/monaco-language-ids.test.ts index 1f736cb1f..6625eb79a 100644 --- a/tests/isolated/monaco-language-ids.test.ts +++ b/tests/isolated/monaco-language-ids.test.ts @@ -47,17 +47,20 @@ * Measured on monaco-editor 0.56.0, 2026-09-13: 89 basic ids, 4 rich ids, and exactly one of the * four rich ids (`json`) absent from the 89. * - * WHY THIS FILE LIVES UNDER `tests/isolated/` (#789). It builds providers through the REAL + * WHAT THIS FILE CANNOT SHARE A PROCESS WITH (#789). It builds providers through the REAL * `createDatabaseProvider`, which is the whole point: a declaration census that read a double * would certify the double. Every file under `tests/api/` mocks `@/lib/db` with a * `createDatabaseProvider: mock()` answering undefined, and that mock reaches * `@/lib/db/factory` through the index re-export, so in a shared process this file reads * `provider.getCapabilities` off undefined. Measured 2026-09-13: alone it is green; beside * `tests/api/db-objects.test.ts` it is not. Nothing this file can do prevents it, because - * mocking the factory is what the api layer is for, so the isolation sits here and - * `tests/run-components.sh` gives it a group of its own. + * mocking the factory is what the api layer is for. The runner gives every test file a bun + * process of its own, so that isolation is already in force and this paragraph, rather than a + * directory or an entry in a runner script, is where the requirement is written down. */ import { readdirSync, readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; import { describe, expect, test } from "bun:test"; import { EXTERNAL_DATABASE_TYPES } from "@/lib/db/compatibility"; import { createDatabaseProvider } from "@/lib/db/factory"; @@ -65,9 +68,22 @@ import { declaredKinds } from "@/lib/db/object-kinds"; import type { DatabaseConnection } from "@/lib/db/types"; import type { DatabaseType } from "@/lib/types"; -const MONACO_ROOT = "node_modules/monaco-editor"; -const BASIC_CONTRIBUTION = `${MONACO_ROOT}/min/vs/basic-languages/monaco.contribution.js`; -const RICH_LANGUAGE_DIR = `${MONACO_ROOT}/min/vs/language`; +/** + * The installed package, located through the resolver rather than by spelling out a path. + * + * `"node_modules/monaco-editor"` is relative to the cwd, and both reads below run at MODULE + * scope: a process that did not start in the repo root fails this file with ENOENT before a + * single test registers, which reads as a missing bundle rather than as a wrong cwd. Resolving + * from `import.meta.url` also follows a hoisted or nested install instead of assuming the flat + * one. `join` rather than string concatenation, so the separator is the platform's. + * + * monaco-editor's `exports` map has no `./package.json` entry, so this leans on bun's resolver + * answering it anyway (verified: it returns the installed package's own manifest). If that ever + * stops being true the failure is a named resolution error here, not a silent wrong path. + */ +const MONACO_ROOT = dirname(createRequire(import.meta.url).resolve("monaco-editor/package.json")); +const BASIC_CONTRIBUTION = join(MONACO_ROOT, "min/vs/basic-languages/monaco.contribution.js"); +const RICH_LANGUAGE_DIR = join(MONACO_ROOT, "min/vs/language"); /** * The version the two counts below are counts OF. diff --git a/tests/isolated/object-edit-declarations.test.ts b/tests/isolated/object-edit-declarations.test.ts index ddc9405c1..66ea29edc 100644 --- a/tests/isolated/object-edit-declarations.test.ts +++ b/tests/isolated/object-edit-declarations.test.ts @@ -10,18 +10,24 @@ * two disagree, exactly one of them is wrong, and the repair is to the DECLARATION or to the * design, never to the expectation. * - * WHY IT LIVES UNDER `tests/isolated/`, which is the same reason the Phase 2 source census does + * WHAT IT CANNOT SHARE A PROCESS WITH, which is the same thing the Phase 2 source census cannot * and is measured rather than inherited: it builds every provider through the REAL * `createDatabaseProvider`, and every file under `tests/api/` mocks `@/lib/db` with a * `createDatabaseProvider: mock()` answering undefined, which reaches `@/lib/db/factory` through * the index re-export. In a shared process this file would read `provider.getCapabilities` off - * undefined. `tests/run-components.sh` gives Group 0b3 to both censuses so they share one - * process and one population. + * undefined. The runner gives every test file a bun process of its own, so that isolation is + * already in force and this paragraph is where the requirement is written down. * - * `CENSUS_CONNECTION` is IMPORTED from the source census beside it rather than copied. A second - * seventeen-row `Record` is a map that goes stale the first - * time an engine's port moves in only one of them, and the record exists so a new member of the - * union is a compile error rather than a missing row: two records defeat exactly that. + * `CENSUS_CONNECTION` comes from `tests/helpers/census-connection.ts`, which both censuses + * import. It used to be imported from the source census itself, which works and costs the run + * that census twice: importing a TEST file registers its suite in this process too, so + * `bun test ./tests/isolated/object-edit-declarations.test.ts` reported fifteen tests where this + * file declares six, each of the nine strays building all seventeen providers a second time. + * Under one bun process per test file that double count is in every run. Copying the record + * instead was the other option and it is the worse one: a second seventeen-row + * `Record` goes stale the first time an engine's port moves in + * only one of them, and the record exists so a new member of the union is a compile error rather + * than a missing row, which two records defeat exactly. * * THE MARIADB LEVER, and it is measured rather than a worry. `createDatabaseProvider("mysql")` * is UNCONNECTED, and mysql is the one provider whose `objectKinds` is not a constant: @@ -43,7 +49,7 @@ import { EXPECTED_EDITABLE_KINDS, EXPECTED_EDIT_ABSTAINERS, } from "../helpers/object-edit-expectation"; -import { CENSUS_CONNECTION } from "./object-source-declarations.test"; +import { CENSUS_CONNECTION } from "../helpers/census-connection"; /** * The version string a MariaDB server answers `SELECT VERSION()` with, measured on diff --git a/tests/isolated/object-source-declarations.test.ts b/tests/isolated/object-source-declarations.test.ts index e28ea428a..2a2ee8cf3 100644 --- a/tests/isolated/object-source-declarations.test.ts +++ b/tests/isolated/object-source-declarations.test.ts @@ -20,8 +20,9 @@ * * 1. The type-id list is DRIVEN from `EXTERNAL_DATABASE_TYPES` plus the embedded store, never * typed here, so a new engine is censused the day it lands rather than being silently - * omitted. `CENSUS_CONNECTION` is a `Record`, so a new member of the - * union is a COMPILE error rather than a missing row. + * omitted. `CENSUS_CONNECTION` (`tests/helpers/census-connection.ts`, shared with the edit + * census) is a `Record`, so a new member of the union is a COMPILE error + * rather than a missing row. * 2. `createDatabaseProvider("mysql")` is UNCONNECTED, and mysql is the one provider whose * `objectKinds` is not a constant: `objectKindsFor(undefined)` answers the MySQL six and * structurally excludes MariaDB's `package` and `sequence`, which the design flags as the @@ -53,15 +54,16 @@ * sees every provider's declarations at once, so the half-declaration guard lives here, and it is * not a duplicate of anything: deleting it makes the class invisible again. * - * WHY THIS FILE LIVES UNDER `tests/isolated/` (#789). It builds providers through the REAL + * WHAT THIS FILE CANNOT SHARE A PROCESS WITH (#789). It builds providers through the REAL * `createDatabaseProvider`, which is the whole point: a declaration census that read a double * would certify the double. Every file under `tests/api/` mocks `@/lib/db` with a * `createDatabaseProvider: mock()` answering undefined, and that mock reaches * `@/lib/db/factory` through the index re-export, so in a shared process this file reads * `provider.getCapabilities` off undefined. Measured 2026-09-13: alone it is green; beside * `tests/api/db-objects.test.ts` it is not. Nothing this file can do prevents it, because - * mocking the factory is what the api layer is for, so the isolation sits here and - * `tests/run-components.sh` gives it a group of its own. + * mocking the factory is what the api layer is for. The runner gives every test file a bun + * process of its own, so that isolation is already in force and this paragraph, rather than a + * directory or an entry in a runner script, is where the requirement is written down. */ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; @@ -71,62 +73,9 @@ import { rowActions, type TreeRowActionHandlers } from "@/components/object-tree import { EXTERNAL_DATABASE_TYPES, SHIPPED_DATABASE_TYPES } from "@/lib/db/compatibility"; import { createDatabaseProvider } from "@/lib/db/factory"; import { declaredKinds, findKind } from "@/lib/db/object-kinds"; -import type { DatabaseConnection, DatabaseObject, ObjectKindSpec, ProviderCapabilities } from "@/lib/db/types"; +import type { DatabaseObject, ObjectKindSpec, ProviderCapabilities } from "@/lib/db/types"; import type { DatabaseType } from "@/lib/types"; - -/** - * The fields every provider's `validate()` demands, none of which is ever dialled. - * - * Nothing here connects: `createDatabaseProvider` is a switch over dynamic imports and a - * constructor, and the constructors validate their configuration without opening a socket or a - * file. The host is the loopback address and the port is 1 so that a provider which ever did - * try to dial would fail loudly rather than reach something real. - */ -const UNCONNECTED = { - id: "census", - name: "census", - host: "127.0.0.1", - port: 1, - database: "census", - user: "census", - password: "census", - filePath: ":memory:", - url: "http://127.0.0.1:1", - connectionString: "mongodb://127.0.0.1:1/census", - // Cassandra's driver refuses to build a client without one, so the census cannot reach that - // provider's declarations at all without it. A stock single-node install reports datacenter1. - localDataCenter: "datacenter1", - createdAt: new Date(0), -} as const; - -const unconnected = (type: DatabaseType): DatabaseConnection => ({ ...UNCONNECTED, type }) as DatabaseConnection; - -/** - * One connection per shipped type-id, as a Record so the compiler owns exhaustiveness. - * - * A new member of `DatabaseType` fails to compile here, which is a stronger failure than the - * runtime one the driven population below also gives: the census cannot be extended to a new - * engine by accident, and it cannot skip one either. - */ -export const CENSUS_CONNECTION: Readonly> = Object.freeze({ - postgres: unconnected("postgres"), - mysql: unconnected("mysql"), - sqlite: unconnected("sqlite"), - libsql: unconnected("libsql"), - duckdb: unconnected("duckdb"), - oracle: unconnected("oracle"), - mssql: unconnected("mssql"), - clickhouse: unconnected("clickhouse"), - druid: unconnected("druid"), - trino: unconnected("trino"), - cassandra: unconnected("cassandra"), - elasticsearch: unconnected("elasticsearch"), - opensearch: unconnected("opensearch"), - mongodb: unconnected("mongodb"), - redis: unconnected("redis"), - couchbase: unconnected("couchbase"), - libredb: unconnected("libredb"), -}); +import { CENSUS_CONNECTION } from "../helpers/census-connection"; /** * The committed expectation, transcribed from the design's kind-declaration table, one entry diff --git a/tests/isolated/use-storage-sync.test.ts b/tests/isolated/use-storage-sync.test.ts index 598742c1a..7114c210c 100644 --- a/tests/isolated/use-storage-sync.test.ts +++ b/tests/isolated/use-storage-sync.test.ts @@ -568,7 +568,7 @@ describe("useStorageSync", () => { * The retry timer and the debounce timer are different clocks. * * A push takes as long as the network does, and the user keeps working while - * it is in flight — so when a failure comes back, the debounce slot usually + * it is in flight, so when a failure comes back, the debounce slot usually * holds a fresh timer for an edit that has nothing to do with it. Sharing one * ref meant the retry replaced that timer with its own backoff, and a single * failed push held a later, healthy write off the server for as long as the @@ -576,13 +576,42 @@ describe("useStorageSync", () => { * * The tell is WHEN that write lands. On its own debounce it goes out ~500ms * after the edit; hostage to the backoff it cannot go out before the first - * retry step, which is a full second after the failure. The threshold sits - * between the two with room on both sides. + * retry step, which is a full second after the failure. + * + * HOW THAT IS WATCHED, and it is not the wall clock. This used to read + * `historyAt - editedAt < 800`, two `Date.now()` samples and a budget, which + * says "the machine got from here to there in under 800ms" and not "the write + * kept its own debounce": a loaded box that spends 900ms of that window + * descheduled reports a hook defect that is not there. The reference is now a + * TIMER this test arms itself, due at 800ms, between the 500ms debounce and + * the 1000ms first retry step, and the question is only which of the two fired + * first. Timers fire in deadline order however slow the machine is, so a stall + * delays both and can never swap them, and the fetch double records the answer + * synchronously inside the flush the debounce fired, so nothing interleaves. + * + * The wait on the reference timer afterwards is the control. Without it, + * `false` would also be the reading when the timer never ran at all. + * + * MEASURED, AND ONE OBVIOUS REFERENCE IS THE WRONG ONE. Comparing the history + * write against the connections RETRY does not discriminate: the failed + * collection is requeued, so the debounce flush at +500ms carries connections + * too and IS connections attempt 2. Both landed at +500ms. Moving this + * deadline to 200ms makes the test fail with the edit landing at +501ms, which + * is what proves the 800ms reading is a measurement rather than a formality. */ test("a failed push does not swallow a later edit's debounce", async () => { localStorage.setItem("libredb_server_migrated", "true"); let connectionsAttempts = 0; - let historyAt: number | null = null; + /** + * True when the history write went out before the reference timer, null until it goes out. + * + * Held on an object rather than in a bare `let` so the reads below keep the union: the + * only assignment is inside the fetch double, and the compiler narrows a `let` that is + * never assigned in this flow back to `null`, which makes `toBe(true)` a type error + * rather than a question. + */ + const race: { historyBeatReference: boolean | null } = { historyBeatReference: null }; + let referencePassed = false; mockGlobalFetch({ "/api/storage/config": { ok: true, status: 200, json: { provider: "postgres", serverMode: true } }, @@ -595,7 +624,7 @@ describe("useStorageSync", () => { return { ok: false, status: 500, json: { error: "Write failed" } }; }, "/api/storage/history": () => { - historyAt = Date.now(); + race.historyBeatReference = !referencePassed; return { ok: true, status: 200, json: { ok: true } }; }, "/api/storage": { ok: true, status: 200, json: {} }, @@ -614,21 +643,31 @@ describe("useStorageSync", () => { await waitFor(() => { expect(connectionsAttempts).toBe(1); }); - const editedAt = Date.now(); + // Armed with the edit, so the two deadlines start together. + const reference = setTimeout(() => { + referencePassed = true; + }, 800); act(() => { window.dispatchEvent(new CustomEvent("libredb-storage-change", { detail: { collection: "history" } })); }); await waitFor( () => { - expect(historyAt).not.toBeNull(); + expect(race.historyBeatReference).not.toBeNull(); + }, + { timeout: 5000 }, + ); + // The control: the reference timer really does fire, so `false` above can only + // mean the write lost the race and never "the timer was never scheduled". + await waitFor( + () => { + expect(referencePassed).toBe(true); }, { timeout: 5000 }, ); + clearTimeout(reference); - // ~500ms when the edit keeps its own debounce; no sooner than ~1100ms when - // the retry's backoff has replaced it. - expect(historyAt! - editedAt).toBeLessThan(800); + expect(race.historyBeatReference).toBe(true); }); /** diff --git a/tests/live/mysql-object-vocabulary.ts b/tests/live/mysql-object-vocabulary.ts index 21b5b99c9..6f1a95fc0 100644 --- a/tests/live/mysql-object-vocabulary.ts +++ b/tests/live/mysql-object-vocabulary.ts @@ -18,10 +18,10 @@ * - by hand, against a disposable server, with the command below; * - in Task 27's live acceptance run (#789), which is where it belongs permanently. * - * It is NOT in `bun run test` or `bun run test:ci`. `tests/run-core.sh` globs - * `tests/unit tests/api tests/integration tests/hooks tests/security tests/evals`, so nothing - * under `tests/live/` is collected, which is the same arrangement - * `tests/live/schema-diff-dialects.ts` has. + * It is NOT in `bun run test`. The runner collects every `*.test.ts` / `*.test.tsx` file under + * `tests/`, and excludes `tests/live/` by name (`EXCLUDED` in `tests/runner/discover.ts`), so this + * file is outside that set twice over: by its directory and by its name. That is the same + * arrangement `tests/live/schema-diff-dialects.ts` has. * * WHAT IT CAN AND CANNOT SEE, stated plainly because a guard nobody can calibrate is worse * than none. `SELECT DISTINCT` reports the spellings the server's DATA exhibits, not the diff --git a/tests/run-components.sh b/tests/run-components.sh deleted file mode 100755 index 40e28ef73..000000000 --- a/tests/run-components.sh +++ /dev/null @@ -1,448 +0,0 @@ -#!/bin/bash -# Component test runner with mock isolation groups. -# -# bun's mock.module() is process-wide, so when one test file mocks a module, -# every other file in the same bun process sees the mock instead of the real -# module. This script groups test files so that no file runs in the same -# process as a file that mocks its component module. -# -# Grouping rationale: -# Group 1 — Studio.test.tsx (mocks sidebar, schema-explorer, QueryEditor, -# studio/index, ConnectionModal, CommandPalette, SchemaDiagram, -# DataProfiler, CodeGenerator, TestDataGenerator, CreateTableModal, -# SaveQueryModal, etc.) -# Group 2 — Sidebar.test.tsx (mocks ConnectionsList, schema-explorer) -# Group 3 — BottomPanel.test.tsx (mocks ResultsGrid, QueryHistory, -# DataCharts, SchemaDiff, SavedQueries, VisualExplain, etc.) -# Group 4 — AdminDashboard shell + admin section/layout/index pages -# Group 5 — SecurityTab.test.tsx (mocks MaskingSettings) -# Group 6 — All remaining files (safe together — only mock libraries, -# ui primitives, or sub-components with no test files) - -set -e - -PASS=0 -FAIL=0 -# Count the `run_group` CALLS below when adding one (not the definition) - this is the -# number the final summary reports, and it had already drifted by one before Group 0e -# was added. Verify with `grep -c '^run_group ' tests/run-components.sh`, which is how -# the third drift was caught (#331 T5): 26 was declared while 27 calls existed, so the -# green summary line reported a group count no run had. -# Drifted again before this line was touched: it read 30 while 32 `run_group` calls -# existed, so every green run reported a group count no run had. The comment then went -# stale a fourth time by naming a DIGIT for the current value, which is the one thing -# here that cannot stay true: the value is whatever that grep prints, never a number -# written in prose. -TOTAL_GROUPS=46 -EXTRA_BUN_ARGS=("$@") -GROUP_INDEX=0 -COVERAGE_MODE=0 -COVERAGE_BASE_DIR="" - -for arg in "${EXTRA_BUN_ARGS[@]}"; do - if [ "$arg" = "--coverage" ]; then - COVERAGE_MODE=1 - fi - if [[ "$arg" == --coverage-dir=* ]]; then - COVERAGE_BASE_DIR="${arg#--coverage-dir=}" - fi -done - -run_group() { - local label="$1" - shift - # Optional --nocov flag: run the group WITHOUT coverage collection. Used for - # groups that import modules without exercising them (e.g. the exports shim, - # which loads the whole component chain) — their load-only lcov records would - # otherwise merge as phantom uncovered lines on files other groups fully cover. - local nocov=0 - if [ "$1" = "--nocov" ]; then - nocov=1 - shift - fi - GROUP_INDEX=$((GROUP_INDEX + 1)) - echo "" - echo "=== $label ===" - - local RUN_ARGS=() - for arg in "${EXTRA_BUN_ARGS[@]}"; do - if [[ "$arg" == --coverage-dir=* ]]; then - continue - fi - if [ "$nocov" -eq 1 ] && [[ "$arg" == --coverage* ]]; then - continue - fi - RUN_ARGS+=("$arg") - done - - if [ "$COVERAGE_MODE" -eq 1 ] && [ -n "$COVERAGE_BASE_DIR" ] && [ "$nocov" -eq 0 ]; then - RUN_ARGS+=("--coverage-dir=${COVERAGE_BASE_DIR}/group-${GROUP_INDEX}") - fi - - if bun test "${RUN_ARGS[@]}" "$@"; then - PASS=$((PASS + 1)) - else - FAIL=$((FAIL + 1)) - echo "FAILED: $label" - fi -} - -# Group 0a: useStorageSync hook (isolated — mocks @/lib/storage which contaminates other hook tests) -run_group "Group 0a: useStorageSync hook" \ - tests/isolated/use-storage-sync.test.ts - -# Group 0b: Factory singleton (isolated — mocks provider modules which contaminates provider unit tests) -run_group "Group 0b: Factory singleton" \ - tests/isolated/factory-singleton.test.ts - -# Group 0b2: The factory's cache, execution profiles and shutdown handlers (#789). -# Isolated for the reason Group 0c's comment already named from the other side: this file -# can only pass while it is the FIRST thing in its process to evaluate `@/lib/db/factory`. -# It mocks six native driver packages and `@/lib/ssh/tunnel`, then imports the factory under -# NODE_ENV=production to capture the SIGTERM and SIGINT handlers the module registers on load. -# Both of those happen once per process, so any earlier evaluation of the factory by another -# file leaves this one with an already-built module: no handler to capture, and unmocked -# drivers behind `getOrCreateProvider`, whose cached entry then throws inside the -# `clearProviderCache()` in `beforeEach` and fails every remaining test in the file. -# Measured 2026-09-13: a three-line probe under `tests/unit/` whose only content is an import -# of `@/lib/db/factory` takes this file from 99 pass 0 fail to 44 pass 56 fail, in either CLI -# order, because bun does not run files in the order they are listed. A probe importing -# `@/lib/ssh/tunnel` or `@/lib/db/compatibility` instead reproduces nothing. -run_group "Group 0b2: Factory cache and execution profiles" \ - tests/isolated/factory.test.ts - -# Group 0b3: The fleet census of object source declarations and the editor language guard (#789). -# Both build all seventeen providers through the REAL `createDatabaseProvider`, which is the only -# way to census what each provider declares rather than what somebody typed. Every file under -# `tests/api/` mocks `@/lib/db` with a `createDatabaseProvider: mock()` that answers undefined, -# and that mock reaches `@/lib/db/factory` through the index re-export, so both files read -# `provider.getCapabilities` off undefined the moment they share a process with the api layer. -# Measured 2026-09-13: census plus `tests/api/db-objects.test.ts` is 3 fail, the language guard -# plus the same file is 1 fail, and each of them alone is 0 fail. There is nothing either file can -# do about it: mocking the factory is what the api layer is for. -# The edit census joins them for the same reason and adds nothing new to it: it builds every -# provider through the REAL `createDatabaseProvider` too, and it imports `CENSUS_CONNECTION` from -# the source census beside it, so the two files share one population and one process. -run_group "Group 0b3: Object source declaration census" \ - tests/isolated/object-source-declarations.test.ts \ - tests/isolated/object-edit-declarations.test.ts \ - tests/isolated/monaco-language-ids.test.ts - -# Group 0c: exports CJS shim (isolated — importing it pulls @/lib/db/factory into the -# module cache, which breaks factory.test.ts's first-import signal-handler capture). -# --nocov: the shim import loads the entire component chain without rendering it, -# which would inject load-only zero-hit lcov records for files other groups cover. -run_group "Group 0c: Exports shim" --nocov \ - tests/isolated/exports-shim.test.ts - -# Group 0d: Monaco loader wiring (isolated — mocks @monaco-editor/react and must observe -# the loader call QueryEditor makes at module-evaluation time, so it dynamic-imports the -# component after the mock is registered). -# --nocov: same load-only concern as Group 0c — importing QueryEditor pulls its whole -# module chain without rendering it. -run_group "Group 0d: Monaco loader wiring" --nocov \ - tests/isolated/monaco-loader-wiring.test.ts - -# Group 0e: The standalone execution path against the REAL confirmation gate -# (isolated — tests/hooks/use-query-execution.test.ts stubs -# @/components/QuerySafetyDialog with mock.module, which is process-wide, so a -# test sharing that process cannot observe what the real predicate answers). -run_group "Group 0e: Query safety gate (standalone path)" \ - tests/isolated/query-safety-gate-standalone.test.ts - -# Group 0f: The agent's model layer against the REAL LLM error classes -# (isolated — every tests/api/ai/*.test.ts replaces @/lib/llm/types with stub -# error classes whose constructors take a message only, and mock.module is -# process-wide, so a test sharing that process sees the mapper's provider tag -# dropped even though the class identity still matches). The four files share one -# process: none of them mocks a module, so they only need isolating from those. -run_group "Group 0f: Agent model layer" \ - tests/isolated/agent-model-adapter.test.ts \ - tests/isolated/agent-provider-registry.test.ts \ - tests/isolated/agent-capability-probe.test.ts \ - tests/isolated/agent-investigation.test.ts - -# Group 0g: The agent's composition root. Its own group, NOT part of 0f: it mocks -# @/lib/db, @/lib/agent/investigation and @/lib/seed/resolve-connection, and -# mock.module is process-wide, so sharing 0f's process would hand the loop suite -# above a stubbed investigation module. -run_group "Group 0g: Agent runtime composition" \ - tests/isolated/agent-runtime.test.ts - -# Group 0h: The agent's end-to-end investigation against real engines. Its own -# group, NOT part of 0f: it mocks `pg` (the PostgreSQL suite's engine-fixture -# technique) and mock.module is process-wide, so sharing a process would hand every -# other file in it a pg module that answers only this fixture's statements. -run_group "Group 0h: Agent end-to-end investigation" \ - tests/isolated/agent-investigation-e2e.test.ts - -# Group 0i: The start path's model gate. Its own group, NOT part of 0f: it mocks -# @/lib/agent/capability-probe and @/lib/agent/model-adapter, and mock.module is -# process-wide — 0f contains the suites for both of those modules, so sharing its -# process would hand them the stubs written for this one. -run_group "Group 0i: Agent capability gate" \ - tests/isolated/agent-capability-gate.test.ts - -# Group 0j: The workflow classifier. Its own group, NOT part of 0f: it mocks -# @/lib/agent/model-adapter and the `ai` package, and mock.module is process-wide -# — 0f holds the model adapter's own suite, so sharing its process would hand that -# suite the stub written here instead of the module it is testing. -run_group "Group 0j: Agent workflow classifier" \ - tests/isolated/agent-workflow-classifier.test.ts - -# Group 1: Studio (isolated — mocks almost every child component) -run_group "Group 1/6: Studio" \ - tests/components/Studio.test.tsx - -# Group 1b: the palette-item-to-rail path (isolated — it must see the REAL -# use-agent-prefill, use-tab-manager and CommandPalette, all three of which Group 1 -# replaces with mock.module stubs, and mock.module is process-wide). -run_group "Group 1b/6: Studio agent ask" \ - tests/components/studio-agent-ask.test.tsx - -# Group 2: Sidebar (isolated — mocks ConnectionsList, SchemaExplorer) -run_group "Group 2/6: Sidebar" \ - tests/components/sidebar/Sidebar.test.tsx - -# Group 3: BottomPanel (isolated — mocks ResultsGrid, QueryHistory, DataCharts, SchemaDiff) -run_group "Group 3/6: BottomPanel" \ - tests/components/studio/BottomPanel.test.tsx - -# Group 4: AdminDashboard shell + section pages -run_group "Group 4/6: AdminDashboard" \ - tests/components/admin/AdminDashboard.test.tsx \ - tests/components/admin/AdminOverviewPage.test.tsx \ - tests/components/admin/AdminSectionPages.test.tsx \ - tests/components/AdminPage.test.tsx - -# Group 4b: AdminLayout (isolated — mocks AdminDashboard) -run_group "Group 4b/6: AdminLayout" \ - tests/components/admin/AdminLayout.test.tsx - -# Group 5: SecurityTab (isolated — mocks MaskingSettings) -run_group "Group 5/6: SecurityTab" \ - tests/components/admin/SecurityTab.test.tsx - -# Group 6: MonitoringDashboard (isolated - mocks all monitoring tabs) -run_group "Group 6/7: MonitoringDashboard" \ - tests/components/monitoring/MonitoringDashboard.test.tsx - -# Group 7: Results-grid subcomponents (isolated from ResultsGrid.test.tsx mocks) -run_group "Group 7/10: Results-grid subcomponents" \ - tests/components/results-grid/StatsBar.test.tsx \ - tests/components/results-grid/ResultCard.test.tsx \ - tests/components/results-grid/RowDetailSheet.test.tsx - -# Group 8: SavedQueries (isolated - mocks @/lib/storage) -run_group "Group 8/10: SavedQueries" \ - tests/components/SavedQueries.test.tsx - -# Group 9: StudioHeaders + TableItem (isolated - mock dropdown-menu) -run_group "Group 9/12: StudioHeaders & TableItem" \ - tests/components/studio/StudioMobileHeader.test.tsx \ - tests/components/studio/StudioDesktopHeader.test.tsx \ - tests/components/schema-explorer/TableItem.test.tsx - -# Group 10: PoolTab (isolated - mock globalThis.fetch) -run_group "Group 10/12: PoolTab" \ - tests/components/monitoring/PoolTab.test.tsx - -# Group 10b: PivotTable (isolated - mocks @/lib/export/download and dropdown-menu, which -# the DatabaseDocs export tests in the smoke group need real) -run_group "Group 10b: PivotTable" \ - tests/components/PivotTable.test.tsx - -# Group 11: Smoke tests (isolated - mock globalThis.fetch + MonitoringEmbed) -run_group "Group 11/12: Smoke tests" \ - tests/components/agent/AgentRail.test.tsx \ - tests/components/agent/AnswerCard.test.tsx \ - tests/components/agent/ConsentCard.test.tsx \ - tests/components/agent/SafetyStrip.test.tsx \ - tests/components/agent/use-agent-run.test.tsx \ - tests/components/admin/MonitoringEmbed.test.tsx \ - tests/components/VisualExplain.test.tsx \ - tests/components/DatabaseDocs.test.tsx \ - tests/components/SnapshotTimeline.test.tsx \ - tests/components/CodeGenerator.test.tsx \ - tests/components/TestDataGenerator.test.tsx \ - tests/components/CreateTableModal.test.tsx \ - tests/components/SaveQueryModal.test.tsx \ - tests/components/MobileNav.test.tsx \ - tests/components/DataImportModal.test.tsx \ - tests/components/RootLayout.test.tsx \ - tests/components/AppErrorPages.test.tsx \ - tests/components/LazyView.test.tsx \ - tests/components/Page.test.tsx \ - tests/components/LoginPage.test.tsx \ - tests/components/LoginPageOIDC.test.tsx \ - tests/components/CommunitySection.test.tsx \ - tests/components/ConnectionSignature.test.tsx \ - tests/components/GitHubRepoLink.test.tsx \ - tests/components/MonitoringPage.test.tsx \ - tests/components/monitoring/PanelUnavailable.test.tsx \ - tests/components/monitoring/MetricChart.test.tsx - -# Group 12: MaskingSettings (isolated — mocks @/lib/data-masking with different shape than ResultsGrid/DataProfiler) -run_group "Group 12/13: MaskingSettings" \ - tests/components/MaskingSettings.test.tsx - -# Group 13: SchemaDiff (isolated — mocks @/components/ui/badge, @/components/ui/select) -run_group "Group 13/14: SchemaDiff" \ - tests/components/SchemaDiff.test.tsx - -# Group 16: ConnectionModal Mobile Drawer (isolated - useIsMobile returns true) -run_group "Group 16/16: ConnectionModal Mobile" \ - tests/components/ConnectionModal.mobile.test.tsx - -# Group 14: DataCharts (isolated — mocks @/lib/storage with chart methods) -run_group "Group 14/16: DataCharts" \ - tests/components/DataCharts.test.tsx - -# Group 15: All remaining files (safe together) -run_group "Group 15/16: Remaining components" \ - tests/components/copy-button.test.tsx \ - tests/components/rich-text.test.tsx \ - tests/components/QueryEditor.test.tsx \ - tests/components/QuerySafetyDialog.test.tsx \ - tests/components/QueryHistory.test.tsx \ - tests/components/ConnectionModal.test.tsx \ - tests/components/CommandPalette.test.tsx \ - tests/components/ResultsGrid.test.tsx \ - tests/components/SchemaDiagram.test.tsx \ - tests/components/DataProfiler.test.tsx \ - tests/components/schema-explorer/SchemaExplorer.test.tsx \ - tests/components/schema-explorer/ColumnList.test.tsx \ - tests/components/sidebar/ConnectionItem.test.tsx \ - tests/components/sidebar/ConnectionsList.test.tsx \ - tests/components/studio/QueryToolbar.test.tsx \ - tests/components/studio/StudioTabBar.test.tsx \ - tests/components/admin/OverviewTab.test.tsx \ - tests/components/admin/OperationsTab.test.tsx \ - tests/components/admin/AuditTab.test.tsx \ - tests/components/monitoring/StorageTab.test.tsx \ - tests/components/monitoring/SessionsTab.test.tsx \ - tests/components/monitoring/TablesTab.test.tsx \ - tests/components/monitoring/QueriesTab.test.tsx - -# Read the real threshold storage without the other group's partial storage mocks. -run_group "Group 22: Saved monitoring thresholds" \ - tests/components/monitoring/PerformanceTab.test.tsx \ - tests/components/monitoring/OverviewTab.test.tsx - -# Group 18: ui/resizable (isolated — installs a global DOMRect that -# react-resizable-panels 4 needs, and is the one suite that renders -# the real library instead of mocking @/components/ui/resizable) -run_group "Group 18: ui/resizable" \ - tests/components/ui/resizable.test.tsx - -# Group 17: StudioWorkspace (isolated — mocks the same child families as Studio: -# sidebar, QueryEditor, studio/index, SchemaDiagram, DataProfiler, -# CodeGenerator, TestDataGenerator, SaveQueryModal, DataImportModal, -# QuerySafetyDialog, plus the workspace adapter hooks) -run_group "Group 17: StudioWorkspace" \ - tests/components/StudioWorkspace.test.tsx - -# Groups 18 and 19: the two theme files. Each mocks `next-themes` — process-wide, -# and with a DIFFERENT shape (one replaces `useTheme`, the other `ThemeProvider`), -# so they cannot share a process with each other, nor with anything that reaches -# the real next-themes through ui/sonner. -run_group "Group 18: ThemeToggle" \ - tests/components/ThemeToggle.test.tsx - -run_group "Group 19: ThemeProvider" \ - tests/components/ThemeProvider.test.tsx - -# Group 20: WireCompatibilityHint. Its own group, and it had NO group at all until now: -# the file shipped with #426 and was never added to this script, so its seven tests had -# never run in CI once. It cannot join Group 11 or 15 either - it mocks -# @/lib/db/compatibility, and mock.module is process-wide, so it would hand LoginPage's -# engine-count assertions and ConnectionModal's own hint render a two-entry stub registry. -run_group "Group 20: WireCompatibilityHint" \ - tests/components/WireCompatibilityHint.test.tsx - -# Group 23: The object tree (#789). Its own group: it replaces globalThis.fetch for every test -# and restores it afterwards, and a file that assigns the global at MODULE scope (the pattern -# tests/components/monitoring/PoolTab.test.tsx uses) would be captured as this file's "real" fetch -# when the two share a process. It mocks no module, so nothing else needs isolating from it. -run_group "Group 23: Object tree" \ - tests/components/object-tree.test.tsx - -# Group 24: First paint (#789, #765). Its own group for Group 23's reason, and separate from -# it because it counts EVERY request by pathname: sharing a process with a file that answers -# other routes from the same global would make "exactly two catalog reads" count somebody -# else's reads. It renders the real tree against a fetch double rather than a mocked module. -run_group "Group 24: Object tree first paint" \ - tests/components/object-tree/first-paint.test.tsx - -# Group 25: The object tree's row menu (U22, #789). Its own group for Group 23's reason - -# it replaces globalThis.fetch for every test and restores it afterwards - and separate from -# 23 and 24 because it renders the REAL menu against the real tree: a file sharing its -# process that replaced a menu primitive with mock.module would make every assertion in it a -# statement about the stub. -run_group "Group 25: Object tree row menu" \ - tests/components/object-tree/row-menu.test.tsx - -# Group 26: the embedded workspace's object tree (#789, B76). Its own group for Group 23's -# reason - it replaces globalThis.fetch to prove no route is asked - and separate from Group 17, -# which mocks the sidebar and the workspace adapter hooks process-wide: this file exists to drive -# the REAL adapter and the REAL sidebar from the published prop, which is exactly what those -# mocks would replace. It mocks only the editor and the panel library, neither of which Group 17 -# asserts against. -run_group "Group 26: Embedded workspace object tree" \ - tests/components/studio/embedded-object-tree.test.tsx - -# Group 21: ui/scroll-area. Its own group for the same reason ui/resizable has one: -# it is the only suite that renders the REAL @radix-ui/react-scroll-area, while -# Sidebar and RowDetailSheet both mock that module process-wide - sharing a process -# with either would hand these tests a stub with no Radix wrapper div to assert on. -# It also installs a global ResizeObserver, which Radix mounts on the viewport. -run_group "Group 21: ui/scroll-area" \ - tests/components/ui/scroll-area.test.tsx - -# Group 27: The read-only object source viewer (#789). Its own group: it replaces -# @monaco-editor/react with mock.module, which is process-wide, and Group 15 holds -# QueryEditor.test.tsx, which installs a DIFFERENT double of that same module - sharing a -# process would hand one of the two suites the other's editor. It also asserts against -# globalThis.fetch for the default-reader case. -run_group "Group 27: Object source viewer" \ - tests/components/object-source/ObjectSourceView.test.tsx - -# Group 28: The standalone shell's Source tab (#789). Its own group, for three reasons that -# each rule out sharing one: it replaces @monaco-editor/react with mock.module, which is -# process-wide, and both Group 15 (QueryEditor.test.tsx) and Group 27 install a DIFFERENT -# double of that same module; it mocks the same child families as Group 1 while deliberately -# using the REAL use-tab-manager and the REAL StudioTabBar, which Group 1 replaces; and it -# answers globalThis.fetch for the source route. -run_group "Group 28: Studio source tab" \ - tests/components/studio/source-tab.test.tsx - -# Group 29: The EMBEDDED shell's Source tab (#789). Its own group for Group 28's three reasons -# and one more that is this file's alone. It installs a process-wide @monaco-editor/react double, -# as Groups 15, 27 and 28 each install a different one; it mounts the REAL adapter, the REAL -# sidebar, the REAL use-tab-manager and the REAL StudioTabBar, all of which Group 17 replaces -# process-wide; and it replaces globalThis.fetch to prove NO route is asked, which is the whole -# point on this shell, since the published package ships no API routes at all. It is separate -# from Group 26, which is the same shell's tree, because this one also doubles the studio barrel -# to capture the bottom panel's props. -run_group "Group 29: Embedded workspace source tab" \ - tests/components/studio/embedded-source.test.tsx - -# Group 30: The apply preview dialog (#789 Phase 3). Its own group for Group 27's reason: it -# installs a process-wide @monaco-editor/react double, and Groups 15, 27, 28 and 29 each install a -# different one. It doubles `DiffEditor` rather than `Editor`, which is a different export of the -# same module, so sharing a process with any of them would hand one suite the other's editor. -run_group "Group 30: Apply preview dialog" \ - tests/components/object-source/ApplyPreviewDialog.test.tsx - -# Summary -echo "" -echo "========================================" -if [ $FAIL -eq 0 ]; then - echo "All $TOTAL_GROUPS groups passed!" - if [ "$COVERAGE_MODE" -eq 1 ] && [ -n "$COVERAGE_BASE_DIR" ]; then - node scripts/merge-lcov.mjs "${COVERAGE_BASE_DIR}"/group-*/lcov.info "${COVERAGE_BASE_DIR}/lcov.info" - fi -else - echo "$FAIL/$TOTAL_GROUPS groups FAILED" - exit 1 -fi diff --git a/tests/run-core.sh b/tests/run-core.sh deleted file mode 100755 index a38b972f2..000000000 --- a/tests/run-core.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# Core test runner with PER-FILE mock isolation. -# -# bun's mock.module() is process-wide: when one test file mocks a shared module -# (e.g. @/lib/db/factory, @/lib/oidc, the audit module) the mock leaks into every -# other file that runs in the same bun process. A file that imports the real -# export then sees the partial mock — producing nondeterministic failures such as -# "clearProviderCache is not a function" or "Export named 'removeProvider' not -# found", depending purely on file load order. This passes locally and fails in -# CI (different order). -# -# Running each core test file in its OWN bun process makes cross-file -# contamination structurally impossible. This mirrors tests/run-components.sh, -# which isolates the component tests for the same reason. It is slower than a -# single invocation, but correctness beats speed for the coverage gate. -# -# Usage: bash tests/run-core.sh [extra bun test args] -# When --coverage-dir=DIR is passed, each file writes to DIR/file-N so the -# per-file lcov reports can be merged afterwards. - -set -uo pipefail - -EXTRA_BUN_ARGS=() -COVERAGE_BASE_DIR="" -for arg in "$@"; do - if [[ "$arg" == --coverage-dir=* ]]; then - COVERAGE_BASE_DIR="${arg#--coverage-dir=}" - else - EXTRA_BUN_ARGS+=("$arg") - fi -done - -# Deterministic, sorted list of every core test file. -# -# `tests/evals` is here rather than in run-components.sh because it needs exactly -# what this script already gives: one process per file. The eval suites drive the -# real agent run loop over the real ratified provider package, so they must not -# share a process with `tests/api/ai/*.test.ts`, which replaces `@/lib/llm/types` -# process-wide with stub error classes. Per-file isolation makes that structural -# instead of something a group comment has to remember. -mapfile -t FILES < <(find tests/unit tests/api tests/integration tests/hooks tests/security tests/evals \ - -type f \( -name '*.test.ts' -o -name '*.test.tsx' \) | sort) - -if [ "${#FILES[@]}" -eq 0 ]; then - echo "run-core.sh: no core test files found" >&2 - exit 1 -fi - -TOTAL="${#FILES[@]}" -PASS=0 -FAIL=0 -FAILED_FILES=() -INDEX=0 - -for file in "${FILES[@]}"; do - INDEX=$((INDEX + 1)) - RUN_ARGS=("${EXTRA_BUN_ARGS[@]}") - if [ -n "$COVERAGE_BASE_DIR" ]; then - RUN_ARGS+=("--coverage-dir=${COVERAGE_BASE_DIR}/file-${INDEX}") - fi - - echo "=== [${INDEX}/${TOTAL}] ${file} ===" - if bun test "${RUN_ARGS[@]}" "$file"; then - PASS=$((PASS + 1)) - else - FAIL=$((FAIL + 1)) - FAILED_FILES+=("$file") - fi -done - -echo "" -echo "========================================" -if [ "$FAIL" -eq 0 ]; then - echo "All ${TOTAL} core test files passed!" -else - echo "${FAIL}/${TOTAL} core test files FAILED:" - for f in "${FAILED_FILES[@]}"; do - echo " - $f" - done - exit 1 -fi diff --git a/tests/run-tests.ts b/tests/run-tests.ts new file mode 100644 index 000000000..31e33bd7a --- /dev/null +++ b/tests/run-tests.ts @@ -0,0 +1,499 @@ +#!/usr/bin/env bun +/** + * The test runner: `bun run test`. + * + * It runs every test file in its own bun process, several files at a time, and it + * is written in TypeScript rather than shell so that the one command a contributor + * is told to run behaves the same on Linux, macOS and Windows. The two bash scripts + * it replaces could not: `tests/run-core.sh` used `mapfile`, a bash 4 builtin, and + * macOS ships bash 3.2, so the documented gate never ran there at all. + * + * Why a process per file rather than `bun test `: see the docblock in + * `tests/runner/execute.ts`. + * + * bun tests/run-tests.ts every test file + * bun tests/run-tests.ts tests/api one layer + * bun tests/run-tests.ts tests/api/db.test.ts one file + * bun tests/run-tests.ts --jobs=4 bound the concurrency + * bun tests/run-tests.ts --list what would run + * bun tests/run-tests.ts --coverage --merge-into=coverage/lcov.info + * bun tests/run-tests.ts tests/unit -- --bail pass flags to bun test + * + * That last one needs both halves as written: a selector before the `--`, and the + * runner invoked directly. `bun run test -- --bail` does not work, because `bun run` + * consumes the first `--` itself, and neither does a `--` straight after the script + * path, for the same reason (measured on 1.4.2; the refusal in runner/options.ts says + * so). + * + * It exits 0 when every file passed, 1 when a file failed, and 2 when the runner could + * not do its job (a usage error, a selector that names nothing, a scratch directory it + * could not remove, a write that failed for the runner's own reason such as a full + * disk). A reader that goes away is NOT one of those: `bun run test | head -1` keeps + * the run's own 0 or 1, because turning it into 2 would hide the very distinction the + * 2 is for. Stopped by SIGINT, SIGTERM, SIGHUP or SIGBREAK, it stops scheduling, kills + * the files still running, removes its scratch directory and exits 128 + the signal's + * number: 130, 143, 129, and 149 for SIGBREAK on Windows (see tests/runner/signals.ts). + */ +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { availableParallelism, tmpdir } from "node:os"; +import path from "node:path"; +import { captureBounded } from "./runner/capture"; +import { assertCoverageDirIsOurs, assertMergeTargetIsOurs } from "./runner/coverage"; +import { COVERAGE_EXEMPT_FILES, selectTestFiles } from "./runner/discover"; +import { coverageDirFor, type RunFile, runTestFiles, type SpawnOutcome } from "./runner/execute"; +import { parseRunnerArgs, type RunnerOptions } from "./runner/options"; +import { formatFileLine, formatSummary } from "./runner/report"; +import { missingHelm, planRequirements, requiredCapabilities, systemHelmProbe } from "./runner/requirements"; +import { exitCodeForSignal, STOP_SIGNALS, type StopSignal } from "./runner/signals"; + +const root = path.resolve(import.meta.dir, ".."); + +/** How long a child that was asked to stop is given before it is killed outright. */ +const KILL_ESCALATION_MS = 5_000; + +/** + * How long a signal's last line is given to reach its reader before the process ends + * anyway. + * + * A signal has to end the run promptly whatever the reader is doing. Measured on bun + * 1.4.2 with a consumer that had stopped reading: the handler's own write queued + * behind a full pipe (64 KiB on Linux), the process sat there for the whole stall and + * survived a second SIGINT, a SIGTERM and a SIGHUP, and when the consumer finally + * drained it exited 1 with the run's summary and "Interrupted (SIGINT)." after it. So + * the write is raced against this, and when it loses, the Interrupted line is lost: + * the reader that was not reading is the one that does not get it, and the exit code + * and the scratch-directory removal still say what happened. Three seconds is far more + * than a reader that is reading needs, even on a loaded 4-CPU CI runner. + */ +const SIGNAL_WRITE_GRACE_MS = 3_000; + +const live = new Set(); + +/** + * The run's own temporary directory (the children's junit reports), removed on every + * way out: a normal end, an error (exit 2), and any of SIGINT, SIGTERM, SIGHUP and + * SIGBREAK, which end the run with 128 + the signal's number (130, 143, 129) or with + * exit 2 when the directory cannot be removed. The children themselves need no such + * care, because `--no-orphans` takes them down with this process. + */ +let runScratch: string | null = null; + +/** + * Removes it, and answers with what went wrong rather than throwing. + * + * `force: true` only ignores a path that is not there; a directory that cannot be + * removed (no write permission on its parent, a Windows handle still open on a junit + * file) still throws. Every caller here is on its way out, two of them from a signal + * listener, where a throw is worse than useless: measured on bun 1.4.2, a throw from + * inside a SIGINT listener left the process RUNNING and the queue started the next + * file. So the failure comes back as a sentence for the caller to print before it + * exits 2, and is never swallowed. + */ +function removeRunScratch(): string | null { + if (runScratch === null) return null; + const directory = runScratch; + runScratch = null; + try { + rmSync(directory, { recursive: true, force: true }); + return null; + } catch (error) { + return `The run's scratch directory ${directory} could not be removed: ${error instanceof Error ? error.message : String(error)}`; + } +} + +/** + * True once the run has been stopped from outside (a signal, or an error that ends + * it): no further file is started, and the results of the children being killed are + * not printed as though they were the run's own verdict. + */ +let stopping = false; + +/** True once the run's header has been written, so stdout has something to drain. */ +let runStarted = false; + +/** Stops scheduling and asks every running child to stop. */ +function stopRun(): void { + stopping = true; + for (const child of live) child.kill("SIGTERM"); +} + +/** + * Writes, and resolves when the bytes have actually left this process. + * + * bun writes to a pipe asynchronously, and `process.exit` throws away whatever is + * still pending: measured on 1.4.2, a run whose stdout was a pipe (a CI log, `| tee`, + * a test driving the runner) lost about a third of a failing file's megabyte of + * output AND the whole summary with it, exit code intact. So every exit path writes + * its last line through here first. The callback of a write waits for the writes + * queued before it as well, which is what drains the output printed as files landed; + * an EMPTY write's callback does not (measured), so this is only ever called with + * text. + * + * What it covers is what THIS process writes. A child can still drop part of its own + * queued console output when it exits under load, with no runner in the picture at + * all (measured on 1.4.2, and filed as D96 in docs/BACKLOG.md), and no drain here can + * put that back. + */ +function written(stream: NodeJS.WriteStream, text: string): Promise { + return new Promise((resolve, reject) => { + stream.write(text, (error) => (error ? reject(error) : resolve())); + }); +} + +/** + * True when a write failed because the reader has gone rather than because this + * process could not write: `| head -1`, a CI log tailer that stopped, a closed + * terminal. Measured on bun 1.4.2, the write callback's error carries `code` "EPIPE" + * for those, and a real runner problem carries its own ("ENOSPC" for a full disk, + * measured against /dev/full), so the two are told apart by that and not by guesswork. + * The other two names are reasoned rather than measured: neither came up here, and + * both describe a descriptor that has gone rather than a run that could not be made. + */ +function readerHasGone(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException | null)?.code; + return code === "EPIPE" || code === "ERR_STREAM_DESTROYED" || code === "EBADF"; +} + +/** Writes and waits, treating a reader that has gone as nothing to report. */ +async function writtenOrReaderGone(stream: NodeJS.WriteStream, text: string): Promise { + try { + await written(stream, text); + } catch (error) { + if (!readerHasGone(error)) throw error; + } +} + +/** + * The one way out: the last lines reach their reader, then the process ends. + * + * `graceMs` bounds the wait. Only the signal path passes it, because only the signal + * path has something more urgent than its own last line (see SIGNAL_WRITE_GRACE_MS); + * everywhere else the write is the reason the process is still alive. + */ +async function exitAfterWriting( + code: number, + text: { stdout?: string; stderr?: string }, + graceMs?: number, +): Promise { + const writes: Promise[] = []; + if (text.stdout !== undefined) writes.push(written(process.stdout, text.stdout)); + if (text.stderr !== undefined) writes.push(written(process.stderr, text.stderr)); + // allSettled, not all: the other stream still has something to say. A scratch + // directory that could not be removed is explained on stderr while the run's last + // line goes to stdout, and Promise.all would abandon one the moment the other broke. + // Answered inside the promise rather than in a catch around the await, so the grace + // below cannot walk away from a rejection and leave an unhandled one behind. + const drained = Promise.allSettled(writes).then((results) => + // The stream that refused the bytes is the stream that would have carried the + // explanation, so nothing more can be said there; what is left to say is said by + // the exit code. A reader that went away is not the runner failing, and answering + // 2 there would take the run's own verdict away from `bun run test | head -1`. + // Any other write error IS the runner's problem, which is what 2 is for. + results + .filter((result): result is PromiseRejectedResult => result.status === "rejected") + .every((result) => readerHasGone(result.reason)) + ? code + : 2, + ); + process.exit( + graceMs === undefined ? await drained : await Promise.race([drained, Bun.sleep(graceMs).then(() => code)]), + ); +} + +/** + * Children inherit the environment, plus one decision: `FORCE_COLOR` when this + * runner is on a terminal. Each child's output is a pipe, so bun would drop its + * colour and the failure diffs are much harder to read without it. `NO_COLOR` wins + * over that, because it is the user's own word. + * + * Nothing else is set here. The environment tests run under is pinned by + * `tests/setup.ts`, which bunfig preloads into every child. + */ +function childEnvironment(): Record { + const wantsColour = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR; + return wantsColour ? { ...process.env, FORCE_COLOR: "1" } : { ...process.env }; +} + +function spawnTestFile(bunArgs: string[], junitDir: string): RunFile { + return async ({ file, index, coverageDir, timeoutMs }): Promise => { + const coverageArgs = coverageDir ? ["--coverage", "--coverage-reporter=lcov", `--coverage-dir=${coverageDir}`] : []; + // Every child writes a junit report, and it is the runner's only source of truth + // about what the file did: the counts, and the titles of the tests it skipped, + // which bun names nowhere else. Its console output is for a reader and decides + // nothing (see readTestReport). The file is small, even for a child that printed + // hundreds of megabytes, and the whole directory is removed when the run ends. + const junitPath = path.join(junitDir, `file-${index + 1}.xml`); + + const command = [ + process.execPath, + // Reap whatever the file spawned (helm, node, sh) if this child is killed: + // bun uses PR_SET_PDEATHSIG on Linux, EVFILT_PROC on macOS and a + // kill-on-close Job Object on Windows, so a timeout leaves nothing behind. + "--no-orphans", + "test", + ...bunArgs, + // AFTER the user's arguments, because bun takes the last of a repeated option: + // a forwarded `-- --reporter-outfile=x` would otherwise send the report + // somewhere else and leave every file looking as though it wrote none. + "--reporter=junit", + `--reporter-outfile=${junitPath}`, + ...coverageArgs, + // "./" matters: bun reads a bare relative path as a SUBSTRING FILTER over the + // whole tree, so `bun test tests/a/b.test.ts` also runs any other file whose + // path contains that string. With the prefix it is a path, on every platform. + `./${file}`, + ]; + + const startedAt = Date.now(); + const child = Bun.spawn(command, { + // Every child runs from the repository root: bunfig.toml's preload, the `@/` + // alias and the tests that read repository files all resolve from there. + cwd: root, + stdout: "pipe", + stderr: "pipe", + env: childEnvironment(), + }); + live.add(child); + + let timedOut = false; + const softKill = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + }, timeoutMs); + const hardKill = setTimeout(() => { + if (timedOut) child.kill("SIGKILL"); + }, timeoutMs + KILL_ESCALATION_MS); + + // bun writes its file header, failure diffs and per-file summary to stderr, and + // the tests' own console output to stdout, so both are captured. Each is bounded + // at both ends rather than read whole: see tests/runner/capture.ts. + const [stdout, stderr] = await Promise.all([ + captureBounded(child.stdout as ReadableStream, { name: "stdout" }), + captureBounded(child.stderr as ReadableStream, { name: "stderr" }), + ]); + const exitCode = await child.exited; + + clearTimeout(softKill); + clearTimeout(hardKill); + live.delete(child); + + return { + exitCode: child.signalCode ? null : exitCode, + signal: child.signalCode, + output: `${stderr}${stdout}`, + durationMs: Date.now() - startedAt, + timedOut, + junitReport: existsSync(junitPath) ? readFileSync(junitPath, "utf8") : null, + }; + }; +} + +function mergeCoverage(options: RunnerOptions, files: string[]): void { + const reports = files + .map((file, index) => + coverageDirFor(file, index, { + coverage: options.coverage, + coverageDir: options.coverageDir, + coverageExempt: COVERAGE_EXEMPT_FILES, + }), + ) + .filter((directory): directory is string => directory !== null) + .map((directory) => `${directory}/lcov.info`) + // bun writes no report at all for a test file that covered no source file, so a + // missing one is expected here rather than an error. + // A coverage directory may be given as an absolute path, so resolve rather + // than join: path.join("/repo", "/tmp/raw") is "/repo/tmp/raw". + .filter((report) => existsSync(path.resolve(root, report))); + + if (reports.length === 0) { + throw new Error(`No coverage report was written under ${options.coverageDir}.`); + } + + // The list goes in a file rather than in argv: Windows caps a command line at + // 32767 characters and this repository already has over 500 test files. + const manifest = `${options.coverageDir}/inputs.txt`; + writeFileSync(path.resolve(root, manifest), `${reports.join("\n")}\n`); + + const merged = Bun.spawnSync( + ["node", "scripts/merge-lcov.mjs", `--inputs-from=${manifest}`, options.mergeInto as string], + { cwd: root, stdout: "inherit", stderr: "inherit" }, + ); + if (merged.exitCode !== 0) { + // A child that died by a signal reports exitCode null and signalCode instead + // (measured on bun 1.4.2), and a terminal Ctrl+C reaches this child too, because + // it goes to the whole foreground group. "failed with exit null" names no cause. + const how = merged.signalCode ? `was killed by ${merged.signalCode}` : `failed with exit ${merged.exitCode}`; + throw new Error(`Merging ${reports.length} coverage reports ${how}.`); + } +} + +async function main(): Promise { + const options = parseRunnerArgs(process.argv.slice(2), { cpuCount: availableParallelism() }); + const files = selectTestFiles(root, options.selectors); + + if (options.list) { + await writtenOrReaderGone(process.stdout, `${files.join("\n")}\n`); + return 0; + } + + // Decided before anything is deleted or started: a run that has to be refused (a CI job whose + // Helm is missing) refuses with the coverage directory and the merged report still intact. + const plan = planRequirements({ + files, + readSource: (file) => readFileSync(path.join(root, file), "utf8"), + missing: { helm: () => missingHelm(systemHelmProbe(root)) }, + required: requiredCapabilities(process.env), + }); + + if (options.coverage) { + const coverageDir = path.resolve(root, options.coverageDir); + // --coverage-dir is a path the caller chooses and this line deletes it, so it is + // checked before it is emptied. See tests/runner/coverage.ts. + if (existsSync(coverageDir)) assertCoverageDirIsOurs(options.coverageDir, readdirSync(coverageDir)); + rmSync(coverageDir, { recursive: true, force: true }); + mkdirSync(coverageDir, { recursive: true }); + } + // The merged report goes too, and before the run rather than after it: a run that + // ends red never reaches the merge, and a stale lcov left beside it is a report of + // a tree that no longer exists, which `coverage:check` would happily pass. + if (options.mergeInto) { + const mergeTarget = path.resolve(root, options.mergeInto); + if (existsSync(mergeTarget)) assertMergeTargetIsOurs(options.mergeInto, readFileSync(mergeTarget, "utf8")); + rmSync(mergeTarget, { force: true }); + } + + const junitDir = mkdtempSync(path.join(tmpdir(), "libredb-test-junit-")); + runScratch = junitDir; + + const selection = options.selectors.length > 0 ? options.selectors.join(" ") : "tests/"; + const notRunNote = + plan.notRun.length > 0 ? ` (${plan.notRun.length} not run on this machine, listed at the end)` : ""; + runStarted = true; + process.stdout.write( + `bun ${Bun.version} on ${process.platform}-${process.arch}: ${plan.run.length} files from ${selection}${notRunNote}, ` + + `${options.jobs} at a time${options.coverage ? ", with coverage" : ""}\n\n`, + ); + + const summary = await runTestFiles({ + files: plan.run, + jobs: options.jobs, + timeoutMs: options.fileTimeoutMs, + coverage: options.coverage, + coverageDir: options.coverageDir, + coverageExempt: COVERAGE_EXEMPT_FILES, + runFile: spawnTestFile(options.bunArgs, junitDir), + shouldStop: () => stopping, + onResult: (outcome, position, total) => { + // Belt and braces. The intent is that what a child has just been killed for is + // not this file's verdict, but by the time a killed child's outcome could arrive + // the signal handler has usually already called process.exit, so removing this + // guard leaves the whole suite green (measured by mutation). It is written from + // the shape of the code, for the window where the handler's own exit is still + // pending, and not from an observed behaviour. + if (stopping) return; + process.stdout.write(`${formatFileLine(outcome, position, total)}\n`); + // A failing file's whole output is printed where it lands rather than kept for + // the end: a CI log is read from the first red line downwards. + if (outcome.status !== "passed") process.stdout.write(`${outcome.output}\n`); + }, + }); + + // A run that was signalled prints no summary and carries no code of its own: the + // handler is already on its way out with 128 + the signal's number, and returning + // its promise waits for it rather than reporting the files it had killed as + // failures. Belt and braces like the guard above: the handler normally exits before + // this is reached, so removing it leaves the suite green (measured by mutation). + if (interruption !== null) return interruption; + + await writtenOrReaderGone(process.stdout, `${formatSummary(summary, plan.notRun)}\n`); + // Looked at again, because a megabyte of summary takes a while to drain and a signal + // taken while it did still owns the way out. Belt and braces once more: the window is + // between this write resolving and the handler's own exit, and nothing here can drive + // it to order, so no test pins this line. + if (interruption !== null) return interruption; + if (summary.failures.length > 0) return 1; + + if (options.mergeInto) { + mergeCoverage(options, plan.run); + // mergeCoverage is synchronous and merges 500+ reports, so a signal delivered + // during it is still queued when it returns: measured on bun 1.4.2, the listener + // has NOT run at that point, and without this turn of the event loop the + // process.exit(0) below wins and the user's Ctrl+C vanishes with no Interrupted + // line and a green exit. One setImmediate is enough (measured, 5 runs of 5), + // because libuv polls its signal handles before the check phase. + await new Promise((resolve) => setImmediate(resolve)); + if (interruption !== null) return interruption; + } + return 0; +} + +/** + * Ends the run on a signal, with 128 + the signal's number, which is what a shell + * reports: 130 for SIGINT, 143 for SIGTERM, 129 for SIGHUP (see tests/runner/signals.ts, + * which also carries the number for SIGBREAK, that no POSIX table has). + * + * The children are asked to stop and are not waited for: `--no-orphans` takes every + * child and its descendants down once this process has gone (measured), and on POSIX + * a directory is removed happily while a dying child still holds a file in it open. + * A removal that fails is named and exits 2 instead, never swallowed: a throw from + * inside a signal listener leaves bun running, and the queue then starts the next + * file (measured 1.4.2), which is worse than either. + * + * Nothing here waits on anything it does not control. The listeners come off first, so + * the default disposition is back and a second Ctrl+C really kills (measured on 1.4.2: + * removing every listener for a signal restores it, and the second SIGINT then ends the + * process with 130 by itself). The last line is then raced against a grace, because a + * reader that has stopped reading must not be able to keep a signalled run alive. + */ +function stopOnSignal(signal: StopSignal): Promise { + stopRun(); + for (const other of STOP_SIGNALS) process.removeAllListeners(other); + const removal = removeRunScratch(); + return exitAfterWriting( + removal === null ? exitCodeForSignal(signal) : 2, + { + stdout: `\nInterrupted (${signal}).\n`, + stderr: removal === null ? undefined : `${removal}\n`, + }, + SIGNAL_WRITE_GRACE_MS, + ); +} + +/** Set once a signal has been taken: the run has no verdict of its own after that. */ +let interruption: Promise | null = null; + +for (const signal of STOP_SIGNALS) { + // The first signal owns the way out; a second one arriving while it writes its last + // line must not start a second cleanup over the top of it. It is not swallowed + // either: stopOnSignal has already put the default disposition back, so the second + // one kills the process outright rather than reaching this listener at all. + process.on(signal, () => { + interruption ??= stopOnSignal(signal); + }); +} + +try { + const code = await main(); + const removal = removeRunScratch(); + // Usage and setup errors exit 2, so a caller can tell "the tests failed" (1) from + // "the runner could not run them" (2). A scratch directory left behind is the + // second kind, so it takes the run's own code away. + if (removal !== null) await exitAfterWriting(2, { stderr: `${removal}\n` }); + // The last look: a signal taken in the turn between main's own final check and this + // line still owns the way out, and this promise never resolves, so the handler's + // exit is what happens. Belt and braces, like main's own two checks. + if (interruption !== null) await interruption; + process.exit(code); +} catch (error) { + // The children are stopped before the message is written: the write is awaited, and + // while it is, a worker would otherwise start the next file over a run that is over. + stopRun(); + const removal = removeRunScratch(); + const reason = error instanceof Error ? error.message : String(error); + await exitAfterWriting(2, { + // Whoever is reading the run is reading stdout, and whatever landed there before + // the error still has to reach them, which needs a write of its own. + stdout: runStarted ? "\nThe run stopped before it finished; the reason is on stderr.\n" : undefined, + stderr: `${[reason, removal].filter((line) => line !== null).join("\n")}\n`, + }); +} diff --git a/tests/runner/capture.ts b/tests/runner/capture.ts new file mode 100644 index 000000000..4aae50e11 --- /dev/null +++ b/tests/runner/capture.ts @@ -0,0 +1,102 @@ +/** + * Reading a child's stdout or stderr without letting one file's output size the run. + * + * The output is captured rather than inherited because several files run at once, so + * it has to be held in memory until the file lands. Held with no bound, one runaway + * test file sizes the runner: measured on bun 1.4.2, a file printing 300 MB took the + * runner to 577 MB of RSS, and the whole run's output added up because every outcome + * kept its own. + * + * So each stream keeps its first CAPTURE_HEAD_BYTES and its last CAPTURE_TAIL_BYTES, + * and says in one line how many bytes fell between them. Both ends are kept because + * both are read: the head has bun's file header and the first failure diff, and the + * tail has the rest of the diffs and bun's own per-file summary. + * + * The limit is a megabyte at each end, measured against this repository rather than + * guessed: over the 543 test files the tree held when this was measured, the largest + * printed 154,526 bytes on stdout and the largest stderr was 48,369 bytes, with a + * median of 104 bytes. A megabyte is about 7x + * the largest stdout and about 20x the largest stderr, so nothing that runs here + * today is ever cut, and the worst case per running child is 4 MB. + * + * Nothing is decided from this text (the counts and the skips come from the child's + * junit report, see tests/runner/report.ts), so a cut can never change a verdict. + */ + +/** Bytes kept from the start of each stream. */ +export const CAPTURE_HEAD_BYTES = 1024 * 1024; + +/** Bytes kept from the end of each stream. */ +export const CAPTURE_TAIL_BYTES = 1024 * 1024; + +export type CaptureLimits = { + /** The stream's name, for the line that says what was left out. */ + name: "stdout" | "stderr"; + headBytes?: number; + tailBytes?: number; +}; + +function join(chunks: Uint8Array[], length: number): Uint8Array { + const joined = new Uint8Array(length); + let at = 0; + for (const chunk of chunks) { + joined.set(chunk, at); + at += chunk.length; + } + return joined; +} + +export async function captureBounded( + stream: ReadableStream, + { name, headBytes = CAPTURE_HEAD_BYTES, tailBytes = CAPTURE_TAIL_BYTES }: CaptureLimits, +): Promise { + const head: Uint8Array[] = []; + let headLength = 0; + const tail: Uint8Array[] = []; + let tailLength = 0; + let elidedBytes = 0; + + const reader = stream.getReader(); + for (;;) { + // oxlint-disable-next-line no-await-in-loop -- a stream is read chunk by chunk, in order. + const { done, value } = await reader.read(); + if (done) break; + let rest = value; + + if (headLength < headBytes) { + const take = Math.min(headBytes - headLength, rest.length); + head.push(rest.subarray(0, take)); + headLength += take; + rest = rest.subarray(take); + } + if (rest.length === 0) continue; + + tail.push(rest); + tailLength += rest.length; + while (tailLength > tailBytes) { + const oldest = tail[0] as Uint8Array; + const excess = tailLength - tailBytes; + if (oldest.length <= excess) { + tail.shift(); + elidedBytes += oldest.length; + tailLength -= oldest.length; + continue; + } + // slice rather than subarray: a copy lets the chunk that held the dropped bytes go. + tail[0] = oldest.slice(excess); + elidedBytes += excess; + tailLength -= excess; + } + } + + const decoder = new TextDecoder(); + if (elidedBytes === 0) return decoder.decode(join([...head, ...tail], headLength + tailLength)); + // The two ends are decoded separately, so a multi-byte character or an escape + // sequence that straddles a cut decodes as a replacement character. That is the + // honest reading of text with a hole in it, and the hole is stated where it is. + return [ + decoder.decode(join(head, headLength)), + `\n[runner: ${elidedBytes} bytes of ${name} elided here]\n`, + decoder.decode(join(tail, tailLength)), + ].join(""); +} diff --git a/tests/runner/coverage.ts b/tests/runner/coverage.ts new file mode 100644 index 000000000..01b02d2a3 --- /dev/null +++ b/tests/runner/coverage.ts @@ -0,0 +1,57 @@ +/** + * The coverage directory, which the runner empties before every coverage run. + * + * Emptying it is necessary: a stale report from a previous run would be merged into + * this one, and the merge cannot tell the two apart. But `--coverage-dir` is a path + * the caller chooses, and `rmSync(dir, { recursive: true })` on a mistyped one is + * not a mistake anybody recovers from: `--coverage-dir=src` would delete the + * product. Every other option in this runner validates what it is given; this is + * that validation. + * + * The rule is ownership, not a name: a directory the runner may empty either does + * not exist yet, is empty, or holds nothing but what a previous coverage run of this + * runner put there. + */ + +/** What a coverage run leaves behind: one directory per test file, plus the merge's own inputs. */ +const OWNED = /^(file-\d+|lcov\.info|inputs\.txt)$/; + +export function unownedCoverageEntries(entries: string[]): string[] { + return entries.filter((entry) => !OWNED.test(entry)).sort(); +} + +/** + * Raises when the directory holds anything this runner did not write, naming what it + * found. The caller passes the entries rather than a path so the rule is testable + * without building a directory that proves the point by being deleted. + */ +export function assertCoverageDirIsOurs(directory: string, entries: string[]): void { + const unowned = unownedCoverageEntries(entries); + if (unowned.length === 0) return; + + throw new Error( + `Refusing to empty ${directory}: it holds ${unowned.length} entr${unowned.length === 1 ? "y" : "ies"} ` + + `this runner did not write (${unowned.slice(0, 5).join(", ")}). ` + + "Point --coverage-dir at a directory that is empty or holds only a previous coverage run.", + ); +} + +/** + * Raises unless an existing `--merge-into` target is a coverage report. + * + * The runner removes that file before a coverage run, so that a run which ends red + * cannot leave a stale report for `coverage:check` to pass. Removing is only safe + * for a file this runner or `scripts/merge-lcov.mjs` wrote: `--merge-into=package.json` + * would otherwise delete package.json. An lcov report starts with `TN:` (as bun writes + * it) or `SF:` (as the merge writes it), and an empty file is a merge that found no + * records, so those three are ours and anything else is somebody's work. + */ +export function assertMergeTargetIsOurs(target: string, content: string): void { + const firstLine = content.split("\n", 1)[0] ?? ""; + if (content.trim() === "" || /^(TN|SF):/.test(firstLine)) return; + + throw new Error( + `Refusing to replace ${target}: it is not a coverage report (it starts ${JSON.stringify(firstLine.slice(0, 40))}). ` + + "Point --merge-into at an lcov file or at a path that does not exist yet.", + ); +} diff --git a/tests/runner/discover.ts b/tests/runner/discover.ts new file mode 100644 index 000000000..8034ac398 --- /dev/null +++ b/tests/runner/discover.ts @@ -0,0 +1,139 @@ +/** + * Which files the test runner runs, and which of them are measured for coverage. + * + * One rule, in one place: every `*.test.ts` / `*.test.tsx` file under `tests/`, + * except `tests/live/`, whose files drive real database engines and are started by + * hand (`bun tests/live/.ts`). The rule is deliberately a rule and not a + * hand-written list: `tests/components/WireCompatibilityHint.test.tsx` shipped with + * #426 and never ran once, because the runner of the day named its files one by one. + * + * A symbolic link or junction under `tests/` is refused by name rather than followed: + * following one can run files from outside the repository, loop on a link to a parent + * and list one file twice under two names, while skipping it drops its tests silently. + * + * `scripts/security-check.mjs` asks this module (through `bun tests/run-tests.ts + * --list`) whether a test named by `docs/SECURITY.md` is actually executed, so the + * discovery rule is also the repository's definition of "this test runs". + */ +import { existsSync, lstatSync, readdirSync, realpathSync } from "node:fs"; +import path from "node:path"; + +const TESTS_DIRECTORY = "tests"; +const TEST_FILE = /\.test\.tsx?$/; + +/** Directories under `tests/` that the runner never collects, with the reason. */ +const EXCLUDED = new Map([["live", "drives real engines, started by hand"]]); + +/** + * Files that run WITHOUT coverage collection. + * + * Both import a whole module chain without exercising it: the CJS shim pulls in + * every component, and the loader wiring file imports the editor to observe a call + * it makes at module scope. bun's lcov is per-function, so a process that only + * LOADS a module emits a coarse zero-hit block for it, and `scripts/merge-lcov.mjs` + * picks the record with the most executed lines as the authority for which lines + * are coverable. When one of these two processes is the only one that ever loaded a + * file, its coarse block becomes that authority and its zero lines are reported as + * uncovered: measured 2026-09-15, `src/lib/llm/factory.ts` gains 31 phantom + * uncovered lines from the shim alone. Today the core layer happens to supply a + * better record for each of them, so the merged gate still reaches 100%; this list + * is what makes that a property rather than a coincidence. + * + * `sonar-project.properties` states the same exemption for `src/exports/index.js` + * from the other side. + */ +export const COVERAGE_EXEMPT_FILES: readonly string[] = [ + "tests/isolated/exports-shim.test.ts", + "tests/isolated/monaco-loader-wiring.test.ts", +]; + +/** + * Every entry is classified with lstat rather than by its Dirent type. A Dirent for + * a link is neither a directory nor a file (measured on bun 1.4.2), which is how a + * linked test used to be skipped without a word, and what bun's Dirent reports for a + * Windows junction could not be measured; lstat reports a junction as a symbolic + * link, as it does on POSIX. The extra lstat per entry is cheap: measured on Linux + * over the 543 files the tree held then, `discoverTestFiles` went from 0.43 ms to + * 1.1 ms a call. + */ +function collect(root: string, directory: string): string[] { + return readdirSync(path.join(root, directory)).flatMap((name) => { + const child = `${directory}/${name}`; + const stats = lstatSync(path.join(root, child)); + if (stats.isSymbolicLink()) { + throw new Error( + `${child} is a symbolic link or junction: the test runner does not follow links, so a test behind one would never run. Replace it with the real file or directory.`, + ); + } + if (stats.isDirectory()) { + return directory === TESTS_DIRECTORY && EXCLUDED.has(name) ? [] : collect(root, child); + } + return stats.isFile() && TEST_FILE.test(name) ? [child] : []; + }); +} + +/** Every test file the runner runs, as repository-relative POSIX paths, sorted. */ +export function discoverTestFiles(root: string): string[] { + return collect(root, TESTS_DIRECTORY).sort(); +} + +/** + * A selector as the user typed it, reduced to a repository-relative POSIX path. + * + * A relative selector is resolved against the working directory, not against the + * repository root, because that is what the person typing it meant: from + * `tests/unit`, `bun ../run-tests.ts lib/lazy.test.ts` names the file beside them. + * Resolving against the root instead answered "is not under tests/", which is a true + * sentence about a path they never wrote. + */ +function normalizeSelector(root: string, selector: string): string { + // Both sides in real-path space, because one directory can have two spellings and + // path.relative compares spellings. Measured on windows-latest: os.tmpdir() is the + // 8.3 short form (C:\Users\RUNNER~1\...) and import.meta.dir the long one, so a + // runner started from a temp directory called a correct selector "not under + // tests/". A junction or a symlinked checkout does the same anywhere. The two + // realpaths that carry it are the root's and the resolved selector's: the resolved + // path is realpathed when it exists, which covers a link inside a relative selector + // (`u/x.test.ts` with u -> tests/unit) and, on Windows, a wrong-case or 8.3 segment. + // The working directory itself is not realpathed again: measured on bun 1.4.2, after + // chdir into a link `process.cwd()` already answers the real path, so the call could + // only ever have changed which message a selector that does not EXIST is refused + // with, and an unfalsifiable line is worse than the message it might improve. + // The realpath is of the resolved path, never of the raw selector: bun's existsSync + // follows a link before a "..", its realpath collapses the ".." as text first, so + // for `u/../unit/x.test.ts` the two disagree and realpath threw a raw ENOENT + // (measured on 1.4.2). A selector that does not exist keeps its spelling, and + // matches no file either way. + const resolved = path.resolve(process.cwd(), selector); + const absolute = existsSync(resolved) ? realpathSync.native(resolved) : resolved; + return path.relative(realpathSync.native(root), absolute).split(path.sep).join("/").replace(/\/+$/, ""); +} + +/** + * The files named by the command line, or all of them when nothing is named. + * + * A selector that matches nothing raises: an empty run that exits 0 is the one + * outcome a test runner must never produce. + */ +export function selectTestFiles(root: string, selectors: string[]): string[] { + const all = discoverTestFiles(root); + if (selectors.length === 0) return all; + + const selected = new Set(); + for (const selector of selectors) { + const target = normalizeSelector(root, selector); + if (target !== TESTS_DIRECTORY && !target.startsWith(`${TESTS_DIRECTORY}/`)) { + throw new Error(`"${selector}" is not under tests/: name a test file or a directory under tests/.`); + } + + const matches = all.filter((file) => file === target || file.startsWith(`${target}/`)); + if (matches.length === 0) { + throw new Error( + `"${selector}" matched no test files. Run "bun tests/run-tests.ts --list" to see what the runner runs.`, + ); + } + for (const file of matches) selected.add(file); + } + + return [...selected].sort(); +} diff --git a/tests/runner/execute.ts b/tests/runner/execute.ts new file mode 100644 index 000000000..de6a291f1 --- /dev/null +++ b/tests/runner/execute.ts @@ -0,0 +1,216 @@ +/** + * Running the files: one bun process per test file, several at a time. + * + * One process per file is not a performance choice, it is the isolation the suite + * needs. bun's `mock.module()` is process-wide with no undo, and whole-module mocks + * are the standard pattern in `tests/api/` (29 of its files mock `@/lib/auth`, and 36 + * files do across the whole suite), so + * any file that needs the real module fails when it shares a process with one that + * mocked it. bun 1.4.2 has `--isolate`, which resets the module registry per file + * in ONE process, and it does contain `mock.module`, but it is also the subject of + * oven-sh/bun#41655 (a NAPI finalizer SIGSEGV that reproduces serially on 1.4.2) + * and this suite loads three NAPI addons: `better-sqlite3`, `oracledb` and + * `@duckdb/node-api`. A process boundary needs no upstream fix, so that is what + * this uses, and the concurrency is what pays for it. + * + * Spawning is injected so this module can be tested without processes. + */ +import { parseSkippedTests, readTestReport } from "./report"; + +export type TestCounts = { pass: number; fail: number; skip: number; todo: number }; + +export type SpawnOutcome = { + exitCode: number | null; + signal: string | null; + /** Everything the child printed, for a reader. Nothing is decided from it. */ + output: string; + durationMs: number; + timedOut: boolean; + /** The junit report the child wrote, or null when it wrote none. */ + junitReport: string | null; +}; + +export type FileOutcome = { + file: string; + status: "passed" | "failed" | "timed-out"; + exitCode: number | null; + signal: string | null; + durationMs: number; + output: string; + counts: TestCounts | null; + /** Whether the file's junit report was read, missing, or there but unreadable. */ + report: "read" | "missing" | "unreadable"; + skippedTests: string[]; +}; + +export type RunSummary = { + outcomes: FileOutcome[]; + failures: FileOutcome[]; + durationMs: number; + jobs: number; + /** The per-file budget a timed-out file ran into. */ + timeoutMs: number; + totals: { + files: number; + filesPassed: number; + filesFailed: number; + filesTimedOut: number; + filesWithoutCounts: number; + tests: TestCounts; + }; +}; + +export type RunFile = (input: { + file: string; + index: number; + coverageDir: string | null; + timeoutMs: number; +}) => Promise; + +export type RunTestFilesInput = { + files: string[]; + jobs: number; + timeoutMs: number; + coverage: boolean; + coverageDir: string; + coverageExempt: readonly string[]; + runFile: RunFile; + /** + * Asked before a worker picks up a file: true stops the run where it is. + * + * It is what a stop signal needs, and it is here rather than in the caller because + * only this loop knows when the next file would start. The files already running + * are not this gate's business; the signal handler kills those children itself. + */ + shouldStop?: () => boolean; + onResult?: (outcome: FileOutcome, position: number, total: number) => void; + now?: () => number; +}; + +function toOutcome(file: string, spawned: SpawnOutcome): FileOutcome { + const { state, counts } = readTestReport(spawned.junitReport); + // A signal death arrives as exitCode null. `exitCode === 0` is correctly false for + // it, but any reading that coerces (`exitCode || 0`, `!exitCode`) turns a SIGSEGV + // into a pass, so the status is derived once, here. + // + // Three more shapes are failures although the child exited 0: + // + // - No report, or one that cannot be read. bun writes the report when it reaches + // the end of a file, so its absence means the process left early: a test calling + // `process.exit(0)` does it (measured 1.4.2: exit 0, no report, and the tests + // after it never run), and so would a native addon calling exit(). Reporting + // that as a pass is how this runner would turn a red tree green, for every shape + // the report can see; the one it cannot see is `.only`, which bun honours, so a + // file with a committed `it.only` writes a report naming that test alone and + // exits 0 (measured 1.4.2: four other registered tests, one of them failing, + // absent from the report, from the totals and from the verdict). Nothing here can + // tell that report from a file that really holds one test, so a committed `.only` + // has to be refused before the run, not read out of what the run wrote. + // - A report saying zero of everything. The runner's own rule is that a + // discovered file runs, so a file that registered nothing is either a + // registration that silently stopped happening or a file that should not exist. + // - A report that records a failure. The counts are what the verdict is read from, + // so a green exit code does not overrule them: otherwise the summary would print + // that failure in its own totals under a passing file line. + const registeredNothing = counts !== null && counts.pass + counts.fail + counts.skip + counts.todo === 0; + // `timedOut` is the runner's own flag, set when it fired the kill. A child that + // finished cleanly in the same millisecond still exited 0, and it did not time out. + const killedByTimeout = spawned.timedOut && spawned.exitCode !== 0; + const status = killedByTimeout + ? "timed-out" + : spawned.exitCode === 0 && counts !== null && !registeredNothing && counts.fail === 0 + ? "passed" + : "failed"; + + return { + file, + status, + exitCode: spawned.exitCode, + signal: spawned.signal, + durationMs: spawned.durationMs, + output: spawned.output, + counts, + report: state, + // Read even from a report the counts could not be taken from: what it did reach + // still names tests, and this is the only place a skip's reason is ever printed. + // The summary prints those titles under an unknown count rather than dropping them + // (see formatSummary), so this parse is read by someone in that case too. + skippedTests: spawned.junitReport === null ? [] : parseSkippedTests(spawned.junitReport), + }; +} + +/** + * The per-file coverage directory, or null for a file that must not be measured. + * + * The index is the file's position in the sorted selection, which is what makes the + * directory names stable and collision-free for `scripts/merge-lcov.mjs` to read + * back. + */ +export function coverageDirFor( + file: string, + index: number, + { coverage, coverageDir, coverageExempt }: Pick, +): string | null { + if (!coverage || coverageExempt.includes(file)) return null; + return `${coverageDir}/file-${index + 1}`; +} + +export async function runTestFiles(input: RunTestFilesInput): Promise { + const { files, jobs, timeoutMs, runFile, onResult, shouldStop, now = () => Date.now() } = input; + if (files.length === 0) { + throw new Error("The runner was handed no test files, so there is nothing to report as passing."); + } + + const startedAt = now(); + const outcomes: FileOutcome[] = []; + let next = 0; + let finished = 0; + + async function worker(): Promise { + while (next < files.length) { + if (shouldStop?.()) return; + const index = next; + next += 1; + const file = files[index] as string; + // oxlint-disable-next-line no-await-in-loop -- one file at a time per worker; the jobs come from the workers. + const spawned = await runFile({ file, index, coverageDir: coverageDirFor(file, index, input), timeoutMs }); + const outcome = toOutcome(file, spawned); + finished += 1; + onResult?.(outcome, finished, files.length); + // Every outcome is kept until the run ends, and nothing reads a passing file's + // output after it has been reported, so it is dropped here rather than carried: + // otherwise the runner's memory grows with the whole run's output, passing + // files included (measured on 1.4.2: four passing files printing 100 MB each + // peaked at 406 MB, against 249 MB for one). + outcomes.push(outcome.status === "passed" ? { ...outcome, output: "" } : outcome); + } + } + + await Promise.all(Array.from({ length: Math.min(jobs, files.length) }, () => worker())); + + const totals = { + files: files.length, + filesPassed: outcomes.filter((outcome) => outcome.status === "passed").length, + filesFailed: outcomes.filter((outcome) => outcome.status === "failed").length, + filesTimedOut: outcomes.filter((outcome) => outcome.status === "timed-out").length, + filesWithoutCounts: outcomes.filter((outcome) => outcome.counts === null).length, + tests: outcomes.reduce( + (sum, outcome) => ({ + pass: sum.pass + (outcome.counts?.pass ?? 0), + fail: sum.fail + (outcome.counts?.fail ?? 0), + skip: sum.skip + (outcome.counts?.skip ?? 0), + todo: sum.todo + (outcome.counts?.todo ?? 0), + }), + { pass: 0, fail: 0, skip: 0, todo: 0 }, + ), + }; + + // Failures in the order the files were selected, not the order they happened to + // finish in, so two runs of the same red tree print the same list. + const order = new Map(files.map((file, index) => [file, index])); + const failures = outcomes + .filter((outcome) => outcome.status !== "passed") + .sort((a, b) => (order.get(a.file) ?? 0) - (order.get(b.file) ?? 0)); + + return { outcomes, failures, durationMs: now() - startedAt, jobs, timeoutMs, totals }; +} diff --git a/tests/runner/options.ts b/tests/runner/options.ts new file mode 100644 index 000000000..8b412aaaa --- /dev/null +++ b/tests/runner/options.ts @@ -0,0 +1,120 @@ +/** + * The test runner's command line. + * + * Every option is `--name=value`; a value written as a separate argument is + * refused with the form that works, rather than being read as a selector and + * silently running the wrong thing. An unknown option is refused by name for the + * same reason: a forwarded bun flag goes after `--`, where it is visible. + * + * That refusal also names the trap a contributor is most likely to have hit, because + * the obvious command does not work: measured on bun 1.4.2, `bun run test -- --bail` + * reaches this parser as `["--bail"]`, since `bun run` consumes the first `--` itself, + * and so does bun when the `--` sits straight after the script path. Only + * `bun tests/run-tests.ts -- ` arrives whole. + */ + +export type RunnerOptions = { + /** Paths naming what to run; empty means every discovered test file. */ + selectors: string[]; + /** How many test files run at once, each in its own bun process. */ + jobs: number; + /** Collect an lcov report per file. */ + coverage: boolean; + /** Where the per-file reports go, one subdirectory per file. */ + coverageDir: string; + /** When set, merge the per-file reports into this path after the run. */ + mergeInto: string | null; + /** How long one file may take before the runner kills it and fails it. */ + fileTimeoutMs: number; + /** Print what would run and exit. */ + list: boolean; + /** Arguments after `--`, handed to every `bun test` child verbatim. */ + bunArgs: string[]; +}; + +export const DEFAULT_COVERAGE_DIR = "coverage/raw"; + +/** + * Five minutes. No single test file comes near it (the slowest in this repository + * is about 25 seconds), so it only ever fires on a hang - which it then reports as + * that file's failure, with its output, instead of letting a CI job sit until the + * job timeout kills the whole run with nothing to read. + */ +export const DEFAULT_FILE_TIMEOUT_MS = 300_000; + +function positiveInteger(name: string, raw: string): number { + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) { + throw new Error(`${name} needs a whole number of 1 or more, got "${raw}".`); + } + return value; +} + +export function parseRunnerArgs(argv: string[], { cpuCount }: { cpuCount: number }): RunnerOptions { + const options: RunnerOptions = { + selectors: [], + jobs: Math.max(1, cpuCount), + coverage: false, + coverageDir: DEFAULT_COVERAGE_DIR, + mergeInto: null, + fileTimeoutMs: DEFAULT_FILE_TIMEOUT_MS, + list: false, + bunArgs: [], + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] as string; + + if (arg === "--") { + options.bunArgs = argv.slice(index + 1); + break; + } + + if (!arg.startsWith("-")) { + options.selectors.push(arg); + continue; + } + + const separator = arg.indexOf("="); + const name = separator === -1 ? arg : arg.slice(0, separator); + const value = separator === -1 ? null : arg.slice(separator + 1); + + switch (name) { + case "--coverage": + options.coverage = true; + break; + case "--list": + options.list = true; + break; + case "--jobs": + if (value === null) + throw new Error(`${name} takes its value with an equals sign, as ${name}=${argv[index + 1] ?? "N"}.`); + options.jobs = positiveInteger(name, value); + break; + case "--file-timeout": + if (value === null) + throw new Error(`${name} takes its value with an equals sign, as ${name}=${argv[index + 1] ?? "SECONDS"}.`); + options.fileTimeoutMs = positiveInteger(name, value) * 1000; + break; + case "--coverage-dir": + if (value === null) + throw new Error(`${name} takes its value with an equals sign, as ${name}=${argv[index + 1] ?? "DIR"}.`); + options.coverageDir = value; + break; + case "--merge-into": + if (value === null) + throw new Error(`${name} takes its value with an equals sign, as ${name}=${argv[index + 1] ?? "FILE"}.`); + options.mergeInto = value; + options.coverage = true; + break; + default: + throw new Error( + `Unknown option "${name}". The runner's own options are --jobs, --coverage, --coverage-dir, --merge-into, --file-timeout and --list; everything for bun test goes after --. ` + + "If you did write one: `bun run` removes the first --, and so does bun when it sits straight after the script path, " + + "so the form that arrives whole is `bun tests/run-tests.ts -- `.", + ); + } + } + + return options; +} diff --git a/tests/runner/report.ts b/tests/runner/report.ts new file mode 100644 index 000000000..e73d537f9 --- /dev/null +++ b/tests/runner/report.ts @@ -0,0 +1,260 @@ +/** + * What the runner prints. + * + * The children's output is captured rather than inherited, because several files + * run at once and interleaved output belongs to nobody. Each file therefore gets + * one line when it lands, a failing file gets its whole output printed with it, and + * the run ends with the population: how many files, how many tests, how many + * skipped, and how to re-run any file that failed on its own. + */ +import type { FileOutcome, RunSummary, TestCounts } from "./execute"; +import type { NotRunFile } from "./requirements"; + +/** + * What a file's junit report said, or why it said nothing. + * + * "missing" and "unreadable" are told apart because they mean different things to + * whoever reads the failure: a missing report is a child that never got to the end + * (it registered no test, or a `process.exit` or a signal cut it short), while an + * unreadable one is a report this parser does not understand, which is a defect in + * the runner or a bun that changed its format. Neither ever becomes zero counts. + */ +export type TestReport = { state: "read"; counts: TestCounts } | { state: "missing" | "unreadable"; counts: null }; + +/** + * The counts of one file, read from the junit report bun wrote for it. + * + * NOT from the console output, although bun prints " 13 pass" / " 1 fail" lines + * there. Those lines are free-form text mixed with whatever the tests themselves + * printed, and a test that prints one is indistinguishable from bun printing it: + * measured on 1.4.2, a file that registered no test and printed " 1 pass" was + * reported PASS with exit 0, and so was a test that printed a whole summary on + * stderr and then called `process.exit(0)` so the tests after it never ran. Both + * shapes defeat exactly the guards that keep this runner from turning a red tree + * green. `--bail` makes the console reading wrong in the other direction: it prints + * no count line at all, while the report still carries the failure. + * + * bun counts a todo test among the skipped ones and writes it as + * ``, so the todos are counted from the elements and + * taken out of the skips. parseSkippedTests leaves them out of its titles for the + * same reason, so the count and the list below it are the same tests. + */ +export function readTestReport(report: string | null): TestReport { + if (report === null) return { state: "missing", counts: null }; + const unreadable: TestReport = { state: "unreadable", counts: null }; + const root = /]*)>/.exec(report); + // The closing tag matters: the todo count comes from the elements, so a report cut + // off half way (a child killed mid-write) would undercount rather than be unknown. + if (root === null || !report.includes("")) return unreadable; + + const attributes = root[1] as string; + const count = (name: string): number | null => { + const raw = new RegExp(`\\b${name}="(\\d+)"`).exec(attributes)?.[1]; + return raw === undefined ? null : Number(raw); + }; + const tests = count("tests"); + const failures = count("failures"); + const skipped = count("skipped"); + if (tests === null || failures === null || skipped === null) return unreadable; + + const todo = [...report.matchAll(/]*\bmessage="TODO"/g)].length; + const pass = tests - failures - skipped; + const skip = skipped - todo; + // Counts that contradict each other are not counts: reporting them would put a + // negative number in the run's totals and call it measured. + if (pass < 0 || skip < 0) return unreadable; + return { state: "read", counts: { pass, fail: failures, skip, todo } }; +} + +const XML_ENTITY: Record = { + "&": "&", + "<": "<", + ">": ">", + """: '"', + "'": "'", +}; + +/** + * The titles of the tests a file skipped, read from bun's own junit report. + * + * bun prints a skipped test's title NOWHERE: measured on 1.4.2 piped, with + * FORCE_COLOR set, and under a real pty, the output carries the count (" 4 skip") + * and nothing else. In this repository a skip always states its reason in its title + * (a deb postinstall, a snap launcher, an AppImage permission audit: artifacts that + * cannot exist on the platform), so the count alone hides the only thing worth + * reading. The junit reporter names them, so the runner asks each child for one. + * + * The describe path matters as much as the name: a skip made with `describe.skip` + * carries its reason in the DESCRIBE title, and the tests inside it are named only + * for what they check. That path is read from the nested elements, one + * per describe, and NOT from the `classname` attribute, which also lists them: in + * classname bun joins the titles with " > " and writes a literal ">" inside a + * title as ">" too, so the two cannot be told apart. Measured on 1.4.2, splitting + * classname turned a describe titled "rows where count > 100" into + * "100 > rows where count". The element nesting says it unambiguously. + * + * A todo is not here. bun writes one as ``, and readTestReport + * takes the todos out of the skip count, so listing them would put titles under a + * header that does not count them. A todo also has no reason to state: it is work + * nobody has written, which the "N todo" count says in full. + * + * A report that is missing or truncated (a child killed mid-write) names what it + * reached, and raises nothing: the run's counts and verdict are read separately, by + * readTestReport, which refuses a report it cannot read rather than guessing. Those + * titles are printed under an unknown count (see formatSummary), which is why they are + * worth reading out of a report nothing else could use. + */ +export function parseSkippedTests(report: string): string[] { + const decode = (text: string) => text.replace(/&(amp|lt|gt|quot|apos);/g, (entity) => XML_ENTITY[entity] as string); + const skipped: string[] = []; + // The outermost suite is the file itself, so the stack is read from its second + // element on. ``. + const suites: string[] = []; + for (const match of report.matchAll(/]*)>|<\/testsuite>|]*)>\s*]*)>/g)) { + const [element, suiteAttributes, caseAttributes, skippedAttributes] = match; + if (element === "") { + suites.pop(); + continue; + } + if (suiteAttributes !== undefined) { + // A self-closing `` opens nothing: it has no `` to + // close it, so treating it as a describe would put its name in front of every + // title after it. bun 1.4.2 writes no element at all for an empty describe, so + // this is a shape of the junit format that the parser tolerates, not one measured. + if (!suiteAttributes.endsWith("/")) suites.push(decode(/\bname="([^"]*)"/.exec(suiteAttributes)?.[1] ?? "")); + continue; + } + // A todo is ``, the same element with a message. It is + // counted separately by readTestReport and it has no reason to state, so it is left + // out here too: a list that names it under a "(N skipped)" header states one number + // and shows another. + if ((skippedAttributes as string).includes('message="TODO"')) continue; + const name = /\bname="([^"]*)"/.exec(caseAttributes as string)?.[1]; + if (name === undefined) continue; + skipped.push([...suites.slice(1), decode(name)].join(" > ")); + } + return skipped; +} + +function seconds(durationMs: number): string { + return `${(durationMs / 1000).toFixed(1)}s`; +} + +function countsSuffix(outcome: FileOutcome): string { + const counts = outcome.counts; + if (!counts) return outcome.report === "missing" ? "no test report" : "unreadable test report"; + const parts = [`${counts.pass} pass`]; + if (counts.fail > 0) parts.push(`${counts.fail} fail`); + if (counts.skip > 0) parts.push(`${counts.skip} skip`); + if (counts.todo > 0) parts.push(`${counts.todo} todo`); + return parts.join(" "); +} + +const STATUS_LABEL = { passed: "PASS", failed: "FAIL", "timed-out": "TIMEOUT" } as const; + +export function formatFileLine(outcome: FileOutcome, position: number, total: number): string { + const width = String(total).length; + const place = `[${String(position).padStart(width)}/${total}]`; + const label = STATUS_LABEL[outcome.status].padEnd(7); + return `${place} ${label} ${seconds(outcome.durationMs).padStart(6)} ${outcome.file} ${countsSuffix(outcome)}`; +} + +function failureReason(outcome: FileOutcome, timeoutMs: number): string { + // The BUDGET, not the elapsed time: a killed child is given a few more seconds to + // die before SIGKILL, so the elapsed time is always the larger, unrelated number. + if (outcome.status === "timed-out") return `timed out, the budget is ${seconds(timeoutMs)} per file`; + // The runner sends SIGKILL itself only to a child that outran its budget, and that + // child is reported above as timed out. So a SIGKILL here came from outside, and on + // Linux that is nearly always the OOM killer: measured, a real kernel OOM kill of + // one child read only "killed by SIGKILL", which tells a reader nothing to act on. + if (outcome.signal === "SIGKILL") + return "killed by SIGKILL from outside the runner (its own timeout kill is reported as a timeout); on Linux that is usually the OOM killer, so re-run with a lower --jobs=N"; + if (outcome.signal) return `killed by ${outcome.signal}`; + if (outcome.counts && outcome.counts.fail > 0) return `${outcome.counts.fail} failing`; + if (outcome.report === "missing") + return `exit ${outcome.exitCode}, and it wrote no test report, which usually means it registered no test or stopped before bun finished, so its tests are unaccounted for`; + if (outcome.report === "unreadable") + return `exit ${outcome.exitCode}, and its test report could not be read, so its tests are unaccounted for`; + if ( + outcome.counts !== null && + outcome.counts.pass + outcome.counts.fail + outcome.counts.skip + outcome.counts.todo === 0 + ) + return `exit ${outcome.exitCode}, and it registered no test`; + return `exit ${outcome.exitCode}`; +} + +export function formatSummary(summary: RunSummary, notRun: NotRunFile[] = []): string { + const { totals } = summary; + const tests = [`${totals.tests.pass} pass`]; + if (totals.tests.fail > 0) tests.push(`${totals.tests.fail} fail`); + if (totals.tests.skip > 0) tests.push(`${totals.tests.skip} skip`); + if (totals.tests.todo > 0) tests.push(`${totals.tests.todo} todo`); + + const files = [`${totals.filesPassed} passed`]; + if (totals.filesFailed > 0) files.push(`${totals.filesFailed} failed`); + if (totals.filesTimedOut > 0) files.push(`${totals.filesTimedOut} timed out`); + + const plural = (count: number, noun: string) => `${count} ${noun}${count === 1 ? "" : "s"}`; + const totalTests = totals.tests.pass + totals.tests.fail + totals.tests.skip + totals.tests.todo; + const lines = [ + "", + "=".repeat(72), + `${plural(totals.files, "file")}: ${files.join(", ")} | ${plural(totalTests, "test")}: ${tests.join(", ")} | ${seconds(summary.durationMs)} with ${plural(summary.jobs, "job")}`, + ]; + + if (totals.filesWithoutCounts > 0) { + lines.push( + `${plural(totals.filesWithoutCounts, "file")} left no readable test report, so ${totals.filesWithoutCounts === 1 ? "its" : "their"} tests are not in the totals above.`, + ); + } + + // A file that needs something this machine does not have was never started, so it is in none + // of the totals above. It is named here under its reason, printed once: on a machine without + // Helm that is twelve chart test files and one sentence (see tests/runner/requirements.ts). + if (notRun.length > 0) { + lines.push("", "Files not run on this machine:"); + const byReason = new Map(); + for (const { file, reason } of notRun) byReason.set(reason, [...(byReason.get(reason) ?? []), file]); + for (const [reason, files] of byReason) { + lines.push(` ${reason}`); + for (const file of files.sort()) lines.push(` ${file}`); + } + } + + // A skipped test is not a passing test, and bun prints its title nowhere (see + // parseSkippedTests), so this is where a reader meets it. Each such title states + // the reason: a platform that cannot host the artifact, a tool that is not + // installed. A run that says only "3 skip" has told nobody anything. + // + // Selected by what there is to say, which is either of two things: a skip count, or + // titles. A truncated report has titles and no count, and selecting on the count + // alone dropped exactly those files, parsed and then thrown away; a report this + // parser read but could not name titles in has the count and no titles, and saying + // "4 skipped" without them is still more than saying nothing. + const skipping = summary.outcomes.filter( + (outcome) => outcome.skippedTests.length > 0 || (outcome.counts?.skip ?? 0) > 0, + ); + if (skipping.length > 0) { + lines.push("", "Files with skipped tests:"); + for (const outcome of skipping.sort((a, b) => a.file.localeCompare(b.file))) { + const counted = outcome.counts + ? `${outcome.counts.skip} skipped` + : `unreadable report; it named ${plural(outcome.skippedTests.length, "skipped test")}`; + lines.push(` ${outcome.file} (${counted})`); + for (const title of outcome.skippedTests) lines.push(` ${title}`); + } + } + + if (summary.failures.length > 0) { + lines.push("", "Failed files:"); + for (const outcome of summary.failures) { + lines.push(` ${outcome.file} (${failureReason(outcome, summary.timeoutMs)})`); + // The runner, not bare `bun test `: bare bun reads the verdict off its + // console and takes no --jobs, which the SIGKILL reason above tells the reader to + // lower. It is also what CONTRIBUTING.md and CLAUDE.md tell a contributor to run. + lines.push(` re-run alone with: bun tests/run-tests.ts ${outcome.file}`); + } + } + + return lines.join("\n"); +} diff --git a/tests/runner/requirements.ts b/tests/runner/requirements.ts new file mode 100644 index 000000000..8878465b5 --- /dev/null +++ b/tests/runner/requirements.ts @@ -0,0 +1,141 @@ +/** + * What a test file needs from the machine it runs on, and what the runner does about it. + * + * A file declares a requirement with one line of its own, `// @requires helm`, and the runner + * reads it before starting the file: + * + * - where the requirement is met, the file runs like any other; + * - where it is not, the file is not started, and the summary names it under the reason, once; + * - where the run REQUIRES it (`LIBREDB_REQUIRE_HELM=1`, which every CI job that runs the suite + * sets), a missing requirement stops the whole run before anything starts. + * + * The chart tests are why this exists. They drive the real `helm` binary against a PostgreSQL + * subchart that has to be downloaded, and a contributor who never touches the chart should not + * have to install either to see `bun run test` go green. The decision is made per FILE rather + * than per test because every one of those files needs Helm for everything it does: skipping + * inside them would list about 180 test names on a machine without Helm, where one line per file + * says the same thing. CI keeps them mandatory, so nothing is quietly left out of the gate. + * + * `tests/unit/test-runner-requirements.test.ts` holds the marker true of the whole tree in both + * directions: a file that runs helm declares it, and a file that declares it runs helm. + */ +import { existsSync, readdirSync } from "node:fs"; +import path from "node:path"; + +export type Capability = "helm"; + +const KNOWN: readonly Capability[] = ["helm"]; + +/** The marker, alone on its line. Anywhere else (quoted, indented) it is not a declaration. */ +const MARKER = /^\/\/ @requires ([a-z][a-z-]*)[ \t]*$/gm; + +export type NotRunFile = { file: string; reason: string }; + +export function parseRequirements(source: string, file: string): Capability[] { + const found: Capability[] = []; + for (const match of source.matchAll(MARKER)) { + const name = match[1] as string; + if (!(KNOWN as readonly string[]).includes(name)) { + throw new Error(`${file} declares "@requires ${name}", and the runner knows only: ${KNOWN.join(", ")}.`); + } + if (!found.includes(name as Capability)) found.push(name as Capability); + } + return found; +} + +export type HelmProbe = { + which: (name: string) => string | null; + subchartBuilt: () => boolean; +}; + +/** + * Why Helm is not usable here, or null when it is. + * + * Usable means two things, because every `helm template` of this chart needs both: the binary, + * and the chart's PostgreSQL dependency built into `charts/libredb-studio/charts/`, which is + * gitignored and so absent from every fresh clone. Without it Helm refuses every render with + * "missing in charts/ directory: postgresql", whatever the render asks for. + */ +export function missingHelm(probe: HelmProbe): string | null { + if (probe.which("helm") === null) { + return "Helm is not installed. The chart tests drive the real helm binary; install Helm 4.1.3 to run them (CONTRIBUTING.md, Prerequisites)."; + } + if (!probe.subchartBuilt()) { + return "The chart's postgresql dependency is not built, and every chart render needs it. Run: helm repo add bitnami https://charts.bitnami.com/bitnami, then helm dependency build charts/libredb-studio --skip-refresh"; + } + return null; +} + +export function systemHelmProbe(root: string): HelmProbe { + return { + which: (name) => Bun.which(name), + subchartBuilt: () => { + const vendored = path.join(root, "charts/libredb-studio/charts"); + return ( + existsSync(vendored) && + readdirSync(vendored).some((entry) => entry.startsWith("postgresql-") && entry.endsWith(".tgz")) + ); + }, + }; +} + +/** The capabilities this run may not do without, from the environment. */ +export function requiredCapabilities(env: Record): Set { + const value = env.LIBREDB_REQUIRE_HELM; + if (value === undefined || value === "" || value === "0") return new Set(); + if (value === "1") return new Set(["helm"]); + throw new Error( + `LIBREDB_REQUIRE_HELM must be 1 or 0, got "${value}": a typo must not quietly make the chart tests optional.`, + ); +} + +/** + * Splits the selection into the files that run and the files that cannot run here. + * + * Each capability is asked about once, however many files need it. A selection in which nothing + * can run is an error rather than an empty green run, which is the one outcome a test runner must + * never produce. + */ +export function planRequirements({ + files, + readSource, + missing, + required, +}: { + files: string[]; + readSource: (file: string) => string; + missing: Record string | null>; + required: ReadonlySet; +}): { run: string[]; notRun: NotRunFile[] } { + const answers = new Map(); + const ask = (capability: Capability): string | null => { + if (!answers.has(capability)) answers.set(capability, missing[capability]()); + return answers.get(capability) ?? null; + }; + + const run: string[] = []; + const notRun: NotRunFile[] = []; + for (const file of files) { + const blocking = parseRequirements(readSource(file), file) + .map((capability) => ({ capability, reason: ask(capability) })) + .find((entry): entry is { capability: Capability; reason: string } => entry.reason !== null); + + if (blocking === undefined) { + run.push(file); + continue; + } + if (required.has(blocking.capability)) { + throw new Error( + `${file} needs ${blocking.capability}, and LIBREDB_REQUIRE_HELM=1 says this run must include it: ${blocking.reason}`, + ); + } + notRun.push({ file, reason: blocking.reason }); + } + + if (run.length === 0 && notRun.length > 0) { + throw new Error( + `None of the ${notRun.length} selected ${notRun.length === 1 ? "file" : "files"} can run here: ${(notRun[0] as NotRunFile).reason}`, + ); + } + return { run, notRun }; +} diff --git a/tests/runner/signals.ts b/tests/runner/signals.ts new file mode 100644 index 000000000..cda9ae060 --- /dev/null +++ b/tests/runner/signals.ts @@ -0,0 +1,49 @@ +/** + * The signals that stop a run, and the exit code each one ends it with. + * + * SIGINT is Ctrl+C. SIGTERM is `timeout(1)`, `docker stop`, Kubernetes and systemd, + * and it is what `bun run test` forwards. SIGHUP is a closed terminal, and on Windows + * a closed console window. SIGBREAK is Ctrl+Break, which GitHub Actions on Windows + * sends 7.5 seconds after Ctrl+C; it is delivered only there, and naming it here is + * valid on every platform. Handling only SIGINT, as this did, meant a run stopped any + * other way printed nothing and left its scratch directory behind (measured 1.4.2). + * + * A signalled run exits 128 + the signal's number, which is what a shell reports. The + * number comes from the platform's own table where the platform has it, because that + * is the number the shell will use; the table below carries it where it does not. + * That second case is real, not defensive: `os.constants.signals` has no SIGBREAK on + * Linux or macOS (measured on bun 1.4.2), so `128 + constants.signals.SIGBREAK` is + * NaN, and `process.exit(NaN)` throws RangeError ERR_OUT_OF_RANGE from inside the + * signal listener, which is the one place a throw must not happen: measured on 1.4.2, + * a throw from inside a SIGINT listener left the process RUNNING and the queue + * started the next file. @types/node types every signal as present, so the compiler + * cannot see it either. + * + * This lives beside the runner rather than inside it so the mapping can be tested on + * every platform: SIGBREAK exists only on Windows, and Windows delivers none of these + * to a piped child, so no end-to-end case can reach that arm anywhere. + */ +import { constants } from "node:os"; + +export const STOP_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP", "SIGBREAK"] as const; + +export type StopSignal = (typeof STOP_SIGNALS)[number]; + +/** + * The POSIX numbers, plus Windows's SIGBREAK, written out because no single platform's + * table carries all four. A test pins these against the platform's own numbers for the + * signals it does name, so a wrong one here cannot pass unnoticed. + */ +export const STOP_SIGNAL_NUMBERS: Record = { + SIGHUP: 1, + SIGINT: 2, + SIGTERM: 15, + SIGBREAK: 21, +}; + +export function exitCodeForSignal( + signal: StopSignal, + platformSignals: Partial> = constants.signals, +): number { + return 128 + (platformSignals[signal] ?? STOP_SIGNAL_NUMBERS[signal]); +} diff --git a/tests/security/agent-statement-boundary.test.ts b/tests/security/agent-statement-boundary.test.ts index 9a086f8a8..f25ab1d62 100644 --- a/tests/security/agent-statement-boundary.test.ts +++ b/tests/security/agent-statement-boundary.test.ts @@ -324,7 +324,7 @@ describe("agent statement boundary — layer (b): SQLite refuses the same statem afterAll(async () => { if (profile?.isConnected()) await profile.disconnect(); - rmSync(SCRATCH, { recursive: true, force: true }); + rmSync(SCRATCH, { recursive: true }); }); const sqliteAttacks = ATTACKS.filter((attack) => attack.sqlite !== undefined); diff --git a/tests/security/audit-channel-callsites.test.ts b/tests/security/audit-channel-callsites.test.ts index d8083d82b..6a2f2eaf5 100644 --- a/tests/security/audit-channel-callsites.test.ts +++ b/tests/security/audit-channel-callsites.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test, afterAll } from "bun:test"; import { mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join, relative } from "node:path"; +import { join, relative, sep } from "node:path"; /** * Threat: a route that records an audit event in the ring buffer WITHOUT the authoritative @@ -51,7 +51,10 @@ function listSources(rootDir: string): string[] { if (entry.isDirectory()) { walk(full); } else if (entry.name.endsWith(".ts") || entry.name.endsWith(".tsx")) { - files.push(relative(rootDir, full)); + // Normalised here, at the one place a path becomes a lookup key: `relative` returns the + // platform separator, so on Windows the key would be "lib\\audit.ts" while every + // allowlist key and every Map lookup in this file is written with forward slashes. + files.push(relative(rootDir, full).split(sep).join("/")); } } } @@ -143,7 +146,7 @@ describe("the authoritative audit channel has no unlisted bypass", () => { // pattern matches. Both are cheap to forbid outright, so they are. test("no file outside src/lib/audit.ts builds its own buffer or renames the accessor", () => { for (const file of listSources(SRC_DIR)) { - if (file === join("lib", "audit.ts")) continue; + if (file === "lib/audit.ts") continue; const source = readFileSync(join(SRC_DIR, file), "utf8"); expect(source).not.toMatch(/new\s+AuditRingBuffer\s*\(/); expect(source).not.toMatch(/getServerAuditBuffer\s+as\s+/); diff --git a/tests/security/helpers/discover-routes.ts b/tests/security/helpers/discover-routes.ts index fe9452b5b..1399a4fbb 100644 --- a/tests/security/helpers/discover-routes.ts +++ b/tests/security/helpers/discover-routes.ts @@ -1,5 +1,6 @@ import { existsSync, readdirSync } from "node:fs"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; export type RouteModule = Record Promise) | undefined>; @@ -15,7 +16,10 @@ export type RouteModule = Record Promise) | u * plain path has no such restriction - bun's dynamic import() resolves it like any other * runtime module specifier, and the "@/" imports *inside* each route.ts still resolve normally * there, since that resolution happens in that file's own context, independent of how the - * importer named it. + * importer named it. It is handed to import() as a file: URL, which is the portable spelling of + * that same specifier: an ESM import of a bare Windows absolute path ("C:\\...") reads "C:" as a + * URL scheme and is rejected, and every route enumeration in the suite goes through this one + * function, so it would take all three files out at once. * * Shared by every route-enumeration test under tests/security/ - the AI-only enumeration in * rate-limit-routes.test.ts and the whole-tree enumeration in route-auth.test.ts both call this @@ -30,7 +34,8 @@ export function discoverRoutes(rootDir: string): Array<[string, () => Promise import(routeFile) as Promise]); + const specifier = pathToFileURL(routeFile).href; + results.push([keySegments.join("/"), () => import(specifier) as Promise]); } for (const entry of readdirSync(dir, { withFileTypes: true })) { if (entry.isDirectory()) { diff --git a/tests/unit/agent-dependency-boundary.test.ts b/tests/unit/agent-dependency-boundary.test.ts index ad6b06695..f1eeb39f0 100644 --- a/tests/unit/agent-dependency-boundary.test.ts +++ b/tests/unit/agent-dependency-boundary.test.ts @@ -26,8 +26,12 @@ interface PackageManifest { peerDependencies?: Record; } +// Anchored to this file, not to process.cwd(): this read happens at module scope, so a runner +// that launched the file from anywhere but the repository root would kill it before a test ran. +const ROOT = path.resolve(import.meta.dir, "../.."); + const manifest: PackageManifest = JSON.parse( - fs.readFileSync(path.join(process.cwd(), "package.json"), "utf8"), + fs.readFileSync(path.join(ROOT, "package.json"), "utf8"), ) as PackageManifest; /** Exact versions ratified by the owner when the runtime spike closed. */ @@ -145,7 +149,7 @@ describe("the knip ignore list stays bounded", () => { test("ignores no dependency beyond tailwindcss and the ratified runtime", () => { const knip: { ignoreDependencies?: string[] } = JSON.parse( - fs.readFileSync(path.join(process.cwd(), "knip.json"), "utf8"), + fs.readFileSync(path.join(ROOT, "knip.json"), "utf8"), ) as { ignoreDependencies?: string[] }; // A subset assertion, deliberately: removing an entry once the run loop // imports the package is the desired direction of travel, and an equality diff --git a/tests/unit/agent-documentation.test.ts b/tests/unit/agent-documentation.test.ts index f09004589..1627f896c 100644 --- a/tests/unit/agent-documentation.test.ts +++ b/tests/unit/agent-documentation.test.ts @@ -35,6 +35,23 @@ import { AGENT_WORKFLOW_BUDGETS } from "@/lib/agent/execution-policy"; const ROOT = path.resolve(import.meta.dir, "../.."); const read = (relative: string): string => readFileSync(path.join(ROOT, relative), "utf8"); +/** + * `Bun.Glob().scanSync()` yields HOST-separated paths, so on Windows a hit arrives as + * `src\app\api\agent\config\route.ts` (measured on windows-latest, where this file's own + * non-vacuity control failed along with the nine assertions it guards). Every use of a + * scanned path below is POSIX-spelled - the `src/app` prefix stripped off a route, the + * route path looked up in `docs/API_DOCS.md`, the comparison against + * `src/lib/agent/config.ts` - and so is the documentation being searched, which no + * platform rewrites. So the separator is normalised once where the paths are produced + * rather than in each comparison. + * + * Unconditional, not a win32 branch: a tracked path holding a backslash cannot be + * checked out on Windows at all, and this suite runs there, so no name under `src/` can + * carry one. That makes this a no-op on POSIX rather than a branch nothing here runs. + */ +const scan = (pattern: string): string[] => + [...new Bun.Glob(pattern).scanSync(ROOT)].map((hit) => hit.replaceAll("\\", "/")); + const AGENT_DOC_PATH = "docs/AGENT.md"; const AGENT_DOC = read(AGENT_DOC_PATH); const ARCHITECTURE = read("docs/ARCHITECTURE.md"); @@ -90,7 +107,7 @@ describe("the agent's environment surface is documented where an operator looks" "src/hooks/use-agent-*.ts", "src/lib/api/agent-run-access.ts", ]; - const files = roots.flatMap((pattern) => [...new Bun.Glob(pattern).scanSync(ROOT)]); + const files = roots.flatMap(scan); expect(files.length).toBeGreaterThan(20); const readers = files.filter((file) => read(file).includes("process.env")); @@ -244,7 +261,7 @@ describe("the agent's HTTP surface is documented where a reader looks for a rout */ const API_DOCS = read("docs/API_DOCS.md"); - const routePaths = [...new Bun.Glob("src/app/api/agent/**/route.ts").scanSync(ROOT)] + const routePaths = scan("src/app/api/agent/**/route.ts") .map((file) => file .replace(/^src\/app/, "") diff --git a/tests/unit/agent-package-boundary.test.ts b/tests/unit/agent-package-boundary.test.ts index 3143a0708..fe4b61f33 100644 --- a/tests/unit/agent-package-boundary.test.ts +++ b/tests/unit/agent-package-boundary.test.ts @@ -42,7 +42,8 @@ import path from "node:path"; import { describe, expect, test } from "bun:test"; import { DEFAULT_WORKSPACE_FEATURES } from "@/workspace/types"; -const ROOT = process.cwd(); +// Anchored to this file, not to process.cwd(): the test is then correct whoever launches it. +const ROOT = path.resolve(import.meta.dir, "../.."); const SRC = path.join(ROOT, "src"); // --------------------------------------------------------------------------- @@ -199,7 +200,12 @@ function reExportClosure(): Set { return exported; } -const relative = (file: string): string => path.relative(ROOT, file); +// path.relative returns the platform separator, so on Windows this would be +// "src\\exports\\components.ts". Every literal compared against it below is POSIX-shaped, and +// isAgentModule() matches on "/" - a backslash path makes that regex match nothing, so the +// boundary gate at "no agent module reaches the package surface" would pass vacuously while a +// real violation went unreported. Normalise once here, at the only place a path becomes a string. +const relative = (file: string): string => path.relative(ROOT, file).split(path.sep).join("/"); /** What the bundler emits as code. */ const emitted = walkFrom("only-values"); diff --git a/tests/unit/aws-ami-descriptor.test.ts b/tests/unit/aws-ami-descriptor.test.ts index 8cf95f87d..611164012 100644 --- a/tests/unit/aws-ami-descriptor.test.ts +++ b/tests/unit/aws-ami-descriptor.test.ts @@ -13,8 +13,19 @@ import { describe, expect, test } from "bun:test"; import * as fs from "fs"; import * as os from "os"; import * as path from "path"; +import { MISSING_POSIX_FILE_MODES, missingPosixShell, posixShell, testIf } from "../helpers/posix-tools"; const AMI = path.join(__dirname, "../../deploy/aws/ami"); +/* + Everything in this file is text analysis except one case, which EXECUTES the Ubuntu MOTD hook + against a stub `curl` it makes runnable with mode 0755 and finds through PATH. That is a + /etc/update-motd.d artifact, run by pam_motd on a buyer's Ubuntu instance: Windows has neither the + shell on PATH (`Bun.spawnSync(["sh", ...])` throws "Executable not found in $PATH") nor a mode bit + for the stub, so that one case says why it is skipped instead of failing as if the hook were + broken. The shape checks around it keep running everywhere. +*/ +const SHELL = posixShell("sh"); +const HOOK_CANNOT_RUN = missingPosixShell("sh") ?? MISSING_POSIX_FILE_MODES; const read = (relative: string): string => fs.readFileSync(path.join(AMI, relative), "utf8"); const template = read("template.pkr.hcl"); @@ -258,7 +269,7 @@ describe("AWS AMI banner and MOTD", () => { expect(motd.trimEnd().endsWith("exit 0")).toBe(true); }); - test("running the hook against a fixture never prints the password", () => { + testIf(HOOK_CANNOT_RUN, "running the hook against a fixture never prints the password", () => { // The assertions above are shape checks, and shape checks passed while the // hook could still be made to print the value (a capture group in the sed, or // a second grep after it). This runs the real hook. @@ -284,8 +295,8 @@ describe("AWS AMI banner and MOTD", () => { fs.writeFileSync(path.join(dir, "curl"), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); try { - const run = Bun.spawnSync(["sh", hookPath], { - env: { ...process.env, PATH: `${dir}:${process.env.PATH}` }, + const run = Bun.spawnSync([SHELL!, hookPath], { + env: { ...process.env, PATH: `${dir}${path.delimiter}${process.env.PATH}` }, }); const stdout = new TextDecoder().decode(run.stdout); expect(run.exitCode).toBe(0); diff --git a/tests/unit/backlog-structure.test.ts b/tests/unit/backlog-structure.test.ts index 41372bb1a..d9c14e0bb 100644 --- a/tests/unit/backlog-structure.test.ts +++ b/tests/unit/backlog-structure.test.ts @@ -23,7 +23,7 @@ * as shapes and the assertions derive the real ones from the document. */ import { describe, expect, test } from "bun:test"; -import { readFileSync } from "node:fs"; +import { readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; const ROOT = path.resolve(import.meta.dir, "../.."); @@ -375,21 +375,72 @@ describe("a quoted grep command answers what the entry says it answers", () => { where: `${BACKLOG_PATH}:${lineAt(match.index)}`, })); + /** + * Every file `grep -r` would read under this path, relative to ROOT. + * + * A symlink met inside the tree is skipped, which is what `-r` does (only `-R` follows one), and + * the WORKING TREE is what is walked rather than the index: entries cite paths that are + * gitignored, so a tracked-files listing would answer a different question. + */ + const filesUnder = (target: string): string[] => { + if (!statSync(path.join(ROOT, target)).isDirectory()) return [target]; + return readdirSync(path.join(ROOT, target), { withFileTypes: true }).flatMap((child) => { + if (child.isDirectory()) return filesUnder(`${target}/${child.name}`); + return child.isFile() ? [`${target}/${child.name}`] : []; + }); + }; + + /** + * The quoted pattern as a JS regex. + * + * grep reads a POSIX BASIC regular expression, where `+ ? | ( ) { }` are literal characters and + * JS reads every one of them as syntax - so handing the raw pattern to `new RegExp` would answer + * a different question than the sentence claims. Backslash escapes and bracket expressions are + * not translated at all: a command that uses one fails by name here rather than being run under + * the wrong dialect, which is the same discipline as the SHAPE above. + */ + const toRegExp = (pattern: string): RegExp => { + if (/[\\[]/.test(pattern)) { + throw new Error( + `${BACKLOG_PATH} quotes a grep whose pattern this guard does not translate ` + + `(no backslash escapes, no bracket expressions): ${pattern}`, + ); + } + // The backslash is in the escape class as well, although the refusal above means one + // never reaches this line: a translation that escapes some metacharacters and not the + // escape character itself is only correct while its caller is, and this one should be + // correct on its own terms. `. * ^ $` are deliberately NOT escaped: BRE and JS read + // those the same way, so escaping them would change the question the entry asks. + return new RegExp(pattern.replaceAll(/[\\+?|(){}]/g, "\\$&")); + }; + + /** + * The command, run in process. + * + * It used to be `Bun.spawnSync(["grep", ...])`, which made the whole guard depend on a binary a + * PowerShell session does not have - Git for Windows keeps grep.exe in usr\bin, which is not on + * that PATH - and `Bun.spawnSync` THROWS there ("Executable not found in $PATH", measured in this + * worktree) rather than returning a non-zero exit code, so every claim test died at the first + * command. Searching here also removes the GNU/BSD divergence for any flag a future entry uses. + */ const run = (command: string): number => { const shape = SHAPE.exec(command); if (shape === null) throw new Error(`${BACKLOG_PATH} quotes a grep this guard cannot run: ${command}`); - const result = Bun.spawnSync({ - cmd: ["grep", shape[1], "--", shape[2], ...shape[3].split(/ +/)], - cwd: ROOT, - }); - // grep answers 1 for "no lines matched", which is an outcome here rather than a failure. - if (result.exitCode !== 0 && result.exitCode !== 1) { - throw new Error(`${command} failed with ${result.exitCode}: ${result.stderr.toString()}`); + // -n counts matched LINES and -l counts matched FILES; anything else (a -i, a non-recursive + // grep over a directory) would need its own modelling, so it is refused instead of guessed. + if (!/^-r[nl]$/.test(shape[1])) { + throw new Error(`${BACKLOG_PATH} quotes a grep with flags this guard does not model: ${command}`); + } + const pattern = toRegExp(shape[2]); + let hits = 0; + for (const file of shape[3].split(/ +/).flatMap(filesUnder)) { + const lines = readFileSync(path.join(ROOT, file), "utf8").split("\n"); + // A trailing newline ends the last line, it does not start an empty one. + if (lines.at(-1) === "") lines.pop(); + const matched = lines.filter((line) => pattern.test(line)).length; + hits += shape[1].includes("l") ? Math.min(matched, 1) : matched; } - return result.stdout - .toString() - .split("\n") - .filter((line) => line !== "").length; + return hits; }; test("the extractor found every grep the file quotes", () => { diff --git a/tests/unit/build-azure-package.test.ts b/tests/unit/build-azure-package.test.ts index 013802a31..e8e7bb16f 100644 --- a/tests/unit/build-azure-package.test.ts +++ b/tests/unit/build-azure-package.test.ts @@ -26,10 +26,27 @@ import { pinnedRef, resolveImageDigest, } from "../../scripts/build-azure-package.mjs"; +import { describeIf, missingUnixTool, resolveUnixTool, testIf } from "../helpers/posix-tools"; const SCRIPT = join(import.meta.dir, "../../scripts/build-azure-package.mjs"); const REPO_ROOT = join(import.meta.dir, "../.."); +/* + The builder itself shells out to `zip` (scripts/build-azure-package.mjs:273), so every case that + actually builds a package needs that binary, and reading the result back needs `unzip`. A stock + Windows 11 has neither, and `execFileSync`/`Bun.spawnSync` answer a missing binary by THROWING + ("Executable not found in $PATH", measured in this worktree), so those cases now say what is + missing instead of dying at the first spawn. macOS and Linux both ship the pair and run them + unchanged. + + Reading the archive in process would not widen that: the gate is `zip`, which the script needs + whatever the test does. What WOULD widen it is making the builder write the two-file archive + without an external tool - a change to the script, not to this file. +*/ +const UNZIP = resolveUnixTool("unzip"); +const NO_ZIP = missingUnixTool("zip"); +const NO_ZIP_TOOLS = NO_ZIP ?? missingUnixTool("unzip"); + const APP_DIGEST = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const CADDY_DIGEST = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; @@ -219,7 +236,7 @@ describe("checkApiVersionAges (the 540/700-day gate)", () => { }); }); -describe("buildPackage (end to end against a fixture repo)", () => { +describeIf(NO_ZIP_TOOLS, "buildPackage (end to end against a fixture repo)", () => { const NOW = Date.parse("2026-08-05T12:00:00Z"); function makeFixtureRepo({ apiVersion = "2025-07-01" }: { apiVersion?: string } = {}): string { @@ -257,7 +274,7 @@ describe("buildPackage (end to end against a fixture repo)", () => { expect(result.zipPath).toBe(join(root, "dist/azure/libredb-studio-azure-1.2.3.zip")); expect(existsSync(result.zipPath)).toBe(true); - const listing = execFileSync("unzip", ["-l", result.zipPath], { encoding: "utf8" }); + const listing = execFileSync(UNZIP!, ["-l", result.zipPath], { encoding: "utf8" }); const entries = listing .split("\n") .map((line) => line.trim().split(/\s+/).slice(3).join(" ")) @@ -331,7 +348,7 @@ describe("CLI", () => { expect(result.stderr).toContain("integer.integer.integer"); }); - test("the full success path works end to end against a local registry stub", async () => { + testIf(NO_ZIP, "the full success path works end to end against a local registry stub", async () => { const NOW = Date.parse("2026-08-05T12:00:00Z"); const root = mkdtempSync(join(tmpdir(), "azure-package-cli-")); const src = join(root, "deploy/azure/src"); diff --git a/tests/unit/check-appimage-perms.test.ts b/tests/unit/check-appimage-perms.test.ts index fc0c33bf8..9dec0e880 100644 --- a/tests/unit/check-appimage-perms.test.ts +++ b/tests/unit/check-appimage-perms.test.ts @@ -21,6 +21,7 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; import { auditAppDirPermissions, formatOffenders } from "../../scripts/check-appimage-perms.mjs"; +import { MISSING_POSIX_FILE_MODES, describeIf } from "../helpers/posix-tools"; /** Build a throwaway AppDir. `entries` maps a relative path to its octal mode. */ const appDir = (entries: Record): string => { @@ -34,7 +35,15 @@ const appDir = (entries: Record): string => { return root; }; -describe("auditAppDirPermissions", () => { +/* + What this audit measures IS the POSIX mode bits, so the fixture cannot be built on Windows: chmod + there only toggles the read-only flag and stat reads every file back as 0o666 or 0o444, which + would leave auditAppDirPermissions seeing a clean AppDir and the three "reports an offender" cases + asserting against an empty list. The underlying artifact is a Linux AppImage AppDir. The two cases + that need no mode - the refusal on a missing directory, and formatOffenders - are kept out of this + group so they keep running everywhere. +*/ +describeIf(MISSING_POSIX_FILE_MODES, "auditAppDirPermissions over POSIX mode bits", () => { test("passes an AppDir whose files are all world-readable", () => { const root = appDir({ AppRun: 0o755, "usr/bin/app": 0o755, "usr/share/icon.png": 0o644 }); expect(auditAppDirPermissions(root)).toEqual([]); @@ -72,7 +81,9 @@ describe("auditAppDirPermissions", () => { fs.symlinkSync("usr/share/icons/hicolor/32x32/apps/app.png", path.join(root, ".DirIcon")); expect(auditAppDirPermissions(root)).toEqual([]); }); +}); +describe("auditAppDirPermissions", () => { test("throws when the directory does not exist, rather than reporting a clean audit", () => { // A silent pass on a mistyped path would turn this gate into decoration. expect(() => auditAppDirPermissions(path.join(os.tmpdir(), "appdir-perms-absent-xyz"))).toThrow(/not a directory/); diff --git a/tests/unit/ci-install.test.ts b/tests/unit/ci-install.test.ts index 6e77d4b49..31cf9c138 100644 --- a/tests/unit/ci-install.test.ts +++ b/tests/unit/ci-install.test.ts @@ -7,12 +7,24 @@ * publish. The script is exercised against a stub `bun` on PATH that fails a * chosen number of times, so the retry policy is verified without a network. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, expect, test } from "bun:test"; import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { MISSING_POSIX_FILE_MODES, describeIf, missingPosixShell, posixShell } from "../helpers/posix-tools"; const SCRIPT = join(import.meta.dir, "../../scripts/ci-install.sh"); +/* + The script itself is not POSIX-only - .github/actions/bun-install runs it under Git Bash on + windows-latest too - but this fixture is: it replaces PATH with `:/usr/bin:/bin` and puts + two `#!/usr/bin/env bash` stubs there, made runnable with chmod 0755. Windows has no exec bit, no + shebang dispatch for an extension-less file, and no /usr/bin to fall back on, and + `Bun.spawnSync(["bash", ...])` THROWS there ("Executable not found in $PATH", measured in this + worktree) or resolves to WSL's Linux bash, which cannot see the Win32 stub directory. So the + retry policy is measured on Linux and macOS and the skip says so by name. +*/ +const SHELL = posixShell("bash"); +const CANNOT_RUN = missingPosixShell("bash") ?? MISSING_POSIX_FILE_MODES; const roots: string[] = []; afterEach(() => { @@ -64,7 +76,7 @@ exit 0 } function run(bin: string, env: Record = {}) { - return Bun.spawnSync(["bash", SCRIPT], { + return Bun.spawnSync([SHELL!, SCRIPT], { // PATH is replaced, not prepended: the stub must be the only bun in reach. env: { PATH: `${bin}:/usr/bin:/bin`, CI_INSTALL_BACKOFF_SECONDS: "0", ...env }, stdout: "pipe", @@ -80,7 +92,7 @@ function callCount(log: string): number { } } -describe("scripts/ci-install.sh", () => { +describeIf(CANNOT_RUN, "scripts/ci-install.sh", () => { test("installs once when the first attempt succeeds", () => { const { bin, log } = stubBun(0); const result = run(bin); diff --git a/tests/unit/component-runner-coverage.test.ts b/tests/unit/component-runner-coverage.test.ts deleted file mode 100644 index 0079edd3f..000000000 --- a/tests/unit/component-runner-coverage.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { readdirSync, readFileSync } from "node:fs"; -import path from "node:path"; - -const root = path.resolve(import.meta.dir, "../.."); -const runner = readFileSync(path.join(root, "tests/run-components.sh"), "utf8"); -const commands = runner - .split("\n") - .filter((line) => !line.trimStart().startsWith("#")) - .join("\n"); -const named = new Set(commands.match(/tests\/(?:components|isolated)\/[^\s"']+\.test\.tsx?\b/g) ?? []); - -function testFiles(directory: string): string[] { - return readdirSync(path.join(root, directory), { withFileTypes: true }).flatMap((entry) => { - const file = `${directory}/${entry.name}`; - if (entry.isDirectory()) return testFiles(file); - return entry.isFile() && /\.test\.tsx?$/.test(entry.name) ? [file] : []; - }); -} - -const onDisk = new Set(["tests/components", "tests/isolated"].flatMap(testFiles)); - -describe("component runner coverage", () => { - test("every component and isolated test file is named by the runner", () => { - expect([...onDisk].filter((file) => !named.has(file)).sort()).toEqual([]); - }); - - test("every test path named by the runner exists on disk", () => { - expect([...named].filter((file) => !onDisk.has(file)).sort()).toEqual([]); - }); - - test("TOTAL_GROUPS matches the number of run_group calls", () => { - const declared = runner.match(/^TOTAL_GROUPS=(\d+)$/m); - expect(declared).not.toBeNull(); - expect(Number(declared![1])).toBe((runner.match(/^run_group /gm) ?? []).length); - }); -}); diff --git a/tests/unit/components/agent-hydration.test.ts b/tests/unit/components/agent-hydration.test.ts index 9e9be00ee..b1537f406 100644 --- a/tests/unit/components/agent-hydration.test.ts +++ b/tests/unit/components/agent-hydration.test.ts @@ -145,7 +145,8 @@ describe("hydrateAgentArtifact", () => { * rail fails here rather than shipping a second one to keep correct. */ describe("the agent rail's module boundary", () => { - const AGENT_DIR = path.join(process.cwd(), "src/components/agent"); + // Anchored to this file rather than to process.cwd(), so the scan is correct whoever launches it. + const AGENT_DIR = path.resolve(import.meta.dir, "../../..", "src/components/agent"); const RAIL_MODULES = [ "AgentRail.tsx", // The three the 2026-08-21 redesign split out of `AgentRail.tsx`, plus the module diff --git a/tests/unit/components/results-grid-renderers.test.ts b/tests/unit/components/results-grid-renderers.test.ts index 9ac374ee9..2b1d47ba4 100644 --- a/tests/unit/components/results-grid-renderers.test.ts +++ b/tests/unit/components/results-grid-renderers.test.ts @@ -1,6 +1,6 @@ import { describe, test, expect } from "bun:test"; import { readFileSync, readdirSync } from "node:fs"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { classifyValue } from "@/components/results-grid/renderers/classify"; import { getRenderer } from "@/components/results-grid/renderers/registry"; import { jsonRenderer } from "@/components/results-grid/renderers/json"; @@ -230,9 +230,11 @@ describe("renderDetail", () => { describe("rendering layer is provider-agnostic", () => { test("no connection-type identifiers in the renderer modules or the formatter", () => { - const renderersDir = join(process.cwd(), "src/components/results-grid/renderers"); + // Anchored to this file rather than to process.cwd(), so the scan is correct whoever launches it. + const root = resolve(import.meta.dir, "../../.."); + const renderersDir = join(root, "src/components/results-grid/renderers"); const sources = readdirSync(renderersDir).map((f) => join(renderersDir, f)); - sources.push(join(process.cwd(), "src/components/results-grid/utils.ts")); + sources.push(join(root, "src/components/results-grid/utils.ts")); const providerTypeIds = /\b(postgres|mysql|sqlite|oracle|mssql|mongodb|redis|libredb)\b/i; for (const file of sources) { diff --git a/tests/unit/copy-monaco.test.ts b/tests/unit/copy-monaco.test.ts index d9c23b81e..57ebfbfe3 100644 --- a/tests/unit/copy-monaco.test.ts +++ b/tests/unit/copy-monaco.test.ts @@ -70,7 +70,10 @@ describe("copy-monaco CLI", () => { const proc = Bun.spawnSync(["node", script], { cwd: root }); expect(proc.exitCode).toBe(0); - expect(proc.stdout.toString()).toContain("public/monaco/vs"); + // The script prints path.relative(cwd, target) (scripts/copy-monaco.mjs:54), and target is + // built with path.join, so the separator is the platform's: "public\\monaco\\vs" on Windows. + // Build the expected fragment the same way instead of hardcoding a POSIX spelling. + expect(proc.stdout.toString()).toContain(join("public", "monaco", "vs")); }); test("exits 1 with an actionable message when the dependency is missing", () => { diff --git a/tests/unit/db-tunnel-discipline.test.ts b/tests/unit/db-tunnel-discipline.test.ts index 9b80fc9f6..257136127 100644 --- a/tests/unit/db-tunnel-discipline.test.ts +++ b/tests/unit/db-tunnel-discipline.test.ts @@ -28,7 +28,16 @@ import * as path from "path"; * repeat, which is the case that actually happened, not a determined one. */ -const SRC = path.join(process.cwd(), "src"); +// Anchored to this file rather than to process.cwd(), so the scan is correct whoever launches it. +const ROOT = path.resolve(import.meta.dir, "../.."); +const SRC = path.join(ROOT, "src"); + +/** + * The one place an absolute path becomes a rule key. path.relative hands back the platform + * separator, so on Windows this would be "src\\lib\\db\\index.ts": the EXEMPT table is keyed + * with forward slashes and would stop matching, reporting exempt files as violations. + */ +const repoRelative = (full: string): string => path.relative(ROOT, full).split(path.sep).join("/"); /** Import specifiers that resolve to the factory. */ const FACTORY_SPECIFIERS = ["@/lib/db/factory", "@/lib/db"] as const; @@ -162,7 +171,7 @@ describe("SSH tunnel discipline (#457)", () => { test("every connecting caller of createDatabaseProvider tunnels", () => { const violations = walk(SRC) .map((full) => { - const relPath = path.relative(process.cwd(), full); + const relPath = repoRelative(full); return analyseTunnelDiscipline(relPath, fs.readFileSync(full, "utf8")); }) .filter((v): v is Violation => v !== null); @@ -178,11 +187,11 @@ describe("SSH tunnel discipline (#457)", () => { // outside both caches and so had to open its own tunnel, and it is deleted with the // flat schema reading it read (#789). Its consumer reads `/api/db/objects/inventory`, // which goes through `getOrCreateProvider` and is tunnelled by the factory. - const scanned = walk(SRC).map((full) => path.relative(process.cwd(), full)); + const scanned = walk(SRC).map(repoRelative); expect(scanned).toContain("src/app/api/db/test-connection/route.ts"); for (const route of ["src/app/api/db/test-connection/route.ts"]) { - const source = fs.readFileSync(path.join(process.cwd(), route), "utf8"); + const source = fs.readFileSync(path.join(ROOT, route), "utf8"); // Compliant today, and provably in scope: strip the scope and the rule bites. expect(analyseTunnelDiscipline(route, source)).toBeNull(); expect(analyseTunnelDiscipline(route, source.replace(/withOneShotTunnel/g, "somethingElse"))).not.toBeNull(); @@ -193,7 +202,7 @@ describe("SSH tunnel discipline (#457)", () => { // A stale exemption is a hole nobody can see. If a file moves, the entry must move // with it or be deleted. for (const file of Object.keys(EXEMPT)) { - expect(fs.existsSync(path.join(process.cwd(), file))).toBe(true); + expect(fs.existsSync(path.join(ROOT, file))).toBe(true); } }); }); diff --git a/tests/unit/distribution-check.test.ts b/tests/unit/distribution-check.test.ts index ed2a62269..c5521679b 100644 --- a/tests/unit/distribution-check.test.ts +++ b/tests/unit/distribution-check.test.ts @@ -839,6 +839,20 @@ describe("CLI (subprocess against temp fixtures)", () => { }); }); +/** + * A case that starts a node process and makes an HTTP round trip to a server in this + * process, which bun's 5000 ms per-test default is not sized for. Measured on + * windows-latest on 2026-09-15, with the suite running four files at once: "a reachable + * remote pin is compared like a local one" took 4480 ms end to end and failed, most of + * that spent inside a 3000 ms fetch budget the test itself had set. + * + * That fetch budget is gone rather than raised. No case here needs a fetch to time out: + * every failure it exercises answers at once, with a refused connection or a status + * code, so a short budget could only ever fail the SUCCESS path on a slow machine. The + * script's own default applies instead, which is also what a real run gets. + */ +const spawnTest = (name: string, body: () => Promise) => test(name, body, 30_000); + describe("CLI (remote pins against a local server)", () => { const fixtureRoots: string[] = []; const servers: Array<{ stop: () => void }> = []; @@ -884,7 +898,7 @@ describe("CLI (remote pins against a local server)", () => { // in-process Bun.serve fixture, so the remote tests spawn asynchronously. async function runCheckAsync(root: string) { const proc = Bun.spawn(["node", SCRIPT, "--root", root], { - env: { ...process.env, GITHUB_STEP_SUMMARY: "", DISTRIBUTION_CHECK_TIMEOUT_MS: "3000" }, + env: { ...process.env, GITHUB_STEP_SUMMARY: "" }, stdout: "pipe", stderr: "pipe", }); @@ -896,7 +910,7 @@ describe("CLI (remote pins against a local server)", () => { return { stdout, stderr, exitCode }; } - test("a reachable remote pin is compared like a local one", async () => { + spawnTest("a reachable remote pin is compared like a local one", async () => { const url = serve(() => new Response("image: ghcr.io/libredb/libredb-studio:0.9.27\n")); const result = await runCheckAsync(remoteFixture(url)); expect(result.exitCode).toBe(0); @@ -904,14 +918,14 @@ describe("CLI (remote pins against a local server)", () => { expect(result.stdout).toContain("0.9.27"); }); - test("a failing remote fetch degrades to UNKNOWN and still exits 0", async () => { + spawnTest("a failing remote fetch degrades to UNKNOWN and still exits 0", async () => { const url = serve(() => new Response("boom", { status: 500 })); const result = await runCheckAsync(remoteFixture(url)); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("UNKNOWN"); }); - test("an unreachable host degrades to UNKNOWN and still exits 0", async () => { + spawnTest("an unreachable host degrades to UNKNOWN and still exits 0", async () => { // Port 1 is reserved and closed: connection refused, no timeout wait. const result = await runCheckAsync(remoteFixture("http://127.0.0.1:1/pin.yml")); expect(result.exitCode).toBe(0); @@ -946,7 +960,7 @@ describe("CLI (probes against a local registry/store/catalog)", () => { async function runCheckAsync(root: string) { const proc = Bun.spawn(["node", SCRIPT, "--root", root], { - env: { ...process.env, GITHUB_STEP_SUMMARY: "", DISTRIBUTION_CHECK_TIMEOUT_MS: "3000" }, + env: { ...process.env, GITHUB_STEP_SUMMARY: "" }, stdout: "pipe", stderr: "pipe", }); @@ -995,7 +1009,7 @@ describe("CLI (probes against a local registry/store/catalog)", () => { }); } - test("GHCR latest pointing at the released version is OK", async () => { + spawnTest("GHCR latest pointing at the released version is OK", async () => { const base = registry({ latest: "sha256:same", "0.9.53": "sha256:same" }); const result = await runCheckAsync(probeFixture(ghcrRow(base))); expect(result.exitCode).toBe(0); @@ -1003,7 +1017,7 @@ describe("CLI (probes against a local registry/store/catalog)", () => { expect(result.stdout).toContain("0.9.53"); }); - test("GHCR latest still pointing at the previous image is DRIFT with both digests", async () => { + spawnTest("GHCR latest still pointing at the previous image is DRIFT with both digests", async () => { const base = registry({ latest: "sha256:previous", "0.9.53": "sha256:current" }); const result = await runCheckAsync(probeFixture(ghcrRow(base))); expect(result.stdout).toContain("| DRIFT | docker-ghcr |"); @@ -1011,20 +1025,20 @@ describe("CLI (probes against a local registry/store/catalog)", () => { expect(result.stdout).toContain("sha256:current"); }); - test("a GHCR token exchange that fails degrades to UNKNOWN", async () => { + spawnTest("a GHCR token exchange that fails degrades to UNKNOWN", async () => { const base = serveBase(() => new Response("no token for you", { status: 403 })); const result = await runCheckAsync(probeFixture(ghcrRow(base))); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("| UNKNOWN | docker-ghcr |"); }); - test("an unreachable registry degrades to UNKNOWN, never drift", async () => { + spawnTest("an unreachable registry degrades to UNKNOWN, never drift", async () => { const result = await runCheckAsync(probeFixture(ghcrRow("http://127.0.0.1:1"))); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("| UNKNOWN | docker-ghcr |"); }); - test("a Docker Hub mirror missing the released tag is DRIFT - the silently skipped push", async () => { + spawnTest("a Docker Hub mirror missing the released tag is DRIFT - the silently skipped push", async () => { const base = serveBase((req) => { const ref = new URL(req.url).pathname.split("/").pop(); return ref === "latest" ? Response.json({ digest: "sha256:stale" }) : new Response("not found", { status: 404 }); @@ -1076,24 +1090,27 @@ describe("CLI (probes against a local registry/store/catalog)", () => { `; } - test("the Snap Store stable channel is measured per architecture and the device-series header is sent", async () => { - const seenHeaders: Array = []; - const base = serveBase((req) => { - seenHeaders.push(req.headers.get("snap-device-series")); - return Response.json({ - "channel-map": [ - { channel: { track: "latest", risk: "stable", architecture: "amd64" }, version: "0.9.53" }, - { channel: { track: "latest", risk: "stable", architecture: "arm64" }, version: "0.9.53" }, - { channel: { track: "latest", risk: "edge", architecture: "amd64" }, version: "0.9.40" }, - ], + spawnTest( + "the Snap Store stable channel is measured per architecture and the device-series header is sent", + async () => { + const seenHeaders: Array = []; + const base = serveBase((req) => { + seenHeaders.push(req.headers.get("snap-device-series")); + return Response.json({ + "channel-map": [ + { channel: { track: "latest", risk: "stable", architecture: "amd64" }, version: "0.9.53" }, + { channel: { track: "latest", risk: "stable", architecture: "arm64" }, version: "0.9.53" }, + { channel: { track: "latest", risk: "edge", architecture: "amd64" }, version: "0.9.40" }, + ], + }); }); - }); - const result = await runCheckAsync(probeFixture(snapRow(base))); - expect(seenHeaders).toEqual(["16"]); - expect(result.stdout).toContain("| OK | snap |"); - }); + const result = await runCheckAsync(probeFixture(snapRow(base))); + expect(seenHeaders).toEqual(["16"]); + expect(result.stdout).toContain("| OK | snap |"); + }, + ); - test("one lagging Snap architecture is DRIFT even though the other is current", async () => { + spawnTest("one lagging Snap architecture is DRIFT even though the other is current", async () => { const base = serveBase(() => Response.json({ "channel-map": [ @@ -1107,7 +1124,7 @@ describe("CLI (probes against a local registry/store/catalog)", () => { expect(result.stdout).toContain("arm64=0.9.40"); }); - test("a Snap Store error degrades to UNKNOWN", async () => { + spawnTest("a Snap Store error degrades to UNKNOWN", async () => { const base = serveBase(() => new Response("maintenance", { status: 503 })); const result = await runCheckAsync(probeFixture(snapRow(base))); expect(result.exitCode).toBe(0); @@ -1162,33 +1179,33 @@ describe("CLI (probes against a local registry/store/catalog)", () => { `; } - test("winget is measured by the highest version its catalog enumerates", async () => { + spawnTest("winget is measured by the highest version its catalog enumerates", async () => { const base = serveBase(() => Response.json([{ name: ".validation" }, { name: "0.9.40" }, { name: "0.9.53" }])); const result = await runCheckAsync(probeFixture(wingetRow(base))); expect(result.stdout).toContain("| OK | winget |"); }); - test("a winget catalog stuck on an older version is DRIFT", async () => { + spawnTest("a winget catalog stuck on an older version is DRIFT", async () => { const base = serveBase(() => Response.json([{ name: "0.9.40" }])); const result = await runCheckAsync(probeFixture(wingetRow(base))); expect(result.stdout).toContain("| DRIFT | winget |"); expect(result.stdout).toContain("0.9.40"); }); - test("a winget catalog listing that cannot be read degrades to UNKNOWN", async () => { + spawnTest("a winget catalog listing that cannot be read degrades to UNKNOWN", async () => { const base = serveBase(() => new Response("rate limited", { status: 429 })); const result = await runCheckAsync(probeFixture(wingetRow(base))); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("| UNKNOWN | winget |"); }); - test("two catalog listings that agree are one OK row", async () => { + spawnTest("two catalog listings that agree are one OK row", async () => { const base = serveBase(() => Response.json([{ name: "ci.yaml" }, { name: "0.9.53" }])); const result = await runCheckAsync(probeFixture(twoCatalogRow(base))); expect(result.stdout).toContain("| OK | operatorhub-community |"); }); - test("one lagging catalog is DRIFT naming both listings", async () => { + spawnTest("one lagging catalog is DRIFT naming both listings", async () => { const base = serveBase((req) => Response.json(new URL(req.url).pathname === "/hub" ? [{ name: "0.9.40" }] : [{ name: "0.9.53" }]), ); @@ -1198,7 +1215,7 @@ describe("CLI (probes against a local registry/store/catalog)", () => { expect(result.stdout).toContain("openshift-console=0.9.53"); }); - test("an unreadable listing names which catalog failed", async () => { + spawnTest("an unreadable listing names which catalog failed", async () => { const base = serveBase((req) => new URL(req.url).pathname === "/hub" ? new Response("rate limited", { status: 429 }) @@ -1209,7 +1226,7 @@ describe("CLI (probes against a local registry/store/catalog)", () => { expect(result.stdout).toContain("operatorhub-io: catalog listing unavailable"); }); - test("a listing with no published version names which catalog is empty", async () => { + spawnTest("a listing with no published version names which catalog is empty", async () => { const base = serveBase((req) => Response.json(new URL(req.url).pathname === "/hub" ? [{ name: "ci.yaml" }] : [{ name: "0.9.53" }]), ); diff --git a/tests/unit/docker-bind-address.test.ts b/tests/unit/docker-bind-address.test.ts index 933f875d7..f520ae46c 100644 --- a/tests/unit/docker-bind-address.test.ts +++ b/tests/unit/docker-bind-address.test.ts @@ -36,7 +36,7 @@ * both the address it chose and the evidence it chose it on. */ import { describe, expect, test } from "bun:test"; -import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -516,13 +516,22 @@ describe("isDirectExecution - the module's own on/off switch", () => { test("invocation through a symlink matches - import.meta.url is already realpath'd", () => { const dir = mkdtempSync(join(tmpdir(), "libredb-guard-link-")); try { - const target = join(dir, "bind.mjs"); - const link = join(dir, "link.mjs"); + const real = join(dir, "real"); + mkdirSync(real); + const target = join(real, "bind.mjs"); writeFileSync(target, ""); - symlinkSync(target, link); + const link = join(dir, "link"); + /* + The link is a DIRECTORY link created as a junction, and argv[1] reaches the module through + it. A file symlink needs SeCreateSymbolicLinkPrivilege on Windows - Developer Mode is off on + a fresh machine - so symlinkSync throws EPERM there and the test errors instead of asserting + anything. A junction needs no privilege, node ignores the type argument on POSIX, and what is + under test is unchanged: argv[1] arriving through a link still matches. + */ + symlinkSync(real, link, "junction"); // Resolved for the same reason as above: what is under test is that argv[1] reaching // this through a SYMLINK still matches, not that the temp directory has none. - expect(isDirectExecution(link, pathToFileURL(realpathSync(target)).href)).toBe(true); + expect(isDirectExecution(join(link, "bind.mjs"), pathToFileURL(realpathSync(target)).href)).toBe(true); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/tests/unit/docker-entrypoint.test.ts b/tests/unit/docker-entrypoint.test.ts index b6edbeb9c..659e8f8b9 100644 --- a/tests/unit/docker-entrypoint.test.ts +++ b/tests/unit/docker-entrypoint.test.ts @@ -16,12 +16,27 @@ * OTHER (root -> gosu) path, and resolution placed inside either branch would * silently apply to only half of the deployments. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, expect, test } from "bun:test"; import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { delimiter, join } from "node:path"; +import { MISSING_POSIX_FILE_MODES, describeIf, missingPosixShell, posixShell } from "../helpers/posix-tools"; const ENTRYPOINT = join(import.meta.dir, "../../docker-entrypoint.sh"); +/* + The shell is resolved, not spawned by bare name: `Bun.spawnSync(["sh", ...])` THROWS + ("Executable not found in $PATH", measured in this worktree) rather than returning a non-zero + exit code, and in a PowerShell session there is no `sh` - Git for Windows keeps sh.exe in usr\bin, + which is not on that PATH. + + And where the shell resolves, the fixture still cannot stand up: the stub `node` below is a + `#!/bin/sh` script made runnable with chmod 0755 and found through PATH, which is POSIX process + execution. Windows has no exec bit and cannot exec an extension-less #! file. The entrypoint + itself only ever runs as /bin/sh PID 1 inside the Linux image, so a Windows contributor has + nothing to verify here - better said by name than passed quietly. +*/ +const SHELL = posixShell("sh"); +const CANNOT_RUN = missingPosixShell("sh") ?? MISSING_POSIX_FILE_MODES; /** * A stub `node`: when handed the resolver path it runs the fixture as a shell @@ -39,7 +54,7 @@ const STUB_NODE_SCRIPT = [ "", ].join("\n"); -describe("docker-entrypoint.sh bind address (#432)", () => { +describeIf(CANNOT_RUN, "docker-entrypoint.sh bind address (#432)", () => { const fixtureRoots: string[] = []; afterEach(() => { @@ -59,10 +74,10 @@ describe("docker-entrypoint.sh bind address (#432)", () => { const resolver = join(root, "bind-address.mjs"); if (resolverScript !== null) writeFileSync(resolver, resolverScript); - return Bun.spawnSync(["sh", ENTRYPOINT, ...args], { + return Bun.spawnSync([SHELL!, ENTRYPOINT, ...args], { env: { ...process.env, - PATH: `${binDir}:${process.env.PATH}`, + PATH: `${binDir}${delimiter}${process.env.PATH}`, LIBREDB_BIND_RESOLVER: resolver, HOSTNAME: "", }, diff --git a/tests/unit/document-keydown-listeners.test.ts b/tests/unit/document-keydown-listeners.test.ts index 89b1b4f7f..304c9b97f 100644 --- a/tests/unit/document-keydown-listeners.test.ts +++ b/tests/unit/document-keydown-listeners.test.ts @@ -33,10 +33,13 @@ const sourceFiles = (dir: string): string[] => /** `.addEventListener("keydown", …)`, with the target as written. */ const REGISTRATION = /(\w+)\s*\.addEventListener\(\s*"keydown"/g; +/** Repository-relative and POSIX-spelled, so the sites read the same on Windows as on Linux. */ +const repoRelative = (file: string): string => path.relative(ROOT, file).split(path.sep).join("/"); + const registrations = sourceFiles(SRC) .flatMap((file) => [...readFileSync(file, "utf8").matchAll(REGISTRATION)].map((match) => ({ - file: path.relative(ROOT, file), + file: repoRelative(file), target: match[1], })), ) diff --git a/tests/unit/flatpark-descriptor.test.ts b/tests/unit/flatpark-descriptor.test.ts index 2941275e2..5820948e8 100644 --- a/tests/unit/flatpark-descriptor.test.ts +++ b/tests/unit/flatpark-descriptor.test.ts @@ -81,10 +81,25 @@ describe("flatpark.yml catalog descriptor (#241)", () => { // FlatPark runs this relative to the registry directory. expect(descriptor.update.command).toMatch(/^\.\/[A-Za-z0-9._-]+$/); const resolver = descriptor.update.command.replace(/^\.\//, ""); - const stat = fs.statSync(path.join(DIR, resolver)); - expect(stat.isFile()).toBe(true); - // Any execute bit: FlatPark invokes it directly, not through a shell. - expect(stat.mode & 0o111).toBeGreaterThan(0); + expect(fs.statSync(path.join(DIR, resolver)).isFile()).toBe(true); + + // The exec bit as GIT records it, not as this working tree happens to hold it. + // FlatPark runs what is COMMITTED - it clones the registry - so the index is what + // has to say 100755, and it says so on every platform: a Windows checkout sets + // core.fileMode=false and keeps the recorded mode, while NTFS carries no POSIX mode + // bits for stat to read at all, which is why the mode check that used to stand here + // read 0 on windows-latest for a file that is executable everywhere it matters. + // FlatPark invokes the resolver directly rather than through a shell, so a 100644 + // upstream is an update check that never runs. + const indexed = Bun.spawnSync(["git", "ls-files", "-s", "--", resolver], { + cwd: DIR, + stdout: "pipe", + stderr: "pipe", + }); + expect(indexed.exitCode).toBe(0); + // An untracked resolver prints nothing, so this fails on that too rather than on a + // mode it never read. + expect(indexed.stdout.toString()).toStartWith("100755 "); }); test("stays readable by a line scanner, not just by a YAML parser", () => { diff --git a/tests/unit/helm-chart-agent.test.ts b/tests/unit/helm-chart-agent.test.ts index be1680eeb..99990f2dd 100644 --- a/tests/unit/helm-chart-agent.test.ts +++ b/tests/unit/helm-chart-agent.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * The chart's `agent` block (#331 T8). * diff --git a/tests/unit/helm-chart-auth-cookie-secure.test.ts b/tests/unit/helm-chart-auth-cookie-secure.test.ts index 0afa31865..2d406099e 100644 --- a/tests/unit/helm-chart-auth-cookie-secure.test.ts +++ b/tests/unit/helm-chart-auth-cookie-secure.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * config.authCookieSecure is three-state (backlog N2). * diff --git a/tests/unit/helm-chart-auth-provider.test.ts b/tests/unit/helm-chart-auth-provider.test.ts index 8f5e14fce..4abd7089c 100644 --- a/tests/unit/helm-chart-auth-provider.test.ts +++ b/tests/unit/helm-chart-auth-provider.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * Auth-provider scoping in the chart's secret handling (issue #170). * diff --git a/tests/unit/helm-chart-base-path.test.ts b/tests/unit/helm-chart-base-path.test.ts index 57939415c..f713f091a 100644 --- a/tests/unit/helm-chart-base-path.test.ts +++ b/tests/unit/helm-chart-base-path.test.ts @@ -1,3 +1,4 @@ +// @requires helm import { describe, expect, test } from "bun:test"; import { join } from "node:path"; import { parseAllDocuments } from "yaml"; diff --git a/tests/unit/helm-chart-data-volume.test.ts b/tests/unit/helm-chart-data-volume.test.ts index 5972b777f..38c5fd4cf 100644 --- a/tests/unit/helm-chart-data-volume.test.ts +++ b/tests/unit/helm-chart-data-volume.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * /app/data volume knobs from the Rancher E2E follow-ups (issue #170). * diff --git a/tests/unit/helm-chart-dualstack.test.ts b/tests/unit/helm-chart-dualstack.test.ts index 292b46312..70f9037d8 100644 --- a/tests/unit/helm-chart-dualstack.test.ts +++ b/tests/unit/helm-chart-dualstack.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * Regression tests for the chart's dual-stack Service surface * (service.ipFamilyPolicy / service.ipFamilies), added for #432: @@ -62,7 +63,7 @@ * compared against it too: a hand-edit that reaches only one of the two trees * must fail here as well. */ -import { afterAll, describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -377,14 +378,23 @@ describe("charts/libredb-studio install notes warn about an IPv4-pinned pod (#43 // file's own bytes are wrapped in a named template and emitted as a // ConfigMap. Helm does the rendering, the template text is the shipped one, // and nothing here reimplements the condition under test. - const notesChart = mkdtempSync(join(tmpdir(), "libredb-notes-probe-")); - cpSync(CHART_DIR, notesChart, { recursive: true }); const PROBE_TEMPLATE = "templates/zz-notes-probe.yaml"; - writeFileSync( - join(notesChart, PROBE_TEMPLATE), - `{{- define "notesProbe" -}}\n${readFileSync(join(CHART_DIR, "templates/NOTES.txt"), "utf8")}\n{{- end -}}\n` + - 'apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: notes-probe\ndata:\n notes: {{ include "notesProbe" . | quote }}\n', - ); + // Built in beforeAll rather than in the describe body: a recursive copy of the whole chart is + // real work, and work done at import time is attributed to the file rather than to a hook, so a + // per-file timeout starts counting against it before any test is named. Nothing below reads + // notesChart until a test runs, so the move changes nothing about what is tested. + let notesChart: string; + + beforeAll(() => { + notesChart = mkdtempSync(join(tmpdir(), "libredb-notes-probe-")); + cpSync(CHART_DIR, notesChart, { recursive: true }); + writeFileSync( + join(notesChart, PROBE_TEMPLATE), + `{{- define "notesProbe" -}}\n${readFileSync(join(CHART_DIR, "templates/NOTES.txt"), "utf8")}\n{{- end -}}\n` + + 'apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: notes-probe\ndata:\n notes: {{ include "notesProbe" . | quote }}\n', + ); + }); + afterAll(() => rmSync(notesChart, { recursive: true, force: true })); function notes(args: string[]): string { diff --git a/tests/unit/helm-chart-hardening.test.ts b/tests/unit/helm-chart-hardening.test.ts index 57aa635ab..3df6aa603 100644 --- a/tests/unit/helm-chart-hardening.test.ts +++ b/tests/unit/helm-chart-hardening.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * Regression tests for issue #45: Helm chart hardening items deferred from * the chart-introduction review (#44). diff --git a/tests/unit/helm-chart-openshift.test.ts b/tests/unit/helm-chart-openshift.test.ts index 94e095e82..ef66d27f2 100644 --- a/tests/unit/helm-chart-openshift.test.ts +++ b/tests/unit/helm-chart-openshift.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * Regression tests for the chart-0.1.20 OpenShift and seed-connection * behaviors introduced with the operator PR (#152): @@ -20,8 +21,9 @@ * `--api-versions security.openshift.io/v1`, which feeds * .Capabilities.APIVersions exactly like a live API server would. */ -import { beforeAll, describe, expect, test } from "bun:test"; -import { existsSync, readdirSync } from "node:fs"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { cpSync, existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { parseAllDocuments } from "yaml"; @@ -42,8 +44,11 @@ interface RenderedManifest { }; } -function helmTemplate(args: string[]): { exitCode: number; stdout: string; stderr: string } { - const run = Bun.spawnSync(["helm", "template", "release-under-test", CHART_DIR, ...args], { +function helmTemplate( + args: string[], + chartDir: string = CHART_DIR, +): { exitCode: number; stdout: string; stderr: string } { + const run = Bun.spawnSync(["helm", "template", "release-under-test", chartDir, ...args], { stdout: "pipe", stderr: "pipe", }); @@ -164,18 +169,44 @@ describe("charts/libredb-studio seedConnections source guard (#152)", () => { * fails loudly when that is impossible rather than silently skipping. */ describe("charts/libredb-studio PostgreSQL subchart contracts (#152)", () => { + /** + * A private copy of the chart, because vendoring writes: `helm dependency build` drops + * `charts/postgresql-*.tgz` and rewrites `Chart.lock`. Doing that in `charts/libredb-studio` + * mutated the repository working tree, which a drift guard running beside the suite sees, and + * with several test files rendering that one directory at once, a file reading it mid-write + * gets a render failure that has nothing to do with what it asserts. Helm's repository config + * and cache are redirected into the copy for the same reason: `helm repo add` otherwise edits + * one shared file under the contributor's home directory. + */ + let pgChartDir: string; + let helmHome: string; + beforeAll(() => { + helmHome = mkdtempSync(join(tmpdir(), "libredb-helm-openshift-")); + pgChartDir = join(helmHome, "libredb-studio"); + cpSync(CHART_DIR, pgChartDir, { recursive: true }); + const vendored = - existsSync(join(CHART_DIR, "charts")) && - readdirSync(join(CHART_DIR, "charts")).some((f) => f.startsWith("postgresql-") && f.endsWith(".tgz")); + existsSync(join(pgChartDir, "charts")) && + readdirSync(join(pgChartDir, "charts")).some((f) => f.startsWith("postgresql-") && f.endsWith(".tgz")); if (vendored) { return; } + // Only a clone that has never vendored the subchart reaches the network here. + const helmEnv = { + ...process.env, + HELM_REPOSITORY_CONFIG: join(helmHome, "repositories.yaml"), + HELM_REPOSITORY_CACHE: join(helmHome, "cache"), + }; const repoAdd = Bun.spawnSync( ["helm", "repo", "add", "bitnami", "https://charts.bitnami.com/bitnami", "--force-update"], - { stdout: "pipe", stderr: "pipe" }, + { stdout: "pipe", stderr: "pipe", env: helmEnv }, ); - const depBuild = Bun.spawnSync(["helm", "dependency", "build", CHART_DIR], { stdout: "pipe", stderr: "pipe" }); + const depBuild = Bun.spawnSync(["helm", "dependency", "build", pgChartDir], { + stdout: "pipe", + stderr: "pipe", + env: helmEnv, + }); if (repoAdd.exitCode !== 0 || depBuild.exitCode !== 0) { throw new Error( `could not vendor the postgresql subchart dependency: ${repoAdd.stderr.toString()} ${depBuild.stderr.toString()}`, @@ -183,10 +214,14 @@ describe("charts/libredb-studio PostgreSQL subchart contracts (#152)", () => { } }); + afterAll(() => { + rmSync(helmHome, { recursive: true, force: true }); + }); + const PG_ARGS = ["--set", "postgresql.enabled=true", "--set", "postgresql.auth.password=test-pg-pass"]; function statefulSetSource(args: string[]): string { - const run = helmTemplate([...PG_ARGS, ...args]); + const run = helmTemplate([...PG_ARGS, ...args], pgChartDir); if (run.exitCode !== 0) { throw new Error(`helm template failed (exit ${run.exitCode}): ${run.stderr}`); } diff --git a/tests/unit/helm-chart-persistence.test.ts b/tests/unit/helm-chart-persistence.test.ts index f7240fa03..c9b1c214d 100644 --- a/tests/unit/helm-chart-persistence.test.ts +++ b/tests/unit/helm-chart-persistence.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * Regression test for issue #137: a default Helm install * (persistence.enabled=false) must render a writable mount at /app/data so diff --git a/tests/unit/helm-chart-route.test.ts b/tests/unit/helm-chart-route.test.ts index 517e5bbb4..c0603cbb3 100644 --- a/tests/unit/helm-chart-route.test.ts +++ b/tests/unit/helm-chart-route.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * Regression tests for the Gateway API route surface of the chart * (templates/route.yaml), added by #362 and corrected by #366: diff --git a/tests/unit/helm-chart-totp.test.ts b/tests/unit/helm-chart-totp.test.ts index 23a9426e3..eb31d6e23 100644 --- a/tests/unit/helm-chart-totp.test.ts +++ b/tests/unit/helm-chart-totp.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * TOTP second-factor wiring in the chart. * diff --git a/tests/unit/helm-chart-user-password.test.ts b/tests/unit/helm-chart-user-password.test.ts index c666d2f5f..a96c85163 100644 --- a/tests/unit/helm-chart-user-password.test.ts +++ b/tests/unit/helm-chart-user-password.test.ts @@ -1,3 +1,4 @@ +// @requires helm /** * Regression test for issue #136: the documented minimal Helm install * (only secrets.jwtSecret + secrets.adminPassword) must render cleanly - diff --git a/tests/unit/helm-pin-matrix.test.ts b/tests/unit/helm-pin-matrix.test.ts index 9c1b3f569..5367e6d40 100644 --- a/tests/unit/helm-pin-matrix.test.ts +++ b/tests/unit/helm-pin-matrix.test.ts @@ -1,6 +1,6 @@ /** * Unit tests for the Helm CLI version matrix pinned across the five workflows - * that run `azure/setup-helm` (seven sites in total). + * that run `azure/setup-helm` (eight sites in total). * * This exists because of #434. Its NOTES.txt assertions used * `helm install --dry-run=client`, which is green on Helm 4 (the maintainer's @@ -64,8 +64,14 @@ const HELM_3 = "v3.16.0"; */ const EXPECTED_PINS: Record = { // Required "Unit & Integration Tests" check: spawns `helm template` from the - // ten helm-chart-*.test.ts files. Produces no published byte. + // helm-chart-*.test.ts files. Produces no published byte. "ci.yml:test": HELM_4, + // The same suite on windows-latest and macos-latest, not a required check. + // Same pin as ci.yml:test for the same reason the two suite sites below share + // one: the helm-chart tests assert on what `helm template` renders, so a + // platform leg on a different client would report a difference that is the + // client's, not the platform's. + "ci.yml:test-cross-platform": HELM_4, // Advisory chart lint + a conditional kind `ct install`. Raised on purpose so // chart-testing under Helm 4 is exercised somewhere non-blocking. "ci.yml:helm-lint": HELM_4, @@ -84,8 +90,8 @@ const EXPECTED_PINS: Record = { /** The site whose Helm 3 pin is load-bearing evidence, not an oversight. */ const HELM3_PINNED_SITE = "helm-release.yml:lint-test"; -/** The two jobs that run the helm-touching test suite; #434 was their drift. */ -const SUITE_SITES = ["ci.yml:test", "npm-publish.yml:validate"]; +/** The jobs that run the helm-touching test suite; #434 was their drift. */ +const SUITE_SITES = ["ci.yml:test", "ci.yml:test-cross-platform", "npm-publish.yml:validate"]; interface HelmPin { site: string; @@ -230,9 +236,9 @@ describe("jobCommandLines", () => { }); }); -describe("the seven setup-helm sites", () => { - test("there are exactly seven, and every one is classified", () => { - expect(ALL_PINS).toHaveLength(7); +describe("the eight setup-helm sites", () => { + test("there are exactly eight, and every one is classified", () => { + expect(ALL_PINS).toHaveLength(8); expect([...BY_SITE.keys()].sort()).toEqual(Object.keys(EXPECTED_PINS).sort()); }); @@ -258,12 +264,34 @@ describe("the seven setup-helm sites", () => { }); }); -describe("#434 regression: the two suite-running jobs cannot drift apart", () => { - test("ci.yml:test and npm-publish.yml:validate pin the identical Helm version", () => { +describe("#434 regression: the suite-running jobs cannot drift apart", () => { + test("every job that runs the helm-touching suite pins the identical Helm version", () => { // Asserted against each other, not against a literal: the defect in #434 was // one helm here and another there, whatever the versions happened to be. - const [ci, npm] = SUITE_SITES.map((site) => BY_SITE.get(site)?.version); - expect(ci).toBe(npm as string); + // + // Every site against the first, not the first two against each other. This used + // to destructure `const [ci, npm]`, which was right while there were exactly two + // sites and silently stopped comparing npm-publish.yml:validate the moment a third + // was inserted between them: ci.yml:test was then compared with its own + // cross-platform twin and the release validation job with nothing. + const pins = SUITE_SITES.map((site) => [site, BY_SITE.get(site)?.version] as const); + expect(pins.every(([, version]) => version !== undefined)).toBe(true); + const [, first] = pins[0]!; + expect(pins.filter(([, version]) => version !== first)).toEqual([]); + }); + + test("every job that runs the suite requires the chart tests, so none of them can quietly skip", () => { + // A test file marked `@requires helm` runs only where helm and the built chart dependency are + // there, and is listed as not run elsewhere (tests/runner/requirements.ts). That is the right + // answer on a contributor's machine and the wrong one in CI, where a lost setup-helm step would + // otherwise turn twelve chart test files into a line in a green log. LIBREDB_REQUIRE_HELM=1 makes + // the runner refuse instead, and this holds every suite-running job to setting it. + const unenforced = SUITE_SITES.filter((site) => { + const pin = BY_SITE.get(site); + if (pin === undefined) return true; + return !jobCommandLines(readWorkflow(pin.file), pin.job).some((line) => /LIBREDB_REQUIRE_HELM:\s*"1"/.test(line)); + }); + expect(unenforced).toEqual([]); }); test("reintroducing `helm install --dry-run=client` requires a Helm 4 suite pin", () => { @@ -357,7 +385,7 @@ describe("`helm registry login` targets a bare domain", () => { describe("the release runbook records the split", () => { test("cut-release SKILL.md points at this test as the matrix's enforcement", () => { - // SKILL.md is the single written inventory of the seven sites; without this + // SKILL.md is the single written inventory of the sites; without this // pointer the next reader re-unifies them from the runbook. const skill = readFileSync(join(REPO_ROOT, ".claude/skills/cut-release/SKILL.md"), "utf8"); const mentions = skill.split("\n").filter((line) => line.includes("helm-pin-matrix.test.ts")); diff --git a/tests/unit/instrumentation.test.ts b/tests/unit/instrumentation.test.ts index 8edaca5cb..61f370ad6 100644 --- a/tests/unit/instrumentation.test.ts +++ b/tests/unit/instrumentation.test.ts @@ -235,22 +235,28 @@ describe("instrumentation register()", () => { expect(output).not.toContain("Star the project"); }); - test("logs a warning and keeps boot alive when seeding fails", async () => { - if (process.platform === "win32" || process.getuid?.() === 0) return; // perms not enforceable - process.env.NEXT_RUNTIME = "nodejs"; - process.env.AUTH_BOOTSTRAP = "off"; - const lockedDir = path.join(tmpDir, "locked"); - fs.mkdirSync(lockedDir); - fs.chmodSync(lockedDir, 0o500); - process.env.LIBREDB_EMBEDDED_SAMPLE_PATH = path.join(lockedDir, "sample.libredb"); - const warn = spyOn(logger, "warn").mockImplementation(() => {}); - try { - await expect(register()).resolves.toBeUndefined(); - expect(warn).toHaveBeenCalled(); - expect(String(warn.mock.calls[0]?.[0])).toContain("seeding skipped"); - } finally { - warn.mockRestore(); - fs.chmodSync(lockedDir, 0o700); - } - }); + // The failure is produced by a directory whose mode forbids writing, which nothing can arrange + // on Windows (chmod there only toggles the read-only bit) or as root (mode bits do not apply). + // Named in the title rather than returned early from the body: a bare `return` reports a pass on + // a machine that never ran the assertion, which is the same output a real pass gives. + test.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "logs a warning and keeps boot alive when seeding fails (POSIX non-root only: needs an unwritable directory)", + async () => { + process.env.NEXT_RUNTIME = "nodejs"; + process.env.AUTH_BOOTSTRAP = "off"; + const lockedDir = path.join(tmpDir, "locked"); + fs.mkdirSync(lockedDir); + fs.chmodSync(lockedDir, 0o500); + process.env.LIBREDB_EMBEDDED_SAMPLE_PATH = path.join(lockedDir, "sample.libredb"); + const warn = spyOn(logger, "warn").mockImplementation(() => {}); + try { + await expect(register()).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalled(); + expect(String(warn.mock.calls[0]?.[0])).toContain("seeding skipped"); + } finally { + warn.mockRestore(); + fs.chmodSync(lockedDir, 0o700); + } + }, + ); }); diff --git a/tests/unit/launcher-utils.test.ts b/tests/unit/launcher-utils.test.ts index 61b8916e2..7bf4aa8f7 100644 --- a/tests/unit/launcher-utils.test.ts +++ b/tests/unit/launcher-utils.test.ts @@ -6,6 +6,7 @@ import { afterAll, describe, expect, test } from "bun:test"; import * as fs from "fs"; import * as os from "os"; import * as path from "path"; +import { pathToFileURL } from "url"; import { artifactName, startupUrl, @@ -23,8 +24,17 @@ import { resolveLedgerDir, sha256File, } from "../../bin/lib/launcher-utils.mjs"; +import { describeIf, missingUnixTool, resolveUnixTool } from "../helpers/posix-tools"; const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "launcher-utils-test-")); +/* + Resolved rather than spawned by bare name: `Bun.spawnSync` THROWS ("Executable not found in + $PATH", measured in this worktree) when a name does not resolve, which would take the whole file + down instead of failing one assertion. tar is not POSIX-only here - Windows 11 ships bsdtar as + System32\tar.exe, which both writes the fixture .tar.gz and honours the --strip-components the + launcher passes - so the archive case runs everywhere a tar exists. +*/ +const TAR = resolveUnixTool("tar"); afterAll(() => { fs.rmSync(tempDir, { recursive: true, force: true }); @@ -388,7 +398,7 @@ describe("preservePayloadData", () => { }); }); -describe("extractArchive", () => { +describeIf(missingUnixTool("tar"), "extractArchive", () => { // Release tarballs are packed with a top-level libredb-studio-/ // root (issue #133, scripts/lib/pack-standalone-tarball.sh) instead of a // tarbomb; extractArchive must strip that one path component so the @@ -404,7 +414,7 @@ describe("extractArchive", () => { fs.writeFileSync(path.join(versionedRoot, "nested", "file.txt"), "nested contents"); const tarballPath = path.join(sourceDir, "fixture.tar.gz"); - const result = Bun.spawnSync(["tar", "-czf", tarballPath, "-C", sourceDir, rootName], { + const result = Bun.spawnSync([TAR!, "-czf", tarballPath, "-C", sourceDir, rootName], { stdout: "pipe", stderr: "pipe", }); @@ -622,12 +632,35 @@ describe("launcher startup URL", () => { 'import os from "node:os"; import { syncBuiltinESMExports } from "node:module"; ' + `os.homedir = () => ${JSON.stringify(home)}; syncBuiltinESMExports();`, ); - const run = Bun.spawnSync([node!, "--import", preload, path.join(root, "bin/studio.js"), "--host", host], { - env: { PATH: process.env.PATH }, - stdout: "pipe", - stderr: "pipe", - }); - expect(run.exitCode).toBe(0); + /* + The environment is kept and only what would change the answer is removed. The launcher reads + PORT for the URL it prints and LIBREDB_STUDIO_ARCHIVE to skip the download, so a contributor + who exports either would see this fail for a reason that is not the test's. + + It used to pass `{ PATH: process.env.PATH }`, which drops SystemRoot, windir, TEMP and TMP: a + node child started without SystemRoot can fail to initialise on Windows, and the failure + arrives as a bare non-zero exit code - exactly the opaque shape the assertion above tries to + avoid. + */ + const env = { ...process.env }; + delete env.PORT; + delete env.LIBREDB_STUDIO_ARCHIVE; + /* + The preload goes to `--import` as a file: URL, not as the path it is. `--import` resolves its + value by ESM rules, where an absolute Windows path is not a path at all: it parses as a URL + whose scheme is the drive letter. Measured with node 24.14.0, `--import 'C:\tmp\fixture.mjs'` + dies at startup with ERR_UNSUPPORTED_ESM_URL_SCHEME ("Received protocol 'c:'") before reading + a line of bin/studio.js, while the same argument on Linux fails to parse as a URL and falls + back to a path - which is why this spawned fine everywhere but Windows. + */ + const run = Bun.spawnSync( + [node!, "--import", pathToFileURL(preload).href, path.join(root, "bin/studio.js"), "--host", host], + { env, stdout: "pipe", stderr: "pipe" }, + ); + // The launcher's own stderr rides on the exit code: node reports a startup failure there and + // nowhere else, and "Received: 1" on its own sends the next reader back to a machine they may + // not have. + expect(run.exitCode, `launcher stderr: ${run.stderr.toString()}`).toBe(0); const output = run.stdout.toString(); expect(output).toContain(`Starting LibreDB Studio ${version} on ${url}\n`); expect(output).toContain(`BIND=${host}\n`); diff --git a/tests/unit/lib/agent/config.test.ts b/tests/unit/lib/agent/config.test.ts index a061af1a0..3d1af1550 100644 --- a/tests/unit/lib/agent/config.test.ts +++ b/tests/unit/lib/agent/config.test.ts @@ -432,7 +432,11 @@ describe("resolveAgentAvailability under concurrency", () => { // re-pointing WORKFLOW_LOCAL_DATA_DIR while a probe is in flight — neither of // which the memo can de-duplicate. const alias = path.join(freshLedgerDir(), "alias"); - fs.symlinkSync(dataDir, alias); + // "junction", not the default: fs.symlinkSync with no type creates a FILE symlink on Windows + // even when the target is a directory, and any symlink there needs a privilege a normal shell + // does not hold. A junction is a directory link that needs none and reaches the same inode, + // which is the whole point of the alias here. Ignored on POSIX, where the type is not used. + fs.symlinkSync(dataDir, alias, "junction"); // Hold both probes at the point where each has written its file and neither has // removed it. A probe filename that is constant for the process lifetime cannot @@ -740,7 +744,8 @@ describe("getAgentRuntimeConfig", () => { // ─── operator documentation ───────────────────────────────────────────────── describe(".env.example", () => { - const envExample = fs.readFileSync(path.join(process.cwd(), ".env.example"), "utf8"); + // Anchored to this file rather than to process.cwd(), so the read is correct whoever launches it. + const envExample = fs.readFileSync(path.resolve(import.meta.dir, "../../../..", ".env.example"), "utf8"); test.each([AGENT_ENABLED_ENV, AGENT_WORLD_TARGET_ENV])("documents %s", (key) => { expect(envExample).toContain(key); diff --git a/tests/unit/lib/agent/plan-draft-boundary.test.ts b/tests/unit/lib/agent/plan-draft-boundary.test.ts index ebc5b4ecd..865cef16b 100644 --- a/tests/unit/lib/agent/plan-draft-boundary.test.ts +++ b/tests/unit/lib/agent/plan-draft-boundary.test.ts @@ -24,7 +24,11 @@ import * as path from "node:path"; * instead of a browser. */ -const SOURCE = fs.readFileSync(path.join(process.cwd(), "src/lib/agent/plan-draft.ts"), "utf8"); +// Anchored to this file, not to process.cwd(): this read happens at module scope, so a runner +// that launched the file from anywhere but the repository root would kill it before a test ran. +const ROOT = path.resolve(import.meta.dir, "../../../.."); + +const SOURCE = fs.readFileSync(path.join(ROOT, "src/lib/agent/plan-draft.ts"), "utf8"); /** Every module specifier the file imports, type-only imports included. */ const specifiers = (source: string): readonly string[] => @@ -91,7 +95,7 @@ describe("the plan-draft reader stays reachable from a browser", () => { than a browser. */ const read = (specifier: string) => - fs.readFileSync(path.join(process.cwd(), "src/lib/sql", `${specifier.replace("./", "")}.ts`), "utf8"); + fs.readFileSync(path.join(ROOT, "src/lib/sql", `${specifier.replace("./", "")}.ts`), "utf8"); const closure = new Set(); const pending = ["./statement-splitter"]; @@ -127,7 +131,7 @@ describe("the plan-draft reader stays reachable from a browser", () => { test("the validation half still holds the guard, so the split moved the reader and not the rule", () => { // The other direction of the same boundary: had the guard simply been dropped, this // test file would pass while the drafted statement stopped being classified at all. - const validation = fs.readFileSync(path.join(process.cwd(), "src/lib/agent/plan-statement.ts"), "utf8"); + const validation = fs.readFileSync(path.join(ROOT, "src/lib/agent/plan-statement.ts"), "utf8"); expect(validation).toContain("@/lib/db/operations/statement-guard"); }); diff --git a/tests/unit/lib/api/object-edit-wire.test.ts b/tests/unit/lib/api/object-edit-wire.test.ts index cce4e1a51..7cdb21549 100644 --- a/tests/unit/lib/api/object-edit-wire.test.ts +++ b/tests/unit/lib/api/object-edit-wire.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { existsSync, readFileSync } from "node:fs"; -import { dirname, join, relative } from "node:path"; +import { dirname, join, relative, sep } from "node:path"; import { isObjectEditBuildResponseShape, isObjectEditOutcomeShape, @@ -830,6 +830,9 @@ describe("every host-supplied string is bounded", () => { const REPO_ROOT = join(import.meta.dir, "../../../../"); +/** Repository-relative and POSIX-spelled, so the closure reads the same on Windows as on Linux. */ +const repoRelative = (file: string): string => relative(REPO_ROOT, file).split(sep).join("/"); + /** The file a specifier names, resolved the way the bundler resolves it, or `undefined`. */ function moduleFileOf(specifier: string, fromDirectory: string): string | undefined { const base = specifier.startsWith("@/") @@ -859,7 +862,7 @@ describe("the wire module's import closure stays free of the server", () => { for (const match of readFileSync(file, "utf8").matchAll(/^import\s+(type\s+)?[^;]*?from\s+"([^"]+)";/gm)) { if (match[1] !== undefined) continue; const specifier = match[2] as string; - const named = `${relative(REPO_ROOT, file)} imports ${specifier}`; + const named = `${repoRelative(file)} imports ${specifier}`; if (!specifier.startsWith("@/") && !specifier.startsWith(".")) { external.push(named); continue; @@ -874,7 +877,7 @@ describe("the wire module's import closure stays free of the server", () => { // A control on the walk itself, because an assertion over an empty or one-file closure would // pass for the wrong reason: the two value imports this module's docblock names, and what THEY // reach, must all be in it. - expect([...walked].map((file) => relative(REPO_ROOT, file)).sort()).toEqual([ + expect([...walked].map(repoRelative).sort()).toEqual([ "src/lib/api/error-codes.ts", "src/lib/api/object-edit-wire.ts", "src/lib/db/errors.ts", diff --git a/tests/unit/lib/auth-bootstrap.test.ts b/tests/unit/lib/auth-bootstrap.test.ts index 3af6f7e48..9215a8876 100644 --- a/tests/unit/lib/auth-bootstrap.test.ts +++ b/tests/unit/lib/auth-bootstrap.test.ts @@ -57,11 +57,17 @@ describe("auth-bootstrap bootstrapAuth()", () => { expect(resolveBootstrapPath()).toBe(path.join(tmpDir, BOOTSTRAP_FILE_NAME)); }); - test("persists with owner-only file mode", () => { - if (process.platform === "win32") return; // chmod is a no-op on Windows - bootstrapAuth(); - expect(fs.statSync(resolveBootstrapPath()).mode & 0o777).toBe(0o600); - }); + // Named in the title rather than returned early from the body, the way + // tests/unit/instrumentation.test.ts does it: a bare `return` reports a pass on a + // machine that never ran the assertion, which is the same output a real pass gives, + // and the runner lists a skip by its title while it cannot see an early return at all. + test.skipIf(process.platform === "win32")( + "persists with owner-only file mode (POSIX only: NTFS has no mode bits)", + () => { + bootstrapAuth(); + expect(fs.statSync(resolveBootstrapPath()).mode & 0o777).toBe(0o600); + }, + ); test("reuses persisted credentials across restarts instead of regenerating", () => { bootstrapAuth(); @@ -196,18 +202,20 @@ describe("auth-bootstrap bootstrapAuth()", () => { expect(readStored().jwtSecret).toBe(process.env.JWT_SECRET!); }); - test("fails open when the data dir is not writable: no throw, no injection", () => { - if (process.platform === "win32" || process.getuid?.() === 0) return; // perms not enforceable - fs.mkdirSync(tmpDir, { recursive: true }); - fs.chmodSync(tmpDir, 0o500); - try { - expect(() => bootstrapAuth()).not.toThrow(); - expect(process.env.JWT_SECRET).toBeUndefined(); - expect(process.env.ADMIN_PASSWORD).toBeUndefined(); - } finally { - fs.chmodSync(tmpDir, 0o700); - } - }); + test.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "fails open when the data dir is not writable: no throw, no injection (POSIX non-root only: needs an unwritable directory)", + () => { + fs.mkdirSync(tmpDir, { recursive: true }); + fs.chmodSync(tmpDir, 0o500); + try { + expect(() => bootstrapAuth()).not.toThrow(); + expect(process.env.JWT_SECRET).toBeUndefined(); + expect(process.env.ADMIN_PASSWORD).toBeUndefined(); + } finally { + fs.chmodSync(tmpDir, 0o700); + } + }, + ); test("prints the password in a banner on generation, but not on reuse", () => { const log = spyOn(console, "log").mockImplementation(() => {}); diff --git a/tests/unit/lib/auth-jwt-config.test.ts b/tests/unit/lib/auth-jwt-config.test.ts index 98527acd8..6044f3461 100644 --- a/tests/unit/lib/auth-jwt-config.test.ts +++ b/tests/unit/lib/auth-jwt-config.test.ts @@ -5,7 +5,7 @@ import { AuthConfigError } from "@/lib/auth-errors"; // module imports cleanly in the test runtime (signJWT itself never uses it). // The mock replaces the whole module, so every import auth.ts makes must appear // here - a missing name is a link-time "Export named 'x' not found" that fails -// the file in isolation (which is how tests/run-core.sh runs it). +// the whole file, which is the unit the runner works in: one bun process each. mock.module("next/headers", () => ({ cookies: async () => ({ get: () => undefined, set: () => {}, delete: () => {} }), headers: async () => ({ get: () => null }), diff --git a/tests/unit/lib/lazy.test.ts b/tests/unit/lib/lazy.test.ts index 1941abf08..3bf114363 100644 --- a/tests/unit/lib/lazy.test.ts +++ b/tests/unit/lib/lazy.test.ts @@ -37,8 +37,12 @@ describe("lazyRetry", () => { throw new Error(`attempt ${calls}`); }); - expect(load()).rejects.toThrow("attempt 2"); - await Bun.sleep(600); + // Awaited, and no sleep. `Bun.sleep(600)` was a bet that the loader's own 400ms retry delay + // had elapsed, with 200ms of margin that one bun process per CPU spends; and the assertion + // above was never awaited, so a rejection that arrived late or never was not asserted at all. + // The returned promise settles only after the SECOND attempt has failed, so awaiting it is + // both the wait and the fact. + await expect(load()).rejects.toThrow("attempt 2"); expect(calls).toBe(2); }); }); diff --git a/tests/unit/lib/saved-query-import.test.ts b/tests/unit/lib/saved-query-import.test.ts index d0df47b06..9624e9022 100644 --- a/tests/unit/lib/saved-query-import.test.ts +++ b/tests/unit/lib/saved-query-import.test.ts @@ -1,4 +1,6 @@ import { describe, expect, test } from "bun:test"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; import { parseSavedQueries } from "@/lib/saved-query-import"; import type { SavedQuery } from "@/lib/types"; @@ -22,11 +24,19 @@ describe("parseSavedQueries", () => { globalThis.Function = new Proxy(Function, { construct() { evaluations++; throw new EvalError("CSP blocks eval"); }, }); - const { parseSavedQueries } = await import(${JSON.stringify(import.meta.dir + "/../../../src/lib/saved-query-import.ts")}); + const { parseSavedQueries } = await import(${JSON.stringify(pathToFileURL(resolve(import.meta.dir, "../../../src/lib/saved-query-import.ts")).href)}); const rows = parseSavedQueries(${JSON.stringify(JSON.stringify([query]))}); process.stdout.write(JSON.stringify({ evaluations, name: rows[0].name, date: rows[0].createdAt instanceof Date })); `; - const processResult = Bun.spawnSync([process.execPath, "-e", script], { stdout: "pipe", stderr: "pipe" }); + // A file: URL, not a bare path: a Windows absolute path ("C:\\...") reads "C:" as a URL + // scheme in an ESM specifier. cwd is pinned to the repository root because the module under + // import resolves "@/lib/db/compatibility", and that alias comes from the tsconfig.json found + // from the child's working directory, not from the importing file. + const processResult = Bun.spawnSync([process.execPath, "-e", script], { + cwd: resolve(import.meta.dir, "../../.."), + stdout: "pipe", + stderr: "pipe", + }); expect(new TextDecoder().decode(processResult.stderr)).toBe(""); expect(processResult.exitCode).toBe(0); expect(JSON.parse(new TextDecoder().decode(processResult.stdout))).toEqual({ diff --git a/tests/unit/loop-scripts.test.ts b/tests/unit/loop-scripts.test.ts index 7db597728..566438fda 100644 --- a/tests/unit/loop-scripts.test.ts +++ b/tests/unit/loop-scripts.test.ts @@ -14,22 +14,55 @@ * pipeline fixture stubs the agent command (loop.sh's generic-agent path) so no * real model runs. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, expect, test } from "bun:test"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { MISSING_POSIX_FILE_MODES, describeIf, missingPosixShell, posixShell } from "../helpers/posix-tools"; const REPO_ROOT = join(import.meta.dir, "../.."); const NEW_MILESTONE = join(REPO_ROOT, "loop/scripts/new-milestone.sh"); const LOOP_SH = join(REPO_ROOT, "loop/scripts/loop.sh"); const PIPELINE_SH = join(REPO_ROOT, "loop/scripts/pipeline.sh"); +/* + The maintainer loop is bash all the way down, and the pipeline fixture hands loop.sh a stub agent + it makes runnable with chmod 0755 and reaches through `eval "$AGENT_CMD"`. Windows has no exec + bit, cannot exec an extension-less #! file, and `Bun.spawnSync(["bash", ...])` THROWS there + ("Executable not found in $PATH", measured in this worktree) or resolves to WSL's Linux bash, + which cannot see the Win32 fixture root. These are maintainer scripts rather than product code, so + a Windows contributor loses nothing by the skip - as long as it says so. +*/ +const SHELL = posixShell("bash"); +const CANNOT_RUN = missingPosixShell("bash") ?? MISSING_POSIX_FILE_MODES; const fixtureRoots: string[] = []; +let emptyConfig: string | null = null; afterEach(() => { for (const root of fixtureRoots.splice(0)) rmSync(root, { recursive: true, force: true }); + emptyConfig = null; }); +/** + * A real, empty config file for GIT_CONFIG_GLOBAL/SYSTEM, in its own directory so it is never inside + * a fixture repository and never seen by `git add -A`. + * + * Without it the fixture repo inherits the contributor's own git config: `commit.gpgsign=true` (a + * common macOS setup) makes the fixture commit prompt or fail, and `core.hooksPath` pointing at a + * missing directory does the same, so makePipelineFixture throws "fixture git setup failed" for + * every pipeline test on a machine where nothing is wrong. tests/unit/sync-chart-version.test.ts + * already isolates git this way for the same reason. + */ +function emptyConfigPath(): string { + if (emptyConfig === null) { + const dir = mkdtempSync(join(tmpdir(), "loop-gitconfig-")); + fixtureRoots.push(dir); + emptyConfig = join(dir, "empty.gitconfig"); + writeFileSync(emptyConfig, ""); + } + return emptyConfig; +} + function write(root: string, rel: string, content: string): void { mkdirSync(dirname(join(root, rel)), { recursive: true }); writeFileSync(join(root, rel), content); @@ -42,8 +75,18 @@ function read(root: string, rel: string): string { function run(cmd: string[], cwd?: string) { // Fixtures must be hermetic. LOOP_ENV_FILE is a real input to loop.sh/pipeline.sh, // so an inherited one (a shell that ran a stage by hand) would point a fixture at - // the LIVE loop config instead of its stub agent. - const env = { ...process.env }; + // the LIVE loop config instead of its stub agent. The git identity and the empty + // config go to every command, not just the fixture's own: pipeline.sh runs `git + // status` and `git branch` itself, so it must see the same isolated git. + const env: Record = { + ...process.env, + GIT_CONFIG_GLOBAL: emptyConfigPath(), + GIT_CONFIG_SYSTEM: emptyConfigPath(), + GIT_AUTHOR_NAME: "fixture", + GIT_AUTHOR_EMAIL: "fixture@test", + GIT_COMMITTER_NAME: "fixture", + GIT_COMMITTER_EMAIL: "fixture@test", + }; delete env.LOOP_ENV_FILE; return Bun.spawnSync(cmd, { cwd, env, stdout: "pipe", stderr: "pipe" }); } @@ -198,10 +241,10 @@ function makeFreshFixture(): string { return root; } -describe("loop/scripts/new-milestone.sh", () => { +describeIf(CANNOT_RUN, "loop/scripts/new-milestone.sh", () => { test("archives the previous milestone and resets the working set into .loop/", () => { const root = makeMilestoneFixture(); - const result = run(["bash", NEW_MILESTONE, "sweep-3", root]); + const result = run([SHELL!, NEW_MILESTONE, "sweep-3", root]); expect(result.exitCode).toBe(0); // Previous milestone name derived from the sentinel; whole set archived under .loop/. @@ -241,7 +284,7 @@ describe("loop/scripts/new-milestone.sh", () => { test("seeds .loop/ from the templates on a fresh repo (no prior state, no archive)", () => { const root = makeFreshFixture(); - const result = run(["bash", NEW_MILESTONE, "sweep-1", root]); + const result = run([SHELL!, NEW_MILESTONE, "sweep-1", root]); expect(result.exitCode).toBe(0); // Live working set created from templates. @@ -262,14 +305,14 @@ describe("loop/scripts/new-milestone.sh", () => { test("rejects a non-kebab-case milestone name", () => { const root = makeMilestoneFixture(); - const result = run(["bash", NEW_MILESTONE, "Sweep_3", root]); + const result = run([SHELL!, NEW_MILESTONE, "Sweep_3", root]); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain("kebab-case"); }); test("refuses to reopen the current milestone name", () => { const root = makeMilestoneFixture(); - const result = run(["bash", NEW_MILESTONE, "sweep-2", root]); + const result = run([SHELL!, NEW_MILESTONE, "sweep-2", root]); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain("already the current one"); }); @@ -277,7 +320,7 @@ describe("loop/scripts/new-milestone.sh", () => { test("refuses to overwrite an existing archive", () => { const root = makeMilestoneFixture(); mkdirSync(join(root, ".loop/archive/sweep-2"), { recursive: true }); - const result = run(["bash", NEW_MILESTONE, "sweep-3", root]); + const result = run([SHELL!, NEW_MILESTONE, "sweep-3", root]); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain("refusing to overwrite"); }); @@ -341,10 +384,13 @@ function makePipelineFixture({ planningViolation = false } = {}): string { chmodSync(join(root, "loop/scripts/loop.sh"), 0o755); chmodSync(join(root, "loop/scripts/pipeline.sh"), 0o755); + // The identity comes from GIT_AUTHOR_*/GIT_COMMITTER_* in run(), alongside the empty + // GIT_CONFIG_GLOBAL/SYSTEM: `-c user.email` set an identity but left the rest of the + // contributor's config in force, which is the half that breaks the commit. for (const cmd of [ ["git", "init", "--quiet", "--initial-branch=main"], - ["git", "-c", "user.email=t@t", "-c", "user.name=t", "add", "-A"], - ["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "--quiet", "-m", "fixture"], + ["git", "add", "-A"], + ["git", "commit", "--quiet", "-m", "fixture"], ["git", "checkout", "--quiet", "-b", "loop/fixture-run"], ]) { const r = run(cmd, root); @@ -353,10 +399,10 @@ function makePipelineFixture({ planningViolation = false } = {}): string { return root; } -describe("loop/scripts/pipeline.sh", () => { +describeIf(CANNOT_RUN, "loop/scripts/pipeline.sh", () => { test("runs triage, planning and build in order and exits 0", () => { const root = makePipelineFixture(); - const result = run(["bash", join(root, "loop/scripts/pipeline.sh"), "3", "3"], root); + const result = run([SHELL!, join(root, "loop/scripts/pipeline.sh"), "3", "3"], root); expect(result.exitCode).toBe(0); expect(result.stdout.toString()).toContain("pipeline COMPLETE"); @@ -374,7 +420,7 @@ describe("loop/scripts/pipeline.sh", () => { // agent - observed as a fixture case hanging for minutes inside the loop's own // gate, with a stray iteration running in the fixture repo. const root = makePipelineFixture(); - const result = run(["bash", join(root, "loop/scripts/pipeline.sh"), "3", "3"], root); + const result = run([SHELL!, join(root, "loop/scripts/pipeline.sh"), "3", "3"], root); expect(result.exitCode).toBe(0); const seen = read(root, ".loop/stub-env.log").trim().split("\n"); @@ -384,7 +430,7 @@ describe("loop/scripts/pipeline.sh", () => { test("aborts when planning creates the completion marker (contract violation)", () => { const root = makePipelineFixture({ planningViolation: true }); - const result = run(["bash", join(root, "loop/scripts/pipeline.sh"), "3", "3"], root); + const result = run([SHELL!, join(root, "loop/scripts/pipeline.sh"), "3", "3"], root); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain("planning must never complete"); // Build never ran. @@ -394,7 +440,7 @@ describe("loop/scripts/pipeline.sh", () => { test("refuses a dirty working tree", () => { const root = makePipelineFixture(); write(root, "uncommitted.txt", "dirty"); - const result = run(["bash", join(root, "loop/scripts/pipeline.sh")], root); + const result = run([SHELL!, join(root, "loop/scripts/pipeline.sh")], root); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain("not clean"); }); @@ -402,7 +448,7 @@ describe("loop/scripts/pipeline.sh", () => { test("refuses to run on main", () => { const root = makePipelineFixture(); run(["git", "checkout", "--quiet", "main"], root); - const result = run(["bash", join(root, "loop/scripts/pipeline.sh")], root); + const result = run([SHELL!, join(root, "loop/scripts/pipeline.sh")], root); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain("dedicated loop branch"); }); diff --git a/tests/unit/merge-lcov.test.ts b/tests/unit/merge-lcov.test.ts index 8d770283a..147a10bd9 100644 --- a/tests/unit/merge-lcov.test.ts +++ b/tests/unit/merge-lcov.test.ts @@ -101,3 +101,50 @@ describe("merge-lcov authority-universe rule", () => { expect(merged.get(5)).toBe(0); }); }); + +describe("merge-lcov on Windows", () => { + test("a backslash SF path is the same file as its forward-slash spelling", () => { + // bun writes SF: with the host separator, so a Windows contributor running + // `bun run test:coverage` produces `SF:src\virtual\win.tsx`. Without this + // normalisation the two spellings merge as two files, and the `src/` filter + // at the end of the script drops both, leaving an empty report that + // check-coverage rejects for a reason that names nothing real. + const windows = lcov("src\\virtual\\win.tsx", [ + [1, 4], + [7, 0], + ]); + const posix = lcov("src/virtual/win.tsx", [[7, 2]]); + + const merged = runMerge("separators", [windows, posix]); + expect([...merged.keys()]).toEqual(["src/virtual/win.tsx"]); + expect(merged.get("src/virtual/win.tsx")!.get(7)).toBe(2); + }); +}); + +describe("merge-lcov input manifest", () => { + test("--inputs-from reads the input list from a file", () => { + // The test runner passes 500-odd reports, and Windows caps a command line at + // 32767 characters, so the list travels in a file instead of in argv. + const first = path.join(workDir, "manifest-in-0.info"); + const second = path.join(workDir, "manifest-in-1.info"); + writeFileSync(first, lcov("src/virtual/manifest.ts", [[1, 1]])); + writeFileSync(second, lcov("src/virtual/manifest.ts", [[2, 3]])); + const manifest = path.join(workDir, "manifest.txt"); + writeFileSync(manifest, `${first}\n${second}\n`); + const outPath = path.join(workDir, "manifest-out.info"); + + const result = Bun.spawnSync(["node", SCRIPT, `--inputs-from=${manifest}`, outPath]); + expect(result.exitCode).toBe(0); + expect(readFileSync(outPath, "utf8")).toContain("SF:src/virtual/manifest.ts"); + expect(result.stdout.toString()).toContain("Merged 2 LCOV file(s)"); + }); + + test("an empty manifest is an error, not an empty report", () => { + const manifest = path.join(workDir, "empty-manifest.txt"); + writeFileSync(manifest, "\n\n"); + + const result = Bun.spawnSync(["node", SCRIPT, `--inputs-from=${manifest}`, path.join(workDir, "empty-out.info")]); + expect(result.exitCode).toBe(1); + expect(result.stderr.toString()).toContain(manifest); + }); +}); diff --git a/tests/unit/operator-catalog-submission.test.ts b/tests/unit/operator-catalog-submission.test.ts index bb3e696b9..701843757 100644 --- a/tests/unit/operator-catalog-submission.test.ts +++ b/tests/unit/operator-catalog-submission.test.ts @@ -16,7 +16,7 @@ * FBC side. A skipRange satisfies neither. */ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { parse } from "yaml"; @@ -491,8 +491,11 @@ describe("CLI", () => { return `http://127.0.0.1:${server.port}`; } - async function run(args: string[]) { - const proc = Bun.spawn(["node", join(import.meta.dir, "../../scripts/operator-catalog-submission.mjs"), ...args], { + /** The one path the harness spawns, so the copy below cannot drift from it. */ + const CLI = join(import.meta.dir, "../../scripts/operator-catalog-submission.mjs"); + + async function runScript(script: string, args: string[]) { + const proc = Bun.spawn(["node", script, ...args], { env: { ...process.env, GITHUB_TOKEN: "" }, stdout: "pipe", stderr: "pipe", @@ -505,6 +508,40 @@ describe("CLI", () => { return { stdout, stderr, exitCode }; } + const run = (args: string[]) => runScript(CLI, args); + + /** + * The entry-point guard is the CLI's on/off switch, and it fails silently: + * when it reads "imported", node loads the module, runs nothing, exits 0 and + * prints not a line, so the workflow step that asked for a decision gets a + * green run and an empty GITHUB_OUTPUT. It compared `import.meta.url` against + * `file://${process.argv[1]}` - a URL against a path - which holds only while + * the path needs no encoding and already uses forward slashes. + * + * Measured on windows-latest: every CLI case in this file got exit 0 and an + * empty stdout, because argv[1] arrives as + * `D:\a\libredb-studio\libredb-studio\scripts\operator-catalog-submission.mjs` + * while the URL holds `file:///D:/a/...`. The same defect is reachable from a + * POSIX machine, which is what this drives: a directory name with a space is + * percent-encoded in the URL and not in argv[1]. + */ + test("runs when its own path needs URL encoding, where comparing a URL to a path stops", async () => { + // realpath'd because tmpdir() is /var/folders/... on macOS and /var is a + // symlink to /private/var: an unresolved path would make this fail for a + // second reason that is not under test (measured in + // tests/unit/docker-bind-address.test.ts, same guard, same trap). + const root = realpathSync(mkdtempSync(join(tmpdir(), "operator cli-"))); + roots.push(root); + const copy = join(root, "operator-catalog-submission.mjs"); + // A copy rather than a link: the script imports node builtins only, so it + // runs from anywhere, and a link would resolve back to the unencoded path. + copyFileSync(CLI, copy); + + const result = await runScript(copy, ["publish"]); + expect(result.exitCode).toBe(2); + expect(result.stderr).toMatch(/unknown command/); + }); + test("prints the outputs a workflow step reads when a submission is due", async () => { const result = await run([ "decide", diff --git a/tests/unit/packaging-bind-address.test.ts b/tests/unit/packaging-bind-address.test.ts index 9a9aebe0d..a345deb59 100644 --- a/tests/unit/packaging-bind-address.test.ts +++ b/tests/unit/packaging-bind-address.test.ts @@ -4,10 +4,24 @@ * real subprocess against a stub "node" binary that only echoes the * HOSTNAME it was started with - no real server ever starts. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, expect, test } from "bun:test"; import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { MISSING_POSIX_FILE_MODES, describeIf, missingPosixShell, posixShell } from "../helpers/posix-tools"; + +/* + Both wrappers are /bin/sh files shipped in the .deb/.rpm and in the Homebrew formula, and each is + exercised against a stub `node` that is itself a `#!/bin/sh` script made runnable with chmod 0755. + Windows has neither: the shell is not on a PowerShell PATH (`Bun.spawnSync(["sh", ...])` THROWS + "Executable not found in $PATH", measured in this worktree), NTFS carries no exec bit, and + CreateProcess cannot exec an extension-less #! file. There is no Windows artifact behind these + tests, so the skip names that rather than pretending the suite covered it. +*/ +const SH = posixShell("sh"); +const BASH = posixShell("bash"); +const NO_SH = missingPosixShell("sh") ?? MISSING_POSIX_FILE_MODES; +const NO_BASH = missingPosixShell("bash") ?? MISSING_POSIX_FILE_MODES; const STUB_NODE_SCRIPT = '#!/bin/sh\necho "HOSTNAME=$HOSTNAME"\n'; /** Looks like what Docker exports as HOSTNAME for every container process. */ @@ -21,7 +35,7 @@ function writeStubNode(binDir: string): string { return nodePath; } -describe("packaging/linux/libredb-studio bind address (#134)", () => { +describeIf(NO_SH, "packaging/linux/libredb-studio bind address (#134)", () => { const WRAPPER = join(import.meta.dir, "../../packaging/linux/libredb-studio"); const fixtureRoots: string[] = []; @@ -34,7 +48,7 @@ describe("packaging/linux/libredb-studio bind address (#134)", () => { fixtureRoots.push(home); writeStubNode(join(home, "node/bin")); writeFileSync(join(home, "server.js"), ""); - return Bun.spawnSync(["sh", WRAPPER], { + return Bun.spawnSync([SH!, WRAPPER], { env: { ...process.env, LIBREDB_STUDIO_HOME: home, @@ -73,7 +87,7 @@ describe("packaging/linux/libredb-studio bind address (#134)", () => { }); }); -describe("packaging/homebrew/libredb-studio.rb.tmpl bind address (#134)", () => { +describeIf(NO_BASH, "packaging/homebrew/libredb-studio.rb.tmpl bind address (#134)", () => { const template = readFileSync(join(import.meta.dir, "../../packaging/homebrew/libredb-studio.rb.tmpl"), "utf8"); const heredocMatch = /\(bin\/"libredb-studio"\)\.write <<~SCRIPT\n([\s\S]*?)\n\s*SCRIPT\b/.exec(template); if (!heredocMatch) throw new Error('could not locate the bin/"libredb-studio" heredoc in the Homebrew template'); @@ -94,7 +108,7 @@ describe("packaging/homebrew/libredb-studio.rb.tmpl bind address (#134)", () => const script = rawScript .replaceAll('#{Formula["node@24"].opt_bin}/node', nodePath) .replaceAll("#{libexec}/server.js", serverPath); - return Bun.spawnSync(["bash", "-c", script], { + return Bun.spawnSync([BASH!, "-c", script], { env: { ...process.env, HOME: dir, HOSTNAME: "", LIBREDB_BIND: "", ...env }, stdout: "pipe", stderr: "pipe", diff --git a/tests/unit/packaging-homebrew-datadir.test.ts b/tests/unit/packaging-homebrew-datadir.test.ts index 6a92cf245..29614db23 100644 --- a/tests/unit/packaging-homebrew-datadir.test.ts +++ b/tests/unit/packaging-homebrew-datadir.test.ts @@ -4,10 +4,21 @@ * `bin/"libredb-studio"` heredoc in the .rb.tmpl) as a subprocess against a * stub "node" that only echoes STORAGE_SQLITE_PATH - no real server starts. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, expect, test } from "bun:test"; import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { MISSING_POSIX_FILE_MODES, describeIf, missingPosixShell, posixShell } from "../helpers/posix-tools"; + +/* + A Homebrew formula's wrapper, run against a stub `node` that is a `#!/bin/sh` script made runnable + with chmod 0755. Homebrew has no Windows edition, and Windows has neither the shell on a + PowerShell PATH (`Bun.spawnSync(["bash", ...])` THROWS "Executable not found in $PATH", measured + in this worktree) nor a mode bit for the stub, so the skip names the artifact instead of leaving a + Windows contributor to read a spawn error as a defect in the formula. +*/ +const BASH = posixShell("bash"); +const CANNOT_RUN = missingPosixShell("bash") ?? MISSING_POSIX_FILE_MODES; const STUB_NODE_SCRIPT = '#!/bin/sh\necho "STORAGE_SQLITE_PATH=$STORAGE_SQLITE_PATH"\n'; @@ -19,7 +30,7 @@ function writeStubNode(binDir: string): string { return nodePath; } -describe("packaging/homebrew/libredb-studio.rb.tmpl data dir (#135)", () => { +describeIf(CANNOT_RUN, "packaging/homebrew/libredb-studio.rb.tmpl data dir (#135)", () => { const template = readFileSync(join(import.meta.dir, "../../packaging/homebrew/libredb-studio.rb.tmpl"), "utf8"); const heredocMatch = /\(bin\/"libredb-studio"\)\.write <<~SCRIPT\n([\s\S]*?)\n\s*SCRIPT\b/.exec(template); if (!heredocMatch) throw new Error('could not locate the bin/"libredb-studio" heredoc in the Homebrew template'); @@ -44,7 +55,7 @@ describe("packaging/homebrew/libredb-studio.rb.tmpl data dir (#135)", () => { .replaceAll('#{Formula["node@24"].opt_bin}/node', nodePath) .replaceAll("#{libexec}/server.js", serverPath) .replaceAll("#{var}/libredb-studio/libredb-storage.db", join(brewVar, "libredb-studio/libredb-storage.db")); - const result = Bun.spawnSync(["bash", "-c", script], { + const result = Bun.spawnSync([BASH!, "-c", script], { env: { ...process.env, HOME: dir, HOSTNAME: "", LIBREDB_BIND: "", STORAGE_SQLITE_PATH: "", ...env }, stdout: "pipe", stderr: "pipe", diff --git a/tests/unit/packaging-payload-prune.test.ts b/tests/unit/packaging-payload-prune.test.ts index 08959a1bc..d8d063186 100644 --- a/tests/unit/packaging-payload-prune.test.ts +++ b/tests/unit/packaging-payload-prune.test.ts @@ -8,12 +8,22 @@ * subprocess against a fixture payload dir - no full `bun run build` * needed, since the helper only prunes an already-assembled payload. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, expect, test } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { describeIfPosixShell, posixShell } from "../helpers/posix-tools"; const SCRIPT = join(import.meta.dir, "../../scripts/lib/prune-standalone-payload.sh"); +/* + Resolved rather than spawned by bare name. `Bun.spawnSync(["bash", ...])` THROWS ("Executable not + found in $PATH", measured in this worktree) where there is no bash, and in a PowerShell session + with WSL installed the bare name resolves to C:\Windows\System32\bash.exe - a Linux shell that + cannot stat the Win32 fixture path this passes it, so the script would refuse with "not found" and + the test would read that as a prune failure. Nothing else here is platform-bound: the script is + the one the release-artifacts workflow already runs under Git Bash on windows-latest. +*/ +const SHELL = posixShell("bash"); /** Runtime files the prune must never remove (mirrors the build script's * payload assembly - including the hidden .next dir: see the snap 0.9.52 @@ -111,7 +121,7 @@ const EXTRA_FILES = [ "local-cert.pem", ]; -describe("scripts/lib/prune-standalone-payload.sh (#124)", () => { +describeIfPosixShell("bash", "scripts/lib/prune-standalone-payload.sh (#124)", () => { const fixtureRoots: string[] = []; afterEach(() => { @@ -130,7 +140,7 @@ describe("scripts/lib/prune-standalone-payload.sh (#124)", () => { } function runPrune(...args: string[]) { - return Bun.spawnSync(["bash", SCRIPT, ...args], { stdout: "pipe", stderr: "pipe" }); + return Bun.spawnSync([SHELL!, SCRIPT, ...args], { stdout: "pipe", stderr: "pipe" }); } test("removes the repo-root extras from the payload root", () => { diff --git a/tests/unit/packaging-postinstall-restart.test.ts b/tests/unit/packaging-postinstall-restart.test.ts index 5708f940e..76e9fdab5 100644 --- a/tests/unit/packaging-postinstall-restart.test.ts +++ b/tests/unit/packaging-postinstall-restart.test.ts @@ -9,18 +9,29 @@ * systemd probe at a temp directory, so the outcome never depends on whether * the host running the tests is booted with systemd. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, expect, test } from "bun:test"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { delimiter, join } from "node:path"; +import { MISSING_POSIX_FILE_MODES, describeIf, missingPosixShell, posixShell } from "../helpers/posix-tools"; const SCRIPT = join(import.meta.dir, "../../packaging/linux/scripts/postinstall.sh"); +/* + A .deb/.rpm maintainer script, driven through a stub `systemctl` that is a `#!/bin/sh` file made + runnable with chmod 0755 and found through PATH. Neither the package format nor systemd nor the + POSIX exec bit exists on Windows, and `Bun.spawnSync(["sh", ...])` THROWS there ("Executable not + found in $PATH", measured in this worktree) rather than returning a non-zero exit code, so the + skip names the artifact. macOS runs it unchanged: the script is plain POSIX sh and systemd is + stubbed. +*/ +const SHELL = posixShell("sh"); +const CANNOT_RUN = missingPosixShell("sh") ?? MISSING_POSIX_FILE_MODES; function stubSystemctl(exitCode: number, logPath: string): string { return `#!/bin/sh\nprintf '%s\\n' "$*" >> ${JSON.stringify(logPath)}\nexit ${exitCode}\n`; } -describe("packaging/linux/scripts/postinstall.sh service restart", () => { +describeIf(CANNOT_RUN, "packaging/linux/scripts/postinstall.sh service restart", () => { const fixtureRoots: string[] = []; afterEach(() => { @@ -49,10 +60,10 @@ describe("packaging/linux/scripts/postinstall.sh service restart", () => { const runtimeDir = join(root, "run-systemd-system"); if (systemd) mkdirSync(runtimeDir, { recursive: true }); - const result = Bun.spawnSync(["sh", SCRIPT, arg], { + const result = Bun.spawnSync([SHELL!, SCRIPT, arg], { env: { ...process.env, - PATH: `${binDir}:${process.env.PATH ?? ""}`, + PATH: `${binDir}${delimiter}${process.env.PATH ?? ""}`, LIBREDB_SYSTEMD_RUNTIME_DIR: runtimeDir, }, stdout: "pipe", diff --git a/tests/unit/packaging-standalone-tarball.test.ts b/tests/unit/packaging-standalone-tarball.test.ts index 9503c0d95..d2fa0d4a2 100644 --- a/tests/unit/packaging-standalone-tarball.test.ts +++ b/tests/unit/packaging-standalone-tarball.test.ts @@ -6,15 +6,45 @@ * as a subprocess against a small fixture payload dir - no full `bun run * build` needed, since that script only wraps an already-assembled payload. */ -import { afterEach, describe, expect, test } from "bun:test"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { afterEach, expect, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { + describeIf, + missingPosixShell, + missingUnixTool, + posixShell, + resolveUnixTool, + testIf, +} from "../helpers/posix-tools"; const SCRIPT = join(import.meta.dir, "../../scripts/lib/pack-standalone-tarball.sh"); const VERSION = "9.9.9"; -describe("scripts/lib/pack-standalone-tarball.sh (#133)", () => { +/* + Both tools are resolved rather than spawned by bare name: `Bun.spawnSync` THROWS ("Executable not + found in $PATH", measured in this worktree) when a name does not resolve, and in a PowerShell + session `bash` either is absent or is WSL's Linux shell, which cannot stat the Win32 temp path + this hands it. The script itself packs with tar, so listing with tar adds no dependency the + artifact does not already have; Windows 11 ships bsdtar in System32 and Git for Windows ships GNU + tar, and both read the .tar.gz this produces. +*/ +const SHELL = posixShell("bash"); +const TAR = resolveUnixTool("tar"); +const CANNOT_PACK = missingPosixShell("bash") ?? missingUnixTool("tar"); + +/* + A colon is an ordinary character in a POSIX file name and an impossible one on Windows: NTFS reads + it as the alternate data stream separator, so `out:1.tar.gz` is not a name Win32 can hold and + existsSync would go looking for a stream on a file called `out`. Platform rather than a probe, for + the reason posix-tools gives for MISSING_POSIX_FILE_MODES: a probe that answered "no" on Linux + would turn a real regression into a skip. +*/ +const MISSING_COLON_FILE_NAMES: string | null = + process.platform === "win32" ? "colon in a file name: NTFS reads ':' as the stream separator" : null; + +describeIf(CANNOT_PACK, "scripts/lib/pack-standalone-tarball.sh (#133)", () => { const fixtureRoots: string[] = []; afterEach(() => { @@ -31,20 +61,25 @@ describe("scripts/lib/pack-standalone-tarball.sh (#133)", () => { return { root, payloadDir, tarball: join(root, "out.tar.gz") }; } - test("packs the payload under a top-level libredb-studio-/ root", () => { - const { payloadDir, tarball } = makeFixturePayload(); - - const run = Bun.spawnSync(["bash", SCRIPT, payloadDir, VERSION, tarball], { stdout: "pipe", stderr: "pipe" }); - expect(run.exitCode).toBe(0); - - const list = Bun.spawnSync(["tar", "tzf", tarball], { stdout: "pipe", stderr: "pipe" }); - expect(list.exitCode).toBe(0); - const entries = list.stdout + /** The archive's entry names. The script's own stderr rides on the assertion: a shell script that + dies under `set -e` says why there and nowhere else, and an exit code alone names no cause. */ + function listEntries(tarball: string): string[] { + const list = Bun.spawnSync([TAR!, "tzf", tarball], { stdout: "pipe", stderr: "pipe" }); + expect(list.exitCode, `tar tzf ${tarball} stderr: ${list.stderr.toString()}`).toBe(0); + return list.stdout .toString() .split("\n") .map((line) => line.trim()) .filter(Boolean); + } + test("packs the payload under a top-level libredb-studio-/ root", () => { + const { payloadDir, tarball } = makeFixturePayload(); + + const run = Bun.spawnSync([SHELL!, SCRIPT, payloadDir, VERSION, tarball], { stdout: "pipe", stderr: "pipe" }); + expect(run.exitCode, `pack-standalone-tarball.sh stderr: ${run.stderr.toString()}`).toBe(0); + + const entries = listEntries(tarball); expect(entries.length).toBeGreaterThan(0); for (const entry of entries) { expect(entry.startsWith(`libredb-studio-${VERSION}/`)).toBe(true); @@ -53,8 +88,32 @@ describe("scripts/lib/pack-standalone-tarball.sh (#133)", () => { expect(entries.some((entry) => entry === "./" || entry.startsWith("./"))).toBe(false); }); + testIf(MISSING_COLON_FILE_NAMES, "resolves an output path tar would otherwise dial as a remote host", () => { + const { root, payloadDir } = makeFixturePayload(); + + /* + GNU tar reads a -f argument whose first colon comes before any slash as `host:file` and tries + to reach that host: measured with tar 1.35, `-f out:1.tar.gz` answers "Cannot connect to out: + resolve failed" and exits 2 having written nothing. Windows reaches that line through every + absolute path it has - `C:\payload\out.tar.gz` is `host:file` by the same rule - so a script + that forwards the caller's path straight to tar breaks for any native caller there. A colon in + a plain file name is how a machine with no drive letters states the same case; resolving the + path first is what makes both local. + */ + const run = Bun.spawnSync([SHELL!, SCRIPT, payloadDir, VERSION, "out:1.tar.gz"], { + cwd: root, + stdout: "pipe", + stderr: "pipe", + }); + expect(run.exitCode, `pack-standalone-tarball.sh stderr: ${run.stderr.toString()}`).toBe(0); + + const tarball = join(root, "out:1.tar.gz"); + expect(existsSync(tarball)).toBe(true); + expect(listEntries(tarball)).toContain(`libredb-studio-${VERSION}/server.js`); + }); + test("rejects a wrong number of arguments", () => { - const run = Bun.spawnSync(["bash", SCRIPT, "/tmp/x"], { stdout: "pipe", stderr: "pipe" }); + const run = Bun.spawnSync([SHELL!, SCRIPT, join(tmpdir(), "x")], { stdout: "pipe", stderr: "pipe" }); expect(run.exitCode).not.toBe(0); expect(run.stderr.toString()).toContain("Usage:"); }); diff --git a/tests/unit/packaging-standalone-zip.test.ts b/tests/unit/packaging-standalone-zip.test.ts index 2fc39115a..0a5982b61 100644 --- a/tests/unit/packaging-standalone-zip.test.ts +++ b/tests/unit/packaging-standalone-zip.test.ts @@ -6,27 +6,60 @@ * zip root and `wingetcreate update` never rewrites that path, so a versioned * wrapper would break every subsequent release. Exercises the real script as * a subprocess against a small fixture payload (mirrors - * packaging-standalone-tarball.test.ts; 7z is preinstalled on the CI runners). + * packaging-standalone-tarball.test.ts), and reads the archive it produced in + * process, so only the packing side depends on 7-Zip. */ -import { afterEach, describe, expect, test } from "bun:test"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { afterEach, expect, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { describeIf, missingPosixShell, missingUnixTool, posixShell } from "../helpers/posix-tools"; const SCRIPT = join(import.meta.dir, "../../scripts/lib/pack-standalone-zip.sh"); +/* + Packing needs the shell and 7-Zip; READING the result does not, and used to anyway. + + `Bun.spawnSync(["7z", ...])` THROWS ("Executable not found in $PATH", measured in this worktree) + rather than returning a non-zero exit code, so on a fresh clone with no 7-Zip - a stock macOS or + Windows machine - every test in this file died at its first listing. The layout contract (#114) is + what these tests protect, so the listing is now read from the archive's own central directory and + only the packing side is gated, on the same two places the script itself looks for 7-Zip. +*/ +const SHELL = posixShell("bash"); +const SEVENZIP_WINDOWS_DEFAULT = "C:/Program Files/7-Zip/7z.exe"; +const CANNOT_PACK = missingPosixShell("bash") ?? (existsSync(SEVENZIP_WINDOWS_DEFAULT) ? null : missingUnixTool("7z")); + +/** + * The archive's entry names, read from its end-of-central-directory record. + * + * Directory entries are returned without the trailing "/" the zip format marks them with, because + * the contract under test is the PATH an installer resolves (`.next/BUILD_ID` under `.next`), not + * how the archiver spells a directory - `7z l`, which this replaced, prints them unmarked too. + */ function listZipEntries(zipPath: string): string[] { - const list = Bun.spawnSync(["7z", "l", "-ba", "-slt", zipPath], { stdout: "pipe", stderr: "pipe" }); - expect(list.exitCode).toBe(0); - return list.stdout - .toString() - .split("\n") - .filter((line) => line.startsWith("Path = ")) - .map((line) => line.slice("Path = ".length).trim()) - .filter(Boolean); + const zip = readFileSync(zipPath); + const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength); + let eocd = zip.length - 22; + while (eocd >= 0 && view.getUint32(eocd, true) !== 0x06054b50) eocd -= 1; + if (eocd < 0) throw new Error(`${zipPath}: no end-of-central-directory record, so this is not a zip`); + const entryCount = view.getUint16(eocd + 10, true); + let offset = view.getUint32(eocd + 16, true); + const names: string[] = []; + for (let index = 0; index < entryCount; index += 1) { + if (view.getUint32(offset, true) !== 0x02014b50) { + throw new Error(`${zipPath}: central directory entry ${index} does not start with its signature`); + } + const nameLength = view.getUint16(offset + 28, true); + const extraLength = view.getUint16(offset + 30, true); + const commentLength = view.getUint16(offset + 32, true); + names.push(zip.toString("utf8", offset + 46, offset + 46 + nameLength).replace(/\/$/, "")); + offset += 46 + nameLength + extraLength + commentLength; + } + return names; } -describe("scripts/lib/pack-standalone-zip.sh (#114)", () => { +describeIf(CANNOT_PACK, "scripts/lib/pack-standalone-zip.sh (#114)", () => { const fixtureRoots: string[] = []; afterEach(() => { @@ -48,7 +81,7 @@ describe("scripts/lib/pack-standalone-zip.sh (#114)", () => { test("packs the payload contents FLAT at the archive root, including dot-directories", () => { const { payloadDir, zip } = makeFixturePayload(); - const run = Bun.spawnSync(["bash", SCRIPT, payloadDir, zip], { stdout: "pipe", stderr: "pipe" }); + const run = Bun.spawnSync([SHELL!, SCRIPT, payloadDir, zip], { stdout: "pipe", stderr: "pipe" }); expect(run.stderr.toString()).toBe(""); expect(run.exitCode).toBe(0); @@ -68,7 +101,7 @@ describe("scripts/lib/pack-standalone-zip.sh (#114)", () => { const { payloadDir, zip } = makeFixturePayload(); writeFileSync(zip, "not a zip"); - const run = Bun.spawnSync(["bash", SCRIPT, payloadDir, zip], { stdout: "pipe", stderr: "pipe" }); + const run = Bun.spawnSync([SHELL!, SCRIPT, payloadDir, zip], { stdout: "pipe", stderr: "pipe" }); expect(run.exitCode).toBe(0); expect(listZipEntries(zip)).toContain("server.js"); }); @@ -77,7 +110,7 @@ describe("scripts/lib/pack-standalone-zip.sh (#114)", () => { const { payloadDir, zip } = makeFixturePayload(); rmSync(join(payloadDir, "server.js")); - const run = Bun.spawnSync(["bash", SCRIPT, payloadDir, zip], { stdout: "pipe", stderr: "pipe" }); + const run = Bun.spawnSync([SHELL!, SCRIPT, payloadDir, zip], { stdout: "pipe", stderr: "pipe" }); expect(run.exitCode).not.toBe(0); expect(run.stderr.toString()).toContain("server.js"); }); @@ -87,7 +120,7 @@ describe("scripts/lib/pack-standalone-zip.sh (#114)", () => { rmSync(join(payloadDir, "server.js")); writeFileSync(join(payloadDir, "serverXjs"), "// imposter"); - const run = Bun.spawnSync(["bash", SCRIPT, payloadDir, zip], { stdout: "pipe", stderr: "pipe" }); + const run = Bun.spawnSync([SHELL!, SCRIPT, payloadDir, zip], { stdout: "pipe", stderr: "pipe" }); expect(run.exitCode).not.toBe(0); expect(run.stderr.toString()).toContain("server.js"); }); @@ -95,13 +128,13 @@ describe("scripts/lib/pack-standalone-zip.sh (#114)", () => { test("fails loudly for a missing payload directory", () => { const { root, zip } = makeFixturePayload(); - const run = Bun.spawnSync(["bash", SCRIPT, join(root, "nope"), zip], { stdout: "pipe", stderr: "pipe" }); + const run = Bun.spawnSync([SHELL!, SCRIPT, join(root, "nope"), zip], { stdout: "pipe", stderr: "pipe" }); expect(run.exitCode).not.toBe(0); expect(run.stderr.toString()).toContain("Payload dir not found"); }); test("rejects wrong usage", () => { - const run = Bun.spawnSync(["bash", SCRIPT, "only-one-arg"], { stdout: "pipe", stderr: "pipe" }); + const run = Bun.spawnSync([SHELL!, SCRIPT, "only-one-arg"], { stdout: "pipe", stderr: "pipe" }); expect(run.exitCode).not.toBe(0); expect(run.stderr.toString()).toContain("Usage:"); }); diff --git a/tests/unit/posix-tools.test.ts b/tests/unit/posix-tools.test.ts new file mode 100644 index 000000000..6566d3748 --- /dev/null +++ b/tests/unit/posix-tools.test.ts @@ -0,0 +1,234 @@ +/** + * Unit tests for tests/helpers/posix-tools.ts - the explicit resolution of the POSIX shell and the + * unix tools the packaging tests drive. + * + * The whole point of the helper is the Windows branch, and the machine running this is not Windows, + * so every case below drives an INJECTED lookup: a platform branch that only a Windows contributor + * can execute is a branch nobody checks until it breaks on their laptop. The real lookup is + * exercised too, directly, because "ask git where it lives" has to be a measurement rather than a + * shape. + */ +import { describe, expect, test } from "bun:test"; +import { + MISSING_POSIX_FILE_MODES, + type ToolLookup, + describeIf, + describeIfPosixShell, + missingPosixShell, + missingUnixTool, + posixShell, + resolveUnixTool, + systemLookup, + testIf, +} from "../helpers/posix-tools"; + +const GIT_ROOT = "C:/Program Files/Git"; +/** What `git --exec-path` prints on Windows, verbatim shape: forward slashes, three levels deep. */ +const GIT_EXEC_PATH = `${GIT_ROOT}/mingw64/libexec/git-core`; + +interface FakeWindows { + /** What is on PATH, by command name. */ + path?: Record; + /** Which absolute paths exist. */ + files?: string[]; + /** What `git --exec-path` answers. */ + execPath?: string | null; + localAppData?: string | null; +} + +function fakeWindows(options: FakeWindows = {}): ToolLookup { + return { + platform: "win32", + which: (command) => options.path?.[command] ?? null, + exists: (candidate) => (options.files ?? []).includes(candidate), + gitExecPath: () => options.execPath ?? null, + localAppData: options.localAppData ?? null, + }; +} + +describe("resolveUnixTool on Linux and macOS", () => { + test("is the PATH lookup, nothing more", () => { + const lookup: ToolLookup = { + platform: "linux", + which: (command) => (command === "bash" ? "/usr/bin/bash" : null), + exists: () => true, + gitExecPath: () => GIT_EXEC_PATH, + localAppData: null, + }; + expect(resolveUnixTool("bash", lookup)).toBe("/usr/bin/bash"); + // No Git-for-Windows guessing off PATH, even though every candidate above "exists". + expect(resolveUnixTool("7z", lookup)).toBeNull(); + }); + + test("resolves this machine's own shell through the real lookup", () => { + expect(resolveUnixTool("sh")).toContain("sh"); + expect(resolveUnixTool("libredb-not-a-binary-4b7c")).toBeNull(); + }); +}); + +describe("resolveUnixTool on Windows", () => { + test("refuses WSL's bash and takes the Git for Windows one instead", () => { + // The trap this helper exists for: with WSL installed, `bash` resolves - to a Linux shell that + // cannot stat the Win32 temp path every fixture hands it, so the script under test looks broken. + const lookup = fakeWindows({ + path: { bash: "C:\\Windows\\System32\\bash.exe", git: `${GIT_ROOT}/cmd/git.exe` }, + execPath: GIT_EXEC_PATH, + files: [`${GIT_ROOT}/bin/bash.exe`], + }); + expect(resolveUnixTool("bash", lookup)).toBe(`${GIT_ROOT}/bin/bash.exe`); + }); + + test("keeps a System32 hit that WSL does not shadow", () => { + // Windows 10+ ships a genuine bsdtar as System32\tar.exe; only the shells are shadowed. + const lookup = fakeWindows({ path: { tar: "C:\\Windows\\System32\\tar.exe" } }); + expect(resolveUnixTool("tar", lookup)).toBe("C:\\Windows\\System32\\tar.exe"); + }); + + test("keeps a shell that is genuinely on PATH (MSYS2, or a PATH that carries Git's usr/bin)", () => { + const lookup = fakeWindows({ path: { sh: "C:\\msys64\\usr\\bin\\sh.exe" } }); + expect(resolveUnixTool("sh", lookup)).toBe("C:\\msys64\\usr\\bin\\sh.exe"); + }); + + test("asks the git binary it found, rather than guessing the install location", () => { + const asked: string[] = []; + const lookup: ToolLookup = { + platform: "win32", + which: (command) => (command === "git" ? "D:/tools/PortableGit/cmd/git.exe" : null), + exists: (candidate) => candidate === "D:/tools/PortableGit/usr/bin/grep.exe", + gitExecPath: (gitBinary) => { + asked.push(gitBinary); + return "D:/tools/PortableGit/mingw64/libexec/git-core"; + }, + localAppData: null, + }; + expect(resolveUnixTool("grep", lookup)).toBe("D:/tools/PortableGit/usr/bin/grep.exe"); + expect(asked).toEqual(["D:/tools/PortableGit/cmd/git.exe"]); + }); + + test("reads back the backslashes git prints when it prints them", () => { + const lookup = fakeWindows({ + path: { git: `${GIT_ROOT}/cmd/git.exe` }, + execPath: "C:\\Program Files\\Git\\mingw64\\libexec\\git-core", + files: [`${GIT_ROOT}/usr/bin/unzip.exe`], + }); + expect(resolveUnixTool("unzip", lookup)).toBe(`${GIT_ROOT}/usr/bin/unzip.exe`); + }); + + test("falls back to a per-user install when git is not on PATH at all", () => { + const home = "C:\\Users\\dev\\AppData\\Local"; + const lookup = fakeWindows({ + localAppData: home, + files: ["C:/Users/dev/AppData/Local/Programs/Git/bin/bash.exe"], + }); + expect(resolveUnixTool("bash", lookup)).toBe("C:/Users/dev/AppData/Local/Programs/Git/bin/bash.exe"); + }); + + test("falls back to the standard install locations", () => { + const lookup = fakeWindows({ files: ["C:/Program Files (x86)/Git/usr/bin/grep.exe"] }); + expect(resolveUnixTool("grep", lookup)).toBe("C:/Program Files (x86)/Git/usr/bin/grep.exe"); + }); + + test("ignores a git that answers nothing, and a --exec-path with no ancestors to climb", () => { + const silent = fakeWindows({ path: { git: "git.exe" }, execPath: null, files: [`${GIT_ROOT}/bin/sh.exe`] }); + expect(resolveUnixTool("sh", silent)).toBe(`${GIT_ROOT}/bin/sh.exe`); + + const truncated = fakeWindows({ path: { git: "git.exe" }, execPath: "git-core", files: [] }); + expect(resolveUnixTool("sh", truncated)).toBeNull(); + }); + + test("answers null when the machine has no Git for Windows anywhere", () => { + expect(resolveUnixTool("bash", fakeWindows())).toBeNull(); + }); +}); + +describe("the real lookup", () => { + test("finds a file that exists and not one that does not", () => { + expect(systemLookup.exists(import.meta.path)).toBe(true); + expect(systemLookup.exists(`${import.meta.path}.absent`)).toBe(false); + }); + + test("git answers --exec-path with a directory that exists", () => { + // Every clone of this repository needed git, so an absent one is a broken machine, not a skip. + const git = systemLookup.which("git"); + expect(git).not.toBeNull(); + const execPath = systemLookup.gitExecPath(git!); + expect(execPath).not.toBeNull(); + expect(systemLookup.exists(execPath!)).toBe(true); + }); + + test("platform and LOCALAPPDATA are read from this process", () => { + expect(systemLookup.platform).toBe(process.platform); + expect(systemLookup.localAppData).toBe(process.env.LOCALAPPDATA ?? null); + }); +}); + +describe("the reasons a skip carries", () => { + test("a resolvable shell has no reason at all", () => { + expect(posixShell("sh")).not.toBeNull(); + expect(posixShell("bash")).not.toBeNull(); + expect(missingPosixShell("bash")).toBeNull(); + expect(missingUnixTool("git")).toBeNull(); + }); + + test("an unresolvable one names what is missing and where it would have come from", () => { + const none = fakeWindows(); + expect(posixShell("bash", none)).toBeNull(); + expect(missingPosixShell("sh", none)).toBe( + "no POSIX sh: none on PATH and no Git for Windows installation carries one", + ); + expect(missingUnixTool("7z", none)).toBe("no 7z: not on PATH and not in a Git for Windows installation"); + }); + + test("POSIX file modes are a platform fact, not a probe", () => { + // A probe could answer "no" on Linux (a noexec /tmp, a container quirk) and turn a genuine + // regression into a skip; the platform cannot. + expect(MISSING_POSIX_FILE_MODES).toBe( + process.platform === "win32" + ? "POSIX file modes: NTFS has no exec bit and Windows cannot exec an extension-less #! stub" + : null, + ); + }); +}); + +/* + The skip path, registered for real: a requirement no machine can meet, so the describe below is + collected as skipped everywhere. The warnings are captured rather than printed because these two + are self-tests and a reader scanning a green run should not have to decide whether they matter - + every other caller's skip goes to the terminal. +*/ +const warnings: string[] = []; +const realWarn = console.warn; +console.warn = (message: string) => { + warnings.push(message); +}; +describeIf(missingUnixTool("libredb-not-a-binary-4b7c"), "describeIf's own skip path", () => { + test("never runs, because the describe above is skipped", () => { + throw new Error("a skipped describe must not execute its test bodies"); + }); +}); +// Both calls sit at module scope on purpose: bun defers a describe body, so a testIf inside one +// would log after the capture below is put back. +testIf(null, "testIf runs a test when nothing is missing", () => { + expect(posixShell("sh")).not.toBeNull(); +}); +testIf(missingUnixTool("libredb-not-a-binary-4b7c"), "testIf's own skip path", () => { + throw new Error("a skipped test must not execute its body"); +}); +console.warn = realWarn; + +describe("a skipped requirement is visible", () => { + test("names the title and the reason, in the title and in the log", () => { + expect(warnings).toEqual([ + 'posix-tools: skipping "describeIf\'s own skip path" - no libredb-not-a-binary-4b7c: ' + + "not on PATH and not in a Git for Windows installation", + 'posix-tools: skipping "testIf\'s own skip path" - no libredb-not-a-binary-4b7c: ' + + "not on PATH and not in a Git for Windows installation", + ]); + }); +}); + +describeIfPosixShell("bash", "describeIfPosixShell", () => { + test("runs the body on a machine that has bash", () => { + expect(posixShell("bash")).not.toBeNull(); + }); +}); diff --git a/tests/unit/security-check.test.ts b/tests/unit/security-check.test.ts index 9ab7dbb7f..02cec3852 100644 --- a/tests/unit/security-check.test.ts +++ b/tests/unit/security-check.test.ts @@ -16,10 +16,18 @@ const SCRIPT = path.resolve(import.meta.dir, "../../scripts/security-check.mjs") * cases exist so this one is not. Each violation family gets a case that PRODUCES it. */ -const COMPONENTS_RUNNER = ` -run_group "Group 0b: Factory singleton" \\ - tests/isolated/factory-singleton.test.ts -`; +/** + * What `bun tests/run-tests.ts --list` would answer, as the gate asks it for real. + * One entry per layer the rule covers, including `tests/isolated/`, which used to be + * counted only because a bash runner happened to name the file. + */ +const DISCOVERED_TESTS = new Set([ + ...PROGRAMME_CONTROL_IDS.map((id) => `tests/security/c-${id}.test.ts`), + "tests/security/headers.test.ts", + "tests/unit/a.test.tsx", + "tests/isolated/factory-singleton.test.ts", + "tests/evals/agent-loop.test.ts", +]); const PLAYWRIGHT_CONFIG = `export default defineConfig({ testDir: "./e2e", projects: [] });`; const HEADER = "| ID | Control | Status | Enforced in | Verified by |"; @@ -45,7 +53,7 @@ const CLEAN_SECURITY_TESTS = PROGRAMME_CONTROL_IDS.map((id) => `tests/security/c function run(overrides: Record = {}) { return checkPosture({ posture: page(cleanRows()), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS, @@ -86,9 +94,9 @@ describe("linkTargets", () => { }); describe("isExecuted", () => { - const context = { componentsRunner: COMPONENTS_RUNNER, playwrightConfig: PLAYWRIGHT_CONFIG }; + const context = { discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG }; - test("a tests/security file is run by tests/run-core.sh", () => { + test("a tests/security file the runner collects is run", () => { expect(isExecuted("tests/security/headers.test.ts", context).executed).toBe(true); }); @@ -100,9 +108,15 @@ describe("isExecuted", () => { expect(isExecuted("tests/security/headers.test.ts.disabled", context).executed).toBe(false); }); - test("a tests/isolated file is run only because run-components.sh names it", () => { + test("a tests/isolated file is run because the runner discovers it, and a file that is not there is not", () => { expect(isExecuted("tests/isolated/factory-singleton.test.ts", context).executed).toBe(true); - expect(isExecuted("tests/isolated/never-listed.test.ts", context).executed).toBe(false); + // The negative is what the rule is for: a path named by docs/SECURITY.md that no + // longer exists (renamed, deleted, or never created) is not a verified control. + expect(isExecuted("tests/isolated/never-written.test.ts", context).executed).toBe(false); + }); + + test("an eval test counts too, which the old hardcoded directory list missed", () => { + expect(isExecuted("tests/evals/agent-loop.test.ts", context).executed).toBe(true); }); test("an e2e spec is run when playwright's testDir is the e2e directory", () => { @@ -150,7 +164,7 @@ describe("checkPosture", () => { const violations = checkPosture({ posture: page(rows), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS.filter((f) => f !== "tests/security/c-0.1.test.ts"), @@ -175,7 +189,7 @@ describe("checkPosture", () => { const violations = checkPosture({ posture: page(rows), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS, @@ -190,7 +204,7 @@ describe("checkPosture", () => { const violations = checkPosture({ posture: page(rows), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS.filter((f) => f !== "tests/security/c-0.1.test.ts"), @@ -205,7 +219,7 @@ describe("checkPosture", () => { const violations = checkPosture({ posture: page(rows), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS.filter((f) => f !== "tests/security/c-0.1.test.ts"), @@ -217,7 +231,7 @@ describe("checkPosture", () => { test("names a programme control the page forgot", () => { const violations = checkPosture({ posture: page(cleanRows().slice(1)), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS.filter((f) => f !== "tests/security/c-0.1.test.ts"), @@ -230,7 +244,7 @@ describe("checkPosture", () => { const rows = [...cleanRows(), row("9.9", "Implemented", "[`t`](../tests/security/c-9.9.test.ts)")]; const violations = checkPosture({ posture: page(rows), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: [...CLEAN_SECURITY_TESTS, "tests/security/c-9.9.test.ts"], @@ -244,7 +258,7 @@ describe("checkPosture", () => { rows[0] = row("0.1", "Implemented", "[`policy`](../SECURITY.md)"); const violations = checkPosture({ posture: page(rows), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS.filter((f) => f !== "tests/security/c-0.1.test.ts"), @@ -265,7 +279,7 @@ describe("checkPosture", () => { expect( checkPosture({ posture: page(rows), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS, @@ -277,7 +291,7 @@ describe("checkPosture", () => { const rows = [...cleanRows(), row("0.1", "Implemented", "[`t`](../tests/security/c-0.1-again.test.ts)")]; const violations = checkPosture({ posture: page(rows), - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: [...CLEAN_SECURITY_TESTS, "tests/security/c-0.1-again.test.ts"], @@ -289,7 +303,7 @@ describe("checkPosture", () => { test("a page whose control table cannot be found fails loudly instead of passing vacuously", () => { const violations = checkPosture({ posture: "# Security Posture\n\nno table here\n", - componentsRunner: COMPONENTS_RUNNER, + discoveredTests: DISCOVERED_TESTS, playwrightConfig: PLAYWRIGHT_CONFIG, exists: () => true, securityTestFiles: CLEAN_SECURITY_TESTS, diff --git a/tests/unit/snap-launcher.test.ts b/tests/unit/snap-launcher.test.ts index 14293d281..70c21b959 100644 --- a/tests/unit/snap-launcher.test.ts +++ b/tests/unit/snap-launcher.test.ts @@ -11,9 +11,19 @@ import { afterEach, describe, expect, test } from "bun:test"; import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { MISSING_POSIX_FILE_MODES, describeIf, missingPosixShell, posixShell } from "../helpers/posix-tools"; const REPO_ROOT = join(import.meta.dir, "../.."); const LAUNCHER = join(REPO_ROOT, "snap/local/launch.sh"); +/* + Only the second describe runs anything: it executes the snap's own launcher against a stub `node` + that is a `#!/bin/sh` file made runnable with chmod 0755. snapd is Linux-only, Windows has no exec + bit and cannot exec an extension-less #! file, and `Bun.spawnSync(["sh", ...])` THROWS there + ("Executable not found in $PATH", measured in this worktree). The manifest describe above it is + YAML parsing and keeps running everywhere. +*/ +const SHELL = posixShell("sh"); +const CANNOT_RUN = missingPosixShell("sh") ?? MISSING_POSIX_FILE_MODES; /** Every key the launcher defaults, and so an operator must be able to override. */ const DEFAULTED_KEYS = [ @@ -44,7 +54,7 @@ describe("snap/snapcraft.yaml app environment (#807)", () => { }); }); -describe("snap/local/launch.sh defaults (#807)", () => { +describeIf(CANNOT_RUN, "snap/local/launch.sh defaults (#807)", () => { const fixtureRoots: string[] = []; afterEach(() => { @@ -62,7 +72,7 @@ describe("snap/local/launch.sh defaults (#807)", () => { writeFileSync(join(snap, "server.js"), ""); const cleared = Object.fromEntries([...DEFAULTED_KEYS, "INVOCATION_ID", "LIBREDB_BIND"].map((k) => [k, ""])); - const result = Bun.spawnSync(["sh", LAUNCHER], { + const result = Bun.spawnSync([SHELL!, LAUNCHER], { env: { ...process.env, ...cleared, SNAP: snap, SNAP_DATA: join(root, "data"), ...env }, stdout: "pipe", stderr: "pipe", diff --git a/tests/unit/sql/spans.test.ts b/tests/unit/sql/spans.test.ts index bd55cd4bb..ed885d71d 100644 --- a/tests/unit/sql/spans.test.ts +++ b/tests/unit/sql/spans.test.ts @@ -4,6 +4,31 @@ import { hasUnterminatedSpan, readSqlSpan } from "@/lib/sql/spans"; // ─── Helpers ──────────────────────────────────────────────────────────────── +/** + * Run a bounded-time probe ten times and report its cheapest run, in milliseconds. + * + * The guards below are about algorithmic SHAPE, not about the machine: what they pin is that a + * character scanner cannot backtrack, and the linear scanner costs hundredths of a millisecond + * on these 20k inputs against a ceiling of 200. A single sample measures the machine as well, + * though, and one bun process per CPU makes a scheduler stall inside the measured region + * ordinary - so a stall reports a performance regression that did not happen, and it gets + * blamed on the scanner. Being preempted only ever ADDS time, so the cheapest of several runs + * is the sample least polluted by the other test processes, and it leaves the ceiling exactly + * where it was rather than widening it until it stops catching anything. Measured on this tree: + * the slowest of these inputs is 1.7ms idle and 5.5ms with forty bun processes on twenty cores. + */ +function bestOfTen(probe: () => T): { readonly result: T; readonly elapsed: number } { + // The first run is outside the measurement on purpose: it is the cold one. + let result = probe(); + let elapsed = Infinity; + for (let attempt = 0; attempt < 10; attempt++) { + const started = performance.now(); + result = probe(); + elapsed = Math.min(elapsed, performance.now() - started); + } + return { result, elapsed }; +} + /** The span at index 0, as `kind|text` (or `null`), which is what most cases assert. */ function spanOf(sql: string, index = 0, grammar?: SqlGrammar): string | null { const span = readSqlSpan(sql, index, grammar); @@ -457,9 +482,7 @@ describe("readSqlSpan", () => { test("answers in bounded time on a long body that never closes", () => { const sql = `q'{${"a".repeat(20000)}`; - const started = performance.now(); - const span = readSqlSpan(sql, 0, ORACLE); - const elapsed = performance.now() - started; + const { result: span, elapsed } = bestOfTen(() => readSqlSpan(sql, 0, ORACLE)); expect(span).toEqual({ kind: "string", end: sql.length, terminated: false }); expect(elapsed, `took ${elapsed.toFixed(1)}ms`).toBeLessThan(200); @@ -506,9 +529,7 @@ describe("readSqlSpan", () => { ]; for (const [label, sql, expectSpan] of adversarial) { - const started = performance.now(); - const span = readSqlSpan(sql, 0); - const elapsed = performance.now() - started; + const { result: span, elapsed } = bestOfTen(() => readSqlSpan(sql, 0)); // A correct answer AND a bounded one: a fast wrong answer is not a pass. expect(span === null, label).toBe(!expectSpan); @@ -688,9 +709,7 @@ describe("hasUnterminatedSpan", () => { // Unbalanced on purpose: the answer has to be reached by scanning to the end. const deep = `${"/*".repeat(20000)} SELECT 1`; - const started = performance.now(); - const unresolved = hasUnterminatedSpan(deep, resolveSqlGrammar("postgres")); - const elapsed = performance.now() - started; + const { result: unresolved, elapsed } = bestOfTen(() => hasUnterminatedSpan(deep, resolveSqlGrammar("postgres"))); expect(unresolved).toBe(true); expect(elapsed, `took ${elapsed.toFixed(1)}ms`).toBeLessThan(200); @@ -711,9 +730,7 @@ describe("hasUnterminatedSpan", () => { // time cannot backtrack - the property this asserts is kept, not assumed. const many = `SELECT ${"'lit' /* note */ -- line\n".repeat(20000)}1`; - const started = performance.now(); - const unresolved = hasUnterminatedSpan(many); - const elapsed = performance.now() - started; + const { result: unresolved, elapsed } = bestOfTen(() => hasUnterminatedSpan(many)); expect(unresolved).toBe(false); expect(elapsed, `took ${elapsed.toFixed(1)}ms`).toBeLessThan(200); diff --git a/tests/unit/sync-chart-version.test.ts b/tests/unit/sync-chart-version.test.ts index fb4b37b21..a70df8684 100644 --- a/tests/unit/sync-chart-version.test.ts +++ b/tests/unit/sync-chart-version.test.ts @@ -509,9 +509,11 @@ describe("CLI (--check via subprocess)", () => { describe("CLI (--check against git fixtures, #151/#167)", () => { const fixtureRoots: string[] = []; + let emptyConfig: string | null = null; afterEach(() => { for (const root of fixtureRoots.splice(0)) rmSync(root, { recursive: true, force: true }); + emptyConfig = null; }); function makeDir(prefix: string): string { @@ -520,14 +522,29 @@ describe("CLI (--check against git fixtures, #151/#167)", () => { return dir; } + /** + * A real, empty config file for GIT_CONFIG_GLOBAL/SYSTEM, in its own directory so it is never + * inside a fixture repository and never seen by `git add -A`. "/dev/null" was a POSIX device + * path: a git build that refuses a config path it cannot open would turn every fixture in this + * describe into a thrown "git ... failed", and on Windows there is no such path at all, so the + * isolation was accidental rather than stated. + */ + function emptyConfigPath(): string { + if (emptyConfig === null) { + emptyConfig = join(makeDir("chart-sync-gitconfig-"), "empty.gitconfig"); + writeFileSync(emptyConfig, ""); + } + return emptyConfig; + } + // Hermetic git: no user/system config, fixed identity, so fixtures behave the same on any box. function runGit(cwd: string, ...args: string[]): string { const result = Bun.spawnSync(["git", ...args], { cwd, env: { ...process.env, - GIT_CONFIG_GLOBAL: "/dev/null", - GIT_CONFIG_SYSTEM: "/dev/null", + GIT_CONFIG_GLOBAL: emptyConfigPath(), + GIT_CONFIG_SYSTEM: emptyConfigPath(), GIT_AUTHOR_NAME: "fixture", GIT_AUTHOR_EMAIL: "fixture@test", GIT_COMMITTER_NAME: "fixture", diff --git a/tests/unit/test-runner-capture.test.ts b/tests/unit/test-runner-capture.test.ts new file mode 100644 index 000000000..003b73c87 --- /dev/null +++ b/tests/unit/test-runner-capture.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from "bun:test"; +import { CAPTURE_HEAD_BYTES, CAPTURE_TAIL_BYTES, captureBounded } from "../runner/capture"; + +/** A stream that hands out exactly these chunks, the way a child's pipe does. */ +function streamOf(chunks: Uint8Array[]): ReadableStream { + return new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }); +} + +function bytes(text: string): Uint8Array { + return new TextEncoder().encode(text); +} + +/** `text` repeated until it is exactly `length` bytes, so a slice is recognisable. */ +function filler(text: string, length: number): string { + return text.repeat(Math.ceil(length / text.length)).slice(0, length); +} + +describe("capturing a child's output", () => { + test("a stream under the limit comes back byte for byte, with no marker", async () => { + const text = "bun test v1.4.2\nπ over several chunks, ünicode included\n 3 pass\n"; + const chunks = [bytes(text.slice(0, 20)), bytes(text.slice(20))]; + + const captured = await captureBounded(streamOf(chunks), { name: "stdout", headBytes: 64, tailBytes: 64 }); + + expect(captured).toBe(text); + expect(captured).not.toContain("elided"); + }); + + test("an empty stream is an empty string", async () => { + expect(await captureBounded(streamOf([]), { name: "stderr", headBytes: 8, tailBytes: 8 })).toBe(""); + }); + + test("a stream longer than head plus tail keeps both ends and says how much went, and from which stream", async () => { + const head = filler("HEAD-", 1000); + const middle = filler("middle-", 5000); + const tail = `${filler("TAIL-", 991)}\n 3 pass\n`; + const chunks = [bytes(head), bytes(middle), bytes(tail)]; + + const captured = await captureBounded(streamOf(chunks), { name: "stderr", headBytes: 1000, tailBytes: 1000 }); + + expect(captured).toBe(`${head}\n[runner: 5000 bytes of stderr elided here]\n${tail}`); + // The end of the stream is what a reader needs most, because bun writes its + // failure diffs and its per-file summary there. + expect(captured.endsWith(" 3 pass\n")).toBe(true); + }); + + test("limits that fall in the middle of a chunk keep exactly the head and tail bytes asked for", async () => { + const chunks = ["abc", "def", "ghi", "jkl", "mno"].map(bytes); + + const captured = await captureBounded(streamOf(chunks), { name: "stdout", headBytes: 5, tailBytes: 5 }); + + expect(captured).toBe("abcde\n[runner: 5 bytes of stdout elided here]\nklmno"); + }); + + test("a stream of exactly head plus tail bytes is kept whole", async () => { + // The paired control for the case above: one byte more is what starts the eliding. + const whole = filler("x", 10); + + expect(await captureBounded(streamOf([bytes(whole)]), { name: "stdout", headBytes: 5, tailBytes: 5 })).toBe(whole); + expect( + await captureBounded(streamOf([bytes(`${whole}y`)]), { name: "stdout", headBytes: 5, tailBytes: 5 }), + ).toContain("[runner: 1 bytes of stdout elided here]"); + }); + + test("the limits the runner uses are a megabyte at each end of each stream", () => { + // The basis is measured, and is argued in the module's docblock: the largest real + // file prints 154,526 bytes on stdout and the largest stderr is 48,369 bytes. + expect(CAPTURE_HEAD_BYTES).toBe(1024 * 1024); + expect(CAPTURE_TAIL_BYTES).toBe(1024 * 1024); + }); + + test("the default limits keep a real file's output whole", async () => { + const output = filler("a real test file's console output\n", 200_000); + + expect(await captureBounded(streamOf([bytes(output)]), { name: "stdout" })).toBe(output); + }); +}); diff --git a/tests/unit/test-runner-cli.test.ts b/tests/unit/test-runner-cli.test.ts new file mode 100644 index 000000000..0acee986e --- /dev/null +++ b/tests/unit/test-runner-cli.test.ts @@ -0,0 +1,870 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + chmodSync, + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { spawn } from "node:child_process"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { Readable } from "node:stream"; + +// The runner end to end, driven the way a contributor and CI drive it. The unit +// tests beside this one cover discovery, the command line, the pool and the +// report against injected data; these cases are here because three of the +// runner's decisions can only be wrong against the real bun binary: that a file +// is addressed as a path and not as a substring filter, that a child's exit code +// reaches the runner's own exit code, and that coverage lands where the merge +// expects it. +const root = path.resolve(import.meta.dir, "../.."); +const RUNNER = "tests/run-tests.ts"; + +// The cases that need a deliberately failing, skipping or empty test file run the +// runner in a SANDBOX: a temporary directory holding a copy of tests/run-tests.ts and +// tests/runner/, whose own tests/ tree holds nothing but the fixture. The runner finds +// its root from its own location, so the copy discovers only that tree. +// +// The fixture used to be written into this repository's tests/unit/. That had two +// costs: a run interrupted between the write and the cleanup left a failing file the +// discovery rule then collects in every later run, and every other test file that +// walks tests/ while this one runs (the discovery test, the backlog guard, the +// security gate asking --list) could see a file that exists for half a second. +const sandboxes: string[] = []; + +/** Private TMPDIRs, so a stopped run's scratch directory can be seen (or not) on its own. */ +const privateTmpDirs: string[] = []; + +afterEach(() => { + for (const directory of privateTmpDirs.splice(0)) { + // The removal-failure case takes the write permission away, and it has to come back + // here: without it this cleanup would fail for the same reason the runner did. + if (process.platform !== "win32") chmodSync(directory, 0o755); + rmSync(directory, { recursive: true, force: true }); + } + for (const sandbox of sandboxes.splice(0)) rmSync(sandbox, { recursive: true, force: true }); +}); + +function sandboxWith(fixture: string): string { + const sandbox = mkdtempSync(path.join(tmpdir(), "runner-sandbox-")); + sandboxes.push(sandbox); + cpSync(path.join(root, "tests/run-tests.ts"), path.join(sandbox, "tests/run-tests.ts")); + cpSync(path.join(root, "tests/runner"), path.join(sandbox, "tests/runner"), { recursive: true }); + mkdirSync(path.join(sandbox, "tests/unit"), { recursive: true }); + // writeFileSync, not Bun.write: Bun.write returns a promise, and leaving it + // unawaited let the runner start against a file that was still empty. bun then + // ran 0 tests and exited 0, so this passed on Linux and failed on windows-latest + // (measured 2026-09-15). + writeFileSync(path.join(sandbox, "tests/unit/fixture.test.ts"), fixture); + return sandbox; +} + +function runInSandbox( + sandbox: string, + selectors: string[] = ["tests/unit/fixture.test.ts"], + env: Record = withoutRequirements(), +): { exitCode: number; stdout: string; stderr: string } { + const result = Bun.spawnSync([process.execPath, "tests/run-tests.ts", ...selectors], { cwd: sandbox, env }); + return { exitCode: result.exitCode, stdout: result.stdout.toString(), stderr: result.stderr.toString() }; +} + +/** + * This process's environment without the run requirements CI sets. The CI job that runs this + * file exports LIBREDB_REQUIRE_HELM=1 for the real suite, and a sandbox child inheriting it + * would refuse to run for a reason that belongs to the parent, not to the case under test. + */ +function withoutRequirements(): NodeJS.ProcessEnv { + const env = { ...process.env }; + delete env.LIBREDB_REQUIRE_HELM; + return env; +} + +/** A test file that says it started and then waits for the runner to be stopped. */ +function waitingFixture(name: string): string { + return [ + 'import { test } from "bun:test";', + 'import { writeFileSync } from "node:fs";', + 'import path from "node:path";', + 'test("waits to be interrupted", async () => {', + ` writeFileSync(path.join(process.env.RUNNER_MARKERS as string, "${name}.started"), "");`, + " await Bun.sleep(30_000);", + "}, 60_000);", + "", + ].join("\n"); +} + +type WaitingRun = { sandbox: string; markers: string; scratchParent: string }; + +/** Two waiting files, run one at a time, with a TMPDIR of this run's own. */ +function sandboxThatWaits(): WaitingRun { + const sandbox = sandboxWith(waitingFixture("fixture")); + writeFileSync(path.join(sandbox, "tests/unit/second.test.ts"), waitingFixture("second")); + const markers = path.join(sandbox, "markers"); + mkdirSync(markers); + const scratchParent = mkdtempSync(path.join(tmpdir(), "runner-tmpdir-")); + privateTmpDirs.push(scratchParent); + return { sandbox, markers, scratchParent }; +} + +/** + * A test file that prints `kibibytes` KiB on its own stdout and then fails. + * + * The pause before it fails is load-bearing, not politeness: measured, bun 1.4.2 drops + * part of a child's queued stdout when the child exits under CPU load, and without the + * pause 2 of 6 loaded runs delivered too little for the runner's own pipe to fill, so + * the cases built on a stuck reader stopped being about a stuck reader at all. + */ +function floodingFixture(kibibytes: number): string { + return [ + 'import { expect, test } from "bun:test";', + 'test("prints a lot and then fails", async () => {', + ' const line = `${"k".repeat(1023)}\\n`;', + ` for (let index = 0; index < ${kibibytes}; index += 1) process.stdout.write(line);`, + " await Bun.sleep(500);", + " expect(1).toBe(2);", + "});", + "", + ].join("\n"); +} + +/** + * A run whose first file floods stdout and fails, and whose second file then waits. + * + * The flood is what makes a signal case about a STUCK reader: a pipe holds 64 KiB on + * Linux, so once the first file's output has been printed and nobody is reading, the + * runner's next write cannot complete until the reader comes back. + */ +function sandboxThatFloodsThenWaits(): WaitingRun { + const sandbox = sandboxWith(floodingFixture(512)); + writeFileSync(path.join(sandbox, "tests/unit/second.test.ts"), waitingFixture("second")); + const markers = path.join(sandbox, "markers"); + mkdirSync(markers); + const scratchParent = mkdtempSync(path.join(tmpdir(), "runner-tmpdir-")); + privateTmpDirs.push(scratchParent); + return { sandbox, markers, scratchParent }; +} + +/** + * A sandbox whose run reaches the coverage merge, with a merge script of its own. + * + * The merge only happens when there is something to merge, so the fixture has to cover + * a source file: with no src/ the run ends at "No coverage report was written". The + * script is spawned as `node scripts/merge-lcov.mjs` from the root the runner found + * beside itself, which is this sandbox. + */ +function sandboxThatMerges(mergeScript: string): { sandbox: string; markers: string; args: string[] } { + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\n' + + 'import { covered } from "../../src/covered";\n' + + 'test("covers a source file", () => {\n expect(covered()).toBe(1);\n});\n', + ); + mkdirSync(path.join(sandbox, "src"), { recursive: true }); + writeFileSync(path.join(sandbox, "src/covered.ts"), "export function covered(): number {\n return 1;\n}\n"); + mkdirSync(path.join(sandbox, "scripts"), { recursive: true }); + writeFileSync(path.join(sandbox, "scripts/merge-lcov.mjs"), mergeScript); + const markers = path.join(sandbox, "markers"); + mkdirSync(markers); + return { + sandbox, + markers, + args: [ + "tests/unit/fixture.test.ts", + "--coverage", + `--coverage-dir=${path.join(sandbox, "raw")}`, + `--merge-into=${path.join(sandbox, "lcov.info")}`, + ], + }; +} + +async function waitForFile(file: string, within = 15_000): Promise { + const deadline = Date.now() + within; + while (!existsSync(file)) { + if (Date.now() > deadline) throw new Error(`${file} was not written within ${within} ms`); + // oxlint-disable-next-line no-await-in-loop -- polling: the next look has to come after this wait. + await Bun.sleep(25); + } +} + +/** Starts the run, waits until its first file is running, then signals it. */ +async function signalTheRun( + { sandbox, markers, scratchParent }: WaitingRun, + signal: NodeJS.Signals, + beforeSignalling?: () => void, +): Promise<{ exitCode: number | null; signalCode: string | null; stdout: string; stderr: string }> { + const runner = Bun.spawn([process.execPath, "tests/run-tests.ts", "--jobs=1", "tests/unit"], { + cwd: sandbox, + env: { ...withoutRequirements(), TMPDIR: scratchParent, RUNNER_MARKERS: markers }, + stdout: "pipe", + stderr: "pipe", + }); + await waitForFile(path.join(markers, "fixture.started")); + beforeSignalling?.(); + runner.kill(signal); + + const [stdout, stderr] = await Promise.all([ + new Response(runner.stdout as ReadableStream).text(), + new Response(runner.stderr as ReadableStream).text(), + ]); + const exitCode = await runner.exited; + return { exitCode, signalCode: runner.signalCode, stdout, stderr }; +} + +/** + * Starts the run, waits until its SECOND file is running, and signals it without ever + * reading stdout, so the runner is stuck on a write it cannot finish. + * + * node:child_process, not Bun.spawn: measured on bun 1.4.2, a Bun.spawn parent drains + * a "pipe" stdout into a buffer of its own whether or not anything reads the stream, so + * the child never meets backpressure and there is no stall to test. A node child's + * stdio stream starts paused and stays paused, and the kernel pipe (64 KiB on Linux) + * then fills behind the first file's 512 KiB. + * + * Nothing of that stdout is returned: measured, bun destroys the stream at the exit + * event, so what the pipe still held is gone by the time there is anything to read it + * with. How long the run took is the evidence instead, and it is the better evidence: + * only a write that never completed can spend the whole grace. + */ +async function signalTheStuckRun( + { sandbox, markers, scratchParent }: WaitingRun, + signals: NodeJS.Signals[], + gapMs = 500, +): Promise<{ exitCode: number | null; signalCode: string | null; elapsedMs: number }> { + const runner = spawn(process.execPath, ["tests/run-tests.ts", "--jobs=1", "tests/unit"], { + cwd: sandbox, + env: { ...withoutRequirements(), TMPDIR: scratchParent, RUNNER_MARKERS: markers }, + stdio: ["ignore", "pipe", "pipe"] as const, + }); + // stderr is read, so only stdout is the stuck stream: a scratch-removal failure + // would otherwise be stuck too, and that is a different case, tested on its own. + (runner.stderr as Readable).resume(); + await waitForFile(path.join(markers, "second.started")); + + const startedAt = Date.now(); + const ended = new Promise<{ exitCode: number | null; signalCode: string | null }>((resolve) => { + runner.once("exit", (exitCode, signalCode) => resolve({ exitCode, signalCode })); + }); + for (const [index, signal] of signals.entries()) { + // oxlint-disable-next-line no-await-in-loop -- the gap between two signals is the point. + if (index > 0) await Bun.sleep(gapMs); + runner.kill(signal); + } + const { exitCode, signalCode } = await ended; + return { exitCode, signalCode, elapsedMs: Date.now() - startedAt }; +} + +/** + * Drives the runner with a reader that is always a little behind, the way a CI log + * consumer is: it takes a chunk, stops for a moment, and takes the next. + * + * node:child_process again, for the reason signalTheStuckRun gives: a Bun.spawn parent + * drains the pipe itself, so there is no way to be behind it. + */ +async function runWithASlowReader( + sandbox: string, + args: string[], +): Promise<{ exitCode: number | null; stdout: string; stderr: string }> { + const runner = spawn(process.execPath, ["tests/run-tests.ts", ...args], { + cwd: sandbox, + env: withoutRequirements(), + stdio: ["ignore", "pipe", "pipe"] as const, + }); + const stdout = runner.stdout as Readable; + const chunks: Buffer[] = []; + stdout.on("data", (chunk: Buffer) => { + chunks.push(chunk); + stdout.pause(); + setTimeout(() => stdout.resume(), 200); + }); + + const [stderr, exitCode] = await Promise.all([ + new Promise((resolve, reject) => { + const parts: Buffer[] = []; + (runner.stderr as Readable).on("data", (chunk: Buffer) => parts.push(chunk)); + (runner.stderr as Readable).once("end", () => resolve(Buffer.concat(parts).toString())); + (runner.stderr as Readable).once("error", reject); + }), + new Promise((resolve) => { + runner.once("close", (code) => resolve(code)); + }), + ]); + return { exitCode, stdout: Buffer.concat(chunks).toString(), stderr }; +} + +/** Drives the runner with a piped stdout, takes one chunk, and then goes away for good. */ +async function runWithAReaderThatLeaves( + sandbox: string, + selectors: string[], +): Promise<{ exitCode: number | null; firstChunk: string; stderr: string }> { + const runner = Bun.spawn([process.execPath, "tests/run-tests.ts", "--jobs=1", ...selectors], { + cwd: sandbox, + env: withoutRequirements(), + stdout: "pipe", + stderr: "pipe", + }); + const reader = (runner.stdout as ReadableStream).getReader(); + const { value } = await reader.read(); + // Measured on bun 1.4.2: cancelling closes the read end, and the runner's next write + // fails with EPIPE, which is exactly what `| head -1` does to it. + await reader.cancel(); + const stderr = await new Response(runner.stderr as ReadableStream).text(); + const exitCode = await runner.exited; + return { exitCode, firstChunk: new TextDecoder().decode(value), stderr }; +} + +function runRunner(args: string[]): { exitCode: number; stdout: string; stderr: string } { + const result = Bun.spawnSync([process.execPath, RUNNER, ...args], { cwd: root }); + return { + exitCode: result.exitCode, + stdout: result.stdout.toString(), + stderr: result.stderr.toString(), + }; +} + +describe("the test runner, end to end", () => { + test("--list prints one repository-relative path per line and nothing else", () => { + const { exitCode, stdout } = runRunner(["--list", "tests/unit/test-runner-cli.test.ts"]); + + expect(exitCode).toBe(0); + expect(stdout).toBe("tests/unit/test-runner-cli.test.ts\n"); + }); + + test("a passing file exits 0 and is reported with its test count", () => { + const { exitCode, stdout } = runRunner(["tests/unit/test-runner-options.test.ts"]); + + expect(exitCode).toBe(0); + expect(stdout).toContain("tests/unit/test-runner-options.test.ts"); + expect(stdout).toContain("PASS"); + expect(stdout).toContain("1 file: 1 passed"); + }); + + test("a single file is addressed as a path, so a name that is a substring of another does not drag it in", () => { + // `bun test tests/unit/x.test.ts` without a leading ./ is a SUBSTRING FILTER, + // which would also run every file whose path contains that string. The runner + // passes ./, so exactly one file runs. tests/unit/lib/auth.test.ts is the + // live example: tests/unit/lib/auth-jwt-config.test.ts and + // tests/unit/lib/auth-compare.test.ts share its prefix. + const { exitCode, stdout } = runRunner(["tests/unit/lib/auth.test.ts"]); + + expect(exitCode).toBe(0); + expect(stdout).toContain("1 file: 1 passed"); + expect(stdout).not.toContain("auth-jwt-config"); + }); + + test("a selector that names nothing exits 2 and says so, rather than passing an empty run", () => { + const { exitCode, stdout, stderr } = runRunner(["tests/unit/there-is-no-such-file.test.ts"]); + + expect(exitCode).toBe(2); + expect(stderr).toContain("matched no test files"); + // Nothing on stdout: the error path adds "the reason is on stderr" only once the + // run has started and stdout has something of its own to drain, and this one never + // started. The paired control is "an error after the run started", further down, + // which reaches the same error path and DOES print that line. + expect(stdout).toBe(""); + }); + + test("an unknown option exits 2 and names the option", () => { + const { exitCode, stderr } = runRunner(["--parallel"]); + + expect(exitCode).toBe(2); + expect(stderr).toContain("--parallel"); + }); + + test("a failing test file makes the runner exit 1 and prints the child's own failure output", () => { + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\ntest("deliberately failing fixture", () => {\n expect(1).toBe(2);\n});\n', + ); + const { exitCode, stdout, stderr } = runInSandbox(sandbox); + + expect(exitCode, `runner stderr: ${stderr}`).toBe(1); + expect(stdout).toContain("FAIL"); + expect(stdout).toContain("deliberately failing fixture"); + expect(stdout).toContain("re-run alone with: bun tests/run-tests.ts tests/unit/fixture.test.ts"); + }); + + test("a skipped test reaches the summary by name, because bun prints that name nowhere", () => { + // The reason a test did not run lives in its title by convention here, and bun + // reports only a count, so the runner reads its junit report. Without this, a + // Windows run that skips a dozen files says "0 fail" and names nothing. + // The second shape is the one the Windows packaging tests use: the reason sits on a + // skipped DESCRIBE, and the test inside is named only for what it checks. + const sandbox = sandboxWith( + 'import { describe, expect, test } from "bun:test";\n' + + 'test.skipIf(true)("needs a POSIX shell, which this platform has not", () => {\n' + + " expect(1).toBe(1);\n});\n" + + 'describe.skip("snap launcher [skipped: no sh on this platform]", () => {\n' + + ' test("exports SNAP_DATA", () => {\n expect(1).toBe(1);\n });\n});\n' + + 'test("runs anyway", () => {\n expect(1).toBe(1);\n});\n', + ); + const { exitCode, stdout, stderr } = runInSandbox(sandbox); + + expect(exitCode, `runner stderr: ${stderr}`).toBe(0); + expect(stdout).toContain("Files with skipped tests:"); + expect(stdout).toContain("needs a POSIX shell, which this platform has not"); + expect(stdout).toContain("snap launcher [skipped: no sh on this platform] > exports SNAP_DATA"); + }); + + test("the summary survives a failing file that printed a megabyte into a stdout that is a pipe", () => { + // bun writes to a pipe asynchronously, so anything still pending when the runner + // calls process.exit is thrown away. Measured on 1.4.2 through this same + // Bun.spawnSync capture: a failing file printing 1 MiB lost about a third of it + // AND the whole summary, so a CI log said the run was red and never said which + // file or how to re-run it. A megabyte, not the 200 KB where the loss starts, + // because the loss is timing dependent and every capture lost it at this size. + // + // What is asserted is what the RUNNER guarantees: its own last lines, and that the + // child's output reached stdout at all. The exact line count is not guaranteed and + // is not asserted: measured under CPU load, bun 1.4.2 drops part of a child's own + // queued stdout when the child exits (3 of 10 loaded runs here lost between 20% + // and 90% of it), and it does so with no runner in the picture at all, so the + // runner's drain cannot fix it and a count would be red on a busy CI machine. + const line = "k".repeat(1023); + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\n' + + 'test("prints a megabyte and then fails", () => {\n' + + ` for (let index = 0; index < 1024; index += 1) process.stdout.write("${line}\\n");\n` + + " expect(1).toBe(2);\n});\n", + ); + const { exitCode, stdout, stderr } = runInSandbox(sandbox); + + expect(exitCode, `runner stderr: ${stderr}`).toBe(1); + expect(stdout).toContain("Failed files:"); + expect(stdout).toContain("re-run alone with: bun tests/run-tests.ts tests/unit/fixture.test.ts"); + expect(stdout).toContain("1 file: 0 passed, 1 failed"); + // The paired control for the assertions above: the child's output really was in + // the way, so the summary was written after a megabyte rather than instead of it. + expect(stdout).toContain(line); + }); + + test("a reader that takes one line and leaves keeps the run's own exit code, green and red", async () => { + // `bun run test | head -1` answered 2, "the runner could not do its job", for a run + // where every test passed: the awaited summary write got EPIPE and the error path + // took the verdict away. The committed runner answered 0 and 1 here, so this was a + // regression, and it destroys the one distinction exit code 2 exists to carry. + const green = sandboxWith( + 'import { expect, test } from "bun:test";\n' + + 'test("takes a moment and passes", async () => {\n await Bun.sleep(700);\n expect(1).toBe(1);\n});\n', + ); + const passing = await runWithAReaderThatLeaves(green, ["tests/unit/fixture.test.ts"]); + + expect({ exitCode: passing.exitCode, stderr: passing.stderr }).toEqual({ exitCode: 0, stderr: "" }); + // Not vacuous: the summary had NOT been written when the reader went, so the run + // really did meet a broken pipe rather than finishing before the reader left. + expect(passing.firstChunk).toContain("1 files from tests/unit/fixture.test.ts"); + expect(passing.firstChunk).not.toContain("1 file: 1 passed"); + + // The paired control: the same sandbox, read to the end, is 0 WITH its summary. + const drained = runInSandbox(green); + expect(drained.exitCode).toBe(0); + expect(drained.stdout).toContain("1 file: 1 passed"); + + const red = sandboxWith(floodingFixture(512)); + const failing = await runWithAReaderThatLeaves(red, ["tests/unit/fixture.test.ts"]); + + expect(failing.exitCode).toBe(1); + expect(failing.firstChunk).not.toContain("Failed files:"); + expect(runInSandbox(red).exitCode).toBe(1); + }, 30_000); + + test("a file that prints more than the capture keeps is cut in the middle, and says so once", () => { + // tests/runner/capture.ts keeps 1 MiB at each end of each stream. Nothing else + // reaches that bound: the real tree's largest stdout is 154 KB, and the megabyte + // case above sits exactly on the head limit, so without this case reverting the + // two captureBounded calls to an unbounded read passes the whole suite (measured + // by mutation) and no reader ever meets the elision line in a CI log. + // + // 6 MiB against a 2 MiB threshold, and a pause before the file ends: measured under + // load, bun 1.4.2 loses part of a child's queued stdout at the child's own exit, so + // the fixture gives the runner time to read it and keeps a 3x margin over the bound. + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\n' + + 'test("prints six mebibytes and then fails", async () => {\n' + + ' const line = `${"k".repeat(1023)}\\n`;\n' + + " for (let index = 0; index < 6 * 1024; index += 1) process.stdout.write(line);\n" + + " await Bun.sleep(500);\n" + + " expect(1).toBe(2);\n});\n", + ); + const { exitCode, stdout, stderr } = runInSandbox(sandbox); + + const elisions = stdout.match(/\[runner: \d+ bytes of stdout elided here\]/g) ?? []; + expect({ exitCode, elisions: elisions.length }, `runner stderr: ${stderr}`).toEqual({ exitCode: 1, elisions: 1 }); + // The verdict is unchanged by the cut: nothing is decided from this text. + expect(stdout).toContain("FAIL"); + expect(stdout).toContain("1 file: 0 passed, 1 failed | 1 test: 0 pass, 1 fail"); + // The paired control for the count above: stderr stayed well inside its own bound, + // so "exactly one" is a statement about which stream was cut, not about parsing. + expect(stdout).not.toContain("bytes of stderr elided here"); + }, 60_000); + + test("a file that needs Helm is named as not run where Helm is unusable, and refused where it is required", () => { + // The sandbox has no charts/ directory, so Helm is unusable in it on every machine: either + // there is no helm binary, or there is one and no built chart dependency beside it. + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\ntest("adds", () => {\n expect(1 + 1).toBe(2);\n});\n', + ); + writeFileSync( + path.join(sandbox, "tests/unit/chart.test.ts"), + [ + ["//", "@requires", "helm"].join(" "), + 'import { test } from "bun:test";', + 'test("renders", () => {});', + "", + ].join("\n"), + ); + + const relaxed = runInSandbox(sandbox, ["tests/unit"]); + expect(relaxed.exitCode, `runner stderr: ${relaxed.stderr}`).toBe(0); + expect(relaxed.stdout).toContain("Files not run on this machine:"); + expect(relaxed.stdout).toContain("tests/unit/chart.test.ts"); + expect(relaxed.stdout).toContain("1 file: 1 passed"); + + const strict = runInSandbox(sandbox, ["tests/unit"], { ...withoutRequirements(), LIBREDB_REQUIRE_HELM: "1" }); + expect(strict.exitCode).toBe(2); + expect(strict.stderr).toContain("tests/unit/chart.test.ts needs helm"); + expect(strict.stderr).toContain("LIBREDB_REQUIRE_HELM=1"); + }); + + test("a file that registers no test is a failure, not a green line", () => { + const sandbox = sandboxWith('import { expect } from "bun:test";\nexpect(1).toBe(1);\n'); + const { exitCode, stdout, stderr } = runInSandbox(sandbox); + + expect(exitCode, `runner stderr: ${stderr}`).toBe(1); + expect(stdout).toContain("FAIL"); + }); + + test("a file that registers no test but prints a count line is still a failure", () => { + // Measured on 1.4.2 before the counts came from the junit report: this was PASS, exit 0. + const sandbox = sandboxWith('console.log(" 1 pass");\n'); + const { exitCode, stdout, stderr } = runInSandbox(sandbox); + + expect(exitCode, `runner stderr: ${stderr}`).toBe(1); + expect(stdout).toContain("FAIL"); + expect(stdout).toContain("wrote no test report"); + }); + + test("a test that prints a whole summary on stderr and then exits 0 cannot turn the file green", () => { + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\n' + + 'test("a", () => {\n expect(1).toBe(1);\n});\n' + + 'test("b", () => {\n console.error("\\n 7 pass\\n 0 fail\\nRan 7 tests across 1 file.");\n process.exit(0);\n});\n' + + 'test("c", () => {\n throw new Error("never reached");\n});\n', + ); + const { exitCode, stdout, stderr } = runInSandbox(sandbox); + + expect(exitCode, `runner stderr: ${stderr}`).toBe(1); + expect(stdout).toContain("FAIL"); + // The spoofed block is printed as part of the failing file's output, which is the + // control for the assertion that matters: none of it reached the run's totals. + expect(stdout).toContain(" 7 pass"); + expect(stdout).toContain("1 file: 0 passed, 1 failed | 0 tests: 0 pass"); + }); + + test("a passing test that prints fail counts is reported with the counts bun recorded", () => { + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\n' + + 'test("a", () => {\n console.log(" 1 fail");\n console.error(" 1 fail");\n expect(1).toBe(1);\n});\n', + ); + const { exitCode, stdout, stderr } = runInSandbox(sandbox); + + expect(exitCode, `runner stderr: ${stderr}`).toBe(0); + expect(stdout).toContain("1 file: 1 passed | 1 test: 1 pass |"); + expect(stdout).not.toContain("1 pass 2 fail"); + }); + + test("a --reporter-outfile forwarded to bun cannot move the report the runner reads", () => { + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\ntest("adds", () => {\n expect(1 + 1).toBe(2);\n});\n', + ); + const elsewhere = path.join(sandbox, "elsewhere.xml"); + const { exitCode, stdout, stderr } = runInSandbox(sandbox, [ + "tests/unit/fixture.test.ts", + "--", + `--reporter-outfile=${elsewhere}`, + ]); + + expect(exitCode, `runner stdout: ${stdout}\nrunner stderr: ${stderr}`).toBe(0); + expect(stdout).toContain("1 file: 1 passed | 1 test: 1 pass |"); + expect(existsSync(elsewhere)).toBe(false); + }); + + // Windows delivers none of these to a piped child (SIGINT and SIGBREAK come from a + // console event the test would have to share a console to raise, and SIGTERM is never + // delivered there at all), so the end-to-end cases are POSIX only. The stop gate they + // rest on is unit-tested on every platform in tests/unit/test-runner-execute.test.ts. + const onlyPosixSignals = process.platform === "win32"; + + for (const [signal, code] of [ + ["SIGTERM", 143], + ["SIGHUP", 129], + ["SIGINT", 130], + ] as const) { + test.skipIf(onlyPosixSignals)( + `${signal} stops the run: exit ${code}, the scratch directory removed, and no further file started [skipped: Windows delivers no ${signal} to a piped child]`, + async () => { + const waiting = sandboxThatWaits(); + const { exitCode, signalCode, stdout } = await signalTheRun(waiting, signal); + + expect({ exitCode, signalCode }).toEqual({ exitCode: code, signalCode: null }); + expect(stdout).toContain(`Interrupted (${signal}).`); + // Nothing of the run's own verdict: the files it killed are not failures. + expect(stdout).not.toContain("Failed files:"); + expect(readdirSync(waiting.scratchParent)).toEqual([]); + // The paired control for the negative: the first file did start, the second did not. + expect(existsSync(path.join(waiting.markers, "fixture.started"))).toBe(true); + expect(existsSync(path.join(waiting.markers, "second.started"))).toBe(false); + }, + 30_000, + ); + } + + test.skipIf(onlyPosixSignals)( + "SIGINT ends a run whose reader has stopped reading, rather than waiting for a write it cannot finish [skipped: Windows delivers no SIGINT to a piped child]", + async () => { + // Measured on bun 1.4.2 before this: with a consumer that had stopped reading, + // the handler's own write queued behind the full pipe, the process sat there for + // the whole stall, and when the consumer finally drained it exited 1 with the + // run's summary and "Interrupted (SIGINT)." tacked on after it. The signal has + // to win whatever the reader is doing, which costs the Interrupted line: that + // line goes to the reader that is not reading, so it is dropped when the grace + // runs out, and only the exit code and the cleanup are promised here. + const waiting = sandboxThatFloodsThenWaits(); + const { exitCode, signalCode, elapsedMs } = await signalTheStuckRun(waiting, ["SIGINT"]); + + expect({ exitCode, signalCode }).toEqual({ exitCode: 130, signalCode: null }); + expect(readdirSync(waiting.scratchParent)).toEqual([]); + expect(existsSync(path.join(waiting.markers, "second.started"))).toBe(true); + // The control that makes the exit code non-vacuous: the run really was stuck on a + // write. The handler's grace is 3 s, and only a write that never completed can + // spend it, so anything at or above 2 s says the stall was real; a signal to a run + // whose reader is reading is answered in tens of milliseconds. The upper bound is + // deliberately loose: what the runner guarantees is that the grace ENDS the wait, + // not that the process is reaped within any particular time on a loaded machine. + expect({ stalled: elapsedMs >= 2_000, prompt: elapsedMs < 20_000, elapsedMs }).toEqual({ + stalled: true, + prompt: true, + elapsedMs, + }); + }, + 45_000, + ); + + test.skipIf(onlyPosixSignals)( + "a signal to a run whose reader has left still exits 128 plus the signal, not 2 [skipped: Windows delivers no SIGINT to a piped child]", + async () => { + // The stalled case above is a write that cannot finish; this is a write that + // cannot happen at all, because the reader closed the pipe. Both end the same + // way: the run was stopped by a signal, and saying "the runner could not do its + // job" instead would hide that from whoever reads the exit code. + const waiting = sandboxThatWaits(); + const runner = Bun.spawn([process.execPath, "tests/run-tests.ts", "--jobs=1", "tests/unit"], { + cwd: waiting.sandbox, + env: { ...withoutRequirements(), TMPDIR: waiting.scratchParent, RUNNER_MARKERS: waiting.markers }, + stdout: "pipe", + stderr: "pipe", + }); + const reader = (runner.stdout as ReadableStream).getReader(); + await reader.read(); + await reader.cancel(); + await waitForFile(path.join(waiting.markers, "fixture.started")); + runner.kill("SIGINT"); + + const stderr = await new Response(runner.stderr as ReadableStream).text(); + const exitCode = await runner.exited; + + expect({ exitCode, signalCode: runner.signalCode }, `runner stderr: ${stderr}`).toEqual({ + exitCode: 130, + signalCode: null, + }); + // The cleanup is still done, which is the part that does not depend on a reader. + expect(readdirSync(waiting.scratchParent)).toEqual([]); + }, + 45_000, + ); + + test.skipIf(onlyPosixSignals)( + "a second SIGINT kills a run whose first one cannot write [skipped: Windows delivers no SIGINT to a piped child]", + async () => { + // The first signal takes the way out and restores the default disposition, so a + // user who presses Ctrl+C again is not told to go and find SIGKILL. Against the + // reviewed code the second signal was swallowed by the `interruption ??=` guard + // and by the listener still being installed. + const waiting = sandboxThatFloodsThenWaits(); + const { exitCode, signalCode } = await signalTheStuckRun(waiting, ["SIGINT", "SIGINT"]); + + // A process that dies by the default action reports no exit code of its own, + // which is what separates this from the handler's own process.exit(130) in the + // case above. Both are 130 to a shell. + expect({ exitCode, signalCode }).toEqual({ exitCode: null, signalCode: "SIGINT" }); + }, + 45_000, + ); + + test.skipIf(onlyPosixSignals)( + "a signal taken while the coverage merge is running still ends the run [skipped: Windows delivers no SIGINT to a piped child]", + async () => { + // `bun run test:coverage` ends in a synchronous Bun.spawnSync that merges 500+ + // reports. Measured on bun 1.4.2 before this: a SIGINT arriving during it was + // queued behind the sync call, and the `await main()` continuation (a microtask) + // called process.exit(0) before the signal listener (a macrotask) ever ran. The + // run exited 0, said nothing, and the user's Ctrl+C had vanished. + const merging = sandboxThatMerges( + [ + 'import { writeFileSync } from "node:fs";', + 'import path from "node:path";', + 'writeFileSync(path.join(process.env.RUNNER_MARKERS, "merging"), "");', + "const until = Date.now() + 3000;", + "while (Date.now() < until) {}", + 'writeFileSync(process.argv[3], "merged\\n");', + "", + ].join("\n"), + ); + const runner = Bun.spawn([process.execPath, "tests/run-tests.ts", ...merging.args], { + cwd: merging.sandbox, + env: { ...withoutRequirements(), RUNNER_MARKERS: merging.markers }, + stdout: "pipe", + stderr: "pipe", + }); + await waitForFile(path.join(merging.markers, "merging")); + runner.kill("SIGINT"); + + const stdout = await new Response(runner.stdout as ReadableStream).text(); + const exitCode = await runner.exited; + + expect({ exitCode, signalCode: runner.signalCode }).toEqual({ exitCode: 130, signalCode: null }); + expect(stdout).toContain("Interrupted (SIGINT)."); + // The paired control: the merge did run and did finish, so this is a signal taken + // DURING the merge and not one that arrived before it started. + expect(existsSync(path.join(merging.sandbox, "lcov.info"))).toBe(true); + }, + 45_000, + ); + + test.skipIf(onlyPosixSignals)( + "a coverage merge killed by a signal is reported by that signal, not as exit null [skipped: needs POSIX signal semantics]", + async () => { + // A terminal Ctrl+C goes to the whole foreground group, so the merge child dies + // too. Bun.spawnSync then reports exitCode null, and "failed with exit null" + // names no cause at all. + const merging = sandboxThatMerges('process.kill(process.pid, "SIGKILL");\n'); + const { exitCode, stderr, stdout } = runInSandbox(merging.sandbox, merging.args); + + expect({ exitCode, stderr: stderr.trim() }).toEqual({ + exitCode: 2, + stderr: "Merging 1 coverage reports was killed by SIGKILL.", + }); + expect(stdout).toContain("The run stopped before it finished; the reason is on stderr."); + }, + 30_000, + ); + + test("a coverage merge that exits non-zero is still reported by its exit code", () => { + // The paired control for the case above: the signal wording must not have taken + // over the ordinary failure, which is the one the exit code can describe. + const merging = sandboxThatMerges("process.exit(3);\n"); + const { exitCode, stderr } = runInSandbox(merging.sandbox, merging.args); + + expect({ exitCode, stderr: stderr.trim() }).toEqual({ + exitCode: 2, + stderr: "Merging 1 coverage reports failed with exit 3.", + }); + }, 30_000); + + test.skipIf(onlyPosixSignals || process.getuid?.() === 0)( + "a scratch directory that cannot be removed is named, and ends the run with exit 2 [skipped: needs POSIX permissions and a non-root user]", + async () => { + const waiting = sandboxThatWaits(); + const { exitCode, stderr, stdout } = await signalTheRun(waiting, "SIGINT", () => + chmodSync(waiting.scratchParent, 0o555), + ); + + expect({ exitCode, stderr }).toEqual({ exitCode: 2, stderr: expect.stringContaining("could not be removed") }); + expect(stderr).toContain(path.join(waiting.scratchParent, "libredb-test-junit-")); + expect(stdout).toContain("Interrupted (SIGINT)."); + expect(existsSync(path.join(waiting.markers, "second.started"))).toBe(false); + }, + 30_000, + ); + + test("the error path's last stdout line arrives behind 300 KB the merge left in the pipe", async () => { + // The case below is the same error path with an empty pipe in front of it, and it + // passes whether or not exitAfterWriting waits for its writes: measured on bun + // 1.4.2, process.exit does flush a small write into a pipe that has room. So the + // drain the docblock is written about is only observable with the pipe FULL when + // the last line is written, and the merge child is the one writer that can leave it + // that way, because it writes to the inherited descriptor directly and the runner's + // own stream knows nothing about what it queued there. + const merging = sandboxThatMerges( + [ + "const line = `${'m'.repeat(1023)}\\n`;", + "for (let index = 0; index < 300; index += 1) process.stdout.write(line);", + "process.exit(3);", + "", + ].join("\n"), + ); + const { exitCode, stdout, stderr } = await runWithASlowReader(merging.sandbox, merging.args); + + expect({ exitCode, last: stdout.trimEnd().split("\n").at(-1) }, `runner stderr: ${stderr}`).toEqual({ + exitCode: 2, + last: "The run stopped before it finished; the reason is on stderr.", + }); + expect(stderr).toContain("Merging 1 coverage reports failed with exit 3."); + // The paired control: the merge's own output really was in front of the last line, + // so this is a write that had to wait rather than one into an empty pipe. Far below + // the 300 KB the script writes, because two things cut it and neither is guaranteed: + // node drops its own queued stdout at process.exit, and whatever is still in the + // pipe when the runner goes is lost to the reader. What IS guaranteed is that the + // 64 KiB the pipe holds was accepted before the child left, and the runner's own + // wait is what drains it, so the reader sees at least that. + expect(stdout.length).toBeGreaterThan(60_000); + }, 60_000); + + test("an error after the run started is reported on stderr, and stdout is told the run stopped", () => { + // The error path exits 2 straight away rather than letting the run finish, so + // whatever it has already printed to stdout has to be drained with a write of its + // own. The sandbox has no src/, so no child covers a source file and the merge has + // nothing to merge, which is the one error this can reach after the run has begun. + const sandbox = sandboxWith( + 'import { expect, test } from "bun:test";\ntest("adds", () => {\n expect(1 + 1).toBe(2);\n});\n', + ); + const { exitCode, stdout, stderr } = runInSandbox(sandbox, [ + "tests/unit/fixture.test.ts", + "--coverage", + `--coverage-dir=${path.join(sandbox, "raw")}`, + `--merge-into=${path.join(sandbox, "lcov.info")}`, + ]); + + expect(exitCode, `runner stdout: ${stdout}`).toBe(2); + expect(stderr).toContain("No coverage report was written"); + expect(stdout).toContain("1 file: 1 passed"); + expect(stdout).toContain("The run stopped before it finished; the reason is on stderr."); + }); + + test("--coverage writes one report per file and --merge-into merges them", () => { + const workDir = mkdtempSync(path.join(tmpdir(), "runner-coverage-")); + try { + const rawDir = path.join(workDir, "raw"); + const merged = path.join(workDir, "lcov.info"); + const { exitCode, stdout } = runRunner([ + "tests/unit/test-runner-discovery.test.ts", + "--coverage", + `--coverage-dir=${rawDir}`, + `--merge-into=${merged}`, + ]); + + expect(exitCode).toBe(0); + expect(stdout).toContain("with coverage"); + expect(existsSync(path.join(rawDir, "file-1", "lcov.info"))).toBe(true); + // The merged report keeps only src/ records, so the runner's own module is + // absent from it by design; what matters here is that the merge ran and + // wrote a report the coverage gate can read. + expect(existsSync(merged)).toBe(true); + expect(readFileSync(path.join(rawDir, "inputs.txt"), "utf8")).toContain("file-1/lcov.info"); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/unit/test-runner-coverage.test.ts b/tests/unit/test-runner-coverage.test.ts new file mode 100644 index 000000000..d22817f7a --- /dev/null +++ b/tests/unit/test-runner-coverage.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from "bun:test"; +import { assertCoverageDirIsOurs, assertMergeTargetIsOurs, unownedCoverageEntries } from "../runner/coverage"; + +// The runner empties its coverage directory before every coverage run, so the one +// argument that decides which directory that is has to be the one argument it +// validates. Without this, `--coverage-dir=src` deletes the product. +describe("emptying the coverage directory", () => { + test("a directory that does not exist yet, or is empty, is ours to use", () => { + expect(unownedCoverageEntries([])).toEqual([]); + expect(() => assertCoverageDirIsOurs("coverage/raw", [])).not.toThrow(); + }); + + test("a previous coverage run of this runner is ours to empty", () => { + const previous = ["file-1", "file-2", "file-538", "lcov.info", "inputs.txt"]; + + expect(unownedCoverageEntries(previous)).toEqual([]); + expect(() => assertCoverageDirIsOurs("coverage/raw", previous)).not.toThrow(); + }); + + test("anything else stops the run, and the message names what was found", () => { + expect(() => assertCoverageDirIsOurs("src", ["app", "lib", "components", "file-1"])).toThrow( + /Refusing to empty src: it holds 3 entries this runner did not write \(app, components, lib\)/, + ); + }); + + test("a single stranger is reported in the singular, because the sentence is read by a person", () => { + expect(() => assertCoverageDirIsOurs("coverage", ["html"])).toThrow(/holds 1 entry this runner did not write/); + }); + + test("a name that merely looks like ours is not ours", () => { + // `file-1` is the runner's; `file-one` and `lcov.info.bak` are somebody's work. + expect(unownedCoverageEntries(["file-1", "file-one", "lcov.info.bak"])).toEqual(["file-one", "lcov.info.bak"]); + }); +}); + +// The merged report is removed before a coverage run, so a red run cannot leave a +// stale one behind for coverage:check to pass. The path it removes is the caller's +// choice, so it gets the same treatment as the directory. +describe("replacing the merged report", () => { + test("a report bun wrote, or the merge wrote, or an empty one, is ours to replace", () => { + expect(() => assertMergeTargetIsOurs("coverage/lcov.info", "TN:\nSF:src/a.ts\nend_of_record\n")).not.toThrow(); + expect(() => assertMergeTargetIsOurs("coverage/lcov.info", "SF:src/a.ts\nDA:1,1\nend_of_record\n")).not.toThrow(); + expect(() => assertMergeTargetIsOurs("coverage/lcov.info", "")).not.toThrow(); + }); + + test("anything else stops the run before it is deleted, and says what it found", () => { + expect(() => assertMergeTargetIsOurs("package.json", '{\n "name": "@libredb/studio",\n')).toThrow( + /Refusing to replace package.json: it is not a coverage report \(it starts "\{"\)/, + ); + }); +}); diff --git a/tests/unit/test-runner-discovery.test.ts b/tests/unit/test-runner-discovery.test.ts new file mode 100644 index 000000000..3b4b750b2 --- /dev/null +++ b/tests/unit/test-runner-discovery.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, test } from "bun:test"; +import { lstatSync, mkdirSync, mkdtempSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { testIf } from "../helpers/posix-tools"; +import { COVERAGE_EXEMPT_FILES, discoverTestFiles, selectTestFiles } from "../runner/discover"; + +// The guard that replaces tests/unit/component-runner-coverage.test.ts: that file +// existed because tests/run-components.sh named its files by hand, so a new file +// could be added and never run (#426 shipped seven tests that never ran once). +// Discovery is now automatic, so the invariant worth pinning is the other way +// round: the rule the runner applies must equal what is on disk, and no directory +// may quietly fall outside it. +const root = path.resolve(import.meta.dir, "../.."); + +// The oracle classifies with lstat, not with the Dirent: a Dirent for a link is +// neither a directory nor a file, so an oracle built on it would share the blind +// spot this test exists to catch, and agree with a runner that skipped a link. +function walk(directory: string): string[] { + return readdirSync(path.join(root, directory)).flatMap((name) => { + const child = `${directory}/${name}`; + const stats = lstatSync(path.join(root, child)); + if (stats.isSymbolicLink()) throw new Error(`${child} is a link, which the runner refuses`); + if (stats.isDirectory()) return walk(child); + return stats.isFile() && /\.test\.tsx?$/.test(name) ? [child] : []; + }); +} + +/** A throwaway repository root holding only `tests/unit/real.test.ts`; the caller removes it. */ +function throwawayRoot(): string { + const scratch = mkdtempSync(path.join(tmpdir(), "runner-links-")); + mkdirSync(path.join(scratch, "tests/unit"), { recursive: true }); + writeFileSync(path.join(scratch, "tests/unit/real.test.ts"), ""); + return scratch; +} + +// A file symlink on Windows needs SeCreateSymbolicLinkPrivilege or Developer Mode, +// which a runner cannot count on; a directory junction needs neither, so the +// junction case carries the Windows measurement and only the file case is skipped. +const MISSING_FILE_SYMLINKS: string | null = + process.platform === "win32" ? "file symlinks need a privilege or Developer Mode on Windows" : null; + +describe("test discovery", () => { + test("runs every *.test.ts(x) file under tests/, except tests/live", () => { + const onDisk = walk("tests") + .filter((file) => !file.startsWith("tests/live/")) + .sort(); + + expect(discoverTestFiles(root)).toEqual(onDisk); + }); + + test("discovers the layers the suite is made of, and each one is non-empty", () => { + const files = discoverTestFiles(root); + const layers = ["unit", "api", "integration", "hooks", "security", "evals", "components", "isolated"]; + + for (const layer of layers) { + expect(files.filter((file) => file.startsWith(`tests/${layer}/`)).length).toBeGreaterThan(0); + } + // Nothing outside those layers: a new top-level directory has to be added to + // the list above deliberately, which is where someone reads this test. + const outside = files.filter((file) => !layers.some((layer) => file.startsWith(`tests/${layer}/`))); + expect(outside).toEqual([]); + }); + + test("excludes tests/live, which drives real engines and is run by hand", () => { + expect(discoverTestFiles(root).some((file) => file.startsWith("tests/live/"))).toBe(false); + }); + + test("returns POSIX-separated paths, sorted, with no duplicates", () => { + const files = discoverTestFiles(root); + + expect(files.some((file) => file.includes("\\"))).toBe(false); + expect([...files].sort()).toEqual(files); + expect(new Set(files).size).toBe(files.length); + }); + + test("a selector may be a layer directory", () => { + const selected = selectTestFiles(root, ["tests/unit"]); + + expect(selected.length).toBeGreaterThan(0); + expect(selected.every((file) => file.startsWith("tests/unit/"))).toBe(true); + expect(selected).toEqual(discoverTestFiles(root).filter((file) => file.startsWith("tests/unit/"))); + }); + + test("a selector may be a single test file, spelled with either separator", () => { + const one = "tests/unit/test-runner-discovery.test.ts"; + + expect(selectTestFiles(root, [one])).toEqual([one]); + expect(selectTestFiles(root, [one.replaceAll("/", path.sep)])).toEqual([one]); + expect(selectTestFiles(root, [path.join(root, one)])).toEqual([one]); + }); + + test("a relative selector means what it means in the directory it was typed in", () => { + // From tests/unit, `bun ../run-tests.ts lib/lazy.test.ts` names the file beside + // you. Resolving against the repository root instead answered "is not under + // tests/", a true sentence about a path nobody wrote. + const cwd = process.cwd(); + try { + process.chdir(path.join(root, "tests/unit")); + expect(selectTestFiles(root, ["lib/lazy.test.ts"])).toEqual(["tests/unit/lib/lazy.test.ts"]); + } finally { + process.chdir(cwd); + } + }); + + test("a root spelled differently from the working directory is still the same directory", () => { + // Measured on windows-latest, 2026-09-15: os.tmpdir() answers the 8.3 short form + // (C:\Users\RUNNER~1\...) while import.meta.dir answers the long one + // (C:\Users\runneradmin\...), so a runner started from a temp directory resolved a + // correct relative selector to a path "not under tests/" and exited 2. A junction + // reproduces the same two spellings of one directory on every platform (on POSIX + // the type argument is ignored and it is an ordinary directory symlink). + const link = path.join(mkdtempSync(path.join(tmpdir(), "runner-spelling-")), "repo"); + symlinkSync(root, link, "junction"); + try { + expect(selectTestFiles(link, ["tests/unit/lib/lazy.test.ts"])).toEqual(["tests/unit/lib/lazy.test.ts"]); + } finally { + rmSync(path.dirname(link), { recursive: true, force: true }); + } + }); + + test("a relative selector that passes through a link names the file behind the link", () => { + // `u/x.test.ts`, typed where u is a link to tests/unit, is tests/unit/x.test.ts, as + // the same path given absolutely already was. Only the working directory used to + // be resolved, so the link's own spelling reached path.relative and the selector + // was "not under tests/". A junction needs no privilege on Windows; on POSIX the + // type argument is ignored and it is an ordinary directory symlink. + const unitFiles = discoverTestFiles(root).filter((file) => file.startsWith("tests/unit/")); + const one = unitFiles[0]; + if (one === undefined) throw new Error("tests/unit holds no test file to select through a link"); + const scratch = mkdtempSync(path.join(tmpdir(), "runner-selector-link-")); + symlinkSync(path.join(root, "tests/unit"), path.join(scratch, "u"), "junction"); + symlinkSync(path.join(root, "src"), path.join(scratch, "s"), "junction"); + const cwd = process.cwd(); + try { + process.chdir(scratch); + expect(selectTestFiles(root, [`u/${one.slice("tests/unit/".length)}`])).toEqual([one]); + expect(selectTestFiles(root, ["u"])).toEqual(unitFiles); + // A link that leads outside tests/ is still outside tests/. + expect(() => selectTestFiles(root, ["s"])).toThrow(/"s" is not under tests\//); + } finally { + process.chdir(cwd); + rmSync(scratch, { recursive: true, force: true }); + } + }); + + test("a directory link under tests/ is refused by name, never silently skipped", () => { + const repository = throwawayRoot(); + const elsewhere = mkdtempSync(path.join(tmpdir(), "runner-links-target-")); + try { + writeFileSync(path.join(elsewhere, "other.test.ts"), ""); + // The control: the same root without the link is discovered normally. + expect(discoverTestFiles(repository)).toEqual(["tests/unit/real.test.ts"]); + + symlinkSync(elsewhere, path.join(repository, "tests/unit/linked"), "junction"); + expect(() => discoverTestFiles(repository)).toThrow(/tests\/unit\/linked .*does not follow links/); + expect(() => selectTestFiles(repository, ["tests/unit/real.test.ts"])).toThrow(/tests\/unit\/linked/); + } finally { + rmSync(repository, { recursive: true, force: true }); + rmSync(elsewhere, { recursive: true, force: true }); + } + }); + + testIf(MISSING_FILE_SYMLINKS, "a file link under tests/ is refused by name, never silently skipped", () => { + const repository = throwawayRoot(); + const elsewhere = mkdtempSync(path.join(tmpdir(), "runner-links-target-")); + try { + writeFileSync(path.join(elsewhere, "target.test.ts"), ""); + // The control: the same root without the link is discovered normally. + expect(discoverTestFiles(repository)).toEqual(["tests/unit/real.test.ts"]); + + symlinkSync(path.join(elsewhere, "target.test.ts"), path.join(repository, "tests/unit/file.test.ts"), "file"); + expect(() => discoverTestFiles(repository)).toThrow(/tests\/unit\/file\.test\.ts .*does not follow links/); + } finally { + rmSync(repository, { recursive: true, force: true }); + rmSync(elsewhere, { recursive: true, force: true }); + } + }); + + test("selecting nothing selects everything", () => { + expect(selectTestFiles(root, [])).toEqual(discoverTestFiles(root)); + }); + + test("a selector that matches no test file is an error, never a quiet empty run", () => { + expect(() => selectTestFiles(root, ["tests/unit/there-is-no-such.test.ts"])).toThrow( + /tests\/unit\/there-is-no-such\.test\.ts/, + ); + expect(() => selectTestFiles(root, ["tests/live"])).toThrow(/no test files/); + expect(() => selectTestFiles(root, ["src/lib"])).toThrow(/tests\//); + }); + + test("a relative selector that does not exist is refused for what it is, not for where it is", () => { + // The relative resolution has to hold for a path that is not on disk too, or the + // runner answers the wrong question: resolved against the repository root instead, + // `no-such.test.ts` typed in tests/unit becomes /no-such.test.ts and is + // refused as "not under tests/", which is a true sentence about a path nobody wrote + // and sends the reader looking for a directory problem instead of a typo. + const cwd = process.cwd(); + try { + process.chdir(path.join(root, "tests/unit")); + expect(() => selectTestFiles(root, ["no-such.test.ts"])).toThrow(/matched no test files/); + // The control: the same working directory, a selector that is really outside tests/. + expect(() => selectTestFiles(root, ["../../src/lib"])).toThrow(/is not under tests\//); + } finally { + process.chdir(cwd); + } + }); + + test("every coverage-exempt file exists and is discovered", () => { + const files = new Set(discoverTestFiles(root)); + + expect(COVERAGE_EXEMPT_FILES.length).toBeGreaterThan(0); + for (const file of COVERAGE_EXEMPT_FILES) { + expect(files.has(file)).toBe(true); + } + }); +}); diff --git a/tests/unit/test-runner-execute.test.ts b/tests/unit/test-runner-execute.test.ts new file mode 100644 index 000000000..61fa068c3 --- /dev/null +++ b/tests/unit/test-runner-execute.test.ts @@ -0,0 +1,352 @@ +import { describe, expect, test } from "bun:test"; +import type { SpawnOutcome } from "../runner/execute"; +import { runTestFiles } from "../runner/execute"; + +/** A junit report in the shape bun 1.4.2 writes, with these counts. */ +function junit({ pass = 0, fail = 0, skip = 0 }: { pass?: number; fail?: number; skip?: number }): string { + const cases = [ + ...Array.from({ length: pass }, (_, index) => ` `), + ...Array.from( + { length: fail }, + (_, index) => ` `, + ), + ...Array.from( + { length: skip }, + (_, index) => ` `, + ), + ]; + const tests = pass + fail + skip; + return ` + + +${cases.join("\n")} + +`; +} + +function passed(overrides: Partial = {}): SpawnOutcome { + return { + exitCode: 0, + signal: null, + output: "(a passing file's output)", + durationMs: 1, + timedOut: false, + junitReport: junit({ pass: 3 }), + ...overrides, + }; +} + +const files = ["tests/unit/a.test.ts", "tests/unit/b.test.ts", "tests/unit/c.test.ts"]; + +async function run(overrides: Partial[0]> = {}) { + return runTestFiles({ + files, + jobs: 2, + timeoutMs: 1000, + coverage: false, + coverageDir: "coverage/raw", + coverageExempt: [], + runFile: async () => passed(), + ...overrides, + }); +} + +describe("running the files", () => { + test("a run where every file passes is a pass, with the tests counted", async () => { + const summary = await run(); + + expect(summary.failures).toEqual([]); + expect(summary.totals.files).toBe(3); + expect(summary.totals.filesPassed).toBe(3); + expect(summary.totals.tests.pass).toBe(9); + expect(summary.totals.tests.fail).toBe(0); + }); + + test("never more than `jobs` files at once, and every file runs exactly once", async () => { + let running = 0; + let peak = 0; + const seen: string[] = []; + + await run({ + jobs: 2, + files: Array.from({ length: 9 }, (_, index) => `tests/unit/${index}.test.ts`), + runFile: async ({ file }) => { + seen.push(file); + running += 1; + peak = Math.max(peak, running); + await Promise.resolve(); + await Promise.resolve(); + running -= 1; + return passed(); + }, + }); + + expect(peak).toBe(2); + expect(seen).toHaveLength(9); + expect(new Set(seen).size).toBe(9); + }); + + test("a non-zero exit is a failed file, whatever the code is", async () => { + // A test calling process.exit(7) propagates 7 verbatim: measured with bun 1.4.2. + const summary = await run({ + runFile: async ({ file }) => + file.endsWith("b.test.ts") ? passed({ exitCode: 7, output: "boom", junitReport: null }) : passed(), + }); + + expect(summary.failures.map((outcome) => outcome.file)).toEqual(["tests/unit/b.test.ts"]); + expect(summary.failures[0]?.status).toBe("failed"); + expect(summary.totals.filesFailed).toBe(1); + }); + + test("a child killed by a signal is a failed file, not a passed one", async () => { + // bun reports a signal death as exitCode null; `exitCode === 0` is false for it, + // but `exitCode || 0` would turn it into a pass, which is the trap this pins. + const summary = await run({ + runFile: async ({ file }) => + file.endsWith("c.test.ts") + ? passed({ exitCode: null, signal: "SIGSEGV", output: "", junitReport: null }) + : passed(), + }); + + expect(summary.failures.map((outcome) => outcome.file)).toEqual(["tests/unit/c.test.ts"]); + expect(summary.failures[0]?.signal).toBe("SIGSEGV"); + }); + + test("a file that outran the timeout is reported as timed out, with what it printed", async () => { + const summary = await run({ + runFile: async ({ file }) => + file.endsWith("a.test.ts") + ? passed({ + exitCode: null, + signal: "SIGTERM", + output: "hung here", + durationMs: 1000, + timedOut: true, + junitReport: null, + }) + : passed(), + }); + + expect(summary.failures[0]?.status).toBe("timed-out"); + expect(summary.failures[0]?.output).toBe("hung here"); + expect(summary.totals.filesTimedOut).toBe(1); + }); + + test("a failing file's counts are still read, so the summary is the whole run", async () => { + const summary = await run({ + runFile: async ({ file }) => + file.endsWith("b.test.ts") ? passed({ exitCode: 1, junitReport: junit({ pass: 2, fail: 1 }) }) : passed(), + }); + + expect(summary.totals.tests.pass).toBe(8); + expect(summary.totals.tests.fail).toBe(1); + }); + + test("a file that wrote no report is counted as unknown, never as zero", async () => { + const summary = await run({ + runFile: async () => passed({ exitCode: 1, output: "segfault", junitReport: null }), + }); + + expect(summary.totals.tests.pass).toBe(0); + expect(summary.totals.filesWithoutCounts).toBe(3); + expect(summary.outcomes.map((outcome) => outcome.report)).toEqual(["missing", "missing", "missing"]); + }); + + test("console text that looks like a summary cannot make a file pass", async () => { + // Measured on 1.4.2: a file that registers no test and prints " 1 pass" exits 0, and + // so does a test that prints a whole summary on stderr and then calls process.exit(0). + // Neither writes a junit report, and a console reader took both for a pass. + const spoofed = " 1 pass\n 0 pass\n 0 fail\nRan 7 tests across 1 file. [3.00ms]\n"; + const summary = await run({ + runFile: async ({ file }) => + file.endsWith("a.test.ts") ? passed({ output: spoofed, junitReport: null }) : passed({ output: spoofed }), + }); + + expect(summary.failures.map((outcome) => outcome.file)).toEqual(["tests/unit/a.test.ts"]); + expect(summary.totals.filesWithoutCounts).toBe(1); + // The control: the same console text beside a real report changes nothing. + expect(summary.totals.filesPassed).toBe(2); + expect(summary.totals.tests.pass).toBe(6); + }); + + test("a passing file that prints ' 1 fail' has no failure in the totals", async () => { + const summary = await run({ + runFile: async () => passed({ output: " 1 fail\n 0 pass\n 1 fail\n", junitReport: junit({ pass: 1 }) }), + }); + + expect(summary.failures).toEqual([]); + expect(summary.totals.tests).toEqual({ pass: 3, fail: 0, skip: 0, todo: 0 }); + }); + + test("a report that exists but cannot be read fails the file, and is told apart from a missing one", async () => { + const summary = await run({ + runFile: async ({ file }) => + file.endsWith("b.test.ts") ? passed({ junitReport: ' outcome.file)).toEqual(["tests/unit/b.test.ts"]); + expect(summary.failures[0]?.report).toBe("unreadable"); + expect(summary.outcomes.filter((outcome) => outcome.report === "read")).toHaveLength(2); + }); + + test("the skipped titles come from the report too", async () => { + const summary = await run({ + files: ["tests/unit/a.test.ts"], + runFile: async () => passed({ junitReport: junit({ pass: 1, skip: 1 }) }), + }); + + expect(summary.outcomes[0]?.skippedTests).toEqual(["skipped 0"]); + expect(summary.outcomes[0]?.counts).toEqual({ pass: 1, fail: 0, skip: 1, todo: 0 }); + }); + + test("a passing file's output is not kept once it has been reported, while a failing file's is", async () => { + // Every outcome lives until the run ends, so keeping the output of files nobody + // will look at again is what makes the runner's memory grow with the file count: + // measured on 1.4.2, four passing files printing 100 MB each peaked at 406 MB + // against 249 MB for one. + const reported: string[] = []; + const summary = await run({ + jobs: 1, + runFile: async ({ file }) => + file.endsWith("b.test.ts") + ? passed({ exitCode: 1, output: "the failure diff", junitReport: junit({ fail: 1 }) }) + : passed({ output: "chatter" }), + onResult: (outcome) => reported.push(outcome.output), + }); + + // The control: a passing file's output does reach whoever reports it. + expect(reported).toEqual(["chatter", "the failure diff", "chatter"]); + expect(summary.outcomes.map((outcome) => outcome.output)).toEqual(["", "the failure diff", ""]); + expect(summary.failures[0]?.output).toBe("the failure diff"); + }); + + test("each file is given its own coverage directory, and exempt files get none", async () => { + const given: Array = []; + + await run({ + coverage: true, + coverageDir: "out/raw", + coverageExempt: ["tests/unit/b.test.ts"], + runFile: async ({ coverageDir }) => { + given.push(coverageDir); + return passed(); + }, + }); + + expect(given.sort()).toEqual([null, "out/raw/file-1", "out/raw/file-3"]); + }); + + test("without --coverage no child is given a coverage directory", async () => { + const given: Array = []; + + await run({ + runFile: async ({ coverageDir }) => { + given.push(coverageDir); + return passed(); + }, + }); + + expect(given).toEqual([null, null, null]); + }); + + test("every result is reported as it lands, with a running position", async () => { + const progress: string[] = []; + + await run({ + jobs: 1, + onResult: (outcome, position, total) => progress.push(`${position}/${total} ${outcome.file}`), + }); + + expect(progress).toEqual(["1/3 tests/unit/a.test.ts", "2/3 tests/unit/b.test.ts", "3/3 tests/unit/c.test.ts"]); + }); + + test("a file whose report counts no test at all is a failure, even though bun exits 0", async () => { + // Measured with bun 1.4.2: a file with no test in it exits 0 and writes no report at + // all, which the missing-report rule already fails. A report saying tests="0" gets + // the same verdict: the runner's own rule is that a discovered file runs. + const summary = await run({ + runFile: async ({ file }) => (file.endsWith("a.test.ts") ? passed({ junitReport: junit({}) }) : passed()), + }); + + expect(summary.failures.map((outcome) => outcome.file)).toEqual(["tests/unit/a.test.ts"]); + expect(summary.totals.filesFailed).toBe(1); + }); + + test("a file that exits 0 without writing a report is a failure, not a pass", async () => { + // A test calling process.exit(0) ends the process there: bun exits 0, writes no + // report, and every test after that line never runs. Measured with bun 1.4.2. + // Counting that as a pass is the one way this runner could report a green run over + // a tree whose tests did not all run. + const summary = await run({ + runFile: async ({ file }) => + file.endsWith("c.test.ts") ? passed({ output: "bun test v1.4.2\n", junitReport: null }) : passed(), + }); + + expect(summary.failures.map((outcome) => outcome.file)).toEqual(["tests/unit/c.test.ts"]); + expect(summary.totals.filesWithoutCounts).toBe(1); + }); + + test("a file whose report records a failure is a failed file, whatever its exit code says", async () => { + // The verdict is read from the report, so a report carrying a failure is a failure + // even when the child exited 0: otherwise the same summary prints "1 fail" in its + // totals under a green file line and a green run. I could not make bun 1.4.2 exit 0 + // with a failure in its report (process.on("exit") setting exitCode, a beforeExit + // handler calling process.exit(0), a stubbed process.exit, test.failing, retry, + // --todo and --bail all still exit 1), so this is injected: it is the guard for a + // shape the next bun may allow, and the counts are what the runner trusts. + const summary = await run({ + runFile: async ({ file }) => + file.endsWith("b.test.ts") ? passed({ exitCode: 0, junitReport: junit({ pass: 2, fail: 1 }) }) : passed(), + }); + + expect(summary.failures.map((outcome) => outcome.file)).toEqual(["tests/unit/b.test.ts"]); + expect(summary.totals.filesFailed).toBe(1); + // The control: the other two files exit 0 with a report holding no failure and pass. + expect(summary.totals.filesPassed).toBe(2); + expect(summary.totals.tests.fail).toBe(1); + }); + + test("a child that finished as the timeout fired is read by its exit code, not by the timer", async () => { + const summary = await run({ + runFile: async () => passed({ timedOut: true }), + }); + + expect(summary.failures).toEqual([]); + expect(summary.totals.filesTimedOut).toBe(0); + }); + + test("a run that has been stopped starts no further file", async () => { + // What a stop signal needs: the handler kills the children it has and sets the + // gate, and no worker may pick up the next file while the handler is writing its + // last line and removing the run's scratch directory. Measured on 1.4.2 without + // it: a Ctrl+C whose cleanup failed left the runner alive and the queue ran on. + const started: string[] = []; + let stop = false; + + const summary = await run({ + jobs: 1, + runFile: async ({ file }) => { + started.push(file); + stop = true; + return passed(); + }, + shouldStop: () => stop, + }); + + expect(started).toEqual(["tests/unit/a.test.ts"]); + expect(summary.outcomes).toHaveLength(1); + }); + + test("a run nobody stopped runs every file", async () => { + // The control for the case above: the same shape with the gate closed. + const started: string[] = []; + + await run({ jobs: 1, runFile: async ({ file }) => (started.push(file), passed()), shouldStop: () => false }); + + expect(started).toEqual(files); + }); + + test("a runner that is handed no files refuses rather than reporting a green run", async () => { + await expect(run({ files: [] })).rejects.toThrow(/no test files/i); + }); +}); diff --git a/tests/unit/test-runner-options.test.ts b/tests/unit/test-runner-options.test.ts new file mode 100644 index 000000000..7a924f5d4 --- /dev/null +++ b/tests/unit/test-runner-options.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from "bun:test"; +import { parseRunnerArgs } from "../runner/options"; + +const defaults = { cpuCount: 8 }; + +describe("runner command line", () => { + test("no arguments runs everything, one job per CPU", () => { + const options = parseRunnerArgs([], defaults); + + expect(options.selectors).toEqual([]); + expect(options.jobs).toBe(8); + expect(options.coverage).toBe(false); + expect(options.mergeInto).toBeNull(); + expect(options.list).toBe(false); + expect(options.bunArgs).toEqual([]); + }); + + test("positional arguments are selectors, in the order given", () => { + expect(parseRunnerArgs(["tests/unit", "tests/api/db-objects.test.ts"], defaults).selectors).toEqual([ + "tests/unit", + "tests/api/db-objects.test.ts", + ]); + }); + + test("--jobs overrides the CPU count", () => { + expect(parseRunnerArgs(["--jobs=3"], defaults).jobs).toBe(3); + }); + + test("a single job is allowed, zero and negative and fractional are not", () => { + expect(parseRunnerArgs(["--jobs=1"], defaults).jobs).toBe(1); + expect(() => parseRunnerArgs(["--jobs=0"], defaults)).toThrow(/--jobs/); + expect(() => parseRunnerArgs(["--jobs=-2"], defaults)).toThrow(/--jobs/); + expect(() => parseRunnerArgs(["--jobs=2.5"], defaults)).toThrow(/--jobs/); + expect(() => parseRunnerArgs(["--jobs=many"], defaults)).toThrow(/--jobs/); + }); + + test("a machine that reports no CPU count still gets one job", () => { + expect(parseRunnerArgs([], { cpuCount: 0 }).jobs).toBe(1); + }); + + test("--coverage collects per-file reports under the default directory", () => { + const options = parseRunnerArgs(["--coverage"], defaults); + + expect(options.coverage).toBe(true); + expect(options.coverageDir).toBe("coverage/raw"); + expect(options.mergeInto).toBeNull(); + }); + + test("--merge-into implies coverage and names the merged report", () => { + const options = parseRunnerArgs(["--merge-into=coverage/lcov.info"], defaults); + + expect(options.coverage).toBe(true); + expect(options.mergeInto).toBe("coverage/lcov.info"); + }); + + test("--coverage-dir moves the per-file reports", () => { + expect(parseRunnerArgs(["--coverage", "--coverage-dir=out/raw"], defaults).coverageDir).toBe("out/raw"); + }); + + test("--file-timeout is seconds, and must be a positive number", () => { + expect(parseRunnerArgs(["--file-timeout=90"], defaults).fileTimeoutMs).toBe(90_000); + expect(() => parseRunnerArgs(["--file-timeout=0"], defaults)).toThrow(/--file-timeout/); + expect(() => parseRunnerArgs(["--file-timeout=nope"], defaults)).toThrow(/--file-timeout/); + }); + + test("--list asks what would run", () => { + expect(parseRunnerArgs(["--list"], defaults).list).toBe(true); + }); + + test("everything after -- goes to bun test verbatim", () => { + const options = parseRunnerArgs(["tests/unit", "--", "--bail", "--timeout=20000"], defaults); + + expect(options.selectors).toEqual(["tests/unit"]); + expect(options.bunArgs).toEqual(["--bail", "--timeout=20000"]); + }); + + test("an unknown option is refused by name, never ignored", () => { + expect(() => parseRunnerArgs(["--parallel"], defaults)).toThrow(/--parallel/); + }); + + test("a bun flag that arrived without its -- is told why, and given the form that works", () => { + // `bun run test -- --bail` reaches this parser as ["--bail"]: `bun run` eats the + // first --, and so does bun when it sits straight after the script path (measured + // on 1.4.2 with an argv probe). Only `bun tests/run-tests.ts -- ` + // survives. Before this, the refusal answered "everything for bun test goes after + // --", which is exactly what the user had just written. + const refusal = (() => { + try { + parseRunnerArgs(["--bail"], defaults); + return null; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + })(); + + expect(refusal).toContain('Unknown option "--bail"'); + expect(refusal).toContain("bun run"); + expect(refusal).toContain("removes the first --"); + expect(refusal).toContain("bun tests/run-tests.ts -- "); + }); + + test("the sentence about bun run is not printed for an option that came through --", () => { + // The paired control: past a --, nothing is refused at all, so the advice above + // belongs to the refusal and not to every run. + expect(parseRunnerArgs(["tests/unit", "--", "--bail"], defaults).bunArgs).toEqual(["--bail"]); + }); + + test("a value written as a separate argument is refused with the form that works", () => { + expect(() => parseRunnerArgs(["--jobs", "4"], defaults)).toThrow(/--jobs=4/); + }); +}); diff --git a/tests/unit/test-runner-report.test.ts b/tests/unit/test-runner-report.test.ts new file mode 100644 index 000000000..3eb42b7f7 --- /dev/null +++ b/tests/unit/test-runner-report.test.ts @@ -0,0 +1,556 @@ +import { describe, expect, test } from "bun:test"; +import type { FileOutcome, RunSummary } from "../runner/execute"; +import type { TestReport } from "../runner/report"; +import { formatFileLine, formatSummary, parseSkippedTests, readTestReport } from "../runner/report"; + +describe("reading the counts from bun's junit report", () => { + // Written by bun 1.4.2 for a file with one passing, one failing, one skipped and one + // todo test (`bun test --reporter=junit`), verbatim. + const mixed = ` + + + + + AssertionError: expect(received).toBe(expected) Expected: 2 Received: 1 at tests/mixed.test.ts:3:29 + + + + + + + + +`; + + test("reads pass, fail, skip and todo, where bun counts a todo among the skipped", () => { + expect(readTestReport(mixed)).toEqual({ state: "read", counts: { pass: 1, fail: 1, skip: 1, todo: 1 } }); + }); + + test("a --bail run still counts its failure, although bun prints no count lines for it", () => { + // Measured on 1.4.2: under --bail stderr carries "Bailed out after 1 failure" and no + // " N fail" line at all, so a console reader saw nothing. The report still has it. + const bail = ` + + + + AssertionError + + +`; + + expect(readTestReport(bail)).toEqual({ state: "read", counts: { pass: 0, fail: 1, skip: 0, todo: 0 } }); + }); + + test("no report is unknown, never zero", () => { + expect(readTestReport(null)).toEqual({ state: "missing", counts: null }); + }); + + test("a report that exists but cannot be read is unknown too, and is unreadable rather than missing", () => { + const unreadable: TestReport = { state: "unreadable", counts: null }; + // No element at all. + expect(readTestReport("")).toEqual(unreadable); + expect(readTestReport('')).toEqual(unreadable); + // An attribute the counts need is missing, or is not a count. + expect(readTestReport('')).toEqual(unreadable); + expect(readTestReport('')).toEqual(unreadable); + // Cut off before its end: the todo count is read from the elements, so it cannot be trusted. + expect(readTestReport(mixed.slice(0, mixed.indexOf("")))).toEqual(unreadable); + // Counts that contradict each other. + expect(readTestReport('')).toEqual(unreadable); + // The control: the same shape with counts that agree is read. + expect(readTestReport('')).toEqual({ + state: "read", + counts: { pass: 0, fail: 1, skip: 1, todo: 0 }, + }); + }); +}); + +describe("reading which tests were skipped", () => { + // bun prints a skipped test's title NOWHERE: measured on 1.4.2 piped, with + // FORCE_COLOR, and under a real pty, the output carries only " 1 skip". Its junit + // reporter does name them, which is why the runner asks for one per file. + const report = ` + + + + + + + + + + +`; + + // Written by bun 1.4.2 for a file with describes titled "rows where count > 100" and + // `inner & "quoted" `, a test.skip named "skipped a > b", a describe.skip titled + // "outer skip > reason", and a three-level plain/mid/deep. Verbatim, because the point + // of the case is the layout bun really writes. + const nested = ` + + + + + + + + + + + + + + + + + + + + + + + + + +`; + + test("names every skipped test, and nothing else", () => { + expect(parseSkippedTests(report)).toEqual([ + "packs the payload (needs a POSIX shell)", + 'mode bits & the "x" bit (POSIX only)', + ]); + }); + + test("a test skipped by its describe carries that describe's reason, outermost first", () => { + // The Windows packaging skips are made with describe.skip, so the reason is in the + // describe title and the test inside is only named for what it checks. The path comes + // from the nested elements, never from classname: bun writes the separator + // between two describe titles and a literal ">" inside one title identically, as + // " > ", so a title containing ">" came out split and reversed (measured 1.4.2: + // "rows where count > 100" printed as "100 > rows where count"). + expect(parseSkippedTests(nested)).toEqual([ + 'rows where count > 100 > inner & "quoted" > skipped a > b', + "outer skip > reason > inside a skipped describe", + "plain > mid > deep > leaf", + ]); + }); + + test("a todo is not a skipped test, because bun writes it with the same element", () => { + // bun 1.4.2 writes a todo as ``, and readTestReport takes + // the todos back out of the skip count. The titles have to leave them out too, or + // the summary states one number and lists another (a file with 1 skip and 2 todos + // printed "(1 skipped)" over three titles). + const withTodos = ` + + + + + + + + + + + + + +`; + + expect(parseSkippedTests(withTodos)).toEqual(["packs the payload (needs a POSIX shell)"]); + // The control: the counts read from the same report separate them the same way. + expect(readTestReport(withTodos)).toEqual({ state: "read", counts: { pass: 1, fail: 0, skip: 1, todo: 2 } }); + }); + + test("a report with no skips names nothing", () => { + expect(parseSkippedTests('')).toEqual([]); + }); + + test("a self-closing opens no describe, so the titles after it keep their path", () => { + // Tolerated, not observed: bun 1.4.2 writes no element at all for a describe with + // nothing in it (measured against an empty describe, one holding only a hook, and an + // empty describe.skip: none of the three appears in the report), so this shape comes + // from the junit format rather than from today's writer. A suite element that opened + // a scope it never closes would put its name in front of every title after it, which + // is a wrong reason attached to a real skip, so the shape is pinned rather than left + // to be discovered by whatever writes the next report. + const selfClosing = ` + + + + + + + + + +`; + + expect(parseSkippedTests(selfClosing)).toEqual(["outer > the skip (no dpkg here)"]); + }); + + test("a report that was never written, or was written half way, names what it reached rather than throwing", () => { + expect(parseSkippedTests("")).toEqual([]); + expect(parseSkippedTests(' 100 > inner & "quoted" > skipped a > b', + ]); + }); +}); + +function outcome(overrides: Partial = {}): FileOutcome { + return { + file: "tests/unit/a.test.ts", + status: "passed", + exitCode: 0, + signal: null, + durationMs: 420, + output: "", + counts: { pass: 13, fail: 0, skip: 0, todo: 0 }, + report: "read", + skippedTests: [], + ...overrides, + }; +} + +describe("what the runner prints", () => { + test("a passing file is one line with its position, time and counts", () => { + const line = formatFileLine(outcome(), 12, 533); + + expect(line).toContain("12/533"); + expect(line).toContain("tests/unit/a.test.ts"); + expect(line).toContain("13 pass"); + expect(line).toContain("0.4s"); + expect(line).toContain("PASS"); + }); + + test("a skipped test is visible on the file's line, so a platform skip is never silent", () => { + const line = formatFileLine(outcome({ counts: { pass: 3, fail: 0, skip: 4, todo: 0 } }), 1, 1); + + expect(line).toContain("4 skip"); + }); + + test("a failed file says so, and a timed-out file says how long it was given", () => { + expect(formatFileLine(outcome({ status: "failed", exitCode: 1 }), 1, 1)).toContain("FAIL"); + + const timedOut = formatFileLine(outcome({ status: "timed-out", exitCode: null, durationMs: 305_000 }), 1, 1); + expect(timedOut).toContain("TIMEOUT"); + expect(timedOut).toContain("305.0s"); + }); + + function summary(overrides: Partial = {}): RunSummary { + return { + outcomes: [outcome()], + failures: [], + durationMs: 77_400, + jobs: 16, + timeoutMs: 300_000, + totals: { + files: 533, + filesPassed: 533, + filesFailed: 0, + filesTimedOut: 0, + filesWithoutCounts: 0, + tests: { pass: 13_960, fail: 0, skip: 3, todo: 0 }, + }, + ...overrides, + }; + } + + test("a green run reports the whole population, not just the failures", () => { + const text = formatSummary(summary()); + + expect(text).toContain("533 files"); + expect(text).toContain("13960"); + expect(text).toContain("3 skip"); + expect(text).toContain("77.4s"); + expect(text).toContain("16 jobs"); + }); + + test("a timed-out file is reported against the budget, not against the time it took to die", () => { + // The elapsed time is the budget plus the kill escalation, so printing it would + // answer a question nobody asked: a 3 s budget reported "timed out after 8.0s". + const timedOut = outcome({ file: "tests/unit/hang.test.ts", status: "timed-out", durationMs: 305_000 }); + const text = formatSummary( + summary({ + failures: [timedOut], + timeoutMs: 300_000, + totals: { ...summary().totals, filesPassed: 532, filesTimedOut: 1 }, + }), + ); + + expect(text).toContain("the budget is 300.0s per file"); + expect(text).not.toContain("305.0s per file"); + }); + + test("a failing file that wrote no test report says what that usually means, and that its tests are unaccounted for", () => { + const silent = outcome({ + file: "tests/unit/c.test.ts", + status: "failed", + exitCode: 0, + counts: null, + report: "missing", + }); + const text = formatSummary(summary({ failures: [silent], totals: { ...summary().totals, filesFailed: 1 } })); + + expect(text).toContain( + "tests/unit/c.test.ts (exit 0, and it wrote no test report, which usually means it registered no test or stopped before bun finished, so its tests are unaccounted for)", + ); + expect(formatFileLine(silent, 1, 1)).toContain("no test report"); + }); + + test("a report that could not be read is named as unreadable, not as missing", () => { + const garbled = outcome({ + file: "tests/unit/c.test.ts", + status: "failed", + exitCode: 0, + counts: null, + report: "unreadable", + }); + const text = formatSummary(summary({ failures: [garbled], totals: { ...summary().totals, filesFailed: 1 } })); + + expect(text).toContain("tests/unit/c.test.ts (exit 0, and its test report could not be read"); + expect(text).not.toContain("wrote no test report"); + expect(formatFileLine(garbled, 1, 1)).toContain("unreadable test report"); + }); + + test("a file killed by SIGKILL from outside says where that comes from and what to do", () => { + // The runner sends SIGKILL itself only after a timeout, and that file is reported + // as timed out, so a SIGKILL on a failed file came from outside. Measured: a real + // kernel OOM kill of one child read only "killed by SIGKILL", which names neither + // the cause nor anything the reader can act on. + const killed = outcome({ file: "tests/unit/heavy.test.ts", status: "failed", exitCode: null, signal: "SIGKILL" }); + const text = formatSummary(summary({ failures: [killed], totals: { ...summary().totals, filesFailed: 1 } })); + + expect(text).toContain("OOM killer"); + expect(text).toContain("--jobs"); + }); + + test("another signal keeps the plain wording, and a timed-out file keeps its own", () => { + // The two controls for the case above: only an outside SIGKILL gets the advice. + const segfault = outcome({ status: "failed", exitCode: null, signal: "SIGSEGV" }); + const segfaultText = formatSummary( + summary({ failures: [segfault], totals: { ...summary().totals, filesFailed: 1 } }), + ); + + expect(segfaultText).toContain("(killed by SIGSEGV)"); + expect(segfaultText).not.toContain("--jobs"); + + // A timed-out file is killed with SIGKILL by the runner itself. + const timedOut = outcome({ status: "timed-out", exitCode: null, signal: "SIGKILL", durationMs: 305_000 }); + const timedOutText = formatSummary( + summary({ failures: [timedOut], totals: { ...summary().totals, filesTimedOut: 1 } }), + ); + + expect(timedOutText).toContain("timed out, the budget is 300.0s per file"); + expect(timedOutText).not.toContain("OOM killer"); + }); + + test("a file whose report counts no test at all says it registered none", () => { + const empty = outcome({ status: "failed", counts: { pass: 0, fail: 0, skip: 0, todo: 0 } }); + const text = formatSummary(summary({ failures: [empty], totals: { ...summary().totals, filesFailed: 1 } })); + + expect(text).toContain("tests/unit/a.test.ts (exit 0, and it registered no test)"); + }); + + test("one of something is not plural", () => { + const text = formatSummary( + summary({ + jobs: 1, + totals: { + files: 1, + filesPassed: 1, + filesFailed: 0, + filesTimedOut: 0, + filesWithoutCounts: 0, + tests: { pass: 1, fail: 0, skip: 0, todo: 0 }, + }, + }), + ); + + expect(text).toContain("1 file: 1 passed"); + expect(text).toContain("1 test: 1 pass"); + expect(text).toContain("with 1 job"); + }); + + test("a red run names every failing file and how to re-run it alone", () => { + const failure = outcome({ file: "tests/unit/b.test.ts", status: "failed", exitCode: 1 }); + const text = formatSummary( + summary({ + failures: [failure], + totals: { ...summary().totals, filesPassed: 532, filesFailed: 1 }, + }), + ); + + expect(text).toContain("tests/unit/b.test.ts"); + expect(text).toContain("re-run alone with: bun tests/run-tests.ts tests/unit/b.test.ts"); + // Never bare `bun test ./file`: it takes no --jobs, which the SIGKILL reason above + // tells the reader to lower, and it reads the verdict off the console instead of the + // junit report, so a file calling process.exit(0) reads green when re-run that way. + // It is also the form CONTRIBUTING.md and CLAUDE.md tell contributors not to use. + expect(text).not.toContain("bun test ./"); + }); + + test("a file that left no readable test report is called out, so the total is honest", () => { + expect(formatSummary(summary({ totals: { ...summary().totals, filesWithoutCounts: 2 } }))).toContain( + "2 files left no readable test report, so their tests are not in the totals above.", + ); + expect(formatSummary(summary())).not.toContain("readable test report"); + }); + + test("every file that skipped a test is named, with the titles that carry the reason", () => { + // A test skipped because the artifact it drives cannot exist on this platform + // (a deb postinstall, a snap launcher) says so in its own title, and bun prints + // that title nowhere, so the summary is the only place a reader meets it. + const skipping = outcome({ + file: "tests/unit/snap-launcher.test.ts", + counts: { pass: 4, fail: 0, skip: 9, todo: 0 }, + skippedTests: ["the launcher exports SNAP_DATA (POSIX shell only)"], + }); + const text = formatSummary( + summary({ + outcomes: [outcome(), skipping], + totals: { ...summary().totals, tests: { pass: 13_960, fail: 0, skip: 9, todo: 0 } }, + }), + ); + + expect(text).toContain("Files with skipped tests:"); + expect(text).toContain("tests/unit/snap-launcher.test.ts (9 skipped)"); + expect(text).toContain("the launcher exports SNAP_DATA (POSIX shell only)"); + expect(text).not.toContain("tests/unit/a.test.ts ("); + }); + + test("the count over the titles is the number of titles under it, todos in neither", () => { + // Read and printed the way the runner does it, from one report, so the section and + // its header cannot drift apart: a todo is written as ``, + // and it belongs in neither, since "todo" is already its own count on the file line. + const withTodos = ` + + + + + + + + + + + + + +`; + const read = readTestReport(withTodos); + if (read.state !== "read") throw new Error("the fixture is a report this parser reads"); + const mixed = outcome({ + file: "tests/unit/mixed.test.ts", + counts: read.counts, + skippedTests: parseSkippedTests(withTodos), + }); + const text = formatSummary( + summary({ + outcomes: [mixed], + totals: { ...summary().totals, tests: { pass: 1, fail: 0, skip: 1, todo: 2 } }, + }), + ); + + expect(text).toContain("tests/unit/mixed.test.ts (1 skipped)"); + expect(text).toContain(" packs the payload (needs a POSIX shell)"); + expect(text).not.toContain("a todo nobody has written yet"); + expect(text).not.toContain("a second todo"); + }); + + test("a file whose unrun tests are all todos is named nowhere as skipping something", () => { + // The mirror of the case above: "3 todo" on the file line and in the totals is the + // whole story, and a todo has no reason to state, so there is no section to print. + const todosOnly = outcome({ file: "tests/unit/todo.test.ts", counts: { pass: 1, fail: 0, skip: 0, todo: 3 } }); + const text = formatSummary( + summary({ + outcomes: [todosOnly], + totals: { ...summary().totals, tests: { pass: 1, fail: 0, skip: 0, todo: 3 } }, + }), + ); + + expect(text).toContain("3 todo"); + expect(text).not.toContain("Files with skipped tests"); + // The control: the same file with one real skip is named, with its title. + const withSkip = formatSummary( + summary({ + outcomes: [outcome({ ...todosOnly, counts: { pass: 1, fail: 0, skip: 1, todo: 3 }, skippedTests: ["a skip"] })], + totals: { ...summary().totals, tests: { pass: 1, fail: 0, skip: 1, todo: 3 } }, + }), + ); + + expect(withSkip).toContain("tests/unit/todo.test.ts (1 skipped)"); + expect(withSkip).toContain("a skip"); + }); + + test("a file that counted skips whose titles it could not name is still named, with its count", () => { + // The count and the titles come from different parts of the report (the + // attributes and the elements), so one can be there without the other. + // "4 skipped" with no titles is less than this section is for and more than silence. + const quiet = outcome({ + file: "tests/unit/quiet.test.ts", + counts: { pass: 1, fail: 0, skip: 4, todo: 0 }, + skippedTests: [], + }); + const text = formatSummary( + summary({ + outcomes: [quiet], + totals: { ...summary().totals, tests: { pass: 1, fail: 0, skip: 4, todo: 0 } }, + }), + ); + + expect(text).toContain("tests/unit/quiet.test.ts (4 skipped)"); + }); + + test("a file whose report could not be read still names the skips it reached, under an unknown count", () => { + // A child killed while bun was writing its report leaves a truncated one: the counts + // are refused (they would undercount), but what it did reach still names tests, and + // this section is the only place a skip's reason is ever printed. Selecting the + // section by the skip count dropped exactly these files, since their count is null. + const cut = outcome({ + file: "tests/unit/packaging.test.ts", + status: "failed", + exitCode: null, + signal: "SIGKILL", + counts: null, + report: "unreadable", + skippedTests: ["windows packaging (no dpkg on this platform) > packs the payload", "signs the installer"], + }); + const text = formatSummary( + summary({ + outcomes: [outcome(), cut], + failures: [cut], + totals: { ...summary().totals, filesFailed: 1, filesWithoutCounts: 1 }, + }), + ); + + expect(text).toContain("tests/unit/packaging.test.ts (unreadable report; it named 2 skipped tests)"); + expect(text).toContain(" windows packaging (no dpkg on this platform) > packs the payload"); + expect(text).toContain(" signs the installer"); + // The control: a file whose report was read keeps the plain count in its header. + expect(text).not.toContain("tests/unit/a.test.ts (unreadable report"); + }); + + test("files that could not run here are listed under the reason, which is printed once", () => { + const notRun = [ + { file: "tests/unit/helm-chart-agent.test.ts", reason: "Helm is not installed." }, + { file: "tests/unit/helm-chart-route.test.ts", reason: "Helm is not installed." }, + ]; + const text = formatSummary(summary(), notRun); + + expect(text).toContain("Files not run on this machine:"); + expect(text.split("Helm is not installed.").length - 1).toBe(1); + expect(text).toContain("tests/unit/helm-chart-agent.test.ts"); + expect(text).toContain("tests/unit/helm-chart-route.test.ts"); + }); + + test("a run where everything could run says nothing about it", () => { + expect(formatSummary(summary(), [])).not.toContain("not run on this machine"); + }); + + test("a run with no skips says nothing about skips", () => { + expect(formatSummary(summary())).not.toContain("Files with skipped tests"); + }); +}); diff --git a/tests/unit/test-runner-requirements.test.ts b/tests/unit/test-runner-requirements.test.ts new file mode 100644 index 000000000..57bc14b53 --- /dev/null +++ b/tests/unit/test-runner-requirements.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { discoverTestFiles } from "../runner/discover"; +import { missingHelm, parseRequirements, planRequirements, requiredCapabilities } from "../runner/requirements"; + +// A test file that needs an external tool says so in one line, and the runner decides what +// that means on this machine: run it, name it as not run, or refuse the whole run. The chart +// tests are the reason: they drive the real `helm` binary against a subchart that has to be +// downloaded, and a contributor who never touches the chart should not have to install either +// to get a green `bun run test`. CI sets LIBREDB_REQUIRE_HELM=1, so there nothing is ever +// quietly left out. +const root = path.resolve(import.meta.dir, "../.."); + +// Built from pieces so that no line of THIS file starts with the marker: the invariant test +// below reads every test file in the tree, this one included. +const MARKER = ["//", "@requires", "helm"].join(" "); + +describe("reading what a test file requires", () => { + test("a marker on its own line names a requirement", () => { + expect(parseRequirements(`${MARKER}\nimport { test } from "bun:test";\n`, "tests/unit/a.test.ts")).toEqual([ + "helm", + ]); + }); + + test("a file saved with CRLF line endings still declares its requirement", () => { + // .gitattributes checks every text file out as LF, but an editor on Windows can save a new + // test file with CRLF long before git ever normalises it. A multiline `$` already treats the + // CR as a line end; this pins that, so a parser rewritten to split on "\n" cannot lose it. + expect(parseRequirements(`${MARKER}\r\nimport { test } from "bun:test";\r\n`, "tests/unit/a.test.ts")).toEqual([ + "helm", + ]); + }); + + test("the marker anywhere else is not a declaration", () => { + const quoted = `const text = "${MARKER}";\n ${MARKER}\n`; + + expect(parseRequirements(quoted, "tests/unit/a.test.ts")).toEqual([]); + expect(parseRequirements('import { test } from "bun:test";\n', "tests/unit/a.test.ts")).toEqual([]); + }); + + test("a requirement the runner does not know is refused by name, never ignored", () => { + expect(() => parseRequirements("// @requires docker\n", "tests/unit/b.test.ts")).toThrow( + /tests\/unit\/b\.test\.ts declares "@requires docker"/, + ); + }); +}); + +describe("whether Helm is usable here", () => { + test("no helm binary is the first thing said", () => { + expect(missingHelm({ which: () => null, subchartBuilt: () => false })).toMatch(/Helm is not installed/); + }); + + test("a helm binary without the chart's built dependency names the command that builds it", () => { + expect(missingHelm({ which: () => "/usr/bin/helm", subchartBuilt: () => false })).toContain( + "helm dependency build charts/libredb-studio --skip-refresh", + ); + }); + + test("a helm binary and a built dependency is usable", () => { + expect(missingHelm({ which: () => "/usr/bin/helm", subchartBuilt: () => true })).toBeNull(); + }); +}); + +describe("what a run requires", () => { + test("LIBREDB_REQUIRE_HELM=1 makes Helm a requirement of the run", () => { + expect([...requiredCapabilities({ LIBREDB_REQUIRE_HELM: "1" })]).toEqual(["helm"]); + }); + + test("an unset or zero variable requires nothing", () => { + expect([...requiredCapabilities({})]).toEqual([]); + expect([...requiredCapabilities({ LIBREDB_REQUIRE_HELM: "0" })]).toEqual([]); + expect([...requiredCapabilities({ LIBREDB_REQUIRE_HELM: "" })]).toEqual([]); + }); + + test("any other value is refused, because a typo must not quietly mean 'not required'", () => { + expect(() => requiredCapabilities({ LIBREDB_REQUIRE_HELM: "true" })).toThrow(/LIBREDB_REQUIRE_HELM/); + }); +}); + +describe("planning the run", () => { + const sources: Record = { + "tests/unit/chart.test.ts": `${MARKER}\ntest("renders", () => {});\n`, + "tests/unit/plain.test.ts": 'test("adds", () => {});\n', + }; + const readSource = (file: string) => sources[file] as string; + const files = Object.keys(sources).sort(); + + test("a file whose requirement is missing is not run, and says why; the rest run", () => { + const plan = planRequirements({ + files, + readSource, + missing: { helm: () => "Helm is not installed." }, + required: new Set(), + }); + + expect(plan.run).toEqual(["tests/unit/plain.test.ts"]); + expect(plan.notRun).toEqual([{ file: "tests/unit/chart.test.ts", reason: "Helm is not installed." }]); + }); + + test("a file whose requirement is present simply runs", () => { + const plan = planRequirements({ files, readSource, missing: { helm: () => null }, required: new Set() }); + + expect(plan.run).toEqual(files); + expect(plan.notRun).toEqual([]); + }); + + test("a required capability that is missing stops the run and names the file and the variable", () => { + expect(() => + planRequirements({ + files, + readSource, + missing: { helm: () => "Helm is not installed." }, + required: new Set(["helm"]), + }), + ).toThrow(/tests\/unit\/chart\.test\.ts needs helm, and LIBREDB_REQUIRE_HELM=1 .*Helm is not installed\./); + }); + + test("a selection in which nothing can run is an error, not an empty green run", () => { + expect(() => + planRequirements({ + files: ["tests/unit/chart.test.ts"], + readSource, + missing: { helm: () => "Helm is not installed." }, + required: new Set(), + }), + ).toThrow(/None of the 1 selected file can run here/); + }); + + test("the machine is asked once per capability, not once per file", () => { + let asked = 0; + planRequirements({ + files: ["tests/unit/chart-a.test.ts", "tests/unit/chart-b.test.ts"], + readSource: () => `${MARKER}\n`, + missing: { + helm: () => { + asked += 1; + return null; + }, + }, + required: new Set(), + }); + + expect(asked).toBe(1); + }); +}); + +describe("every test file that drives helm declares it, and no other file does", () => { + // The marker is only worth anything while it is true of the whole tree: a new chart test + // that forgot it would fail on every machine without Helm, and a stale marker would hide a + // file that needs nothing. Detection is the helm invocation itself, the one shape all twelve + // chart tests share. + const SPAWNS_HELM = /\[\s*"helm"\s*,\s*"(?:template|repo|dependency)"/; + const files = discoverTestFiles(root).map((file) => ({ file, source: readFileSync(path.join(root, file), "utf8") })); + const declaring = files.filter(({ file, source }) => parseRequirements(source, file).includes("helm")); + const spawning = files.filter(({ source }) => SPAWNS_HELM.test(source)); + + test("the tree has chart tests at all, so the two checks below are not vacuous", () => { + expect(spawning.length).toBeGreaterThan(0); + }); + + test("a file that runs helm declares @requires helm", () => { + expect(spawning.filter(({ file }) => !declaring.some((d) => d.file === file)).map(({ file }) => file)).toEqual([]); + }); + + test("a file that declares @requires helm runs helm", () => { + expect(declaring.filter(({ file }) => !spawning.some((s) => s.file === file)).map(({ file }) => file)).toEqual([]); + }); +}); diff --git a/tests/unit/test-runner-signals.test.ts b/tests/unit/test-runner-signals.test.ts new file mode 100644 index 000000000..04d627521 --- /dev/null +++ b/tests/unit/test-runner-signals.test.ts @@ -0,0 +1,64 @@ +import { constants } from "node:os"; +import { describe, expect, test } from "bun:test"; +import { exitCodeForSignal, STOP_SIGNAL_NUMBERS, STOP_SIGNALS } from "../runner/signals"; + +// The end-to-end signal cases in tests/unit/test-runner-cli.test.ts can only drive the +// signals the running platform delivers, and they are skipped altogether on Windows. +// SIGBREAK is therefore untestable end to end everywhere: it exists only on Windows, +// and Windows delivers none of these to a piped child. So the mapping is pinned here, +// against injected tables, where every arm runs on every platform. +describe("the signals that stop a run", () => { + test("the four signals are named, and each has a number written out", () => { + expect([...STOP_SIGNALS]).toEqual(["SIGINT", "SIGTERM", "SIGHUP", "SIGBREAK"]); + expect(STOP_SIGNAL_NUMBERS).toEqual({ SIGHUP: 1, SIGINT: 2, SIGTERM: 15, SIGBREAK: 21 }); + }); + + test("a signal the platform names takes its exit code from the platform's own number", () => { + const platform = { SIGHUP: 1, SIGINT: 2, SIGTERM: 15, SIGBREAK: 21 }; + + expect(exitCodeForSignal("SIGINT", platform)).toBe(130); + expect(exitCodeForSignal("SIGTERM", platform)).toBe(143); + expect(exitCodeForSignal("SIGHUP", platform)).toBe(129); + expect(exitCodeForSignal("SIGBREAK", platform)).toBe(149); + }); + + test("the platform's number is used rather than the written-out one, where they differ", () => { + // The control for the case above: if the table were ignored, both would answer 130. + expect(exitCodeForSignal("SIGINT", { SIGINT: 3 })).toBe(131); + }); + + test("a signal the platform does not name falls back to the written-out number, never NaN", () => { + // This is the live case: os.constants.signals has no SIGBREAK on Linux or macOS, + // so `128 + constants.signals.SIGBREAK` is NaN, and process.exit(NaN) throws + // RangeError from inside the signal listener, where a throw leaves bun running. + expect(exitCodeForSignal("SIGBREAK", {})).toBe(149); + expect(exitCodeForSignal("SIGINT", {})).toBe(130); + expect(exitCodeForSignal("SIGTERM", {})).toBe(143); + expect(exitCodeForSignal("SIGHUP", {})).toBe(129); + }); + + test("on this platform, every stop signal resolves to a code process.exit accepts", () => { + for (const signal of STOP_SIGNALS) { + const code = exitCodeForSignal(signal); + // process.exit refuses anything that is not a whole number in range, and the + // whole point of the fallback is that no arm can reach it with NaN. + expect({ signal, integer: Number.isInteger(code), inRange: code >= 128 && code <= 255 }).toEqual({ + signal, + integer: true, + inRange: true, + }); + } + }); + + test("the written-out numbers agree with this platform's own table wherever it has them", () => { + // Paired control for the fallback: if the written-out numbers were wrong, the + // fallback would quietly report a different code from the one the shell reports. + const platform = constants.signals as Partial>; + const named = STOP_SIGNALS.filter((signal) => platform[signal] !== undefined); + + // On POSIX that is SIGINT, SIGTERM and SIGHUP; naming them proves the loop is not empty. + expect(named.length).toBeGreaterThan(0); + for (const signal of named) + expect({ signal, number: platform[signal] }).toEqual({ signal, number: STOP_SIGNAL_NUMBERS[signal] }); + }); +}); diff --git a/tests/unit/theme-token-usage.test.ts b/tests/unit/theme-token-usage.test.ts index f664d63d6..3f8ef5126 100644 --- a/tests/unit/theme-token-usage.test.ts +++ b/tests/unit/theme-token-usage.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { readdirSync, readFileSync, statSync } from "node:fs"; -import { join, relative } from "node:path"; +import { join, relative, sep } from "node:path"; /** * The token layer only holds if adding a literal is harder than adding a token. @@ -35,8 +35,12 @@ function sourceFiles(dir: string): string[] { }); } +// `relative` hands back the platform separator, so on Windows this would be +// "src\\components\\ui\\button.tsx". Every comparison below is written with forward slashes: +// the `src/components/ui/` exclusion would silently stop excluding the vendored shadcn tree and +// its literal Tailwind hues would trip the token assertions. Normalise once, here. const files = sourceFiles(SRC).map((path) => ({ - path: relative(ROOT, path), + path: relative(ROOT, path).split(sep).join("/"), text: readFileSync(path, "utf8"), }));