Merge pull request #108 from mapbox/feat/commands-search-eval

Report commands search eval scores against the base in CI

Author: zmofei

.github/workflows/ci.yml

@@ -113,6 +113,87 @@ jobs:
         shell: bash
         run: sh scripts/test-completion.sh
 
+  # Which `commands search` queries a PR fixed and broke, against its base,
+  # written to the job summary. It reports and never fails: the floors that
+  # `cargo test` holds in the build job are the gate. A job summary rather
+  # than a PR comment because it needs no write token, so a PR from a fork
+  # gets one too. One runner: ranking is the same everywhere.
+  search-eval:
+    name: commands search eval
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    steps:
+      # Whole history, so the base commit is there to check out beside it.
+      # No credential left behind: nothing here pushes or fetches again.
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      # As in the build job: install exactly the pinned toolchain.
+      - name: Read the pinned toolchain
+        id: toolchain
+        run: |
+          echo "toolchain=$(sed -n 's/^channel = "\(.*\)"/\1/p' rust-toolchain.toml | head -1)" >>"$GITHUB_OUTPUT"
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ steps.toolchain.outputs.toolchain }}
+
+      # The exit status is recorded rather than failing the step: a run below
+      # its floor still writes its reports, the build job is where that
+      # failure is reported, and the summary needs the status to say whether
+      # missing reports mean a failed run or one with no eval to run.
+      - name: Run the eval on this branch
+        id: head
+        run: |
+          status=0
+          MAPBOX_SEARCH_EVAL_REPORT="$RUNNER_TEMP/eval-head" cargo test --locked --bin mapbox eval_set || status=$?
+          echo "status=$status" >>"$GITHUB_OUTPUT"
+
+      # In a worktree sharing this target directory, so only the crate itself
+      # rebuilds. It rebuilds only because the worktree is added after the
+      # head build: cargo goes by mtime, and base sources older than that
+      # build would reuse its binary. Keep this step after the head run.
+      # A base from before the eval or its reports passes and writes none;
+      # one that fails to build or check out exits non-zero. The summary
+      # tells the two apart by the status.
+      - name: Run the eval on the base
+        id: base
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          status=0
+          {
+            git worktree add --detach "$RUNNER_TEMP/base" "$BASE_SHA" &&
+              cd "$RUNNER_TEMP/base" &&
+              CARGO_TARGET_DIR="$GITHUB_WORKSPACE/target" \
+                MAPBOX_SEARCH_EVAL_REPORT="$RUNNER_TEMP/eval-base" \
+                cargo test --locked --bin mapbox eval_set
+          } || status=$?
+          echo "status=$status" >>"$GITHUB_OUTPUT"
+
+      # No `--base` on a push: the script then says there was no base rather
+      # than that the base had no reports. A status left empty by a step that
+      # never finished reads as "did not finish".
+      - name: Write the summary
+        if: ${{ !cancelled() }}
+        env:
+          EVENT: ${{ github.event_name }}
+          HEAD_STATUS: ${{ steps.head.outputs.status }}
+          BASE_STATUS: ${{ steps.base.outputs.status }}
+        run: |
+          set -- --head "$RUNNER_TEMP/eval-head" --head-status "$HEAD_STATUS" \
+            --sets tests/fixtures/commands_search
+          if [ "$EVENT" = pull_request ]; then
+            set -- "$@" --base "$RUNNER_TEMP/eval-base" --base-status "$BASE_STATUS"
+          fi
+          python3 scripts/commands-search-report.py "$@" >>"$GITHUB_STEP_SUMMARY"
+
   # No Rust: this is where the repo's shell gets checked, install.sh above all
   # — it is served to machines we do not control, and this is the only thing
   # that checks it. Both runners, because the portability it has to get right
@@ -136,6 +217,7 @@ jobs:
           sh -n scripts/install.sh
           sh -n scripts/test-install.sh
           sh -n scripts/test-completion.sh
+          sh -n scripts/test-commands-search-report.sh
 
       # -s sh, not the default: bash would accept things `curl | sh` will not.
       # Once is enough — the findings do not depend on the runner. The version
@@ -145,11 +227,19 @@ jobs:
       # Kept clean against 0.9, 0.10 and 0.11.
       - name: Shellcheck
         if: runner.os == 'Linux'
-        run: shellcheck -s sh scripts/install.sh scripts/test-install.sh scripts/test-completion.sh
+        run: shellcheck -s sh scripts/install.sh scripts/test-install.sh scripts/test-completion.sh scripts/test-commands-search-report.sh
 
       - name: Install, verify and prompt against a local channel
         run: sh scripts/test-install.sh
 
+      # The `commands search eval` job's summary script, against hand-written
+      # reports: that job never fails, so this is where a summary that says
+      # the wrong thing would be caught. Python only, so it lives here rather
+      # than beside the Rust. Once is enough, as for shellcheck.
+      - name: Summarize hand-written search eval reports
+        if: runner.os == 'Linux'
+        run: sh scripts/test-commands-search-report.sh
+
   # The other half of the same job, for the installer install.sh stops short
   # of. Both shells, because both are on a stock Windows 11 and they differ
   # exactly where this script lives: the default TLS versions, what `2>` does

AGENTS.md

@@ -126,6 +126,24 @@ One table outside that file works the same way: `GROUPS` in
 an entry outlives its command. A command it misses still shows, under
 "Other", but CI will not let that ship.
 
+## `commands search` and its query sets
+
+`mapbox commands search` ranks commands by the words in their names,
+summaries, descriptions, help-group lines and argument help, so **editing a
+command's `about`, an argument's `help` or a line in `DESCRIPTIONS` changes
+search results**, not only `--help`. The `eval_set_*` tests in
+`src/commands.rs` run the query sets in `tests/fixtures/commands_search/` and
+fail when a set ranks more than a few queries fewer right than its recorded
+score. Each file's header says where its queries came from and how much the
+search was tuned against them. `cargo test eval_set -- --nocapture` prints
+the scores and every miss, and that directory's README explains each set and
+how to read its score.
+
+When a change raises a score, record it in the same commit. Dropping past the
+slack is a decision to explain in the PR. Don't tune against the sets by
+special-casing a query; fix the kind of miss. New real queries go in a new
+file, so there is always one set the search has not been fitted to.
+
 ## What the tests can and cannot tell you
 
 `cargo test` answers to fake tokens and a fake server. It runs offline, on a

CHANGELOG.md

@@ -56,6 +56,15 @@ that may never merge. They are not releases and are not listed here.
   from `tilequery get` and the renderer kept matching the old name.
   `-o json` is unchanged.
 
+- New `mapbox commands search "<task>"` finds the commands that match a
+  task, best first, and `mapbox commands list` lists every command with its
+  summary. Both are local keyword lookups: no request, no token. `json`
+  gives `[{command, summary, score}]`, where `score` (0 to 1) is how much
+  of the query the command's own text covers. The query is sent in telemetry as its length
+  only. Top-level help now points to `commands search` (agents see it in
+  their hint), and an unknown command with no near spelling suggests it in
+  the error's `fix`. No existing command or output changes.
+
 - `mapbox usage` reads as a chart: one row per product with no blank
   lines between, every sparkline as wide as the period, bars scaled from
   zero so their height is proportional to the value, and the tallest

Cargo.lock

@@ -65,6 +65,9 @@ clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] }
 # top-level help, which src/help_layout.rs renders itself and so has to wrap
 # to the terminal itself.
 terminal_size = "0.4"
+# Typo tolerance for `mapbox commands search`. Already in the graph through
+# clap's own "did you mean" suggestions; the same version.
+strsim = "0.11"
 # Renders `mapbox completion <shell>` from the same `Command` tree everything
 # else here is built from. Kept to the default features on purpose: the
 # dynamic-completion half is `unstable-dynamic`, and value completion is out

Cargo.toml

@@ -21,6 +21,7 @@ mapbox styles list
   - [Named profiles](#named-profiles)
 - [Commands](#commands)
   - [API commands](#api-commands)
+  - [Finding a command](#finding-a-command)
   - [Diagnostics and settings](#diagnostics-and-settings)
   - [Shell completion](#shell-completion)
   - [Tileset CLI](#tileset-cli)
@@ -307,6 +308,17 @@ the token an event was sent with alongside the event. When the CLI's token
 can't be used, the event is dropped, and it is never sent twice.
 `MAPBOX_CLI_NO_TELEMETRY=1` or `mapbox config set telemetry off` turns it off.
 
+### Finding a command
+
+```sh
+mapbox commands search upload a font   # the commands that match a task, best first
+mapbox commands list                   # every command with its summary
+```
+
+Both are local keyword lookups over the CLI's own commands: no model, no
+request and no token. Agents get the same matches as JSON. See
+[Commands](./docs/commands.md#commands).
+
 ### Diagnostics and settings
 
 ```sh

README.md

@@ -134,6 +134,9 @@ nests, and is typed `mapbox styles draft get`.
 **[History](#history)** — [history.list](#mapbox-history-list) ·
 [history.show](#mapbox-history-show)
 
+**[Commands](#commands)** — [commands.search](#mapbox-commands-search) ·
+[commands.list](#mapbox-commands-list)
+
 **[Doctor](#doctor)** — [doctor](#mapbox-doctor)
 
 **[Usage](#usage)** — [usage](#mapbox-usage)
@@ -4736,6 +4739,138 @@ Log       no longer available: it expired or was removed to keep diagnostic logs
 
 ---
 
+## Commands
+
+Finding a command by what it does. Both read the same command surface
+`mapbox --schema` describes, so they cover every command this build has,
+and both are local: no request and no token. `json` gives each command as
+`{command, summary}`, plus a `score` from `search`; `mapbox <command>
+--schema` has the rest.
+
+### `mapbox commands search`
+
+The commands that match a task, best first. It is keyword matching, not a
+model: each word of the query is matched against a command's name, summary
+and description, its group's description and its arguments' help, with a
+command's name counting most. A word can be a prefix (`geo`), a different
+ending of the same stem (`geocoding` for `geocoder`) or one typo away
+(`uplaod`), unless some command's text uses the word as it is: "match" is a
+word, not a typo for "batch". A plural matches its singular. A few common
+words stand in for the ones commands use ("remove" for "delete", "rename"
+for "update").
+Words like "how", "my", "to" and "mapbox" are ignored. It cannot find a
+command whose text uses none of the query's words, so a query names the
+action and the thing acted on: "upload a font", not "get my typeface onto
+Mapbox". Matches far below the best one are left out. A task no command
+does usually still comes back with a few weak guesses, so check the
+summaries before running one.
+
+Under `json`, each match has a `score` from 0 to 1: how much of the query
+the command's own text covers. 1 means its name and summary use every word
+of the query, a few tenths is a word found elsewhere, and under 0.1 is
+barely a match. It is not a confidence, so don't use it as a cutoff: the
+right command can score lower than a wrong one on another query.
+
+The query can be quoted or not. One with no words at all, such as an
+empty string, is a usage error (exit 2). No match is not an error: `text`
+prints nothing and says so on stderr, `json` prints `[]`, and both exit 0. The
+`commands` group itself is left out of the results. Telemetry records the
+query's length, never its text.
+
+#### Parameters
+
+| Parameter | Effect |
+| --- | --- |
+| `<query>...` | What you want to do, in a few words. |
+| `--limit <n>` | How many matches to show. Defaults to `5`; `0` shows every match. |
+
+#### Examples
+
+```sh
+mapbox commands search upload a font
+
+mapbox commands search "snap a gps trace to roads"
+
+mapbox commands search tiles --limit 0
+```
+
+#### Outputs
+
+<table>
+<tr><th width="50%"><code>text</code></th><th width="50%"><code>json</code></th></tr>
+<tr><td>
+
+```
+COMMAND              SUMMARY
+mapbox fonts upload  Upload a font
+mapbox fonts list    List fonts for an owner
+mapbox fonts delete  Delete a font
+```
+
+</td><td>
+
+```json
+[
+  {
+    "command": "mapbox fonts upload",
+    "score": 1.0,
+    "summary": "Upload a font"
+  },
+  {
+    "command": "mapbox fonts list",
+    "score": 0.65,
+    "summary": "List fonts for an owner"
+  },
+  ...
+]
+```
+
+</td></tr>
+</table>
+
+### `mapbox commands list`
+
+Every command with its summary, sorted by name.
+
+#### Examples
+
+```sh
+mapbox commands list
+```
+
+#### Outputs
+
+<table>
+<tr><th width="50%"><code>text</code></th><th width="50%"><code>json</code></th></tr>
+<tr><td>
+
+```
+COMMAND                         SUMMARY
+mapbox accounts list-scopes     List scopes
+mapbox accounts list-tokens     List tokens
+mapbox accounts retrieve-token  Retrieve a token
+...
+```
+
+</td><td>
+
+```json
+[
+  {
+    "command": "mapbox accounts list-scopes",
+    "summary": "List scopes"
+  },
+  {
+    "command": "mapbox accounts list-tokens",
+    "summary": "List tokens"
+  },
+  ...
+]
+```
+
+</td></tr>
+</table>
+
 ## Doctor
 
 ### `mapbox doctor`