|
| 1 | +"""The self-test: every read-only command, run against names you choose, to find bugs. |
| 2 | +
|
| 3 | +Hidden from --help: it is for trying a build in a real tenant before a release. Each command |
| 4 | +runs in this process with its output captured and thrown away; only what happened is kept: |
| 5 | +
|
| 6 | +- ok: it succeeded; |
| 7 | +- attention: it exited 3, having found something, as designed; |
| 8 | +- refused: it stopped with an error it explained, most often a missing permission or scope; |
| 9 | +- usage: it rejected its own arguments, which is a bug in the test or the command; |
| 10 | +- CRASH: an exception escaped, which is a bug, reported with the line of ldo it came from. |
| 11 | +
|
| 12 | +Nothing here changes anything, and no token is printed. Run it with a profile that is |
| 13 | +already signed in: a sign-in cannot be answered while it runs. |
| 14 | +""" |
| 15 | + |
| 16 | +import json |
| 17 | +import os |
| 18 | +import time |
| 19 | +import traceback |
| 20 | +from dataclasses import dataclass, field |
| 21 | +from pathlib import Path |
| 22 | +from typing import Annotated, Any |
| 23 | + |
| 24 | +import typer |
| 25 | + |
| 26 | +from libre_devops_helpers.cli import render |
| 27 | +from libre_devops_helpers.cli.options import OutputOption, get_runtime |
| 28 | +from libre_devops_helpers.cli.render import Output |
| 29 | +from libre_devops_helpers.core import brand |
| 30 | +from libre_devops_helpers.core.errors import LdoError |
| 31 | +from libre_devops_helpers.core.util import short_name |
| 32 | + |
| 33 | +_PACKAGE = __name__.split(".", 1)[0] |
| 34 | +_JSON_SAMPLE = '{"value": [{"id": "1", "name": "web01", "on": true, "at": null}]}' |
| 35 | + |
| 36 | + |
| 37 | +@dataclass(frozen=True) |
| 38 | +class Case: |
| 39 | + """One command to run: its arguments, with {device}, {short}, {user}, {group} and |
| 40 | + {workspace} filled in, and what it needs to be worth running.""" |
| 41 | + |
| 42 | + args: tuple[str, ...] |
| 43 | + needs: tuple[str, ...] = () |
| 44 | + slow: bool = False |
| 45 | + stdin: str | None = None |
| 46 | + |
| 47 | + |
| 48 | +@dataclass |
| 49 | +class Outcome: |
| 50 | + command: str |
| 51 | + result: str |
| 52 | + exit_code: int | None |
| 53 | + seconds: float |
| 54 | + detail: str = "" |
| 55 | + hint: str | None = None |
| 56 | + where: list[str] = field(default_factory=list) |
| 57 | + |
| 58 | + |
| 59 | +CASES = ( |
| 60 | + Case(("welcome",)), |
| 61 | + Case(("profiles",)), |
| 62 | + Case(("config", "path")), |
| 63 | + Case(("network", "test")), |
| 64 | + Case(("az", "whoami")), |
| 65 | + Case(("json",), stdin=_JSON_SAMPLE), |
| 66 | + Case(("json", "--yaml"), stdin=_JSON_SAMPLE), |
| 67 | + Case(("entra", "token", "graph")), |
| 68 | + Case(("entra", "token", "mde")), |
| 69 | + Case(("entra", "token", "arm")), |
| 70 | + Case(("graph", "whoami")), |
| 71 | + Case(("graph", "get", "me")), |
| 72 | + Case(("graph", "get", "organization", "--select", "id,displayName")), |
| 73 | + Case(("entra", "ca-policies")), |
| 74 | + Case(("entra", "sign-ins", "--since", "1d", "--limit", "5")), |
| 75 | + Case(("entra", "app-credentials", "--expiring", "30d"), slow=True), |
| 76 | + Case(("graph", "get-device", "{device}"), ("device",)), |
| 77 | + Case(("entra", "devices", "{device}"), ("device",)), |
| 78 | + Case(("entra", "devices", "{short}"), ("device",)), |
| 79 | + Case(("entra", "device-groups", "{device}"), ("device",)), |
| 80 | + Case(("xdr", "machines", "{device}"), ("device",)), |
| 81 | + Case(("xdr", "machines", "{short}"), ("device",)), |
| 82 | + Case(("xdr", "alerts", "--device", "{device}", "--include-resolved"), ("device",)), |
| 83 | + Case(("xdr", "vulns", "{device}"), ("device",)), |
| 84 | + Case(("devices", "check", "{device}"), ("device",)), |
| 85 | + Case(("devices", "show", "{device}"), ("device",)), |
| 86 | + Case(("devices", "av-signature", "{device}", "--endpoint"), ("device",)), |
| 87 | + Case(("devices", "av-signature", "{device}"), ("device",)), |
| 88 | + Case(("intune", "devices", "{device}"), ("device",)), |
| 89 | + Case(("graph", "get-group", "{group}"), ("group",)), |
| 90 | + Case(("entra", "devices", "{device}", "--group", "{group}"), ("device", "group")), |
| 91 | + Case(("devices", "check", "{device}", "--group", "{group}"), ("device", "group")), |
| 92 | + Case(("entra", "group-devices", "{group}"), ("group",), slow=True), |
| 93 | + Case(("graph", "get-user", "{user}"), ("user",)), |
| 94 | + Case(("entra", "user-groups", "{user}"), ("user",)), |
| 95 | + Case(("entra", "user-roles", "{user}"), ("user",)), |
| 96 | + Case(("entra", "sign-ins", "--user", "{user}", "--since", "7d", "--limit", "5"), ("user",)), |
| 97 | + Case(("azure", "rbac", "{user}"), ("user",), slow=True), |
| 98 | + Case(("xdr", "alerts", "--since", "24h", "--limit", "5")), |
| 99 | + Case(("xdr", "indicators")), |
| 100 | + Case(("xdr", "stale", "--older-than", "180d"), slow=True), |
| 101 | + Case(("xdr", "hunt", "DeviceInfo | take 1", "--endpoint")), |
| 102 | + Case(("xdr", "hunt", "DeviceInfo | take 1")), |
| 103 | + Case(("xdr", "incidents", "top")), |
| 104 | + Case(("xdr", "incidents", "summary", "--since", "1d")), |
| 105 | + Case(("azure", "subscriptions")), |
| 106 | + Case(("azure", "resource-graph", "resources | take 1")), |
| 107 | + Case(("azure", "secure-score")), |
| 108 | + Case(("azure", "defender-plans")), |
| 109 | + Case(("azure", "recommendations", "--severity", "high"), slow=True), |
| 110 | + Case(("azure", "automation", "accounts")), |
| 111 | + Case(("keyvault", "expiry", "--all-vaults", "--within", "30d"), slow=True), |
| 112 | + Case(("logs", "query", "Heartbeat | take 1", "--workspace", "{workspace}"), ("workspace",)), |
| 113 | + Case(("pim", "eligible", "--azure")), |
| 114 | + Case(("pim", "active", "--azure")), |
| 115 | + Case(("pim", "eligible")), |
| 116 | + Case(("snow", "whoami"), ("snow",)), |
| 117 | + Case(("snow", "instance"), ("snow",)), |
| 118 | +) |
| 119 | + |
| 120 | + |
| 121 | +def register(app: typer.Typer) -> None: |
| 122 | + app.command("self-test", hidden=True)(self_test) |
| 123 | + |
| 124 | + |
| 125 | +def self_test( |
| 126 | + ctx: typer.Context, |
| 127 | + device: Annotated[ |
| 128 | + str | None, typer.Option("--device", help="A device to look up: its FQDN is best.") |
| 129 | + ] = None, |
| 130 | + user: Annotated[str | None, typer.Option("--user", help="A user's UPN to look up.")] = None, |
| 131 | + group: Annotated[ |
| 132 | + str | None, typer.Option("--group", help="An Entra group, by display name or object id.") |
| 133 | + ] = None, |
| 134 | + workspace: Annotated[ |
| 135 | + str | None, typer.Option("--workspace", help="A Log Analytics workspace id.") |
| 136 | + ] = None, |
| 137 | + snow: Annotated[ |
| 138 | + bool, typer.Option("--snow", help="Also test the ServiceNow commands.") |
| 139 | + ] = False, |
| 140 | + everything: Annotated[ |
| 141 | + bool, typer.Option("--all", help="Also run the slow ones (whole-tenant listings).") |
| 142 | + ] = False, |
| 143 | + only: Annotated[ |
| 144 | + list[str] | None, |
| 145 | + typer.Option("--only", help="Only commands starting with this, e.g. xdr. Repeatable."), |
| 146 | + ] = None, |
| 147 | + profile: Annotated[ |
| 148 | + str | None, typer.Option("--profile", "-p", help="The profile every command uses.") |
| 149 | + ] = None, |
| 150 | + report: Annotated[ |
| 151 | + Path | None, |
| 152 | + typer.Option("--report", help="Also write every outcome, and where each crash was, here."), |
| 153 | + ] = None, |
| 154 | + output: OutputOption = Output.TABLE, |
| 155 | +) -> None: |
| 156 | + """Run every read-only command against the names given, and report what broke. |
| 157 | +
|
| 158 | + Output is thrown away; only each command's outcome is shown: ok, attention (exit 3, as |
| 159 | + designed), refused (an explained error, often a permission), usage, or CRASH (a bug, |
| 160 | + with the line of ldo it came from). Exits 1 when anything crashed. |
| 161 | + """ |
| 162 | + names = { |
| 163 | + "device": device, |
| 164 | + "short": short_name(device) if device else None, |
| 165 | + "user": user, |
| 166 | + "group": group, |
| 167 | + "workspace": workspace, |
| 168 | + "snow": "yes" if snow else None, |
| 169 | + } |
| 170 | + chosen = [ |
| 171 | + case |
| 172 | + for case in CASES |
| 173 | + if all(names.get(need) for need in case.needs) |
| 174 | + and (everything or not case.slow) |
| 175 | + and (not only or any(" ".join(case.args).startswith(prefix) for prefix in only)) |
| 176 | + ] |
| 177 | + filled = {key: value or "" for key, value in names.items()} |
| 178 | + # Filled in, a short --device makes {device} and {short} the same: run it once. |
| 179 | + runs = list( |
| 180 | + dict.fromkeys( |
| 181 | + (tuple(part.format(**filled) for part in case.args), case.stdin) for case in chosen |
| 182 | + ) |
| 183 | + ) |
| 184 | + config_path = get_runtime(ctx).config_path |
| 185 | + render.note(f"running {len(runs)} commands; each one's output is discarded") |
| 186 | + outcomes: list[Outcome] = [] |
| 187 | + try: |
| 188 | + for number, (args, stdin) in enumerate(runs, 1): |
| 189 | + outcome = _run(list(args), stdin, config_path, profile) |
| 190 | + outcomes.append(outcome) |
| 191 | + render.note( |
| 192 | + f"{number}/{len(runs)} {outcome.result:<9} {brand.COMMAND} {outcome.command}" |
| 193 | + ) |
| 194 | + except KeyboardInterrupt: |
| 195 | + render.warn(f"stopped after {len(outcomes)} of {len(runs)}") |
| 196 | + _show(outcomes, output) |
| 197 | + if report is not None: |
| 198 | + report.write_text(json.dumps([vars(item) for item in outcomes], indent=2) + "\n", "utf-8") |
| 199 | + render.note(f"wrote {report}") |
| 200 | + counts = {name: sum(1 for item in outcomes if item.result == name) for name in _RESULTS} |
| 201 | + render.note(", ".join(f"{count} {name}" for name, count in counts.items() if count)) |
| 202 | + if counts["CRASH"] or counts["usage"]: |
| 203 | + raise typer.Exit(1) |
| 204 | + |
| 205 | + |
| 206 | +_RESULTS = ("ok", "attention", "refused", "usage", "CRASH") |
| 207 | + |
| 208 | + |
| 209 | +def _run(args: list[str], stdin: str | None, config: Path | None, profile: str | None) -> Outcome: |
| 210 | + """One command, in this process, with its output captured and dropped.""" |
| 211 | + from typer.testing import CliRunner |
| 212 | + |
| 213 | + from libre_devops_helpers.cli.app import app |
| 214 | + |
| 215 | + command = " ".join(args) |
| 216 | + full = [*(["--config", str(config)] if config else []), *args] |
| 217 | + previous = os.environ.get(brand.PROFILE_ENV) |
| 218 | + if profile: |
| 219 | + os.environ[brand.PROFILE_ENV] = profile |
| 220 | + started = time.monotonic() |
| 221 | + try: |
| 222 | + result = CliRunner().invoke(app, full, input=stdin, catch_exceptions=True) |
| 223 | + finally: |
| 224 | + if profile: |
| 225 | + if previous is None: |
| 226 | + os.environ.pop(brand.PROFILE_ENV, None) |
| 227 | + else: |
| 228 | + os.environ[brand.PROFILE_ENV] = previous |
| 229 | + seconds = round(time.monotonic() - started, 2) |
| 230 | + error = result.exception |
| 231 | + if error is None or isinstance(error, SystemExit): |
| 232 | + code = result.exit_code |
| 233 | + if code == 0: |
| 234 | + return Outcome(command, "ok", code, seconds) |
| 235 | + if code == 3: |
| 236 | + return Outcome(command, "attention", code, seconds, "exit 3: a finding, as designed") |
| 237 | + if code == 2: |
| 238 | + return Outcome(command, "usage", code, seconds, _last_line(result.output)) |
| 239 | + return Outcome(command, "refused", code, seconds, _last_line(result.output)) |
| 240 | + if isinstance(error, LdoError): |
| 241 | + return Outcome(command, "refused", 1, seconds, str(error), error.hint) |
| 242 | + frames = traceback.extract_tb(error.__traceback__) |
| 243 | + ours = [ |
| 244 | + f"{Path(frame.filename).name}:{frame.lineno} in {frame.name}" |
| 245 | + for frame in frames |
| 246 | + if _PACKAGE in frame.filename |
| 247 | + ] |
| 248 | + return Outcome( |
| 249 | + command, "CRASH", None, seconds, f"{type(error).__name__}: {error}", None, ours[-3:] |
| 250 | + ) |
| 251 | + |
| 252 | + |
| 253 | +def _last_line(text: str) -> str: |
| 254 | + lines = [line.strip() for line in (text or "").splitlines() if line.strip()] |
| 255 | + return lines[-1][:200] if lines else "" |
| 256 | + |
| 257 | + |
| 258 | +def _show(outcomes: list[Outcome], output: Output) -> None: |
| 259 | + colours = { |
| 260 | + "ok": "green", |
| 261 | + "attention": "yellow", |
| 262 | + "refused": "yellow", |
| 263 | + "usage": "red", |
| 264 | + "CRASH": "red", |
| 265 | + } |
| 266 | + rows: list[list[render.Cell]] = [] |
| 267 | + for item in outcomes: |
| 268 | + detail = item.detail |
| 269 | + if item.where: |
| 270 | + detail += f" (at {item.where[-1]})" |
| 271 | + rows.append( |
| 272 | + [item.command, (item.result, colours[item.result]), f"{item.seconds:.1f}s", detail] |
| 273 | + ) |
| 274 | + records: list[dict[str, Any]] = [vars(item) for item in outcomes] |
| 275 | + render.emit(output, ["COMMAND", "RESULT", "TIME", "DETAIL"], rows, records) |
0 commit comments