Repository navigation
Add the LambdaTest gateway to incident-scribe and a one-command samples setup #72
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,22 +2,29 @@ | |
|
|
||
| from __future__ import annotations | ||
|
|
||
| import base64 | ||
| import json | ||
| import os | ||
| import time | ||
| import urllib.error | ||
| import urllib.parse | ||
| import urllib.request | ||
| from typing import Any | ||
|
|
||
|
|
||
| class ModelError(RuntimeError): | ||
| """The model could not be reached, or refused. `status` is the HTTP status when an endpoint answered.""" | ||
|
|
||
| def __init__(self, message: str, status: int | None = None) -> None: | ||
| super().__init__(message) | ||
| self.status = status | ||
|
|
||
|
|
||
| ANTHROPIC = {"anthropic", "glm", "zai"} | ||
| RESPONSES = {"responses", "azure", "azure_responses", "openai_responses"} | ||
| CHAT = {"chat", "chat_completions", "openai", "openai_chat"} | ||
| GEMINI = {"gemini", "google", "google_gemini"} | ||
| LAMBDATEST = {"lambdatest", "lt", "gateway"} | ||
|
|
||
|
|
||
| def _config() -> tuple[str, str, str, str]: | ||
|
|
@@ -29,6 +36,10 @@ def _config() -> tuple[str, str, str, str]: | |
| requested = "gemini" | ||
| elif os.environ.get("OPENAI_ENDPOINT", "").strip(): | ||
| requested = "responses" | ||
| elif all(os.environ.get(name, "").strip() for name in ("GATEWAY_URL", "LT_USERNAME", "LT_ACCESS_KEY")) and not any( | ||
| os.environ.get(name, "").strip() for name in ("OPENAI_API_KEY", "OPENAI_INDIA_API_KEY") | ||
| ): | ||
| requested = "lambdatest" | ||
| else: | ||
| requested = "chat" | ||
|
|
||
|
|
@@ -54,6 +65,21 @@ def _config() -> tuple[str, str, str, str]: | |
| raise ModelError("the Gemini model needs GEMINI_API_KEY and GEMINI_MODEL") | ||
| return "gemini", endpoint.rstrip("/"), key, model | ||
|
|
||
| if requested in LAMBDATEST: | ||
| endpoint = os.environ.get("GATEWAY_URL", "").strip() | ||
| username = os.environ.get("LT_USERNAME", "").strip() | ||
| access_key = os.environ.get("LT_ACCESS_KEY", "").strip() | ||
| model = os.environ.get("LT_MODEL", "").strip() or "auto" | ||
| missing = [name for name, value in ( | ||
| ("GATEWAY_URL", endpoint), ("LT_USERNAME", username), ("LT_ACCESS_KEY", access_key) | ||
| ) if not value] | ||
| if missing: | ||
| raise ModelError("the LambdaTest gateway is not configured: " + ", ".join(missing)) | ||
| parts = urllib.parse.urlsplit(endpoint) | ||
| if parts.scheme != "https" and not (parts.scheme == "http" and parts.hostname in ("localhost", "127.0.0.1", "::1")): | ||
| raise ModelError("GATEWAY_URL must start with https:// (the account access key is sent with every call)") | ||
| return "lambdatest", _chat_url(endpoint), base64.b64encode(f"{username}:{access_key}".encode()).decode(), model | ||
|
Contributor
There was a problem hiding this comment. These Basic credentials are the LambdaTest account username and access key, and they go to whatever |
||
|
|
||
| if requested in CHAT: | ||
| endpoint = os.environ.get("OPENAI_CHAT_ENDPOINT", "").strip() | ||
| if not endpoint: | ||
|
|
@@ -242,11 +268,11 @@ def _request(url: str, body: dict[str, Any], headers: dict[str, str]) -> dict[st | |
| with urllib.request.urlopen(req, timeout=_timeout_seconds()) as response: | ||
| data = json.load(response) | ||
| if data.get("error"): | ||
| raise ModelError(f"model returned an error: {data['error']}", status=response.status) | ||
| return data | ||
| except urllib.error.HTTPError as error: | ||
| detail = error.read()[:400].decode("utf-8", "replace") | ||
| raise ModelError(f"model answered {error.code}: {detail}", status=error.code) from None | ||
| except (urllib.error.URLError, TimeoutError, OSError) as error: | ||
| if attempt >= retries: | ||
| raise ModelError(f"could not reach the model: {error}") from None | ||
|
|
@@ -271,11 +297,11 @@ def _call_anthropic(items: list[dict[str, Any]], tools: list[dict[str, Any]] | N | |
| return {"text": "\n".join(block.get("text", "") for block in blocks if block.get("type") == "text").strip(), "tool_calls": calls, "raw_output": [{"role": "assistant", "content": blocks}], "stop_reason": data.get("stop_reason"), "usage": {"input_tokens": usage.get("input_tokens", 0), "output_tokens": usage.get("output_tokens", 0)}} | ||
|
|
||
|
|
||
| def _call_chat(items: list[dict[str, Any]], tools: list[dict[str, Any]] | None, endpoint: str, key: str, model: str, max_tokens: int, scheme: str = "Bearer") -> dict[str, Any]: | ||
| body: dict[str, Any] = {"model": model, "messages": _chat_items(items), "max_tokens": max_tokens} | ||
|
Contributor
There was a problem hiding this comment. The lambdatest provider goes through
Contributor
There was a problem hiding this comment. Still open on ba9646a: the lambdatest provider still goes through |
||
| if tools: | ||
| body["tools"] = _chat_tools(tools) | ||
| data = _request(endpoint, body, {"content-type": "application/json", "Authorization": f"{scheme} {key}"}) | ||
| message = (data.get("choices") or [{}])[0].get("message") or {} | ||
| calls = [] | ||
| for call in message.get("tool_calls") or []: | ||
|
|
@@ -314,6 +340,8 @@ def call(items: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None, | |
| return _call_anthropic(items, tools, endpoint, key, model, max_output_tokens) | ||
| if provider == "chat": | ||
| return _call_chat(items, tools, endpoint, key, model, max_output_tokens) | ||
| if provider == "lambdatest": | ||
| return _call_chat(items, tools, endpoint, key, model, max_output_tokens, scheme="Basic") | ||
| if provider == "gemini": | ||
| return _call_gemini(items, tools, endpoint, key, model, max_output_tokens) | ||
|
|
||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| """Prove the model provider in `.env` works: keys present, endpoint up, auth accepted. | ||
|
|
||
| python scripts/check_llm.py # all three checks | ||
| python scripts/check_llm.py --config-only # keys only, no network call | ||
|
|
||
| Exit codes: 0 all good · 1 not configured · 2 endpoint up but credentials | ||
| refused · 3 endpoint unreachable or the request was rejected. | ||
|
|
||
| Secrets are never printed: only the NAMES of the variables that are set. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import os | ||
| import sys | ||
| import time | ||
| import urllib.error | ||
| import urllib.request | ||
| from pathlib import Path | ||
|
|
||
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | ||
|
|
||
| from incident_scribe.__main__ import _load_dotenv # noqa: E402 | ||
| from incident_scribe import llm # noqa: E402 | ||
|
|
||
| _load_dotenv() | ||
|
|
||
| # Variable names each provider reads, so the report can say which are set. | ||
| VARIABLES = { | ||
| "anthropic": ("ANTHROPIC_BASE_URL", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY", "GLM_API_KEY", "ZAI_API_KEY", "ANTHROPIC_MODEL", "GLM_MODEL"), | ||
| "gemini": ("GEMINI_API_KEY", "GEMINI_MODEL", "GEMINI_ENDPOINT"), | ||
| "chat": ("OPENAI_API_KEY", "OPENAI_INDIA_API_KEY", "OPENAI_CHAT_MODEL", "OPENAI_MODEL", "OPENAI_CHAT_ENDPOINT", "OPENAI_BASE_URL"), | ||
| "responses": ("OPENAI_ENDPOINT", "OPENAI_INDIA_API_KEY", "OPENAI_MODEL"), | ||
| "lambdatest": ("GATEWAY_URL", "LT_USERNAME", "LT_ACCESS_KEY", "LT_MODEL"), | ||
| } | ||
|
|
||
|
|
||
| def _paint(code: str, text: str) -> str: | ||
| return f"\033[{code}m{text}\033[0m" if sys.stdout.isatty() and "NO_COLOR" not in os.environ else text | ||
|
|
||
|
|
||
| def ok(message: str) -> None: | ||
| print(f" {_paint('32', 'ok')} {message}") | ||
|
|
||
|
|
||
| def bad(message: str) -> None: | ||
| print(f" {_paint('31', '!!')} {message}") | ||
|
|
||
|
|
||
| def info(message: str) -> None: | ||
| print(f" {message}") | ||
|
|
||
|
|
||
| def gateway_health() -> bool: | ||
| """GET <GATEWAY_URL>/healthz — unauthenticated, so it isolates "is it up" from "are my keys good". | ||
|
|
||
| A gateway with no such route (404) is not a failure; the credential check below still decides. | ||
| """ | ||
| url = os.environ["GATEWAY_URL"].strip().rstrip("/") + "/healthz" | ||
| try: | ||
| with urllib.request.urlopen(url, timeout=15) as response: | ||
|
Contributor
There was a problem hiding this comment. Only Two ways to hit it:
A final
Contributor
There was a problem hiding this comment. Mostly fixed in ba9646a. The new
Both still end in a traceback with exit 1, so setup shows the "complete the provider settings" hint, doesn't mark incident-scribe failed, and finishes "done.". |
||
| ok(f"health: {url} answered {response.status}") | ||
| return True | ||
| except urllib.error.HTTPError as error: | ||
| if error.code == 404: | ||
| info(f"health: {url} has no health route, relying on the credential check") | ||
| return True | ||
| bad(f"health: {url} answered {error.code}") | ||
| except (urllib.error.URLError, TimeoutError, OSError) as error: | ||
| bad(f"health: {url} unreachable — {error}") | ||
| return False | ||
|
|
||
|
|
||
| def main() -> int: | ||
| config_only = "--config-only" in sys.argv[1:] | ||
|
|
||
| # 1. keys present | ||
| try: | ||
| provider, endpoint, _key, model = llm._config() | ||
| except llm.ModelError as error: | ||
| bad(f"no usable model configuration: {error}") | ||
| return 1 | ||
| present = [name for name in VARIABLES.get(provider, ()) if os.environ.get(name, "").strip()] | ||
| ok(f"provider '{provider}', model '{model}', endpoint {endpoint}") | ||
| ok("credentials present: " + ", ".join(present)) | ||
| if config_only: | ||
| return 0 | ||
|
|
||
| # 2. health. The gateway has a dedicated probe; other providers are judged | ||
| # by the request below, since a 401 from them already proves they are up. | ||
| probed = provider == "lambdatest" | ||
| if probed and not gateway_health(): | ||
| return 3 | ||
|
|
||
| # 3. one tiny real request. Any HTTP answer, even a refusal, proves the | ||
| # endpoint is up; only a 2xx proves the credentials are accepted. | ||
| os.environ.setdefault("LLM_RETRIES", "0") | ||
| started = time.monotonic() | ||
| try: | ||
| llm.call([llm.user("Reply with the single word: ok")], max_output_tokens=64) | ||
| except llm.ModelError as error: | ||
| message = str(error) | ||
| if error.status is None: | ||
| bad(f"health: endpoint unreachable — {message[:300]}") | ||
| return 3 | ||
| if not probed: | ||
| ok(f"health: endpoint is up ({error.status} answered)") | ||
| if error.status in (401, 403): | ||
| bad(f"auth: credentials were refused — check the key/username in .env ({message[:200]})") | ||
| return 2 | ||
| bad(f"auth: could not be confirmed, the request was rejected — {message[:300]}") | ||
| return 3 | ||
| except Exception as error: # an unexpected answer must never read as "not configured" (exit 1) | ||
| bad(f"the endpoint answered something unexpected — {type(error).__name__}: {str(error)[:300]}") | ||
| return 3 | ||
| seconds = time.monotonic() - started | ||
| if not probed: | ||
| ok(f"health: endpoint is up ({seconds:.1f}s round trip)") | ||
| ok(f"auth: credentials accepted, the model answered ({seconds:.1f}s)") | ||
| return 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| raise SystemExit(main()) | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit: the code also skips LambdaTest when
OPENAI_INDIA_API_KEYis set (llm.py line 40), so this could say "all set and neitherOPENAI_API_KEYnorOPENAI_INDIA_API_KEYis".