Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 51 additions & 7 deletions statek/llm_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,20 @@ def resolve_reasoning_payload(
default_provider: Optional[str],
) -> Tuple[str, Optional[Dict]]:
"""Resolve model parameters into an upstream model ID and provider payload."""
upstream_model, payload, _ = _resolve_reasoning_request(
model,
provider_config,
default_provider,
)
return upstream_model, payload


def _resolve_reasoning_request(
model: str,
provider_config: Optional[ProviderConfig],
default_provider: Optional[str],
) -> Tuple[str, Optional[Dict], List[str]]:
"""Resolve model parameters into request model, reasoning payload, and conflicts."""
model_name = ensure_model_name(model)
provider = select_model_provider(model_name, default_provider=default_provider)
aliases = [
Expand All @@ -120,6 +134,7 @@ def resolve_reasoning_payload(
if key in model_name.params
]
payload = None
ignored_parameters: List[str] = []
if aliases:
if len(aliases) == 2 and aliases[0] != aliases[1]:
raise ValueError("rl and reasoning_level specify conflicting values")
Expand All @@ -137,14 +152,39 @@ def resolve_reasoning_payload(
if payload is None:
raise ValueError(
"positive reasoning level has no matching provider configuration payload")
ignored_parameters = provider_config.find_ignored_parameters(
provider,
model_name.model_family,
model_name.model,
) or []
upstream_model = (
format_model_for_provider(model_name, provider)
if provider is not None else (
f"{model_name.model_family}/{model_name.model}"
if model_name.model_family else model_name.model
)
)
return upstream_model, payload
return upstream_model, payload, ignored_parameters


def _ignore_request_parameters(
payload: Dict,
ignored_parameters: Sequence[str],
nested_container: Optional[str] = None,
) -> Dict:
"""Remove reasoning-conflicting parameters from final provider request JSON."""
for parameter in ignored_parameters:
payload.pop(parameter, None)
if nested_container is None:
return payload
nested_payload = payload.get(nested_container)
if not isinstance(nested_payload, dict):
return payload
for parameter in ignored_parameters:
nested_payload.pop(parameter, None)
if not nested_payload:
payload.pop(nested_container)
return payload

LLM_Stats = namedtuple(
"LLM_Stats",
Expand Down Expand Up @@ -759,7 +799,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit
from .utils import format_tool_spec # pylint: disable=import-outside-toplevel

provider = select_model_provider(model, default_provider=(metadata or {}).get("PROVIDER"))
model, reasoning_payload = resolve_reasoning_payload(
model, reasoning_payload, ignored_parameters = _resolve_reasoning_request(
self.require_model(model),
provider_config,
provider,
Expand All @@ -782,7 +822,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit
payload.update(self.kwargs)
if reasoning_payload is not None:
payload = _deep_merge(payload, reasoning_payload)
return payload
return _ignore_request_parameters(payload, ignored_parameters)

async def _process_request( # pylint: disable=too-many-locals,too-many-arguments,too-many-positional-arguments
self,
Expand Down Expand Up @@ -1263,7 +1303,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit
from .utils import format_tool_spec # pylint: disable=import-outside-toplevel

provider = select_model_provider(model, default_provider=(metadata or {}).get("PROVIDER"))
model, reasoning_payload = resolve_reasoning_payload(
model, reasoning_payload, ignored_parameters = _resolve_reasoning_request(
self.require_model(model),
provider_config,
provider,
Expand Down Expand Up @@ -1296,7 +1336,11 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit
self.require_model(model)
if reasoning_payload is not None:
payload = _deep_merge(payload, reasoning_payload)
return payload
return _ignore_request_parameters(
payload,
ignored_parameters,
nested_container="generationConfig",
)

@staticmethod
def _parse_response( # pylint: disable=too-many-locals
Expand Down Expand Up @@ -1630,7 +1674,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit
from .utils import format_tool_spec # pylint: disable=import-outside-toplevel

provider = select_model_provider(model, default_provider=(metadata or {}).get("PROVIDER"))
model, reasoning_payload = resolve_reasoning_payload(
model, reasoning_payload, ignored_parameters = _resolve_reasoning_request(
self.require_model(model),
provider_config,
provider,
Expand All @@ -1650,7 +1694,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit
payload["temperature"] = temperature
if reasoning_payload is not None:
payload = _deep_merge(payload, reasoning_payload)
return payload
return _ignore_request_parameters(payload, ignored_parameters)

async def _process_request( # pylint: disable=too-many-locals,too-many-arguments,too-many-positional-arguments
self,
Expand Down
42 changes: 40 additions & 2 deletions statek/provider_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
@db0.memo
@dataclass
class ProviderConfig:
"""Durable snapshot of provider-specific model parameter payload mappings."""
"""Durable snapshot of provider-specific reasoning payload and conflict mappings."""

provider_config: Dict[str, Any]

Expand All @@ -53,11 +53,26 @@ def find_payload(self, *args: Optional[str], **kwargs: Any) -> Optional[Dict[str
node = _find_path_node(self.provider_config, path)
if node is None:
continue
payload = _matching_payload(node, reasoning_level)
reasoning = _reasoning_config(node)
if reasoning is None:
continue
payload = _matching_payload(reasoning, reasoning_level)
if payload is not None:
return payload
return None

def find_ignored_parameters(self, *args: Optional[str]) -> Optional[List[str]]:
"""Return the deepest explicit reasoning parameter ignore list for a model path."""
for path in _candidate_paths(args):
node = _find_path_node(self.provider_config, path)
if node is None:
continue
reasoning = _reasoning_config(node)
if reasoning is None or "ignore_parameters" not in reasoning:
continue
return _ignored_parameters(reasoning["ignore_parameters"])
return None


def resolve_provider_config(provider_config: Mapping[str, Any]) -> ProviderConfig:
"""Reuse or persist a durable provider-configuration snapshot by exact content."""
Expand Down Expand Up @@ -141,6 +156,29 @@ def _matching_payload(config: Mapping[str, Any], reasoning_level: int) -> Option
return None


def _reasoning_config(config: Mapping[str, Any]) -> Optional[Mapping[str, Any]]:
"""Return the canonical nested reasoning configuration at one hierarchy node."""
if "reasoning" not in config:
return None
reasoning = config["reasoning"]
if not _is_mapping(reasoning):
raise ValueError("Provider configuration reasoning must be a mapping")
return reasoning


def _ignored_parameters(parameters: Any) -> List[str]:
"""Validate and copy a reasoning parameter ignore list."""
error_message = (
"Provider configuration reasoning ignore_parameters must be a sequence of strings"
)
if not _is_sequence(parameters):
raise ValueError(error_message)
copied_parameters = list(parameters)
if any(not isinstance(parameter, str) for parameter in copied_parameters):
raise ValueError(error_message)
return copied_parameters


def _is_mapping(value: Any) -> bool:
"""Return whether a regular or dbzero persistent value exposes mapping items."""
return isinstance(value, Mapping) or callable(getattr(value, "items", None))
Expand Down
116 changes: 100 additions & 16 deletions tests/test_provider_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,36 @@ def provider_config(db0_fixture):
del db0_fixture
return ProviderConfig({
"openrouter": {
"reasoning_level": [
{
"range": {"from": 1, "to": 100},
"payload": {"reasoning": {"effort": "low"}},
},
],
"openai": {
"reasoning": {
"ignore_parameters": ["temperature"],
"reasoning_level": [
{
"range": {"from": 26, "to": 75},
"payload": {"reasoning": {"effort": "medium"}},
"range": {"from": 1, "to": 100},
"payload": {"reasoning": {"effort": "low"}},
},
],
"gpt-5.4": {
},
"openai": {
"reasoning": {
"ignore_parameters": ["temperature", "top_p"],
"reasoning_level": [
{
"range": {"from": 76},
"payload": {"reasoning": {"effort": "high"}},
"range": {"from": 26, "to": 75},
"payload": {"reasoning": {"effort": "medium"}},
},
],
},
"gpt-5.4": {
"reasoning": {
"ignore_parameters": ["temperature", "logprobs"],
"reasoning_level": [
{
"range": {"from": 76},
"payload": {"reasoning": {"effort": "high"}},
},
],
},
},
},
},
})
Expand Down Expand Up @@ -100,10 +109,12 @@ def test_find_payload_resolves_provider_model_path_when_family_is_redundant(db0_
config = ProviderConfig({
"openai": {
"gpt-5": {
"reasoning_level": [{
"range": {"from": 1},
"payload": {"reasoning_effort": "high"},
}],
"reasoning": {
"reasoning_level": [{
"range": {"from": 1},
"payload": {"reasoning_effort": "high"},
}],
},
},
},
})
Expand All @@ -123,6 +134,79 @@ def test_find_payload_returns_a_defensive_copy(provider_config):
}


def test_find_ignored_parameters_prefers_the_deepest_explicit_mapping(provider_config):
"""The most-specific explicit ignore list overrides broader reasoning policy."""
assert provider_config.find_ignored_parameters(
"openrouter", "openai", "gpt-5.4",
) == ["temperature", "logprobs"]


def test_find_ignored_parameters_falls_back_when_model_omits_the_list(provider_config):
"""A nested reasoning mapping inherits the closest explicit parent ignore list."""
assert provider_config.find_ignored_parameters(
"openrouter", "openai", "gpt-5.3",
) == ["temperature", "top_p"]


def test_find_ignored_parameters_allows_an_explicit_empty_override(db0_fixture):
"""An empty list disables a broader ignore policy for one model mapping."""
del db0_fixture
config = ProviderConfig({
"openrouter": {
"reasoning": {"ignore_parameters": ["temperature"]},
"openai": {
"gpt-5.4": {
"reasoning": {"ignore_parameters": []},
},
},
},
})

assert config.find_ignored_parameters("openrouter", "openai", "gpt-5.4") == []


def test_find_ignored_parameters_returns_a_defensive_copy(provider_config):
"""Request formatting cannot mutate the durable ignored-parameter list."""
parameters = provider_config.find_ignored_parameters("openrouter", "openai")
parameters.append("logprobs")

assert provider_config.find_ignored_parameters("openrouter", "openai") == [
"temperature", "top_p",
]


@pytest.mark.parametrize(
"reasoning_config",
[
[],
{"ignore_parameters": "temperature"},
{"ignore_parameters": ["temperature", 1]},
],
)
def test_reasoning_configuration_rejects_invalid_ignore_parameters(db0_fixture, reasoning_config):
"""The canonical reasoning object requires a list of parameter names."""
del db0_fixture
config = ProviderConfig({"openrouter": {"reasoning": reasoning_config}})

with pytest.raises(ValueError, match="reasoning"):
config.find_ignored_parameters("openrouter")


def test_flat_reasoning_level_configuration_is_not_supported(db0_fixture):
"""Only the canonical nested reasoning configuration resolves mappings."""
del db0_fixture
config = ProviderConfig({
"openrouter": {
"reasoning_level": [{
"range": {"from": 1},
"payload": {"reasoning": {"effort": "high"}},
}],
},
})

assert config.find_payload("openrouter", reasoning_level=1) is None


@pytest.mark.parametrize("reasoning_level", [-1, 101, "invalid"])
def test_find_payload_rejects_invalid_reasoning_levels(provider_config, reasoning_level):
"""Reasoning levels must be integer values in the Statek 0–100 range."""
Expand Down
Loading
Loading