From d7921b6b07ac6ba95984daa2ab091bddcc612470 Mon Sep 17 00:00:00 2001 From: Kiki McArron Date: Wed, 29 Jul 2026 13:35:17 +0200 Subject: [PATCH] Make 'temperature' ignoreable --- statek/llm_api.py | 58 ++++++++++++++-- statek/provider_config.py | 42 ++++++++++- tests/test_provider_config.py | 116 ++++++++++++++++++++++++++----- tests/test_reasoning_provider.py | 115 ++++++++++++++++++++++++++++-- 4 files changed, 300 insertions(+), 31 deletions(-) diff --git a/statek/llm_api.py b/statek/llm_api.py index 102cbd8..f0e5f3e 100644 --- a/statek/llm_api.py +++ b/statek/llm_api.py @@ -112,6 +112,20 @@ def resolve_reasoning_payload( default_provider: Optional[str], ) -> Tuple[str, Optional[Dict]]: """Resolve model parameters into an upstream model ID and provider payload.""" + upstream_model, payload, _ = _resolve_reasoning_request( + model, + provider_config, + default_provider, + ) + return upstream_model, payload + + +def _resolve_reasoning_request( + model: str, + provider_config: Optional[ProviderConfig], + default_provider: Optional[str], +) -> Tuple[str, Optional[Dict], List[str]]: + """Resolve model parameters into request model, reasoning payload, and conflicts.""" model_name = ensure_model_name(model) provider = select_model_provider(model_name, default_provider=default_provider) aliases = [ @@ -120,6 +134,7 @@ def resolve_reasoning_payload( if key in model_name.params ] payload = None + ignored_parameters: List[str] = [] if aliases: if len(aliases) == 2 and aliases[0] != aliases[1]: raise ValueError("rl and reasoning_level specify conflicting values") @@ -137,6 +152,11 @@ def resolve_reasoning_payload( if payload is None: raise ValueError( "positive reasoning level has no matching provider configuration payload") + ignored_parameters = provider_config.find_ignored_parameters( + provider, + model_name.model_family, + model_name.model, + ) or [] upstream_model = ( format_model_for_provider(model_name, provider) if provider is not None else ( @@ -144,7 +164,27 @@ def resolve_reasoning_payload( if model_name.model_family else model_name.model ) ) - return upstream_model, payload + return upstream_model, payload, ignored_parameters + + +def _ignore_request_parameters( + payload: Dict, + ignored_parameters: Sequence[str], + nested_container: Optional[str] = None, +) -> Dict: + """Remove reasoning-conflicting parameters from final provider request JSON.""" + for parameter in ignored_parameters: + payload.pop(parameter, None) + if nested_container is None: + return payload + nested_payload = payload.get(nested_container) + if not isinstance(nested_payload, dict): + return payload + for parameter in ignored_parameters: + nested_payload.pop(parameter, None) + if not nested_payload: + payload.pop(nested_container) + return payload LLM_Stats = namedtuple( "LLM_Stats", @@ -759,7 +799,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit from .utils import format_tool_spec # pylint: disable=import-outside-toplevel provider = select_model_provider(model, default_provider=(metadata or {}).get("PROVIDER")) - model, reasoning_payload = resolve_reasoning_payload( + model, reasoning_payload, ignored_parameters = _resolve_reasoning_request( self.require_model(model), provider_config, provider, @@ -782,7 +822,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit payload.update(self.kwargs) if reasoning_payload is not None: payload = _deep_merge(payload, reasoning_payload) - return payload + return _ignore_request_parameters(payload, ignored_parameters) async def _process_request( # pylint: disable=too-many-locals,too-many-arguments,too-many-positional-arguments self, @@ -1263,7 +1303,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit from .utils import format_tool_spec # pylint: disable=import-outside-toplevel provider = select_model_provider(model, default_provider=(metadata or {}).get("PROVIDER")) - model, reasoning_payload = resolve_reasoning_payload( + model, reasoning_payload, ignored_parameters = _resolve_reasoning_request( self.require_model(model), provider_config, provider, @@ -1296,7 +1336,11 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit self.require_model(model) if reasoning_payload is not None: payload = _deep_merge(payload, reasoning_payload) - return payload + return _ignore_request_parameters( + payload, + ignored_parameters, + nested_container="generationConfig", + ) @staticmethod def _parse_response( # pylint: disable=too-many-locals @@ -1630,7 +1674,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit from .utils import format_tool_spec # pylint: disable=import-outside-toplevel provider = select_model_provider(model, default_provider=(metadata or {}).get("PROVIDER")) - model, reasoning_payload = resolve_reasoning_payload( + model, reasoning_payload, ignored_parameters = _resolve_reasoning_request( self.require_model(model), provider_config, provider, @@ -1650,7 +1694,7 @@ def _build_request_payload( # pylint: disable=too-many-arguments,too-many-posit payload["temperature"] = temperature if reasoning_payload is not None: payload = _deep_merge(payload, reasoning_payload) - return payload + return _ignore_request_parameters(payload, ignored_parameters) async def _process_request( # pylint: disable=too-many-locals,too-many-arguments,too-many-positional-arguments self, diff --git a/statek/provider_config.py b/statek/provider_config.py index 7879a88..ac38382 100644 --- a/statek/provider_config.py +++ b/statek/provider_config.py @@ -29,7 +29,7 @@ @db0.memo @dataclass class ProviderConfig: - """Durable snapshot of provider-specific model parameter payload mappings.""" + """Durable snapshot of provider-specific reasoning payload and conflict mappings.""" provider_config: Dict[str, Any] @@ -53,11 +53,26 @@ def find_payload(self, *args: Optional[str], **kwargs: Any) -> Optional[Dict[str node = _find_path_node(self.provider_config, path) if node is None: continue - payload = _matching_payload(node, reasoning_level) + reasoning = _reasoning_config(node) + if reasoning is None: + continue + payload = _matching_payload(reasoning, reasoning_level) if payload is not None: return payload return None + def find_ignored_parameters(self, *args: Optional[str]) -> Optional[List[str]]: + """Return the deepest explicit reasoning parameter ignore list for a model path.""" + for path in _candidate_paths(args): + node = _find_path_node(self.provider_config, path) + if node is None: + continue + reasoning = _reasoning_config(node) + if reasoning is None or "ignore_parameters" not in reasoning: + continue + return _ignored_parameters(reasoning["ignore_parameters"]) + return None + def resolve_provider_config(provider_config: Mapping[str, Any]) -> ProviderConfig: """Reuse or persist a durable provider-configuration snapshot by exact content.""" @@ -141,6 +156,29 @@ def _matching_payload(config: Mapping[str, Any], reasoning_level: int) -> Option return None +def _reasoning_config(config: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: + """Return the canonical nested reasoning configuration at one hierarchy node.""" + if "reasoning" not in config: + return None + reasoning = config["reasoning"] + if not _is_mapping(reasoning): + raise ValueError("Provider configuration reasoning must be a mapping") + return reasoning + + +def _ignored_parameters(parameters: Any) -> List[str]: + """Validate and copy a reasoning parameter ignore list.""" + error_message = ( + "Provider configuration reasoning ignore_parameters must be a sequence of strings" + ) + if not _is_sequence(parameters): + raise ValueError(error_message) + copied_parameters = list(parameters) + if any(not isinstance(parameter, str) for parameter in copied_parameters): + raise ValueError(error_message) + return copied_parameters + + def _is_mapping(value: Any) -> bool: """Return whether a regular or dbzero persistent value exposes mapping items.""" return isinstance(value, Mapping) or callable(getattr(value, "items", None)) diff --git a/tests/test_provider_config.py b/tests/test_provider_config.py index 91e861d..c521b35 100644 --- a/tests/test_provider_config.py +++ b/tests/test_provider_config.py @@ -18,27 +18,36 @@ def provider_config(db0_fixture): del db0_fixture return ProviderConfig({ "openrouter": { - "reasoning_level": [ - { - "range": {"from": 1, "to": 100}, - "payload": {"reasoning": {"effort": "low"}}, - }, - ], - "openai": { + "reasoning": { + "ignore_parameters": ["temperature"], "reasoning_level": [ { - "range": {"from": 26, "to": 75}, - "payload": {"reasoning": {"effort": "medium"}}, + "range": {"from": 1, "to": 100}, + "payload": {"reasoning": {"effort": "low"}}, }, ], - "gpt-5.4": { + }, + "openai": { + "reasoning": { + "ignore_parameters": ["temperature", "top_p"], "reasoning_level": [ { - "range": {"from": 76}, - "payload": {"reasoning": {"effort": "high"}}, + "range": {"from": 26, "to": 75}, + "payload": {"reasoning": {"effort": "medium"}}, }, ], }, + "gpt-5.4": { + "reasoning": { + "ignore_parameters": ["temperature", "logprobs"], + "reasoning_level": [ + { + "range": {"from": 76}, + "payload": {"reasoning": {"effort": "high"}}, + }, + ], + }, + }, }, }, }) @@ -100,10 +109,12 @@ def test_find_payload_resolves_provider_model_path_when_family_is_redundant(db0_ config = ProviderConfig({ "openai": { "gpt-5": { - "reasoning_level": [{ - "range": {"from": 1}, - "payload": {"reasoning_effort": "high"}, - }], + "reasoning": { + "reasoning_level": [{ + "range": {"from": 1}, + "payload": {"reasoning_effort": "high"}, + }], + }, }, }, }) @@ -123,6 +134,79 @@ def test_find_payload_returns_a_defensive_copy(provider_config): } +def test_find_ignored_parameters_prefers_the_deepest_explicit_mapping(provider_config): + """The most-specific explicit ignore list overrides broader reasoning policy.""" + assert provider_config.find_ignored_parameters( + "openrouter", "openai", "gpt-5.4", + ) == ["temperature", "logprobs"] + + +def test_find_ignored_parameters_falls_back_when_model_omits_the_list(provider_config): + """A nested reasoning mapping inherits the closest explicit parent ignore list.""" + assert provider_config.find_ignored_parameters( + "openrouter", "openai", "gpt-5.3", + ) == ["temperature", "top_p"] + + +def test_find_ignored_parameters_allows_an_explicit_empty_override(db0_fixture): + """An empty list disables a broader ignore policy for one model mapping.""" + del db0_fixture + config = ProviderConfig({ + "openrouter": { + "reasoning": {"ignore_parameters": ["temperature"]}, + "openai": { + "gpt-5.4": { + "reasoning": {"ignore_parameters": []}, + }, + }, + }, + }) + + assert config.find_ignored_parameters("openrouter", "openai", "gpt-5.4") == [] + + +def test_find_ignored_parameters_returns_a_defensive_copy(provider_config): + """Request formatting cannot mutate the durable ignored-parameter list.""" + parameters = provider_config.find_ignored_parameters("openrouter", "openai") + parameters.append("logprobs") + + assert provider_config.find_ignored_parameters("openrouter", "openai") == [ + "temperature", "top_p", + ] + + +@pytest.mark.parametrize( + "reasoning_config", + [ + [], + {"ignore_parameters": "temperature"}, + {"ignore_parameters": ["temperature", 1]}, + ], +) +def test_reasoning_configuration_rejects_invalid_ignore_parameters(db0_fixture, reasoning_config): + """The canonical reasoning object requires a list of parameter names.""" + del db0_fixture + config = ProviderConfig({"openrouter": {"reasoning": reasoning_config}}) + + with pytest.raises(ValueError, match="reasoning"): + config.find_ignored_parameters("openrouter") + + +def test_flat_reasoning_level_configuration_is_not_supported(db0_fixture): + """Only the canonical nested reasoning configuration resolves mappings.""" + del db0_fixture + config = ProviderConfig({ + "openrouter": { + "reasoning_level": [{ + "range": {"from": 1}, + "payload": {"reasoning": {"effort": "high"}}, + }], + }, + }) + + assert config.find_payload("openrouter", reasoning_level=1) is None + + @pytest.mark.parametrize("reasoning_level", [-1, 101, "invalid"]) def test_find_payload_rejects_invalid_reasoning_levels(provider_config, reasoning_level): """Reasoning levels must be integer values in the Statek 0–100 range.""" diff --git a/tests/test_reasoning_provider.py b/tests/test_reasoning_provider.py index 7fe502a..730ada2 100644 --- a/tests/test_reasoning_provider.py +++ b/tests/test_reasoning_provider.py @@ -17,9 +17,12 @@ from statek.provider_config import ProviderConfig -def _config(provider, family=None, model=None, payload=None): +def _config(provider, family=None, model=None, payload=None, ignored_parameters=None): """Create a single reasoning-level mapping at an optional model path.""" - node = {"reasoning_level": [{"range": {"from": 1}, "payload": payload or {}}]} + reasoning = {"reasoning_level": [{"range": {"from": 1}, "payload": payload or {}}]} + if ignored_parameters is not None: + reasoning["ignore_parameters"] = ignored_parameters + node = {"reasoning": reasoning} if model is not None: node = {family: {model: node}} elif family is not None: @@ -81,10 +84,12 @@ def test_metadata_provider_resolves_model_specific_reasoning(db0_fixture): config = ProviderConfig({ "openai": { "gpt-5": { - "reasoning_level": [{ - "range": {"from": 1}, - "payload": {"reasoning": {"effort": "high"}}, - }], + "reasoning": { + "reasoning_level": [{ + "range": {"from": 1}, + "payload": {"reasoning": {"effort": "high"}}, + }], + }, }, }, }) @@ -108,6 +113,104 @@ def test_provider_payload_deep_merges_after_provider_defaults(db0_fixture): assert payload["reasoning"] == {"effort": "high", "summary": "auto"} +@pytest.mark.parametrize( + ("api_class", "api_kwargs", "model", "expected_container"), + [ + (DefaultLLM_API_Impl, {}, "openrouter/openai/gpt-5/rl=1", None), + (ClaudeAI_API, {"use_prompt_caching": False}, "claudeai//claude-4/rl=1", None), + (VertexAI_API, {}, "vertexai//gemini-3/rl=1", "generationConfig"), + ], +) +def test_positive_reasoning_removes_configured_temperature_from_final_payload( + db0_fixture, + api_class, + api_kwargs, + model, + expected_container, +): + """Configured conflicts are removed after every provider payload source is merged.""" + api = api_class(_settings(), temperature=0.7, **api_kwargs) + config = _config( + model.split("/", 1)[0], + payload={"temperature": 0.5}, + ignored_parameters=["temperature"], + ) + + payload = api.preview_request( + model=model, + temperature=0.3, + provider_config=config, + ) + + if expected_container is None: + assert "temperature" not in payload + else: + assert "temperature" not in payload.get(expected_container, {}) + + +def test_positive_reasoning_with_empty_payload_still_ignores_temperature(db0_fixture): + """An empty mapping is still an enabled reasoning configuration.""" + api = DefaultLLM_API_Impl(_settings()) + config = _config("openai", payload={}, ignored_parameters=["temperature"]) + + payload = api.preview_request( + model="openai//gpt-5/rl=1", + temperature=0.3, + provider_config=config, + ) + + assert "temperature" not in payload + + +def test_zero_reasoning_keeps_temperature_when_configuration_ignores_it(db0_fixture): + """Ignoring applies only to a positive configured reasoning level.""" + api = DefaultLLM_API_Impl(_settings()) + config = _config("openai", ignored_parameters=["temperature"]) + + payload = api.preview_request( + model="openai//gpt-5/rl=0", + temperature=0.3, + provider_config=config, + ) + + assert payload["temperature"] == 0.3 + + +@pytest.mark.asyncio +async def test_reasoning_preview_matches_filtered_execution_payload(db0_fixture): + """Preview exposes the same post-merge parameter filtering as execution.""" + api = DefaultLLM_API_Impl(_settings(), temperature=0.7) + config = _config( + "openrouter", + payload={"reasoning": {"effort": "high"}, "temperature": 0.5}, + ignored_parameters=["temperature"], + ) + captured_payload = {} + + async def fake_post(*_args, **kwargs): + captured_payload.update(kwargs["json"]) + response = MagicMock(content=b'{"choices":[{"message":{"content":"ok"}}]}') + response.raise_for_status = MagicMock() + response.json.return_value = {"choices": [{"message": {"content": "ok"}}]} + return response + + preview = api.preview_request( + model="openrouter/openai/gpt-5/rl=1", + temperature=0.3, + provider_config=config, + ) + + with patch("httpx.AsyncClient.post", fake_post): + await api.process_request( + model="openrouter/openai/gpt-5/rl=1", + temperature=0.3, + provider_config=config, + ) + + assert preview == captured_payload + assert "temperature" not in preview + + @pytest.mark.asyncio async def test_openai_reasoning_details_round_trip_without_entering_visible_text(db0_fixture): api = DefaultLLM_API_Impl(_settings())