From d57ac0a99344d223b056048031731e867bf54f2a Mon Sep 17 00:00:00 2001 From: "Lindo St. Angel" Date: Sat, 17 Oct 2026 19:01:00 -0700 Subject: [PATCH 1/2] fix: always offer HassTurnOn on an actuation request (#795) The actuation safety leg ranks tools by similarity and keeps three, so for "open the study shade" vacuum and media tools that scored higher crowded out HassTurnOn, the only tool HA gives an assistant to open a cover. The model guessed a tool that does not exist and was refused. Bind HassTurnOn outside the limit on any query the actuation safety net accepts (the on/off partner step then binds HassTurnOff), unless the user excluded it. Fixes #795 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SFM2hXVUELSh13U7FEKGJu --- CHANGELOG.md | 4 + .../home_generative_agent/agent/graph.py | 42 +++++++- .../test_tool_retrieval.py | 99 +++++++++++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9942605e..4106b277 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ their own. See [RELEASING.md](RELEASING.md). ## [Unreleased] +### Fixed + +- **Opening or closing a cover by voice no longer fails when other device tools rank higher** ([#795](https://github.com/goruck/home-generative-agent/issues/795)). "Open the study shade" could fail with "I couldn't find a suitable tool": the agent offers the model only the few tools that best match the request, and device tools such as a vacuum's start or a media player's unpause could outscore the general "turn on" tool, which is the only one Home Assistant gives an assistant for opening a cover. The model then guessed a tool that does not exist and the call was refused. Any request with an action word ("turn", "open", "close", ...) now always offers the turn on and turn off tools, alongside what the ranking chose. A tool you excluded is still never offered. + ### Changed - The model-provider libraries are no longer pinned to one exact version each: `langchain-openai`, `langchain-anthropic` and `langchain-ollama` accept any 1.x release (`langchain-ollama` from 0.3.10), `langchain-google-genai` any 3.x from 3.1.0, and the `anthropic` SDK anything from 0.125.0 below 2. When a Home Assistant release moved a library these depend on, such as `openai` in 2026.10, an exact pin could not be installed next to it and the integration failed to set up for everyone until a new release shipped (#735). Home Assistant can now pick a compatible release on its own. A fresh install, or the reinstall that follows a Home Assistant update, gets the newest release in each range: today `langchain-openai` 1.7.2, `langchain-anthropic` 1.7.6 with `anthropic` 1.15.0, `langchain-ollama` 1.1.0 and `langchain-google-genai` 3.2.0. The test suite runs against the newest releases every day. diff --git a/custom_components/home_generative_agent/agent/graph.py b/custom_components/home_generative_agent/agent/graph.py index 149a8d29..aff4e940 100644 --- a/custom_components/home_generative_agent/agent/graph.py +++ b/custom_components/home_generative_agent/agent/graph.py @@ -1327,7 +1327,8 @@ async def _append_history_tool( # noqa: PLR0913 return candidates if fetched is None else [*candidates, fetched] -_ON_OFF_PARTNER = {"HassTurnOn": "HassTurnOff", "HassTurnOff": "HassTurnOn"} +_TURN_ON_TOOL = "HassTurnOn" +_ON_OFF_PARTNER = {_TURN_ON_TOOL: "HassTurnOff", "HassTurnOff": _TURN_ON_TOOL} async def _append_on_off_partners( @@ -1371,6 +1372,39 @@ async def _append_on_off_partners( return candidates +async def _append_turn_on_tool( # noqa: PLR0913 + candidates: list[RawTool], + store: BaseStore, + config: RunnableConfig, + query: str, + allowed_api_ids: set[str], + live_tool_ids: set[tuple[str, str]] | None, +) -> list[RawTool]: + """ + Bind HassTurnOn on an actuation request when ranking left it out (#795). + + HassTurnOn is what opens and closes a cover, turns on a switch and presses + a button: Home Assistant offers no cover-specific open tool. The actuation + safety leg ranks by similarity and keeps three, so for "open the study + shade" device tools (vacuum, media) that scored higher crowded it out, and + the model guessed a tool that does not exist. Appended outside the limit + like GetLiveContext; `_append_on_off_partners` then binds HassTurnOff. + """ + if not _query_needs_actuation_safety(query) or any( + base_tool_name(t["name"]) in _ON_OFF_PARTNER for t in candidates + ): + return candidates + fetched = await _get_tool_by_name( + store, config, _TURN_ON_TOOL, allowed_api_ids, live_tool_ids + ) + if fetched is None: + return candidates + excluded_names = tool_exclusions(config.get("configurable", {}).get("options", {})) + if is_tool_excluded(fetched["name"], excluded_names.get(fetched["api_id"], ())): + return candidates + return [*candidates, fetched] + + # How many trailing HUMAN turns establish automation-creation context for # follow-up turns: the current message plus two prior user exchanges. Counting # human turns (not raw messages) keeps the window stable when a single turn @@ -2830,6 +2864,12 @@ async def _retrieve_tools( # noqa: PLR0915 all_candidates, store, config, live_tool_ids ) + # 3i. Bind HassTurnOn on an actuation request ranking left it out (#795). + # Before 3h, which then binds HassTurnOff with it. + all_candidates = await _append_turn_on_tool( + all_candidates, store, config, query, allowed_api_ids, live_tool_ids + ) + # 3h. Bind HassTurnOn and HassTurnOff together. After 3e, so an # always-included one gets its partner too; it dedupes by (api, name). all_candidates = await _append_on_off_partners( diff --git a/tests/custom_components/home_generative_agent/test_tool_retrieval.py b/tests/custom_components/home_generative_agent/test_tool_retrieval.py index 0b492607..1cbe0c6d 100644 --- a/tests/custom_components/home_generative_agent/test_tool_retrieval.py +++ b/tests/custom_components/home_generative_agent/test_tool_retrieval.py @@ -40,6 +40,7 @@ _retrieve_tools, _split_query_intents, ) +from custom_components.home_generative_agent.agent.helpers import tool_index_key from custom_components.home_generative_agent.const import ACTUATION_KEYWORDS_REGEX if TYPE_CHECKING: @@ -3728,3 +3729,101 @@ async def test_retrieve_tools_binds_no_on_off_tool_when_neither_ranked() -> None ) assert "intent__HassTurnOn" not in result["tool_routing_map"] assert "intent__HassTurnOff" not in result["tool_routing_map"] + + +_COVER_FIELD_TOOLS = ( + "vacuum__HassVacuumStart", + "media_player__HassMediaUnpause", + "media_player__HassMediaPlayerUnmute", + "intent__HassTurnOn", + "intent__HassTurnOff", + "homeassistant__GetLiveContext", +) + + +def _cover_field_store() -> MagicMock: + """ + Rank as the box did for "Turn on Everything" (#795): device tools first. + + HassTurnOn is indexed but never among the hits, so only force-binding can + offer it. + """ + hits = [] + for name, score in ( + ("vacuum__HassVacuumStart", 0.591), + ("media_player__HassMediaUnpause", 0.567), + ("media_player__HassMediaPlayerUnmute", 0.545), + ): + item = _indexed_item(name) + item.value["is_actuation"] = True + item.score = score + hits.append(item) + indexed = { + tool_index_key("assist", name): _indexed_item(name) + for name in _COVER_FIELD_TOOLS + } + store = MagicMock() + store.asearch = AsyncMock(return_value=hits) + store.aget = AsyncMock(side_effect=lambda _ns, key: indexed.get(key)) + return store + + +def _cover_field_config(*, excluded: list[str] | None = None) -> RunnableConfig: + options: dict[str, Any] = { + "llm_hass_api": ["assist"], + "tool_relevance_threshold": 0.15, + "tool_retrieval_limit": 5, + } + if excluded: + options["tool_exclusions"] = {"assist": excluded} + return cast( + "RunnableConfig", + { + "configurable": { + "options": options, + "tool_index_ready": True, + "langchain_tools": {}, + "ha_llm_api": _live_llm_api(*_COVER_FIELD_TOOLS), + } + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "query", ["Open the study shade", "Turn on Everything", "close the garage door"] +) +async def test_an_actuation_request_always_offers_turn_on_and_off( + query: str, +) -> None: + """Device tools that outrank HassTurnOn no longer crowd it out (#795).""" + result = await _retrieve_tools( + _query_state(query), _cover_field_config(), store=_cover_field_store() + ) + routed = result["tool_routing_map"] + assert "intent__HassTurnOn" in routed + assert "intent__HassTurnOff" in routed + # Appended past the limit: nothing the ranking chose was evicted. + assert "vacuum__HassVacuumStart" in routed + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "query", ["What's the weather tomorrow?", "is the garage door open?"] +) +async def test_turn_on_is_not_bound_without_an_actuation_request(query: str) -> None: + result = await _retrieve_tools( + _query_state(query), _cover_field_config(), store=_cover_field_store() + ) + assert "intent__HassTurnOn" not in result["tool_routing_map"] + + +@pytest.mark.asyncio +async def test_an_excluded_turn_on_is_not_bound() -> None: + """The user's per-tool exclusion still wins.""" + result = await _retrieve_tools( + _query_state("Open the study shade"), + _cover_field_config(excluded=["intent__HassTurnOn"]), + store=_cover_field_store(), + ) + assert "intent__HassTurnOn" not in result["tool_routing_map"] From 9cbd951293803342e1df3c31bed7038ff96cad77 Mon Sep 17 00:00:00 2001 From: "Lindo St. Angel" Date: Sat, 17 Oct 2026 21:40:16 -0700 Subject: [PATCH 2/2] fix: offer HassTurnOn on a "yes" to an offer, exclusion before fetch (#795) Review follow-ups: a "Yes" to "Want me to open the Study Shade?" carries no action word, so the agent's offer is checked too. The exclusion is checked by base name before the fetch, so a namespaced exclusion also blocks the bare pre-2026.9 index row when the live filter is off. Steps renumbered 3h/3i. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SFM2hXVUELSh13U7FEKGJu --- CHANGELOG.md | 2 +- .../home_generative_agent/agent/graph.py | 50 +++++++++++---- .../test_tool_retrieval.py | 62 +++++++++++++++---- 3 files changed, 90 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4106b277..42b7e4c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ their own. See [RELEASING.md](RELEASING.md). ### Fixed -- **Opening or closing a cover by voice no longer fails when other device tools rank higher** ([#795](https://github.com/goruck/home-generative-agent/issues/795)). "Open the study shade" could fail with "I couldn't find a suitable tool": the agent offers the model only the few tools that best match the request, and device tools such as a vacuum's start or a media player's unpause could outscore the general "turn on" tool, which is the only one Home Assistant gives an assistant for opening a cover. The model then guessed a tool that does not exist and the call was refused. Any request with an action word ("turn", "open", "close", ...) now always offers the turn on and turn off tools, alongside what the ranking chose. A tool you excluded is still never offered. +- **Opening or closing a cover by voice no longer fails when other device tools rank higher** ([#795](https://github.com/goruck/home-generative-agent/issues/795)). "Open the study shade" could fail with "I couldn't find a suitable tool": the agent offers the model only the few tools that best match the request, and device tools such as a vacuum's start or a media player's unpause could outscore the general "turn on" tool, which is the only one Home Assistant gives an assistant for opening a cover. The model then guessed a tool that does not exist and the call was refused. A request with an English action word ("turn", "open", "close", ...), or a "yes" to the agent offering one ("Want me to open the Study Shade?"), now always offers the turn on and turn off tools, alongside what the ranking chose. Requests in other languages, and replies such as "the left one", still rely on the ranking. A tool you excluded is still never offered. ### Changed diff --git a/custom_components/home_generative_agent/agent/graph.py b/custom_components/home_generative_agent/agent/graph.py index aff4e940..6f44133c 100644 --- a/custom_components/home_generative_agent/agent/graph.py +++ b/custom_components/home_generative_agent/agent/graph.py @@ -1376,7 +1376,7 @@ async def _append_turn_on_tool( # noqa: PLR0913 candidates: list[RawTool], store: BaseStore, config: RunnableConfig, - query: str, + messages: Sequence[BaseMessage], allowed_api_ids: set[str], live_tool_ids: set[tuple[str, str]] | None, ) -> list[RawTool]: @@ -1389,19 +1389,41 @@ async def _append_turn_on_tool( # noqa: PLR0913 shade" device tools (vacuum, media) that scored higher crowded it out, and the model guessed a tool that does not exist. Appended outside the limit like GetLiveContext; `_append_on_off_partners` then binds HassTurnOff. - """ - if not _query_needs_actuation_safety(query) or any( + + A "Yes" to "Want me to open the Study Shade?" has no action word, so the + agent's offer is checked too. That widens the tool list only and gates + nothing, which is why the raw-query rule for intent detectors does not + apply here. The offer alone, not the ranking query: that one also carries + the user's earlier "is it open?", which reads as a read-only question. + """ + query = _message_text(messages[-1]) if messages else "" + wants_action = ( + _query_needs_actuation_safety(query) + or ( + _is_affirmative_follow_up(query) + # Action word only: an offer often opens with the state ("It is + # closed."), which the read-only check would take as a question. + and bool(re.search(ACTUATION_KEYWORDS_REGEX, _offer_text(messages[:-1]))) + ) + ) + if not wants_action or any( base_tool_name(t["name"]) in _ON_OFF_PARTNER for t in candidates ): return candidates + excluded_names = tool_exclusions(config.get("configurable", {}).get("options", {})) + # Before the fetch, and by base name: with the live filter off the bare + # lookup can return the pre-2026.9 "HassTurnOn" row, which a namespaced + # exclusion ("intent__HassTurnOn") would not match by name. + excluded_apis = { + api_id + for api_id, names in excluded_names.items() + if any(base_tool_name(n) == _TURN_ON_TOOL for n in names) + } fetched = await _get_tool_by_name( - store, config, _TURN_ON_TOOL, allowed_api_ids, live_tool_ids + store, config, _TURN_ON_TOOL, allowed_api_ids - excluded_apis, live_tool_ids ) if fetched is None: return candidates - excluded_names = tool_exclusions(config.get("configurable", {}).get("options", {})) - if is_tool_excluded(fetched["name"], excluded_names.get(fetched["api_id"], ())): - return candidates return [*candidates, fetched] @@ -2864,13 +2886,19 @@ async def _retrieve_tools( # noqa: PLR0915 all_candidates, store, config, live_tool_ids ) - # 3i. Bind HassTurnOn on an actuation request ranking left it out (#795). - # Before 3h, which then binds HassTurnOff with it. + # 3h. Bind HassTurnOn on an actuation request ranking left it out (#795), + # including a "Yes" to the agent's offer. Before 3i, which then binds + # HassTurnOff with it. all_candidates = await _append_turn_on_tool( - all_candidates, store, config, query, allowed_api_ids, live_tool_ids + all_candidates, + store, + config, + state["messages"], + allowed_api_ids, + live_tool_ids, ) - # 3h. Bind HassTurnOn and HassTurnOff together. After 3e, so an + # 3i. Bind HassTurnOn and HassTurnOff together. After 3e, so an # always-included one gets its partner too; it dedupes by (api, name). all_candidates = await _append_on_off_partners( all_candidates, store, config, live_tool_ids diff --git a/tests/custom_components/home_generative_agent/test_tool_retrieval.py b/tests/custom_components/home_generative_agent/test_tool_retrieval.py index 1cbe0c6d..b9946c43 100644 --- a/tests/custom_components/home_generative_agent/test_tool_retrieval.py +++ b/tests/custom_components/home_generative_agent/test_tool_retrieval.py @@ -3768,7 +3768,9 @@ def _cover_field_store() -> MagicMock: return store -def _cover_field_config(*, excluded: list[str] | None = None) -> RunnableConfig: +def _cover_field_config( + *, excluded: list[str] | None = None, live: bool = True +) -> RunnableConfig: options: dict[str, Any] = { "llm_hass_api": ["assist"], "tool_relevance_threshold": 0.15, @@ -3776,17 +3778,14 @@ def _cover_field_config(*, excluded: list[str] | None = None) -> RunnableConfig: } if excluded: options["tool_exclusions"] = {"assist": excluded} - return cast( - "RunnableConfig", - { - "configurable": { - "options": options, - "tool_index_ready": True, - "langchain_tools": {}, - "ha_llm_api": _live_llm_api(*_COVER_FIELD_TOOLS), - } - }, - ) + configurable: dict[str, Any] = { + "options": options, + "tool_index_ready": True, + "langchain_tools": {}, + } + if live: + configurable["ha_llm_api"] = _live_llm_api(*_COVER_FIELD_TOOLS) + return cast("RunnableConfig", {"configurable": configurable}) @pytest.mark.asyncio @@ -3827,3 +3826,42 @@ async def test_an_excluded_turn_on_is_not_bound() -> None: store=_cover_field_store(), ) assert "intent__HassTurnOn" not in result["tool_routing_map"] + + +@pytest.mark.asyncio +async def test_yes_to_an_open_offer_offers_turn_on() -> None: + """A "Yes" to the agent's offer carries no action word of its own (#795).""" + state = _query_state("Yes") + state["messages"] = [ + HumanMessage(content="Is the study shade open?"), + AIMessage(content="It is closed. Want me to open the Study Shade?"), + HumanMessage(content="Yes"), + ] + result = await _retrieve_tools( + state, _cover_field_config(), store=_cover_field_store() + ) + assert "intent__HassTurnOn" in result["tool_routing_map"] + + +@pytest.mark.asyncio +async def test_a_namespaced_exclusion_blocks_the_bare_row_with_live_filter_off() -> ( + None +): + """ + With the live filter off, the bare lookup finds the pre-2026.9 row. + + "intent__HassTurnOn" never matches the name "HassTurnOn", so the exclusion + is checked by base name before the fetch. + """ + config = _cover_field_config(excluded=["intent__HassTurnOn"], live=False) + store = _cover_field_store() + stale = _indexed_item("HassTurnOn") + store.aget = AsyncMock( + side_effect=lambda _ns, key: ( + stale if key == tool_index_key("assist", "HassTurnOn") else None + ) + ) + result = await _retrieve_tools( + _query_state("Open the study shade"), config, store=store + ) + assert not any(name.endswith("HassTurnOn") for name in result["tool_routing_map"])