Add chatdsg_repair example + constraint-aware prompts + pddl_repair_tool

examples/chatdsg/agent_config.yaml

@@ -18,7 +18,7 @@ agent_info:
           uri: neo4j://$ADT4_HERACLES_IP:$ADT4_HERACLES_PORT
     - name: send_multirobot_pddl_goal
       bound_args:
-        planner_topic: /hilbert/omniplanner_node/multirobot_region_rearrange_objects_pddl/pddl_goal
+        planner_topic: /${ADT4_ROBOT_NAME}/omniplanner_node/multirobot_region_rearrange_objects_pddl/pddl_goal
     - name: visualize_objects
       bound_args:
         viz_topic: /viz_objects_marker_array

examples/chatdsg_repair/agent_config.yaml

@@ -0,0 +1,24 @@
+---
+client:
+  client_type: openai
+    # auth key is supplied by HERACLES_OPENAI_API_KEY
+  timeout: 20
+model_info:
+  model: gpt-4.1
+  temperature: 0.2
+  seed: 123
+agent_info:
+  prompt_settings:
+    base_prompt: agent_prompt.yaml
+  tools:
+    - name: run_cypher_query
+      bound_args:
+        dsgdb_conf:
+          dsg_interface_type: heracles
+          uri: neo4j://$ADT4_HERACLES_IP:7683
+    - name: send_pddl_goal_with_constraints
+      bound_args:
+        robot_name: hilbert
+        planner_topic: /hilbert/commanded_goal
+  tool_interface: openai
+  max_iterations: 6

examples/chatdsg_repair/agent_prompt.yaml

@@ -0,0 +1,30 @@
+---
+system: |
+  You are a helpful assistant that controls a robot by sending PDDL goals and runtime constraints.
+  You have access to a Neo4j database representing a 3D scene graph of the robot's environment.
+  Your workflow:
+  1. If the user refers to a specific place/object by ID (e.g., "go to place 23778"), send the goal directly.
+  2. If the user refers to something by name (e.g., "go to the tree"), first query the scene graph with run_cypher_query to find the ID, then send the goal.
+  3. If the user wants to avoid a location, add constraints using the constraints_json parameter.
+  4. You can interrupt the robot's current plan by sending a new goal. The system decides automatically whether to keep executing the current plan (when the new goal is already covered) or to replan from the current robot state.
+  IMPORTANT RULES:
+  - All IDs in PDDL goals must be LOWERCASE: p23778, o188 (not P23778, O188)
+  - Places use prefix "p", objects use prefix "o"
+  - Use run_cypher_query to look up IDs when the user refers to objects by name/class
+  - Ask the user for confirmation before sending robot commands
+  - Constraints in constraints_json apply to the specific goal you are sending. When the user adds a new constraint, include any previously-stated constraints they still want active. To clear, send a goal with constraints_json="" (or omitted).
+  - CRITICAL: The robot planner only knows about MeshPlace nodes (labeled "MeshPlace") and Object nodes. It does NOT know about Place nodes (layer 3). When looking up places to visit, ALWAYS query MeshPlace, NEVER Place. Objects (labeled "Object") are always valid.
+  - MeshPlace nodeSymbols are uppercase in the database (e.g., "P1734") but must be sent LOWERCASE in PDDL goals (e.g., "p1734").
+  - If the user asks for a random or nearby point, query MeshPlace nodes.
+  - Example Cypher for MeshPlaces: MATCH (p:MeshPlace) RETURN p.nodeSymbol LIMIT 10
+  When using run_cypher_query, remember:
+  - Objects are labeled "Object" with properties: nodeSymbol, class, center
+  - MeshPlaces are labeled "MeshPlace" with properties: nodeSymbol, center
+  - Rooms are labeled "Room" with properties: nodeSymbol, class
+  - Use lowercase node symbols in PDDL goals (the database may return uppercase like P5041 — convert to p5041)
+scene_graph_description: ${HERACLES_EVALUATION_PATH}/examples/prompts/common/scene_graph_description.yaml
+labelspace_description: ${HERACLES_EVALUATION_PATH}/examples/prompts/common/building_45_labelspace.yaml
+interface_description: ${HERACLES_EVALUATION_PATH}/examples/prompts/common/cypher_interface_description.yaml
+domain_description: ${HERACLES_EVALUATION_PATH}/examples/prompts/common/pddl_domain_description_with_constraints.yaml
+in_context_examples: ${HERACLES_EVALUATION_PATH}/examples/prompts/common/pddl_in_context_examples_with_constraints.yaml
+novel_instruction_template: '{question}'

examples/chatdsg_repair/chatdsg.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""ChatDSG with plan repair tool support."""
+
+# Register the repair tool before the agent loads
+# Import the app class from the original chatdsg
+import importlib.util
+import os
+
+import yaml
+
+import heracles_agents.tools.pddl_repair_tool  # noqa: F401
+from heracles_agents.llm_agent import LlmAgent
+
+original = os.path.join(os.path.dirname(__file__), "..", "chatdsg", "chatdsg.py")
+spec = importlib.util.spec_from_file_location("chatdsg_original", original)
+mod = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mod)
+
+if __name__ == "__main__":
+    with open("agent_config.yaml", "r") as fo:
+        yml = yaml.safe_load(fo)
+    agent = LlmAgent(**yml)
+    app = mod.InputDisplayApp(agent)
+    app.run()

examples/prompts/common/building_45_labelspace.yaml

@@ -1,55 +1,29 @@
 labelspace_description: |
   <Labelspace Description>
-  These are the labels available in the scene graph at each layer. Synonyms should be mapped to a label in the the list.
+  These are subset of labels in a scene graph. The full list should be quiered (if it is not in your context), before looking up objects/locations. 
+  Cypher Query: (MATCH (o:Object) RETURN DISTINCT o.class AS object_type). Note similar queries can be used for rooms.
+  Synonyms should be mapped to existing labels in the graph.
   <object_labels>
     tree
-    vehicle
-    signal
-    rock
-    fence
-    boat
-    sign
-    door
-    pole
-    rail
-    window
-    flower
-    bed
+    car
+    van
+    truck
+    cone
+    hospital
+    government_center
+    barracks
+    warehouse
     box
-    storage
-    barrel
-    bag
-    basket
-    seating
     flag
-    decor
-    light
-    appliance
-    trash
-    bicycle
-    food
-    clothes
+    window
+    door
+    sign
+    seating
+    pole
   </object_labels>
   <room_labels>
-    lounge
-    hallway
-  </room_labels>
-  <mesh_places_labels>
-    water
-    ground
-    grass
-    sand
-    sidewalk
-    dock
+    parking lot
+    intersection
     road
-    path
-    hill
-    bridge
-    wall
-    floor
-    stairs
-    structure
-    surface
-    flora
-  </mesh_places_labels>
+  </room_labels>
   </Labelspace Description>

examples/prompts/common/cypher_interface_description.yaml

@@ -1,26 +1,22 @@
 interface_description: |
   Labels in Database:
       - Object: a node representing an object in the world.
-          - nodeSymbol: a unique string identifier
+          - nodeSymbol: a unique string identifier ('O(X)')
           - class: a string identifying the object's semantic class or type
           - center: the 3D position of the object, as a POINT type
       - MeshPlace: a node representing a 2D segment of space the robot might be able to move to.
-          - nodeSymbol: a unique string identifier
-          - class: a string identifying the place's semantic class or type
+          - nodeSymbol: a unique string identifier ('t(X)')
           - center: the 3D position of the mesh place, as a POINT type
-      - Place: a node representing a 3D region of free space
-          - nodeSymbol: a unique string identifier
-          - center: the 3D position of the place, as a POINT type
       - Room: a node representing a room or higher-level region
-          - nodeSymbol: a unique string identifier
+          - nodeSymbol: a unique string identifier ('R(X)')
           - class: a string identifying the room's semantic class or type
           - center: the 3D position of the room, as a POINT type
 
-  Object, MeshPlace, Place, and Room are all Cypher labels attached to nodes.
+  Object, MeshPlace, and Room are all Cypher labels attached to nodes.
 
-  Places and Mesh Places represent a higher level of the hierarchy compared to objects, but lower level than rooms.
+  Mesh Places represent a higher level of the hierarchy compared to Objects, but lower level than Rooms.
 
-  There are two kinds of existing edges. First is (a)-[:CONTAINS]->(b), which connects nodes between different layers and means that b is contained within a. Nodes in higher levels of the hierarchy may contain nodes in lower levels of the hierarchy, but nodes in the lower level of the hierarchy will not contain higher-level nodes. The other kind of edges represent connectivity within a layer: [:OBJECT_CONNECTED], [:PLACE_CONNECTED], [:MESH_PLACE_CONNECTED], [:ROOM_CONNECTED].
-  Remember that (a)-[:CONTAINS*]->(b) will match transitive relationships.
+  There are two kinds of existing edges. First is (a)-[:CONTAINS]->(b), which connects nodes between different layers and means that b is contained within a. Nodes in higher levels of the hierarchy may contain nodes in lower levels of the hierarchy, but nodes in the lower level of the hierarchy will not contain higher-level nodes. The other kind of edges represent connectivity within a layer: [:OBJECT_CONNECTED], [:MESH_PLACE_CONNECTED], [:ROOM_CONNECTED].
+  Remember that (a)-[:CONTAINS*]->(b) will match transitive relationships. To check if an object is in a room you should use a transitive containment operation.
 
   Note that in the current version of cypher, `distance` has been replaced by `point.distance`. Also, do not use any apoc functions in your queries.

examples/prompts/common/pddl_domain_description.yaml

@@ -1,21 +1,21 @@
 domain_description: |
   <PDDL Domain>
       The PDDL domain consists of the following predicates described below. These predicates get parameterized by symbols from a 3D scene graph.
-      (visited-place ?P): This predicate indicates that a robot must visit Place '?P' at some point, where '?P' is a placeholder for a Place ID.
-      (at-place ?P): This predicate indicates that a robot must be at Place '?P', where '?P' is a placeholder for a Place ID.
+      (visited-place ?t): This predicate indicates that a robot must visit MeshPlace '?t' at some point, where '?t' is a placeholder for a MeshPlace ID.
+      (at-place ?t): This predicate indicates that a robot must be at MeshPlace '?t', where '?t' is a placeholder for a MeshPlace ID.
       (visited-object ?O): This predicate indicates that a robot must visit Object '?O' at some point, where '?O' is a placeholder for an Object ID.
-      (at-object ?O): This predicate indicates that a robot must be at Object '?O', where '?O' is a placeholder for an Object ID.
+      (at-object ?O): This predicate indicates that a robot must be at Object '?O', where '?O' is a placeholder for an Object ID. 
       (safe ?O): This predicate indicates that a robot must inspect Object '?O', where '?O' is a placeholder for an Object ID.
       (visited-room ?R): This predicate indicates that a robot must visit Room '?R' at some point, where '?R' is a placeholder for a Room ID.
       (in-room ?R): This predicate indicates that a robot must be at Room '?R', where '?R' is a placeholder for a Room ID.
       The 'at' and 'visited' predicates are useful for specifying locations for robots to go.
-      When an instruction indicates an order, you should use (at-place ?P), (at-object ?O), or (in-room ?R) to specify the final goal and (visited-place ?P), (visited-object ?O), or (visited-room ?R) to specify the intermediate goals.
+      When an instruction indicates an order, you should use (at-place ?t), (at-object ?O), or (in-room ?R) to specify the final goal and (visited-place ?t), (visited-object ?O), or (visited-room ?R) to specify the intermediate goals.
 
       (holding ?O): This predicate indicates the a robot must be holding an Object '?O', where '?O' is a placeholder for an Object ID.
       The 'holding' predicate is useful for specifying that a robot should pick up an object.
 
-      (object-in-place ?O ?P): This predicate indicates that an Object '?O' must be located inside a Place '?P', where '?O' is a placeholder for an Object ID and '?P' is a placeholder for a Place ID.
-      The 'object-in-place' predicate is useful for specifying that a robot should place an object somewhere. If the robot should move an object from one place to another, you should use the 'object-in-place' predicate for the goal instead of the 'holding' predicate.
+      (object-in-place ?O ?t): This predicate indicates that an Object '?O' must be located inside a MeshPlace'?t', where '?O' is a placeholder for an Object ID and '?t' is a placeholder for a MeshPlaceID.
+      The 'object-in-place' predicate is useful for specifying that a robot should place an object somewhere. If the robot should move an object from one MeshPlaceto another, you should use the 'object-in-place' predicate for the goal instead of the 'holding' predicate.
 
       You can compose PDDL goal predicates into more complex goals using the following operators:
       not: the 'not' operator negates the truth value of the predicate. For example '(not (visited-room R1))' means Room 'R1' should not be visited.

examples/prompts/common/pddl_domain_description_with_constraints.yaml

@@ -0,0 +1,49 @@
+domain_description: |
+  <PDDL Domain>
+      The PDDL domain consists of the following predicates. These predicates get parameterized by symbols from a 3D scene graph.
+      IMPORTANT: All IDs in PDDL goals and constraints must be LOWERCASE. Places use prefix "p" (e.g., p23778), objects use prefix "o" (e.g., o188).
+
+      (visited-place ?p): Robot must visit place ?p at some point. Example: (visited-place p23778)
+      (visited-object ?o): Robot must visit object ?o at some point. Example: (visited-object o188)
+      (at-place ?p): Robot must end at place ?p.
+      (at-object ?o): Robot must end at object ?o.
+      (safe ?o): Robot must inspect object ?o. Example: (safe o188)
+
+      When an instruction indicates an order, use "at-" predicates for the final destination and "visited-" predicates for intermediate stops.
+
+      You can compose goals using:
+      and: multiple predicates must be true. Example: (and (visited-place p23778)(visited-object o188))
+      or: at least one must be true. Example: (or (visited-place p23778)(visited-place p34383))
+      not: negates a predicate. Example: (not (visited-object o188))
+
+      If the user is not specific about which object or place, use "or" to let the planner decide.
+
+      To find the correct place or object IDs, use the run_cypher_query tool to query the scene graph database.
+      Example Cypher: MATCH (o:Object) WHERE o.class = 'tree' RETURN o.nodeSymbol
+      Example Cypher: MATCH (p:MeshPlace) RETURN p.nodeSymbol LIMIT 10
+  </PDDL Domain>
+
+  <Runtime Constraints>
+      In addition to goals, you can specify runtime constraints to forbid the robot from visiting certain locations or edges.
+      Constraints are passed as a JSON list via the constraints_json parameter to send_pddl_goal_with_constraints.
+
+      Constraint types:
+      - forbidden-poi: forbids the robot from visiting (or transiting through) a specific place or object.
+        Example: [["forbidden-poi", "o188"]] forbids object o188.
+        Example: [["forbidden-poi", "p23778"]] forbids place p23778.
+      - forbidden-edge: forbids the robot from traversing the edge between two points.
+        Example: [["forbidden-edge", "p1234", "p5678"]] blocks the corridor between p1234 and p5678.
+        NOTE: forbidden-edge is unordered. You only need ONE direction, not both.
+
+      ALL constraint IDs must be LOWERCASE (e.g., "o188" not "O188", "p1234" not "P1234").
+
+      When the user says:
+      - "avoid object X" or "don't go near place Y" -> use forbidden-poi
+      - "that path is blocked" or "the corridor between A and B is closed" -> use forbidden-edge (a single entry)
+
+      Constraints scoped to a single goal: pass them in constraints_json alongside the goal.
+      The user does not need to re-state existing constraints when sending a new goal.
+
+      You can also send a goal with no constraints — just pass an empty string for constraints_json
+      (or omit the parameter entirely).
+  </Runtime Constraints>

examples/prompts/common/pddl_in_context_examples_with_constraints.yaml

@@ -0,0 +1,37 @@
+in_context_examples:
+  # Simple visit place
+  - user: |
+      Instruction: Go to place 5041.
+    assistant: "I'll send the robot to place p5041.\n\nUsing send_pddl_goal_with_constraints with pddl_goal_string=(visited-place p5041)"
+  # Simple visit object
+  - user: |
+      Instruction: Go to object 188.
+    assistant: "I'll send the robot to object o188.\n\nUsing send_pddl_goal_with_constraints with pddl_goal_string=(visited-object o188)"
+  # Visit place and object together
+  - user: |
+      Instruction: Visit place 23778 and object 188.
+    assistant: "I'll send the robot to visit both place p23778 and object o188.\n\nUsing send_pddl_goal_with_constraints with pddl_goal_string=(and (visited-place p23778)(visited-object o188))"
+  # Visit multiple places
+  - user: |
+      Instruction: Go to place 23778 and place 34383.
+    assistant: "I'll have the robot visit both places.\n\nUsing send_pddl_goal_with_constraints with pddl_goal_string=(and (visited-place p23778)(visited-place p34383))"
+  # Goal with forbidden-poi constraint
+  - user: |
+      Instruction: Go to place 23778 but avoid object 187.
+    assistant: "I'll route the robot to place p23778 while avoiding object o187.\n\nUsing send_pddl_goal_with_constraints with pddl_goal_string=(visited-place p23778) and constraints_json=[[\"forbidden-poi\", \"o187\"]]"
+  # Goal with multiple constraints
+  - user: |
+      Instruction: Visit place 23778 but stay away from object 188 and object 130.
+    assistant: "I'll send the robot to p23778 while avoiding objects o188 and o130.\n\nUsing send_pddl_goal_with_constraints with pddl_goal_string=(visited-place p23778) and constraints_json=[[\"forbidden-poi\", \"o188\"], [\"forbidden-poi\", \"o130\"]]"
+  # Goal with forbidden-edge constraint (unordered — one direction is enough)
+  - user: |
+      Instruction: Go to place 23778 but the path between place 1234 and place 5678 is blocked.
+    assistant: "I'll route the robot to p23778 while avoiding the blocked edge.\n\nUsing send_pddl_goal_with_constraints with pddl_goal_string=(visited-place p23778) and constraints_json=[[\"forbidden-edge\", \"p1234\", \"p5678\"]]"
+  # Interrupt with new goal
+  - user: |
+      Instruction: Change the plan. Go to place 34383 instead.
+    assistant: "I'll interrupt the current plan and send the robot to p34383.\n\nUsing send_pddl_goal_with_constraints with pddl_goal_string=(visited-place p34383)"
+  # Natural language with scene graph query needed
+  - user: |
+      Instruction: Go to the nearest tree.
+    assistant: "I need to find tree objects in the scene graph first. Let me query for them.\n\nUsing run_cypher_query to find trees, then I'll send the goal with the correct object ID."