Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/reference/cli.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ echo "List the Python files here" | gaia -p --output-format json
| `--output-format text\|json` | `text` (default) prints the answer alone; `json` prints one JSON object (below) |
| `--verbose` | With `--output-format json`: add the turn's full `conversation` (every model message and tool call) |
| `--max-steps <n>` | Cap on agent steps for this run, at least 1. Default: the agent's own limit ([Agent step limit](#agent-step-limit)) |
| `--time-limit <seconds>` | Wall-clock seconds the turn may take, at least 1. At 70% the agent is told to write what was asked for. Near the limit (at 90%, or earlier when its steps are slow) it stops working, writes any requested file still missing, and answers. A turn still busy at the limit is stopped there: a running shell or Python command ends, a model call already sent is waited for, and the answer says the time limit stopped the turn, followed by the agent's latest findings. A turn the limit ended reports `status: incomplete` and exits 1, even when it wrote its file and answered. Counted from the start of the turn, after server start-up, so give a wrapper's own timeout some headroom above it. Can't be combined with `--attempts` yet (a usage error). Default: no limit |
| `--time-limit <seconds>` | Wall-clock seconds the turn may take, at least 1. At 70% the agent is told to write what was asked for. Near the limit (at 90%, or earlier when its steps are slow) it stops working, writes any requested file still missing, and answers. A shell or Python command, a wait or a sleep the agent starts gets a timeout no longer than the time left before that stop, and one with no time left is refused. A turn still busy at the limit is stopped there: a running shell or Python command ends, a model call already sent is waited for, and the answer says the time limit stopped the turn, followed by the agent's latest findings. A turn the limit ended reports `status: incomplete` and exits 1, even when it wrote its file and answered. Counted from the start of the turn, after server start-up, so give a wrapper's own timeout some headroom above it. Can't be combined with `--attempts` yet (a usage error). Default: no limit |
| `--attempts <n>` | Run `n` (up to 4) independent attempts, then one turn that checks and combines them ([below](#independent-attempts)). Default: 1 |
| `--dangerously-skip-permissions` | Run confirmation-gated tools (shell commands, file writes) without asking. A request that only hands over quoted text to translate, reword or proofread still has them refused |
| `--full-access` | Full access, as the TUI's `/full-access`: lifts the shell guardrails, so test runners, build tools and other commands outside the no-prompt list run, and grants paths outside the current folder without asking. Implies `--dangerously-skip-permissions` |
Expand Down
28 changes: 25 additions & 3 deletions hub/agents/gaia/python/gaia_agent/chat_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,12 @@
thin_tool_schema,
)
from gaia.agents.base.tools import _TOOL_REGISTRY
from gaia.agents.base.verification import NOT_EXECUTED
from gaia.agents.base.verification import (
NOT_EXECUTED,
OUT_OF_TIME,
TIME_LIMIT_CUT_RUN,
cut_by_time_limit,
)
from gaia.agents.install_hints import missing_package_hint
from gaia.agents.registry import get_embedding_model_for_device
from gaia.agents.tools import FileSystemToolsMixin # Enhanced file system navigation
Expand Down Expand Up @@ -409,6 +414,17 @@ def _missing_library_hint(stderr_line: str) -> Optional[str]:
)


def _stopped_run_result(agent: Any, job: Any, timeout: int) -> Dict[str, Any]:
"""A killed job's result; one the time limit cut short is not told to ask for longer."""
result = stopped_job_result(job, timeout)
if job.stopped == "timeout" and cut_by_time_limit(agent):
result["error"] = (
f"Timed out after {timeout}s; the script and every process it "
f"started were stopped. {TIME_LIMIT_CUT_RUN}"
)
return result


def _python_run_result(
run: Any,
stdout: str,
Expand Down Expand Up @@ -2729,6 +2745,9 @@ def execute_python_file(
timeout = resolve_python_timeout(timeout)
except ValueError as e:
return {"status": "error", "error": str(e), "has_errors": True}
timeout = self.time_capped_run_seconds(timeout)
if not timeout:
return dict(OUT_OF_TIME)

file_path = str(
_resolve_target(file_path, None, _workspace_default(self))
Expand Down Expand Up @@ -2819,7 +2838,7 @@ def execute_python_file(
may_write=bool(written) or not complete,
)
if job.stopped:
return stopped_job_result(job, timeout)
return _stopped_run_result(self, job, timeout)
r = job.run
from gaia.agents.base.artifacts import retain_excerpt

Expand Down Expand Up @@ -2879,6 +2898,9 @@ def run_python(code: str, timeout: int = DEFAULT_PYTHON_TIMEOUT) -> dict:
timeout = resolve_python_timeout(timeout)
except ValueError as e:
return {"status": "error", "error": str(e), "has_errors": True}
timeout = self.time_capped_run_seconds(timeout)
if not timeout:
return dict(OUT_OF_TIME)

tool_names = set(_TOOL_REGISTRY) | set(
getattr(self, "_tools_registry", None) or ()
Expand Down Expand Up @@ -3037,7 +3059,7 @@ def run_python(code: str, timeout: int = DEFAULT_PYTHON_TIMEOUT) -> dict:
may_write=bool(written) or not complete,
)
if job.stopped:
return stopped_job_result(job, timeout)
return _stopped_run_result(self, job, timeout)
r = job.run
except OSError as e:
logger.error("run_python could not start in %s: %s", run_dir, e)
Expand Down
36 changes: 30 additions & 6 deletions src/gaia/agents/base/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -952,6 +952,8 @@ def _loaded_context_is_too_small(loaded_ctx: Any, requested_ctx: Any) -> bool:
)
_TIME_NOTICE_AT = 0.7
_TIME_STOP_AT = 0.9
# Shortest run worth starting when a time limit cuts a tool's timeout.
_MIN_CAPPED_RUN_S = 5

# How much of the step budget must be gone before each notice is sent, and
# what it says. Until these existed the only thing the model ever heard about
Expand Down Expand Up @@ -12026,20 +12028,42 @@ def _announce_step_budget(
return notice
return None

def _time_limit_passed(self) -> bool:
"""True once the turn is past its time limit's stop point.
def time_capped_run_seconds(self, seconds: int) -> int:
"""*seconds*, cut so a run a tool starts ends before the turn's stop point.

A command left at its class default can outlast the whole turn; this
keeps the time to write up what was found. Unchanged with no time
limit. 0 when under ``_MIN_CAPPED_RUN_S`` is left: start nothing.
"""
left = self._time_to_stop_s()
if left is None:
return seconds
if left < _MIN_CAPPED_RUN_S:
return 0
return min(seconds, int(left))

def _time_to_stop_s(self) -> Optional[float]:
"""Seconds until the turn's stop point, or None with no time limit.

The stop point leaves room for the write-only step and the closing
answer: a tenth of the limit, or two of the turn's slowest steps so
far when that is more. It is also why a stop the caller's own timer
forces at the limit counts as reaching it.
far when that is more.
"""
limit = getattr(self, "time_limit_s", None)
started = getattr(self, "_turn_started_at", None)
if not limit or limit <= 0 or started is None:
return False
return None
reserve = max(limit * (1 - _TIME_STOP_AT), 2 * self._slowest_step_s)
return time.monotonic() - started >= limit - reserve
return limit - reserve - (time.monotonic() - started)

def _time_limit_passed(self) -> bool:
"""True once the turn is past its time limit's stop point.

See ``_time_to_stop_s`` for where that is. It is also why a stop the
caller's own timer forces at the limit counts as reaching it.
"""
left = self._time_to_stop_s()
return left is not None and left <= 0

def _time_is_up(self, messages: List[Dict], conversation: List[Dict]) -> bool:
"""Warn once near the turn's time limit; True once it is time to stop.
Expand Down
34 changes: 34 additions & 0 deletions src/gaia/agents/base/verification.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,40 @@
#: Spread into a tool's pre-execution refusal: ``{**NOT_EXECUTED, "status": …}``.
NOT_EXECUTED: Dict[str, Any] = {EXECUTED_KEY: False}

#: What a tool returns instead of starting a run the turn's time limit has no
#: room for.
OUT_OF_TIME: Dict[str, Any] = {
**NOT_EXECUTED,
"status": "error",
"error": (
"Nothing ran: this turn's time limit is nearly up. Write the result "
"you have now instead of starting another run."
),
"has_errors": True,
"out_of_time": True,
}

#: Replaces "raise the timeout" advice on a run the time limit cut short.
TIME_LIMIT_CUT_RUN = (
"The turn's time limit cut this run short, so a longer timeout is not "
"available. Write the result you have now."
)


def time_capped(host: Any, seconds: int) -> int:
"""*seconds* cut to the time *host*'s turn has left; 0 means start nothing.

A host that is not an Agent has no turn and no time limit.
"""
cap = getattr(host, "time_capped_run_seconds", None)
return cap(seconds) if callable(cap) else seconds


def cut_by_time_limit(host: Any) -> bool:
"""True when a run that just timed out was stopped by the turn's time limit."""
return time_capped(host, 1) == 0


#: On a result that looked on disk and found the path missing: nothing ran,
#: but the absence was checked, which is what a "doesn't exist" claim needs.
CHECKED_ABSENT_KEY = "checked_absent"
Expand Down
20 changes: 18 additions & 2 deletions src/gaia/agents/tools/shell_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,14 @@
)
from gaia.agents.base.errors import exit_error
from gaia.agents.base.file_snapshots import SnapshotError, before_change, note_untracked
from gaia.agents.base.verification import EXECUTED_KEY, NOT_EXECUTED
from gaia.agents.base.verification import (
EXECUTED_KEY,
NOT_EXECUTED,
OUT_OF_TIME,
TIME_LIMIT_CUT_RUN,
cut_by_time_limit,
time_capped,
)
from gaia.agents.tools.command_timeouts import (
MAX_COMMAND_TIMEOUT,
TIMEOUT_CLASSES,
Expand Down Expand Up @@ -7392,6 +7399,8 @@ def _timeout_hint(self, timeout_class: str, cwd: str) -> str:
class default usually blows a larger one too. A build or download is
pointed at a background run instead.
"""
if cut_by_time_limit(self):
return TIME_LIMIT_CUT_RUN
cls = TIMEOUT_CLASSES[timeout_class]
hint = (
f"It ran as a '{timeout_class}' command ({cls.summary}, "
Expand Down Expand Up @@ -8178,7 +8187,6 @@ def run_shell_command(
"command": command,
"has_errors": True,
}

# Check rate limits first to prevent DOS
allowed, reason, wait_time, waited = (
(True, "", 0.0, 0.0) if bypass else self._pace_rate_limit()
Expand All @@ -8193,6 +8201,10 @@ def run_shell_command(
"wait_time_seconds": wait_time,
"hint": "Rate limiting prevents excessive command execution",
}
# After the pacing wait, which is time the turn has also spent.
timeout = time_capped(self, timeout)
if not timeout:
return {**OUT_OF_TIME, "command": command}

# Validate an explicit directory after resolving it from the
# chat workspace; keep the original argument for approval keys.
Expand Down Expand Up @@ -8701,6 +8713,10 @@ def wait_for_condition(
"has_errors": True,
}

timeout = time_capped(self, timeout)
if not timeout:
return dict(OUT_OF_TIME)

# The wait is charged as one command up front; its probes are then
# exempt (see _check_rate_limit) rather than each tripping the limit.
allowed, reason, wait_time = self._check_rate_limit()
Expand Down
6 changes: 6 additions & 0 deletions src/gaia/agents/tools/wait_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from datetime import datetime
from typing import Any, Dict

from gaia.agents.base.verification import OUT_OF_TIME, time_capped
from gaia.logger import get_logger
from gaia.tool_cancellation import tool_cancelled

Expand Down Expand Up @@ -84,6 +85,11 @@ def _sleep(self, seconds: float, reason: str) -> Dict[str, Any]:
),
}

allowed = time_capped(self, math.ceil(seconds))
if not allowed:
return dict(OUT_OF_TIME)
seconds = min(seconds, allowed)

logger.info("Waiting %gs: %s", seconds, reason or "(no reason given)")
start = time.monotonic()
deadline = start + seconds
Expand Down
135 changes: 135 additions & 0 deletions tests/unit/agents/test_turn_time_limit.py
Original file line number Diff line number Diff line change
Expand Up @@ -371,3 +371,138 @@ def test_the_limit_is_measured_per_turn(agent, clock):

assert result["time_limit_reached"] is False
assert _notices(chat) == []


class _Timed:
"""The agent attributes the run-time cap reads, with a clock the test sets."""

time_capped_run_seconds = Agent.time_capped_run_seconds
_time_to_stop_s = Agent._time_to_stop_s

def __init__(self, limit, used, slowest=0.0):
self.time_limit_s = limit
self._turn_started_at = 1000.0 - used if used is not None else None
self._slowest_step_s = slowest


@pytest.mark.parametrize(
"limit, used, asked, expected",
[
(None, 0, 600, 600), # no limit: untouched
(720, None, 600, 600), # no turn running: untouched
(720, 100, 600, 548), # 720 * 0.9 - 100 left before the stop point
(720, 100, 30, 30), # a short run already fits
(720, 645, 600, 0), # under 5 s left: start nothing
(720, 700, 600, 0), # past the stop point
],
)
def test_a_run_is_cut_to_end_before_the_stop_point(limit, used, asked, expected):
clock = _Clock()
with patch.object(agent_module, "time", clock):
assert _Timed(limit, used).time_capped_run_seconds(asked) == expected


def test_the_shell_caps_a_default_timeout_and_refuses_with_no_time_left():
from gaia.agents.base.verification import OUT_OF_TIME
from gaia.agents.tools.shell_tools import ShellToolsMixin

class _Host(ShellToolsMixin):
time_capped_run_seconds = Agent.time_capped_run_seconds
_time_to_stop_s = Agent._time_to_stop_s
_slowest_step_s = 0.0

def __init__(self, limit, used):
self.time_limit_s = limit
self._turn_started_at = 1000.0 - used
self.applied = None

def full_access_active(self):
return True

def _run_full_access(self, command, cwd, timeout, timeout_class):
self.applied = timeout
return {"status": "success", "timeout": timeout}

def run(host):
host.register_shell_tools()
return _TOOL_REGISTRY["run_shell_command"]["function"]("python train.py")

saved = dict(_TOOL_REGISTRY)
clock = _Clock()
try:
with patch.object(agent_module, "time", clock):
capped = _Host(720, 100)
assert run(capped)["timeout"] == 548
spent = _Host(720, 700)
refused = run(spent)
finally:
_TOOL_REGISTRY.clear()
_TOOL_REGISTRY.update(saved)
assert spent.applied is None
assert refused == {**OUT_OF_TIME, "command": "python train.py"}
assert refused["executed"] is False


def test_the_cap_ends_at_the_same_stop_point_the_loop_uses():
"""Slow steps move the stop earlier; a run capped to 90% would overrun it."""
clock = _Clock()
with patch.object(agent_module, "time", clock):
# 600 s limit, slowest step 60 s: the stop is at 600 - 120 = 480 s.
assert _Timed(600, 300, slowest=60).time_capped_run_seconds(900) == 180
assert _Timed(600, 300).time_capped_run_seconds(900) == 240


def test_the_loop_stops_exactly_where_the_cap_ends(agent, clock):
agent.time_limit_s = 600
agent._turn_started_at = clock.now
agent._slowest_step_s = 60.0
clock.now += 479
assert agent._time_limit_passed() is False
assert agent.time_capped_run_seconds(900) == 0 # 1 s left: start nothing
clock.now += 1
assert agent._time_limit_passed() is True


def test_a_run_the_limit_cut_short_is_not_told_to_rescope_or_ask_for_longer():
from gaia.agents.base.verification import TIME_LIMIT_CUT_RUN
from gaia.agents.tools.shell_tools import ShellToolsMixin

class _Host(ShellToolsMixin, _Timed):
pass

clock = _Clock()
with patch.object(agent_module, "time", clock):
cut = _Host(720, 648)._timeout_hint("script", ".")
not_cut = _Host(720, 100)._timeout_hint("script", ".")
no_limit = _Host(None, 0)._timeout_hint("script", ".")
assert cut == TIME_LIMIT_CUT_RUN
assert "time limit" not in not_cut and "It ran as a 'script' command" in not_cut
assert no_limit == not_cut


def test_sleep_and_wait_for_condition_stop_short_of_the_limit():
from gaia.agents.base.verification import OUT_OF_TIME
from gaia.agents.tools.shell_tools import ShellToolsMixin
from gaia.agents.tools.wait_tools import WaitToolsMixin

class _Host(ShellToolsMixin, WaitToolsMixin, _Timed):
slept = None

def _require_stop_channels(self):
return None

def _wait_interrupted(self):
return False

saved = dict(_TOOL_REGISTRY)
clock = _Clock()
try:
with patch.object(agent_module, "time", clock):
spent = _Host(720, 646)
assert spent._sleep(120, "rate limit") == OUT_OF_TIME
spent.register_shell_tools()
waited = _TOOL_REGISTRY["wait_for_condition"]["function"]("true")
finally:
_TOOL_REGISTRY.clear()
_TOOL_REGISTRY.update(saved)
assert waited == OUT_OF_TIME
Loading
Loading