Skip to content

Commit 1c249d8

Browse files
wedamijaclaude
andauthored
feat(huey): Add an option to monitor Huey periodic tasks (#8052)
Part of a series adding automatic Crons monitoring for the most-used job schedulers in each SDK. Check-ins for a monitor that doesn't exist yet are dropped, so today each monitor has to be created by hand with its schedule copied from the code. Sending the job's schedule as the monitor config lets Sentry create the monitor and keep it in sync. This adds `monitor_periodic_tasks` to `HueyIntegration`, matching Celery's `monitor_beat_tasks`. When it's on, each `@huey.periodic_task` run sends check-ins with its `crontab(...)` schedule in UTC, so Sentry creates the monitor on the first run. The slug is the task name. It's off by default, and regular tasks are unaffected. ```python sentry_sdk.init( integrations=[HueyIntegration(monitor_periodic_tasks=True, exclude_periodic_tasks=["cleanup"])], ) ``` Tasks aren't monitored when they use a custom `validate_datetime`, restrict both `day` and `day_of_week`, or run on a Huey instance with `utc=False`. Docs: getsentry/sentry-docs#20057 --------- Co-authored-by: Claude <noreply@anthropic.com>
1 parent 4e19592 commit 1c249d8

2 files changed

Lines changed: 330 additions & 3 deletions

File tree

‎sentry_sdk/integrations/huey.py‎

Lines changed: 105 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import sys
22
from datetime import datetime
3+
from functools import wraps
34
from typing import TYPE_CHECKING
45

56
import sentry_sdk
67
from sentry_sdk.api import continue_trace, get_baggage, get_traceparent
78
from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS
9+
from sentry_sdk.crons import monitor
810
from sentry_sdk.integrations import DidNotEnable, Integration
911
from sentry_sdk.scope import should_send_default_pii
1012
from sentry_sdk.traces import SegmentNameSource, SpanStatus, StreamedSpan
@@ -21,19 +23,21 @@
2123
ensure_integration_enabled,
2224
event_from_exception,
2325
has_data_collection_enabled,
26+
logger,
27+
match_regex_list,
2428
reraise,
2529
)
2630

2731
if TYPE_CHECKING:
28-
from typing import Any, Callable, Optional, TypeVar, Union
32+
from typing import Any, Callable, List, Optional, TypeVar, Union
2933

30-
from sentry_sdk._types import Event, EventProcessor, Hint
34+
from sentry_sdk._types import Event, EventProcessor, Hint, MonitorConfig
3135
from sentry_sdk.utils import ExcInfo
3236

3337
F = TypeVar("F", bound=Callable[..., Any])
3438

3539
try:
36-
from huey.api import Huey, PeriodicTask, Result, ResultGroup, Task
40+
from huey.api import Huey, PeriodicTask, Result, ResultGroup, Task, crontab
3741
from huey.exceptions import CancelExecution, RetryTask, TaskLockedException
3842
except ImportError:
3943
raise DidNotEnable("Huey is not installed")
@@ -53,6 +57,14 @@ class HueyIntegration(Integration):
5357
identifier = "huey"
5458
origin = f"auto.queue.{identifier}"
5559

60+
def __init__(
61+
self,
62+
monitor_periodic_tasks: bool = False,
63+
exclude_periodic_tasks: "Optional[List[str]]" = None,
64+
) -> None:
65+
self.monitor_periodic_tasks = monitor_periodic_tasks
66+
self.exclude_periodic_tasks = exclude_periodic_tasks
67+
5668
@staticmethod
5769
def setup_once() -> None:
5870
patch_enqueue()
@@ -249,9 +261,99 @@ def _sentry_execute(
249261

250262
if not getattr(task, "_sentry_is_patched", False):
251263
task.execute = _wrap_task_execute(task.execute)
264+
with capture_internal_exceptions():
265+
task.execute = _monitor_periodic_task(self, task, task.execute)
252266
task._sentry_is_patched = True
253267

254268
with span_ctx:
255269
return old_execute(self, task, timestamp)
256270

257271
Huey._execute = _sentry_execute
272+
273+
274+
def _monitor_periodic_task(
275+
huey: "Huey", task: "Task", execute: "Callable[..., Any]"
276+
) -> "Callable[..., Any]":
277+
integration = sentry_sdk.get_client().get_integration(HueyIntegration)
278+
if (
279+
integration is None
280+
or not integration.monitor_periodic_tasks
281+
or not isinstance(task, PeriodicTask)
282+
or match_regex_list(task.name, integration.exclude_periodic_tasks)
283+
):
284+
return execute
285+
286+
monitor_config = _get_monitor_config(huey, task)
287+
if monitor_config is None:
288+
logger.debug(
289+
"Not monitoring Huey periodic task %r: its schedule can't be "
290+
"expressed as a Sentry Crons schedule.",
291+
task.name,
292+
)
293+
return execute
294+
295+
@wraps(execute)
296+
def _sentry_monitored_execute(*args: "Any", **kwargs: "Any") -> "Any":
297+
with monitor(monitor_slug=task.name, monitor_config=monitor_config):
298+
return execute(*args, **kwargs)
299+
300+
return _sentry_monitored_execute
301+
302+
303+
_CRONTAB_CODE = crontab().__code__
304+
305+
306+
def _get_closure_var(func: "Any", name: str) -> "Any":
307+
code = getattr(func, "__code__", None)
308+
closure = getattr(func, "__closure__", None)
309+
if code is None or closure is None or name not in code.co_freevars:
310+
return None
311+
return closure[code.co_freevars.index(name)].cell_contents
312+
313+
314+
def _get_crontab_field(values: "List[int]", first: int, last: int) -> str:
315+
full = list(range(first, last + 1))
316+
if values == full:
317+
return "*"
318+
if len(values) > 2:
319+
step = values[1] - values[0]
320+
if values == list(range(values[0], values[-1] + 1, step)):
321+
if step == 1:
322+
return "%d-%d" % (values[0], values[-1])
323+
if values == full[::step]:
324+
return "*/%d" % step
325+
return "%d-%d/%d" % (values[0], values[-1], step)
326+
return ",".join(str(v) for v in values)
327+
328+
329+
def _get_monitor_config(huey: "Huey", task: "Task") -> "Optional[MonitorConfig]":
330+
# Huey only evaluates crontabs in UTC when `utc=True` (the default).
331+
if not huey.utc:
332+
return None
333+
334+
# `periodic_task` wraps the `crontab()` validator in a method.
335+
validate = _get_closure_var(type(task).validate_datetime, "validate_datetime")
336+
if getattr(validate, "__code__", None) is not _CRONTAB_CODE:
337+
return None
338+
months, days, weekdays, hours, minutes = _get_closure_var(validate, "cron_settings")
339+
if not (months and days and weekdays and hours and minutes):
340+
return None
341+
342+
day = _get_crontab_field(days, 1, 31)
343+
# Huey 3 expands "*" to 0-7, with 7 also meaning Sunday.
344+
weekday = _get_crontab_field(sorted({v % 7 for v in weekdays}), 0, 6)
345+
# Huey requires both to match, cron requires either.
346+
if day != "*" and weekday != "*":
347+
return None
348+
349+
fields = (
350+
_get_crontab_field(minutes, 0, 59),
351+
_get_crontab_field(hours, 0, 23),
352+
day,
353+
_get_crontab_field(months, 1, 12),
354+
weekday,
355+
)
356+
return {
357+
"schedule": {"type": "crontab", "value": " ".join(fields)},
358+
"timezone": "UTC",
359+
}
Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
from datetime import datetime
2+
3+
import pytest
4+
from huey import crontab
5+
from huey.api import MemoryHuey
6+
from huey.exceptions import CancelExecution, RetryTask, TaskLockedException
7+
8+
from sentry_sdk.integrations.huey import HueyIntegration, _get_monitor_config
9+
10+
11+
def _check_ins(envelopes):
12+
return [
13+
item.payload.json
14+
for envelope in envelopes
15+
for item in envelope.items
16+
if item.headers.get("type") == "check_in"
17+
]
18+
19+
20+
def _run_periodic_tasks(huey, timestamp=datetime(2026, 1, 1)):
21+
"""Run the periodic tasks due at `timestamp` the way the consumer does."""
22+
for task in huey.read_periodic(timestamp):
23+
huey.enqueue(task)
24+
while True:
25+
task = huey.dequeue()
26+
if task is None:
27+
break
28+
huey.execute(task)
29+
30+
31+
def test_periodic_task_ok(sentry_init, capture_envelopes):
32+
sentry_init(integrations=[HueyIntegration(monitor_periodic_tasks=True)])
33+
envelopes = capture_envelopes()
34+
huey = MemoryHuey()
35+
36+
@huey.periodic_task(crontab(minute="*/5"))
37+
def cleanup():
38+
pass
39+
40+
_run_periodic_tasks(huey)
41+
42+
in_progress, ok = _check_ins(envelopes)
43+
assert in_progress["monitor_slug"] == "cleanup"
44+
assert in_progress["status"] == "in_progress"
45+
assert in_progress["monitor_config"] == {
46+
"schedule": {"type": "crontab", "value": "*/5 * * * *"},
47+
"timezone": "UTC",
48+
}
49+
assert ok["monitor_slug"] == "cleanup"
50+
assert ok["status"] == "ok"
51+
assert ok["check_in_id"] == in_progress["check_in_id"]
52+
assert ok["duration"] >= 0
53+
54+
55+
def test_periodic_task_error(sentry_init, capture_envelopes):
56+
sentry_init(integrations=[HueyIntegration(monitor_periodic_tasks=True)])
57+
envelopes = capture_envelopes()
58+
huey = MemoryHuey()
59+
60+
@huey.periodic_task(crontab(minute="0", hour="0"), name="nightly")
61+
def failing():
62+
raise ValueError("oops")
63+
64+
_run_periodic_tasks(huey)
65+
66+
check_ins = _check_ins(envelopes)
67+
assert [c["status"] for c in check_ins] == ["in_progress", "error"]
68+
assert {c["monitor_slug"] for c in check_ins} == {"nightly"}
69+
70+
events = [envelope.get_event() for envelope in envelopes]
71+
(error,) = [e for e in events if e is not None and e.get("exception")]
72+
assert error["exception"]["values"][0]["type"] == "ValueError"
73+
74+
75+
@pytest.mark.parametrize("exception", [CancelExecution, RetryTask, TaskLockedException])
76+
def test_periodic_task_control_flow(sentry_init, capture_envelopes, exception):
77+
sentry_init(integrations=[HueyIntegration(monitor_periodic_tasks=True)])
78+
envelopes = capture_envelopes()
79+
huey = MemoryHuey()
80+
81+
calls = []
82+
83+
@huey.periodic_task(crontab())
84+
def cleanup():
85+
if not calls:
86+
calls.append(1)
87+
raise exception()
88+
89+
_run_periodic_tasks(huey)
90+
91+
statuses = [c["status"] for c in _check_ins(envelopes)]
92+
assert statuses[:2] == ["in_progress", "error"]
93+
94+
95+
def test_regular_tasks_not_monitored(sentry_init, capture_envelopes):
96+
sentry_init(integrations=[HueyIntegration(monitor_periodic_tasks=True)])
97+
envelopes = capture_envelopes()
98+
huey = MemoryHuey()
99+
100+
@huey.task()
101+
def add(a, b):
102+
return a + b
103+
104+
add(1, 2)
105+
huey.execute(huey.dequeue())
106+
107+
assert _check_ins(envelopes) == []
108+
109+
110+
def test_periodic_tasks_not_monitored_by_default(sentry_init, capture_envelopes):
111+
sentry_init(integrations=[HueyIntegration()])
112+
envelopes = capture_envelopes()
113+
huey = MemoryHuey()
114+
115+
@huey.periodic_task(crontab())
116+
def cleanup():
117+
pass
118+
119+
_run_periodic_tasks(huey)
120+
121+
assert _check_ins(envelopes) == []
122+
123+
124+
def test_exclude_periodic_tasks(sentry_init, capture_envelopes):
125+
sentry_init(
126+
integrations=[
127+
HueyIntegration(
128+
monitor_periodic_tasks=True, exclude_periodic_tasks=["clean.*"]
129+
)
130+
]
131+
)
132+
envelopes = capture_envelopes()
133+
huey = MemoryHuey()
134+
135+
@huey.periodic_task(crontab())
136+
def cleanup():
137+
pass
138+
139+
@huey.periodic_task(crontab())
140+
def report():
141+
pass
142+
143+
_run_periodic_tasks(huey)
144+
145+
assert {c["monitor_slug"] for c in _check_ins(envelopes)} == {"report"}
146+
147+
148+
def _every_minute(timestamp):
149+
return True
150+
151+
152+
@pytest.mark.parametrize(
153+
"utc, validate_datetime",
154+
[
155+
(True, _every_minute),
156+
(True, crontab(minute="0", hour="0", day="1", day_of_week="1")),
157+
(False, crontab()),
158+
],
159+
)
160+
def test_unsupported_schedule_not_monitored(
161+
sentry_init, capture_envelopes, utc, validate_datetime
162+
):
163+
sentry_init(integrations=[HueyIntegration(monitor_periodic_tasks=True)])
164+
envelopes = capture_envelopes()
165+
huey = MemoryHuey(utc=utc)
166+
calls = []
167+
168+
@huey.periodic_task(validate_datetime)
169+
def cleanup():
170+
calls.append(1)
171+
172+
_run_periodic_tasks(huey, timestamp=datetime(2026, 6, 1))
173+
174+
assert calls == [1]
175+
assert _check_ins(envelopes) == []
176+
177+
178+
def _config(huey, validate_datetime):
179+
@huey.periodic_task(validate_datetime)
180+
def task():
181+
pass
182+
183+
return _get_monitor_config(huey, task.task_class())
184+
185+
186+
@pytest.mark.parametrize(
187+
"kwargs, value",
188+
[
189+
({}, "* * * * *"),
190+
({"minute": "0"}, "0 * * * *"),
191+
({"minute": 30, "hour": "9"}, "30 9 * * *"),
192+
({"minute": "*/15"}, "*/15 * * * *"),
193+
({"minute": "0", "hour": "*/6"}, "0 */6 * * *"),
194+
({"minute": "0", "hour": "9-17"}, "0 9-17 * * *"),
195+
({"minute": "0", "hour": "9,17"}, "0 9,17 * * *"),
196+
({"minute": "0", "hour": "8,12,16,20"}, "0 8-20/4 * * *"),
197+
({"minute": "0", "hour": "1,2,5"}, "0 1,2,5 * * *"),
198+
({"minute": "0", "hour": "0", "day": "1"}, "0 0 1 * *"),
199+
({"minute": "0", "hour": "0", "day": "1-31"}, "0 0 * * *"),
200+
({"minute": "0", "hour": "0", "day": "1", "month": "*/3"}, "0 0 1 */3 *"),
201+
({"minute": "0", "hour": "0", "day_of_week": "1-5"}, "0 0 * * 1-5"),
202+
({"minute": "0", "hour": "0", "day_of_week": "0,6"}, "0 0 * * 0,6"),
203+
],
204+
)
205+
def test_get_monitor_config(kwargs, value):
206+
assert _config(MemoryHuey(), crontab(**kwargs)) == {
207+
"schedule": {"type": "crontab", "value": value},
208+
"timezone": "UTC",
209+
}
210+
211+
212+
@pytest.mark.skipif(
213+
not hasattr(crontab, "hourly"), reason="crontab.hourly not available"
214+
)
215+
def test_get_monitor_config_hourly():
216+
assert _config(MemoryHuey(), crontab.hourly())["schedule"]["value"] == ("0 * * * *")
217+
218+
219+
def test_get_monitor_config_day_and_weekday():
220+
# Huey requires both to match, cron either.
221+
assert _config(MemoryHuey(), crontab(day="1", day_of_week="1")) is None
222+
223+
224+
def test_get_monitor_config_local_time():
225+
assert _config(MemoryHuey(utc=False), crontab()) is None

0 commit comments

Comments
 (0)