From ee1cc2a25dd992c4116862504bee2c39e351f417 Mon Sep 17 00:00:00 2001 From: Ignasi Barrera Date: Tue, 13 Oct 2026 10:54:57 +0200 Subject: [PATCH 1/2] redis: count REDIS_POOL_ON_EMPTY_BEHAVIOR=ERROR rejections in cmd_rejected REDIS_POOL_ON_EMPTY_BEHAVIOR=ERROR and the Redis pool metrics landed separately; count commands rejected because the pool has no connection in cmd_rejected, as is done for REDIS_MAX_INFLIGHT. Also check in the end-to-end outage test that no rate limit call is left in flight. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ignasi Barrera --- README.md | 2 +- src/redis/deadline.go | 2 +- src/redis/deadline_test.go | 1 + src/redis/e2e_test.go | 4 +++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 26719904..f8f35141 100644 --- a/README.md +++ b/README.md @@ -1111,7 +1111,7 @@ cmd_success # commands that succeeded cmd_error # commands that failed with a Redis or connection error cmd_io_timeout # commands that failed because Redis did not answer within the I/O timeout cmd_deadline_exceeded # commands whose caller's deadline expired first -cmd_rejected # commands rejected without reaching Redis (REDIS_MAX_INFLIGHT) +cmd_rejected # commands rejected without reaching Redis (REDIS_MAX_INFLIGHT, REDIS_POOL_ON_EMPTY_BEHAVIOR=ERROR) cmd_time # command latency in milliseconds, until the command completed, also in the background (timer) ``` diff --git a/src/redis/deadline.go b/src/redis/deadline.go index f8407756..d67c7f0f 100644 --- a/src/redis/deadline.go +++ b/src/redis/deadline.go @@ -37,7 +37,7 @@ func (c *clientImpl) do(ctx context.Context, fn func(context.Context) error, onS switch { case err == nil: c.stats.cmdSuccess.Inc() - case errors.Is(err, ErrTooManyInflight): + case errors.Is(err, ErrNoConnection) || errors.Is(err, ErrTooManyInflight): c.stats.cmdRejected.Inc() case errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled): c.stats.cmdDeadlineExceeded.Inc() diff --git a/src/redis/deadline_test.go b/src/redis/deadline_test.go index bc4d1a61..f80e2da9 100644 --- a/src/redis/deadline_test.go +++ b/src/redis/deadline_test.go @@ -177,6 +177,7 @@ func TestPoolOnEmptyErrorFailsFastWithoutConnections(t *testing.T) { err := c.PipeDo(context.Background(), incrPipeline(c, &rcv)) assert.ErrorIs(t, err, ErrNoConnection) assert.Less(t, time.Since(start), 10*time.Millisecond) + assert.Positive(t, c.stats.cmdRejected.Value()) } func TestCommandStats(t *testing.T) { diff --git a/src/redis/e2e_test.go b/src/redis/e2e_test.go index 1ea99173..6ceff9a8 100644 --- a/src/redis/e2e_test.go +++ b/src/redis/e2e_test.go @@ -273,8 +273,10 @@ func TestServiceReportsRedisOutage(t *testing.T) { require.Error(t, err) } - // Every failed call is counted, and the pool reports why. + // Every failed call is counted, none is left in flight, and the pool + // reports why. assert.EqualValues(t, calls, svc.counter("ratelimit.service.call.should_rate_limit.redis_error")) + assert.EqualValues(t, 0, svc.store.NewGauge("ratelimit.service.call.should_rate_limit.inflight").Value()) assert.Positive(t, svc.counter("ratelimit.redis_pool.cmd_io_timeout")) assert.Eventually(t, func() bool { return svc.counter("ratelimit.redis_pool.cx_local_close_error") > 0 }, 2*time.Second, 20*time.Millisecond) From 8a15db674d112b1ea2ab396344392d2fa31ca7bd Mon Sep 17 00:00:00 2001 From: Ignasi Barrera Date: Mon, 12 Oct 2026 18:53:13 +0200 Subject: [PATCH 2/2] docs: Redis failover and unresponsive Redis recommendations Document how ratelimit behaves when Redis fails over or stops answering, the layers that bound the impact (request deadline, I/O timeout, TCP_USER_TIMEOUT, health probe), recommended settings for a Redis with failover, how to size the I/O timeout against the client's own timeout, and which metrics to alert on. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ignasi Barrera --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index f8f35141..1eb7ac88 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ - [Two Redis Instances](#two-redis-instances) - [Health Checking for Redis Active Connection](#health-checking-for-redis-active-connection) - [Recovering from a failover (READONLY errors)](#recovering-from-a-failover-readonly-errors) + - [Redis failover and unresponsive Redis](#redis-failover-and-unresponsive-redis) - [Credentials that expire or rotate](#credentials-that-expire-or-rotate) - [AWS ElastiCache IAM authentication](#aws-elasticache-iam-authentication) - [Calendar-aligned MONTH rate limits](#calendar-aligned-month-rate-limits) @@ -1495,6 +1496,31 @@ configured address and reaches the current master. The failing command still ret to the caller; only the connection handling changes. Applies to both the main and the per-second Redis clients. +## Redis failover and unresponsive Redis + +A Redis failover, or any event that makes Redis stop answering without closing its connections (a managed Redis that moves the primary behind the same IP, a network partition, a node that hangs), leaves ratelimit's connections open with no reply coming. Ratelimit handles it in layers, from the fastest to the slowest: + +1. **Request deadline.** A call returns an error as soon as the deadline set by its client expires (see [Request Deadlines](#request-deadlines)). Envoy then applies its failure mode right away (allow by default, or deny with `failure_mode_deny`). +1. **I/O timeout.** A connection that does not reply within `REDIS_TIMEOUT` (or `REDIS_IO_TIMEOUT`) is closed and the pool dials a new one, which reaches the current primary. This is what ends the outage on the ratelimit side; without it the connections stay blocked until the OS closes them, about 15 minutes with default Linux settings. +1. **`TCP_USER_TIMEOUT`.** With `REDIS_TCP_USER_TIMEOUT`, the kernel itself closes connections whose data stays unacknowledged, as a second line of defense. +1. **Readiness.** With `REDIS_HEALTH_CHECK_PING_INTERVAL`, replicas that cannot reach Redis report not ready on `/ready`, so a readiness probe can take them out of rotation while the rest keep serving. `/healthcheck`, used for liveness, is not affected: restarting replicas cannot fix Redis. + +Recommended settings for a Redis with failover (e.g. Sentinel, a managed Redis such as GCP Memorystore or AWS ElastiCache, or a Kubernetes Service in front of a primary): + +``` +REDIS_TIMEOUT=1s # dial and I/O timeout: well above the slowest normal reply +REDIS_TCP_USER_TIMEOUT=5s # Linux only +REDIS_CLOSE_CONNECTION_ON_READONLY_ERROR=true # if the failover demotes the old primary behind the same address +REDIS_HEALTH_CHECK_PING_INTERVAL=1s # point readiness probes at /ready, keep liveness on /healthcheck +REDIS_MAX_INFLIGHT= +``` + +Use the same values for the `REDIS_PERSECOND_*` settings when a per-second Redis is configured. + +The I/O timeout fails every command in flight on a connection that timed out, so keep it well above the slowest reply Redis normally gives, including under load and across zones; a few hundred milliseconds to a couple of seconds suits most deployments. The client's own timeout (e.g. Envoy's rate limit filter `timeout`) should in turn be above the normal Redis round-trip: calls that miss it are not rate limited (or are denied, with `failure_mode_deny`) even though Redis is healthy. + +To watch for these situations, alert on `ratelimit.service.call.should_rate_limit.redis_error`, `cmd_io_timeout` and `cx_local_close_error`, and on `ratelimit.service.call.should_rate_limit.inflight` or `cmd_inflight` growing (see [Service statistics](#service-statistics) and [Redis statistics](#redis-statistics)). + ## Credentials that expire or rotate 1. `REDIS_AUTH_FILE` : (default is "")