From af1151ab0c084376ac4997ce902a3804277efb97 Mon Sep 17 00:00:00 2001 From: Rodrigo Nogueira Date: Sat, 8 Aug 2026 22:21:02 -0300 Subject: [PATCH 1/4] fix(base): scope clear() to the cache namespace clear() forwarded its namespace argument straight to the backend, so a cache built with a namespace passed None and every backend took its "clear everything" path: FLUSHDB on Valkey, flush_all on Memcached, and a fresh dict for in-memory. Two caches sharing one server could therefore destroy each other's keys, and the docstring already promised the opposite. Resolve the namespace the way every other operation does. Valkey's namespaced branch only ever deleted the first SCAN batch, which left almost everything behind once a namespace grew past a few keys, so iterate until the cursor returns to 0. Memcached cannot clear by namespace and already raised ValueError when given one explicitly; a namespaced instance now gets that same error instead of silently flushing the server. Pass namespace="" to flush. --- CHANGES.rst | 1 + aiocache/backends/valkey.py | 14 ++++++---- aiocache/base.py | 4 +++ tests/acceptance/test_base.py | 44 +++++++++++++++++++++++++++----- tests/ut/backends/test_valkey.py | 14 ++++++++++ tests/ut/test_base.py | 14 ++++++++++ 6 files changed, 80 insertions(+), 11 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8962edf4b..92d036bf0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,7 @@ There are a number of backwards-incompatible changes. These points should help w * The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"`` * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). * ``BaseCache`` and ``BaseSerializer`` are now ``ABC``s, so cannot be instantiated directly. +* ``clear()`` now defaults to the cache's own ``namespace`` instead of clearing the whole backend. A cache built with ``namespace`` only removes its own keys, so a shared server keeps the keys written by other caches. ``MemcachedCache`` cannot clear by namespace, so ``clear()`` on a namespaced instance now raises ``ValueError`` rather than flushing the server; call ``clear(namespace="")`` to flush explicitly. * If subclassing ``BaseCache`` to implement a custom backend: * The cache key type used by the backend must now be specified when inheriting (e.g. ``BaseCache[str]`` typically). diff --git a/aiocache/backends/valkey.py b/aiocache/backends/valkey.py index d07c70fe6..64113e393 100644 --- a/aiocache/backends/valkey.py +++ b/aiocache/backends/valkey.py @@ -148,11 +148,15 @@ async def _clear(self, namespace=None, _conn=None): if not namespace: return await self.client.flushdb() - _, keys = await self.client.scan(b"0", "{}:*".format(namespace)) - if keys: - return bool(await self.client.delete(keys)) - - return True + # SCAN returns a batch at a time, so keep going until the cursor comes + # back to 0 or only the keys in the first batch would be deleted. + cursor = b"0" + while True: + cursor, keys = await self.client.scan(cursor, "{}:*".format(namespace)) + if keys: + await self.client.delete(keys) + if cursor == b"0": + return True async def _raw(self, command, *args, encoding="utf-8", _conn=None, **kwargs): value = await getattr(self.client, command)(*args, **kwargs) diff --git a/aiocache/base.py b/aiocache/base.py index 8aed65706..85481e890 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -447,6 +447,9 @@ async def clear(self, namespace=None, _conn=None): Clears the cache in the cache namespace. If an alternative namespace is given, it will clear those ones instead. + Caches configured without a namespace clear the whole backend, which for a shared + server means every key, including those written by other caches. + :param namespace: str alternative namespace to use :param timeout: int or float in seconds specifying maximum timeout for the operations to last @@ -454,6 +457,7 @@ async def clear(self, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() + namespace = namespace if namespace is not None else self.namespace ret = await self._clear(namespace, _conn=_conn) logger.debug("CLEAR %s %d (%.4f)s", namespace, ret, time.monotonic() - start) return ret diff --git a/tests/acceptance/test_base.py b/tests/acceptance/test_base.py index 614bb0b0b..47fdd6875 100644 --- a/tests/acceptance/test_base.py +++ b/tests/acceptance/test_base.py @@ -121,12 +121,6 @@ async def test_expire_with_0(self, cache): async def test_expire_missing(self, cache): assert await cache.expire(Keys.KEY, 1) is False - async def test_clear(self, cache): - await cache.set(Keys.KEY, "value") - await cache.clear() - - assert await cache.exists(Keys.KEY) is False - async def test_close_pool_only_clears_resources(self, cache): await cache.set(Keys.KEY, "value") await cache.close() @@ -169,6 +163,15 @@ async def test_clear_with_namespace_memory(self, memory_cache): assert await memory_cache.exists(Keys.KEY, namespace="test") is False + async def test_clear_only_removes_own_namespace_memory(self, memory_cache): + await memory_cache.set(Keys.KEY, "value") + await memory_cache.set(Keys.KEY, "other", namespace="other") + + await memory_cache.clear() + + assert await memory_cache.exists(Keys.KEY) is False + assert await memory_cache.exists(Keys.KEY, namespace="other") is True + @pytest.mark.memcached class TestMemcachedCache: @@ -208,6 +211,14 @@ async def test_clear_with_namespace_memcached(self, memcached_cache): assert await memcached_cache.exists(Keys.KEY, namespace="test") is True + async def test_clear_namespaced_cache_memcached(self, memcached_cache): + await memcached_cache.set(Keys.KEY, "value") + + with pytest.raises(ValueError): + await memcached_cache.clear() + + assert await memcached_cache.exists(Keys.KEY) is True + async def test_close(self, memcached_cache): await memcached_cache.set(Keys.KEY, "value") await memcached_cache._close() @@ -259,6 +270,27 @@ async def test_clear_with_namespace_valkey(self, valkey_cache): assert await valkey_cache.exists(Keys.KEY, namespace="test") is False + async def test_clear_only_removes_own_namespace_valkey(self, valkey_cache): + await valkey_cache.set(Keys.KEY, "value") + await valkey_cache.set(Keys.KEY, "other", namespace="other") + + await valkey_cache.clear() + + assert await valkey_cache.exists(Keys.KEY) is False + assert await valkey_cache.exists(Keys.KEY, namespace="other") is True + await valkey_cache.delete(Keys.KEY, namespace="other") + + async def test_clear_removes_keys_beyond_one_scan_batch(self, valkey_cache): + keys = [f"{Keys.KEY.value}-{i}" for i in range(2000)] + for start in range(0, len(keys), 200): + batch = keys[start:start + 200] + await asyncio.gather(*(valkey_cache.set(k, "value") for k in batch)) + + await valkey_cache.clear() + + assert await valkey_cache.exists(keys[0]) is False + assert await valkey_cache.exists(keys[-1]) is False + async def test_close(self, valkey_cache): await valkey_cache.set(Keys.KEY, "value") await valkey_cache._close() diff --git a/tests/ut/backends/test_valkey.py b/tests/ut/backends/test_valkey.py index eed39c021..ed8cbad8c 100644 --- a/tests/ut/backends/test_valkey.py +++ b/tests/ut/backends/test_valkey.py @@ -200,6 +200,20 @@ async def test_clear_no_keys(self, valkey): await valkey._clear("nm") valkey.client.delete.assert_not_called() + async def test_clear_scans_until_cursor_is_exhausted(self, valkey): + valkey.client.scan.side_effect = [ + [b"17", ["nm:a"]], + [b"0", ["nm:b"]], + ] + + await valkey._clear("nm") + + assert [c.args[0] for c in valkey.client.scan.call_args_list] == [b"0", b"17"] + assert [c.args[0] for c in valkey.client.delete.call_args_list] == [ + ["nm:a"], + ["nm:b"], + ] + async def test_clear_no_namespace(self, valkey): await valkey._clear() assert valkey.client.flushdb.call_count == 1 diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index cfda509e0..b8130d18d 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -577,6 +577,20 @@ async def test_clear(self, mock_base_cache): assert mock_base_cache.plugins[0].pre_clear.call_count == 1 assert mock_base_cache.plugins[0].post_clear.call_count == 1 + async def test_clear_uses_cache_namespace(self, mock_base_cache): + mock_base_cache.namespace = "ns" + + await mock_base_cache.clear() + + mock_base_cache._clear.assert_called_with("ns", _conn=ANY) + + async def test_clear_namespace_argument_takes_precedence(self, mock_base_cache): + mock_base_cache.namespace = "ns" + + await mock_base_cache.clear("other") + + mock_base_cache._clear.assert_called_with("other", _conn=ANY) + async def test_clear_timeouts(self, mock_base_cache): mock_base_cache._clear = self.asleep From 30ed7bcddf6be74c4a14f523486fce1045d7076a Mon Sep 17 00:00:00 2001 From: Rodrigo Nogueira Date: Sat, 8 Aug 2026 22:37:15 -0300 Subject: [PATCH 2/4] Match the namespace prefix the key_builder produces Both namespaced backends assumed the default key layout: Valkey scanned ":*" and memory compared against the bare namespace. A cache with a custom key_builder therefore cleared nothing at all on Valkey, which is worse than the flush it used to do. Derive the prefix from build_key() instead. With a builder that does not separate the namespace from the key, a namespace still matches longer namespaces starting with it, since those keys are indistinguishable. Say so rather than promise otherwise. Also pin that an empty namespace still clears the whole backend; that is the documented way to flush and nothing covered it. --- CHANGES.rst | 2 +- aiocache/backends/memory.py | 7 ++++++- aiocache/backends/valkey.py | 11 ++++++++--- aiocache/base.py | 9 ++++++++- tests/acceptance/test_base.py | 26 +++++++++++++++++++------- tests/ut/test_base.py | 8 ++++++++ 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 92d036bf0..5814807b0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,7 +17,7 @@ There are a number of backwards-incompatible changes. These points should help w * The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"`` * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). * ``BaseCache`` and ``BaseSerializer`` are now ``ABC``s, so cannot be instantiated directly. -* ``clear()`` now defaults to the cache's own ``namespace`` instead of clearing the whole backend. A cache built with ``namespace`` only removes its own keys, so a shared server keeps the keys written by other caches. ``MemcachedCache`` cannot clear by namespace, so ``clear()`` on a namespaced instance now raises ``ValueError`` rather than flushing the server; call ``clear(namespace="")`` to flush explicitly. +* ``clear()`` now defaults to the cache's own ``namespace`` instead of clearing the whole backend, so a shared server keeps the keys written by other caches. The namespace is matched by the key prefix the ``key_builder`` produces, which means a builder that does not separate the namespace from the key also matches longer namespaces starting with it. ``MemcachedCache`` cannot clear by namespace, so ``clear()`` on a namespaced instance now raises ``ValueError`` rather than flushing the server; call ``clear(namespace="")`` to flush explicitly. * If subclassing ``BaseCache`` to implement a custom backend: * The cache key type used by the backend must now be specified when inheriting (e.g. ``BaseCache[str]`` typically). diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index eddef5307..8e58c9457 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -125,8 +125,13 @@ async def _delete(self, key, _conn=None): async def _clear(self, namespace=None, _conn=None): if namespace: + # Match the prefix this cache's key_builder produces. With the + # default builder that is the bare namespace, which cannot be told + # apart from a longer namespace sharing it; a builder that appends + # a separator does not have that ambiguity. + prefix = self.build_key("", namespace) for key in list(self._cache): - if key.startswith(namespace): + if key.startswith(prefix): self.__delete(key) else: self._cache = OrderedDict() diff --git a/aiocache/backends/valkey.py b/aiocache/backends/valkey.py index 64113e393..7b7f4803b 100644 --- a/aiocache/backends/valkey.py +++ b/aiocache/backends/valkey.py @@ -148,11 +148,16 @@ async def _clear(self, namespace=None, _conn=None): if not namespace: return await self.client.flushdb() - # SCAN returns a batch at a time, so keep going until the cursor comes - # back to 0 or only the keys in the first batch would be deleted. + # Match whatever prefix this cache's key_builder produces rather than + # assuming the default ":" layout, otherwise a custom + # key_builder makes this delete nothing at all. + pattern = self.build_key("", namespace) + "*" + + # SCAN returns one batch per call, so keep going until the cursor comes + # back to 0. Stopping at the first batch leaves the rest behind. cursor = b"0" while True: - cursor, keys = await self.client.scan(cursor, "{}:*".format(namespace)) + cursor, keys = await self.client.scan(cursor, pattern) if keys: await self.client.delete(keys) if cursor == b"0": diff --git a/aiocache/base.py b/aiocache/base.py index 85481e890..01e0638a3 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -448,13 +448,20 @@ async def clear(self, namespace=None, _conn=None): clear those ones instead. Caches configured without a namespace clear the whole backend, which for a shared - server means every key, including those written by other caches. + server means every key, including those written by other caches. Passing an empty + namespace asks for that explicitly. + + A namespace is matched by the key prefix the ``key_builder`` produces, so with a + builder that does not separate the namespace from the key, a namespace also matches + the longer namespaces starting with it. :param namespace: str alternative namespace to use :param timeout: int or float in seconds specifying maximum timeout for the operations to last :returns: True :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout + :raises: :class:`ValueError` if the backend cannot clear a single namespace, as + :class:`~aiocache.MemcachedCache` cannot """ start = time.monotonic() namespace = namespace if namespace is not None else self.namespace diff --git a/tests/acceptance/test_base.py b/tests/acceptance/test_base.py index 47fdd6875..2ad62fde8 100644 --- a/tests/acceptance/test_base.py +++ b/tests/acceptance/test_base.py @@ -219,6 +219,13 @@ async def test_clear_namespaced_cache_memcached(self, memcached_cache): assert await memcached_cache.exists(Keys.KEY) is True + async def test_clear_empty_namespace_flushes_memcached(self, memcached_cache): + await memcached_cache.set(Keys.KEY, "value") + + await memcached_cache.clear(namespace="") + + assert await memcached_cache.exists(Keys.KEY) is False + async def test_close(self, memcached_cache): await memcached_cache.set(Keys.KEY, "value") await memcached_cache._close() @@ -274,11 +281,13 @@ async def test_clear_only_removes_own_namespace_valkey(self, valkey_cache): await valkey_cache.set(Keys.KEY, "value") await valkey_cache.set(Keys.KEY, "other", namespace="other") - await valkey_cache.clear() + try: + await valkey_cache.clear() - assert await valkey_cache.exists(Keys.KEY) is False - assert await valkey_cache.exists(Keys.KEY, namespace="other") is True - await valkey_cache.delete(Keys.KEY, namespace="other") + assert await valkey_cache.exists(Keys.KEY) is False + assert await valkey_cache.exists(Keys.KEY, namespace="other") is True + finally: + await valkey_cache.delete(Keys.KEY, namespace="other") async def test_clear_removes_keys_beyond_one_scan_batch(self, valkey_cache): keys = [f"{Keys.KEY.value}-{i}" for i in range(2000)] @@ -286,10 +295,13 @@ async def test_clear_removes_keys_beyond_one_scan_batch(self, valkey_cache): batch = keys[start:start + 200] await asyncio.gather(*(valkey_cache.set(k, "value") for k in batch)) - await valkey_cache.clear() + try: + await valkey_cache.clear() - assert await valkey_cache.exists(keys[0]) is False - assert await valkey_cache.exists(keys[-1]) is False + assert await valkey_cache.exists(keys[0]) is False + assert await valkey_cache.exists(keys[-1]) is False + finally: + await valkey_cache.clear() async def test_close(self, valkey_cache): await valkey_cache.set(Keys.KEY, "value") diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index b8130d18d..8ff331b39 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -591,6 +591,14 @@ async def test_clear_namespace_argument_takes_precedence(self, mock_base_cache): mock_base_cache._clear.assert_called_with("other", _conn=ANY) + async def test_clear_empty_namespace_clears_everything(self, mock_base_cache): + """An empty namespace is a request to clear the backend, not a missing value.""" + mock_base_cache.namespace = "ns" + + await mock_base_cache.clear("") + + mock_base_cache._clear.assert_called_with("", _conn=ANY) + async def test_clear_timeouts(self, mock_base_cache): mock_base_cache._clear = self.asleep From 22f08d2bd868e0553a8d80291dda04296e8e5ab4 Mon Sep 17 00:00:00 2001 From: Rodrigo Nogueira Date: Sat, 15 Aug 2026 19:39:58 -0300 Subject: [PATCH 3/4] Keep a namespaced clear() inside its own namespace Deriving the scan pattern from the key_builder's prefix left two ways for clear() to touch keys outside the namespace it was given. A prefix is a glob pattern to Valkey's SCAN, so a namespace containing glob syntax matched something else entirely: "ten[a]nt" became the character class "ten[a]nt:*", which matches "tenant:*". Clearing it deleted the neighbouring namespace's keys and left its own in place, the exact reverse of what was asked. Escape the metacharacters before appending the wildcard. A key_builder that ignores the namespace produces an empty prefix, making the pattern a bare "*" and clear() a full flush of the database, other namespaces included, reported as success. Memory has the same hole, since every key starts with "". Neither can be scoped, so raise rather than delete more than was asked for. A builder that places the namespace anywhere but the start is left alone: it cannot be told apart from a valid prefix, so it deletes nothing and says so in the docs instead. Cover the prefix derivation itself, which nothing pinned before: reverting it to the old hardcoded ":" passed the whole suite, because both default key_builders happen to produce exactly that string. --- CHANGES.rst | 2 +- aiocache/backends/memory.py | 11 ++++ aiocache/backends/valkey.py | 25 +++++++-- aiocache/base.py | 5 ++ tests/acceptance/test_base.py | 98 +++++++++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5814807b0..fb8031424 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,7 +17,7 @@ There are a number of backwards-incompatible changes. These points should help w * The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"`` * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). * ``BaseCache`` and ``BaseSerializer`` are now ``ABC``s, so cannot be instantiated directly. -* ``clear()`` now defaults to the cache's own ``namespace`` instead of clearing the whole backend, so a shared server keeps the keys written by other caches. The namespace is matched by the key prefix the ``key_builder`` produces, which means a builder that does not separate the namespace from the key also matches longer namespaces starting with it. ``MemcachedCache`` cannot clear by namespace, so ``clear()`` on a namespaced instance now raises ``ValueError`` rather than flushing the server; call ``clear(namespace="")`` to flush explicitly. +* ``clear()`` now defaults to the cache's own ``namespace`` instead of clearing the whole backend, so a shared server keeps the keys written by other caches. The namespace is matched by the key prefix the ``key_builder`` produces, which means a builder that does not separate the namespace from the key also matches longer namespaces starting with it. A builder that does not place the namespace at the start of the key leaves nothing to match on, so clearing that namespace deletes nothing; one that drops the namespace entirely now raises ``ValueError`` instead of matching every key. ``MemcachedCache`` cannot clear by namespace, so ``clear()`` on a namespaced instance now raises ``ValueError`` rather than flushing the server; call ``clear(namespace="")`` to flush explicitly. * If subclassing ``BaseCache`` to implement a custom backend: * The cache key type used by the backend must now be specified when inheriting (e.g. ``BaseCache[str]`` typically). diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index 8e58c9457..ed1d9f9fc 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -130,6 +130,17 @@ async def _clear(self, namespace=None, _conn=None): # apart from a longer namespace sharing it; a builder that appends # a separator does not have that ambiguity. prefix = self.build_key("", namespace) + if not prefix: + # The builder ignored the namespace, so every key would match + # this prefix and the whole cache would go. Refuse rather than + # silently clear it. + raise ValueError( + f"key_builder produced an empty prefix for namespace " + f"{namespace!r}, so clearing it would match every key. Use " + f"a key_builder that places the namespace at the start of " + f"the key, or call clear() without a namespace to clear " + f"the cache deliberately." + ) for key in list(self._cache): if key.startswith(prefix): self.__delete(key) diff --git a/aiocache/backends/valkey.py b/aiocache/backends/valkey.py index 7b7f4803b..b34d6f351 100644 --- a/aiocache/backends/valkey.py +++ b/aiocache/backends/valkey.py @@ -1,4 +1,5 @@ import logging +import re import sys from typing import Optional @@ -15,6 +16,9 @@ from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer +#: Characters SCAN treats as glob syntax, escaped before matching a prefix. +_GLOB_METACHARACTERS = re.compile(r"([\\*?\[\]])") + if sys.version_info >= (3, 11): from typing import Self else: @@ -149,9 +153,24 @@ async def _clear(self, namespace=None, _conn=None): return await self.client.flushdb() # Match whatever prefix this cache's key_builder produces rather than - # assuming the default ":" layout, otherwise a custom - # key_builder makes this delete nothing at all. - pattern = self.build_key("", namespace) + "*" + # assuming the default ":" layout, which a custom + # key_builder need not follow. + prefix = self.build_key("", namespace) + if not prefix: + # The builder ignored the namespace, so the pattern would be a bare + # "*" and this would delete every key in the database, including + # other namespaces'. Refuse rather than silently flush. + raise ValueError( + f"key_builder produced an empty prefix for namespace " + f"{namespace!r}, so clearing it would match every key. Use a " + f"key_builder that places the namespace at the start of the " + f"key, or call clear() without a namespace to flush the " + f"database deliberately." + ) + + # Escape the prefix: a namespace holding a glob character would + # otherwise match some other namespace's keys and leave its own behind. + pattern = _GLOB_METACHARACTERS.sub(r"\\\1", prefix) + "*" # SCAN returns one batch per call, so keep going until the cursor comes # back to 0. Stopping at the first batch leaves the rest behind. diff --git a/aiocache/base.py b/aiocache/base.py index 01e0638a3..158e24694 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -455,6 +455,11 @@ async def clear(self, namespace=None, _conn=None): builder that does not separate the namespace from the key, a namespace also matches the longer namespaces starting with it. + This means the namespace has to reach the start of the key. A ``key_builder`` that + places it elsewhere, or hashes it, leaves nothing to match on: clearing that + namespace deletes nothing and still reports success. One that drops the namespace + entirely is rejected with a :class:`ValueError` rather than matching every key. + :param namespace: str alternative namespace to use :param timeout: int or float in seconds specifying maximum timeout for the operations to last diff --git a/tests/acceptance/test_base.py b/tests/acceptance/test_base.py index 2ad62fde8..5054dfb5a 100644 --- a/tests/acceptance/test_base.py +++ b/tests/acceptance/test_base.py @@ -163,6 +163,33 @@ async def test_clear_with_namespace_memory(self, memory_cache): assert await memory_cache.exists(Keys.KEY, namespace="test") is False + async def test_clear_matches_custom_key_builder_prefix_memory(self): + """clear() must match the prefix the key_builder makes, not the bare + namespace, so a builder with its own separator still scopes correctly. + """ + cache = SimpleMemoryCache( + namespace="ns", key_builder=lambda k, ns: f"{ns}__{k}" if ns else k + ) + await cache.set(Keys.KEY, "value") + await cache.set(Keys.KEY, "other", namespace="nsother") + + await cache.clear() + + assert await cache.exists(Keys.KEY) is False + assert await cache.exists(Keys.KEY, namespace="nsother") is True + + async def test_clear_rejects_key_builder_without_namespace_memory(self): + """A builder dropping the namespace makes every key match the prefix, + so clearing one namespace would empty the whole cache. Refuse instead. + """ + cache = SimpleMemoryCache(namespace="ns", key_builder=lambda k, ns: k) + await cache.set(Keys.KEY, "value") + + with pytest.raises(ValueError, match="empty prefix"): + await cache.clear() + + assert await cache.get(Keys.KEY) == "value" + async def test_clear_only_removes_own_namespace_memory(self, memory_cache): await memory_cache.set(Keys.KEY, "value") await memory_cache.set(Keys.KEY, "other", namespace="other") @@ -289,6 +316,77 @@ async def test_clear_only_removes_own_namespace_valkey(self, valkey_cache): finally: await valkey_cache.delete(Keys.KEY, namespace="other") + @pytest.mark.parametrize("namespace", ["ten[a]nt", "ten?nt", "ten*"]) + async def test_clear_escapes_glob_in_namespace_valkey( + self, valkey_config, namespace + ): + """A namespace holding glob syntax must not match another namespace. + + Unescaped, "ten[a]nt:*" is a character class matching "tenant:*", so + clear() deleted the neighbour's keys and left its own in place. + """ + from aiocache.backends.valkey import ValkeyCache + + async with ValkeyCache(valkey_config, namespace=namespace) as odd: + async with ValkeyCache(valkey_config, namespace="tenant") as neighbour: + await odd.set(Keys.KEY, "mine") + await neighbour.set(Keys.KEY, "theirs") + try: + await odd.clear() + + assert await odd.exists(Keys.KEY) is False + assert await neighbour.get(Keys.KEY) == "theirs" + finally: + await odd.delete(Keys.KEY) + await neighbour.delete(Keys.KEY) + + async def test_clear_matches_custom_key_builder_prefix_valkey( + self, valkey_config + ): + """clear() must match the prefix the key_builder makes, not ":". + + A builder using its own separator writes "ns__key". Assuming the + default layout looks for "ns:*", matches nothing, deletes nothing and + still returns True. + """ + from aiocache.backends.valkey import ValkeyCache + + def key_builder(key, namespace): + return f"{namespace}__{key}" if namespace else key + + async with ValkeyCache( + valkey_config, namespace="ns", key_builder=key_builder + ) as cache: + await cache.set(Keys.KEY, "value") + try: + assert await cache.clear() is True + + assert await cache.exists(Keys.KEY) is False + finally: + await cache.delete(Keys.KEY) + + async def test_clear_rejects_key_builder_without_namespace_valkey( + self, valkey_config + ): + """A builder dropping the namespace leaves nothing to scope the scan. + + The pattern would be a bare "*", so clear() would delete every key in + the database. It has to refuse instead. + """ + from aiocache.backends.valkey import ValkeyCache + + async with ValkeyCache( + valkey_config, namespace="ns", key_builder=lambda k, ns: k + ) as cache: + await cache.set(Keys.KEY, "value") + try: + with pytest.raises(ValueError, match="empty prefix"): + await cache.clear() + + assert await cache.get(Keys.KEY) == "value" + finally: + await cache.delete(Keys.KEY) + async def test_clear_removes_keys_beyond_one_scan_batch(self, valkey_cache): keys = [f"{Keys.KEY.value}-{i}" for i in range(2000)] for start in range(0, len(keys), 200): From 2d0ea746d3b4d976b95cec68c7b1128efb7f2670 Mon Sep 17 00:00:00 2001 From: Rodrigo Nogueira Date: Sat, 15 Aug 2026 20:06:33 -0300 Subject: [PATCH 4/4] Check the namespace really leads the key before scanning for it Deriving the prefix from build_key("", namespace) assumed the key_builder puts the namespace first. Nothing enforced that, and the assumption fails destructively rather than harmlessly. A builder appending the namespace, lambda k, ns: f"{k}{ns}", makes keys like "minens" but yields the prefix "ns". That prefix does not match the namespace's own keys, so they survive a clear(), while unrelated keys that merely start with the same characters, "ns-foreign", are deleted. Exactly inverted, on both prefix-matching backends. Hashing the whole key fails the same way, matching whatever happens to share the hash's leading digits. So verify the property instead of assuming it: build a key nothing collides with and confirm the derived prefix leads it. Builders that append, hash or drop the namespace are refused, which also covers the empty prefix the previous guard caught, and leaves ns:key, nskey and ns__key working. --- CHANGES.rst | 2 +- aiocache/backends/memory.py | 13 +---------- aiocache/backends/valkey.py | 16 +++---------- aiocache/base.py | 32 ++++++++++++++++++++++---- tests/acceptance/test_base.py | 42 +++++++++++++++++++++++++---------- 5 files changed, 63 insertions(+), 42 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index fb8031424..4bcbe0f75 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,7 +17,7 @@ There are a number of backwards-incompatible changes. These points should help w * The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"`` * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). * ``BaseCache`` and ``BaseSerializer`` are now ``ABC``s, so cannot be instantiated directly. -* ``clear()`` now defaults to the cache's own ``namespace`` instead of clearing the whole backend, so a shared server keeps the keys written by other caches. The namespace is matched by the key prefix the ``key_builder`` produces, which means a builder that does not separate the namespace from the key also matches longer namespaces starting with it. A builder that does not place the namespace at the start of the key leaves nothing to match on, so clearing that namespace deletes nothing; one that drops the namespace entirely now raises ``ValueError`` instead of matching every key. ``MemcachedCache`` cannot clear by namespace, so ``clear()`` on a namespaced instance now raises ``ValueError`` rather than flushing the server; call ``clear(namespace="")`` to flush explicitly. +* ``clear()`` now defaults to the cache's own ``namespace`` instead of clearing the whole backend, so a shared server keeps the keys written by other caches. The namespace is matched by the key prefix the ``key_builder`` produces, which means a builder that does not separate the namespace from the key also matches longer namespaces starting with it. A builder that does not place the namespace at the start of the key (one that appends it, hashes it, or ignores it) has no prefix to match on, and now raises ``ValueError`` instead of deleting unrelated keys. ``MemcachedCache`` cannot clear by namespace, so ``clear()`` on a namespaced instance now raises ``ValueError`` rather than flushing the server; call ``clear(namespace="")`` to flush explicitly. * If subclassing ``BaseCache`` to implement a custom backend: * The cache key type used by the backend must now be specified when inheriting (e.g. ``BaseCache[str]`` typically). diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index ed1d9f9fc..b167a0617 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -129,18 +129,7 @@ async def _clear(self, namespace=None, _conn=None): # default builder that is the bare namespace, which cannot be told # apart from a longer namespace sharing it; a builder that appends # a separator does not have that ambiguity. - prefix = self.build_key("", namespace) - if not prefix: - # The builder ignored the namespace, so every key would match - # this prefix and the whole cache would go. Refuse rather than - # silently clear it. - raise ValueError( - f"key_builder produced an empty prefix for namespace " - f"{namespace!r}, so clearing it would match every key. Use " - f"a key_builder that places the namespace at the start of " - f"the key, or call clear() without a namespace to clear " - f"the cache deliberately." - ) + prefix = self._namespace_prefix(namespace) for key in list(self._cache): if key.startswith(prefix): self.__delete(key) diff --git a/aiocache/backends/valkey.py b/aiocache/backends/valkey.py index b34d6f351..ea6ab66d3 100644 --- a/aiocache/backends/valkey.py +++ b/aiocache/backends/valkey.py @@ -154,19 +154,9 @@ async def _clear(self, namespace=None, _conn=None): # Match whatever prefix this cache's key_builder produces rather than # assuming the default ":" layout, which a custom - # key_builder need not follow. - prefix = self.build_key("", namespace) - if not prefix: - # The builder ignored the namespace, so the pattern would be a bare - # "*" and this would delete every key in the database, including - # other namespaces'. Refuse rather than silently flush. - raise ValueError( - f"key_builder produced an empty prefix for namespace " - f"{namespace!r}, so clearing it would match every key. Use a " - f"key_builder that places the namespace at the start of the " - f"key, or call clear() without a namespace to flush the " - f"database deliberately." - ) + # key_builder need not follow. Refuses builders that have no such + # prefix, whose pattern would match the wrong keys or all of them. + prefix = self._namespace_prefix(namespace) # Escape the prefix: a namespace holding a glob character would # otherwise match some other namespace's keys and leave its own behind. diff --git a/aiocache/base.py b/aiocache/base.py index 158e24694..728c0a752 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -18,6 +18,9 @@ logger = logging.getLogger(__name__) SENTINEL = object() + +#: Key fed to a key_builder to check the namespace lands at the start of it. +_PREFIX_PROBE = "\x00aiocache-prefix-probe\x00" CacheKeyType = TypeVar("CacheKeyType") @@ -455,10 +458,10 @@ async def clear(self, namespace=None, _conn=None): builder that does not separate the namespace from the key, a namespace also matches the longer namespaces starting with it. - This means the namespace has to reach the start of the key. A ``key_builder`` that - places it elsewhere, or hashes it, leaves nothing to match on: clearing that - namespace deletes nothing and still reports success. One that drops the namespace - entirely is rejected with a :class:`ValueError` rather than matching every key. + This only works if the namespace reaches the start of the key. A ``key_builder`` + that appends it, hashes it, or drops it has no such prefix, and matching on one + anyway would delete unrelated keys while leaving the namespace's own behind, so + those are rejected with a :class:`ValueError`. :param namespace: str alternative namespace to use :param timeout: int or float in seconds specifying maximum timeout @@ -538,6 +541,27 @@ def _str_build_key(self, key: str, namespace: Optional[str] = None) -> str: ns = self.namespace if namespace is None else namespace return self._build_key(key_name, ns) + def _namespace_prefix(self, namespace): + """Key prefix that scopes a scan to ``namespace``. + + Backends that clear a namespace by matching a key prefix need the + ``key_builder`` to put the namespace at the start of the key. That + cannot be assumed, so check it: build a key nothing else can collide + with and confirm the derived prefix really leads it. + + :raises ValueError: if the ``key_builder`` has no such prefix, since + any pattern would then match the wrong keys, or all of them. + """ + prefix = self.build_key("", namespace) + if not prefix or not self.build_key(_PREFIX_PROBE, namespace).startswith(prefix): + raise ValueError( + f"key_builder does not place namespace {namespace!r} at the start " + f"of the key, so clear() cannot scope to it without matching " + f"unrelated keys. Use a key_builder that prefixes keys with the " + f"namespace, or call clear() without a namespace." + ) + return prefix + def _get_ttl(self, ttl): return ttl if ttl is not SENTINEL else self.ttl diff --git a/tests/acceptance/test_base.py b/tests/acceptance/test_base.py index 5054dfb5a..5010d7fda 100644 --- a/tests/acceptance/test_base.py +++ b/tests/acceptance/test_base.py @@ -1,4 +1,5 @@ import asyncio +from hashlib import md5 import pytest @@ -7,6 +8,16 @@ from aiocache.serializers import NullSerializer from ..utils import Keys +#: key_builders that do not leave the namespace at the start of the key, so +#: there is no prefix a namespaced clear() could scan for. +NON_PREFIX_KEY_BUILDERS = [ + pytest.param(lambda k, ns: k, id="drops-namespace"), + pytest.param(lambda k, ns: f"{k}{ns}" if ns else k, id="appends-namespace"), + pytest.param( + lambda k, ns: md5(f"{ns}:{k}".encode()).hexdigest(), id="hashes-whole-key" + ), +] + class TestCache: """ @@ -178,17 +189,23 @@ async def test_clear_matches_custom_key_builder_prefix_memory(self): assert await cache.exists(Keys.KEY) is False assert await cache.exists(Keys.KEY, namespace="nsother") is True - async def test_clear_rejects_key_builder_without_namespace_memory(self): - """A builder dropping the namespace makes every key match the prefix, - so clearing one namespace would empty the whole cache. Refuse instead. + @pytest.mark.parametrize("key_builder", NON_PREFIX_KEY_BUILDERS) + async def test_clear_rejects_non_prefix_key_builder_memory(self, key_builder): + """Without the namespace leading the key there is nothing to match on. + + Matching on the derived prefix anyway deletes unrelated keys and keeps + the namespace's own, so clear() has to refuse. """ - cache = SimpleMemoryCache(namespace="ns", key_builder=lambda k, ns: k) + cache = SimpleMemoryCache(namespace="ns", key_builder=key_builder) await cache.set(Keys.KEY, "value") + # Not in the namespace, but shares its leading characters. + cache._cache["ns-foreign"] = "untouched" - with pytest.raises(ValueError, match="empty prefix"): + with pytest.raises(ValueError, match="at the start of the key"): await cache.clear() assert await cache.get(Keys.KEY) == "value" + assert cache._cache["ns-foreign"] == "untouched" async def test_clear_only_removes_own_namespace_memory(self, memory_cache): await memory_cache.set(Keys.KEY, "value") @@ -365,22 +382,23 @@ def key_builder(key, namespace): finally: await cache.delete(Keys.KEY) - async def test_clear_rejects_key_builder_without_namespace_valkey( - self, valkey_config + @pytest.mark.parametrize("key_builder", NON_PREFIX_KEY_BUILDERS) + async def test_clear_rejects_non_prefix_key_builder_valkey( + self, valkey_config, key_builder ): - """A builder dropping the namespace leaves nothing to scope the scan. + """Without the namespace leading the key there is nothing to scan for. - The pattern would be a bare "*", so clear() would delete every key in - the database. It has to refuse instead. + Matching on the derived prefix anyway deletes unrelated keys and keeps + the namespace's own, so clear() has to refuse. """ from aiocache.backends.valkey import ValkeyCache async with ValkeyCache( - valkey_config, namespace="ns", key_builder=lambda k, ns: k + valkey_config, namespace="ns", key_builder=key_builder ) as cache: await cache.set(Keys.KEY, "value") try: - with pytest.raises(ValueError, match="empty prefix"): + with pytest.raises(ValueError, match="at the start of the key"): await cache.clear() assert await cache.get(Keys.KEY) == "value"