Scope clear() to the cache namespace

CHANGES.rst

@@ -17,6 +17,7 @@ There are a number of backwards-incompatible changes. These points should help w
 * The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"``
 * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour).
 * ``BaseCache`` and ``BaseSerializer`` are now ``ABC``s, so cannot be instantiated directly.
+* ``clear()`` now defaults to the cache's own ``namespace`` instead of clearing the whole backend, so a shared server keeps the keys written by other caches. The namespace is matched by the key prefix the ``key_builder`` produces, which means a builder that does not separate the namespace from the key also matches longer namespaces starting with it. A builder that does not place the namespace at the start of the key (one that appends it, hashes it, or ignores it) has no prefix to match on, and now raises ``ValueError`` instead of deleting unrelated keys. ``MemcachedCache`` cannot clear by namespace, so ``clear()`` on a namespaced instance now raises ``ValueError`` rather than flushing the server; call ``clear(namespace="")`` to flush explicitly.
 * If subclassing ``BaseCache`` to implement a custom backend:
 
   * The cache key type used by the backend must now be specified when inheriting (e.g. ``BaseCache[str]`` typically).

aiocache/backends/memory.py

@@ -125,8 +125,13 @@ async def _delete(self, key, _conn=None):
 
     async def _clear(self, namespace=None, _conn=None):
         if namespace:
+            # Match the prefix this cache's key_builder produces. With the
+            # default builder that is the bare namespace, which cannot be told
+            # apart from a longer namespace sharing it; a builder that appends
+            # a separator does not have that ambiguity.
+            prefix = self._namespace_prefix(namespace)
             for key in list(self._cache):
-                if key.startswith(namespace):
+                if key.startswith(prefix):
                     self.__delete(key)
         else:
             self._cache = OrderedDict()

aiocache/backends/valkey.py

@@ -1,4 +1,5 @@
 import logging
+import re
 import sys
 from typing import Optional
 
@@ -15,6 +16,9 @@
 from aiocache.base import BaseCache
 from aiocache.serializers import JsonSerializer
 
+#: Characters SCAN treats as glob syntax, escaped before matching a prefix.
+_GLOB_METACHARACTERS = re.compile(r"([\\*?\[\]])")
+
 if sys.version_info >= (3, 11):
     from typing import Self
 else:
@@ -148,11 +152,25 @@ async def _clear(self, namespace=None, _conn=None):
         if not namespace:
             return await self.client.flushdb()
 
-        _, keys = await self.client.scan(b"0", "{}:*".format(namespace))
-        if keys:
-            return bool(await self.client.delete(keys))
-
-        return True
+        # Match whatever prefix this cache's key_builder produces rather than
+        # assuming the default "<namespace>:<key>" layout, which a custom
+        # key_builder need not follow. Refuses builders that have no such
+        # prefix, whose pattern would match the wrong keys or all of them.
+        prefix = self._namespace_prefix(namespace)
+
+        # Escape the prefix: a namespace holding a glob character would
+        # otherwise match some other namespace's keys and leave its own behind.
+        pattern = _GLOB_METACHARACTERS.sub(r"\\\1", prefix) + "*"
+
+        # SCAN returns one batch per call, so keep going until the cursor comes
+        # back to 0. Stopping at the first batch leaves the rest behind.
+        cursor = b"0"
+        while True:
+            cursor, keys = await self.client.scan(cursor, pattern)
+            if keys:
+                await self.client.delete(keys)
+            if cursor == b"0":
+                return True
 
     async def _raw(self, command, *args, encoding="utf-8", _conn=None, **kwargs):
         value = await getattr(self.client, command)(*args, **kwargs)

aiocache/base.py

@@ -18,6 +18,9 @@
 logger = logging.getLogger(__name__)
 
 SENTINEL = object()
+
+#: Key fed to a key_builder to check the namespace lands at the start of it.
+_PREFIX_PROBE = "\x00aiocache-prefix-probe\x00"
 CacheKeyType = TypeVar("CacheKeyType")
 
 
@@ -447,13 +450,29 @@ async def clear(self, namespace=None, _conn=None):
         Clears the cache in the cache namespace. If an alternative namespace is given, it will
         clear those ones instead.
 
+        Caches configured without a namespace clear the whole backend, which for a shared
+        server means every key, including those written by other caches. Passing an empty
+        namespace asks for that explicitly.
+
+        A namespace is matched by the key prefix the ``key_builder`` produces, so with a
+        builder that does not separate the namespace from the key, a namespace also matches
+        the longer namespaces starting with it.
+
+        This only works if the namespace reaches the start of the key. A ``key_builder``
+        that appends it, hashes it, or drops it has no such prefix, and matching on one
+        anyway would delete unrelated keys while leaving the namespace's own behind, so
+        those are rejected with a :class:`ValueError`.
+
         :param namespace: str alternative namespace to use
         :param timeout: int or float in seconds specifying maximum timeout
             for the operations to last
         :returns: True
         :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout
+        :raises: :class:`ValueError` if the backend cannot clear a single namespace, as
+            :class:`~aiocache.MemcachedCache` cannot
         """
         start = time.monotonic()
+        namespace = namespace if namespace is not None else self.namespace
         ret = await self._clear(namespace, _conn=_conn)
         logger.debug("CLEAR %s %d (%.4f)s", namespace, ret, time.monotonic() - start)
         return ret
@@ -522,6 +541,27 @@ def _str_build_key(self, key: str, namespace: Optional[str] = None) -> str:
         ns = self.namespace if namespace is None else namespace
         return self._build_key(key_name, ns)
 
+    def _namespace_prefix(self, namespace):
+        """Key prefix that scopes a scan to ``namespace``.
+
+        Backends that clear a namespace by matching a key prefix need the
+        ``key_builder`` to put the namespace at the start of the key. That
+        cannot be assumed, so check it: build a key nothing else can collide
+        with and confirm the derived prefix really leads it.
+
+        :raises ValueError: if the ``key_builder`` has no such prefix, since
+            any pattern would then match the wrong keys, or all of them.
+        """
+        prefix = self.build_key("", namespace)
+        if not prefix or not self.build_key(_PREFIX_PROBE, namespace).startswith(prefix):
+            raise ValueError(
+                f"key_builder does not place namespace {namespace!r} at the start "
+                f"of the key, so clear() cannot scope to it without matching "
+                f"unrelated keys. Use a key_builder that prefixes keys with the "
+                f"namespace, or call clear() without a namespace."
+            )
+        return prefix
+
     def _get_ttl(self, ttl):
         return ttl if ttl is not SENTINEL else self.ttl
 

tests/acceptance/test_base.py

@@ -1,4 +1,5 @@
 import asyncio
+from hashlib import md5
 
 import pytest
 
@@ -7,6 +8,16 @@
 from aiocache.serializers import NullSerializer
 from ..utils import Keys
 
+#: key_builders that do not leave the namespace at the start of the key, so
+#: there is no prefix a namespaced clear() could scan for.
+NON_PREFIX_KEY_BUILDERS = [
+    pytest.param(lambda k, ns: k, id="drops-namespace"),
+    pytest.param(lambda k, ns: f"{k}{ns}" if ns else k, id="appends-namespace"),
+    pytest.param(
+        lambda k, ns: md5(f"{ns}:{k}".encode()).hexdigest(), id="hashes-whole-key"
+    ),
+]
+
 
 class TestCache:
     """
@@ -121,12 +132,6 @@ async def test_expire_with_0(self, cache):
     async def test_expire_missing(self, cache):
         assert await cache.expire(Keys.KEY, 1) is False
 
-    async def test_clear(self, cache):
-        await cache.set(Keys.KEY, "value")
-        await cache.clear()
-
-        assert await cache.exists(Keys.KEY) is False
-
     async def test_close_pool_only_clears_resources(self, cache):
         await cache.set(Keys.KEY, "value")
         await cache.close()
@@ -169,6 +174,48 @@ async def test_clear_with_namespace_memory(self, memory_cache):
 
         assert await memory_cache.exists(Keys.KEY, namespace="test") is False
 
+    async def test_clear_matches_custom_key_builder_prefix_memory(self):
+        """clear() must match the prefix the key_builder makes, not the bare
+        namespace, so a builder with its own separator still scopes correctly.
+        """
+        cache = SimpleMemoryCache(
+            namespace="ns", key_builder=lambda k, ns: f"{ns}__{k}" if ns else k
+        )
+        await cache.set(Keys.KEY, "value")
+        await cache.set(Keys.KEY, "other", namespace="nsother")
+
+        await cache.clear()
+
+        assert await cache.exists(Keys.KEY) is False
+        assert await cache.exists(Keys.KEY, namespace="nsother") is True
+
+    @pytest.mark.parametrize("key_builder", NON_PREFIX_KEY_BUILDERS)
+    async def test_clear_rejects_non_prefix_key_builder_memory(self, key_builder):
+        """Without the namespace leading the key there is nothing to match on.
+
+        Matching on the derived prefix anyway deletes unrelated keys and keeps
+        the namespace's own, so clear() has to refuse.
+        """
+        cache = SimpleMemoryCache(namespace="ns", key_builder=key_builder)
+        await cache.set(Keys.KEY, "value")
+        # Not in the namespace, but shares its leading characters.
+        cache._cache["ns-foreign"] = "untouched"
+
+        with pytest.raises(ValueError, match="at the start of the key"):
+            await cache.clear()
+
+        assert await cache.get(Keys.KEY) == "value"
+        assert cache._cache["ns-foreign"] == "untouched"
+
+    async def test_clear_only_removes_own_namespace_memory(self, memory_cache):
+        await memory_cache.set(Keys.KEY, "value")
+        await memory_cache.set(Keys.KEY, "other", namespace="other")
+
+        await memory_cache.clear()
+
+        assert await memory_cache.exists(Keys.KEY) is False
+        assert await memory_cache.exists(Keys.KEY, namespace="other") is True
+
 
 @pytest.mark.memcached
 class TestMemcachedCache:
@@ -208,6 +255,21 @@ async def test_clear_with_namespace_memcached(self, memcached_cache):
 
         assert await memcached_cache.exists(Keys.KEY, namespace="test") is True
 
+    async def test_clear_namespaced_cache_memcached(self, memcached_cache):
+        await memcached_cache.set(Keys.KEY, "value")
+
+        with pytest.raises(ValueError):
+            await memcached_cache.clear()
+
+        assert await memcached_cache.exists(Keys.KEY) is True
+
+    async def test_clear_empty_namespace_flushes_memcached(self, memcached_cache):
+        await memcached_cache.set(Keys.KEY, "value")
+
+        await memcached_cache.clear(namespace="")
+
+        assert await memcached_cache.exists(Keys.KEY) is False
+
     async def test_close(self, memcached_cache):
         await memcached_cache.set(Keys.KEY, "value")
         await memcached_cache._close()
@@ -259,6 +321,104 @@ async def test_clear_with_namespace_valkey(self, valkey_cache):
 
         assert await valkey_cache.exists(Keys.KEY, namespace="test") is False
 
+    async def test_clear_only_removes_own_namespace_valkey(self, valkey_cache):
+        await valkey_cache.set(Keys.KEY, "value")
+        await valkey_cache.set(Keys.KEY, "other", namespace="other")
+
+        try:
+            await valkey_cache.clear()
+
+            assert await valkey_cache.exists(Keys.KEY) is False
+            assert await valkey_cache.exists(Keys.KEY, namespace="other") is True
+        finally:
+            await valkey_cache.delete(Keys.KEY, namespace="other")
+
+    @pytest.mark.parametrize("namespace", ["ten[a]nt", "ten?nt", "ten*"])
+    async def test_clear_escapes_glob_in_namespace_valkey(
+        self, valkey_config, namespace
+    ):
+        """A namespace holding glob syntax must not match another namespace.
+
+        Unescaped, "ten[a]nt:*" is a character class matching "tenant:*", so
+        clear() deleted the neighbour's keys and left its own in place.
+        """
+        from aiocache.backends.valkey import ValkeyCache
+
+        async with ValkeyCache(valkey_config, namespace=namespace) as odd:
+            async with ValkeyCache(valkey_config, namespace="tenant") as neighbour:
+                await odd.set(Keys.KEY, "mine")
+                await neighbour.set(Keys.KEY, "theirs")
+                try:
+                    await odd.clear()
+
+                    assert await odd.exists(Keys.KEY) is False
+                    assert await neighbour.get(Keys.KEY) == "theirs"
+                finally:
+                    await odd.delete(Keys.KEY)
+                    await neighbour.delete(Keys.KEY)
+
+    async def test_clear_matches_custom_key_builder_prefix_valkey(
+        self, valkey_config
+    ):
+        """clear() must match the prefix the key_builder makes, not "<ns>:".
+
+        A builder using its own separator writes "ns__key". Assuming the
+        default layout looks for "ns:*", matches nothing, deletes nothing and
+        still returns True.
+        """
+        from aiocache.backends.valkey import ValkeyCache
+
+        def key_builder(key, namespace):
+            return f"{namespace}__{key}" if namespace else key
+
+        async with ValkeyCache(
+            valkey_config, namespace="ns", key_builder=key_builder
+        ) as cache:
+            await cache.set(Keys.KEY, "value")
+            try:
+                assert await cache.clear() is True
+
+                assert await cache.exists(Keys.KEY) is False
+            finally:
+                await cache.delete(Keys.KEY)
+
+    @pytest.mark.parametrize("key_builder", NON_PREFIX_KEY_BUILDERS)
+    async def test_clear_rejects_non_prefix_key_builder_valkey(
+        self, valkey_config, key_builder
+    ):
+        """Without the namespace leading the key there is nothing to scan for.
+
+        Matching on the derived prefix anyway deletes unrelated keys and keeps
+        the namespace's own, so clear() has to refuse.
+        """
+        from aiocache.backends.valkey import ValkeyCache
+
+        async with ValkeyCache(
+            valkey_config, namespace="ns", key_builder=key_builder
+        ) as cache:
+            await cache.set(Keys.KEY, "value")
+            try:
+                with pytest.raises(ValueError, match="at the start of the key"):
+                    await cache.clear()
+
+                assert await cache.get(Keys.KEY) == "value"
+            finally:
+                await cache.delete(Keys.KEY)
+
+    async def test_clear_removes_keys_beyond_one_scan_batch(self, valkey_cache):
+        keys = [f"{Keys.KEY.value}-{i}" for i in range(2000)]
+        for start in range(0, len(keys), 200):
+            batch = keys[start:start + 200]
+            await asyncio.gather(*(valkey_cache.set(k, "value") for k in batch))
+
+        try:
+            await valkey_cache.clear()
+
+            assert await valkey_cache.exists(keys[0]) is False
+            assert await valkey_cache.exists(keys[-1]) is False
+        finally:
+            await valkey_cache.clear()
+
     async def test_close(self, valkey_cache):
         await valkey_cache.set(Keys.KEY, "value")
         await valkey_cache._close()

tests/ut/backends/test_valkey.py

@@ -200,6 +200,20 @@ async def test_clear_no_keys(self, valkey):
         await valkey._clear("nm")
         valkey.client.delete.assert_not_called()
 
+    async def test_clear_scans_until_cursor_is_exhausted(self, valkey):
+        valkey.client.scan.side_effect = [
+            [b"17", ["nm:a"]],
+            [b"0", ["nm:b"]],
+        ]
+
+        await valkey._clear("nm")
+
+        assert [c.args[0] for c in valkey.client.scan.call_args_list] == [b"0", b"17"]
+        assert [c.args[0] for c in valkey.client.delete.call_args_list] == [
+            ["nm:a"],
+            ["nm:b"],
+        ]
+
     async def test_clear_no_namespace(self, valkey):
         await valkey._clear()
         assert valkey.client.flushdb.call_count == 1

tests/ut/test_base.py

@@ -577,6 +577,28 @@ async def test_clear(self, mock_base_cache):
         assert mock_base_cache.plugins[0].pre_clear.call_count == 1
         assert mock_base_cache.plugins[0].post_clear.call_count == 1
 
+    async def test_clear_uses_cache_namespace(self, mock_base_cache):
+        mock_base_cache.namespace = "ns"
+
+        await mock_base_cache.clear()
+
+        mock_base_cache._clear.assert_called_with("ns", _conn=ANY)
+
+    async def test_clear_namespace_argument_takes_precedence(self, mock_base_cache):
+        mock_base_cache.namespace = "ns"
+
+        await mock_base_cache.clear("other")
+
+        mock_base_cache._clear.assert_called_with("other", _conn=ANY)
+
+    async def test_clear_empty_namespace_clears_everything(self, mock_base_cache):
+        """An empty namespace is a request to clear the backend, not a missing value."""
+        mock_base_cache.namespace = "ns"
+
+        await mock_base_cache.clear("")
+
+        mock_base_cache._clear.assert_called_with("", _conn=ANY)
+
     async def test_clear_timeouts(self, mock_base_cache):
         mock_base_cache._clear = self.asleep