chore(cmdeploy/dovecot): add multi-dist/Debian trixie support

.github/workflows/docker-dispatch.yaml

@@ -8,7 +8,7 @@ name: Trigger Docker build
 
 on:
   push:
-    branches: [main]
+    branches: [main, j4n/dovecot-multidist]
     tags: ['[0-9]+.[0-9]+.[0-9]+']
   workflow_dispatch:
 

cmdeploy/src/cmdeploy/deployers.py

@@ -99,6 +99,23 @@ def _install_remote_venv_with_chatmaild(deployer) -> None:
         dest=remote_dist_file,
     )
 
+    # Remove venv if its Python major.minor doesn't match the system Python
+    server.shell(
+        name="remove stale chatmaild venv if python version changed",
+        commands=[
+            "\n".join(
+                [
+                    r"re='[0-9]+\.[0-9]+'",  # major.minor out of 'Python X.Y.Z'
+                    'sys_version=$(python3 --version 2>/dev/null | grep -oE "$re")',
+                    f'venv_version=$({remote_venv_dir}/bin/python --version 2>/dev/null | grep -oE "$re")',
+                    # an empty sys_version means we could not tell: keep the venv
+                    f'[ -z "$sys_version" ] || [ "$sys_version" = "$venv_version" ] '
+                    f"|| rm -rf {remote_venv_dir}",
+                ]
+            )
+        ],
+    )
+
     pip.virtualenv(
         name=f"chatmaild virtualenv {remote_venv_dir}",
         path=remote_venv_dir,
@@ -400,6 +417,12 @@ def install(self):
             src=BytesIO(b'APT::Install-Recommends "false";\n'),
             dest="/etc/apt/apt.conf.d/00InstallRecommends",
         )
+        # Pin dovecot-* to priority -1 before any apt operation, apt should
+        # never manage dovecot as our version might be lower than the distro's.
+        self.put_file(
+            src=StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
+            dest="/etc/apt/preferences.d/pin-dovecot",
+        )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,11 +1,10 @@
-import io
 import urllib.request
 
 from chatmaild.config import Config
 from pyinfra import host
 from pyinfra.facts.deb import DebPackages
 from pyinfra.facts.server import Arch, Command, Sysctl
-from pyinfra.operations import apt, files, server
+from pyinfra.operations import files, server
 
 from cmdeploy.basedeploy import (
     Deployer,
@@ -16,7 +15,13 @@
 )
 from cmdeploy.pins import DOVECOT_SHA256, DOVECOT_VERSION
 
-DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_VERSION}"
+VERSION_ID_CMD = "grep '^VERSION_ID=' /etc/os-release"
+
+
+def _stamped_version(deb_release: int) -> str:
+    """Version as built, including the per-distro suffix stamped by
+    chatmail/dovecot CI into package version and filename."""
+    return f"{DOVECOT_VERSION}+deb{deb_release}u1"
 
 
 class DovecotDeployer(Deployer):
@@ -29,34 +34,30 @@ def __init__(self, config, disable_mail):
 
     def install(self):
         arch = host.get_fact(Arch)
+        deb_release = _parse_version_id(host.get_fact(Command, VERSION_ID_CMD))
         with blocked_service_startup():
             debs = []
             for pkg in ("core", "imapd", "lmtpd", "auth-lua"):
-                deb, changed = _download_dovecot_package(pkg, arch)
+                deb, changed = _download_dovecot_package(pkg, arch, deb_release)
                 self.need_restart |= changed
                 if deb:
                     debs.append(deb)
             if debs:
                 deb_list = " ".join(debs)
-                # First dpkg may fail on missing dependencies (stderr suppressed);
-                # apt-get --fix-broken pulls them in, then dpkg retries cleanly.
+                # apt-get install with local .deb paths resolves depends
+                # against the configured repos (e.g. pulls libwrap0),
+                # The pin file written earlier by ChatmailDeployer prevents apt
+                # from installing a 'wrong' version
                 server.shell(
                     name="Install dovecot packages",
                     commands=[
-                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
-                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
-                        f"dpkg --force-confdef --force-confold -i {deb_list}",
+                        "DEBIAN_FRONTEND=noninteractive apt-get install -y "
+                        '-o Dpkg::Options::="--force-confdef" '
+                        '-o Dpkg::Options::="--force-confold" '
+                        f"--allow-downgrades {deb_list}",
                     ],
                 )
                 self.need_restart = True
-        self.put_file(
-            src=io.StringIO(
-                "Package: dovecot-*\n"
-                "Pin: version *\n"
-                "Pin-Priority: -1\n"
-            ),
-            dest="/etc/apt/preferences.d/pin-dovecot",
-        )
 
     def configure(self):
         configure_remote_units(self, self.config.mail_domain_bare, self.units)
@@ -69,7 +70,7 @@ def activate(self):
         if not self.disable_mail and not self.need_restart:
             stale = host.get_fact(
                 Command,
-                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
+                "pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
                 ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
                 " && echo STALE || true",
             )
@@ -84,6 +85,15 @@ def activate(self):
         )
 
 
+def _parse_version_id(version_line: str) -> int:
+    """Debian major release from an /etc/os-release VERSION_ID line."""
+    _, _, raw = (version_line or "").strip().partition("=")
+    try:
+        return int(raw.strip('"'))
+    except ValueError:
+        raise ValueError(f"cannot determine Debian release from {version_line!r}")
+
+
 def _pick_url(primary, fallback):
     try:
         req = urllib.request.Request(primary, method="HEAD")
@@ -93,27 +103,36 @@ def _pick_url(primary, fallback):
         return fallback
 
 
-def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
+def _download_dovecot_package(package: str, arch: str, deb_release: int) -> tuple[str | None, bool]:
     """Download a dovecot .deb if needed, return (path, changed)."""
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
 
     pkg_name = f"dovecot-{package}"
-    sha256 = DOVECOT_SHA256.get((package, arch))
-    if sha256 is None:
-        op = apt.packages(packages=[pkg_name])
-        return None, bool(getattr(op, "changed", False))
+    try:
+        # never fall back to the distro package: it is pinned to -1 and would
+        # in any case be a version we did not build and do not support
+        sha256 = DOVECOT_SHA256[(arch, deb_release, package)]
+    except KeyError:
+        raise ValueError(f"no dovecot build for {pkg_name} on deb{deb_release}/{arch}")
 
+    stamped_version = _stamped_version(deb_release)
     installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
-    if DOVECOT_PACKAGE_VERSION in installed_versions:
+    if f"1:{stamped_version}" in installed_versions:
         return None, False
 
-    url_version = DOVECOT_VERSION.replace("+", "%2B")
-    deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
-    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
-    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
+    # Primary URL: flat structure with distro suffix in filename
+    primary_deb = f"{pkg_name}_{stamped_version}_{arch}.deb"
+    primary_url = f"https://download.delta.chat/dovecot/{primary_deb}"
+    # GitHub release files: escaped + in filename; the release tag stays
+    # distro-neutral, both distros ship in one combined release
+    tag_version = DOVECOT_VERSION.replace("+", "%2B")
+    fallback_deb = f"{pkg_name}_{stamped_version.replace('+', '%2B')}_{arch}.deb"
+    fallback_url = (
+        f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{tag_version}/{fallback_deb}"
+    )
     url = _pick_url(primary_url, fallback_url)
-    deb_filename = f"/root/{deb_base}"
+    deb_filename = f"/root/{primary_deb}"
 
     files.download(
         name=f"Download {pkg_name}",
@@ -125,6 +144,7 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
 
     return deb_filename, True
 
+
 def _configure_dovecot(deployer, config: Config, debug: bool = False):
     """Configures Dovecot IMAP server."""
     deployer.put_template(

cmdeploy/src/cmdeploy/pins.py

@@ -28,16 +28,25 @@
     ),
 }
 
-DOVECOT_VERSION = "2.3.21+dfsg1-3"
+# distro-neutral base version, as committed in chatmail/dovecot debian/changelog
+DOVECOT_VERSION = "2.3.21+dfsg1-3+chatmail2"
 DOVECOT_SHA256 = {
-    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
-    ("auth-lua", "amd64"): "d724f37712faba52e177153114af1831e54da555c57c6474c05f96f176176ce4",
-    ("auth-lua", "arm64"): "7272768e20de148c35891d99cd60205acbe7e732b056caf0a83e8194d49136e6",
+    ("amd64", 12, "auth-lua"): "ef1b8e1db45147a74b48d63125bd61b2cc2f250e1006656ba1c58b9c12f5cde6",
+    ("arm64", 12, "auth-lua"): "c1a06ee9374439893e397ba3b0cacf532a733290c184f4f30ea39df8699be329",
+    ("amd64", 13, "auth-lua"): "6c0946d2516efcbcaa09a27df9b8ea701861cce270d71941056b3c69831a5ea2",
+    ("arm64", 13, "auth-lua"): "5e6c9cfe47f7f3b8aa0d68a3e607161b6abaee1ad696cb1419e997b3099b2985",
+    ("amd64", 12, "core"): "ac3977264d9b9a6fcec53fd3f5cdd2a79ca8aa0324de530c07e535008540826e",
+    ("arm64", 12, "core"): "21626c9c9b52cbdcf1a17b5c09e3c4043e69aa371bf83cc2fcb3b7ddaecdc109",
+    ("amd64", 13, "core"): "47c242ef23c17e700ac19d52d82c9fdb2ebd757d8beb3a7f6781d2de59f87bd0",
+    ("arm64", 13, "core"): "c14c53f112c875f698c4cb6e5870c605cd0a9dd98d35a66e94ceb1827f8020a3",
+    ("amd64", 12, "imapd"): "92a7ab5fc7dc32886a0c34404f919f1335d397b48c467e0c1ef77e56978f60ea",
+    ("arm64", 12, "imapd"): "9369fd566fec4df109ef23debf34ea0417ae85beb29cbe7de619d4d1f31b120c",
+    ("amd64", 13, "imapd"): "e38cc1266455f937ed62f971ea859c47e1a99247841ed0ad946963b524cfdbc5",
+    ("arm64", 13, "imapd"): "11d97dabf23171b37f8b1335dfdb81d408f8b95391aea6d4066aecc9fde01dfe",
+    ("amd64", 12, "lmtpd"): "dc3de473789969f7dd3504ac8783da5e42a446d2d7a305a4e9d7081a6dfe71ab",
+    ("arm64", 12, "lmtpd"): "ae2cbd6c5c43f6d8e2172997b055448f4c79238e2f99cd9ab9200a7d9f548908",
+    ("amd64", 13, "lmtpd"): "833b243e28c7baff141ecf37456e310f5d836e7944a3b9f2fe5074adf0d6a418",
+    ("arm64", 13, "lmtpd"): "55af47a121ba7e23966b20ddaab2dff7feba4b34677864e045e31a702afa180d",
 }
 TURN_VERSION = "v0.4"
 TURN_ARTIFACTS = {