feat: move doveauth from dictproxy to lua/http as recommended by dovecot 2.4

chatmaild/src/chatmaild/config.py

@@ -55,6 +55,7 @@ def __init__(self, inipath, params):
         self.postfix_reinject_port_incoming = int(
             params.pop("postfix_reinject_port_incoming", "10026")
         )
+        self.doveauth_http_port = int(params.pop("doveauth_http_port", "10084"))
         self.mtail_address = params.pop("mtail_address", None)
         self.disable_ipv6 = params.pop("disable_ipv6", "false").lower() == "true"
         self.acme_email = params.pop("acme_email", "")

chatmaild/src/chatmaild/doveauth.py

@@ -1,18 +1,24 @@
-import json
+"""Create chatmail addresses on first login.
+
+Dovecot only asks us about addresses it does not already find in the mailbox:
+the auth.lua we deploy with dovecot (cmdeploy/src/cmdeploy/dovecot/auth.lua.j2)
+verifies existing users itself against a mailbox password file,
+and HTTP-POSTs everything else to the /create endpoint implemented in this module.
+"""
+
 import logging
 import os
 import re
 import sys
-
-import filelock
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 try:
     import crypt_r
 except ImportError:
     import crypt as crypt_r
 
 from .config import Config, read_config
-from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
 from .syslimits import has_sufficient_resources
 
@@ -64,109 +70,117 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
     return True
 
 
-def split_and_unescape(s):
-    """Split strings using double quote as a separator and backslash as escape character
-    into parts."""
-
-    out = ""
-    i = 0
-    while i < len(s):
-        c = s[i]
-        if c == "\\":
-            # Skip escape character.
-            i += 1
-
-            # This will raise IndexError if there is no character
-            # after escape character. This is expected
-            # as this is an invalid input.
-            out += s[i]
-        elif c == '"':
-            # Separator
-            yield out
-            out = ""
-        else:
-            out += c
-        i += 1
-    yield out
-
-
-class AuthDictProxy(DictProxy):
+def verify_password(stored, cleartext_password) -> bool:
+    if stored.startswith("{"):
+        stored = stored.split("}", 1)[1]
+    return crypt_r.crypt(cleartext_password, stored) == stored
+
+
+class DoveAuth:
     def __init__(self, config):
-        super().__init__()
         self.config = config
+        self.creation_lock = threading.Lock()
+
+    def create_user(self, addr, cleartext_password) -> bool:
+        """Create the address, or verify the password if it exists already."""
+        config = self.config
+        if not addr.endswith(f"@{config.mail_domain}"):
+            logging.warning("address not in mail domain: %r", addr)
+            return False
+        try:
+            user = config.get_user(addr)
+        except ValueError:
+            logging.warning("invalid address: %r", addr)
+            return False
+        with self.creation_lock:
+            passhash = user.get_password_hash()
+            if passhash is not None:
+                # a concurrent first login may have just created the address
+                return verify_password(passhash, cleartext_password)
+            if not is_allowed_to_create(config, addr, cleartext_password):
+                return False
+            if not has_sufficient_resources(config):
+                return False
+            user.set_password(encrypt_password(cleartext_password))
+        # mtail counts created_accounts off this exact line
+        print(f"Created address: {addr}", file=sys.stderr)
+        return True
 
-    def handle_lookup(self, parts):
-        # Dovecot <2.3.17 has only one part,
-        # do not attempt to read any other parts for compatibility.
-        keyname = parts[0]
 
-        namespace, type, args = keyname.split("/", 2)
-        args = list(split_and_unescape(args))
+class CreateHandler(BaseHTTPRequestHandler):
+    """Answer POST /create requests from dovecot's auth.lua, body `addr\\tpassword`.
 
-        config = self.config
-        reply_command = "F"
-        res = ""
-        if namespace == "shared":
-            if type == "userdb":
-                user = args[0]
-                if user.endswith(f"@{config.mail_domain}"):
-                    res = self.lookup_userdb(user)
-                if res:
-                    reply_command = "O"
-                else:
-                    reply_command = "N"
-            elif type == "passdb":
-                user = args[1]
-                if user.endswith(f"@{config.mail_domain}"):
-                    res = self.lookup_passdb(user, cleartext_password=args[0])
-                if res:
-                    reply_command = "O"
-                else:
-                    reply_command = "N"
-        json_res = json.dumps(res) if res else ""
-        return f"{reply_command}{json_res}\n"
-
-    def handle_iterate(self, parts):
-        # example: I0\t0\tshared/userdb/
-        if parts[2] == "shared/userdb/":
-            result = "".join(
-                f"Oshared/userdb/{user}\t\n" for user in self.iter_userdb()
-            )
-            return f"{result}\n"
-
-    def iter_userdb(self) -> list:
-        """Get a list of all user addresses."""
-        return [x for x in os.listdir(self.config.mailboxes_dir) if "@" in x]
-
-    def lookup_userdb(self, addr):
-        return self.config.get_user(addr).get_userdb_dict()
-
-    def lookup_passdb(self, addr, cleartext_password):
-        user = self.config.get_user(addr)
-        userdata = user.get_userdb_dict()
-        if userdata:
-            return userdata
-        if not is_allowed_to_create(self.config, addr, cleartext_password):
+    The body must be UTF-8 and only the first tab separates the fields,
+    so a password may itself contain tabs.
+    Any non-UTF8 or \\0 bytes in the body fail the request.
+
+    Addresses are ASCII: dovecot refuses any login name outside its
+    auth_username_chars before auth.lua ever sees it.
+
+    Dovecot hands auth.lua the exact password bytes the client sent;
+    decoding and re-encoding UTF-8 is byte-identical,
+    so dovecot's password_verify later recomputes the same hash crypt() stores here.
+    """
+
+    protocol_version = "HTTP/1.1"  # dovecot's HTTP client reuses connections
+
+    max_body_len = 512  # an address and a password
+
+    def do_POST(self):
+        if self.path != "/create":
+            self.reply(404)
+            return
+        length = self.body_length()
+        if length is None:
+            self.reply(400)
             return
-        if not has_sufficient_resources(self.config):
+        body = self.rfile.read(length)
+        try:
+            addr, _, password = body.decode("utf-8").partition("\t")
+        except UnicodeDecodeError:
+            self.reply(400)
             return
+        if "\0" in addr or "\0" in password:
+            self.reply(400)
+            return
+        self.reply(200 if self.server.doveauth.create_user(addr, password) else 403)
 
-        lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
-        with lock:
-            userdata = user.get_userdb_dict()
-            if userdata:
-                return userdata
-            user.set_password(encrypt_password(cleartext_password))
-            print(f"Created address: {addr}", file=sys.stderr)
-        return user.get_userdb_dict()
+    def body_length(self):
+        try:
+            length = int(self.headers["Content-Length"])
+        except (TypeError, ValueError):
+            return None
+        return length if 0 <= length <= self.max_body_len else None
+
+    def reply(self, status):
+        self.send_response(status)
+        self.send_header("Content-Length", "0")
+        if status != 200:
+            # Just close on any failure, as body might not be fully read.
+            # It's anyway cheap to re-establish http localhost without TLS.
+            self.send_header("Connection", "close")
+        self.end_headers()
+
+    def log_message(self, format, *args):
+        # the per-request access log would only duplicate our own stderr lines
+        pass
+
+
+class DoveAuthServer(ThreadingHTTPServer):
+    # a burst of first-time logins (e.g. from CI) must not overflow
+    # the accept queue, see https://github.com/chatmail/relay/issues/436
+    request_queue_size = 1000
+
+    def __init__(self, config, port):
+        super().__init__(("127.0.0.1", port), CreateHandler)
+        self.doveauth = DoveAuth(config)
 
 
 def main():
-    socket, cfgpath = sys.argv[1:]
+    (cfgpath,) = sys.argv[1:]
     config = read_config(cfgpath)
 
     migrate_from_db_to_maildir(config)
 
-    dictproxy = AuthDictProxy(config=config)
-
-    dictproxy.serve_forever_from_socket(socket)
+    server = DoveAuthServer(config, config.doveauth_http_port)
+    server.serve_forever()

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,6 +1,6 @@
 import time
 
-from chatmaild.doveauth import AuthDictProxy
+from chatmaild.doveauth import DoveAuth
 from chatmaild.expire import daily_expire_main as main_expire
 
 
@@ -18,10 +18,10 @@ def test_login_timestamps(example_config):
 def test_delete_inactive_users(example_config):
     new = time.time()
     old = new - (example_config.delete_inactive_users_after * 86400) - 1
-    dictproxy = AuthDictProxy(example_config)
+    doveauth = DoveAuth(example_config)
 
     def create_user(addr, last_login):
-        dictproxy.lookup_passdb(addr, "q9mr3faue")
+        doveauth.create_user(addr, "q9mr3faue")
         user = example_config.get_user(addr)
         user.maildir.joinpath("cur").mkdir()
         user.maildir.joinpath("cur", "something").mkdir()