From f49a07eedbaa50a45c38b774365a3ae6a2010a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Sat, 5 Sep 2026 17:34:49 +0300 Subject: [PATCH 01/23] initial luks version --- LUKS.md | 13 ++ README.md | 85 +++++++++--- agentd/agentlib.py | 31 ++++- agentd/api.py | 36 +++-- agentd/run.py | 4 +- agentd/tests/test_agentlib.py | 54 ++++---- agentd/tests/test_api.py | 83 +++++++++++- agentd/tests/test_appdata.py | 56 ++++++++ appdata.sh | 82 ++++++++++++ install-release.sh | 50 +++++++ logrotate/deploy-manager19 | 8 ++ nginx/nginx.conf | 4 +- .../{00_agent.conf => 00_agent19.conf} | 8 +- nginx/sites-enabled/odoo.conf | 15 +-- setup-appdata.sh | 126 ++++++++++++++++++ systemd/system/deploy-manager.service | 15 --- systemd/system/deploy-manager19.service | 22 +++ systemd/system/odoo-app.target | 4 + systemd/system/rclone-mount.service | 12 +- ubuntu-install.sh | 81 +++++++---- unlock-appdata | 29 ++++ update.sh | 5 +- 22 files changed, 686 insertions(+), 137 deletions(-) create mode 100644 agentd/tests/test_appdata.py create mode 100644 appdata.sh create mode 100755 install-release.sh create mode 100644 logrotate/deploy-manager19 rename nginx/sites-enabled/{00_agent.conf => 00_agent19.conf} (85%) create mode 100755 setup-appdata.sh delete mode 100644 systemd/system/deploy-manager.service create mode 100644 systemd/system/deploy-manager19.service create mode 100644 systemd/system/odoo-app.target create mode 100755 unlock-appdata diff --git a/LUKS.md b/LUKS.md index b9bdf4d..92de1ba 100644 --- a/LUKS.md +++ b/LUKS.md @@ -47,6 +47,8 @@ Hetzner Volume ├── odoo-config ├── rclone-config ├── rclone-cache + ├── nginx-temp + ├── tmp ├── secrets └── logs ├── nginx @@ -64,6 +66,7 @@ Bind-mount the encrypted directories to the conventional host paths: /srv/secure/rclone-cache → /root/.cache/rclone /srv/secure/logs/nginx → /var/log/nginx /srv/secure/logs/postgresql → /var/log/postgresql +/srv/secure/nginx-temp → /var/lib/nginx ``` Store `cloudflare.ini` under `/srv/secure/secrets` and configure the deployment @@ -80,6 +83,9 @@ The following remain on the unencrypted root filesystem: Do not store customer data in `/tmp` or `/var/tmp`. Disable persistent swap so that application memory cannot be written unencrypted to disk. +Agent subprocesses use `/srv/secure/tmp`; PostgreSQL has a private subdirectory. +Nginx buffers request bodies and upstream responses under `/var/lib/nginx`. +Agent output goes to `/srv/secure/logs/deploy-manager19.log` with weekly rotation. ## One-time fresh-server bootstrap @@ -189,6 +195,13 @@ bootstrap invokes the release installer for Odoo 19. On a server that is already configured with LUKS, a future Odoo release runs only its release installer. +The implementation uses `setup-appdata.sh` for volume provisioning, +`appdata.sh` for shared mount checks, and `install-release.sh` for release setup. +The first release setup leaves SSL sites disabled until the credentials and +wildcard certificate exist. Follow README.md, rerun `install-release.sh`, then +run `unlock-appdata`. Use `unlock-appdata --mount-only` to access storage without +starting services and `unlock-appdata --check` for a noninteractive check. + The 19.0 branch hardcodes the release values. It must: - Verify that it is running from `/opt/odoo19`. diff --git a/README.md b/README.md index 9baba74..f65d973 100644 --- a/README.md +++ b/README.md @@ -98,33 +98,76 @@ flowchart LR ## Installation #### Prerequisite -* `scp .gitconfig agent19.eniemela.fi:` + +Use a fresh Ubuntu LTS server and a separate empty Hetzner Volume. Attach the +volume without formatting or automatically mounting it. The installer refuses +existing application data; this is not an in-place migration. See [LUKS.md](LUKS.md) +for the storage layout, recovery precautions, and acceptance tests. + +Point `odoo19.eniemela.fi` to this server and allow TCP 9019 in the host and +Hetzner firewalls. The agent is available at `https://odoo19.eniemela.fi:9019` +through nginx basic authentication; its backend is loopback-only on port 8019. + +* `scp .gitconfig odoo19.eniemela.fi:` * `cd .ssh && ssh-keygen -f id_ecdsa -t ecdsa -b 521` * `cat id_ecdsa.pub` * go to github / settings / SSH keys / Add 'Odoo 19.0 Hetzner Server key' * https://github.com/settings/keys #### Installation -* `git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/deploy-manager` -* `cd /opt/deploy-manager` -* `./ubuntu-install.sh` + +Run the following as root. Formatting requests confirmation and a LUKS passphrase +interactively; the passphrase is never stored on the server. + +* `git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/odoo19` +* `cd /opt/odoo19` +* `lsblk -f` — identify the empty Hetzner Volume and its `/dev/disk/by-id/` path. +* `bash ./ubuntu-install.sh /dev/disk/by-id/` +* Copy `/root/appdata-luks-header-.img` to offline storage, verify the copy, then delete the server copy. Keep the passphrase separately. * `htpasswd -B -C 12 -c /etc/nginx/.htpasswd cloud` # Use bcrypt (-B) with cost 12 -* `chmod 600 /etc/nginx/.htpasswd` +* `chown root:www-data /etc/nginx/.htpasswd && chmod 640 /etc/nginx/.htpasswd` * `vim /root/.config/rclone/rclone.conf` -* `vim /root/cloudflare.ini` -* `systemctl edit docker.service` -``` -[Service] -ExecStart= -ExecStart=/usr/bin/dockerd -H fd:// -H tcp://127.0.0.1:2375 --containerd=/run/containerd/containerd.sock -``` -* `systemctl daemon-reload` -* `systemctl restart docker.service` -* `systemctl restart postgresql` -* `su - postgres -c "createuser -s root"` -* `source /root/agent-venv/bin/activate` +* `vim /srv/secure/secrets/cloudflare.ini` +* `source /root/agent-venv19/bin/activate` +* `export TMPDIR=/srv/secure/tmp` * `python -m agentd.api ssl_wildcard` -* `systemctl restart nginx` +* `bash ./install-release.sh` — validate and enable the SSL sites after creating the certificate. +* `unlock-appdata` +* `su - postgres -c "createuser -s root"` — first installation only. + +The bootstrap installs Docker's loopback API override automatically. Configure +the Loki Docker logging plugin below before creating Odoo containers. + +After every reboot, Ubuntu and SSH are available but application services stay +stopped. Run `sudo unlock-appdata` to unlock the volume, verify all encrypted bind +mounts, and start PostgreSQL, Docker/containerd, nginx, rclone, and all installed +agents. A wrong passphrase or missing mount prevents startup. Use +`sudo unlock-appdata --mount-only` when you need to edit credentials before starting +services, or `sudo unlock-appdata --check` to verify the mounted storage. + +The encrypted filesystem also contains nginx request-body temporary files, +agent temporary files and rotating agent logs. Do not use unencrypted `/tmp` or +`/var/tmp` for database dumps. Persistent swap is disabled. Native swap units or +generators must be removed if the installer reports them. + +For a host-bootstrap rerun, stop `odoo-app.target` first. Reuse the same configured +volume; the script preserves its filesystem and secrets and rejects conflicting +fstab entries. Interrupted setup may leave runtime service masks; rerun the host +bootstrap to finish configuring guards and remove the masks. + +#### Adding Odoo 20 later + +Create a 20.0 branch and replace release-specific 19 values with 20, including the +service/logrotate filenames. Clone that branch to `/opt/odoo20`, unlock the host, +and run only `bash ./install-release.sh` from the new clone. Configure DNS and +TCP 9020 for `https://odoo20.eniemela.fi:9020`; the backend uses `127.0.0.1:8020`. +Run `unlock-appdata` again to start newly installed agents. + +Each agent discovers and backs up only containers with its release's +`odoo.version` label. UIDs and HTTP/gevent ports must be unique across the entire +server, including stopped containers. Both releases share the PostgreSQL cluster, +backup remote and nginx routing maps. Static files are served by the selected +Odoo container. Updating one clone with `./update.sh` restarts only its agent. #### Promtail setup (TODO: deprecated, migrate to Alloy) * Promtail is an agent which ships the contents of local logs to a private Grafana Loki instance: https://grafana.com/docs/loki/latest/send-data/promtail/ @@ -157,7 +200,7 @@ ExecStart=/usr/bin/dockerd -H fd:// -H tcp://127.0.0.1:2375 --containerd=/run/co * `docker login ghcr.io -u elmeriniemela` #### Building the image -* `docker build -t ghcr.io/elmeriniemela/odoo-src:19.0 /opt/deploy-manager` +* `docker build -t ghcr.io/elmeriniemela/odoo-src:19.0 /opt/odoo19` * https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#building-container-images * `docker push ghcr.io/elmeriniemela/odoo-src:19.0` @@ -181,7 +224,7 @@ ExecStart=/usr/bin/dockerd -H fd:// -H tcp://127.0.0.1:2375 --containerd=/run/co * `systemctl restart rclone-mount.service` * Scheduled cron: * `crontab -e` - * `30 00 * * * cd /opt/deploy-manager && /root/agent-venv/bin/python -m agentd.backup` + * `30 00 * * * /usr/local/sbin/unlock-appdata --check && cd /opt/odoo19 && TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1` ##### Copying existing unencrypted backups to the new encrypted bucket: If you have existing plaintext backups in `odoobackup1` and wish to copy them into the new encrypted bucket: @@ -196,7 +239,7 @@ To manually decrypt a downloaded file without rclone (using only Python and `pip * `python3 docs/decrypt.py ` #### Clone modules -* `cd /opt/deploy-manager/src` +* `cd /opt/odoo19/src` * `git clone -b 19.0 git@github.com:elmeriniemela/tabularium.git` * `git clone -b 19.0 --depth=1 --single-branch git@github.com:odoo/odoo.git` * `git clone -b 19.0 --depth=1 --single-branch git@github.com:OCA/OpenUpgrade.git` diff --git a/agentd/agentlib.py b/agentd/agentlib.py index 2385d69..5da5158 100644 --- a/agentd/agentlib.py +++ b/agentd/agentlib.py @@ -10,12 +10,14 @@ from contextlib import contextmanager from psycopg2.extras import LoggingConnection import datetime +import fcntl +import tempfile _logger = logging.getLogger(__name__) def register(func): def wraps(*args, **kwargs): #XML-RPC doesn't have a concept of 'keyword arguments' - _logger.debug(f"Call {func.__name__}: {args=}, {kwargs=}") + _logger.debug("Call %s", func.__name__) return func(*args, **kwargs) wraps._rpc = True wraps.__name__ = func.__name__ @@ -79,7 +81,7 @@ def parse_version(s): def load_nginx_map(fname): mapping = {} - with open(f'/etc/nginx/conf.d/{fname}') as fp: + with open(os.path.join('/etc/nginx/conf.d', fname)) as fp: lines = fp.readlines() [(match, target)] = re.findall(r'map\s+(\$[\w]+)\s+(\$[\w]+)\s+{', lines[0]) @@ -95,8 +97,24 @@ def store_nginx_map(fname, match, target, mapping): template = Template(fp.read(), keep_trailing_newline=True) conf = template.render(mapping=mapping, match=match, target=target) - with open(f'/etc/nginx/conf.d/{fname}', 'w') as fp: - fp.write(conf) + path = os.path.join('/etc/nginx/conf.d', fname) + with tempfile.NamedTemporaryFile(mode='w', dir=os.path.dirname(path), delete=False) as fp: + try: + fp.write(conf) + fp.flush() + os.fchmod(fp.fileno(), 0o644) + os.replace(fp.name, path) + finally: + if os.path.exists(fp.name): + os.unlink(fp.name) + + +@contextmanager +def nginx_lock(): + # A separate open per call also serializes threads in one agent process. + with open('/run/lock/odoo-nginx.lock', 'a') as fp: + fcntl.flock(fp, fcntl.LOCK_EX) + yield def render_odoo_config(uid, pw, modules): @@ -146,7 +164,8 @@ def odoo_docker_run(uid, http_port, gevent_port): '--log-opt', 'loki-tls-insecure-skip-verify=true', '--log-opt', 'keep-file=true', '--log-opt', 'loki-batch-size=400', - '-v', f'/opt/deploy-manager/src:/mnt:ro', + '--label', 'odoo.version=19.0', + '-v', '/opt/odoo19/src:/mnt:ro', '-v', f'/var/run/postgresql/:/var/run/postgresql/', '-v', f'/etc/odoo/{uid}:/etc/odoo:ro', '-v', f'{uid}:/var/lib/odoo', @@ -182,6 +201,8 @@ def list_instances(): ).json() for container in docker_ps_a: + if container.get('Labels', {}).get('odoo.version') != '19.0': + continue uid = container['Names'][0].lstrip('/') try: validate(uid=uid) diff --git a/agentd/api.py b/agentd/api.py index 305b738..eaaf52a 100644 --- a/agentd/api.py +++ b/agentd/api.py @@ -91,6 +91,7 @@ def module_pull(module, checkout): def backup(uid, trigger='manual'): _logger.info(f"Starting {trigger} backup for {uid}") agentlib.validate(uid=uid) + agentlib.ensure_backups_mounted() fname = agentlib.ts_to_fname(datetime.datetime.now(datetime.timezone.utc)) agentlib.execute(['pg_dump', '--no-owner', '-Fc', '-f', agentlib.dump_path(uid, trigger, fname, makedirs=True), uid]) agentlib.execute([ @@ -232,7 +233,7 @@ def ssl_wildcard(): # https://www.bjornjohansen.com/wildcard-certificate-letsencrypt-cloudflare agentlib.execute([ 'certbot', 'certonly', '--dns-cloudflare', - '--dns-cloudflare-credentials', '/root/cloudflare.ini', + '--dns-cloudflare-credentials', '/srv/secure/secrets/cloudflare.ini', '-d', '*.eniemela.fi', '-d', 'eniemela.fi', '--expand', @@ -245,7 +246,9 @@ def ssl_wildcard(): @agentlib.register def ssl_renew(): proc = agentlib.execute(['certbot', 'renew']) - agentlib.execute(['systemctl', 'reload', 'nginx']) + with agentlib.nginx_lock(): + agentlib.execute(['nginx', '-t']) + agentlib.execute(['systemctl', 'reload', 'nginx']) return '\n'.join([proc.stderr or '', proc.stdout or '']).strip() @agentlib.register @@ -343,17 +346,24 @@ def config(uid, conf): def sync_urls(hostnames, http_port, gevent_port): agentlib.validate(hostnames=hostnames) # ports are not checked, as the instance may be running and ports binded. - for port, fname in [(gevent_port, 'gevent-ports.conf'), (http_port, 'http-ports.conf')]: - match, target, mapping = agentlib.load_nginx_map(fname) - # Remove old ones - mapping = {d: p for d, p in mapping.items() if int(p.split(':')[-1]) != int(port)} - # Add new ones - for hostname in hostnames: - mapping[hostname] = f'127.0.0.1:{port}' - - agentlib.store_nginx_map(fname, match, target, mapping) - - agentlib.execute(['systemctl', 'reload', 'nginx']) + with agentlib.nginx_lock(): + previous = {} + try: + for port, fname in [(gevent_port, 'gevent-ports.conf'), (http_port, 'http-ports.conf')]: + match, target, mapping = agentlib.load_nginx_map(fname) + previous[fname] = match, target, mapping + # Remove old ones + mapping = {d: p for d, p in mapping.items() if int(p.split(':')[-1]) != int(port)} + # Add new ones + for hostname in hostnames: + mapping[hostname] = f'127.0.0.1:{port}' + agentlib.store_nginx_map(fname, match, target, mapping) + agentlib.execute(['nginx', '-t']) + agentlib.execute(['systemctl', 'reload', 'nginx']) + except Exception: + for fname, values in previous.items(): + agentlib.store_nginx_map(fname, *values) + raise @agentlib.register diff --git a/agentd/run.py b/agentd/run.py index 8136245..f678de0 100644 --- a/agentd/run.py +++ b/agentd/run.py @@ -19,8 +19,8 @@ class AgentServer(socketserver.ThreadingMixIn, xmlrpc.server.SimpleXMLRPCServer) def main(argv=None): parser = argparse.ArgumentParser(description='Agent Server') - parser.add_argument("--interface", dest="interface", type=str, default='localhost') - parser.add_argument("--port", dest="port", type=int, default=8000) + parser.add_argument("--interface", dest="interface", type=str, default='127.0.0.1') + parser.add_argument("--port", dest="port", type=int, default=8019) parser.add_argument("--logfile", dest="logfile", type=pathlib.Path) args = parser.parse_args(argv) diff --git a/agentd/tests/test_agentlib.py b/agentd/tests/test_agentlib.py index 84dce29..6a403c3 100644 --- a/agentd/tests/test_agentlib.py +++ b/agentd/tests/test_agentlib.py @@ -1,6 +1,8 @@ import datetime import subprocess import unittest +import tempfile +from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, call, mock_open, patch @@ -18,6 +20,11 @@ def sample(a, b=None): self.assertEqual(wrapped.__name__, "sample") self.assertEqual(wrapped("value", b="keyword"), ("value", "keyword")) + def test_register_does_not_log_argument_values(self): + with patch('agentd.agentlib._logger.debug') as debug: + agentlib.register(lambda conf: None)('db_password=secret') + self.assertNotIn('secret', str(debug.call_args_list)) + class VersionTests(unittest.TestCase): def test_parse_version_orders_release_candidates_before_final(self): @@ -133,8 +140,10 @@ def test_list_backups_extracts_metadata_from_dump_paths(self): class InventoryTests(unittest.TestCase): def test_list_instances_skips_invalid_container_names_and_adds_inspect_data(self): containers = [ - {"Names": ["/1a2b"], "Id": "container-1"}, - {"Names": ["/not-hex"], "Id": "container-2"}, + {"Names": ["/1a2b"], "Id": "container-1", "Labels": {"odoo.version": "19.0"}}, + {"Names": ["/not-hex"], "Id": "container-2", "Labels": {"odoo.version": "19.0"}}, + {"Names": ["/2b3c"], "Id": "container-3", "Labels": {"odoo.version": "20.0"}}, + {"Names": ["/3c4d"], "Id": "container-4"}, ] responses = [ SimpleNamespace(json=lambda: containers), @@ -163,6 +172,7 @@ def test_list_instances_skips_invalid_container_names_and_adds_inspect_data(self "docker": { "Names": ["/1a2b"], "Id": "container-1", + "Labels": {"odoo.version": "19.0"}, "inspect": {"State": {"Running": True}}, }, "backups": [{"fname": "dump.pgc"}], @@ -293,6 +303,8 @@ def test_odoo_docker_run_builds_expected_container_command(self): self.assertIn("127.0.0.1:8069:8069", cmd) self.assertIn("[::1]:8072:8072", cmd) self.assertEqual(cmd[-1], "ghcr.io/elmeriniemela/odoo-src:19.0") + self.assertEqual(cmd[cmd.index('--label') + 1], 'odoo.version=19.0') + self.assertIn('/opt/odoo19/src:/mnt:ro', cmd) def test_execute_wraps_subprocess_errors(self): error = subprocess.CalledProcessError( @@ -435,30 +447,20 @@ def test_store_nginx_map_renders_template_to_nginx_config(self): "{%- endfor %}\n" "}\n" ) - template_handle = mock_open(read_data=template).return_value - output_handle = mock_open().return_value - - with patch("builtins.open") as opened: - opened.side_effect = [template_handle, output_handle] - agentlib.store_nginx_map( - "http-ports.conf", - "$http_host", - "$odoo_http_port", - {"example.test": "127.0.0.1:8069"}, - ) - - self.assertEqual( - opened.call_args_list, - [ - call("templates/nginxmap.conf"), - call("/etc/nginx/conf.d/http-ports.conf", "w"), - ], - ) - output_handle.write.assert_called_once_with( - "map $http_host $odoo_http_port {\n" - " example.test 127.0.0.1:8069;\n" - "}\n" - ) + with tempfile.TemporaryDirectory() as directory: + # An absolute filename selects a disposable directory while still + # exercising the real tempfile, chmod and atomic replacement. + path = Path(directory) / 'http-ports.conf' + with patch("builtins.open", mock_open(read_data=template)): + agentlib.store_nginx_map( + str(path), "$http_host", "$odoo_http_port", + {"example.test": "127.0.0.1:8069"}, + ) + self.assertEqual(path.read_text(), + "map $http_host $odoo_http_port {\n" + " example.test 127.0.0.1:8069;\n}\n") + self.assertEqual(path.stat().st_mode & 0o777, 0o644) + self.assertEqual(list(Path(directory).iterdir()), [path]) if __name__ == "__main__": # pragma: no cover diff --git a/agentd/tests/test_api.py b/agentd/tests/test_api.py index 5f599f1..f1246ee 100644 --- a/agentd/tests/test_api.py +++ b/agentd/tests/test_api.py @@ -1,6 +1,10 @@ import datetime import unittest -from contextlib import contextmanager +import tempfile +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager, nullcontext from types import SimpleNamespace from unittest.mock import MagicMock, call, patch @@ -180,7 +184,8 @@ def test_module_diff_fetches_diff_inside_module_and_filters_patterns(self): class BackupTests(unittest.TestCase): - def test_backup_dumps_database_and_syncs_filestore(self): + @patch('agentd.api.agentlib.ensure_backups_mounted') + def test_backup_dumps_database_and_syncs_filestore(self, ensure_mounted): with patch("agentd.api.agentlib.validate") as validate: with patch("agentd.api.agentlib.ts_to_fname", return_value="dump.pgc"): with patch( @@ -194,6 +199,7 @@ def test_backup_dumps_database_and_syncs_filestore(self): result = api.backup("1a2b", trigger="manual") validate.assert_called_once_with(uid="1a2b") + ensure_mounted.assert_called_once_with() dump_path.assert_called_once_with("1a2b", "manual", "dump.pgc", makedirs=True) self.assertEqual(result, {"backups": [{"fname": "dump.pgc"}]}) self.assertEqual( @@ -223,6 +229,13 @@ def test_backup_dumps_database_and_syncs_filestore(self): ], ) + def test_backup_refuses_to_write_without_rclone_mount(self): + with patch('agentd.agentlib.backups_mounted', return_value=False): + with patch('agentd.agentlib.execute') as execute: + with self.assertRaises(AssertionError): + api.backup('1a2b') + execute.assert_not_called() + def test_fshealth_runs_one_way_rclone_check_and_returns_stderr(self): with patch("agentd.api.agentlib.validate") as validate: with patch( @@ -454,13 +467,16 @@ def test_ssl_wildcard_runs_cloudflare_certbot_command(self): self.assertIn("-d", command) self.assertIn("*.eniemela.fi", command) self.assertIn("--dns-cloudflare-propagation-seconds=120", command) + self.assertIn('/srv/secure/secrets/cloudflare.ini', command) - def test_ssl_renew_reloads_nginx_and_returns_combined_output(self): + @patch('agentd.agentlib.nginx_lock', return_value=nullcontext()) + def test_ssl_renew_reloads_nginx_and_returns_combined_output(self, lock): with patch( "agentd.api.agentlib.execute", side_effect=[ SimpleNamespace(stderr="renew stderr", stdout="renew stdout"), SimpleNamespace(stderr="", stdout=""), + SimpleNamespace(stderr="", stdout=""), ], ) as execute: result = api.ssl_renew() @@ -470,6 +486,7 @@ def test_ssl_renew_reloads_nginx_and_returns_combined_output(self): execute.call_args_list, [ call(["certbot", "renew"]), + call(["nginx", "-t"]), call(["systemctl", "reload", "nginx"]), ], ) @@ -724,6 +741,9 @@ def test_create_configures_urls_database_container_and_admin_password(self): class UrlSyncTests(unittest.TestCase): + def setUp(self): + self.enterContext(patch('agentd.agentlib.nginx_lock', return_value=nullcontext())) + def test_sync_urls_replaces_existing_port_mappings_and_reloads_nginx(self): load_results = [ ( @@ -774,7 +794,62 @@ def test_sync_urls_replaces_existing_port_mappings_and_reloads_nginx(self): ), ], ) - execute.assert_called_once_with(["systemctl", "reload", "nginx"]) + self.assertEqual(execute.call_args_list, [call(['nginx', '-t']), call(['systemctl', 'reload', 'nginx'])]) + + def test_sync_urls_restores_both_maps_when_validation_fails(self): + original = ('$host', '$target', {'keep.example': '127.0.0.1:7000'}) + with patch('agentd.agentlib.load_nginx_map', return_value=original): + with patch('agentd.agentlib.store_nginx_map') as store: + with patch('agentd.agentlib.execute', side_effect=RuntimeError('invalid nginx')) as execute: + with self.assertRaises(RuntimeError): + api.sync_urls(['new.example'], 8001, 9001) + self.assertEqual(store.call_args_list[-2:], [ + call('gevent-ports.conf', *original), call('http-ports.conf', *original), + ]) + execute.assert_called_once_with(['nginx', '-t']) + + +class ConcurrentUrlSyncTests(unittest.TestCase): + def test_two_agents_preserve_each_others_routes(self): + maps = {'http-ports.conf': {}, 'gevent-ports.conf': {}} + barrier = threading.Barrier(2) + + def load(fname): + snapshot = maps[fname].copy() + # Widen the read/write window that previously lost another route. + time.sleep(0.02) + return '$host', '$target', snapshot + + def store(fname, match, target, mapping): + maps[fname] = mapping + + def sync(hostname, http, gevent): + barrier.wait(timeout=5) + api.sync_urls([hostname], http, gevent) + + real_open = open + with tempfile.TemporaryDirectory() as directory: + def lock_open(path, *args, **kwargs): + self.assertEqual(path, '/run/lock/odoo-nginx.lock') + return real_open(f'{directory}/nginx.lock', *args, **kwargs) + + with patch('builtins.open', side_effect=lock_open): + with patch('agentd.agentlib.load_nginx_map', side_effect=load): + with patch('agentd.agentlib.store_nginx_map', side_effect=store): + with patch('agentd.agentlib.execute'): + with ThreadPoolExecutor(max_workers=2) as executor: + jobs = [ + executor.submit(sync, 'v19.example', 8101, 9101), + executor.submit(sync, 'v20.example', 8102, 9102), + ] + for job in jobs: + job.result(timeout=5) + self.assertEqual(maps['http-ports.conf'], { + 'v19.example': '127.0.0.1:8101', 'v20.example': '127.0.0.1:8102', + }) + self.assertEqual(maps['gevent-ports.conf'], { + 'v19.example': '127.0.0.1:9101', 'v20.example': '127.0.0.1:9102', + }) if __name__ == "__main__": # pragma: no cover diff --git a/agentd/tests/test_appdata.py b/agentd/tests/test_appdata.py new file mode 100644 index 0000000..a4e1bb8 --- /dev/null +++ b/agentd/tests/test_appdata.py @@ -0,0 +1,56 @@ +import subprocess +import unittest +from pathlib import Path + + +class AppdataTests(unittest.TestCase): + """Exercise fail-closed checks without root, real mounts or disk writes.""" + + def check(self, override='', command='check_appdata'): + script = ''' +set -euo pipefail +source ./appdata.sh +appdata_device() { echo /dev/test-volume; } +cryptsetup() { echo 'device: /dev/test-volume'; } +readlink() { echo "${@: -1}"; } +mountpoint() { return 0; } +findmnt() { + case "$2" in + MAJ:MIN) echo 253:0 ;; + FSTYPE) echo ext4 ;; + FSROOT) echo / ;; + esac +} +lsblk() { echo 253:0; } +stat() { echo 100:200; } +swapon() { :; } +mount() { echo unexpected-mount; exit 99; } +''' + return subprocess.run( + ['bash', '-c', script + override + '\n' + command], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, text=True, + ) + + def test_accepts_verified_mounts_and_repeated_mount_request(self): + for command in ('check_appdata', 'mount_appdata'): + result = self.check(command=command) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn('unexpected-mount', result.stdout) + + def test_rejects_missing_or_wrong_mounts_and_swap(self): + cases = [ + ('cryptsetup() { echo "device: /dev/other"; }', 'wrong device'), + ('mountpoint() { return 1; }', '/srv/secure is not mounted'), + ('lsblk() { echo 8:1; }', 'not backed by appdata'), + ('mountpoint() { [[ "$2" != /var/lib/docker ]]; }', '/var/lib/docker is not mounted'), + ('stat() { echo "$3"; }', 'not bound to'), + ('stat() { return 1; }', 'Cannot inspect'), + ('readlink() { return 1; }', 'Cannot resolve'), + ('swapon() { echo /swapfile; }', 'Disable swap'), + ] + for override, message in cases: + with self.subTest(message=message): + result = self.check(override) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) diff --git a/appdata.sh b/appdata.sh new file mode 100644 index 0000000..6db4d8c --- /dev/null +++ b/appdata.sh @@ -0,0 +1,82 @@ +# Shared by setup-appdata.sh and /usr/local/sbin/unlock-appdata. +appdata_mounts=( + postgresql:/var/lib/postgresql + docker:/var/lib/docker + containerd:/var/lib/containerd + odoo-config:/etc/odoo + rclone-config:/root/.config/rclone + rclone-cache:/root/.cache/rclone + logs/nginx:/var/log/nginx + logs/postgresql:/var/log/postgresql + nginx-temp:/var/lib/nginx +) + +fail() { + echo "$*" >&2 + exit 1 +} + +appdata_guard() { + # Requisite checks existing mounts without pulling in a locked LUKS device. + local entry mounts=(srv-secure.mount) + for entry in "${appdata_mounts[@]}"; do + mounts+=("$(systemd-escape --path --suffix=mount "${entry#*:}")") + done + # Ubuntu's postgresql@ unit otherwise pulls its data mount in on startup. + printf '[Unit]\nRequiresMountsFor=\nRequisite=%s\nAfter=%s\nPartOf=odoo-app.target\nConditionPathIsMountPoint=/srv/secure\n' "${mounts[*]}" "${mounts[*]}" + if [[ "$1" != *.socket ]]; then + printf '\n[Service]\nExecStartPre=+/usr/local/sbin/unlock-appdata --check\nEnvironment=TMPDIR=/srv/secure/tmp\n' + fi +} + +appdata_device() { + local uuid + uuid=$(awk '$1 == "appdata" {print $2; count++} END {if (count != 1) exit 1}' /etc/crypttab) || fail 'Expected one appdata entry in /etc/crypttab.' + [[ "$uuid" == UUID=* ]] || fail 'appdata must use a LUKS UUID.' + readlink -e "/dev/disk/by-uuid/${uuid#UUID=}" +} + +check_mapper() { + local device expected + device=$(cryptsetup status appdata | awk '$1 == "device:" {print $2}') + device=$(readlink -e "$device") || fail 'Cannot resolve the appdata backing device.' + expected=$(appdata_device) || fail 'Cannot resolve the configured appdata device.' + [[ -n "$device" && "$device" == "$expected" ]] || fail 'appdata is mapped to the wrong device.' +} + +check_secure() { + check_mapper + mountpoint -q /srv/secure || fail '/srv/secure is not mounted.' + [[ $(findmnt -nro MAJ:MIN -M /srv/secure) == "$(lsblk -dnro MAJ:MIN /dev/mapper/appdata)" ]] || fail '/srv/secure is not backed by appdata.' + [[ $(findmnt -nro FSTYPE -M /srv/secure) == ext4 && $(findmnt -nro FSROOT -M /srv/secure) == / ]] || fail '/srv/secure must mount the root of the ext4 filesystem.' +} + +check_bind() { + local source=$1 target=$2 source_inode target_inode + mountpoint -q "$target" || fail "$target is not mounted." + source_inode=$(stat -c '%d:%i' "$source") || fail "Cannot inspect $source." + target_inode=$(stat -c '%d:%i' "$target") || fail "Cannot inspect $target." + [[ "$source_inode" == "$target_inode" ]] || fail "$target is not bound to $source." +} + +check_appdata() { + check_secure + local entry + for entry in "${appdata_mounts[@]}"; do + check_bind "/srv/secure/${entry%%:*}" "${entry#*:}" + done + [[ -z $(swapon --show --noheadings) ]] || fail 'Disable swap before starting application services.' +} + +mount_appdata() { + check_mapper + mountpoint -q /srv/secure || mount /srv/secure + check_secure + local entry target + for entry in "${appdata_mounts[@]}"; do + target=${entry#*:} + mountpoint -q "$target" || mount "$target" + check_bind "/srv/secure/${entry%%:*}" "$target" + done + check_appdata +} diff --git a/install-release.sh b/install-release.sh new file mode 100755 index 0000000..b20094b --- /dev/null +++ b/install-release.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail +cd "$(dirname "$0")" +[[ $EUID == 0 && $(pwd -P) == /opt/odoo19 ]] || { echo 'Run as root from /opt/odoo19.' >&2; exit 1; } +/usr/local/sbin/unlock-appdata --check + +python3 -m venv /root/agent-venv19 +/root/agent-venv19/bin/python -m pip install -r requirements.txt +install -m 0644 systemd/system/deploy-manager19.service /etc/systemd/system/ +install -D -m 0644 logrotate/deploy-manager19 /etc/logrotate.d/deploy-manager19 +install -d /etc/systemd/system/deploy-manager19.service.d +# Reuse the host guard; future release branches install only their own unit. +install -m 0644 /etc/systemd/system/docker.service.d/appdata.conf /etc/systemd/system/deploy-manager19.service.d/appdata.conf +systemctl daemon-reload +systemctl enable deploy-manager19.service + +# First bootstrap installs the venv needed to obtain the wildcard certificate. +if [[ ! -f /etc/letsencrypt/live/eniemela.fi/fullchain.pem || ! -f /etc/letsencrypt/live/eniemela.fi/privkey.pem || ! -f /etc/nginx/.htpasswd ]]; then + echo 'Services have not been started. Configure /root/.config/rclone/rclone.conf and /srv/secure/secrets/cloudflare.ini.' + echo 'Create nginx basic auth and the wildcard certificate (README.md), then rerun ./install-release.sh and sudo unlock-appdata.' + exit 0 +fi + +# Share the agents' routing lock, including the validation and reload. +exec 9>/run/lock/odoo-nginx.lock +flock 9 +staging=$(mktemp -d /etc/nginx/.odoo19.XXXXXX) +trap 'rm -r -- "$staging"' EXIT +mkdir "$staging/sites-enabled" +cp -aL /etc/nginx/sites-enabled/. "$staging/sites-enabled/" +install -m 0644 nginx/sites-enabled/00_agent19.conf "$staging/00_agent19.conf" +ln -sfn "$staging/00_agent19.conf" "$staging/sites-enabled/00_agent19.conf" +if [[ ! -e /etc/nginx/sites-enabled/odoo.conf ]]; then + install -m 0644 nginx/sites-enabled/odoo.conf "$staging/odoo.conf" + ln -sfn "$staging/odoo.conf" "$staging/sites-enabled/odoo.conf" +fi +sed "s@include /etc/nginx/sites-enabled/\*;@include $staging/sites-enabled/*;@" /etc/nginx/nginx.conf > "$staging/nginx.conf" +nginx -t -c "$staging/nginx.conf" +install -d /etc/nginx/sites-available +install -m 0644 "$staging/00_agent19.conf" /etc/nginx/sites-available/00_agent19.conf +ln -sfn /etc/nginx/sites-available/00_agent19.conf /etc/nginx/sites-enabled/00_agent19.conf +if [[ -f "$staging/odoo.conf" ]]; then + install -m 0644 "$staging/odoo.conf" /etc/nginx/sites-available/odoo.conf + ln -sfn /etc/nginx/sites-available/odoo.conf /etc/nginx/sites-enabled/odoo.conf +fi +if systemctl is-active --quiet nginx; then + systemctl reload nginx +fi +echo 'Odoo 19 installed. Run sudo unlock-appdata to start the application services.' diff --git a/logrotate/deploy-manager19 b/logrotate/deploy-manager19 new file mode 100644 index 0000000..f0eb563 --- /dev/null +++ b/logrotate/deploy-manager19 @@ -0,0 +1,8 @@ +/srv/secure/logs/deploy-manager19.log { + weekly + rotate 4 + compress + missingok + notifempty + copytruncate +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 99e6514..01fbd45 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -14,10 +14,12 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; + client_body_temp_path /var/lib/nginx/body; + proxy_temp_path /var/lib/nginx/proxy; access_log /var/log/nginx/access.log; error_log /var/log/nginx/error.log; include /etc/nginx/conf.d/*.conf; include /etc/nginx/sites-enabled/*; -} \ No newline at end of file +} diff --git a/nginx/sites-enabled/00_agent.conf b/nginx/sites-enabled/00_agent19.conf similarity index 85% rename from nginx/sites-enabled/00_agent.conf rename to nginx/sites-enabled/00_agent19.conf index a2144d5..0a2e5cb 100644 --- a/nginx/sites-enabled/00_agent.conf +++ b/nginx/sites-enabled/00_agent19.conf @@ -1,7 +1,7 @@ server { - listen 9000 ssl; - listen [::]:9000 ssl; - server_name agent19.eniemela.fi; + listen 9019 ssl; + listen [::]:9019 ssl; + server_name odoo19.eniemela.fi; proxy_read_timeout 24h; proxy_connect_timeout 24h; @@ -27,6 +27,6 @@ server { auth_basic_user_file /etc/nginx/.htpasswd; location / { - proxy_pass http://127.0.0.1:8000; + proxy_pass http://127.0.0.1:8019; } } diff --git a/nginx/sites-enabled/odoo.conf b/nginx/sites-enabled/odoo.conf index 968deb1..6f45c60 100644 --- a/nginx/sites-enabled/odoo.conf +++ b/nginx/sites-enabled/odoo.conf @@ -68,19 +68,6 @@ server { proxy_pass http://$targetHttp; } - # copy-paste the content of the / location block - location @odoo { - proxy_redirect off; - proxy_pass http://$targetHttp; - } - - # Serve static files right away - location ~ ^/[^/]+/static/.+$ { - root /opt/deploy-manager/src; - try_files /odoo/addons/$uri /tabularium/$uri @odoo; - expires 24h; - } - location = /robots.txt { add_header Content-Type text/plain; return 200 "User-agent: *\nDisallow: /\n"; @@ -89,4 +76,4 @@ server { # common gzip gzip_types text/css text/scss text/plain text/xml application/xml application/json application/javascript; gzip on; -} \ No newline at end of file +} diff --git a/setup-appdata.sh b/setup-appdata.sh new file mode 100755 index 0000000..d180a6d --- /dev/null +++ b/setup-appdata.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +set -euo pipefail +cd "$(dirname "$0")" +source ./appdata.sh + +[[ $EUID == 0 && $# == 1 && $1 == /dev/disk/by-id/* ]] || fail 'Usage: sudo ./setup-appdata.sh /dev/disk/by-id/' +device=$(readlink -e "$1") +[[ -b "$device" && $(lsblk -dnro TYPE "$device") == disk ]] || fail 'Supply a whole, unpartitioned volume.' +echo "Application data volume: $1 ($device)" + +# Reject the boot disk even when / is inside a partition or device mapper. +root_device=$(findmnt -nro MAJ:MIN /) +root_ancestors=$(lsblk -snro MAJ:MIN "/dev/block/$root_device") +[[ -n "$root_ancestors" ]] || fail 'Cannot identify the root disk.' +if grep -Fxq "$(lsblk -dnro MAJ:MIN "$device")" <<< "$root_ancestors"; then + fail 'Refusing the root disk.' +fi + +apt update +apt install -y cryptsetup + +signature=$(wipefs --no-act --noheadings --output TYPE "$device" | sort -u | xargs) +case "$signature" in + '') + [[ $(lsblk -nrpo NAME "$device" | wc -l) == 1 ]] || fail 'Volume has partitions or active holders.' + [[ -z $(lsblk -nro MOUNTPOINTS "$device" | xargs) ]] || fail 'Volume is mounted or used as swap.' + [[ ! -e /dev/mapper/appdata ]] || fail 'appdata already exists.' + if [[ -f /etc/crypttab ]] && grep -qE '^appdata[[:space:]]' /etc/crypttab; then + fail 'appdata is already configured; refusing a different volume.' + fi + ;; + crypto_LUKS) + cryptsetup isLuks --type luks2 "$device" || fail 'Expected LUKS2.' + [[ "$device" == "$(appdata_device)" ]] || fail 'Existing LUKS volume is not the configured appdata device.' + while read -r child; do + [[ "$child" == "$device" || "$child" == /dev/mapper/appdata ]] || fail "Unexpected active device: $child" + done < <(lsblk -nrpo NAME "$device") + if [[ -e /dev/mapper/appdata ]]; then + check_mapper + while read -r target; do + [[ -z "$target" ]] && continue + allowed=false + [[ "$target" == /srv/secure ]] && allowed=true + for entry in "${appdata_mounts[@]}"; do + [[ "$target" == "${entry#*:}" ]] && allowed=true + done + "$allowed" || fail "Unexpected mount on appdata: $target" + done < <(lsblk -nro MOUNTPOINTS /dev/mapper/appdata) + fi + ;; + *) fail "Refusing volume with existing signatures: $signature" ;; +esac + +# Never hide pre-existing plaintext data beneath a bind mount. +for entry in "${appdata_mounts[@]}"; do + target=${entry#*:} + [[ ! -L "$target" ]] || fail "Refusing symlink: $target" + if mountpoint -q "$target"; then + check_secure + check_bind "/srv/secure/${entry%%:*}" "$target" + elif [[ -d "$target" && -n $(find "$target" -mindepth 1 -maxdepth 1 -print -quit) ]]; then + fail "$target is not empty. Existing-server migration is not supported." + fi +done +if mountpoint -q /srv/secure; then + check_secure +elif [[ -d /srv/secure && -n $(find /srv/secure -mindepth 1 -maxdepth 1 -print -quit) ]]; then + fail '/srv/secure is not empty.' +fi +[[ ! -L /srv/secure ]] || fail 'Refusing /srv/secure symlink.' + +if [[ -z "$signature" ]]; then + # cryptsetup retains its interactive destructive-operation confirmation. + cryptsetup luksFormat --type luks2 "$device" + uuid=$(cryptsetup luksUUID "$device") + printf 'appdata UUID=%s none luks,noauto\n' "$uuid" >> /etc/crypttab + (umask 077; cryptsetup luksHeaderBackup "$device" --header-backup-file "/root/appdata-luks-header-$uuid.img") + echo "Copy /root/appdata-luks-header-$uuid.img offline, verify it, then delete the server copy. Store the passphrase separately." + udevadm trigger --action=change --name-match="$device" + udevadm settle +fi +cryptsetup status appdata >/dev/null || cryptsetup open "$device" appdata +check_mapper +signature=$(wipefs --no-act --noheadings --output TYPE /dev/mapper/appdata | sort -u | xargs) +case "$signature" in + '') mkfs.ext4 /dev/mapper/appdata ;; + ext4) ;; + *) fail "Expected ext4, found: $signature" ;; +esac + +# Disable fstab swap persistently. Native swap units/generators must be removed +# by the operator; silently leaving them configured could expose customer data. +swapoff --all +sed -i '/^[^#].*[[:space:]]swap[[:space:]]/s/^/# Disabled for LUKS appdata: /' /etc/fstab +systemctl daemon-reload +[[ -z $(systemctl list-unit-files --type=swap --state=enabled,generated --no-legend) ]] || fail 'Disable native swap units or swap generators, then re-run setup.' +systemctl mask swap.target + +fstab_entry() { + local source=$1 target=$2 type=$3 options=$4 pass=$5 existing + existing=$(awk -v target="$target" '$2 == target {print}' /etc/fstab) + if [[ -n "$existing" ]]; then + [[ "$existing" == "$source $target $type $options 0 $pass" ]] || fail "Conflicting fstab entry for $target." + else + printf '%s %s %s %s 0 %s\n' "$source" "$target" "$type" "$options" "$pass" >> /etc/fstab + fi +} + +install -d /srv/secure +fstab_entry /dev/mapper/appdata /srv/secure ext4 noauto 2 +systemctl daemon-reload +mountpoint -q /srv/secure || mount /srv/secure +check_secure +for entry in "${appdata_mounts[@]}"; do + source_dir=/srv/secure/${entry%%:*} + target=${entry#*:} + mkdir -p "$source_dir" "$target" + fstab_entry "$source_dir" "$target" none noauto,bind,x-systemd.requires=srv-secure.mount 0 +done +install -d -m 0700 /srv/secure/secrets /srv/secure/rclone-config /srv/secure/rclone-cache +install -d -m 0711 /srv/secure/tmp +systemctl daemon-reload +mount_appdata +install -D -m 0644 appdata.sh /usr/local/lib/odoo-appdata.sh +install -m 0755 unlock-appdata /usr/local/sbin/unlock-appdata diff --git a/systemd/system/deploy-manager.service b/systemd/system/deploy-manager.service deleted file mode 100644 index e587cd5..0000000 --- a/systemd/system/deploy-manager.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=deploy-manager - -[Service] -User=root -WorkingDirectory=/opt/deploy-manager -ExecStart=/root/agent-venv/bin/python -m agentd -Restart=always -RestartSec=1 -StartLimitInterval=120 -StartLimitBurst=30 -Environment=PATH=/root/agent-venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -[Install] -WantedBy=multi-user.target diff --git a/systemd/system/deploy-manager19.service b/systemd/system/deploy-manager19.service new file mode 100644 index 0000000..5ef6836 --- /dev/null +++ b/systemd/system/deploy-manager19.service @@ -0,0 +1,22 @@ +[Unit] +Description=deploy-manager Odoo 19 +Requires=docker.service postgresql.service rclone-mount.service +After=docker.service postgresql.service rclone-mount.service +PartOf=odoo-app.target +StartLimitIntervalSec=120 +StartLimitBurst=30 + +[Service] +User=root +WorkingDirectory=/opt/odoo19 +ExecStartPre=/usr/local/sbin/unlock-appdata --check +ExecStart=/root/agent-venv19/bin/python -m agentd --interface 127.0.0.1 --port 8019 +Restart=always +RestartSec=1 +Environment=PATH=/root/agent-venv19/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=TMPDIR=/srv/secure/tmp +StandardOutput=append:/srv/secure/logs/deploy-manager19.log +StandardError=inherit + +[Install] +WantedBy=odoo-app.target diff --git a/systemd/system/odoo-app.target b/systemd/system/odoo-app.target new file mode 100644 index 0000000..e75e584 --- /dev/null +++ b/systemd/system/odoo-app.target @@ -0,0 +1,4 @@ +[Unit] +Description=Odoo application services (start after unlock-appdata) +Requires=postgresql.service containerd.service docker.service nginx.service rclone-mount.service +After=postgresql.service containerd.service docker.service nginx.service rclone-mount.service diff --git a/systemd/system/rclone-mount.service b/systemd/system/rclone-mount.service index 5257049..d8bf065 100644 --- a/systemd/system/rclone-mount.service +++ b/systemd/system/rclone-mount.service @@ -1,13 +1,15 @@ [Unit] Description=rclone-mount -After=multi-user.target +Wants=network-online.target +After=network-online.target +PartOf=odoo-app.target [Service] -Type=simple +Type=notify User=root -ExecStart=/usr/bin/rclone mount backup-crypt: /root/backups --vfs-cache-mode full -ExecStop=fusermount -u /root/backups +ExecStart=/usr/bin/rclone mount backup-crypt: /root/backups --vfs-cache-mode full --cache-dir /root/.cache/rclone +ExecStop=/usr/bin/fusermount3 -u /root/backups Restart=on-abort [Install] -WantedBy=multi-user.target +WantedBy=odoo-app.target diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 71ba48a..49915e7 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -1,12 +1,28 @@ #!/bin/bash -set -e +set -euo pipefail +cd "$(dirname "$0")" +source ./appdata.sh -apt update -apt install postgresql nginx ca-certificates curl gnupg vim tmux patchutils fuse3 python3-pip python3-venv unattended-upgrades apache2-utils +[[ $EUID == 0 && $(pwd -P) == /opt/odoo19 ]] || fail 'Run as root from /opt/odoo19.' +for service in postgresql docker containerd nginx rclone-mount odoo-app.target; do + if systemctl is-active --quiet "$service"; then + fail "$service is active. Stop application services before running the host bootstrap." + fi +done +bash ./setup-appdata.sh "$@" -systemctl enable nginx --now -systemctl enable postgresql --now +# Package post-install scripts must not start services before configuration. +# Leave these runtime masks in place on failure; a successful rerun removes them. +services=(postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service) +systemctl mask --runtime "${services[@]}" +# Install guards before packages: even a reboot during setup must fail closed. +for service in "${services[@]}" deploy-manager19.service; do + mkdir -p "/etc/systemd/system/$service.d" + appdata_guard "$service" > "/etc/systemd/system/$service.d/appdata.conf" +done +systemctl daemon-reload +apt install -y postgresql nginx ca-certificates curl gnupg vim tmux patchutils fuse3 python3-pip python3-venv unattended-upgrades apache2-utils rsync rclone # https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository install -m 0755 -d /etc/apt/keyrings @@ -24,28 +40,40 @@ Signed-By: /etc/apt/keyrings/docker.asc EOF apt update -apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -curl https://rclone.org/install.sh | sudo bash +install -m 0644 systemd/system/odoo-app.target systemd/system/rclone-mount.service /etc/systemd/system/ +install -D -m 0644 systemd/system/docker.service.d/override.conf /etc/systemd/system/docker.service.d/override.conf -python3 -m venv /root/agent-venv -/root/agent-venv/bin/python -m pip install -r requirements.txt +# PostgreSQL needs a writable temporary directory owned by its own user. +install -d -o postgres -g postgres -m 0700 /srv/secure/tmp/postgresql +printf '[Service]\nEnvironment=TMPDIR=/srv/secure/tmp/postgresql\n' > /etc/systemd/system/postgresql@.service.d/tmp.conf -cp systemd/system/deploy-manager.service /etc/systemd/system/ -cp systemd/system/rclone-mount.service /etc/systemd/system/ -systemctl daemon-reload -systemctl enable deploy-manager.service --now -systemctl enable rclone-mount.service --now - -rsync -avz postgres/ /etc/postgresql/*/main -rsync -avz nginx/ /etc/nginx - -mkdir -p /root/backups -mkdir -p /root/.config/rclone/ -cp rclone.conf /root/.config/rclone/rclone.conf -chmod 0600 /root/.config/rclone/rclone.conf -cp cloudflare.ini /root/cloudflare.ini -chmod 0600 /root/cloudflare.ini +# Debian/Ubuntu cluster generators can enable PostgreSQL independently of the +# umbrella service. Start clusters explicitly from our manual target instead. +for pgconf in /etc/postgresql/*/main; do + pgversion=${pgconf#/etc/postgresql/} + pgversion=${pgversion%/main} + rsync -av postgres/ "$pgconf/" + printf 'manual\n' > "$pgconf/start.conf" + systemctl add-wants odoo-app.target "postgresql@$pgversion-main.service" + mkdir -p /etc/systemd/system/docker.service.d + printf '[Unit]\nRequires=postgresql@%s-main.service\nAfter=postgresql@%s-main.service\n' "$pgversion" "$pgversion" > /etc/systemd/system/docker.service.d/postgresql.conf +done +chown postgres:adm /var/log/postgresql +chmod 0750 /var/log/postgresql +chown www-data:adm /var/log/nginx +chmod 0755 /var/log/nginx +chown www-data:adm /var/lib/nginx +chmod 0755 /var/lib/nginx + +# Copy shared routing maps only on first install. A second release must never +# overwrite hostnames that the running agents have already registered. +rsync -av --ignore-existing nginx/conf.d/ /etc/nginx/conf.d/ +install -m 0644 nginx/nginx.conf /etc/nginx/nginx.conf +install -d -m 0700 /root/backups +[[ -f /root/.config/rclone/rclone.conf ]] || install -m 0600 rclone.conf /root/.config/rclone/rclone.conf +[[ -f /srv/secure/secrets/cloudflare.ini ]] || install -m 0600 cloudflare.ini /srv/secure/secrets/cloudflare.ini cp sshd/harden.conf /etc/ssh/sshd_config.d/harden.conf systemctl reload ssh @@ -53,4 +81,7 @@ mkdir -p /etc/apt/apt.conf.d/ cp apt/apt.conf.d/* /etc/apt/apt.conf.d/ systemctl restart unattended-upgrades -exit 0 +systemctl unmask --runtime "${services[@]}" +systemctl disable "${services[@]}" +systemctl daemon-reload +bash ./install-release.sh diff --git a/unlock-appdata b/unlock-appdata new file mode 100755 index 0000000..5989733 --- /dev/null +++ b/unlock-appdata @@ -0,0 +1,29 @@ +#!/bin/bash + +set -euo pipefail +source /usr/local/lib/odoo-appdata.sh + +[[ $EUID == 0 ]] || fail 'Run as root.' +case ${1:-} in + --check) check_appdata; exit 0 ;; + --mount-only|'') ;; + *) fail 'Usage: unlock-appdata [--check|--mount-only]' ;; +esac + +# Serialize interactive unlocks, but never hold this lock in service checks. +exec 9>/run/lock/odoo-appdata.lock +flock 9 +cryptsetup status appdata >/dev/null || cryptsetup open "$(appdata_device)" appdata +mount_appdata +[[ ${1:-} != --mount-only ]] || exit 0 + +flock /run/lock/odoo-nginx.lock nginx -t +systemctl start odoo-app.target +# A target may already be active after an earlier partial failure. Retry its +# services too, including every installed release and PostgreSQL cluster. +read -ra services <<< "$(systemctl show odoo-app.target -p Requires -p Wants --value | tr '\n' ' ')" +for service in "${services[@]}"; do + [[ "$service" == *.service ]] || continue + systemctl start "$service" + systemctl is-active --quiet "$service" || fail "$service did not start." +done diff --git a/update.sh b/update.sh index 435d08e..02fadce 100755 --- a/update.sh +++ b/update.sh @@ -2,8 +2,9 @@ set -e +cd /opt/odoo19 git pull git submodule update -systemctl restart deploy-manager.service +systemctl restart deploy-manager19.service -exit 1 +exit 0 From ae4d27cdf0abf4006e502dcad4099d580b11678d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 18:53:51 +0300 Subject: [PATCH 02/23] rm / simplify bash scripts --- LUKS.md | 422 ++++-------------- README.md | 166 +++++-- agentd/tests/test_appdata.py | 56 --- appdata.sh | 82 ---- apt/docker.sources | 6 + install-release.sh | 50 --- postgres/start.conf | 1 + setup-appdata.sh | 126 ------ systemd/system/appdata-mounts.conf | 5 + systemd/system/deploy-manager19.service | 5 +- .../system/docker.service.d/postgresql.conf | 3 + systemd/system/odoo-app.target | 6 +- systemd/system/postgresql@.service.d/tmp.conf | 2 + ubuntu-install.sh | 105 ++--- unlock-appdata | 29 -- 15 files changed, 291 insertions(+), 773 deletions(-) delete mode 100644 agentd/tests/test_appdata.py delete mode 100644 appdata.sh create mode 100644 apt/docker.sources delete mode 100755 install-release.sh create mode 100644 postgres/start.conf delete mode 100755 setup-appdata.sh create mode 100644 systemd/system/appdata-mounts.conf create mode 100644 systemd/system/docker.service.d/postgresql.conf create mode 100644 systemd/system/postgresql@.service.d/tmp.conf delete mode 100755 unlock-appdata diff --git a/LUKS.md b/LUKS.md index 92de1ba..b2c07d4 100644 --- a/LUKS.md +++ b/LUKS.md @@ -1,362 +1,132 @@ -# LUKS and Multi-Version Odoo Host Plan +# LUKS application storage -## Summary +The Ubuntu root filesystem remains unencrypted so the server can boot and +accept SSH connections. PostgreSQL data, Docker data, Odoo configuration, +rclone credentials and cache, and application logs live on a manually unlocked +LUKS2 Hetzner Volume. -This plan targets a fresh Ubuntu server with an empty Hetzner Cloud Volume. The -Ubuntu root filesystem remains unencrypted so the server can boot normally and -accept SSH connections. Sensitive Odoo runtime state is stored on a manually -unlocked LUKS2 volume. - -PostgreSQL, Docker, containerd, nginx, rclone, and the deployment agents must -remain stopped until the encrypted volume has been unlocked and its bind mounts -have been verified. - -The server supports multiple Odoo versions through separate deploy-manager -clones and agents while sharing PostgreSQL, Docker, nginx, rclone, and the LUKS -volume. Release-specific values are hardcoded in each release branch. - -Odoo 19 uses: - -```text -Repository: /opt/odoo19 -Virtualenv: /root/agent-venv19 -Systemd service: deploy-manager19.service -Agent backend: 127.0.0.1:8019 -Public endpoint: https://odoo19.eniemela.fi:9019 -Container image: ghcr.io/elmeriniemela/odoo-src:19.0 -Container label: odoo.version=19.0 -``` - -When Odoo 20 is released, create a 20.0 branch and replace the relevant 19 -values with 20. The resulting installation will use `/opt/odoo20`, ports 8020 -and 9020, and a separate Odoo 20 agent that can run alongside Odoo 19. +The exact first-install commands are in [README.md](README.md). They are kept in +the documentation because choosing and formatting a block device should be a +deliberate operator action, not hidden in a Bash script. ## Storage layout -Create a LUKS2 container directly on the Hetzner Volume without a partition -table, format the decrypted mapper as ext4, and mount it at `/srv/secure`. - ```text Hetzner Volume └── LUKS2 mapper: appdata - └── ext4 - └── /srv/secure - ├── postgresql - ├── docker - ├── containerd - ├── odoo-config - ├── rclone-config - ├── rclone-cache - ├── nginx-temp - ├── tmp - ├── secrets - └── logs - ├── nginx - └── postgresql + └── ext4 mounted at /srv/secure + ├── postgresql → /var/lib/postgresql + ├── docker → /var/lib/docker + ├── containerd → /var/lib/containerd + ├── odoo-config → /etc/odoo + ├── rclone-config → /root/.config/rclone + ├── rclone-cache → /root/.cache/rclone + ├── nginx-temp → /var/lib/nginx + ├── tmp + ├── secrets + └── logs + ├── nginx → /var/log/nginx + └── postgresql → /var/log/postgresql ``` -Bind-mount the encrypted directories to the conventional host paths: +The mapper and every bind mount use `noauto` in `/etc/fstab`. PostgreSQL, +Docker, containerd, nginx, rclone, and the deployment agent are disabled at +boot. Their systemd drop-ins declare every mount unit as a `Requisite`, so a +direct service start fails while the encrypted storage is not mounted. -```text -/srv/secure/postgresql → /var/lib/postgresql -/srv/secure/docker → /var/lib/docker -/srv/secure/containerd → /var/lib/containerd -/srv/secure/odoo-config → /etc/odoo -/srv/secure/rclone-config → /root/.config/rclone -/srv/secure/rclone-cache → /root/.cache/rclone -/srv/secure/logs/nginx → /var/log/nginx -/srv/secure/logs/postgresql → /var/log/postgresql -/srv/secure/nginx-temp → /var/lib/nginx -``` - -Store `cloudflare.ini` under `/srv/secure/secrets` and configure the deployment -agent to read it from there. The rclone mount continues to use `/root/backups`, -but its configuration and VFS cache reside on the encrypted filesystem. - -The following remain on the unencrypted root filesystem: +Do not store database dumps in `/tmp` or `/var/tmp`. The agent uses +`/srv/secure/tmp`, PostgreSQL uses `/srv/secure/tmp/postgresql`, and nginx uses +its encrypted `/var/lib/nginx` bind mount. Persistent swap is disabled so +application memory is not written to unencrypted disk. -- Ubuntu and SSH configuration -- Deploy-manager and Odoo source under `/opt/odoo19` -- Nginx routing configuration -- TLS certificates and keys -- System logs unrelated to the Odoo application +## Normal boot procedure -Do not store customer data in `/tmp` or `/var/tmp`. Disable persistent swap so -that application memory cannot be written unencrypted to disk. -Agent subprocesses use `/srv/secure/tmp`; PostgreSQL has a private subdirectory. -Nginx buffers request bodies and upstream responses under `/var/lib/nginx`. -Agent output goes to `/srv/secure/logs/deploy-manager19.log` with weekly rotation. - -## One-time fresh-server bootstrap - -Change `ubuntu-install.sh` into a one-time host bootstrap accepting the stable -Hetzner device path: +Replace the device path with the stable path recorded during installation: ```bash -sudo ./ubuntu-install.sh /dev/disk/by-id/ +cryptsetup open /dev/disk/by-id/ appdata +mount /srv/secure +mount /var/lib/postgresql +mount /var/lib/docker +mount /var/lib/containerd +mount /etc/odoo +mount /root/.config/rclone +mount /root/.cache/rclone +mount /var/log/nginx +mount /var/log/postgresql +mount /var/lib/nginx +nginx -t +systemctl start odoo-app.target ``` -The script must perform the following steps in order: - -1. Require root privileges and a `/dev/disk/by-id/...` argument. -2. Resolve and display the target device. -3. Refuse the root device, mounted or otherwise active devices, and devices - containing unexpected signatures. It must never silently erase or reformat - an existing filesystem. -4. Install `cryptsetup` before installing PostgreSQL or Docker. -5. If the device is blank, create a LUKS2 container using an interactively - entered passphrase. Never save the passphrase on the server. -6. Open the volume as `/dev/mapper/appdata`. -7. Create an ext4 filesystem only when the mapper contains no filesystem. - Re-running the script against the same valid LUKS/ext4 volume must not format - it again. -8. Add an idempotent `/etc/crypttab` entry using the LUKS UUID and the `noauto` - option. -9. Add idempotent `/etc/fstab` entries for `/srv/secure` and all bind mounts. - They must use `noauto` and explicit dependencies on `/srv/secure`. -10. Mount `/srv/secure`, create the source and target directories with suitable - ownership and permissions, and activate every bind mount. -11. Install and configure PostgreSQL, Docker, containerd, nginx, rclone, and the - remaining host dependencies only after the encrypted mounts are active. -12. Install the systemd target, mount guards, unlock helper, and the hardcoded - Odoo 19 release service. -13. Disable automatic boot activation of PostgreSQL, Docker, `docker.socket`, - containerd, nginx, rclone, and the deployment agent. -14. Disable persistent swap and fail with a clear message if an unsupported - swap configuration remains. - -The script must use the existing LUKS volume safely when re-run. It must abort -instead of attempting to migrate or overwrite data; migration from an existing -unencrypted server is outside the scope of this plan. +Before unlocking, SSH should work while the mapper, mounts, and application +services remain inactive. A wrong passphrase leaves the mapper closed. If a +mount command fails, stop there and inspect the device and `/etc/fstab` rather +than starting services. -### LUKS header backup - -After formatting, create a mode-0600 header backup named with the LUKS UUID, -for example: - -```text -/root/appdata-luks-header-.img -``` - -The installer must prominently instruct the operator to copy this file to -offline storage, verify the copy, and delete the server copy. The header backup -and the passphrase must be stored separately. Losing both usable LUKS headers -and their backup makes the data unrecoverable. - -## Manual boot and unlock behavior - -Create a custom `odoo-app.target` that groups the protected services. It must -not be wanted by `multi-user.target` and therefore must not start during normal -boot. - -Install `/usr/local/sbin/unlock-appdata` as a root-owned executable. It must be -idempotent and perform these operations: - -1. Resolve the LUKS device from the `appdata` crypttab entry. -2. Run `cryptsetup open` interactively unless the mapper is already open. -3. Mount `/srv/secure` unless it is already mounted. -4. Verify that `/srv/secure` is backed by `/dev/mapper/appdata`. -5. Mount and verify every encrypted bind mount. -6. Start `odoo-app.target` only after all mount checks pass. - -Normal reboot procedure: +Useful inspection commands: ```bash -ssh server -sudo unlock-appdata -``` - -Expected state before unlocking: - -```text -Ubuntu and SSH running -LUKS mapper closed -/srv/secure not mounted -PostgreSQL stopped -Docker and containerd stopped -Nginx stopped -Rclone mount stopped -All deploy-manager agents stopped -``` - -Systemd drop-ins must use mount requirements and mount-point conditions for -PostgreSQL, Docker, containerd, nginx, rclone, and the deployment agents. Even -if a service is started directly or by a package upgrade, it must fail closed -instead of writing into an unencrypted directory hidden beneath a missing bind -mount. - -Docker containers retain `--restart unless-stopped`; they restart normally when -Docker is started by `odoo-app.target` after unlocking. - -## Odoo 19 release installation - -Keep the one-time host bootstrap separate from a small release installer. The -bootstrap invokes the release installer for Odoo 19. On a server that is -already configured with LUKS, a future Odoo release runs only its release -installer. - -The implementation uses `setup-appdata.sh` for volume provisioning, -`appdata.sh` for shared mount checks, and `install-release.sh` for release setup. -The first release setup leaves SSL sites disabled until the credentials and -wildcard certificate exist. Follow README.md, rerun `install-release.sh`, then -run `unlock-appdata`. Use `unlock-appdata --mount-only` to access storage without -starting services and `unlock-appdata --check` for a noninteractive check. - -The 19.0 branch hardcodes the release values. It must: - -- Verify that it is running from `/opt/odoo19`. -- Create and populate `/root/agent-venv19`. -- Install `deploy-manager19.service` with working directory `/opt/odoo19`. -- Start the Python agent with `--interface 127.0.0.1 --port 8019`. -- Make the service wanted by `odoo-app.target`, not `multi-user.target`. -- Install a uniquely named nginx virtual host for - `odoo19.eniemela.fi` listening with TLS on port 9019 and proxying to - `127.0.0.1:8019`. -- Reuse the existing basic-auth file and wildcard certificate. -- Preserve shared nginx routing maps and configuration when another release is - installed. - -The public agent URL is therefore: - -```text -https://odoo19.eniemela.fi:9019 +cryptsetup status appdata +lsblk -f +findmnt /srv/secure +findmnt /var/lib/postgresql +findmnt /var/lib/docker +swapon --show +systemctl status odoo-app.target ``` -DNS must point the hostname to the server, and the server and Hetzner firewalls -must permit TCP port 9019. Port 8019 remains loopback-only. - -Update `update.sh`, module clone instructions, image commands, and scheduled -backup examples to use `/opt/odoo19`, `/root/agent-venv19`, and -`deploy-manager19.service`. - -## Multi-version isolation +To mount the data for maintenance without starting services, stop after the +last `mount` command. To stop the application and lock the volume: -PostgreSQL, Docker, nginx, rclone, and the encrypted volume are shared between -Odoo versions. Instance UIDs, PostgreSQL database/role names, Docker volume -names, and assigned HTTP/gevent ports must therefore be globally unique. - -Add the following label to every Odoo 19 container: - -```text -odoo.version=19.0 +```bash +systemctl stop odoo-app.target +umount /var/lib/nginx +umount /var/log/postgresql +umount /var/log/nginx +umount /root/.cache/rclone +umount /root/.config/rclone +umount /etc/odoo +umount /var/lib/containerd +umount /var/lib/docker +umount /var/lib/postgresql +umount /srv/secure +cryptsetup close appdata ``` -The Odoo 19 agent must filter Docker discovery by this label. As a result: +## Recovery material -- `status()` returns only Odoo 19 containers in `instances`. -- The Odoo 19 scheduled backup job backs up only Odoo 19 containers. -- A future Odoo 20 agent can use its own label without duplicating Odoo 19 - backups. +The first-install procedure creates +`/root/appdata-luks-header-.img`. Copy it to offline storage, verify the +copy, and delete the server copy. Keep the header backup and passphrase +separately. Losing both usable LUKS headers and their backup makes the data +unrecoverable. -Keep XML-RPC method names and arguments unchanged. PostgreSQL status remains -host-wide because PostgreSQL has no equivalent version label and the shared -database cluster is intentional. - -Container creation in the 19.0 branch must consistently use: - -- Image `ghcr.io/elmeriniemela/odoo-src:19.0` -- Source mount `/opt/odoo19/src:/mnt:ro` -- Label `odoo.version=19.0` -- Manifest version prefix `19.0` - -Existing unlabeled containers are not adopted because this plan targets a -fresh server. - -## Shared nginx routing - -All version agents update the same hostname-to-container-port maps. Protect the -complete read-modify-write-validation-reload operation with a shared file lock -under `/run/lock` so simultaneous agent requests cannot discard each other's -changes. - -Write map files through a temporary file and atomically replace the destination. -Run `nginx -t` while holding the lock and reload nginx only when validation -succeeds. - -Remove the static-file shortcut rooted at `/opt/deploy-manager/src`. It cannot -select the correct source tree when Odoo 19 and Odoo 20 coexist. Proxy static -requests through the hostname-selected Odoo container so each instance serves -assets from its own release and mounted modules. - -## Secrets and logs - -- Keep `/etc/odoo` on LUKS because generated configuration files contain - PostgreSQL passwords. -- Keep the rclone configuration and VFS cache on LUKS because the configuration - contains S3 credentials and the backup encryption password, while the cache - may contain plaintext backup data. -- Keep the Cloudflare credential file on LUKS and update `ssl_wildcard()` to use - its encrypted path. -- Keep nginx and PostgreSQL application logs on LUKS because URLs, client - addresses, database names, queries, and errors may contain customer data. -- Do not log complete XML-RPC argument values. In particular, the `config()` - call can contain a database password. Log the method name without its full - arguments. -- Back up the rclone crypt password separately in an offline password manager. - LUKS recovery alone is not sufficient to decrypt remote backups. +The rclone `crypt` password also needs a separate offline backup. Recovering the +LUKS volume does not recover a lost backup-encryption password. ## Volume expansion -The volume has no partition table, so it can be expanded without rebuilding -the encryption layer. - -After increasing the Hetzner Volume size, unlock and mount it normally, then -run: +After expanding the Hetzner Volume, unlock and mount it normally, stop the +application services, and run: ```bash -sudo cryptsetup resize appdata -sudo resize2fs /dev/mapper/appdata +cryptsetup resize appdata +resize2fs /dev/mapper/appdata +lsblk +df -h /srv/secure ``` -Verify the mapper and filesystem sizes with `lsblk` and `df` before returning -the server to normal operation. - -## Validation and acceptance tests - -### Automated checks - -- Initialize the `agentd/cronsyl` submodule before running tests. -- Run the complete Python unit suite. -- Add tests for the Odoo 19 image, source path, Docker label, label-filtered - discovery, encrypted secret paths, and concurrent nginx map preservation. -- Run `bash -n` on every setup, update, and helper script. -- Run `systemd-analyze verify` on the target, release service, mount units, and - service drop-ins. -- Validate nginx configuration before installation and before every reload. -- Validate the README Mermaid diagram using the commands in `AGENTS.md`. - -### Disposable-server acceptance test - -1. Confirm that installation refuses the root disk, active devices, and - nonblank non-LUKS devices. -2. Complete installation on an empty volume and verify all bind mounts resolve - to `/dev/mapper/appdata`. -3. Reboot and verify SSH works while the mapper, mounts, and protected services - remain inactive. -4. Enter a wrong passphrase and confirm that no protected service starts. -5. Run `unlock-appdata` with the correct passphrase and confirm all mounts and - services start. -6. Verify that port 8019 listens only on loopback and that - `https://odoo19.eniemela.fi:9019` returns the expected basic-auth challenge. -7. Create an Odoo 19 instance and verify its image, version label, source mount, - database, encrypted Docker volume, backup, and hostname routing. -8. Run simulated Odoo 19 and Odoo 20 agents concurrently and verify that each - sees and backs up only its labeled containers. -9. Update nginx maps concurrently from both agents and verify that neither - version loses the other's routes. -10. Re-run the installers and confirm that the LUKS filesystem, persistent data, - nginx maps, and existing release services are preserved. +## Fresh-server checks -## Assumptions and exclusions +Before relying on a new installation: -- The target is a fresh Ubuntu server with a separate empty Hetzner Volume. -- Migrating an existing PostgreSQL cluster or Docker data directory is outside - scope. -- One PostgreSQL cluster, Docker daemon, nginx instance, rclone remote, and LUKS - volume are shared by all Odoo versions. -- Instance UIDs and ports are globally allocated and cannot be reused by - another version. -- Release selection is intentionally hardcoded per Git branch; there is no - runtime version argument or shared release configuration file. -- The LUKS passphrase, LUKS header backup, and rclone encryption password are - stored securely off the server. +1. Reboot and confirm SSH works while the application services remain stopped. +2. Confirm a wrong passphrase starts no service. +3. Unlock with the correct passphrase and mount each entry. +4. Confirm `findmnt` shows every sensitive path backed by `/srv/secure` and + `swapon --show` is empty. +5. Run `nginx -t`, start `odoo-app.target`, and confirm PostgreSQL, Docker, + nginx, rclone, and `deploy-manager19.service` are active. +6. Confirm port 8019 listens only on loopback and the public port 9019 requires + nginx basic authentication. diff --git a/README.md b/README.md index f65d973..024e273 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Deployment Manager for Odoo images ## Abstract -This project automates self-hosted Odoo deployments on a Linux host. It provides a custom Odoo Docker image, host bootstrap scripts (Docker/PostgreSQL/nginx/systemd), and an XML-RPC deployment manager for instance lifecycle tasks such as create/reset/restart/upgrade, hostname-to-port routing updates, and SSL certificate management. It also handles client-side encrypted database and filestore backup/restore workflows using `pg_dump`/`pg_restore` and `rclone` (with zero-knowledge `crypt` overlay), with scheduled retention cleanup. +This project supports self-hosted Odoo deployments on a Linux host. It provides a custom Odoo Docker image, a short one-time Ubuntu setup command list, and an XML-RPC deployment manager for instance lifecycle tasks such as create/reset/restart/upgrade, hostname-to-port routing updates, and SSL certificate management. It also handles client-side encrypted database and filestore backup/restore workflows using `pg_dump`/`pg_restore` and `rclone` (with zero-knowledge `crypt` overlay), with scheduled retention cleanup. The XML-RPC API can be used from an Odoo instance to do self upgrades of Odoo source code. @@ -99,10 +99,11 @@ flowchart LR #### Prerequisite -Use a fresh Ubuntu LTS server and a separate empty Hetzner Volume. Attach the -volume without formatting or automatically mounting it. The installer refuses -existing application data; this is not an in-place migration. See [LUKS.md](LUKS.md) -for the storage layout, recovery precautions, and acceptance tests. +Use a fresh Ubuntu 24.04 LTS server and a separate empty Hetzner Volume. Attach the +volume without formatting or automatically mounting it. This is not an in-place +migration. The commands below format the selected device, so inspect it carefully +first. See [LUKS.md](LUKS.md) +for the storage layout and recovery precautions. Point `odoo19.eniemela.fi` to this server and allow TCP 9019 in the host and Hetzner firewalls. The agent is available at `https://odoo19.eniemela.fi:9019` @@ -116,52 +117,139 @@ through nginx basic authentication; its backend is loopback-only on port 8019. #### Installation -Run the following as root. Formatting requests confirmation and a LUKS passphrase -interactively; the passphrase is never stored on the server. - -* `git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/odoo19` -* `cd /opt/odoo19` -* `lsblk -f` — identify the empty Hetzner Volume and its `/dev/disk/by-id/` path. -* `bash ./ubuntu-install.sh /dev/disk/by-id/` -* Copy `/root/appdata-luks-header-.img` to offline storage, verify the copy, then delete the server copy. Keep the passphrase separately. -* `htpasswd -B -C 12 -c /etc/nginx/.htpasswd cloud` # Use bcrypt (-B) with cost 12 -* `chown root:www-data /etc/nginx/.htpasswd && chmod 640 /etc/nginx/.htpasswd` -* `vim /root/.config/rclone/rclone.conf` -* `vim /srv/secure/secrets/cloudflare.ini` -* `source /root/agent-venv19/bin/activate` -* `export TMPDIR=/srv/secure/tmp` -* `python -m agentd.api ssl_wildcard` -* `bash ./install-release.sh` — validate and enable the SSL sites after creating the certificate. -* `unlock-appdata` -* `su - postgres -c "createuser -s root"` — first installation only. +Run the following as root. Replace the device path everywhere. `luksFormat` and +`mkfs.ext4` destroy data on that device, so confirm the `lsblk` and `wipefs` +output before continuing. LUKS asks for the passphrase interactively and does +not store it on the server. + +```bash +git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/odoo19 +cd /opt/odoo19 +apt update +apt install -y cryptsetup +lsblk -f +readlink -e /dev/disk/by-id/ +wipefs --no-act /dev/disk/by-id/ +cryptsetup luksFormat --type luks2 /dev/disk/by-id/ +cryptsetup open /dev/disk/by-id/ appdata +mkfs.ext4 /dev/mapper/appdata +cryptsetup luksUUID /dev/disk/by-id/ +cryptsetup luksHeaderBackup /dev/disk/by-id/ --header-backup-file /root/appdata-luks-header-.img +``` + +Add the UUID printed above to `/etc/crypttab`: + +```text +appdata UUID= none luks,noauto +``` + +Disable swap (`swapoff --all`), remove or comment its entries in `/etc/fstab`, +and run `systemctl mask swap.target`. Then add these entries to `/etc/fstab`: + +```fstab +/dev/mapper/appdata /srv/secure ext4 noauto 0 2 +/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0 +/srv/secure/docker /var/lib/docker none noauto,bind 0 0 +/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0 +/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0 +/srv/secure/rclone-config /root/.config/rclone none noauto,bind 0 0 +/srv/secure/rclone-cache /root/.cache/rclone none noauto,bind 0 0 +/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0 +/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0 +/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0 +``` + +Create and mount the encrypted directories before installing the services: + +```bash +install -d /srv/secure +mount /srv/secure +install -d /srv/secure/postgresql /srv/secure/docker /srv/secure/containerd /srv/secure/odoo-config +install -d /srv/secure/rclone-config /srv/secure/rclone-cache /srv/secure/logs/nginx /srv/secure/logs/postgresql /srv/secure/nginx-temp +install -d -m 0700 /srv/secure/secrets +install -d -m 0711 /srv/secure/tmp +install -d /var/lib/postgresql /var/lib/docker /var/lib/containerd /etc/odoo +install -d /root/.config/rclone /root/.cache/rclone /var/log/nginx /var/log/postgresql /var/lib/nginx +mount /var/lib/postgresql +mount /var/lib/docker +mount /var/lib/containerd +mount /etc/odoo +mount /root/.config/rclone +mount /root/.cache/rclone +mount /var/log/nginx +mount /var/log/postgresql +mount /var/lib/nginx +bash ./ubuntu-install.sh +``` + +The installer is intentionally a one-time, linear list of package and file +installation commands. Read it before running it; it has no device selection, +formatting logic, loops, or conditional branches. + +Finish the configuration and start the application services: + +```bash +htpasswd -B -C 12 -c /etc/nginx/.htpasswd cloud +chown root:www-data /etc/nginx/.htpasswd +chmod 640 /etc/nginx/.htpasswd +vim /root/.config/rclone/rclone.conf +vim /srv/secure/secrets/cloudflare.ini +export TMPDIR=/srv/secure/tmp +/root/agent-venv19/bin/python -m agentd.api ssl_wildcard +ln -s /etc/nginx/sites-available/00_agent19.conf /etc/nginx/sites-enabled/00_agent19.conf +ln -s /etc/nginx/sites-available/odoo.conf /etc/nginx/sites-enabled/odoo.conf +nginx -t +systemctl start odoo-app.target +su - postgres -c "createuser -s root" +``` + +Copy `/root/appdata-luks-header-.img` to offline storage, verify the copy, +then delete the server copy. Keep the passphrase separately. The bootstrap installs Docker's loopback API override automatically. Configure the Loki Docker logging plugin below before creating Odoo containers. After every reboot, Ubuntu and SSH are available but application services stay -stopped. Run `sudo unlock-appdata` to unlock the volume, verify all encrypted bind -mounts, and start PostgreSQL, Docker/containerd, nginx, rclone, and all installed -agents. A wrong passphrase or missing mount prevents startup. Use -`sudo unlock-appdata --mount-only` when you need to edit credentials before starting -services, or `sudo unlock-appdata --check` to verify the mounted storage. +stopped. Unlock, mount, and start them with the same ordinary commands: + +```bash +cryptsetup open /dev/disk/by-id/ appdata +mount /srv/secure +mount /var/lib/postgresql +mount /var/lib/docker +mount /var/lib/containerd +mount /etc/odoo +mount /root/.config/rclone +mount /root/.cache/rclone +mount /var/log/nginx +mount /var/log/postgresql +mount /var/lib/nginx +nginx -t +systemctl start odoo-app.target +``` + +The systemd drop-ins installed by `ubuntu-install.sh` prevent protected services +from starting while any encrypted mount unit is inactive. Use `findmnt /srv/secure` +and `findmnt /var/lib/docker` to inspect mounts, and `systemctl status +odoo-app.target` to inspect the services. The encrypted filesystem also contains nginx request-body temporary files, agent temporary files and rotating agent logs. Do not use unencrypted `/tmp` or -`/var/tmp` for database dumps. Persistent swap is disabled. Native swap units or -generators must be removed if the installer reports them. +`/var/tmp` for database dumps. Persistent swap is disabled. Confirm that +`swapon --show` is empty before starting the application services. -For a host-bootstrap rerun, stop `odoo-app.target` first. Reuse the same configured -volume; the script preserves its filesystem and secrets and rejects conflicting -fstab entries. Interrupted setup may leave runtime service masks; rerun the host -bootstrap to finish configuring guards and remove the masks. +Do not rerun the one-time installer as an update mechanism. Use `./update.sh` for +this release and make later host configuration changes as explicit commands. #### Adding Odoo 20 later Create a 20.0 branch and replace release-specific 19 values with 20, including the -service/logrotate filenames. Clone that branch to `/opt/odoo20`, unlock the host, -and run only `bash ./install-release.sh` from the new clone. Configure DNS and -TCP 9020 for `https://odoo20.eniemela.fi:9020`; the backend uses `127.0.0.1:8020`. -Run `unlock-appdata` again to start newly installed agents. +service/logrotate filenames. Clone that branch to `/opt/odoo20`, create its Python +virtualenv, install its systemd unit, appdata drop-in, logrotate file, and nginx +site with the corresponding individual commands from `ubuntu-install.sh`. Configure +DNS and TCP 9020 for `https://odoo20.eniemela.fi:9020`; the backend uses +`127.0.0.1:8020`. Validate nginx, enable the new agent, and start it through +`odoo-app.target`. Each agent discovers and backs up only containers with its release's `odoo.version` label. UIDs and HTTP/gevent ports must be unique across the entire @@ -224,7 +312,7 @@ Odoo container. Updating one clone with `./update.sh` restarts only its agent. * `systemctl restart rclone-mount.service` * Scheduled cron: * `crontab -e` - * `30 00 * * * /usr/local/sbin/unlock-appdata --check && cd /opt/odoo19 && TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1` + * `30 00 * * * mountpoint -q /srv/secure && cd /opt/odoo19 && TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1` ##### Copying existing unencrypted backups to the new encrypted bucket: If you have existing plaintext backups in `odoobackup1` and wish to copy them into the new encrypted bucket: diff --git a/agentd/tests/test_appdata.py b/agentd/tests/test_appdata.py deleted file mode 100644 index a4e1bb8..0000000 --- a/agentd/tests/test_appdata.py +++ /dev/null @@ -1,56 +0,0 @@ -import subprocess -import unittest -from pathlib import Path - - -class AppdataTests(unittest.TestCase): - """Exercise fail-closed checks without root, real mounts or disk writes.""" - - def check(self, override='', command='check_appdata'): - script = ''' -set -euo pipefail -source ./appdata.sh -appdata_device() { echo /dev/test-volume; } -cryptsetup() { echo 'device: /dev/test-volume'; } -readlink() { echo "${@: -1}"; } -mountpoint() { return 0; } -findmnt() { - case "$2" in - MAJ:MIN) echo 253:0 ;; - FSTYPE) echo ext4 ;; - FSROOT) echo / ;; - esac -} -lsblk() { echo 253:0; } -stat() { echo 100:200; } -swapon() { :; } -mount() { echo unexpected-mount; exit 99; } -''' - return subprocess.run( - ['bash', '-c', script + override + '\n' + command], - cwd=Path(__file__).resolve().parents[2], - capture_output=True, text=True, - ) - - def test_accepts_verified_mounts_and_repeated_mount_request(self): - for command in ('check_appdata', 'mount_appdata'): - result = self.check(command=command) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertNotIn('unexpected-mount', result.stdout) - - def test_rejects_missing_or_wrong_mounts_and_swap(self): - cases = [ - ('cryptsetup() { echo "device: /dev/other"; }', 'wrong device'), - ('mountpoint() { return 1; }', '/srv/secure is not mounted'), - ('lsblk() { echo 8:1; }', 'not backed by appdata'), - ('mountpoint() { [[ "$2" != /var/lib/docker ]]; }', '/var/lib/docker is not mounted'), - ('stat() { echo "$3"; }', 'not bound to'), - ('stat() { return 1; }', 'Cannot inspect'), - ('readlink() { return 1; }', 'Cannot resolve'), - ('swapon() { echo /swapfile; }', 'Disable swap'), - ] - for override, message in cases: - with self.subTest(message=message): - result = self.check(override) - self.assertNotEqual(result.returncode, 0) - self.assertIn(message, result.stderr) diff --git a/appdata.sh b/appdata.sh deleted file mode 100644 index 6db4d8c..0000000 --- a/appdata.sh +++ /dev/null @@ -1,82 +0,0 @@ -# Shared by setup-appdata.sh and /usr/local/sbin/unlock-appdata. -appdata_mounts=( - postgresql:/var/lib/postgresql - docker:/var/lib/docker - containerd:/var/lib/containerd - odoo-config:/etc/odoo - rclone-config:/root/.config/rclone - rclone-cache:/root/.cache/rclone - logs/nginx:/var/log/nginx - logs/postgresql:/var/log/postgresql - nginx-temp:/var/lib/nginx -) - -fail() { - echo "$*" >&2 - exit 1 -} - -appdata_guard() { - # Requisite checks existing mounts without pulling in a locked LUKS device. - local entry mounts=(srv-secure.mount) - for entry in "${appdata_mounts[@]}"; do - mounts+=("$(systemd-escape --path --suffix=mount "${entry#*:}")") - done - # Ubuntu's postgresql@ unit otherwise pulls its data mount in on startup. - printf '[Unit]\nRequiresMountsFor=\nRequisite=%s\nAfter=%s\nPartOf=odoo-app.target\nConditionPathIsMountPoint=/srv/secure\n' "${mounts[*]}" "${mounts[*]}" - if [[ "$1" != *.socket ]]; then - printf '\n[Service]\nExecStartPre=+/usr/local/sbin/unlock-appdata --check\nEnvironment=TMPDIR=/srv/secure/tmp\n' - fi -} - -appdata_device() { - local uuid - uuid=$(awk '$1 == "appdata" {print $2; count++} END {if (count != 1) exit 1}' /etc/crypttab) || fail 'Expected one appdata entry in /etc/crypttab.' - [[ "$uuid" == UUID=* ]] || fail 'appdata must use a LUKS UUID.' - readlink -e "/dev/disk/by-uuid/${uuid#UUID=}" -} - -check_mapper() { - local device expected - device=$(cryptsetup status appdata | awk '$1 == "device:" {print $2}') - device=$(readlink -e "$device") || fail 'Cannot resolve the appdata backing device.' - expected=$(appdata_device) || fail 'Cannot resolve the configured appdata device.' - [[ -n "$device" && "$device" == "$expected" ]] || fail 'appdata is mapped to the wrong device.' -} - -check_secure() { - check_mapper - mountpoint -q /srv/secure || fail '/srv/secure is not mounted.' - [[ $(findmnt -nro MAJ:MIN -M /srv/secure) == "$(lsblk -dnro MAJ:MIN /dev/mapper/appdata)" ]] || fail '/srv/secure is not backed by appdata.' - [[ $(findmnt -nro FSTYPE -M /srv/secure) == ext4 && $(findmnt -nro FSROOT -M /srv/secure) == / ]] || fail '/srv/secure must mount the root of the ext4 filesystem.' -} - -check_bind() { - local source=$1 target=$2 source_inode target_inode - mountpoint -q "$target" || fail "$target is not mounted." - source_inode=$(stat -c '%d:%i' "$source") || fail "Cannot inspect $source." - target_inode=$(stat -c '%d:%i' "$target") || fail "Cannot inspect $target." - [[ "$source_inode" == "$target_inode" ]] || fail "$target is not bound to $source." -} - -check_appdata() { - check_secure - local entry - for entry in "${appdata_mounts[@]}"; do - check_bind "/srv/secure/${entry%%:*}" "${entry#*:}" - done - [[ -z $(swapon --show --noheadings) ]] || fail 'Disable swap before starting application services.' -} - -mount_appdata() { - check_mapper - mountpoint -q /srv/secure || mount /srv/secure - check_secure - local entry target - for entry in "${appdata_mounts[@]}"; do - target=${entry#*:} - mountpoint -q "$target" || mount "$target" - check_bind "/srv/secure/${entry%%:*}" "$target" - done - check_appdata -} diff --git a/apt/docker.sources b/apt/docker.sources new file mode 100644 index 0000000..9beca7a --- /dev/null +++ b/apt/docker.sources @@ -0,0 +1,6 @@ +Types: deb +URIs: https://download.docker.com/linux/ubuntu +Suites: noble +Components: stable +Architectures: amd64 arm64 +Signed-By: /etc/apt/keyrings/docker.asc diff --git a/install-release.sh b/install-release.sh deleted file mode 100755 index b20094b..0000000 --- a/install-release.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -euo pipefail -cd "$(dirname "$0")" -[[ $EUID == 0 && $(pwd -P) == /opt/odoo19 ]] || { echo 'Run as root from /opt/odoo19.' >&2; exit 1; } -/usr/local/sbin/unlock-appdata --check - -python3 -m venv /root/agent-venv19 -/root/agent-venv19/bin/python -m pip install -r requirements.txt -install -m 0644 systemd/system/deploy-manager19.service /etc/systemd/system/ -install -D -m 0644 logrotate/deploy-manager19 /etc/logrotate.d/deploy-manager19 -install -d /etc/systemd/system/deploy-manager19.service.d -# Reuse the host guard; future release branches install only their own unit. -install -m 0644 /etc/systemd/system/docker.service.d/appdata.conf /etc/systemd/system/deploy-manager19.service.d/appdata.conf -systemctl daemon-reload -systemctl enable deploy-manager19.service - -# First bootstrap installs the venv needed to obtain the wildcard certificate. -if [[ ! -f /etc/letsencrypt/live/eniemela.fi/fullchain.pem || ! -f /etc/letsencrypt/live/eniemela.fi/privkey.pem || ! -f /etc/nginx/.htpasswd ]]; then - echo 'Services have not been started. Configure /root/.config/rclone/rclone.conf and /srv/secure/secrets/cloudflare.ini.' - echo 'Create nginx basic auth and the wildcard certificate (README.md), then rerun ./install-release.sh and sudo unlock-appdata.' - exit 0 -fi - -# Share the agents' routing lock, including the validation and reload. -exec 9>/run/lock/odoo-nginx.lock -flock 9 -staging=$(mktemp -d /etc/nginx/.odoo19.XXXXXX) -trap 'rm -r -- "$staging"' EXIT -mkdir "$staging/sites-enabled" -cp -aL /etc/nginx/sites-enabled/. "$staging/sites-enabled/" -install -m 0644 nginx/sites-enabled/00_agent19.conf "$staging/00_agent19.conf" -ln -sfn "$staging/00_agent19.conf" "$staging/sites-enabled/00_agent19.conf" -if [[ ! -e /etc/nginx/sites-enabled/odoo.conf ]]; then - install -m 0644 nginx/sites-enabled/odoo.conf "$staging/odoo.conf" - ln -sfn "$staging/odoo.conf" "$staging/sites-enabled/odoo.conf" -fi -sed "s@include /etc/nginx/sites-enabled/\*;@include $staging/sites-enabled/*;@" /etc/nginx/nginx.conf > "$staging/nginx.conf" -nginx -t -c "$staging/nginx.conf" -install -d /etc/nginx/sites-available -install -m 0644 "$staging/00_agent19.conf" /etc/nginx/sites-available/00_agent19.conf -ln -sfn /etc/nginx/sites-available/00_agent19.conf /etc/nginx/sites-enabled/00_agent19.conf -if [[ -f "$staging/odoo.conf" ]]; then - install -m 0644 "$staging/odoo.conf" /etc/nginx/sites-available/odoo.conf - ln -sfn /etc/nginx/sites-available/odoo.conf /etc/nginx/sites-enabled/odoo.conf -fi -if systemctl is-active --quiet nginx; then - systemctl reload nginx -fi -echo 'Odoo 19 installed. Run sudo unlock-appdata to start the application services.' diff --git a/postgres/start.conf b/postgres/start.conf new file mode 100644 index 0000000..2905494 --- /dev/null +++ b/postgres/start.conf @@ -0,0 +1 @@ +manual diff --git a/setup-appdata.sh b/setup-appdata.sh deleted file mode 100755 index d180a6d..0000000 --- a/setup-appdata.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -set -euo pipefail -cd "$(dirname "$0")" -source ./appdata.sh - -[[ $EUID == 0 && $# == 1 && $1 == /dev/disk/by-id/* ]] || fail 'Usage: sudo ./setup-appdata.sh /dev/disk/by-id/' -device=$(readlink -e "$1") -[[ -b "$device" && $(lsblk -dnro TYPE "$device") == disk ]] || fail 'Supply a whole, unpartitioned volume.' -echo "Application data volume: $1 ($device)" - -# Reject the boot disk even when / is inside a partition or device mapper. -root_device=$(findmnt -nro MAJ:MIN /) -root_ancestors=$(lsblk -snro MAJ:MIN "/dev/block/$root_device") -[[ -n "$root_ancestors" ]] || fail 'Cannot identify the root disk.' -if grep -Fxq "$(lsblk -dnro MAJ:MIN "$device")" <<< "$root_ancestors"; then - fail 'Refusing the root disk.' -fi - -apt update -apt install -y cryptsetup - -signature=$(wipefs --no-act --noheadings --output TYPE "$device" | sort -u | xargs) -case "$signature" in - '') - [[ $(lsblk -nrpo NAME "$device" | wc -l) == 1 ]] || fail 'Volume has partitions or active holders.' - [[ -z $(lsblk -nro MOUNTPOINTS "$device" | xargs) ]] || fail 'Volume is mounted or used as swap.' - [[ ! -e /dev/mapper/appdata ]] || fail 'appdata already exists.' - if [[ -f /etc/crypttab ]] && grep -qE '^appdata[[:space:]]' /etc/crypttab; then - fail 'appdata is already configured; refusing a different volume.' - fi - ;; - crypto_LUKS) - cryptsetup isLuks --type luks2 "$device" || fail 'Expected LUKS2.' - [[ "$device" == "$(appdata_device)" ]] || fail 'Existing LUKS volume is not the configured appdata device.' - while read -r child; do - [[ "$child" == "$device" || "$child" == /dev/mapper/appdata ]] || fail "Unexpected active device: $child" - done < <(lsblk -nrpo NAME "$device") - if [[ -e /dev/mapper/appdata ]]; then - check_mapper - while read -r target; do - [[ -z "$target" ]] && continue - allowed=false - [[ "$target" == /srv/secure ]] && allowed=true - for entry in "${appdata_mounts[@]}"; do - [[ "$target" == "${entry#*:}" ]] && allowed=true - done - "$allowed" || fail "Unexpected mount on appdata: $target" - done < <(lsblk -nro MOUNTPOINTS /dev/mapper/appdata) - fi - ;; - *) fail "Refusing volume with existing signatures: $signature" ;; -esac - -# Never hide pre-existing plaintext data beneath a bind mount. -for entry in "${appdata_mounts[@]}"; do - target=${entry#*:} - [[ ! -L "$target" ]] || fail "Refusing symlink: $target" - if mountpoint -q "$target"; then - check_secure - check_bind "/srv/secure/${entry%%:*}" "$target" - elif [[ -d "$target" && -n $(find "$target" -mindepth 1 -maxdepth 1 -print -quit) ]]; then - fail "$target is not empty. Existing-server migration is not supported." - fi -done -if mountpoint -q /srv/secure; then - check_secure -elif [[ -d /srv/secure && -n $(find /srv/secure -mindepth 1 -maxdepth 1 -print -quit) ]]; then - fail '/srv/secure is not empty.' -fi -[[ ! -L /srv/secure ]] || fail 'Refusing /srv/secure symlink.' - -if [[ -z "$signature" ]]; then - # cryptsetup retains its interactive destructive-operation confirmation. - cryptsetup luksFormat --type luks2 "$device" - uuid=$(cryptsetup luksUUID "$device") - printf 'appdata UUID=%s none luks,noauto\n' "$uuid" >> /etc/crypttab - (umask 077; cryptsetup luksHeaderBackup "$device" --header-backup-file "/root/appdata-luks-header-$uuid.img") - echo "Copy /root/appdata-luks-header-$uuid.img offline, verify it, then delete the server copy. Store the passphrase separately." - udevadm trigger --action=change --name-match="$device" - udevadm settle -fi -cryptsetup status appdata >/dev/null || cryptsetup open "$device" appdata -check_mapper -signature=$(wipefs --no-act --noheadings --output TYPE /dev/mapper/appdata | sort -u | xargs) -case "$signature" in - '') mkfs.ext4 /dev/mapper/appdata ;; - ext4) ;; - *) fail "Expected ext4, found: $signature" ;; -esac - -# Disable fstab swap persistently. Native swap units/generators must be removed -# by the operator; silently leaving them configured could expose customer data. -swapoff --all -sed -i '/^[^#].*[[:space:]]swap[[:space:]]/s/^/# Disabled for LUKS appdata: /' /etc/fstab -systemctl daemon-reload -[[ -z $(systemctl list-unit-files --type=swap --state=enabled,generated --no-legend) ]] || fail 'Disable native swap units or swap generators, then re-run setup.' -systemctl mask swap.target - -fstab_entry() { - local source=$1 target=$2 type=$3 options=$4 pass=$5 existing - existing=$(awk -v target="$target" '$2 == target {print}' /etc/fstab) - if [[ -n "$existing" ]]; then - [[ "$existing" == "$source $target $type $options 0 $pass" ]] || fail "Conflicting fstab entry for $target." - else - printf '%s %s %s %s 0 %s\n' "$source" "$target" "$type" "$options" "$pass" >> /etc/fstab - fi -} - -install -d /srv/secure -fstab_entry /dev/mapper/appdata /srv/secure ext4 noauto 2 -systemctl daemon-reload -mountpoint -q /srv/secure || mount /srv/secure -check_secure -for entry in "${appdata_mounts[@]}"; do - source_dir=/srv/secure/${entry%%:*} - target=${entry#*:} - mkdir -p "$source_dir" "$target" - fstab_entry "$source_dir" "$target" none noauto,bind,x-systemd.requires=srv-secure.mount 0 -done -install -d -m 0700 /srv/secure/secrets /srv/secure/rclone-config /srv/secure/rclone-cache -install -d -m 0711 /srv/secure/tmp -systemctl daemon-reload -mount_appdata -install -D -m 0644 appdata.sh /usr/local/lib/odoo-appdata.sh -install -m 0755 unlock-appdata /usr/local/sbin/unlock-appdata diff --git a/systemd/system/appdata-mounts.conf b/systemd/system/appdata-mounts.conf new file mode 100644 index 0000000..bfddb66 --- /dev/null +++ b/systemd/system/appdata-mounts.conf @@ -0,0 +1,5 @@ +[Unit] +Requisite=srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount root-.config-rclone.mount root-.cache-rclone.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount +After=srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount root-.config-rclone.mount root-.cache-rclone.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount +PartOf=odoo-app.target +ConditionPathIsMountPoint=/srv/secure diff --git a/systemd/system/deploy-manager19.service b/systemd/system/deploy-manager19.service index 5ef6836..0db1a2a 100644 --- a/systemd/system/deploy-manager19.service +++ b/systemd/system/deploy-manager19.service @@ -1,7 +1,7 @@ [Unit] Description=deploy-manager Odoo 19 -Requires=docker.service postgresql.service rclone-mount.service -After=docker.service postgresql.service rclone-mount.service +Requires=docker.service postgresql@16-main.service rclone-mount.service +After=docker.service postgresql@16-main.service rclone-mount.service PartOf=odoo-app.target StartLimitIntervalSec=120 StartLimitBurst=30 @@ -9,7 +9,6 @@ StartLimitBurst=30 [Service] User=root WorkingDirectory=/opt/odoo19 -ExecStartPre=/usr/local/sbin/unlock-appdata --check ExecStart=/root/agent-venv19/bin/python -m agentd --interface 127.0.0.1 --port 8019 Restart=always RestartSec=1 diff --git a/systemd/system/docker.service.d/postgresql.conf b/systemd/system/docker.service.d/postgresql.conf new file mode 100644 index 0000000..e30a87d --- /dev/null +++ b/systemd/system/docker.service.d/postgresql.conf @@ -0,0 +1,3 @@ +[Unit] +Requires=postgresql@16-main.service +After=postgresql@16-main.service diff --git a/systemd/system/odoo-app.target b/systemd/system/odoo-app.target index e75e584..cb060e2 100644 --- a/systemd/system/odoo-app.target +++ b/systemd/system/odoo-app.target @@ -1,4 +1,4 @@ [Unit] -Description=Odoo application services (start after unlock-appdata) -Requires=postgresql.service containerd.service docker.service nginx.service rclone-mount.service -After=postgresql.service containerd.service docker.service nginx.service rclone-mount.service +Description=Odoo application services +Requires=postgresql@16-main.service containerd.service docker.service nginx.service rclone-mount.service +After=postgresql@16-main.service containerd.service docker.service nginx.service rclone-mount.service diff --git a/systemd/system/postgresql@.service.d/tmp.conf b/systemd/system/postgresql@.service.d/tmp.conf new file mode 100644 index 0000000..da03632 --- /dev/null +++ b/systemd/system/postgresql@.service.d/tmp.conf @@ -0,0 +1,2 @@ +[Service] +Environment=TMPDIR=/srv/secure/tmp/postgresql diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 49915e7..8a9d46e 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -1,87 +1,74 @@ #!/bin/bash -set -euo pipefail -cd "$(dirname "$0")" -source ./appdata.sh +set -euxo pipefail +cd /opt/odoo19 -[[ $EUID == 0 && $(pwd -P) == /opt/odoo19 ]] || fail 'Run as root from /opt/odoo19.' -for service in postgresql docker containerd nginx rclone-mount odoo-app.target; do - if systemctl is-active --quiet "$service"; then - fail "$service is active. Stop application services before running the host bootstrap." - fi -done -bash ./setup-appdata.sh "$@" +# One-time Ubuntu 24.04 host install. Prepare and mount the encrypted storage +# first by following README.md. +systemctl mask --runtime postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service -# Package post-install scripts must not start services before configuration. -# Leave these runtime masks in place on failure; a successful rerun removes them. -services=(postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service) -systemctl mask --runtime "${services[@]}" -# Install guards before packages: even a reboot during setup must fail closed. -for service in "${services[@]}" deploy-manager19.service; do - mkdir -p "/etc/systemd/system/$service.d" - appdata_guard "$service" > "/etc/systemd/system/$service.d/appdata.conf" -done +install -d /etc/systemd/system/postgresql@.service.d +install -d /etc/systemd/system/docker.service.d +install -d /etc/systemd/system/docker.socket.d +install -d /etc/systemd/system/containerd.service.d +install -d /etc/systemd/system/nginx.service.d +install -d /etc/systemd/system/rclone-mount.service.d +install -d /etc/systemd/system/deploy-manager19.service.d +install -m 0644 systemd/system/appdata-mounts.conf /etc/systemd/system/postgresql@.service.d/appdata-mounts.conf +install -m 0644 systemd/system/appdata-mounts.conf /etc/systemd/system/docker.service.d/appdata-mounts.conf +install -m 0644 systemd/system/appdata-mounts.conf /etc/systemd/system/docker.socket.d/appdata-mounts.conf +install -m 0644 systemd/system/appdata-mounts.conf /etc/systemd/system/containerd.service.d/appdata-mounts.conf +install -m 0644 systemd/system/appdata-mounts.conf /etc/systemd/system/nginx.service.d/appdata-mounts.conf +install -m 0644 systemd/system/appdata-mounts.conf /etc/systemd/system/rclone-mount.service.d/appdata-mounts.conf +install -m 0644 systemd/system/appdata-mounts.conf /etc/systemd/system/deploy-manager19.service.d/appdata-mounts.conf systemctl daemon-reload + +apt update apt install -y postgresql nginx ca-certificates curl gnupg vim tmux patchutils fuse3 python3-pip python3-venv unattended-upgrades apache2-utils rsync rclone -# https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository +# Docker: https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc chmod a+r /etc/apt/keyrings/docker.asc - -# Add the repository to Apt sources: -tee /etc/apt/sources.list.d/docker.sources < /etc/systemd/system/postgresql@.service.d/tmp.conf - -# Debian/Ubuntu cluster generators can enable PostgreSQL independently of the -# umbrella service. Start clusters explicitly from our manual target instead. -for pgconf in /etc/postgresql/*/main; do - pgversion=${pgconf#/etc/postgresql/} - pgversion=${pgversion%/main} - rsync -av postgres/ "$pgconf/" - printf 'manual\n' > "$pgconf/start.conf" - systemctl add-wants odoo-app.target "postgresql@$pgversion-main.service" - mkdir -p /etc/systemd/system/docker.service.d - printf '[Unit]\nRequires=postgresql@%s-main.service\nAfter=postgresql@%s-main.service\n' "$pgversion" "$pgversion" > /etc/systemd/system/docker.service.d/postgresql.conf -done +rsync -av postgres/ /etc/postgresql/16/main/ chown postgres:adm /var/log/postgresql chmod 0750 /var/log/postgresql + chown www-data:adm /var/log/nginx chmod 0755 /var/log/nginx chown www-data:adm /var/lib/nginx chmod 0755 /var/lib/nginx - -# Copy shared routing maps only on first install. A second release must never -# overwrite hostnames that the running agents have already registered. -rsync -av --ignore-existing nginx/conf.d/ /etc/nginx/conf.d/ +rsync -av nginx/conf.d/ /etc/nginx/conf.d/ install -m 0644 nginx/nginx.conf /etc/nginx/nginx.conf +install -d /etc/nginx/sites-available +install -m 0644 nginx/sites-enabled/00_agent19.conf /etc/nginx/sites-available/00_agent19.conf +install -m 0644 nginx/sites-enabled/odoo.conf /etc/nginx/sites-available/odoo.conf + install -d -m 0700 /root/backups -[[ -f /root/.config/rclone/rclone.conf ]] || install -m 0600 rclone.conf /root/.config/rclone/rclone.conf -[[ -f /srv/secure/secrets/cloudflare.ini ]] || install -m 0600 cloudflare.ini /srv/secure/secrets/cloudflare.ini -cp sshd/harden.conf /etc/ssh/sshd_config.d/harden.conf +install -D -m 0600 rclone.conf /root/.config/rclone/rclone.conf +install -D -m 0600 cloudflare.ini /srv/secure/secrets/cloudflare.ini +install -m 0644 sshd/harden.conf /etc/ssh/sshd_config.d/harden.conf systemctl reload ssh -mkdir -p /etc/apt/apt.conf.d/ -cp apt/apt.conf.d/* /etc/apt/apt.conf.d/ -systemctl restart unattended-upgrades +install -d /etc/apt/apt.conf.d +install -m 0644 apt/apt.conf.d/20auto-upgrades apt/apt.conf.d/50unattended-upgrades /etc/apt/apt.conf.d/ -systemctl unmask --runtime "${services[@]}" -systemctl disable "${services[@]}" +python3 -m venv /root/agent-venv19 +/root/agent-venv19/bin/python -m pip install -r requirements.txt + +systemctl unmask --runtime postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service +systemctl disable postgresql.service postgresql@16-main.service docker.service docker.socket containerd.service nginx.service rclone-mount.service deploy-manager19.service +systemctl enable deploy-manager19.service systemctl daemon-reload -bash ./install-release.sh +systemctl restart unattended-upgrades diff --git a/unlock-appdata b/unlock-appdata deleted file mode 100755 index 5989733..0000000 --- a/unlock-appdata +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -euo pipefail -source /usr/local/lib/odoo-appdata.sh - -[[ $EUID == 0 ]] || fail 'Run as root.' -case ${1:-} in - --check) check_appdata; exit 0 ;; - --mount-only|'') ;; - *) fail 'Usage: unlock-appdata [--check|--mount-only]' ;; -esac - -# Serialize interactive unlocks, but never hold this lock in service checks. -exec 9>/run/lock/odoo-appdata.lock -flock 9 -cryptsetup status appdata >/dev/null || cryptsetup open "$(appdata_device)" appdata -mount_appdata -[[ ${1:-} != --mount-only ]] || exit 0 - -flock /run/lock/odoo-nginx.lock nginx -t -systemctl start odoo-app.target -# A target may already be active after an earlier partial failure. Retry its -# services too, including every installed release and PostgreSQL cluster. -read -ra services <<< "$(systemctl show odoo-app.target -p Requires -p Wants --value | tr '\n' ' ')" -for service in "${services[@]}"; do - [[ "$service" == *.service ]] || continue - systemctl start "$service" - systemctl is-active --quiet "$service" || fail "$service did not start." -done From b9007a15fa4bcb1559310a535d3040f8aa2737a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 19:00:53 +0300 Subject: [PATCH 03/23] stop using homedir --- LUKS.md | 13 +++++++------ README.md | 24 +++++++++--------------- agentd/agentlib.py | 7 +++++-- agentd/api.py | 24 ++++++++++++------------ agentd/tests/test_agentlib.py | 12 ++++++------ agentd/tests/test_api.py | 8 ++++++++ agentd/tests/test_backup.py | 22 +++++++++++----------- systemd/system/appdata-mounts.conf | 4 ++-- systemd/system/rclone-mount.service | 4 ++-- ubuntu-install.sh | 4 ++-- 10 files changed, 64 insertions(+), 58 deletions(-) diff --git a/LUKS.md b/LUKS.md index b2c07d4..5b6f397 100644 --- a/LUKS.md +++ b/LUKS.md @@ -19,8 +19,9 @@ Hetzner Volume ├── docker → /var/lib/docker ├── containerd → /var/lib/containerd ├── odoo-config → /etc/odoo - ├── rclone-config → /root/.config/rclone - ├── rclone-cache → /root/.cache/rclone + ├── rclone-config + ├── rclone-cache + ├── backups (rclone mountpoint) ├── nginx-temp → /var/lib/nginx ├── tmp ├── secrets @@ -34,6 +35,10 @@ Docker, containerd, nginx, rclone, and the deployment agent are disabled at boot. Their systemd drop-ins declare every mount unit as a `Requisite`, so a direct service start fails while the encrypted storage is not mounted. +Rclone reads its configuration from `/srv/secure/rclone-config/rclone.conf`, +uses `/srv/secure/rclone-cache`, and mounts the decrypted backup view at +`/srv/secure/backups`. No rclone path is under `/root`. + Do not store database dumps in `/tmp` or `/var/tmp`. The agent uses `/srv/secure/tmp`, PostgreSQL uses `/srv/secure/tmp/postgresql`, and nginx uses its encrypted `/var/lib/nginx` bind mount. Persistent swap is disabled so @@ -50,8 +55,6 @@ mount /var/lib/postgresql mount /var/lib/docker mount /var/lib/containerd mount /etc/odoo -mount /root/.config/rclone -mount /root/.cache/rclone mount /var/log/nginx mount /var/log/postgresql mount /var/lib/nginx @@ -84,8 +87,6 @@ systemctl stop odoo-app.target umount /var/lib/nginx umount /var/log/postgresql umount /var/log/nginx -umount /root/.cache/rclone -umount /root/.config/rclone umount /etc/odoo umount /var/lib/containerd umount /var/lib/docker diff --git a/README.md b/README.md index 024e273..7022f0f 100644 --- a/README.md +++ b/README.md @@ -152,8 +152,6 @@ and run `systemctl mask swap.target`. Then add these entries to `/etc/fstab`: /srv/secure/docker /var/lib/docker none noauto,bind 0 0 /srv/secure/containerd /var/lib/containerd none noauto,bind 0 0 /srv/secure/odoo-config /etc/odoo none noauto,bind 0 0 -/srv/secure/rclone-config /root/.config/rclone none noauto,bind 0 0 -/srv/secure/rclone-cache /root/.cache/rclone none noauto,bind 0 0 /srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0 /srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0 /srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0 @@ -165,17 +163,15 @@ Create and mount the encrypted directories before installing the services: install -d /srv/secure mount /srv/secure install -d /srv/secure/postgresql /srv/secure/docker /srv/secure/containerd /srv/secure/odoo-config -install -d /srv/secure/rclone-config /srv/secure/rclone-cache /srv/secure/logs/nginx /srv/secure/logs/postgresql /srv/secure/nginx-temp -install -d -m 0700 /srv/secure/secrets +install -d /srv/secure/logs/nginx /srv/secure/logs/postgresql /srv/secure/nginx-temp +install -d -m 0700 /srv/secure/rclone-config /srv/secure/rclone-cache /srv/secure/backups /srv/secure/secrets install -d -m 0711 /srv/secure/tmp install -d /var/lib/postgresql /var/lib/docker /var/lib/containerd /etc/odoo -install -d /root/.config/rclone /root/.cache/rclone /var/log/nginx /var/log/postgresql /var/lib/nginx +install -d /var/log/nginx /var/log/postgresql /var/lib/nginx mount /var/lib/postgresql mount /var/lib/docker mount /var/lib/containerd mount /etc/odoo -mount /root/.config/rclone -mount /root/.cache/rclone mount /var/log/nginx mount /var/log/postgresql mount /var/lib/nginx @@ -192,7 +188,7 @@ Finish the configuration and start the application services: htpasswd -B -C 12 -c /etc/nginx/.htpasswd cloud chown root:www-data /etc/nginx/.htpasswd chmod 640 /etc/nginx/.htpasswd -vim /root/.config/rclone/rclone.conf +vim /srv/secure/rclone-config/rclone.conf vim /srv/secure/secrets/cloudflare.ini export TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.api ssl_wildcard @@ -219,8 +215,6 @@ mount /var/lib/postgresql mount /var/lib/docker mount /var/lib/containerd mount /etc/odoo -mount /root/.config/rclone -mount /root/.cache/rclone mount /var/log/nginx mount /var/log/postgresql mount /var/lib/nginx @@ -297,8 +291,8 @@ Odoo container. Updating one clone with `./update.sh` restarts only its agent. * All database dumps and Odoo filestores are encrypted client-side via rclone's `crypt` backend before upload to AWS S3. * Create a dedicated S3 bucket in AWS: `odoo-backups-crypt` (e.g. in `eu-north-1`). * Generate an obscured password for rclone config: - * `rclone obscure 'YourStrongSecretPassphrase'` -* In `/root/.config/rclone/rclone.conf`, add the `[backup-crypt]` section: + * `rclone obscure 'YourStrongSecretPassphrase' --config /srv/secure/rclone-config/rclone.conf` +* In `/srv/secure/rclone-config/rclone.conf`, add the `[backup-crypt]` section: ```ini [backup-crypt] type = crypt @@ -317,10 +311,10 @@ Odoo container. Updating one clone with `./update.sh` restarts only its agent. ##### Copying existing unencrypted backups to the new encrypted bucket: If you have existing plaintext backups in `odoobackup1` and wish to copy them into the new encrypted bucket: 1. Copy into the encrypted remote (reads unencrypted files, encrypts locally, writes to `odoo-backups-crypt`): - * `rclone copy awsbucket:odoobackup1 backup-crypt: --progress --transfers=16` -2. Verify the files through `/root/backups` or `rclone ls backup-crypt:` + * `rclone copy awsbucket:odoobackup1 backup-crypt: --config /srv/secure/rclone-config/rclone.conf --progress --transfers=16` +2. Verify the files through `/srv/secure/backups` or `rclone --config /srv/secure/rclone-config/rclone.conf ls backup-crypt:` 3. Once verified, the old unencrypted bucket `odoobackup1` can be kept as a fallback or purged: - * `rclone purge awsbucket:odoobackup1` + * `rclone purge awsbucket:odoobackup1 --config /srv/secure/rclone-config/rclone.conf` ##### Decrypting a single file without rclone: To manually decrypt a downloaded file without rclone (using only Python and `pip install pynacl`): diff --git a/agentd/agentlib.py b/agentd/agentlib.py index 5da5158..feab0e1 100644 --- a/agentd/agentlib.py +++ b/agentd/agentlib.py @@ -136,7 +136,7 @@ def save_odoo_config(uid, conf): fp.write(conf) def backups_mounted(): - return os.path.ismount('/root/backups') + return os.path.ismount('/srv/secure/backups') def ensure_backups_mounted(): assert backups_mounted(), "Backup dir not mounted. Something wrong with rclone-mount.service.." @@ -148,11 +148,14 @@ def ts_to_fname(ts): return f"{ts.strftime('%Y-%m-%dT%H-%M-%S')}.pgc" def dump_path(uid, trigger, fname, makedirs=False): - dirs = f'/root/backups/{uid}/{trigger}' + dirs = f'/srv/secure/backups/{uid}/{trigger}' if makedirs: os.makedirs(dirs, mode=0o700, exist_ok=True) return f'{dirs}/{fname}' +def rclone_command(command, *args): + return ['rclone', command, '--config', '/srv/secure/rclone-config/rclone.conf', *args] + def odoo_docker_run(uid, http_port, gevent_port): return [ 'docker', 'run', diff --git a/agentd/api.py b/agentd/api.py index eaaf52a..9c845fa 100644 --- a/agentd/api.py +++ b/agentd/api.py @@ -94,18 +94,18 @@ def backup(uid, trigger='manual'): agentlib.ensure_backups_mounted() fname = agentlib.ts_to_fname(datetime.datetime.now(datetime.timezone.utc)) agentlib.execute(['pg_dump', '--no-owner', '-Fc', '-f', agentlib.dump_path(uid, trigger, fname, makedirs=True), uid]) - agentlib.execute([ - 'rclone', 'copy', + agentlib.execute(agentlib.rclone_command( + 'copy', '--transfers=16', '--ignore-existing', # Odoo filestore checksums prohibit editing an existing filepath. f'/var/lib/docker/volumes/{uid}/_data/filestore/{uid}', f'backup-crypt:{uid}/filestore' - ]) - agentlib.execute([ - 'rclone', 'sync', + )) + agentlib.execute(agentlib.rclone_command( + 'sync', '--transfers=16', '--ignore-existing', # Odoo filestore checksums prohibit editing an existing filepath. f'/var/lib/docker/volumes/{uid}/_data/filestore/{uid}', f'backup-crypt:{uid}/previous_filestore' - ]) + )) _logger.info(f"Backup done: {trigger} backup for {uid}") return { 'backups': agentlib.list_backups(uid), @@ -115,11 +115,11 @@ def backup(uid, trigger='manual'): def fshealth(uid): agentlib.validate(uid=uid) fsproc = agentlib.execute( - cmd=[ - 'rclone', 'check', + cmd=agentlib.rclone_command( + 'check', '--one-way', f'/var/lib/docker/volumes/{uid}/_data/filestore/{uid}', f'backup-crypt:{uid}/filestore' - ], + ), check=False, ) return (fsproc.stderr or '').strip() @@ -144,12 +144,12 @@ def _restore(src_uid, dst_uid, trigger, backup_file): cur.execute(*args) commands = [ - [ - 'rclone', 'sync', + agentlib.rclone_command( + 'sync', '--transfers=16', '--ignore-existing', # Odoo filestore checksums prohibit editing an existing filepath. f'backup-crypt:{src_uid}/previous_filestore', f'/var/lib/docker/volumes/{dst_uid}/_data/filestore/{dst_uid}' - ], + ), ['chown', '1000:1000', '-R', f'/var/lib/docker/volumes/{dst_uid}/_data/filestore/{dst_uid}'], # TODO, better way to assign ownership to container user 'odoo'? ['pg_restore', '-Fc', '--no-owner', f'--role={dst_uid}', '-d', dst_uid, agentlib.dump_path(src_uid, trigger, backup_file)], ] diff --git a/agentd/tests/test_agentlib.py b/agentd/tests/test_agentlib.py index 6a403c3..89552b0 100644 --- a/agentd/tests/test_agentlib.py +++ b/agentd/tests/test_agentlib.py @@ -50,9 +50,9 @@ def test_dump_path_can_create_parent_directories(self): with patch("agentd.agentlib.os.makedirs") as makedirs: path = agentlib.dump_path("1a2b", "daily", "dump.pgc", makedirs=True) - self.assertEqual(path, "/root/backups/1a2b/daily/dump.pgc") + self.assertEqual(path, "/srv/secure/backups/1a2b/daily/dump.pgc") makedirs.assert_called_once_with( - "/root/backups/1a2b/daily", mode=0o700, exist_ok=True + "/srv/secure/backups/1a2b/daily", mode=0o700, exist_ok=True ) def test_save_odoo_config_creates_directory_and_writes_file(self): @@ -102,7 +102,7 @@ def test_backups_mounted_checks_expected_mountpoint(self): with patch("agentd.agentlib.os.path.ismount", return_value=True) as ismount: self.assertTrue(agentlib.backups_mounted()) - ismount.assert_called_once_with("/root/backups") + ismount.assert_called_once_with("/srv/secure/backups") def test_list_backups_returns_empty_list_when_backup_mount_is_absent(self): with patch("agentd.agentlib.backups_mounted", return_value=False): @@ -110,15 +110,15 @@ def test_list_backups_returns_empty_list_when_backup_mount_is_absent(self): def test_list_backups_extracts_metadata_from_dump_paths(self): paths = [ - "/root/backups/1a2b/daily/2026-07-11T12-30-45.pgc", - "/root/backups/1a2b/manual/2026-07-10T01-02-03.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-11T12-30-45.pgc", + "/srv/secure/backups/1a2b/manual/2026-07-10T01-02-03.pgc", ] with patch("agentd.agentlib.backups_mounted", return_value=True): with patch("agentd.agentlib.glob.glob", return_value=paths) as glob: backups = agentlib.list_backups("1a2b") - glob.assert_called_once_with("/root/backups/1a2b/*/*.pgc") + glob.assert_called_once_with("/srv/secure/backups/1a2b/*/*.pgc") self.assertEqual( backups, [ diff --git a/agentd/tests/test_api.py b/agentd/tests/test_api.py index f1246ee..2bcd14f 100644 --- a/agentd/tests/test_api.py +++ b/agentd/tests/test_api.py @@ -210,6 +210,8 @@ def test_backup_dumps_database_and_syncs_filestore(self, ensure_mounted): [ "rclone", "copy", + "--config", + "/srv/secure/rclone-config/rclone.conf", "--transfers=16", "--ignore-existing", "/var/lib/docker/volumes/1a2b/_data/filestore/1a2b", @@ -220,6 +222,8 @@ def test_backup_dumps_database_and_syncs_filestore(self, ensure_mounted): [ "rclone", "sync", + "--config", + "/srv/secure/rclone-config/rclone.conf", "--transfers=16", "--ignore-existing", "/var/lib/docker/volumes/1a2b/_data/filestore/1a2b", @@ -250,6 +254,8 @@ def test_fshealth_runs_one_way_rclone_check_and_returns_stderr(self): cmd=[ "rclone", "check", + "--config", + "/srv/secure/rclone-config/rclone.conf", "--one-way", "/var/lib/docker/volumes/1a2b/_data/filestore/1a2b", "backup-crypt:1a2b/filestore", @@ -296,6 +302,8 @@ def test_restore_helper_recreates_database_and_restores_dump(self): [ "rclone", "sync", + "--config", + "/srv/secure/rclone-config/rclone.conf", "--transfers=16", "--ignore-existing", "backup-crypt:1a2b/previous_filestore", diff --git a/agentd/tests/test_backup.py b/agentd/tests/test_backup.py index 65f5e35..84ddb05 100644 --- a/agentd/tests/test_backup.py +++ b/agentd/tests/test_backup.py @@ -41,11 +41,11 @@ def test_create_scheduled_backups_selects_trigger_from_current_date(self): def test_create_scheduled_backups_removes_all_but_three_latest_dumps(self): paths = [ - "/root/backups/1a2b/daily/2026-07-11T00-00-00.pgc", - "/root/backups/1a2b/daily/2026-07-10T00-00-00.pgc", - "/root/backups/1a2b/daily/2026-07-09T00-00-00.pgc", - "/root/backups/1a2b/daily/2026-07-08T00-00-00.pgc", - "/root/backups/1a2b/daily/2026-07-07T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-11T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-10T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-09T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-08T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-07T00-00-00.pgc", ] with patch("agentd.backup.datetime.date", fake_today(2026, 7, 14)): @@ -61,17 +61,17 @@ def test_create_scheduled_backups_removes_all_but_three_latest_dumps(self): self.assertEqual( remove.call_args_list, [ - call("/root/backups/1a2b/daily/2026-07-08T00-00-00.pgc"), - call("/root/backups/1a2b/daily/2026-07-07T00-00-00.pgc"), + call("/srv/secure/backups/1a2b/daily/2026-07-08T00-00-00.pgc"), + call("/srv/secure/backups/1a2b/daily/2026-07-07T00-00-00.pgc"), ], ) def test_dry_run_does_not_create_or_remove_backups(self): paths = [ - "/root/backups/1a2b/daily/2026-07-11T00-00-00.pgc", - "/root/backups/1a2b/daily/2026-07-10T00-00-00.pgc", - "/root/backups/1a2b/daily/2026-07-09T00-00-00.pgc", - "/root/backups/1a2b/daily/2026-07-08T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-11T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-10T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-09T00-00-00.pgc", + "/srv/secure/backups/1a2b/daily/2026-07-08T00-00-00.pgc", ] with patch("agentd.backup.datetime.date", fake_today(2026, 7, 14)): diff --git a/systemd/system/appdata-mounts.conf b/systemd/system/appdata-mounts.conf index bfddb66..665fa29 100644 --- a/systemd/system/appdata-mounts.conf +++ b/systemd/system/appdata-mounts.conf @@ -1,5 +1,5 @@ [Unit] -Requisite=srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount root-.config-rclone.mount root-.cache-rclone.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount -After=srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount root-.config-rclone.mount root-.cache-rclone.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount +Requisite=srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount +After=srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount PartOf=odoo-app.target ConditionPathIsMountPoint=/srv/secure diff --git a/systemd/system/rclone-mount.service b/systemd/system/rclone-mount.service index d8bf065..4462152 100644 --- a/systemd/system/rclone-mount.service +++ b/systemd/system/rclone-mount.service @@ -7,8 +7,8 @@ PartOf=odoo-app.target [Service] Type=notify User=root -ExecStart=/usr/bin/rclone mount backup-crypt: /root/backups --vfs-cache-mode full --cache-dir /root/.cache/rclone -ExecStop=/usr/bin/fusermount3 -u /root/backups +ExecStart=/usr/bin/rclone mount backup-crypt: /srv/secure/backups --config /srv/secure/rclone-config/rclone.conf --vfs-cache-mode full --cache-dir /srv/secure/rclone-cache +ExecStop=/usr/bin/fusermount3 -u /srv/secure/backups Restart=on-abort [Install] diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 8a9d46e..50afba7 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -55,8 +55,8 @@ install -d /etc/nginx/sites-available install -m 0644 nginx/sites-enabled/00_agent19.conf /etc/nginx/sites-available/00_agent19.conf install -m 0644 nginx/sites-enabled/odoo.conf /etc/nginx/sites-available/odoo.conf -install -d -m 0700 /root/backups -install -D -m 0600 rclone.conf /root/.config/rclone/rclone.conf +install -d -m 0700 /srv/secure/backups +install -m 0600 rclone.conf /srv/secure/rclone-config/rclone.conf install -D -m 0600 cloudflare.ini /srv/secure/secrets/cloudflare.ini install -m 0644 sshd/harden.conf /etc/ssh/sshd_config.d/harden.conf systemctl reload ssh From 700518baf1b90f2dbe916d77bcb0e2932cc22f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 19:28:11 +0300 Subject: [PATCH 04/23] add --- LUKS.md | 7 ++++--- README.md | 21 ++++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/LUKS.md b/LUKS.md index 5b6f397..208e37e 100644 --- a/LUKS.md +++ b/LUKS.md @@ -37,7 +37,7 @@ direct service start fails while the encrypted storage is not mounted. Rclone reads its configuration from `/srv/secure/rclone-config/rclone.conf`, uses `/srv/secure/rclone-cache`, and mounts the decrypted backup view at -`/srv/secure/backups`. No rclone path is under `/root`. +`/srv/secure/backups`. Do not store database dumps in `/tmp` or `/var/tmp`. The agent uses `/srv/secure/tmp`, PostgreSQL uses `/srv/secure/tmp/postgresql`, and nginx uses @@ -46,10 +46,11 @@ application memory is not written to unencrypted disk. ## Normal boot procedure -Replace the device path with the stable path recorded during installation: +Root's `.bashrc` defines `HETZNER_VOL` as the stable device path recorded during +installation. Use it when opening the volume: ```bash -cryptsetup open /dev/disk/by-id/ appdata +cryptsetup open "$HETZNER_VOL" appdata mount /srv/secure mount /var/lib/postgresql mount /var/lib/docker diff --git a/README.md b/README.md index 7022f0f..cd9daf7 100644 --- a/README.md +++ b/README.md @@ -117,8 +117,9 @@ through nginx basic authentication; its backend is loopback-only on port 8019. #### Installation -Run the following as root. Replace the device path everywhere. `luksFormat` and -`mkfs.ext4` destroy data on that device, so confirm the `lsblk` and `wipefs` +Run the following as root. Set the stable device path once in root's `.bashrc`; +later login shells can then use the same `HETZNER_VOL` variable. `luksFormat` +and `mkfs.ext4` destroy data on that device, so confirm the `lsblk` and `wipefs` output before continuing. LUKS asks for the passphrase interactively and does not store it on the server. @@ -128,13 +129,15 @@ cd /opt/odoo19 apt update apt install -y cryptsetup lsblk -f -readlink -e /dev/disk/by-id/ -wipefs --no-act /dev/disk/by-id/ -cryptsetup luksFormat --type luks2 /dev/disk/by-id/ -cryptsetup open /dev/disk/by-id/ appdata +echo 'export HETZNER_VOL="/dev/disk/by-id/"' >> /root/.bashrc +source /root/.bashrc +readlink -e "$HETZNER_VOL" +wipefs --no-act "$HETZNER_VOL" +cryptsetup luksFormat --type luks2 "$HETZNER_VOL" +cryptsetup open "$HETZNER_VOL" appdata mkfs.ext4 /dev/mapper/appdata -cryptsetup luksUUID /dev/disk/by-id/ -cryptsetup luksHeaderBackup /dev/disk/by-id/ --header-backup-file /root/appdata-luks-header-.img +cryptsetup luksUUID "$HETZNER_VOL" +cryptsetup luksHeaderBackup "$HETZNER_VOL" --header-backup-file /root/appdata-luks-header-.img ``` Add the UUID printed above to `/etc/crypttab`: @@ -209,7 +212,7 @@ After every reboot, Ubuntu and SSH are available but application services stay stopped. Unlock, mount, and start them with the same ordinary commands: ```bash -cryptsetup open /dev/disk/by-id/ appdata +cryptsetup open "$HETZNER_VOL" appdata mount /srv/secure mount /var/lib/postgresql mount /var/lib/docker From d6e0dedb410e4e6ca881467c743f9cde56cf8ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 19:45:59 +0300 Subject: [PATCH 05/23] Rename odoo19 to 19 --- README.md | 20 ++++++++++---------- agentd/agentlib.py | 2 +- agentd/tests/test_agentlib.py | 2 +- nginx/sites-enabled/00_agent19.conf | 2 +- systemd/system/deploy-manager19.service | 2 +- ubuntu-install.sh | 2 +- update.sh | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index cd9daf7..abb8c18 100644 --- a/README.md +++ b/README.md @@ -105,11 +105,11 @@ migration. The commands below format the selected device, so inspect it carefull first. See [LUKS.md](LUKS.md) for the storage layout and recovery precautions. -Point `odoo19.eniemela.fi` to this server and allow TCP 9019 in the host and -Hetzner firewalls. The agent is available at `https://odoo19.eniemela.fi:9019` +Point `19.eniemela.fi` to this server and allow TCP 9019 in the host and +Hetzner firewalls. The agent is available at `https://19.eniemela.fi:9019` through nginx basic authentication; its backend is loopback-only on port 8019. -* `scp .gitconfig odoo19.eniemela.fi:` +* `scp .gitconfig 19.eniemela.fi:` * `cd .ssh && ssh-keygen -f id_ecdsa -t ecdsa -b 521` * `cat id_ecdsa.pub` * go to github / settings / SSH keys / Add 'Odoo 19.0 Hetzner Server key' @@ -124,8 +124,8 @@ output before continuing. LUKS asks for the passphrase interactively and does not store it on the server. ```bash -git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/odoo19 -cd /opt/odoo19 +git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/19 +cd /opt/19 apt update apt install -y cryptsetup lsblk -f @@ -241,10 +241,10 @@ this release and make later host configuration changes as explicit commands. #### Adding Odoo 20 later Create a 20.0 branch and replace release-specific 19 values with 20, including the -service/logrotate filenames. Clone that branch to `/opt/odoo20`, create its Python +service/logrotate filenames. Clone that branch to `/opt/20`, create its Python virtualenv, install its systemd unit, appdata drop-in, logrotate file, and nginx site with the corresponding individual commands from `ubuntu-install.sh`. Configure -DNS and TCP 9020 for `https://odoo20.eniemela.fi:9020`; the backend uses +DNS and TCP 9020 for `https://20.eniemela.fi:9020`; the backend uses `127.0.0.1:8020`. Validate nginx, enable the new agent, and start it through `odoo-app.target`. @@ -285,7 +285,7 @@ Odoo container. Updating one clone with `./update.sh` restarts only its agent. * `docker login ghcr.io -u elmeriniemela` #### Building the image -* `docker build -t ghcr.io/elmeriniemela/odoo-src:19.0 /opt/odoo19` +* `docker build -t ghcr.io/elmeriniemela/odoo-src:19.0 /opt/19` * https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#building-container-images * `docker push ghcr.io/elmeriniemela/odoo-src:19.0` @@ -309,7 +309,7 @@ Odoo container. Updating one clone with `./update.sh` restarts only its agent. * `systemctl restart rclone-mount.service` * Scheduled cron: * `crontab -e` - * `30 00 * * * mountpoint -q /srv/secure && cd /opt/odoo19 && TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1` + * `30 00 * * * mountpoint -q /srv/secure && cd /opt/19 && TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1` ##### Copying existing unencrypted backups to the new encrypted bucket: If you have existing plaintext backups in `odoobackup1` and wish to copy them into the new encrypted bucket: @@ -324,7 +324,7 @@ To manually decrypt a downloaded file without rclone (using only Python and `pip * `python3 docs/decrypt.py ` #### Clone modules -* `cd /opt/odoo19/src` +* `cd /opt/19/src` * `git clone -b 19.0 git@github.com:elmeriniemela/tabularium.git` * `git clone -b 19.0 --depth=1 --single-branch git@github.com:odoo/odoo.git` * `git clone -b 19.0 --depth=1 --single-branch git@github.com:OCA/OpenUpgrade.git` diff --git a/agentd/agentlib.py b/agentd/agentlib.py index feab0e1..1b4b946 100644 --- a/agentd/agentlib.py +++ b/agentd/agentlib.py @@ -168,7 +168,7 @@ def odoo_docker_run(uid, http_port, gevent_port): '--log-opt', 'keep-file=true', '--log-opt', 'loki-batch-size=400', '--label', 'odoo.version=19.0', - '-v', '/opt/odoo19/src:/mnt:ro', + '-v', '/opt/19/src:/mnt:ro', '-v', f'/var/run/postgresql/:/var/run/postgresql/', '-v', f'/etc/odoo/{uid}:/etc/odoo:ro', '-v', f'{uid}:/var/lib/odoo', diff --git a/agentd/tests/test_agentlib.py b/agentd/tests/test_agentlib.py index 89552b0..ce33d4d 100644 --- a/agentd/tests/test_agentlib.py +++ b/agentd/tests/test_agentlib.py @@ -304,7 +304,7 @@ def test_odoo_docker_run_builds_expected_container_command(self): self.assertIn("[::1]:8072:8072", cmd) self.assertEqual(cmd[-1], "ghcr.io/elmeriniemela/odoo-src:19.0") self.assertEqual(cmd[cmd.index('--label') + 1], 'odoo.version=19.0') - self.assertIn('/opt/odoo19/src:/mnt:ro', cmd) + self.assertIn('/opt/19/src:/mnt:ro', cmd) def test_execute_wraps_subprocess_errors(self): error = subprocess.CalledProcessError( diff --git a/nginx/sites-enabled/00_agent19.conf b/nginx/sites-enabled/00_agent19.conf index 0a2e5cb..33517cb 100644 --- a/nginx/sites-enabled/00_agent19.conf +++ b/nginx/sites-enabled/00_agent19.conf @@ -1,7 +1,7 @@ server { listen 9019 ssl; listen [::]:9019 ssl; - server_name odoo19.eniemela.fi; + server_name 19.eniemela.fi; proxy_read_timeout 24h; proxy_connect_timeout 24h; diff --git a/systemd/system/deploy-manager19.service b/systemd/system/deploy-manager19.service index 0db1a2a..5517028 100644 --- a/systemd/system/deploy-manager19.service +++ b/systemd/system/deploy-manager19.service @@ -8,7 +8,7 @@ StartLimitBurst=30 [Service] User=root -WorkingDirectory=/opt/odoo19 +WorkingDirectory=/opt/19 ExecStart=/root/agent-venv19/bin/python -m agentd --interface 127.0.0.1 --port 8019 Restart=always RestartSec=1 diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 50afba7..b2aef6d 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euxo pipefail -cd /opt/odoo19 +cd /opt/19 # One-time Ubuntu 24.04 host install. Prepare and mount the encrypted storage # first by following README.md. diff --git a/update.sh b/update.sh index 02fadce..f43e8f9 100755 --- a/update.sh +++ b/update.sh @@ -2,7 +2,7 @@ set -e -cd /opt/odoo19 +cd /opt/19 git pull git submodule update systemctl restart deploy-manager19.service From dba91d1e3f59aab87681c22ddd69e694d61e818d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 20:01:08 +0300 Subject: [PATCH 06/23] change to https --- .gitmodules | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 0144030..dbac6e2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "agentd/cronsyl"] path = agentd/cronsyl - url = git@github.com:elmeriniemela/cronsyl.git + url = https://github.com/elmeriniemela/cronsyl.git diff --git a/README.md b/README.md index abb8c18..615da42 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ Point `19.eniemela.fi` to this server and allow TCP 9019 in the host and Hetzner firewalls. The agent is available at `https://19.eniemela.fi:9019` through nginx basic authentication; its backend is loopback-only on port 8019. +#### SSH key setup (optional, repos are public) * `scp .gitconfig 19.eniemela.fi:` * `cd .ssh && ssh-keygen -f id_ecdsa -t ecdsa -b 521` * `cat id_ecdsa.pub` From 8733b1bd6b16b4dd6421b6b808cb05686734ccd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 20:48:00 +0300 Subject: [PATCH 07/23] documentation --- LUKS.md | 47 ++++++++++++++++++++++++++++++++++++++++++-- README.md | 58 +++++++++++++++++++++++++++++-------------------------- 2 files changed, 76 insertions(+), 29 deletions(-) diff --git a/LUKS.md b/LUKS.md index 208e37e..65e8405 100644 --- a/LUKS.md +++ b/LUKS.md @@ -9,6 +9,12 @@ The exact first-install commands are in [README.md](README.md). They are kept in the documentation because choosing and formatting a block device should be a deliberate operator action, not hidden in a Bash script. +The initial format may use a verified kernel device name such as `/dev/sdb`. +Identify it by matching the `MODEL`, `SERIAL`, and `SIZE` from `lsblk` with the +Hetzner Console. Once LUKS exists, store its stable +`/dev/disk/by-uuid/` path as `HETZNER_VOL` in root's `.bashrc`; do not +persist `/dev/sdb`, because kernel device names can change. + ## Storage layout ```text @@ -98,11 +104,48 @@ cryptsetup close appdata ## Recovery material +The passphrase does not encrypt the data directly. It unlocks a protected copy +of the random volume-encryption key stored in a LUKS keyslot: + +```text +Passphrase + ↓ unlocks +LUKS header and keyslot + ↓ reveals +Volume-encryption key + ↓ decrypts +Data on the volume +``` + +`cryptsetup luksHeaderBackup` creates a binary copy of the LUKS metadata and +keyslot area. This includes the UUID, encryption parameters, keyslot information, +and encrypted copies of the volume key. It does not contain the passphrase in +plaintext, application files, or a complete image of the volume. + +If the live header or its keyslots are damaged, the encrypted data may still be +physically present but impossible to decrypt. A header backup can restore the +information needed to unlock it: + +```bash +cryptsetup luksHeaderRestore "$HETZNER_VOL" --header-backup-file "/offline/path/appdata-luks-header-.img" +``` + +Do not run `luksHeaderRestore` during normal operation. It overwrites the current +header; during recovery, preserve the damaged current header before attempting a +restore. A header backup also does not replace ordinary application-data backups. + The first-install procedure creates `/root/appdata-luks-header-.img`. Copy it to offline storage, verify the copy, and delete the server copy. Keep the header backup and passphrase -separately. Losing both usable LUKS headers and their backup makes the data -unrecoverable. +separately. The header backup alone cannot decrypt the volume, but the backup +together with a passphrase valid when it was created can—even if that passphrase +was later changed or removed from the live header. Create a new header backup +after intentional keyslot changes, and securely destroy old copies if a removed +passphrase must no longer work. Losing both the usable live header and every +header backup makes the data unrecoverable. + +See the official [`luksHeaderBackup` manual](https://man7.org/linux/man-pages/man8/cryptsetup-luksheaderbackup.8.html) +for the recovery and security warnings. The rclone `crypt` password also needs a separate offline backup. Recovering the LUKS volume does not recover a lost backup-encryption password. diff --git a/README.md b/README.md index 615da42..3cde6d3 100644 --- a/README.md +++ b/README.md @@ -118,47 +118,51 @@ through nginx basic authentication; its backend is loopback-only on port 8019. #### Installation -Run the following as root. Set the stable device path once in root's `.bashrc`; -later login shells can then use the same `HETZNER_VOL` variable. `luksFormat` -and `mkfs.ext4` destroy data on that device, so confirm the `lsblk` and `wipefs` -output before continuing. LUKS asks for the passphrase interactively and does -not store it on the server. +Run the following as root. Identify the volume by matching its `MODEL`, `SERIAL`, +and `SIZE` with the Hetzner Console. Use its `/dev/sdX` name only for the initial +format. Afterward, save the stable LUKS UUID path in root's `.bashrc` for later +login shells. `luksFormat` and `mkfs.ext4` destroy data on the selected device, +so confirm the `lsblk`, `findmnt`, and `wipefs` output before continuing. LUKS +asks for the passphrase interactively and does not store it on the server. ```bash git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/19 cd /opt/19 apt update apt install -y cryptsetup -lsblk -f -echo 'export HETZNER_VOL="/dev/disk/by-id/"' >> /root/.bashrc -source /root/.bashrc +lsblk -So NAME,MODEL,SERIAL,SIZE,TYPE +cryptsetup luksFormat --type luks2 /dev/sdX +LUKS_UUID="$(cryptsetup luksUUID /dev/sdX)" && echo "$LUKS_UUID" +udevadm trigger --action=change --name-match=/dev/sdX +udevadm settle +echo "export HETZNER_VOL=\"/dev/disk/by-uuid/$LUKS_UUID\"" >> /root/.bashrc +export HETZNER_VOL="/dev/disk/by-uuid/$LUKS_UUID" readlink -e "$HETZNER_VOL" -wipefs --no-act "$HETZNER_VOL" -cryptsetup luksFormat --type luks2 "$HETZNER_VOL" cryptsetup open "$HETZNER_VOL" appdata mkfs.ext4 /dev/mapper/appdata -cryptsetup luksUUID "$HETZNER_VOL" -cryptsetup luksHeaderBackup "$HETZNER_VOL" --header-backup-file /root/appdata-luks-header-.img +cryptsetup luksHeaderBackup "$HETZNER_VOL" --header-backup-file "/root/appdata-luks-header-$LUKS_UUID.img" ``` -Add the UUID printed above to `/etc/crypttab`: +Append the UUID entry to `/etc/crypttab`: -```text -appdata UUID= none luks,noauto +```bash +echo "appdata UUID=$LUKS_UUID none luks,noauto" >> /etc/crypttab ``` -Disable swap (`swapoff --all`), remove or comment its entries in `/etc/fstab`, -and run `systemctl mask swap.target`. Then add these entries to `/etc/fstab`: - -```fstab -/dev/mapper/appdata /srv/secure ext4 noauto 0 2 -/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0 -/srv/secure/docker /var/lib/docker none noauto,bind 0 0 -/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0 -/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0 -/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0 -/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0 -/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0 +Disable swap, comment its active `/etc/fstab` entries, and append the encrypted +filesystem and bind mounts. These append commands are for the one-time fresh-host +setup; do not run them twice. + +```bash +echo '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' >> /etc/fstab +echo '/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/docker /var/lib/docker none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' >> /etc/fstab +systemctl daemon-reload ``` Create and mount the encrypted directories before installing the services: From 265b387dfd49818277539394c35dbbff915b72ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 20:48:56 +0300 Subject: [PATCH 08/23] move commands to script --- README.md | 40 ++++++---------------------------------- ubuntu-install.sh | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 3cde6d3..a4c6488 100644 --- a/README.md +++ b/README.md @@ -149,46 +149,18 @@ Append the UUID entry to `/etc/crypttab`: echo "appdata UUID=$LUKS_UUID none luks,noauto" >> /etc/crypttab ``` -Disable swap, comment its active `/etc/fstab` entries, and append the encrypted -filesystem and bind mounts. These append commands are for the one-time fresh-host -setup; do not run them twice. +Run the one-time installer after the mapper is open and the crypttab entry is in +place: ```bash -echo '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' >> /etc/fstab -echo '/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/docker /var/lib/docker none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' >> /etc/fstab -systemctl daemon-reload -``` - -Create and mount the encrypted directories before installing the services: - -```bash -install -d /srv/secure -mount /srv/secure -install -d /srv/secure/postgresql /srv/secure/docker /srv/secure/containerd /srv/secure/odoo-config -install -d /srv/secure/logs/nginx /srv/secure/logs/postgresql /srv/secure/nginx-temp -install -d -m 0700 /srv/secure/rclone-config /srv/secure/rclone-cache /srv/secure/backups /srv/secure/secrets -install -d -m 0711 /srv/secure/tmp -install -d /var/lib/postgresql /var/lib/docker /var/lib/containerd /etc/odoo -install -d /var/log/nginx /var/log/postgresql /var/lib/nginx -mount /var/lib/postgresql -mount /var/lib/docker -mount /var/lib/containerd -mount /etc/odoo -mount /var/log/nginx -mount /var/log/postgresql -mount /var/lib/nginx bash ./ubuntu-install.sh ``` The installer is intentionally a one-time, linear list of package and file -installation commands. Read it before running it; it has no device selection, -formatting logic, loops, or conditional branches. +installation commands. It disables swap, appends the fixed `/etc/fstab` entries, +creates and mounts the encrypted directories, and installs the host services. +Read it before running it; it has no device selection, formatting logic, loops, +or conditional branches, and must not be run twice. Finish the configuration and start the application services: diff --git a/ubuntu-install.sh b/ubuntu-install.sh index b2aef6d..fdfd4ca 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -3,8 +3,37 @@ set -euxo pipefail cd /opt/19 -# One-time Ubuntu 24.04 host install. Prepare and mount the encrypted storage -# first by following README.md. +# One-time Ubuntu 24.04 host install. Create and open the LUKS volume and add its +# UUID to /etc/crypttab first by following README.md. +swapoff --all +sed -i '/^[^#].*[[:space:]]swap[[:space:]]/s/^/# Disabled for LUKS appdata: /' /etc/fstab +systemctl mask swap.target +echo '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' >> /etc/fstab +echo '/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/docker /var/lib/docker none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' >> /etc/fstab +systemctl daemon-reload + +install -d /srv/secure +mount /srv/secure +install -d /srv/secure/postgresql /srv/secure/docker /srv/secure/containerd /srv/secure/odoo-config +install -d /srv/secure/logs/nginx /srv/secure/logs/postgresql /srv/secure/nginx-temp +install -d -m 0700 /srv/secure/rclone-config /srv/secure/rclone-cache /srv/secure/backups /srv/secure/secrets +install -d -m 0711 /srv/secure/tmp +install -d /var/lib/postgresql /var/lib/docker /var/lib/containerd /etc/odoo +install -d /var/log/nginx /var/log/postgresql /var/lib/nginx +mount /var/lib/postgresql +mount /var/lib/docker +mount /var/lib/containerd +mount /etc/odoo +mount /var/log/nginx +mount /var/log/postgresql +mount /var/lib/nginx + systemctl mask --runtime postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service install -d /etc/systemd/system/postgresql@.service.d From 6c8aa19715c6202a291a232d4a5f07efdf2067de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 20:51:34 +0300 Subject: [PATCH 09/23] swap is off by default --- ubuntu-install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ubuntu-install.sh b/ubuntu-install.sh index fdfd4ca..d237eef 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -5,8 +5,6 @@ cd /opt/19 # One-time Ubuntu 24.04 host install. Create and open the LUKS volume and add its # UUID to /etc/crypttab first by following README.md. -swapoff --all -sed -i '/^[^#].*[[:space:]]swap[[:space:]]/s/^/# Disabled for LUKS appdata: /' /etc/fstab systemctl mask swap.target echo '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' >> /etc/fstab echo '/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0' >> /etc/fstab From bfe58b82d04d47ae93007a3963eace19dc936b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 20:55:32 +0300 Subject: [PATCH 10/23] migrate to ubuntu 26.04 --- README.md | 2 +- apt/docker.sources | 2 +- postgres/conf.d/pgtune.conf | 2 +- systemd/system/deploy-manager19.service | 4 ++-- systemd/system/docker.service.d/postgresql.conf | 4 ++-- systemd/system/odoo-app.target | 4 ++-- ubuntu-install.sh | 6 +++--- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a4c6488..6d70363 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ flowchart LR #### Prerequisite -Use a fresh Ubuntu 24.04 LTS server and a separate empty Hetzner Volume. Attach the +Use a fresh Ubuntu 26.04 LTS server and a separate empty Hetzner Volume. Attach the volume without formatting or automatically mounting it. This is not an in-place migration. The commands below format the selected device, so inspect it carefully first. See [LUKS.md](LUKS.md) diff --git a/apt/docker.sources b/apt/docker.sources index 9beca7a..dea5926 100644 --- a/apt/docker.sources +++ b/apt/docker.sources @@ -1,6 +1,6 @@ Types: deb URIs: https://download.docker.com/linux/ubuntu -Suites: noble +Suites: resolute Components: stable Architectures: amd64 arm64 Signed-By: /etc/apt/keyrings/docker.asc diff --git a/postgres/conf.d/pgtune.conf b/postgres/conf.d/pgtune.conf index 6f7bd42..fa93a62 100644 --- a/postgres/conf.d/pgtune.conf +++ b/postgres/conf.d/pgtune.conf @@ -1,4 +1,4 @@ -# DB Version: 14 +# DB Version: 18 # OS Type: linux # DB Type: oltp # Total Memory (RAM): 1 GB diff --git a/systemd/system/deploy-manager19.service b/systemd/system/deploy-manager19.service index 5517028..75dae79 100644 --- a/systemd/system/deploy-manager19.service +++ b/systemd/system/deploy-manager19.service @@ -1,7 +1,7 @@ [Unit] Description=deploy-manager Odoo 19 -Requires=docker.service postgresql@16-main.service rclone-mount.service -After=docker.service postgresql@16-main.service rclone-mount.service +Requires=docker.service postgresql@18-main.service rclone-mount.service +After=docker.service postgresql@18-main.service rclone-mount.service PartOf=odoo-app.target StartLimitIntervalSec=120 StartLimitBurst=30 diff --git a/systemd/system/docker.service.d/postgresql.conf b/systemd/system/docker.service.d/postgresql.conf index e30a87d..b8044fa 100644 --- a/systemd/system/docker.service.d/postgresql.conf +++ b/systemd/system/docker.service.d/postgresql.conf @@ -1,3 +1,3 @@ [Unit] -Requires=postgresql@16-main.service -After=postgresql@16-main.service +Requires=postgresql@18-main.service +After=postgresql@18-main.service diff --git a/systemd/system/odoo-app.target b/systemd/system/odoo-app.target index cb060e2..5807a94 100644 --- a/systemd/system/odoo-app.target +++ b/systemd/system/odoo-app.target @@ -1,4 +1,4 @@ [Unit] Description=Odoo application services -Requires=postgresql@16-main.service containerd.service docker.service nginx.service rclone-mount.service -After=postgresql@16-main.service containerd.service docker.service nginx.service rclone-mount.service +Requires=postgresql@18-main.service containerd.service docker.service nginx.service rclone-mount.service +After=postgresql@18-main.service containerd.service docker.service nginx.service rclone-mount.service diff --git a/ubuntu-install.sh b/ubuntu-install.sh index d237eef..29f55b1 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -3,7 +3,7 @@ set -euxo pipefail cd /opt/19 -# One-time Ubuntu 24.04 host install. Create and open the LUKS volume and add its +# One-time Ubuntu 26.04 host install. Create and open the LUKS volume and add its # UUID to /etc/crypttab first by following README.md. systemctl mask swap.target echo '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' >> /etc/fstab @@ -68,7 +68,7 @@ install -m 0644 systemd/system/postgresql@.service.d/tmp.conf /etc/systemd/syste install -D -m 0644 logrotate/deploy-manager19 /etc/logrotate.d/deploy-manager19 install -d -o postgres -g postgres -m 0700 /srv/secure/tmp/postgresql -rsync -av postgres/ /etc/postgresql/16/main/ +rsync -av postgres/ /etc/postgresql/18/main/ chown postgres:adm /var/log/postgresql chmod 0750 /var/log/postgresql @@ -95,7 +95,7 @@ python3 -m venv /root/agent-venv19 /root/agent-venv19/bin/python -m pip install -r requirements.txt systemctl unmask --runtime postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service -systemctl disable postgresql.service postgresql@16-main.service docker.service docker.socket containerd.service nginx.service rclone-mount.service deploy-manager19.service +systemctl disable postgresql.service postgresql@18-main.service docker.service docker.socket containerd.service nginx.service rclone-mount.service deploy-manager19.service systemctl enable deploy-manager19.service systemctl daemon-reload systemctl restart unattended-upgrades From 48cfaf1bf6aa3452467c367cf194132909e7d66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Tue, 8 Sep 2026 21:20:09 +0300 Subject: [PATCH 11/23] split script --- README.md | 16 +++++++++------- ubuntu-install-once.sh | 14 ++++++++++++++ ubuntu-install.sh | 34 ++++++++++++++-------------------- 3 files changed, 37 insertions(+), 27 deletions(-) create mode 100755 ubuntu-install-once.sh diff --git a/README.md b/README.md index 6d70363..319e502 100644 --- a/README.md +++ b/README.md @@ -149,18 +149,20 @@ Append the UUID entry to `/etc/crypttab`: echo "appdata UUID=$LUKS_UUID none luks,noauto" >> /etc/crypttab ``` -Run the one-time installer after the mapper is open and the crypttab entry is in -place: +Append the fixed `/etc/fstab` entries once, then run the host installer. The host +installer can be rerun after a partial failure; do not rerun +`ubuntu-install-once.sh`. ```bash +bash ./ubuntu-install-once.sh bash ./ubuntu-install.sh ``` -The installer is intentionally a one-time, linear list of package and file -installation commands. It disables swap, appends the fixed `/etc/fstab` entries, -creates and mounts the encrypted directories, and installs the host services. -Read it before running it; it has no device selection, formatting logic, loops, -or conditional branches, and must not be run twice. +Both files are linear command lists without loops or conditional branches. +`ubuntu-install-once.sh` only appends the fixed mount configuration. +`ubuntu-install.sh` disables swap, creates and mounts the encrypted directories, +and installs the host services. It uses idempotent systemd mount starts and does +not overwrite configured rclone or Cloudflare credentials when rerun. Finish the configuration and start the application services: diff --git a/ubuntu-install-once.sh b/ubuntu-install-once.sh new file mode 100755 index 0000000..383df6b --- /dev/null +++ b/ubuntu-install-once.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euxo pipefail + +# Run once on a fresh host. Re-running this file duplicates /etc/fstab entries. +echo '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' >> /etc/fstab +echo '/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/docker /var/lib/docker none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' >> /etc/fstab +echo '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' >> /etc/fstab +systemctl daemon-reload diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 29f55b1..132faba 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -3,34 +3,26 @@ set -euxo pipefail cd /opt/19 -# One-time Ubuntu 26.04 host install. Create and open the LUKS volume and add its -# UUID to /etc/crypttab first by following README.md. +# Ubuntu 26.04 host install. Create and open the LUKS volume, add its UUID to +# /etc/crypttab, and run ubuntu-install-once.sh first by following README.md. systemctl mask swap.target -echo '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' >> /etc/fstab -echo '/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/docker /var/lib/docker none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' >> /etc/fstab systemctl daemon-reload install -d /srv/secure -mount /srv/secure +systemctl start srv-secure.mount install -d /srv/secure/postgresql /srv/secure/docker /srv/secure/containerd /srv/secure/odoo-config install -d /srv/secure/logs/nginx /srv/secure/logs/postgresql /srv/secure/nginx-temp install -d -m 0700 /srv/secure/rclone-config /srv/secure/rclone-cache /srv/secure/backups /srv/secure/secrets install -d -m 0711 /srv/secure/tmp install -d /var/lib/postgresql /var/lib/docker /var/lib/containerd /etc/odoo install -d /var/log/nginx /var/log/postgresql /var/lib/nginx -mount /var/lib/postgresql -mount /var/lib/docker -mount /var/lib/containerd -mount /etc/odoo -mount /var/log/nginx -mount /var/log/postgresql -mount /var/lib/nginx +systemctl start var-lib-postgresql.mount +systemctl start var-lib-docker.mount +systemctl start var-lib-containerd.mount +systemctl start etc-odoo.mount +systemctl start var-log-nginx.mount +systemctl start var-log-postgresql.mount +systemctl start var-lib-nginx.mount systemctl mask --runtime postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service @@ -83,8 +75,10 @@ install -m 0644 nginx/sites-enabled/00_agent19.conf /etc/nginx/sites-available/0 install -m 0644 nginx/sites-enabled/odoo.conf /etc/nginx/sites-available/odoo.conf install -d -m 0700 /srv/secure/backups -install -m 0600 rclone.conf /srv/secure/rclone-config/rclone.conf -install -D -m 0600 cloudflare.ini /srv/secure/secrets/cloudflare.ini +cp --update=none rclone.conf /srv/secure/rclone-config/rclone.conf +chmod 0600 /srv/secure/rclone-config/rclone.conf +cp --update=none cloudflare.ini /srv/secure/secrets/cloudflare.ini +chmod 0600 /srv/secure/secrets/cloudflare.ini install -m 0644 sshd/harden.conf /etc/ssh/sshd_config.d/harden.conf systemctl reload ssh From b83f12040bada5621a3527e36faee9bcd957f08f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 17:42:44 +0300 Subject: [PATCH 12/23] fixes --- README.md | 10 +++++++--- ubuntu-install-once.sh => fstab-append.sh | 0 nginx/sites-enabled/00_agent19.conf | 7 +++++-- ubuntu-install.sh | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) rename ubuntu-install-once.sh => fstab-append.sh (100%) diff --git a/README.md b/README.md index 319e502..3e9734f 100644 --- a/README.md +++ b/README.md @@ -151,15 +151,15 @@ echo "appdata UUID=$LUKS_UUID none luks,noauto" >> /etc/crypttab Append the fixed `/etc/fstab` entries once, then run the host installer. The host installer can be rerun after a partial failure; do not rerun -`ubuntu-install-once.sh`. +`fstab-append.sh`. ```bash -bash ./ubuntu-install-once.sh +bash ./fstab-append.sh bash ./ubuntu-install.sh ``` Both files are linear command lists without loops or conditional branches. -`ubuntu-install-once.sh` only appends the fixed mount configuration. +`fstab-append.sh` only appends the fixed mount configuration. `ubuntu-install.sh` disables swap, creates and mounts the encrypted directories, and installs the host services. It uses idempotent systemd mount starts and does not overwrite configured rclone or Cloudflare credentials when rerun. @@ -172,8 +172,12 @@ chown root:www-data /etc/nginx/.htpasswd chmod 640 /etc/nginx/.htpasswd vim /srv/secure/rclone-config/rclone.conf vim /srv/secure/secrets/cloudflare.ini + export TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.api ssl_wildcard +# OR +sudo rsync -aHAX /etc/letsencrypt/ root@NEW_SERVER:/etc/letsencrypt/ + ln -s /etc/nginx/sites-available/00_agent19.conf /etc/nginx/sites-enabled/00_agent19.conf ln -s /etc/nginx/sites-available/odoo.conf /etc/nginx/sites-enabled/odoo.conf nginx -t diff --git a/ubuntu-install-once.sh b/fstab-append.sh similarity index 100% rename from ubuntu-install-once.sh rename to fstab-append.sh diff --git a/nginx/sites-enabled/00_agent19.conf b/nginx/sites-enabled/00_agent19.conf index 33517cb..2d46537 100644 --- a/nginx/sites-enabled/00_agent19.conf +++ b/nginx/sites-enabled/00_agent19.conf @@ -15,8 +15,11 @@ server { ssl_buffer_size 4k; # OCSP stapling - ssl_stapling on; - ssl_stapling_verify on; + # Let’s Encrypt removed OCSP URLs in May 2025 and shut down its OCSP service in August 2025. + # Let’s Encrypt announcement (https://letsencrypt.org/2024/12/05/ending-ocsp.html) + ssl_stapling off; + # ssl_stapling_verify on; + resolver 1.1.1.1 1.0.0.1 [2606:4700:4700::1111] [2606:4700:4700::1001]; # Cloudflare # Set HSTS to 365 days diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 132faba..410f44e 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -4,7 +4,7 @@ set -euxo pipefail cd /opt/19 # Ubuntu 26.04 host install. Create and open the LUKS volume, add its UUID to -# /etc/crypttab, and run ubuntu-install-once.sh first by following README.md. +# /etc/crypttab, and run fstab-append.sh first by following README.md. systemctl mask swap.target systemctl daemon-reload From 772df10f6b98b8c3afff76f005b7c812a59cb678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 18:02:05 +0300 Subject: [PATCH 13/23] fix apparmor mount error --- README.md | 11 ++++++----- apparmor/local/fusermount3 | 2 ++ fstab-append.sh => append.sh | 6 ++++++ ubuntu-install.sh | 2 +- 4 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 apparmor/local/fusermount3 rename fstab-append.sh => append.sh (74%) diff --git a/README.md b/README.md index 3e9734f..fd3a5dc 100644 --- a/README.md +++ b/README.md @@ -149,17 +149,18 @@ Append the UUID entry to `/etc/crypttab`: echo "appdata UUID=$LUKS_UUID none luks,noauto" >> /etc/crypttab ``` -Append the fixed `/etc/fstab` entries once, then run the host installer. The host -installer can be rerun after a partial failure; do not rerun -`fstab-append.sh`. +Append the fixed `/etc/fstab` entries and install the local AppArmor rule once, +then run the host installer. The host installer can be rerun after a partial +failure; do not rerun `append.sh`. ```bash -bash ./fstab-append.sh +bash ./append.sh bash ./ubuntu-install.sh ``` Both files are linear command lists without loops or conditional branches. -`fstab-append.sh` only appends the fixed mount configuration. +`append.sh` appends the fixed mount configuration and permits the rclone FUSE +mount at `/srv/secure/backups` in the local `fusermount3` AppArmor policy. `ubuntu-install.sh` disables swap, creates and mounts the encrypted directories, and installs the host services. It uses idempotent systemd mount starts and does not overwrite configured rclone or Cloudflare credentials when rerun. diff --git a/apparmor/local/fusermount3 b/apparmor/local/fusermount3 new file mode 100644 index 0000000..d9f67e5 --- /dev/null +++ b/apparmor/local/fusermount3 @@ -0,0 +1,2 @@ +mount fstype=fuse.rclone options=(rw,nosuid,nodev) -> /srv/secure/backups/, +umount /srv/secure/backups/, diff --git a/fstab-append.sh b/append.sh similarity index 74% rename from fstab-append.sh rename to append.sh index 383df6b..17bab0f 100755 --- a/fstab-append.sh +++ b/append.sh @@ -11,4 +11,10 @@ echo '/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0' >> /etc/fstab echo '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' >> /etc/fstab echo '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' >> /etc/fstab echo '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' >> /etc/fstab + +# Permit only the rclone filesystem mounted at the encrypted backup path. +grep -qF '' /etc/apparmor.d/fusermount3 +install -D -m 0644 apparmor/local/fusermount3 /etc/apparmor.d/local/fusermount3 +apparmor_parser -r /etc/apparmor.d/fusermount3 + systemctl daemon-reload diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 410f44e..11e7812 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -4,7 +4,7 @@ set -euxo pipefail cd /opt/19 # Ubuntu 26.04 host install. Create and open the LUKS volume, add its UUID to -# /etc/crypttab, and run fstab-append.sh first by following README.md. +# /etc/crypttab, and run append.sh first by following README.md. systemctl mask swap.target systemctl daemon-reload From d8d38ab46154ca6e0f97dba608f46d54f7675d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 18:09:51 +0300 Subject: [PATCH 14/23] fix apparmor mount error --- README.md | 4 +--- append.sh | 5 ----- ubuntu-install.sh | 5 +++++ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fd3a5dc..7a00e4f 100644 --- a/README.md +++ b/README.md @@ -149,9 +149,7 @@ Append the UUID entry to `/etc/crypttab`: echo "appdata UUID=$LUKS_UUID none luks,noauto" >> /etc/crypttab ``` -Append the fixed `/etc/fstab` entries and install the local AppArmor rule once, -then run the host installer. The host installer can be rerun after a partial -failure; do not rerun `append.sh`. +The host installer can be rerun after a partial failure; do not rerun `append.sh`. ```bash bash ./append.sh diff --git a/append.sh b/append.sh index 17bab0f..8a48b4a 100755 --- a/append.sh +++ b/append.sh @@ -12,9 +12,4 @@ echo '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' >> /etc/fstab echo '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' >> /etc/fstab echo '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' >> /etc/fstab -# Permit only the rclone filesystem mounted at the encrypted backup path. -grep -qF '' /etc/apparmor.d/fusermount3 -install -D -m 0644 apparmor/local/fusermount3 /etc/apparmor.d/local/fusermount3 -apparmor_parser -r /etc/apparmor.d/fusermount3 - systemctl daemon-reload diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 11e7812..1055860 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -74,6 +74,11 @@ install -d /etc/nginx/sites-available install -m 0644 nginx/sites-enabled/00_agent19.conf /etc/nginx/sites-available/00_agent19.conf install -m 0644 nginx/sites-enabled/odoo.conf /etc/nginx/sites-available/odoo.conf +# Permit only the rclone filesystem mounted at the encrypted backup path. +grep -qF '' /etc/apparmor.d/fusermount3 +install -D -m 0644 apparmor/local/fusermount3 /etc/apparmor.d/local/fusermount3 +apparmor_parser -r /etc/apparmor.d/fusermount3 + install -d -m 0700 /srv/secure/backups cp --update=none rclone.conf /srv/secure/rclone-config/rclone.conf chmod 0600 /srv/secure/rclone-config/rclone.conf From c432cf70307ba93e484f3f6d2c7f62342f02b80c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 18:21:53 +0300 Subject: [PATCH 15/23] unify mount commands --- LUKS.md | 52 ++++++++++++++++++++++++++++++++++------------------ README.md | 29 ++++++++++++----------------- 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/LUKS.md b/LUKS.md index 65e8405..957294d 100644 --- a/LUKS.md +++ b/LUKS.md @@ -57,21 +57,21 @@ installation. Use it when opening the volume: ```bash cryptsetup open "$HETZNER_VOL" appdata -mount /srv/secure -mount /var/lib/postgresql -mount /var/lib/docker -mount /var/lib/containerd -mount /etc/odoo -mount /var/log/nginx -mount /var/log/postgresql -mount /var/lib/nginx +systemctl start srv-secure.mount +systemctl start var-lib-postgresql.mount +systemctl start var-lib-docker.mount +systemctl start var-lib-containerd.mount +systemctl start etc-odoo.mount +systemctl start var-log-nginx.mount +systemctl start var-log-postgresql.mount +systemctl start var-lib-nginx.mount nginx -t systemctl start odoo-app.target ``` Before unlocking, SSH should work while the mapper, mounts, and application services remain inactive. A wrong passphrase leaves the mapper closed. If a -mount command fails, stop there and inspect the device and `/etc/fstab` rather +mount unit fails, stop there and inspect the device and `/etc/fstab` rather than starting services. Useful inspection commands: @@ -87,7 +87,7 @@ systemctl status odoo-app.target ``` To mount the data for maintenance without starting services, stop after the -last `mount` command. To stop the application and lock the volume: +last `.mount` unit. To stop the application and lock the volume: ```bash systemctl stop odoo-app.target @@ -135,14 +135,30 @@ header; during recovery, preserve the damaged current header before attempting a restore. A header backup also does not replace ordinary application-data backups. The first-install procedure creates -`/root/appdata-luks-header-.img`. Copy it to offline storage, verify the -copy, and delete the server copy. Keep the header backup and passphrase -separately. The header backup alone cannot decrypt the volume, but the backup -together with a passphrase valid when it was created can—even if that passphrase -was later changed or removed from the live header. Create a new header backup -after intentional keyslot changes, and securely destroy old copies if a removed -passphrase must no longer work. Losing both the usable live header and every -header backup makes the data unrecoverable. +`/root/appdata-luks-header-.img`. The header backup alone cannot decrypt +the volume, but the backup together with a passphrase valid when it was created +can—even if that passphrase was later changed or removed from the live header. +Create a new header backup after intentional keyslot changes, and securely +destroy old copies if a removed passphrase must no longer work. Losing both the +usable live header and every header backup makes the data unrecoverable. + +### Encrypting the header backup + +GPG prompts for the password. Encrypt the header before transferring it: + +```bash +HEADER_NAME="appdata-luks-header-$LUKS_UUID.img" +gpg --symmetric "/root/$HEADER_NAME" +``` + +Decrypt it when needed: + +```bash +gpg --output "/root/$HEADER_NAME" --decrypt "/offline/path/$HEADER_NAME.gpg" +``` + +Transfer only the `.gpg` file and remove the plaintext server copy after the +backup is stored safely. Keep the GPG password separately. See the official [`luksHeaderBackup` manual](https://man7.org/linux/man-pages/man8/cryptsetup-luksheaderbackup.8.html) for the recovery and security warnings. diff --git a/README.md b/README.md index 7a00e4f..e95d6e6 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ asks for the passphrase interactively and does not store it on the server. git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/19 cd /opt/19 apt update -apt install -y cryptsetup +apt install -y cryptsetup gnupg lsblk -So NAME,MODEL,SERIAL,SIZE,TYPE cryptsetup luksFormat --type luks2 /dev/sdX LUKS_UUID="$(cryptsetup luksUUID /dev/sdX)" && echo "$LUKS_UUID" @@ -156,12 +156,7 @@ bash ./append.sh bash ./ubuntu-install.sh ``` -Both files are linear command lists without loops or conditional branches. -`append.sh` appends the fixed mount configuration and permits the rclone FUSE -mount at `/srv/secure/backups` in the local `fusermount3` AppArmor policy. -`ubuntu-install.sh` disables swap, creates and mounts the encrypted directories, -and installs the host services. It uses idempotent systemd mount starts and does -not overwrite configured rclone or Cloudflare credentials when rerun. +Both files are simple linear command lists without loops or conditional branches that are easy to review. Finish the configuration and start the application services: @@ -184,8 +179,8 @@ systemctl start odoo-app.target su - postgres -c "createuser -s root" ``` -Copy `/root/appdata-luks-header-.img` to offline storage, verify the copy, -then delete the server copy. Keep the passphrase separately. +Encrypt `/root/appdata-luks-header-.img` before transferring it, then +remove the plaintext copy. See [LUKS.md](LUKS.md#encrypting-the-header-backup). The bootstrap installs Docker's loopback API override automatically. Configure the Loki Docker logging plugin below before creating Odoo containers. @@ -195,14 +190,14 @@ stopped. Unlock, mount, and start them with the same ordinary commands: ```bash cryptsetup open "$HETZNER_VOL" appdata -mount /srv/secure -mount /var/lib/postgresql -mount /var/lib/docker -mount /var/lib/containerd -mount /etc/odoo -mount /var/log/nginx -mount /var/log/postgresql -mount /var/lib/nginx +systemctl start srv-secure.mount +systemctl start var-lib-postgresql.mount +systemctl start var-lib-docker.mount +systemctl start var-lib-containerd.mount +systemctl start etc-odoo.mount +systemctl start var-log-nginx.mount +systemctl start var-log-postgresql.mount +systemctl start var-lib-nginx.mount nginx -t systemctl start odoo-app.target ``` From 1bf18e9d1640e1f649a9ae2cf736416bc36d7af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 18:25:27 +0300 Subject: [PATCH 16/23] add unlock and start --- LUKS.md | 20 ++++++-------------- README.md | 15 +++------------ unlock-and-start.sh | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 26 deletions(-) create mode 100755 unlock-and-start.sh diff --git a/LUKS.md b/LUKS.md index 957294d..48af976 100644 --- a/LUKS.md +++ b/LUKS.md @@ -53,20 +53,11 @@ application memory is not written to unencrypted disk. ## Normal boot procedure Root's `.bashrc` defines `HETZNER_VOL` as the stable device path recorded during -installation. Use it when opening the volume: +installation. From a root login shell, run: ```bash -cryptsetup open "$HETZNER_VOL" appdata -systemctl start srv-secure.mount -systemctl start var-lib-postgresql.mount -systemctl start var-lib-docker.mount -systemctl start var-lib-containerd.mount -systemctl start etc-odoo.mount -systemctl start var-log-nginx.mount -systemctl start var-log-postgresql.mount -systemctl start var-lib-nginx.mount -nginx -t -systemctl start odoo-app.target +cd /opt/19 +./unlock-and-start.sh ``` Before unlocking, SSH should work while the mapper, mounts, and application @@ -86,8 +77,9 @@ swapon --show systemctl status odoo-app.target ``` -To mount the data for maintenance without starting services, stop after the -last `.mount` unit. To stop the application and lock the volume: +To mount the data for maintenance without starting services, run the commands in +`unlock-and-start.sh` only through the last `.mount` unit. To stop the +application and lock the volume: ```bash systemctl stop odoo-app.target diff --git a/README.md b/README.md index e95d6e6..da90e0a 100644 --- a/README.md +++ b/README.md @@ -186,20 +186,11 @@ The bootstrap installs Docker's loopback API override automatically. Configure the Loki Docker logging plugin below before creating Odoo containers. After every reboot, Ubuntu and SSH are available but application services stay -stopped. Unlock, mount, and start them with the same ordinary commands: +stopped. From a root login shell, unlock, mount, and start them with: ```bash -cryptsetup open "$HETZNER_VOL" appdata -systemctl start srv-secure.mount -systemctl start var-lib-postgresql.mount -systemctl start var-lib-docker.mount -systemctl start var-lib-containerd.mount -systemctl start etc-odoo.mount -systemctl start var-log-nginx.mount -systemctl start var-log-postgresql.mount -systemctl start var-lib-nginx.mount -nginx -t -systemctl start odoo-app.target +cd /opt/19 +./unlock-and-start.sh ``` The systemd drop-ins installed by `ubuntu-install.sh` prevent protected services diff --git a/unlock-and-start.sh b/unlock-and-start.sh new file mode 100755 index 0000000..88c56ca --- /dev/null +++ b/unlock-and-start.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euxo pipefail + +: "${HETZNER_VOL:?HETZNER_VOL is not set; start a root login shell first}" + +cryptsetup open "$HETZNER_VOL" appdata +systemctl start srv-secure.mount +systemctl start var-lib-postgresql.mount +systemctl start var-lib-docker.mount +systemctl start var-lib-containerd.mount +systemctl start etc-odoo.mount +systemctl start var-log-nginx.mount +systemctl start var-log-postgresql.mount +systemctl start var-lib-nginx.mount +nginx -t +systemctl start odoo-app.target From 6a8cdc452524f382ab8ccacb5ccca5b52b6406e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 19:40:12 +0300 Subject: [PATCH 17/23] cleanup --- README.md | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index da90e0a..1d7b08c 100644 --- a/README.md +++ b/README.md @@ -206,22 +206,6 @@ agent temporary files and rotating agent logs. Do not use unencrypted `/tmp` or Do not rerun the one-time installer as an update mechanism. Use `./update.sh` for this release and make later host configuration changes as explicit commands. -#### Adding Odoo 20 later - -Create a 20.0 branch and replace release-specific 19 values with 20, including the -service/logrotate filenames. Clone that branch to `/opt/20`, create its Python -virtualenv, install its systemd unit, appdata drop-in, logrotate file, and nginx -site with the corresponding individual commands from `ubuntu-install.sh`. Configure -DNS and TCP 9020 for `https://20.eniemela.fi:9020`; the backend uses -`127.0.0.1:8020`. Validate nginx, enable the new agent, and start it through -`odoo-app.target`. - -Each agent discovers and backs up only containers with its release's -`odoo.version` label. UIDs and HTTP/gevent ports must be unique across the entire -server, including stopped containers. Both releases share the PostgreSQL cluster, -backup remote and nginx routing maps. Static files are served by the selected -Odoo container. Updating one clone with `./update.sh` restarts only its agent. - #### Promtail setup (TODO: deprecated, migrate to Alloy) * Promtail is an agent which ships the contents of local logs to a private Grafana Loki instance: https://grafana.com/docs/loki/latest/send-data/promtail/ * Attach new server to the same private network as "monitoring" in hetzner cloud. @@ -279,23 +263,15 @@ Odoo container. Updating one clone with `./update.sh` restarts only its agent. * `crontab -e` * `30 00 * * * mountpoint -q /srv/secure && cd /opt/19 && TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1` -##### Copying existing unencrypted backups to the new encrypted bucket: -If you have existing plaintext backups in `odoobackup1` and wish to copy them into the new encrypted bucket: -1. Copy into the encrypted remote (reads unencrypted files, encrypts locally, writes to `odoo-backups-crypt`): - * `rclone copy awsbucket:odoobackup1 backup-crypt: --config /srv/secure/rclone-config/rclone.conf --progress --transfers=16` -2. Verify the files through `/srv/secure/backups` or `rclone --config /srv/secure/rclone-config/rclone.conf ls backup-crypt:` -3. Once verified, the old unencrypted bucket `odoobackup1` can be kept as a fallback or purged: - * `rclone purge awsbucket:odoobackup1 --config /srv/secure/rclone-config/rclone.conf` - ##### Decrypting a single file without rclone: To manually decrypt a downloaded file without rclone (using only Python and `pip install pynacl`): * `python3 docs/decrypt.py ` #### Clone modules * `cd /opt/19/src` -* `git clone -b 19.0 git@github.com:elmeriniemela/tabularium.git` -* `git clone -b 19.0 --depth=1 --single-branch git@github.com:odoo/odoo.git` -* `git clone -b 19.0 --depth=1 --single-branch git@github.com:OCA/OpenUpgrade.git` +* `git clone -b 19.0 https://github.com/elmeriniemela/tabularium.git` +* `git clone -b 19.0 --depth=1 --single-branch https://github.com/odoo/odoo.git` +* `git clone -b 19.0 --depth=1 --single-branch https://github.com/OCA/OpenUpgrade.git` ## Other notes From ea04ab0e849aec1d6a4034bdb4269e51b8a94f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 21:13:17 +0300 Subject: [PATCH 18/23] refactor / cleanup after one succesful deployment --- .github/workflows/unit-tests.yml | 15 +- AGENTS.md | 19 +- LUKS.md | 31 ++-- README.md | 171 ++++++------------ agentd/agentlib.py | 34 ++-- agentd/tests/test_agentlib.py | 45 +++-- append.sh | 15 -- apt/apt.conf.d/50unattended-upgrades | 23 --- apt/apt.conf.d/60deploy-manager | 7 + cron/deploy-manager19 | 4 + requirements.txt | 5 - systemd/system/appdata-mounts.conf | 3 +- systemd/system/deploy-manager19.service | 4 +- systemd/system/docker.service.d/override.conf | 4 - systemd/system/odoo-app.target | 4 +- ubuntu-install.sh | 104 +++++++++-- unlock-and-start.sh | 17 -- update.sh | 10 - 18 files changed, 240 insertions(+), 275 deletions(-) delete mode 100755 append.sh delete mode 100644 apt/apt.conf.d/50unattended-upgrades create mode 100644 apt/apt.conf.d/60deploy-manager create mode 100644 cron/deploy-manager19 delete mode 100644 requirements.txt delete mode 100644 systemd/system/docker.service.d/override.conf delete mode 100755 unlock-and-start.sh delete mode 100755 update.sh diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index defbdcb..22129de 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -18,21 +18,14 @@ jobs: with: submodules: true - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: pip - cache-dependency-path: requirements.txt - - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + sudo apt-get update + sudo apt-get install -y python3-jinja2 python3-psycopg2 python3-requests - name: Run agentd unit tests - run: python -m unittest discover -s agentd/tests -t . + run: python3 -m unittest discover -s agentd/tests -t . - name: Run cronsyl unit tests working-directory: agentd/cronsyl - run: python -m unittest discover + run: python3 -m unittest discover diff --git a/AGENTS.md b/AGENTS.md index 20bdc72..9e18f46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,17 @@ +# Repository guidelines -## Validate README.md diagram syntax: +- Minimize maintained code and configuration; remove redundancy before adding tooling. +- Keep `ubuntu-install.sh` rerunnable, simple and easy to review. +- Setup scripts must not clean up or migrate files from older repository versions. +- Keep the custom Odoo image and `rclone-mount.service` backup design. +- Monitoring is mandatory: keep Loki, Promtail, and node exporter. Alloy migration is future work. +- Keep destructive LUKS preparation manual; automate safe setup steps. +- Never overwrite existing secrets under `/srv/secure`. -* `mkdir build` -* `awk '/```mermaid/{flag=1;next}/```/{flag=0}flag' README.md > build/diagram.mmd` -* `mmdc -i build/diagram.mmd -o build/diagram.png` +## Validation + +```bash +mkdir -p build +awk '/```mermaid/{flag=1;next}/```/{flag=0}flag' README.md > build/diagram.mmd +mmdc -i build/diagram.mmd -o build/diagram.png +``` diff --git a/LUKS.md b/LUKS.md index 48af976..0af6c54 100644 --- a/LUKS.md +++ b/LUKS.md @@ -11,8 +11,7 @@ deliberate operator action, not hidden in a Bash script. The initial format may use a verified kernel device name such as `/dev/sdb`. Identify it by matching the `MODEL`, `SERIAL`, and `SIZE` from `lsblk` with the -Hetzner Console. Once LUKS exists, store its stable -`/dev/disk/by-uuid/` path as `HETZNER_VOL` in root's `.bashrc`; do not +Hetzner Console. Once LUKS exists, `/etc/crypttab` stores its stable UUID; do not persist `/dev/sdb`, because kernel device names can change. ## Storage layout @@ -38,8 +37,8 @@ Hetzner Volume The mapper and every bind mount use `noauto` in `/etc/fstab`. PostgreSQL, Docker, containerd, nginx, rclone, and the deployment agent are disabled at -boot. Their systemd drop-ins declare every mount unit as a `Requisite`, so a -direct service start fails while the encrypted storage is not mounted. +boot. Their systemd drop-ins use `RequiresMountsFor` so systemd starts the bind +mounts on demand and fails the service if a required mount cannot be started. Rclone reads its configuration from `/srv/secure/rclone-config/rclone.conf`, uses `/srv/secure/rclone-cache`, and mounts the decrypted backup view at @@ -52,12 +51,12 @@ application memory is not written to unencrypted disk. ## Normal boot procedure -Root's `.bashrc` defines `HETZNER_VOL` as the stable device path recorded during -installation. From a root login shell, run: +The generated `systemd-cryptsetup@appdata.service` reads the stable device UUID +from `/etc/crypttab`. From a root login shell, run: ```bash -cd /opt/19 -./unlock-and-start.sh +systemctl start systemd-cryptsetup@appdata.service +systemctl start odoo-app.target ``` Before unlocking, SSH should work while the mapper, mounts, and application @@ -77,9 +76,15 @@ swapon --show systemctl status odoo-app.target ``` -To mount the data for maintenance without starting services, run the commands in -`unlock-and-start.sh` only through the last `.mount` unit. To stop the -application and lock the volume: +To mount the data for maintenance without starting services, start the encrypted +volume and mount units directly: + +```bash +systemctl start systemd-cryptsetup@appdata.service +systemctl start srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount +``` + +To stop the application and lock the volume: ```bash systemctl stop odoo-app.target @@ -91,7 +96,7 @@ umount /var/lib/containerd umount /var/lib/docker umount /var/lib/postgresql umount /srv/secure -cryptsetup close appdata +systemctl stop systemd-cryptsetup@appdata.service ``` ## Recovery material @@ -119,7 +124,7 @@ physically present but impossible to decrypt. A header backup can restore the information needed to unlock it: ```bash -cryptsetup luksHeaderRestore "$HETZNER_VOL" --header-backup-file "/offline/path/appdata-luks-header-.img" +cryptsetup luksHeaderRestore "/dev/disk/by-uuid/" --header-backup-file "/offline/path/appdata-luks-header-.img" ``` Do not run `luksHeaderRestore` during normal operation. It overwrites the current diff --git a/README.md b/README.md index 1d7b08c..4182f8e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Deployment Manager for Odoo images ## Abstract -This project supports self-hosted Odoo deployments on a Linux host. It provides a custom Odoo Docker image, a short one-time Ubuntu setup command list, and an XML-RPC deployment manager for instance lifecycle tasks such as create/reset/restart/upgrade, hostname-to-port routing updates, and SSL certificate management. It also handles client-side encrypted database and filestore backup/restore workflows using `pg_dump`/`pg_restore` and `rclone` (with zero-knowledge `crypt` overlay), with scheduled retention cleanup. +This project supports self-hosted Odoo deployments on a Linux host. It provides a custom Odoo Docker image, a rerunnable Ubuntu host configuration script, and an XML-RPC deployment manager for instance lifecycle tasks such as create/reset/restart/upgrade, hostname-to-port routing updates, and SSL certificate management. It also handles client-side encrypted database and filestore backup/restore workflows using `pg_dump`/`pg_restore` and `rclone` (with zero-knowledge `crypt` overlay), with scheduled retention cleanup. The XML-RPC API can be used from an Odoo instance to do self upgrades of Odoo source code. @@ -109,38 +109,29 @@ Point `19.eniemela.fi` to this server and allow TCP 9019 in the host and Hetzner firewalls. The agent is available at `https://19.eniemela.fi:9019` through nginx basic authentication; its backend is loopback-only on port 8019. -#### SSH key setup (optional, repos are public) -* `scp .gitconfig 19.eniemela.fi:` -* `cd .ssh && ssh-keygen -f id_ecdsa -t ecdsa -b 521` -* `cat id_ecdsa.pub` -* go to github / settings / SSH keys / Add 'Odoo 19.0 Hetzner Server key' - * https://github.com/settings/keys - #### Installation Run the following as root. Identify the volume by matching its `MODEL`, `SERIAL`, and `SIZE` with the Hetzner Console. Use its `/dev/sdX` name only for the initial -format. Afterward, save the stable LUKS UUID path in root's `.bashrc` for later -login shells. `luksFormat` and `mkfs.ext4` destroy data on the selected device, -so confirm the `lsblk`, `findmnt`, and `wipefs` output before continuing. LUKS -asks for the passphrase interactively and does not store it on the server. +format; `/etc/crypttab` stores the stable UUID used after that. `luksFormat` and +`mkfs.ext4` destroy data on the selected device, so confirm the `lsblk` output before continuing. +LUKS asks for the passphrase interactively and does not store it on the server. ```bash +apt update +apt install -y cryptsetup git vim tmux git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/19 cd /opt/19 -apt update -apt install -y cryptsetup gnupg lsblk -So NAME,MODEL,SERIAL,SIZE,TYPE -cryptsetup luksFormat --type luks2 /dev/sdX -LUKS_UUID="$(cryptsetup luksUUID /dev/sdX)" && echo "$LUKS_UUID" -udevadm trigger --action=change --name-match=/dev/sdX +INSTALL_DEVICE=/dev/sdX +cryptsetup luksFormat --type luks2 "$INSTALL_DEVICE" +LUKS_UUID="$(cryptsetup luksUUID "$INSTALL_DEVICE")" && echo "$LUKS_UUID" +udevadm trigger --action=change --name-match="$INSTALL_DEVICE" udevadm settle -echo "export HETZNER_VOL=\"/dev/disk/by-uuid/$LUKS_UUID\"" >> /root/.bashrc -export HETZNER_VOL="/dev/disk/by-uuid/$LUKS_UUID" -readlink -e "$HETZNER_VOL" -cryptsetup open "$HETZNER_VOL" appdata +readlink -e "/dev/disk/by-uuid/$LUKS_UUID" +cryptsetup open "$INSTALL_DEVICE" appdata mkfs.ext4 /dev/mapper/appdata -cryptsetup luksHeaderBackup "$HETZNER_VOL" --header-backup-file "/root/appdata-luks-header-$LUKS_UUID.img" +cryptsetup luksHeaderBackup "$INSTALL_DEVICE" --header-backup-file "/root/appdata-luks-header-$LUKS_UUID.img" ``` Append the UUID entry to `/etc/crypttab`: @@ -149,131 +140,66 @@ Append the UUID entry to `/etc/crypttab`: echo "appdata UUID=$LUKS_UUID none luks,noauto" >> /etc/crypttab ``` -The host installer can be rerun after a partial failure; do not rerun `append.sh`. +Configure the host. This command is idempotent and can be rerun after a partial +failure or configuration change. It also installs the Loki Docker plugin, +Promtail, node exporter, the scheduled backup job, and the required module +repositories. ```bash -bash ./append.sh bash ./ubuntu-install.sh ``` -Both files are simple linear command lists without loops or conditional branches that are easy to review. - Finish the configuration and start the application services: ```bash htpasswd -B -C 12 -c /etc/nginx/.htpasswd cloud chown root:www-data /etc/nginx/.htpasswd chmod 640 /etc/nginx/.htpasswd +# Add passwrods vim /srv/secure/rclone-config/rclone.conf vim /srv/secure/secrets/cloudflare.ini export TMPDIR=/srv/secure/tmp -/root/agent-venv19/bin/python -m agentd.api ssl_wildcard +/usr/bin/python3 -m agentd.api ssl_wildcard # OR -sudo rsync -aHAX /etc/letsencrypt/ root@NEW_SERVER:/etc/letsencrypt/ +rsync -aHAX /etc/letsencrypt/ root@NEW_SERVER:/etc/letsencrypt/ -ln -s /etc/nginx/sites-available/00_agent19.conf /etc/nginx/sites-enabled/00_agent19.conf -ln -s /etc/nginx/sites-available/odoo.conf /etc/nginx/sites-enabled/odoo.conf nginx -t systemctl start odoo-app.target -su - postgres -c "createuser -s root" ``` Encrypt `/root/appdata-luks-header-.img` before transferring it, then remove the plaintext copy. See [LUKS.md](LUKS.md#encrypting-the-header-backup). -The bootstrap installs Docker's loopback API override automatically. Configure -the Loki Docker logging plugin below before creating Odoo containers. - After every reboot, Ubuntu and SSH are available but application services stay stopped. From a root login shell, unlock, mount, and start them with: ```bash -cd /opt/19 -./unlock-and-start.sh +systemctl start systemd-cryptsetup@appdata.service +systemctl start odoo-app.target ``` -The systemd drop-ins installed by `ubuntu-install.sh` prevent protected services -from starting while any encrypted mount unit is inactive. Use `findmnt /srv/secure` -and `findmnt /var/lib/docker` to inspect mounts, and `systemctl status -odoo-app.target` to inspect the services. +The systemd drop-ins installed by `ubuntu-install.sh` pull in the encrypted bind +mounts and prevent protected services from starting if a required mount fails. +Use `findmnt /srv/secure` and `findmnt /var/lib/docker` to inspect mounts, and +`systemctl status odoo-app.target` to inspect the services. The encrypted filesystem also contains nginx request-body temporary files, agent temporary files and rotating agent logs. Do not use unencrypted `/tmp` or `/var/tmp` for database dumps. Persistent swap is disabled. Confirm that `swapon --show` is empty before starting the application services. -Do not rerun the one-time installer as an update mechanism. Use `./update.sh` for -this release and make later host configuration changes as explicit commands. - -#### Promtail setup (TODO: deprecated, migrate to Alloy) -* Promtail is an agent which ships the contents of local logs to a private Grafana Loki instance: https://grafana.com/docs/loki/latest/send-data/promtail/ -* Attach new server to the same private network as "monitoring" in hetzner cloud. -* docker run \ - -v ./promtail:/etc/promtail \ - -v /var/log:/var/log \ - --restart unless-stopped \ - --name promtail -d \ - grafana/promtail:latest -config.file=/etc/promtail/config.yml - -* https://grafana.com/docs/loki/latest/send-data/docker-driver/ -* https://grafana.com/docs/loki/latest/send-data/docker-driver/configuration/ -* `docker plugin install grafana/loki-docker-driver --alias loki --grant-all-permissions` - -#### Prometheus node exporter monitoring: -* Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors: https://github.com/prometheus/node_exporter -* docker run -d \ - --net="host" \ - --pid="host" \ - -v "/:/host:ro,rslave" \ - --restart unless-stopped \ - quay.io/prometheus/node-exporter:latest \ - --path.rootfs=/host - -#### Creating a personal github access token (READ only): -* https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token -* Go to Settings / Developer / New personal access token (classic) / Add 'Odoo 19.0 Hetzner Server key' + add repo and write:packages - * https://github.com/settings/tokens/new -* `docker login ghcr.io -u elmeriniemela` - -#### Building the image -* `docker build -t ghcr.io/elmeriniemela/odoo-src:19.0 /opt/19` -* https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#building-container-images -* `docker push ghcr.io/elmeriniemela/odoo-src:19.0` - - #### Backup setup (Client-Side Encrypted S3 Backups): * All database dumps and Odoo filestores are encrypted client-side via rclone's `crypt` backend before upload to AWS S3. -* Create a dedicated S3 bucket in AWS: `odoo-backups-crypt` (e.g. in `eu-north-1`). * Generate an obscured password for rclone config: * `rclone obscure 'YourStrongSecretPassphrase' --config /srv/secure/rclone-config/rclone.conf` -* In `/srv/secure/rclone-config/rclone.conf`, add the `[backup-crypt]` section: - ```ini - [backup-crypt] - type = crypt - remote = awsbucket:odoo-backups-crypt - filename_encryption = off - directory_name_encryption = false - password = - ``` *(Important: Back up this passphrase in an offline password manager. If lost, encrypted backups cannot be recovered!)* -* Mount the encrypted remote: - * `systemctl restart rclone-mount.service` -* Scheduled cron: - * `crontab -e` - * `30 00 * * * mountpoint -q /srv/secure && cd /opt/19 && TMPDIR=/srv/secure/tmp /root/agent-venv19/bin/python -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1` +* `/etc/cron.d/deploy-manager19` runs the scheduled backup at 00:30 daily. ##### Decrypting a single file without rclone: To manually decrypt a downloaded file without rclone (using only Python and `pip install pynacl`): * `python3 docs/decrypt.py ` -#### Clone modules -* `cd /opt/19/src` -* `git clone -b 19.0 https://github.com/elmeriniemela/tabularium.git` -* `git clone -b 19.0 --depth=1 --single-branch https://github.com/odoo/odoo.git` -* `git clone -b 19.0 --depth=1 --single-branch https://github.com/OCA/OpenUpgrade.git` - - ## Other notes #### Pulling the image @@ -283,24 +209,33 @@ To manually decrypt a downloaded file without rclone (using only Python and `pip * https://wiki.postgresql.org/wiki/Shared_Database_Hosting * https://wiki.postgresql.org/images/d/d1/Managing_rights_in_postgresql.pdf -### Local setup -* sudo docker run \ - -v /home/elmeri/Odoo/src/16:/mnt:ro \ - -v /var/run/postgresql:/var/run/postgresql \ - -v /home/elmeri/Odoo/src/16/own-docker.conf:/etc/odoo/odoo.conf:ro \ - -v /home/elmeri/.local/share/Odoo:/var/lib/odoo \ - -p 127.0.0.1:8016:8069 \ - -p 127.0.0.1:9016:8072 \ - --name eniemela_16 -ti ghcr.io/elmeriniemela/odoo-src:19.0 -* sudo docker restart eniemela_16 && sudo docker attach eniemela_16 -* sudo docker exec -it -u root eniemela_16 bash -* sudo docker restart eniemela_16 && sudo docker exec -it -u root eniemela_16 odoo -u investment_portfolio --http-port=9999 --stop-after-init && sudo docker restart eniemela_16 && sudo docker attach eniemela_16 + +#### Creating a personal github access token (write packages only): +* https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token +* Go to Settings / Developer / New personal access token (classic) / Add 'Odoo 19.0 Hetzner Server key' + add repo and write:packages + * https://github.com/settings/tokens/new +* `docker login ghcr.io -u elmeriniemela` + +### Image development and publishing + +Production hosts pull the public image without GitHub credentials. Building and +publishing require a developer workstation authenticated to GHCR with package +write permission. + +* `docker build -t ghcr.io/elmeriniemela/odoo-src:19.0 .` +* `docker push ghcr.io/elmeriniemela/odoo-src:19.0` +* https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry + +#### SSH key setup (optional, repos are public) +* `scp .gitconfig 19.eniemela.fi:` +* `cd .ssh && ssh-keygen -f id_ecdsa -t ecdsa -b 521` +* `cat id_ecdsa.pub` +* go to github / settings / SSH keys / Add 'Odoo 19.0 Hetzner Server key' + * https://github.com/settings/keys ### Local mermaid-cli installation for AGENTS.md verification of the diagram: -* `sudo pacman -S nodejs npm` -* `sudo npm install -g @mermaid-js/mermaid-cli` -* `npx puppeteer browsers install chrome-headless-shell@131.0.6778.204`. NOTE: mmdc pins to a specific version, adapt if needed. -* See AGENTS.md for usage. +* Install `sudo pacman -S mermaid-cli` for your development platform. +* See AGENTS.md for the rerunnable validation commands. ### Tests * `python3 -m unittest discover -s agentd/tests -t .` @@ -310,6 +245,4 @@ To manually decrypt a downloaded file without rclone (using only Python and `pip * Docker logs * Docker volumes: `ls /var/lib/docker/volumes` * Delete everything: `docker system prune -a --volumes` -* Remote access: https://docs.docker.com/config/daemon/remote-access/ -* Docker API: https://docs.docker.com/engine/api/latest/ * Login as root: `docker exec -it -u root bash` diff --git a/agentd/agentlib.py b/agentd/agentlib.py index 1b4b946..74456c7 100644 --- a/agentd/agentlib.py +++ b/agentd/agentlib.py @@ -3,7 +3,7 @@ import socket import psycopg2 import re -import requests +import json from jinja2 import Template import os import glob @@ -198,26 +198,34 @@ def list_backups(uid): def list_instances(): instances = [] - docker_ps_a = requests.get( - url='http://127.0.0.1:2375/containers/json', - params={'all': True}, - ).json() - - for container in docker_ps_a: - if container.get('Labels', {}).get('odoo.version') != '19.0': + container_ids = execute([ + 'docker', 'container', 'ls', '--all', '--quiet', '--no-trunc', + '--filter', 'label=odoo.version=19.0', + ]).stdout.split() + if not container_ids: + return instances + + inspected = json.loads(execute([ + 'docker', 'container', 'inspect', *container_ids, + ]).stdout) + for details in inspected: + labels = details.get('Config', {}).get('Labels') or {} + if labels.get('odoo.version') != '19.0': continue - uid = container['Names'][0].lstrip('/') + uid = details.get('Name', '').lstrip('/') try: validate(uid=uid) except ValueError: continue - cid = container['Id'] - container['inspect'] = requests.get(url=f'http://127.0.0.1:2375/containers/{cid}/json').json() - instances.append({ 'uid': uid, - 'docker': container, + 'docker': { + 'Names': [details['Name']], + 'Id': details['Id'], + 'Labels': labels, + 'inspect': details, + }, 'backups': list_backups(uid), }) return instances diff --git a/agentd/tests/test_agentlib.py b/agentd/tests/test_agentlib.py index ce33d4d..aea2cc4 100644 --- a/agentd/tests/test_agentlib.py +++ b/agentd/tests/test_agentlib.py @@ -1,4 +1,5 @@ import datetime +import json import subprocess import unittest import tempfile @@ -139,29 +140,32 @@ def test_list_backups_extracts_metadata_from_dump_paths(self): class InventoryTests(unittest.TestCase): def test_list_instances_skips_invalid_container_names_and_adds_inspect_data(self): - containers = [ - {"Names": ["/1a2b"], "Id": "container-1", "Labels": {"odoo.version": "19.0"}}, - {"Names": ["/not-hex"], "Id": "container-2", "Labels": {"odoo.version": "19.0"}}, - {"Names": ["/2b3c"], "Id": "container-3", "Labels": {"odoo.version": "20.0"}}, - {"Names": ["/3c4d"], "Id": "container-4"}, + inspected = [ + {"Name": "/1a2b", "Id": "container-1", "Config": {"Labels": {"odoo.version": "19.0"}}, "State": {"Running": True}}, + {"Name": "/not-hex", "Id": "container-2", "Config": {"Labels": {"odoo.version": "19.0"}}}, + {"Name": "/2b3c", "Id": "container-3", "Config": {"Labels": {"odoo.version": "20.0"}}}, + {"Name": "/3c4d", "Id": "container-4", "Config": {}}, ] - responses = [ - SimpleNamespace(json=lambda: containers), - SimpleNamespace(json=lambda: {"State": {"Running": True}}), + commands = [ + SimpleNamespace(stdout="container-1\ncontainer-2\ncontainer-3\ncontainer-4\n"), + SimpleNamespace(stdout=json.dumps(inspected)), ] - with patch("agentd.agentlib.requests.get", side_effect=responses) as get: + with patch("agentd.agentlib.execute", side_effect=commands) as execute: with patch("agentd.agentlib.list_backups", return_value=[{"fname": "dump.pgc"}]): instances = agentlib.list_instances() self.assertEqual( - get.call_args_list, + execute.call_args_list, [ - call( - url="http://127.0.0.1:2375/containers/json", - params={"all": True}, - ), - call(url="http://127.0.0.1:2375/containers/container-1/json"), + call([ + "docker", "container", "ls", "--all", "--quiet", "--no-trunc", + "--filter", "label=odoo.version=19.0", + ]), + call([ + "docker", "container", "inspect", "container-1", "container-2", + "container-3", "container-4", + ]), ], ) self.assertEqual( @@ -173,13 +177,22 @@ def test_list_instances_skips_invalid_container_names_and_adds_inspect_data(self "Names": ["/1a2b"], "Id": "container-1", "Labels": {"odoo.version": "19.0"}, - "inspect": {"State": {"Running": True}}, + "inspect": inspected[0], }, "backups": [{"fname": "dump.pgc"}], } ], ) + def test_list_instances_avoids_inspect_when_no_containers_exist(self): + with patch( + "agentd.agentlib.execute", + return_value=SimpleNamespace(stdout=""), + ) as execute: + self.assertEqual(agentlib.list_instances(), []) + + execute.assert_called_once() + def test_list_postgres_converts_cursor_rows_to_dicts(self): cursor = MagicMock() descriptions = [ diff --git a/append.sh b/append.sh deleted file mode 100755 index 8a48b4a..0000000 --- a/append.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -# Run once on a fresh host. Re-running this file duplicates /etc/fstab entries. -echo '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' >> /etc/fstab -echo '/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/docker /var/lib/docker none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' >> /etc/fstab -echo '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' >> /etc/fstab - -systemctl daemon-reload diff --git a/apt/apt.conf.d/50unattended-upgrades b/apt/apt.conf.d/50unattended-upgrades deleted file mode 100644 index 8dd0204..0000000 --- a/apt/apt.conf.d/50unattended-upgrades +++ /dev/null @@ -1,23 +0,0 @@ -// Unattended-Upgrade configuration for automated security patches. - -Unattended-Upgrade::Allowed-Origins { - "${distro_id}:${distro_codename}-security"; - "${distro_id}ESMApps:${distro_codename}-apps-security"; - "${distro_id}ESM:${distro_codename}-infra-security"; -}; - -// Python, Docker or PostgreSQL updates that should not be automatically updated can be blacklisted here -Unattended-Upgrade::Package-Blacklist { -}; - -Unattended-Upgrade::AutoFixInterruptedDpkg "true"; -Unattended-Upgrade::MinimalSteps "true"; -Unattended-Upgrade::InstallOnShutdown "false"; - -// Clean up unused kernels and dependencies automatically -Unattended-Upgrade::Remove-Unused-Kernel-Packages "true"; -Unattended-Upgrade::Remove-New-Unused-Dependencies "true"; -Unattended-Upgrade::Remove-Unused-Dependencies "true"; - -// Do not reboot automatically (preserves LUKS mount and prevents offline downtime) -Unattended-Upgrade::Automatic-Reboot "false"; diff --git a/apt/apt.conf.d/60deploy-manager b/apt/apt.conf.d/60deploy-manager new file mode 100644 index 0000000..17e4066 --- /dev/null +++ b/apt/apt.conf.d/60deploy-manager @@ -0,0 +1,7 @@ +// Local policy layered after Ubuntu's package-owned defaults. +Unattended-Upgrade::Remove-Unused-Kernel-Packages "true"; +Unattended-Upgrade::Remove-New-Unused-Dependencies "true"; +Unattended-Upgrade::Remove-Unused-Dependencies "true"; + +// Preserve the manually unlocked application volume across maintenance. +Unattended-Upgrade::Automatic-Reboot "false"; diff --git a/cron/deploy-manager19 b/cron/deploy-manager19 new file mode 100644 index 0000000..e2a43d0 --- /dev/null +++ b/cron/deploy-manager19 @@ -0,0 +1,4 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +30 0 * * * root mountpoint -q /srv/secure && cd /opt/19 && TMPDIR=/srv/secure/tmp /usr/bin/python3 -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b2be0b1..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -psycopg2-binary -certbot -certbot-nginx -certbot-dns-cloudflare -jinja2 diff --git a/systemd/system/appdata-mounts.conf b/systemd/system/appdata-mounts.conf index 665fa29..f266610 100644 --- a/systemd/system/appdata-mounts.conf +++ b/systemd/system/appdata-mounts.conf @@ -1,5 +1,4 @@ [Unit] -Requisite=srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount -After=srv-secure.mount var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount etc-odoo.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount +RequiresMountsFor=/srv/secure /var/lib/postgresql /var/lib/docker /var/lib/containerd /etc/odoo /var/log/nginx /var/log/postgresql /var/lib/nginx PartOf=odoo-app.target ConditionPathIsMountPoint=/srv/secure diff --git a/systemd/system/deploy-manager19.service b/systemd/system/deploy-manager19.service index 75dae79..96928c7 100644 --- a/systemd/system/deploy-manager19.service +++ b/systemd/system/deploy-manager19.service @@ -9,10 +9,10 @@ StartLimitBurst=30 [Service] User=root WorkingDirectory=/opt/19 -ExecStart=/root/agent-venv19/bin/python -m agentd --interface 127.0.0.1 --port 8019 +ExecStart=/usr/bin/python3 -m agentd --interface 127.0.0.1 --port 8019 Restart=always RestartSec=1 -Environment=PATH=/root/agent-venv19/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin Environment=TMPDIR=/srv/secure/tmp StandardOutput=append:/srv/secure/logs/deploy-manager19.log StandardError=inherit diff --git a/systemd/system/docker.service.d/override.conf b/systemd/system/docker.service.d/override.conf deleted file mode 100644 index 700bd22..0000000 --- a/systemd/system/docker.service.d/override.conf +++ /dev/null @@ -1,4 +0,0 @@ -[Service] -ExecStart= -ExecStart=/usr/bin/dockerd -H fd:// -H tcp://127.0.0.1:2375 --containerd=/run/containerd/containerd.sock - diff --git a/systemd/system/odoo-app.target b/systemd/system/odoo-app.target index 5807a94..1f0880a 100644 --- a/systemd/system/odoo-app.target +++ b/systemd/system/odoo-app.target @@ -1,4 +1,4 @@ [Unit] Description=Odoo application services -Requires=postgresql@18-main.service containerd.service docker.service nginx.service rclone-mount.service -After=postgresql@18-main.service containerd.service docker.service nginx.service rclone-mount.service +Requires=postgresql@18-main.service docker.service nginx.service rclone-mount.service +After=postgresql@18-main.service docker.service nginx.service rclone-mount.service diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 1055860..2d0abfe 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -3,12 +3,25 @@ set -euxo pipefail cd /opt/19 -# Ubuntu 26.04 host install. Create and open the LUKS volume, add its UUID to -# /etc/crypttab, and run append.sh first by following README.md. +# Ubuntu 26.04 host configuration. Create and open the LUKS volume and add its +# UUID to /etc/crypttab first by following README.md. This script is rerunnable. +ensure_line() { + grep -qxF "$1" "$2" || printf '%s\n' "$1" >> "$2" +} + systemctl mask swap.target -systemctl daemon-reload + +ensure_line '/dev/mapper/appdata /srv/secure ext4 noauto 0 2' /etc/fstab +ensure_line '/srv/secure/postgresql /var/lib/postgresql none noauto,bind 0 0' /etc/fstab +ensure_line '/srv/secure/docker /var/lib/docker none noauto,bind 0 0' /etc/fstab +ensure_line '/srv/secure/containerd /var/lib/containerd none noauto,bind 0 0' /etc/fstab +ensure_line '/srv/secure/odoo-config /etc/odoo none noauto,bind 0 0' /etc/fstab +ensure_line '/srv/secure/logs/nginx /var/log/nginx none noauto,bind 0 0' /etc/fstab +ensure_line '/srv/secure/logs/postgresql /var/log/postgresql none noauto,bind 0 0' /etc/fstab +ensure_line '/srv/secure/nginx-temp /var/lib/nginx none noauto,bind 0 0' /etc/fstab install -d /srv/secure +systemctl daemon-reload systemctl start srv-secure.mount install -d /srv/secure/postgresql /srv/secure/docker /srv/secure/containerd /srv/secure/odoo-config install -d /srv/secure/logs/nginx /srv/secure/logs/postgresql /srv/secure/nginx-temp @@ -16,13 +29,8 @@ install -d -m 0700 /srv/secure/rclone-config /srv/secure/rclone-cache /srv/secur install -d -m 0711 /srv/secure/tmp install -d /var/lib/postgresql /var/lib/docker /var/lib/containerd /etc/odoo install -d /var/log/nginx /var/log/postgresql /var/lib/nginx -systemctl start var-lib-postgresql.mount -systemctl start var-lib-docker.mount -systemctl start var-lib-containerd.mount -systemctl start etc-odoo.mount -systemctl start var-log-nginx.mount -systemctl start var-log-postgresql.mount -systemctl start var-lib-nginx.mount +systemctl start var-lib-postgresql.mount var-lib-docker.mount var-lib-containerd.mount +systemctl start etc-odoo.mount var-log-nginx.mount var-log-postgresql.mount var-lib-nginx.mount systemctl mask --runtime postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service @@ -43,7 +51,8 @@ install -m 0644 systemd/system/appdata-mounts.conf /etc/systemd/system/deploy-ma systemctl daemon-reload apt update -apt install -y postgresql nginx ca-certificates curl gnupg vim tmux patchutils fuse3 python3-pip python3-venv unattended-upgrades apache2-utils rsync rclone +apt install -y postgresql nginx ca-certificates curl gnupg git patchutils fuse3 cron unattended-upgrades apache2-utils rsync rclone +apt install -y python3 python3-psycopg2 python3-requests python3-jinja2 certbot python3-certbot-dns-cloudflare # Docker: https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository install -m 0755 -d /etc/apt/keyrings @@ -51,13 +60,13 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/doc chmod a+r /etc/apt/keyrings/docker.asc install -m 0644 apt/docker.sources /etc/apt/sources.list.d/docker.sources apt update -apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +apt install -y docker-ce docker-ce-cli containerd.io install -m 0644 systemd/system/odoo-app.target systemd/system/rclone-mount.service systemd/system/deploy-manager19.service /etc/systemd/system/ -install -D -m 0644 systemd/system/docker.service.d/override.conf /etc/systemd/system/docker.service.d/override.conf install -m 0644 systemd/system/docker.service.d/postgresql.conf /etc/systemd/system/docker.service.d/postgresql.conf install -m 0644 systemd/system/postgresql@.service.d/tmp.conf /etc/systemd/system/postgresql@.service.d/tmp.conf install -D -m 0644 logrotate/deploy-manager19 /etc/logrotate.d/deploy-manager19 +install -D -m 0644 cron/deploy-manager19 /etc/cron.d/deploy-manager19 install -d -o postgres -g postgres -m 0700 /srv/secure/tmp/postgresql rsync -av postgres/ /etc/postgresql/18/main/ @@ -73,13 +82,14 @@ install -m 0644 nginx/nginx.conf /etc/nginx/nginx.conf install -d /etc/nginx/sites-available install -m 0644 nginx/sites-enabled/00_agent19.conf /etc/nginx/sites-available/00_agent19.conf install -m 0644 nginx/sites-enabled/odoo.conf /etc/nginx/sites-available/odoo.conf +ln -sfn /etc/nginx/sites-available/00_agent19.conf /etc/nginx/sites-enabled/00_agent19.conf +ln -sfn /etc/nginx/sites-available/odoo.conf /etc/nginx/sites-enabled/odoo.conf # Permit only the rclone filesystem mounted at the encrypted backup path. grep -qF '' /etc/apparmor.d/fusermount3 install -D -m 0644 apparmor/local/fusermount3 /etc/apparmor.d/local/fusermount3 apparmor_parser -r /etc/apparmor.d/fusermount3 -install -d -m 0700 /srv/secure/backups cp --update=none rclone.conf /srv/secure/rclone-config/rclone.conf chmod 0600 /srv/secure/rclone-config/rclone.conf cp --update=none cloudflare.ini /srv/secure/secrets/cloudflare.ini @@ -88,13 +98,69 @@ install -m 0644 sshd/harden.conf /etc/ssh/sshd_config.d/harden.conf systemctl reload ssh install -d /etc/apt/apt.conf.d -install -m 0644 apt/apt.conf.d/20auto-upgrades apt/apt.conf.d/50unattended-upgrades /etc/apt/apt.conf.d/ +install -m 0644 apt/apt.conf.d/20auto-upgrades apt/apt.conf.d/60deploy-manager /etc/apt/apt.conf.d/ -python3 -m venv /root/agent-venv19 -/root/agent-venv19/bin/python -m pip install -r requirements.txt +if [ ! -d src/tabularium/.git ]; then + git clone -b 19.0 https://github.com/elmeriniemela/tabularium.git src/tabularium +fi +if [ ! -d src/odoo/.git ]; then + git clone -b 19.0 --depth=1 --single-branch https://github.com/odoo/odoo.git src/odoo +fi +if [ ! -d src/OpenUpgrade/.git ]; then + git clone -b 19.0 --depth=1 --single-branch https://github.com/OCA/OpenUpgrade.git src/OpenUpgrade +fi systemctl unmask --runtime postgresql.service postgresql@.service docker.service docker.socket containerd.service nginx.service rclone-mount.service -systemctl disable postgresql.service postgresql@18-main.service docker.service docker.socket containerd.service nginx.service rclone-mount.service deploy-manager19.service -systemctl enable deploy-manager19.service systemctl daemon-reload +/usr/bin/python3 -c 'import jinja2, psycopg2, requests' +systemctl disable postgresql.service postgresql@18-main.service docker.service docker.socket containerd.service nginx.service rclone-mount.service +systemctl enable deploy-manager19.service + +# PostgreSQL must have the peer-authenticated administrative role before the +# deployment manager can receive requests. +systemctl start postgresql@18-main.service +if ! runuser -u postgres -- psql -tAc "SELECT 1 FROM pg_roles WHERE rolname = 'root'" | grep -qx 1; then + runuser -u postgres -- createuser --superuser root +fi + +# Monitoring is part of the required host setup. Odoo containers use the Loki +# log driver, while Promtail ships host logs and node-exporter exposes metrics. +systemctl start docker.service +if ! docker plugin inspect loki >/dev/null 2>&1; then + docker plugin install grafana/loki-docker-driver --alias loki --grant-all-permissions +fi +if [ "$(docker plugin inspect --format '{{.Enabled}}' loki)" != true ]; then + docker plugin enable loki +fi +if ! docker container inspect promtail >/dev/null 2>&1; then + docker run -d \ + --name promtail \ + --restart unless-stopped \ + -v /opt/19/promtail:/etc/promtail:ro \ + -v /var/log:/var/log:ro \ + grafana/promtail:latest \ + -config.file=/etc/promtail/config.yml +else + docker update --restart unless-stopped promtail + docker restart promtail +fi +if ! docker container inspect node-exporter >/dev/null 2>&1; then + docker run -d \ + --name node-exporter \ + --net=host \ + --pid=host \ + --restart unless-stopped \ + -v /:/host:ro,rslave \ + quay.io/prometheus/node-exporter:latest \ + --path.rootfs=/host +else + docker update --restart unless-stopped node-exporter + if [ "$(docker container inspect --format '{{.State.Running}}' node-exporter)" != true ]; then + docker start node-exporter + fi +fi + +test "$(docker plugin inspect --format '{{.Enabled}}' loki)" = true +test "$(docker container inspect --format '{{.State.Running}}' promtail)" = true +test "$(docker container inspect --format '{{.State.Running}}' node-exporter)" = true systemctl restart unattended-upgrades diff --git a/unlock-and-start.sh b/unlock-and-start.sh deleted file mode 100755 index 88c56ca..0000000 --- a/unlock-and-start.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -: "${HETZNER_VOL:?HETZNER_VOL is not set; start a root login shell first}" - -cryptsetup open "$HETZNER_VOL" appdata -systemctl start srv-secure.mount -systemctl start var-lib-postgresql.mount -systemctl start var-lib-docker.mount -systemctl start var-lib-containerd.mount -systemctl start etc-odoo.mount -systemctl start var-log-nginx.mount -systemctl start var-log-postgresql.mount -systemctl start var-lib-nginx.mount -nginx -t -systemctl start odoo-app.target diff --git a/update.sh b/update.sh deleted file mode 100755 index f43e8f9..0000000 --- a/update.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -cd /opt/19 -git pull -git submodule update -systemctl restart deploy-manager19.service - -exit 0 From b4584be4ef756d245a74c46ea791d83a5c3b9d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 21:20:02 +0300 Subject: [PATCH 19/23] cleanup readme --- README.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4182f8e..c0ef097 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,7 @@ flowchart LR #### Prerequisite Use a fresh Ubuntu 26.04 LTS server and a separate empty Hetzner Volume. Attach the -volume without formatting or automatically mounting it. This is not an in-place -migration. The commands below format the selected device, so inspect it carefully -first. See [LUKS.md](LUKS.md) +volume without formatting or automatically mounting it. See [LUKS.md](LUKS.md) for the storage layout and recovery precautions. Point `19.eniemela.fi` to this server and allow TCP 9019 in the host and @@ -111,7 +109,7 @@ through nginx basic authentication; its backend is loopback-only on port 8019. #### Installation -Run the following as root. Identify the volume by matching its `MODEL`, `SERIAL`, +Identify the volume by matching its `MODEL`, `SERIAL`, and `SIZE` with the Hetzner Console. Use its `/dev/sdX` name only for the initial format; `/etc/crypttab` stores the stable UUID used after that. `luksFormat` and `mkfs.ext4` destroy data on the selected device, so confirm the `lsblk` output before continuing. @@ -149,45 +147,46 @@ repositories. bash ./ubuntu-install.sh ``` -Finish the configuration and start the application services: +Finish the configuration by adding secrets: ```bash +# Add passwrods htpasswd -B -C 12 -c /etc/nginx/.htpasswd cloud chown root:www-data /etc/nginx/.htpasswd chmod 640 /etc/nginx/.htpasswd -# Add passwrods vim /srv/secure/rclone-config/rclone.conf vim /srv/secure/secrets/cloudflare.ini +# Add SSL certs export TMPDIR=/srv/secure/tmp /usr/bin/python3 -m agentd.api ssl_wildcard # OR rsync -aHAX /etc/letsencrypt/ root@NEW_SERVER:/etc/letsencrypt/ - -nginx -t -systemctl start odoo-app.target ``` Encrypt `/root/appdata-luks-header-.img` before transferring it, then remove the plaintext copy. See [LUKS.md](LUKS.md#encrypting-the-header-backup). +```bash +reboot +``` + After every reboot, Ubuntu and SSH are available but application services stay stopped. From a root login shell, unlock, mount, and start them with: ```bash +# Mount encrypted partitions systemctl start systemd-cryptsetup@appdata.service +# Start applications systemctl start odoo-app.target ``` The systemd drop-ins installed by `ubuntu-install.sh` pull in the encrypted bind mounts and prevent protected services from starting if a required mount fails. -Use `findmnt /srv/secure` and `findmnt /var/lib/docker` to inspect mounts, and -`systemctl status odoo-app.target` to inspect the services. +Use `systemctl status odoo-app.target` to inspect the services. The encrypted filesystem also contains nginx request-body temporary files, -agent temporary files and rotating agent logs. Do not use unencrypted `/tmp` or -`/var/tmp` for database dumps. Persistent swap is disabled. Confirm that -`swapon --show` is empty before starting the application services. +agent temporary files and rotating agent logs. #### Backup setup (Client-Side Encrypted S3 Backups): * All database dumps and Odoo filestores are encrypted client-side via rclone's `crypt` backend before upload to AWS S3. From 6d8e11fa67a4b0b42bfa736c1473e029ee63e383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 21:21:17 +0300 Subject: [PATCH 20/23] cleanup readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c0ef097..7c979db 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ LUKS asks for the passphrase interactively and does not store it on the server. ```bash apt update apt install -y cryptsetup git vim tmux -git clone -b 19.0 --recurse-submodules --shallow-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/19 +git clone -b 19.0 --recurse-submodules https://github.com/elmeriniemela/deploy-manager.git /opt/19 cd /opt/19 lsblk -So NAME,MODEL,SERIAL,SIZE,TYPE INSTALL_DEVICE=/dev/sdX From 15056789b3ee0257953d2ebb3ef2ec9bac39553f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 21:27:17 +0300 Subject: [PATCH 21/23] simplify TMPDIR --- README.md | 3 +-- agentd/__init__.py | 4 ++++ cron/deploy-manager19 | 2 +- systemd/system/deploy-manager19.service | 1 - 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7c979db..0afabcc 100644 --- a/README.md +++ b/README.md @@ -158,8 +158,7 @@ vim /srv/secure/rclone-config/rclone.conf vim /srv/secure/secrets/cloudflare.ini # Add SSL certs -export TMPDIR=/srv/secure/tmp -/usr/bin/python3 -m agentd.api ssl_wildcard +python3 -m agentd.api ssl_wildcard # OR rsync -aHAX /etc/letsencrypt/ root@NEW_SERVER:/etc/letsencrypt/ ``` diff --git a/agentd/__init__.py b/agentd/__init__.py index 5f37138..4edbb71 100644 --- a/agentd/__init__.py +++ b/agentd/__init__.py @@ -1,2 +1,6 @@ """Deploy manager agent package.""" +import os + + +os.environ["TMPDIR"] = "/srv/secure/tmp" diff --git a/cron/deploy-manager19 b/cron/deploy-manager19 index e2a43d0..b594c95 100644 --- a/cron/deploy-manager19 +++ b/cron/deploy-manager19 @@ -1,4 +1,4 @@ SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -30 0 * * * root mountpoint -q /srv/secure && cd /opt/19 && TMPDIR=/srv/secure/tmp /usr/bin/python3 -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1 +30 0 * * * root mountpoint -q /srv/secure && cd /opt/19 && /usr/bin/python3 -m agentd.backup >> /srv/secure/logs/deploy-manager19.log 2>&1 diff --git a/systemd/system/deploy-manager19.service b/systemd/system/deploy-manager19.service index 96928c7..280a0e3 100644 --- a/systemd/system/deploy-manager19.service +++ b/systemd/system/deploy-manager19.service @@ -13,7 +13,6 @@ ExecStart=/usr/bin/python3 -m agentd --interface 127.0.0.1 --port 8019 Restart=always RestartSec=1 Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -Environment=TMPDIR=/srv/secure/tmp StandardOutput=append:/srv/secure/logs/deploy-manager19.log StandardError=inherit From 963e4646ae71073a07bafac4d18e0207eec86f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 21:37:41 +0300 Subject: [PATCH 22/23] rename file --- .../{postgresql.conf => postgresql-dependency.conf} | 0 ubuntu-install.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename systemd/system/docker.service.d/{postgresql.conf => postgresql-dependency.conf} (100%) diff --git a/systemd/system/docker.service.d/postgresql.conf b/systemd/system/docker.service.d/postgresql-dependency.conf similarity index 100% rename from systemd/system/docker.service.d/postgresql.conf rename to systemd/system/docker.service.d/postgresql-dependency.conf diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 2d0abfe..2314f2e 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -63,7 +63,7 @@ apt update apt install -y docker-ce docker-ce-cli containerd.io install -m 0644 systemd/system/odoo-app.target systemd/system/rclone-mount.service systemd/system/deploy-manager19.service /etc/systemd/system/ -install -m 0644 systemd/system/docker.service.d/postgresql.conf /etc/systemd/system/docker.service.d/postgresql.conf +install -m 0644 systemd/system/docker.service.d/postgresql-dependency.conf /etc/systemd/system/docker.service.d/postgresql-dependency.conf install -m 0644 systemd/system/postgresql@.service.d/tmp.conf /etc/systemd/system/postgresql@.service.d/tmp.conf install -D -m 0644 logrotate/deploy-manager19 /etc/logrotate.d/deploy-manager19 install -D -m 0644 cron/deploy-manager19 /etc/cron.d/deploy-manager19 From 06401037253eea9396549fdaa9aec1f1791f99fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elmeri=20Niemel=C3=A4?= Date: Wed, 9 Sep 2026 21:49:40 +0300 Subject: [PATCH 23/23] fix logging --- promtail/config.yml | 36 ++++++++++++++++++++----- systemd/system/deploy-manager19.service | 2 +- ubuntu-install.sh | 3 +++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/promtail/config.yml b/promtail/config.yml index dab24b4..ef7aab6 100644 --- a/promtail/config.yml +++ b/promtail/config.yml @@ -11,10 +11,32 @@ clients: insecure_skip_verify: true scrape_configs: -- job_name: system - static_configs: - - targets: - - localhost - labels: - job: 19.0-odoo-varlogs - __path__: /var/log/**/*log + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: 19.0-odoo-varlogs + __path__: /var/log/**/*log + + - job_name: deploy-manager + static_configs: + - targets: + - localhost + labels: + job: 19.0-deploy-manager + __path__: /srv/secure/logs/deploy-manager19.log + + - job_name: systemd-journal + journal: + max_age: 12h + labels: + job: 19.0-odoo-systemd + relabel_configs: + - source_labels: ['__journal__systemd_unit'] + regex: '(deploy-manager19|docker|containerd|nginx|postgresql@18-main|rclone-mount)\.service' + action: keep + - source_labels: ['__journal__systemd_unit'] + target_label: unit + - source_labels: ['__journal_priority_keyword'] + target_label: level diff --git a/systemd/system/deploy-manager19.service b/systemd/system/deploy-manager19.service index 280a0e3..6191f80 100644 --- a/systemd/system/deploy-manager19.service +++ b/systemd/system/deploy-manager19.service @@ -14,7 +14,7 @@ Restart=always RestartSec=1 Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin StandardOutput=append:/srv/secure/logs/deploy-manager19.log -StandardError=inherit +StandardError=append:/srv/secure/logs/deploy-manager19.log [Install] WantedBy=odoo-app.target diff --git a/ubuntu-install.sh b/ubuntu-install.sh index 2314f2e..0e7c75c 100755 --- a/ubuntu-install.sh +++ b/ubuntu-install.sh @@ -138,6 +138,9 @@ if ! docker container inspect promtail >/dev/null 2>&1; then --restart unless-stopped \ -v /opt/19/promtail:/etc/promtail:ro \ -v /var/log:/var/log:ro \ + -v /srv/secure/logs:/srv/secure/logs:ro \ + -v /run/log/journal:/run/log/journal:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ grafana/promtail:latest \ -config.file=/etc/promtail/config.yml else