Add native HA clustering

.github/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/pi-hole/ftl-build:v2.25
+FROM ghcr.io/pi-hole/ftl-build:v2.26.1
 
 WORKDIR /app
 

.gitignore

@@ -58,3 +58,6 @@ test/libs/
 # Local assistant tooling artifacts
 /.claude/settings.local.json
 /.claude/test-logs/
+
+# Cluster secret for the local test cluster
+test/cluster/cluster_secret

src/CMakeLists.txt

@@ -339,6 +339,7 @@ add_executable(pihole-FTL
         $<TARGET_OBJECTS:tools>
         $<TARGET_OBJECTS:ntp>
         $<TARGET_OBJECTS:dotdoh>
+        $<TARGET_OBJECTS:cluster>
         )
 if(STATIC)
     set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON)
@@ -410,6 +411,7 @@ add_subdirectory(config)
 add_subdirectory(tools)
 add_subdirectory(ntp)
 add_subdirectory(dotdoh)
+add_subdirectory(cluster)
 
 # Standalone regression harness for the in-memory tar parser (zip/tar.c).
 # It #includes tar.c directly to reach the file-internal helpers, stubs the
@@ -674,6 +676,35 @@ if(LIBSSL AND LIBCRYPTO AND USE_TLS)
         message(STATUS "Building FTL with HTTP/2 support: NO (nghttp2 not found)")
     endif()
 
+    # libcurl is the HTTP client used to talk to the other nodes of a cluster.
+    # It is built against the same OpenSSL as the rest of FTL
+    find_library(LIBCURL NAMES libcurl${LIBRARY_SUFFIX} curl)
+    if(LIBCURL)
+        message(STATUS "Building FTL with cluster support: YES")
+        target_compile_definitions(cluster PRIVATE HAVE_CURL)
+        target_compile_definitions(api PRIVATE HAVE_CURL)
+        # Inside a group with what it draws on: OpenSSL and nghttp2 are already
+        # on the link line, but ahead of libcurl, and a static archive only
+        # answers symbols that are still undefined when the linker reaches it.
+        # Whether that bites depends on what the archives happened to pull in
+        # earlier, which is not something to leave to chance
+        target_link_libraries(pihole-FTL
+            -Wl,--start-group ${LIBCURL} ${LIBSSL} ${LIBCRYPTO} ${LIBNGHTTP2} -Wl,--end-group)
+        # Static libcurl pulls in zlib for the content encodings it offers. The
+        # archive whichever way FTL itself is linked: a dynamic build would
+        # otherwise take the shared one and gain a runtime dependency
+        find_library(LIBZ NAMES libz.a libz${LIBRARY_SUFFIX} z)
+        if(LIBZ)
+            target_link_libraries(pihole-FTL ${LIBZ})
+        endif()
+        # ...and libidn2 for the internationalized hostnames it accepts
+        if(LIBIDN2)
+            target_link_libraries(pihole-FTL ${LIBIDN2})
+        endif()
+    else()
+        message(STATUS "Building FTL with cluster support: NO (libcurl not found)")
+    endif()
+
     # HTTP/3 for the TLS terminator (OpenSSL QUIC + nghttp3). Requires nghttp3
     # AND OpenSSL >= 4.0, which provides SSL_get_peer_addr() (the per-connection
     # QUIC peer getter used to forward the real client IP). Against an older

src/api/2fa.c

@@ -245,6 +245,9 @@ static time_t last_attempt = 0;
 // been accepted, that step and any earlier one can never be replayed, even
 // though the code stays numerically valid across the -1..+1 acceptance window.
 static uint64_t last_counter = 0;
+// ...and the last one a login on this node accepted, which the cluster tells
+// its peers about at once
+static uint64_t local_counter = 0;
 static pthread_mutex_t totp_lock = PTHREAD_MUTEX_INITIALIZER;
 enum totp_status verifyTOTP(const uint32_t incode)
 {
@@ -310,6 +313,7 @@ enum totp_status verifyTOTP(const uint32_t incode)
 			const char *which = i == -1 ? "previous" : i == 0 ? "current" : "next";
 			log_web_debug(DEBUG_API, "2FA code from %s time step is valid", which);
 			last_counter = counter;
+			local_counter = counter;
 			pthread_mutex_unlock(&totp_lock);
 			return TOTP_CORRECT;
 		}
@@ -327,6 +331,35 @@ enum totp_status verifyTOTP(const uint32_t incode)
 	return TOTP_INVALID;
 }
 
+// The last accepted time step, for the cluster to publish so a code used on one
+// node is known to be spent on the others
+uint64_t totp_last_counter(void)
+{
+	pthread_mutex_lock(&totp_lock);
+	const uint64_t counter = last_counter;
+	pthread_mutex_unlock(&totp_lock);
+	return counter;
+}
+
+uint64_t totp_local_counter(void)
+{
+	pthread_mutex_lock(&totp_lock);
+	const uint64_t counter = local_counter;
+	pthread_mutex_unlock(&totp_lock);
+	return counter;
+}
+
+// A time step a peer has already accepted. Raising this node's guard to it
+// makes that step, and any earlier one, unusable here too - the same code
+// cannot be replayed from node to node while it is still numerically valid
+void totp_observe_counter(const uint64_t counter)
+{
+	pthread_mutex_lock(&totp_lock);
+	if(counter > last_counter)
+		last_counter = counter;
+	pthread_mutex_unlock(&totp_lock);
+}
+
 // Print TOTP code to stdout (for CLI use)
 int printTOTP(void)
 {

src/api/CMakeLists.txt

@@ -15,6 +15,7 @@ set(sources
         api.c
         auth.c
         auth.h
+        cluster.c
         config.c
         dhcp.c
         dns.c

src/api/api.c

@@ -105,6 +105,13 @@ static struct {
 	{ "/api/action/restartdns",                 "",                           api_action_restartDNS,                 { API_PARSE_JSON, 0                         }, true,  HTTP_POST },
 	{ "/api/action/flush/logs",                 "",                           api_action_flush_logs,                 { API_PARSE_JSON, 0                         }, true,  HTTP_POST },
 	{ "/api/action/flush/network",              "",                           api_action_flush_network,              { API_PARSE_JSON, 0                         }, true,  HTTP_POST },
+	{ "/api/cluster/lists",                     "",                           api_cluster_lists,                     { API_FLAG_NONE, 0                          }, true,  HTTP_GET },
+	{ "/api/cluster/leases",                    "",                           api_cluster_leases,                    { API_FLAG_NONE, 0                          }, true,  HTTP_GET },
+	{ "/api/cluster/discover",                  "",                           api_cluster_discover,                  { API_FLAG_NONE, 0                          }, true,  HTTP_GET },
+	{ "/api/cluster/enroll",                    "",                           api_cluster_enroll,                    { API_PARSE_JSON, 0                         }, true,  HTTP_POST },
+	{ "/api/cluster/join",                      "",                           api_cluster_join,                      { API_PARSE_JSON, 0                         }, true,  HTTP_POST },
+	{ "/api/cluster/leave",                     "",                           api_cluster_leave,                     { API_PARSE_JSON, 0                         }, true,  HTTP_POST },
+	{ "/api/cluster/status",                    "",                           api_cluster_status,                    { API_PARSE_JSON, 0                         }, true,  HTTP_GET },
 	{ "/api/padd",                              "",                           api_padd,                              { API_PARSE_JSON, 0                         }, true,  HTTP_GET },
 	{ "/api/docs",                              "",                           api_docs,                              { API_PARSE_JSON, 0                         }, false, HTTP_GET },
 };
@@ -215,6 +222,25 @@ int api_handler(struct mg_connection *conn, void *ignored)
 				break;
 			}
 
+			// A peer of our cluster reads what we publish about
+			// ourselves and hands us its configuration. It has no
+			// business anywhere else, so a stolen cluster secret
+			// reaches neither the password hashes in /api/config nor
+			// the Teleporter archive that carries them
+			const bool cluster_read = api.method == HTTP_GET &&
+			    strncmp(api.request->local_uri_raw, "/api/cluster/", sizeof("/api/cluster/") - 1) == 0 &&
+			    strcmp(api.request->local_uri_raw, "/api/cluster/discover") != 0;
+			const bool cluster_push = api.method == HTTP_PATCH &&
+			    strcmp(api.request->local_uri_raw, "/api/config") == 0;
+			if(api.session.used && api.session.cluster && !cluster_read && !cluster_push)
+			{
+				ret = send_json_error(&api, 403,
+				                      "forbidden",
+				                      "Cluster requests are confined to synchronization",
+				                      "A peer reads /api/cluster and hands us its configuration through PATCH /api/config, and needs nothing beyond that");
+				break;
+			}
+
 			// Call the API function and get the return code
 			log_web_debug(DEBUG_API, "Processing %s %s in %s",
 			          api.request->request_method,

src/api/api.h

@@ -78,7 +78,12 @@ int get_version_obj(struct ftl_conn *api, cJSON *version);
 // Config methods
 int api_config(struct ftl_conn *api);
 int api_config_properties(struct ftl_conn *api);
-int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed);
+// cluster selects the document a peer is handed. encrypted says whether the
+// connection it travels over hides it: the credentials are left out when it
+// does not, because a signature keeps them from being changed but not from
+// being read
+int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed, const bool cluster,
+                    const bool encrypted);
 cJSON *addJSONConfValue(const enum conf_type conf_type, union conf_value *val);
 
 // Log methods
@@ -116,6 +121,11 @@ enum totp_status {
 	TOTP_RATE_LIMIT
 } __attribute__ ((packed));
 enum totp_status verifyTOTP(const uint32_t code);
+// The last accepted TOTP time step, shared across a cluster so a code accepted
+// on one node cannot be replayed on another within its validity window
+uint64_t totp_last_counter(void);
+uint64_t totp_local_counter(void);
+void totp_observe_counter(const uint64_t counter);
 int generateTOTP(struct ftl_conn *api);
 int printTOTP(void);
 int generateAppPw(struct ftl_conn *api);
@@ -139,6 +149,15 @@ int api_search(struct ftl_conn *api);
 int api_dhcp_leases_GET(struct ftl_conn *api);
 int api_dhcp_leases_DELETE(struct ftl_conn *api);
 
+// Cluster methods
+int api_cluster_status(struct ftl_conn *api);
+int api_cluster_discover(struct ftl_conn *api);
+int api_cluster_enroll(struct ftl_conn *api);
+int api_cluster_join(struct ftl_conn *api);
+int api_cluster_lists(struct ftl_conn *api);
+int api_cluster_leases(struct ftl_conn *api);
+int api_cluster_leave(struct ftl_conn *api);
+
 // PADD methods
 int api_padd(struct ftl_conn *api);
 

src/api/auth.c

@@ -10,6 +10,8 @@
 
 #include "FTL.h"
 #include "api/auth.h"
+// cluster_verify_request()
+#include "cluster/auth.h"
 #include "webserver/http-common.h"
 #include "webserver/json_macros.h"
 #include "api/api.h"
@@ -185,6 +187,15 @@ static bool known_host(const char *host)
 // Returns >= 0 for any valid authentication
 int check_client_auth(struct ftl_conn *api, const bool is_api)
 {
+	// A peer of our cluster signs every request with the shared secret
+	// instead of holding a session. This is checked before anything else so
+	// it works the same whether or not a web password is set here - but only
+	// for the API, as the pages of the web interface are not what the
+	// cluster secret unlocks and they are served without the confinement
+	// that keeps a peer to the two endpoints it needs
+	if(is_api && cluster_verify_request(api))
+		return API_AUTH_CLUSTER;
+
 	// When the pwhash is unset, authentication is disabled, but not for
 	// other websites using the API through a browser on the network
 	if(config.webserver.api.pwhash.v.s[0] == '\0')

src/api/auth.h

@@ -47,6 +47,8 @@ struct session {
 	bool used;
 	bool app;
 	bool cli;
+	bool cluster;
+	long long cluster_seq;     // the sequence the peer signed this request with
 	struct {
 		bool login;
 		bool mixed;