fix: Reevaluate and restart the run when there is a connection

.changelogs/2.4.0/134-add-node-availability-check-before-migrations.yml

@@ -0,0 +1,2 @@
+added:
+- Check that all cluster nodes still respond to `nodes/{node}/status` before dispatching each migration during balancing; restarts ProxLB from scratch if a node has become unavailable. [#134]

.changelogs/2.4.0/134-fix_segfault_when_pve_api_unreachable.yml

@@ -0,0 +1,2 @@
+fix:
+- Fix segmentation fault when PVE API is unreachable during runtime. [#134]

.changelogs/2.4.0/release_meta.yml

@@ -0,0 +1 @@
+date: TBD

proxlb/__main__.py

@@ -14,6 +14,11 @@
 
 import logging
 import signal
+import time
+
+import proxmoxer
+import proxmoxer.backends.https
+import requests
 
 from proxlb_solver import shadow as _solver_shadow
 from proxlb_solver.models import MigrationPlan
@@ -31,7 +36,7 @@
 from proxlb.models.pools import Pools
 from proxlb.models.ha_rules import HaRules
 from proxlb.models.ha_status import HaStatus
-from proxlb.utils.helper import Helper
+from proxlb.utils.helper import Helper, NodeUnavailableError
 from proxlb.utils.proxlb_data import ProxLbData
 
 
@@ -83,116 +88,139 @@ def reinstall_sigint() -> None:
 
 while True:
 
-    # Validate if HA mode is enabled and if this node is the HA manager.
-    if proxlb_config.service.enable_ha:
-        if not HaStatus.is_node_ha_manager(proxmox_api):
-            logger.info("This node is not the HA manager. Waiting for next run.")
-            Helper.get_daemon_mode(proxlb_config)
-            continue
-        else:
-            logger.debug("This node is the HA manager. Continuing.")
-
-    # Validate if reload signal was sent during runtime
-    # and reload the ProxLB configuration and adjust log level
-    if Helper.proxlb_reload:
-        logger.info("Reloading ProxLB configuration.")
-        proxlb_config = config_parser.get_config()
-        logger.set_log_level(proxlb_config.service.log_level)
-        Helper.proxlb_reload = False
-
-    # Get all required objects from the Proxmox cluster
-    Helper.apply_maintenance_nodes_schedule(proxlb_config)
-    nodes = Nodes.get_nodes(proxmox_api, proxlb_config)
-    meta = Features.validate_any_non_pve9_node(proxlb_config, nodes)
-    pools = Pools.get_pools(proxmox_api)
-    ha_rules = HaRules.get_ha_rules(proxmox_api, meta)
-    guests = Guests.get_guests(proxmox_api, pools, ha_rules, nodes, proxlb_config)
-    groups = Groups.get_groups(guests, nodes)
-
-    # Merge obtained objects from the Proxmox cluster for further usage
-    proxlb_data = ProxLbData(
-        meta=meta,
-        nodes=nodes,
-        guests=guests,
-        pools=pools,
-        ha_rules=ha_rules,
-        groups=groups,
-    )
-    Helper.log_node_metrics(proxlb_data)
-
-    # Validate usable features by PVE versions
-    Features.validate_available_features(proxlb_data)
-
-    # Update the initial node resource assignments
-    # by the previously created groups.
-    Calculations.set_node_assignments(proxlb_data)
-    Helper.log_node_metrics(proxlb_data, init=False)
-    Calculations.set_node_hot(proxlb_data)
-    Calculations.set_guest_hot(proxlb_data)
-    target = Calculations.get_most_free_node(proxlb_data, cli_args.best_node)
-    if target is None:
-        logger.warning("No suitable target node found for balancing. Skipping this run.")
-    else:
-        Calculations.validate_affinity_map(proxlb_data)
-        Calculations.relocate_guests_on_maintenance_nodes(proxlb_data)
-        Calculations.get_balanciness(proxlb_data)
-        Calculations.relocate_guests(proxlb_data)
-
-        # CP-SAT solver (optional) — shadow (read-only) or active mode.
-        _solver_cfg = proxlb_config.solver
-        _run_file: str | None = None
-        _solver_plan: MigrationPlan | None = None
-        if _solver_cfg.enable:
-            _run_file, _solver_plan = _solver_shadow.run_shadow(
-                proxlb_data, _solver_cfg
-            )
-            reinstall_sigint()
-
+    try:
+        # Validate if HA mode is enabled and if this node is the HA manager.
+        if proxlb_config.service.enable_ha:
+            if not HaStatus.is_node_ha_manager(proxmox_api):
+                logger.info("This node is not the HA manager. Waiting for next run.")
+                Helper.get_daemon_mode(proxlb_config)
+                continue
+            else:
+                logger.debug("This node is the HA manager. Continuing.")
+
+        # Validate if reload signal was sent during runtime
+        # and reload the ProxLB configuration and adjust log level
+        if Helper.proxlb_reload:
+            logger.info("Reloading ProxLB configuration.")
+            proxlb_config = config_parser.get_config()
+            logger.set_log_level(proxlb_config.service.log_level)
+            Helper.proxlb_reload = False
+
+        # Get all required objects from the Proxmox cluster
+        Helper.apply_maintenance_nodes_schedule(proxlb_config)
+        nodes = Nodes.get_nodes(proxmox_api, proxlb_config)
+        meta = Features.validate_any_non_pve9_node(proxlb_config, nodes)
+        pools = Pools.get_pools(proxmox_api)
+        ha_rules = HaRules.get_ha_rules(proxmox_api, meta)
+        guests = Guests.get_guests(proxmox_api, pools, ha_rules, nodes, proxlb_config)
+        groups = Groups.get_groups(guests, nodes)
+
+        # Merge obtained objects from the Proxmox cluster for further usage
+        proxlb_data = ProxLbData(
+            meta=meta,
+            nodes=nodes,
+            guests=guests,
+            pools=pools,
+            ha_rules=ha_rules,
+            groups=groups,
+        )
+        Helper.log_node_metrics(proxlb_data)
+
+        # Validate usable features by PVE versions
+        Features.validate_available_features(proxlb_data)
+
+        # Update the initial node resource assignments
+        # by the previously created groups.
+        Calculations.set_node_assignments(proxlb_data)
         Helper.log_node_metrics(proxlb_data, init=False)
+        Calculations.set_node_hot(proxlb_data)
+        Calculations.set_guest_hot(proxlb_data)
+        target = Calculations.get_most_free_node(proxlb_data, cli_args.best_node)
+        if target is None:
+            logger.warning("No suitable target node found for balancing. Skipping this run.")
+        else:
+            Calculations.validate_affinity_map(proxlb_data)
+            Calculations.relocate_guests_on_maintenance_nodes(proxlb_data)
+            Calculations.get_balanciness(proxlb_data)
+            Calculations.relocate_guests(proxlb_data)
+
+            # CP-SAT solver (optional) — shadow (read-only) or active mode.
+            _solver_cfg = proxlb_config.solver
+            _run_file: str | None = None
+            _solver_plan: MigrationPlan | None = None
+            if _solver_cfg.enable:
+                _run_file, _solver_plan = _solver_shadow.run_shadow(
+                    proxlb_data, _solver_cfg
+                )
+                reinstall_sigint()
 
-        # Perform balancing actions via Proxmox API
-        _skipped_greedy_balancing = False
-        if proxlb_data.meta.balancing.enable:
-            if not cli_args.dry_run:
-                if (_solver_cfg.enable
-                        and _solver_cfg.mode == "active"):
-                    if _solver_plan is not None:
-                        try:
-                            _solver_shadow.execute_solver_plan(
-                                proxmox_api, proxlb_data,
-                                _solver_plan, _solver_cfg, _run_file
-                            )
-                        except Exception as exc:
-                            if _solver_cfg.fallback_to_greedy:
-                                logger.warning(
-                                    f"[solver] active execution failed, "
-                                    f"falling back to ProxLB plan: {exc}")
-                                Balancing.balance(proxmox_api, proxlb_data)
-                            else:
-                                _skipped_greedy_balancing = True
-                                logger.warning(
-                                    f"[solver] active execution failed, "
-                                    f"skipping greedy fallback "
-                                    f"(fallback_to_greedy=False): {exc}")
-                    elif _solver_cfg.fallback_to_greedy:
-                        Balancing.balance(proxmox_api, proxlb_data)
+            Helper.log_node_metrics(proxlb_data, init=False)
+
+            # Perform balancing actions via Proxmox API
+            _skipped_greedy_balancing = False
+            if proxlb_data.meta.balancing.enable:
+                if not cli_args.dry_run:
+                    if (_solver_cfg.enable
+                            and _solver_cfg.mode == "active"):
+                        if _solver_plan is not None:
+                            try:
+                                _solver_shadow.execute_solver_plan(
+                                    proxmox_api, proxlb_data,
+                                    _solver_plan, _solver_cfg, _run_file
+                                )
+                            except Exception as exc:
+                                if _solver_cfg.fallback_to_greedy:
+                                    logger.warning(
+                                        f"[solver] active execution failed, "
+                                        f"falling back to ProxLB plan: {exc}")
+                                    Balancing.balance(proxmox_api, proxlb_data)
+                                else:
+                                    _skipped_greedy_balancing = True
+                                    logger.warning(
+                                        f"[solver] active execution failed, "
+                                        f"skipping greedy fallback "
+                                        f"(fallback_to_greedy=False): {exc}")
+                        elif _solver_cfg.fallback_to_greedy:
+                            Balancing.balance(proxmox_api, proxlb_data)
+                        else:
+                            _skipped_greedy_balancing = True
+                            logger.warning(
+                                "[solver] active: no feasible solver plan; "
+                                "skipping greedy fallback "
+                                "(fallback_to_greedy=False)")
                     else:
-                        _skipped_greedy_balancing = True
-                        logger.warning(
-                            "[solver] active: no feasible solver plan; "
-                            "skipping greedy fallback "
-                            "(fallback_to_greedy=False)")
-                else:
-                    Balancing.balance(proxmox_api, proxlb_data)
+                        Balancing.balance(proxmox_api, proxlb_data)
+                    reinstall_sigint()
+
+            # Record whether balancing was executed or skipped (dry-run).
+            if _run_file is not None and not _skipped_greedy_balancing:
+                try:
+                    _solver_shadow.finalize_run(_run_file, dry_run=cli_args.dry_run)
+                except Exception as exc:
+                    logger.warning(f"[solver] finalize_run failed: {exc}")
                 reinstall_sigint()
 
-        # Record whether balancing was executed or skipped (dry-run).
-        if _run_file is not None and not _skipped_greedy_balancing:
-            try:
-                _solver_shadow.finalize_run(_run_file, dry_run=cli_args.dry_run)
-            except Exception as exc:
-                logger.warning(f"[solver] finalize_run failed: {exc}")
-            reinstall_sigint()
+    except (
+        proxmoxer.core.ResourceException,
+        proxmoxer.backends.https.AuthenticationError,
+        requests.exceptions.RequestException,
+        NodeUnavailableError,
+    ) as api_error:
+        logger.critical(
+            f"ProxLB: A Proxmox API call failed or a cluster node became unavailable: {api_error}. "
+            "Reconnecting to the Proxmox API and restarting balancing run from scratch."
+        )
+        time.sleep(proxlb_config.proxmox_api.wait_time)
+
+        # Reload the config to recover the real credentials (masked below after
+        # every connect) and reconnect from scratch, re-running host failover.
+        proxlb_config = config_parser.get_config()
+        logger.set_log_level(proxlb_config.service.log_level)
+        proxmox_api = ProxmoxApi(proxlb_config)
+        proxlb_config.proxmox_api.password = "********"
+        proxlb_config.proxmox_api.token_secret = "********"
+        Helper.get_daemon_mode(proxlb_config)
+        continue
 
     # Validate if the JSON output should be
     # printed to stdout

proxlb/models/balancing.py

@@ -12,6 +12,7 @@
 
 import proxmoxer
 import time
+from proxlb.utils.helper import Helper, NodeUnavailableError
 from proxlb.utils.logger import SystemdLogger
 from proxlb.utils.proxmox_api import ProxmoxApi
 from proxlb.utils.config_parser import Config
@@ -93,14 +94,21 @@ def balance(proxmox_api: ProxmoxApi, proxlb_data: ProxLbData) -> bool:
 
         Keeps up to parallel_job_limit migrations in flight at once and immediately
         submits the next guest as soon as a slot becomes free, rather than waiting
-        for an entire chunk to finish.
+        for an entire chunk to finish. Before dispatching a guest that will actually
+        be migrated, all cluster nodes are re-checked for availability so a node
+        that disappeared mid-run aborts the run instead of migrating guests onto a
+        stale target. Guests that would not be migrated anyway (already on their
+        target node, ignored, or of a type excluded from balancing) skip this check.
 
         Args:
             proxmox_api (ProxmoxApi): The Proxmox API client instance.
             proxlb_data (ProxLbData): ProxLB load balancing data.
 
         Returns:
             bool: True if all migrations completed successfully, False otherwise.
+
+        Raises:
+            NodeUnavailableError: If a cluster node is no longer online.
         """
         logger.debug("Starting: balance.")
         parallel_job_limit = Balancing.get_parallel_job_limit(proxlb_data.meta.balancing)
@@ -118,6 +126,18 @@ def balance(proxmox_api: ProxmoxApi, proxlb_data: ProxLbData) -> bool:
                 if len(jobs_to_wait) >= parallel_job_limit:
                     time.sleep(5)
 
+            guest_will_move = (
+                guest_meta.node_current != guest_meta.node_target
+                and not guest_meta.ignore
+                and guest_meta.type in proxlb_data.meta.balancing.balance_types)
+
+            if guest_will_move and not Helper.check_nodes_available(proxmox_api, proxlb_data.nodes):
+                logger.critical(
+                    "Balancing: At least one cluster node is no longer online. "
+                    "Aborting the current balancing run.")
+                raise NodeUnavailableError(
+                    "At least one cluster node became unavailable during balancing.")
+
             job_id = Balancing._exec_rebalancing(proxmox_api, proxlb_data, guest_name)
             logger.debug(f"Balancing: job_id for {guest_name}: {job_id!r}, jobs_to_wait len: {len(jobs_to_wait)}")
             if job_id is not None: