fix(car): clamp manual car SoC and advance it by measured energy, not the plan

apps/predbat/const.py

@@ -112,6 +112,18 @@
 # A car_charging_now sensor that reports a charging power (in W, after unit conversion) rather than
 # on/off counts as charging from this power: above a charger's standby draw, below the smallest charge
 CAR_CHARGING_NOW_POWER_W = 200
+# A car_charging_energy counter rise is only added to the manual car SoC up to what the car could have taken: an
+# energy budget refilled at car_charging_rate x this margin between readings, starting from a fixed allowance for rounding
+CAR_MANUAL_SOC_RATE_MARGIN = 1.1
+CAR_MANUAL_SOC_ENERGY_MARGIN_KWH = 0.1
+# How long the manual car SoC holds while a car_charging_energy counter it has read before is unreadable, before it
+# drops the counter's baseline and falls back to car_charging_now / the plan until the counter is back
+CAR_MANUAL_SOC_COUNTER_HOLD_MINUTES = 60
+# The budget above holds at most this many minutes of charging at car_charging_rate, bounding it after a long idle spell
+CAR_MANUAL_SOC_MAX_GAP_MINUTES = 60
+# After a car_charging_energy counter drops, rises below its value before the drop are held back this long in case the
+# drop was a glitch (an integration reload) that is about to recover, rather than a new session
+CAR_MANUAL_SOC_GLITCH_MINUTES = 15
 CAR_CHARGING_LIMIT_UNCAPPED = 9999.0  # Model-facing car charge limit (kWh) that makes predict()'s fill clamp inert - larger than any real car battery (#4967)
 DEBUG_ENABLE_MAX_HOURS = 2  # Auto-disable switch.predbat_debug_enable after this long left on, to bound the raw per-cycle debug.yaml disk writes it triggers (and the C++ kernel bypass it forces) if left on by accident - the rotating debug-history buffer covers longer-term history at a coarser interval instead
 # How far ahead a manual override may be placed. The two horizons differ on purpose: a manual

apps/predbat/fetch.py

@@ -1873,6 +1873,18 @@ def car_planned_slot_entity(self, planned_entity_id_list, car_n, slot_entity_id)
             return None
         return planned_entity_id
 
+    def car_manual_soc_kwh(self, car_n):
+        """
+        Read car_n's manual SoC tracker in kWh, kept within [0, battery size] - it is a free-entry number
+        that can hold anything, e.g. 100 kWh entered for a 10 kWh car.
+        """
+        car_postfix = "" if car_n == 0 else "_" + str(car_n)
+        try:
+            soc_kwh = float(self.get_arg("car_charging_manual_soc_kwh" + car_postfix, 0.0))
+        except (ValueError, TypeError):
+            soc_kwh = 0.0
+        return max(0.0, min(soc_kwh, self.car_charging_battery_size[car_n]))
+
     def fetch_sensor_data_cars(self, save=True):
         """
         Fetch car specific data such as Octopus intelligent slots and vehicle data if we can get it, and calculate current SoC and limits based on that
@@ -1890,8 +1902,7 @@ def fetch_sensor_data_cars(self, save=True):
         self.car_charging_soc_next = [None for car_n in range(self.num_cars)]
         for car_n in range(self.num_cars):
             if car_n < len(self.car_charging_manual_soc) and self.car_charging_manual_soc[car_n]:
-                car_postfix = "" if car_n == 0 else "_" + str(car_n)
-                self.car_charging_soc[car_n] = self.get_arg("car_charging_manual_soc_kwh" + car_postfix, 0.0)
+                self.car_charging_soc[car_n] = self.car_manual_soc_kwh(car_n)
             else:
                 self.car_charging_soc[car_n] = (self.get_arg("car_charging_soc", 0.0, index=car_n) * self.car_charging_battery_size[car_n]) / 100.0
 
@@ -1995,7 +2006,10 @@ def fetch_sensor_data_cars(self, save=True):
                 if size:
                     self.car_charging_battery_size[car_n] = size
                     # Recalculate car SoC now that battery size has been updated from Octopus data
-                    if car_n < len(self.car_charging_manual_soc) and not self.car_charging_manual_soc[car_n]:
+                    if car_n < len(self.car_charging_manual_soc) and self.car_charging_manual_soc[car_n]:
+                        # The manual SoC was clamped to the configured size above - clamp it again to the real one
+                        self.car_charging_soc[car_n] = self.car_manual_soc_kwh(car_n)
+                    else:
                         self.car_charging_soc[car_n] = (self.get_arg("car_charging_soc", 0.0, index=car_n) * self.car_charging_battery_size[car_n]) / 100.0
                 if rate:
                     # Octopus's figure only ever raises the configured rate, never lowers it. Where Octopus's catalogue

apps/predbat/predbat.py

@@ -77,6 +77,11 @@
     CONFIG_REFRESH_PERIOD,
     INVERTER_QUICK_UPDATE_SECONDS,
     DEBUG_ENABLE_MAX_HOURS,
+    CAR_MANUAL_SOC_RATE_MARGIN,
+    CAR_MANUAL_SOC_ENERGY_MARGIN_KWH,
+    CAR_MANUAL_SOC_COUNTER_HOLD_MINUTES,
+    CAR_MANUAL_SOC_MAX_GAP_MINUTES,
+    CAR_MANUAL_SOC_GLITCH_MINUTES,
 )
 from config import APPS_SCHEMA, CONFIG_ITEMS
 import debug_history
@@ -551,6 +556,8 @@ def reset(self):
         self.discharge_rate_now = 0
         self.car_charging_hold = False
         self.car_charging_manual_soc = []
+        self.car_charging_manual_energy_last = {}
+        self.car_charging_manual_soc_written = {}
         self.car_charging_threshold = 99
         self.car_charging_energy = {}
         self.car_charging_energy_warned = False
@@ -1158,22 +1165,141 @@ def record_final_run_status(self, status, status_extra):
                 extra=status_extra,
             )
 
+    def car_manual_soc_measured_energy(self, car_n):
+        """
+        Energy in kWh delivered to car_n since the last good reading of the car_charging_energy session counter.
+
+        Returns (configured, kwh). configured is False when there is no usable counter for this car:
+        car_charging_energy is a single total across all cars, so it is only attributed to a car when
+        there is exactly one. A counter that cannot be read (e.g. unavailable) gives (True, None) while
+        there is a baseline from a good reading less than CAR_MANUAL_SOC_COUNTER_HOLD_MINUTES old - the
+        baseline is kept, so the energy delivered across the gap is counted once by the next good reading.
+        With no baseline, or once the outage is longer than that, the baseline is dropped and it is
+        (False, None), so a counter that is missing or broken does not freeze the SoC for good; the next
+        good reading starts a new baseline.
+
+        The baseline is in memory only. The first reading after a start only sets it, so energy between
+        the last write-back before a restart and the first one after it is not counted. A counter that
+        drops (a new session, or an integration reload glitching to 0) starts a new baseline at the new
+        value and adds nothing. A rise is clamped (not discarded) to what the car could have taken at
+        car_charging_rate: an energy budget that refills at that rate between readings, up to
+        CAR_MANUAL_SOC_MAX_GAP_MINUTES' worth, and is spent by each rise - so a counter with coarse steps
+        or slow updates, whose rises lag and then catch up, is counted in full.
+
+        A drop may be a glitch (an integration reload: 500 -> 0 -> 500, or in steps 500 -> 250 -> 0 -> 500
+        or 500 -> 0 -> 250 -> 500), so for CAR_MANUAL_SOC_GLITCH_MINUTES after the latest drop the highest
+        value before the drops is remembered and rises below it are not counted and leave the baseline at
+        the low value. A rise back to about that value is the glitch recovering: only what it shows above
+        it counts. Once the window passes without that, the drop was a real reset (a new session) and the
+        rise from the low value is counted as normal, within the budget.
+        """
+        if self.num_cars != 1:
+            return False, None
+        entity_id = self.get_arg("car_charging_energy", None, indirect=False)
+        if isinstance(entity_id, list) and len(entity_id) == 1:
+            entity_id = entity_id[0]
+        if not is_entity_id(entity_id):
+            return False, None
+        try:
+            energy_now = float(self.resolve_arg("car_charging_energy", entity_id, default=None, required_unit="kWh")) * self.get_arg("car_charging_energy_scale", 1.0)
+        except (ValueError, TypeError):
+            energy_now = None
+        now = self.now_utc
+        state = self.car_charging_manual_energy_last.get(car_n)
+        if energy_now is None:
+            if state is not None and (now - state["read"]).total_seconds() <= CAR_MANUAL_SOC_COUNTER_HOLD_MINUTES * 60:
+                return True, None
+            self.car_charging_manual_energy_last.pop(car_n, None)
+            return False, None
+        if state is None:
+            self.car_charging_manual_energy_last[car_n] = {"value": energy_now, "read": now, "budget": CAR_MANUAL_SOC_ENERGY_MARGIN_KWH, "pre_drop": None, "drop_time": None}
+            return True, 0.0
+        # Refill the budget for the time since the last good reading: the most the car could have taken
+        rate_kw = self.car_charging_rate[car_n] if car_n < len(self.car_charging_rate) else 0.0
+        elapsed_hours = max((now - state["read"]).total_seconds() / 3600.0, 0.0)
+        budget_max = rate_kw * CAR_MANUAL_SOC_MAX_GAP_MINUTES / 60.0 * CAR_MANUAL_SOC_RATE_MARGIN + CAR_MANUAL_SOC_ENERGY_MARGIN_KWH
+        state["budget"] = min(state["budget"] + rate_kw * elapsed_hours * CAR_MANUAL_SOC_RATE_MARGIN, budget_max)
+        state["read"] = now
+        delta = energy_now - state["value"]
+        if delta == 0:
+            return True, 0.0
+        if delta < 0:
+            self.log("Info: Car {} charging energy counter dropped from {} to {} kWh, treating it as a reset".format(car_n, dp3(state["value"]), dp3(energy_now)))
+            # Keep the highest value seen before consecutive drops, so a glitch in steps is still recognised
+            pre_drop = state["value"] if state["pre_drop"] is None else max(state["pre_drop"], state["value"])
+            state.update({"value": energy_now, "pre_drop": pre_drop, "drop_time": now})
+            return True, 0.0
+        pre_drop = state["pre_drop"]
+        if pre_drop is not None:
+            if energy_now >= pre_drop - CAR_MANUAL_SOC_ENERGY_MARGIN_KWH:
+                # Back up to (about) where it was before the drop: a glitch recovering, not new energy
+                self.log("Info: Car {} charging energy counter recovered from a drop to {} kWh (was {} kWh), not counting the jump back".format(car_n, dp3(energy_now), dp3(pre_drop)))
+                delta = max(energy_now - pre_drop, 0.0)
+            elif (now - state["drop_time"]).total_seconds() <= CAR_MANUAL_SOC_GLITCH_MINUTES * 60:
+                # Still below the pre-drop value inside the window: maybe a glitch part way back - wait and see
+                return True, 0.0
+            # else the window passed: the drop was a real reset, count the rise from the low value
+        state.update({"value": energy_now, "pre_drop": None, "drop_time": None})
+        if delta > state["budget"]:
+            self.log("Info: Car {} charging energy counter rose by {} kWh, more than the {} kW charger can deliver - counting {} kWh".format(car_n, dp3(delta), rate_kw, dp3(state["budget"])))
+            delta = state["budget"]
+        state["budget"] -= delta
+        return True, delta
+
     def update_car_manual_soc(self):
         """
-        Write the predicted next car SoC back to the manual car SoC tracker for cars using car_charging_manual_soc
+        Advance the manual car SoC tracker for cars using car_charging_manual_soc.
+
+        A measured source wins over the plan. A car_charging_now sensor reporting the car is not charging
+        holds the SoC (an energy counter is then not read, keeping its baseline). Otherwise, with a
+        car_charging_energy counter the SoC advances by the energy actually delivered (less
+        car_charging_loss), and holds while the counter briefly cannot be read (see
+        car_manual_soc_measured_energy()). Only with neither (no sensor, or it is charging, unknown or
+        unavailable, and no usable counter) is the plan's predicted next SoC written back, which advances
+        the SoC whether or not the charger delivers anything.
+
+        A value that differs from the last one written here was edited by the user, so the energy measured
+        since the last write-back is not added on top of the edit (the counter keeps its new baseline).
         """
         for car_n in range(self.num_cars):
             if car_n < len(self.car_charging_manual_soc) and self.car_charging_manual_soc[car_n]:
                 car_postfix = "" if car_n == 0 else "_" + str(car_n)
-                self.log("Car {} charging Manual SoC current is {} next is {}".format(car_n, self.car_charging_soc[car_n], self.car_charging_soc_next[car_n]))
-                if self.car_charging_soc_next[car_n] is not None:
+                soc_now = self.car_charging_soc[car_n]
+                self.log("Car {} charging Manual SoC current is {} next is {}".format(car_n, soc_now, self.car_charging_soc_next[car_n]))
+                charging_now = None
+                if is_entity_id(self.get_arg("car_charging_now", None, indirect=False, index=car_n)):
+                    # Read live as a tri-state: unknown/unavailable is None (no information), not "not charging"
+                    charging_now = self.car_charging_now_value(self.get_arg("car_charging_now", None, index=car_n, required_unit="W"))
+                soc_written = self.car_charging_manual_soc_written.get(car_n)
+                # Explicitly not charging leaves any energy counter unread so its baseline is kept - energy a
+                # lagging car_charging_now sensor hid is picked up by the next reading, within the budget
+                counter_configured, measured_kwh = (False, None) if charging_now is False else self.car_manual_soc_measured_energy(car_n)
+                if charging_now is False:
+                    soc_next = soc_now
+                    self.log("Car {} charging Manual SoC held at {} - car is not charging".format(car_n, dp3(soc_next)))
+                elif counter_configured:
+                    if measured_kwh is None:
+                        self.log("Car {} charging Manual SoC held at {} - charging energy counter is unavailable".format(car_n, dp3(soc_now)))
+                        continue
+                    if soc_written is not None:
+                        # The read clamps to the battery size, so a battery that shrank is not an edit
+                        soc_written = max(0.0, min(soc_written, self.car_charging_battery_size[car_n]))
+                    if soc_written is not None and abs(soc_now - soc_written) > 0.01:
+                        self.log("Car {} charging Manual SoC was changed from {} to {}, not adding {} kWh measured before the change".format(car_n, dp3(soc_written), dp3(soc_now), dp3(measured_kwh)))
+                        measured_kwh = 0.0
+                    soc_next = max(0.0, min(soc_now + measured_kwh * self.car_charging_loss, self.car_charging_battery_size[car_n]))
+                    self.log("Car {} charging Manual SoC advanced by measured energy {} kWh to {}".format(car_n, dp3(measured_kwh), dp3(soc_next)))
+                elif self.car_charging_soc_next[car_n] is not None:
                     soc_next = self.car_charging_soc_next[car_n]
                     # The modelled car SoC can run past the real charge limit when the prediction's fill
                     # clamp is inert (octopus_intelligent_consider_full off, #4967) - the tracked manual
                     # SoC stands in for a measurement, so keep it within the real per-car limit
                     if car_n < len(self.car_charging_limit):
                         soc_next = min(soc_next, self.car_charging_limit[car_n])
-                    self.expose_config("car_charging_manual_soc_kwh" + car_postfix, dp3(soc_next))
+                else:
+                    continue
+                self.car_charging_manual_soc_written[car_n] = dp3(soc_next)
+                self.expose_config("car_charging_manual_soc_kwh" + car_postfix, dp3(soc_next))
 
     def run_comparison(self, status=None, status_extra=""):
         """