fix: serialize epoch datetimes on Windows

requirements.txt

@@ -1,5 +1,8 @@
 # make sure content of this file can be parsed by setup.load_requirements
 python-dateutil>=2.7
+# resolves the local IANA timezone name so pre-epoch datetimes use real
+# historical UTC offsets on Windows, which only stores the current DST rule
+tzlocal>=4.0
 urllib3
 certifi
 setuptools>=75.1.0

tests/test_api_client/test_deserializer.py

@@ -207,6 +207,9 @@ def test_deserialize_date_error(data):
         ("/Date(315619200000+0000)/", date(1980, 1, 2)),
         ("/Date(1550899400362)/", date(2019, 2, 23)),
         ("/Date(1550899400362+1300)/", date(2019, 2, 23)),
+        # Pre-epoch timestamps are outside the platform range Windows accepts.
+        ("/Date(-2208988800000)/", date(1900, 1, 1)),
+        ("/Date(-315619200000+0000)/", date(1960, 1, 1)),
     ],
 )
 def test_deserialize_date_ms(data, expected):
@@ -306,6 +309,12 @@ def test_deserialize_datetime_error(data):
                 tzinfo=tz.tzoffset(None, timedelta(hours=13)),
             ),
         ),
+        # Pre-epoch timestamps are outside the platform range Windows accepts.
+        ("/Date(-2208988800000)/", datetime(1900, 1, 1, tzinfo=tz.UTC)),
+        (
+            "/Date(-2150881754232+0000)/",
+            datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC),
+        ),
     ],
 )
 def test_deserialize_datetime_ms(data, expected):

tests/test_api_client/test_serializer.py

@@ -7,8 +7,11 @@
 import pytest
 from dateutil import tz
 
+from xero_python.api_client.deserializer import deserialize
 from xero_python.api_client.serializer import (
     data_type,
+    local_timezone,
+    naive_to_utc,
     serialize,
     serialize_routing,
     serialize_dict,
@@ -306,6 +309,7 @@ def test_serialize_datetime(value, expected):
     "value,expected",
     [
         (datetime.fromtimestamp(0.0), "/Date(0)/"),
+        (datetime(1960, 1, 1, tzinfo=tz.UTC), "/Date(-315619200000+0000)/"),
         (datetime.fromtimestamp(1439424000.0), "/Date(1439424000000)/"),
         (datetime.fromtimestamp(1439434356.790), "/Date(1439434356790)/"),
         (datetime(2015, 8, 13, tzinfo=tz.UTC), "/Date(1439424000000+0000)/"),
@@ -337,6 +341,98 @@ def test_serialize_datetime_ms(value, expected):
     assert result == expected
 
 
+@pytest.mark.parametrize(
+    "value,expected",
+    [
+        # Sydney stayed on UTC+10 all year round until 1971, so noon-thirty
+        # local is 02:30 UTC. Applying today's daylight saving rule instead
+        # would place it an hour earlier, at 01:30 UTC.
+        (datetime(1960, 1, 1, 12, 30), datetime(1960, 1, 1, 2, 30, tzinfo=tz.UTC)),
+        # By 1990 daylight saving was in force each January, so UTC+11 applies.
+        (datetime(1990, 1, 1, 12, 30), datetime(1990, 1, 1, 1, 30, tzinfo=tz.UTC)),
+    ],
+)
+def test_naive_to_utc_uses_historical_offsets_of_a_fixed_timezone(
+    monkeypatch, value, expected
+):
+    monkeypatch.setattr(
+        "xero_python.api_client.serializer.local_timezone",
+        lambda: tz.gettz("Australia/Sydney"),
+    )
+
+    assert naive_to_utc(value) == expected
+
+
+def test_local_timezone_agrees_with_the_tz_database_before_the_epoch():
+    zone_name = pytest.importorskip("tzlocal").get_localzone_name()
+    historical_zone = tz.gettz(zone_name)
+    if historical_zone is None:
+        pytest.skip("no tz database entry for {}".format(zone_name))
+    value = datetime(1960, 1, 1, 12, 30)
+
+    assert (
+        value.replace(tzinfo=local_timezone()).utcoffset()
+        == value.replace(tzinfo=historical_zone).utcoffset()
+    )
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        datetime(1971, 1, 1, 12, 30),
+        datetime(2015, 8, 13, 12, 30),
+        datetime(2024, 4, 7, 2, 30, fold=0),
+        datetime(2024, 4, 7, 2, 30, fold=1),
+        datetime(2024, 10, 6, 2, 30, fold=0),
+        datetime(2024, 10, 6, 2, 30, fold=1),
+    ],
+)
+def test_serialize_naive_datetime_ms_preserves_platform_timestamp_semantics(value):
+    expected_ms = int(value.timestamp() * 1000)
+
+    assert serialize_datetime_ms(value) == "/Date({})/".format(expected_ms)
+
+
+@pytest.mark.parametrize(
+    "value,expected",
+    [
+        # Float seconds truncate towards zero, so these lose the final
+        # millisecond and the error changes sign either side of the epoch.
+        (
+            datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC),
+            "/Date(-2150881754232+0000)/",
+        ),
+        (
+            datetime(1935, 5, 6, 15, 50, 10, 433000, tzinfo=tz.UTC),
+            "/Date(-1093680589567+0000)/",
+        ),
+        (
+            datetime(2004, 5, 29, 20, 46, 5, 715000, tzinfo=tz.UTC),
+            "/Date(1085863565715+0000)/",
+        ),
+    ],
+)
+def test_serialize_datetime_ms_keeps_exact_milliseconds(value, expected):
+    assert serialize_datetime_ms(value) == expected
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        datetime(1900, 1, 1, tzinfo=tz.UTC),
+        datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC),
+        datetime(1960, 1, 1, tzinfo=tz.UTC),
+        datetime(1970, 1, 1, tzinfo=tz.UTC),
+        datetime(2004, 5, 29, 20, 46, 5, 715000, tzinfo=tz.UTC),
+        datetime(2016, 10, 13, 20, 13, 36, 437000, tzinfo=tz.UTC),
+    ],
+)
+def test_datetime_ms_round_trips_through_deserialize(value):
+    assert (
+        deserialize("datetime[ms-format]", serialize_datetime_ms(value), None) == value
+    )
+
+
 # serialize_date_ms tests
 @pytest.mark.parametrize(
     "value,expected",
@@ -356,6 +452,15 @@ def test_serialize_date_ms(value, expected):
     assert result == expected
 
 
+def test_serialize_pre_epoch_date_ms_uses_utc_midnight():
+    value = date(1960, 1, 1)
+    utc_value = datetime.combine(value, datetime.min.time()).replace(tzinfo=tz.UTC)
+    epoch = datetime(1970, 1, 1, tzinfo=tz.UTC)
+    expected_ms = int((utc_value - epoch).total_seconds() * 1000)
+
+    assert serialize_date_ms(value) == "/Date({})/".format(expected_ms)
+
+
 # serialize_base_model tests
 def test_serialize_base_model():
     # given test model

xero_python/api_client/deserializer.py

@@ -24,6 +24,8 @@
 MS_DATETIME_RE = re.compile(r"/Date\((?P<timestamp>-?\d+)(?P<tzinfo>[+-]\d{2,4})?\)/$")
 DATE_WITH_NO_DAY_RE = re.compile(r"(\d\d\d\d)-(\d\d)")
 
+UNIX_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=tz.UTC)
+
 
 def deserialize_routing(data_type, data, model_finder):
     """Custom logic to find matching deserialize implementation and
@@ -250,8 +252,11 @@ def deserialize_datetime_ms(data_type, data, model_finder):
             tz_info = tz.UTC
 
         timestamp_ms = int(match.groupdict()["timestamp"])
-        timestamp_s = timestamp_ms / 1000
-        return datetime.datetime.fromtimestamp(timestamp_s, tz=tz_info)
+        # Offsetting the epoch keeps the arithmetic exact and works for dates
+        # outside the platform timestamp range, which Windows rejects.
+        return (UNIX_EPOCH + datetime.timedelta(milliseconds=timestamp_ms)).astimezone(
+            tz_info
+        )
     elif DATE_WITH_NO_DAY_RE.match(str(data)):
         return datetime.datetime.strptime(data + "-01", "%Y-%m-%d")
     else:

xero_python/api_client/serializer.py

@@ -12,6 +12,58 @@
 DICT_DATA_TYPE = re.compile(r"^dict(?:\[(.*)\])?$")
 LIST_DATA_TYPE = re.compile(r"^list(?:\[(.*)\])?$")
 TUPLE_DATA_TYPE = re.compile(r"^tuple(?:\[(.*)\])?$")
+UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=tz.UTC)
+
+
+def local_timezone():
+    """Return the local timezone including its historical UTC offsets.
+
+    Windows only records the currently active DST rule, so tz.tzwinlocal()
+    applies today's rule to every historical date and shifts pre-1971 values
+    by an hour. Resolve the IANA name for the machine instead and read the
+    real transitions from the tz database bundled with python-dateutil.
+    """
+    try:
+        from tzlocal import get_localzone_name
+
+        zone_name = get_localzone_name()
+    except Exception:
+        zone_name = None
+    if zone_name:
+        zone = tz.gettz(zone_name)
+        if zone is not None:
+            return zone
+    return tz.tzlocal()
+
+
+def naive_to_utc(value):
+    """Convert a naive datetime to UTC using the local historical offset."""
+    return value.replace(tzinfo=local_timezone()).astimezone(tz.UTC)
+
+
+def datetime_to_utc(value):
+    """Return value as an aware UTC datetime without platform range limits."""
+    if value.tzinfo is None:
+        try:
+            # Preserve the platform's existing naive-local DST and fold
+            # semantics for every date it is able to represent.
+            return value.astimezone(tz.UTC)
+        except (OSError, OverflowError, ValueError):
+            return naive_to_utc(value)
+    return value.astimezone(tz.UTC)
+
+
+def datetime_timestamp_ms(value):
+    """Return whole milliseconds from the Unix epoch.
+
+    Uses integer arithmetic throughout. Going via float seconds truncates
+    towards zero, which loses a millisecond on values that are not exactly
+    representable and flips the rounding direction either side of the epoch.
+    """
+    elapsed = datetime_to_utc(value) - UNIX_EPOCH
+    return (
+        elapsed.days * 86400000 + elapsed.seconds * 1000 + elapsed.microseconds // 1000
+    )
 
 
 def data_type(value, explicit_type=None):
@@ -159,8 +211,7 @@ def serialize_datetime_ms(value, explicit_type=None):
     :return: serialized object
     """
     tz_str = value.strftime("%z")
-    timestamp_s = value.timestamp()
-    timestamp_ms = int(timestamp_s * 1000)
+    timestamp_ms = datetime_timestamp_ms(value)
     return "/Date({}{})/".format(timestamp_ms, tz_str)
 
 
@@ -180,8 +231,7 @@ def serialize_date_ms(value, explicit_type=None):
     else:
         raise ValueError("Can't serialize {!r} into Microsoft date json format")
 
-    timestamp_s = datetime_value.timestamp()
-    timestamp_ms = int(timestamp_s * 1000)
+    timestamp_ms = datetime_timestamp_ms(datetime_value)
     return "/Date({})/".format(timestamp_ms)