fix(parsing): apply the UTC offset of trajectory timestamps

docs/internal/pyneats_units_audit.md

@@ -4,6 +4,15 @@ Reviewed revision: `f7502fe6344e5e085df1042ac817f02a05bd100d` (`origin/main`, 20
 on branch `docs/units-audit`. Every `file:line` reference below points to that revision,
 before the comment patch was applied. The patch moves lines in the files it touches.
 
+## Follow-up status
+
+The findings below are the audit as written. Each fix updates this table and adds a status
+note under its item. Items not listed here are open.
+
+| Item | Status | Branch / PR | Date |
+| --- | --- | --- | --- |
+| D2 (trajectory `ts` offsets) | Fixed | `fix/d2-timestamp-utc` | 2026-10-20 |
+
 ## 1. Outcome
 
 - **Two known interface differences.** `tas` (knots in the AO JSON, m/s in PyNEATS) and
@@ -447,6 +456,13 @@ Each entry gives:
    document that the array form is required.
 
 **D2. Trajectory timestamp offsets discarded; some valid forms rejected.**
+*Fixed on branch `fix/d2-timestamp-utc` (see the follow-up status table). The UTC offset
+is now applied. Fractional seconds are truncated. A lowercase `t` or `z` and a space
+separator are accepted. A `ts` without offset is read as UTC (maintainer decision). An
+invalid `ts` still aborts the batch, now with an error naming the flight and point; isolating
+it per flight is a separate change. No effect on existing data: on 148 NEATS JSON files on
+the NEATS server (85,128 flights, 10,995,586 points, all `+0000`, `Z` or `+00:00`) the output
+is identical.*
 1. `ts` → `neats_io.py:61-68` → `neats_parser.py:233-242`.
 2. Expected: every RFC 3339 `date-time` maps to the same instant in UTC. Observed:
    - `strptime('%Y-%m-%dT%H:%M:%S%z')` followed by `strftime('%Y-%m-%d %H:%M:%S')` **drops**

src/pyneats/steps/parsing/neats_io.py

@@ -6,8 +6,10 @@
 
 from __future__ import annotations
 
+import re
 from collections.abc import Mapping
 from collections.abc import Sequence
+from datetime import UTC
 from datetime import datetime
 from typing import Any
 
@@ -16,6 +18,53 @@
 from pyneats.steps.parsing.views import Flight4D
 
 
+# Fractional seconds of the seconds field (".5" in "2025-11-02T10:02:00.5Z").
+_FRACTIONAL_SECONDS_RE = re.compile(
+    r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\.\d+", re.ASCII
+)
+
+
+def _ts_to_utc_string(ts_raw: str) -> str:
+    """Convert an AO trajectory ``ts`` to a naive UTC ``"%Y-%m-%d %H:%M:%S"`` string.
+
+    ``ts`` is an RFC 3339 date-time (AO schema ``"format": "date-time"``).
+    The UTC offset is applied, so ``"2025-11-02T12:02:00+02:00"`` becomes
+    ``"2025-11-02 10:02:00"``. A ``ts`` without offset is read as UTC, the
+    time scale the AO schema prescribes. Also accepted: a lowercase ``t`` or
+    ``z`` and a space instead of ``T``. Fractional seconds are truncated to
+    whole seconds. Leap seconds (``:60``) are not supported. UTC values
+    (``Z``, ``+0000``, ``+00:00``) give the same string as before this
+    function existed.
+
+    Raises
+    ------
+    ValueError
+        If ``ts`` is not a string or not a valid date-time, or if the UTC
+        result falls outside years 1 to 9999.
+    """
+    if not isinstance(ts_raw, str):
+        raise ValueError(f"ts {ts_raw!r} is not a string")
+    s = ts_raw.upper()
+    if len(s) > 10 and s[10] == " ":
+        s = s[:10] + "T" + s[11:]
+    s = _FRACTIONAL_SECONDS_RE.sub(r"\1", s, count=1)
+    try:
+        dt = datetime.strptime(s, "%Y-%m-%dT%H:%M:%S%z")
+    except ValueError as err:
+        try:
+            dt = datetime.strptime(s, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC)
+        except ValueError:
+            raise ValueError(
+                f"ts {ts_raw!r} is not a valid RFC 3339 date-time ({err})"
+            ) from None
+    try:
+        return dt.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S")
+    except OverflowError:
+        raise ValueError(
+            f"ts {ts_raw!r} falls outside years 1 to 9999 in UTC"
+        ) from None
+
+
 def neats_json_to_flights(
     flights: Sequence[Mapping[str, Any]],
     *,
@@ -53,14 +102,15 @@ def neats_json_to_flights(
 
     Per-point columns, from ``flight_information.trajectory.trajectory_data[*]``:
 
-    - ``ts`` -> ``time``: string ``"%Y-%m-%d %H:%M:%S"``. Parsed with
-      ``"%Y-%m-%dT%H:%M:%S%z"``; the UTC offset is parsed but discarded, not
-      applied, so a ``+02:00`` value keeps its local clock time and is later
-      read as UTC by ``NeatsTrajectoryParser`` (``utc=True``). Values must
-      already be UTC (``Z``, ``+0000`` or ``+00:00``), as the AO schema
-      requires. Fractional seconds, a lowercase ``z`` or a missing offset
-      raise ``ValueError`` (see docs/internal/pyneats_units_audit.md). Point
-      spacing is not assumed (the interpolation step resamples).
+    - ``ts`` -> ``time``: UTC string ``"%Y-%m-%d %H:%M:%S"``, read as UTC
+      by ``NeatsTrajectoryParser``. ``ts`` is an RFC 3339 date-time and its
+      UTC offset is applied (``"2025-11-02T12:02:00+02:00"`` ->
+      ``"2025-11-02 10:02:00"``); a ``ts`` without offset is read as UTC. A
+      lowercase ``t``/``z`` and a space separator are accepted; fractional
+      seconds are truncated to whole seconds, and points that then share a
+      second are reduced to one by the parser. An invalid ``ts`` raises
+      ``ValueError`` naming the flight and point. Point spacing is not
+      assumed (the interpolation step resamples).
     - ``lat`` / ``lon`` -> ``latitude`` / ``longitude``: decimal degrees
       (WGS 84), unchanged.
     - ``fl`` -> ``altitude``: still flight level (hundreds of ft).
@@ -134,17 +184,21 @@ def neats_json_to_flights(
         tr = fi.get("trajectory", {}) or {}
         pts = tr.get("trajectory_data", []) or []
 
-        for p in pts:
+        for i, p in enumerate(pts):
             ts_raw = p.get("ts")
 
             if ts_raw is None:
                 time_over = None
             else:
-                # "2025-11-02T10:02:00+0000" → "2025-11-02 10:02:00"
-                # The UTC offset is parsed but discarded (not applied); the
-                # parser reads the result as UTC, so ts must already be UTC.
-                dt = datetime.strptime(ts_raw, "%Y-%m-%dT%H:%M:%S%z")
-                time_over = dt.strftime("%Y-%m-%d %H:%M:%S")
+                # "2025-11-02T12:02:00+02:00" → "2025-11-02 10:02:00" (UTC);
+                # the parser reads the result as UTC.
+                try:
+                    time_over = _ts_to_utc_string(ts_raw)
+                except ValueError as exc:
+                    raise ValueError(
+                        f"flight {fi.get('flight_identification')!r}, "
+                        f"trajectory point {i}: {exc}"
+                    ) from exc
 
             rows.append(
                 {

tests/test_neats_io_timestamps.py

@@ -0,0 +1,182 @@
+"""Trajectory ``ts`` parsing in ``neats_json_to_flights`` (units audit item D2).
+
+``ts`` is an RFC 3339 date-time. Its UTC offset must be applied, not
+dropped, and every schema-valid form must be accepted. These tests need
+neither BADA nor weather data.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+import pytest
+
+from pyneats.core.steps_registry import build
+from pyneats.steps.parsing import TrajectoryParser
+from pyneats.steps.parsing.neats_io import _ts_to_utc_string
+from pyneats.steps.parsing.neats_io import neats_json_to_flights
+
+
+def _flight(ts_values: list[str | None], flight_id: str = "TEST01") -> dict:
+    """Minimal NEATS JSON flight at cruise level, one point per ``ts``."""
+    return {
+        "flight_information": {
+            "flight_identification": flight_id,
+            "departure_airport": "LFPG",
+            "arrival_airport": "EDDF",
+            "departure_date_time": "2025-11-02T10:00:00+0000",
+            "aircraft_properties": {"aircraft_type": "A320"},
+            "trajectory": {
+                "trajectory_data": [
+                    {"ts": ts, "lat": 49.0 + 0.1 * k, "lon": 2.5 + 0.1 * k, "fl": 350}
+                    for k, ts in enumerate(ts_values)
+                ]
+            },
+        }
+    }
+
+
+# ---------------------------------------------------------------------------
+# _ts_to_utc_string
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "ts",
+    [
+        "2025-11-02T10:02:00+0000",
+        "2025-11-02T10:02:00Z",
+        "2025-11-02T10:02:00+00:00",
+        "2025-11-02T10:02:00-00:00",
+        "2025-11-02t10:02:00z",
+        "2025-11-02 10:02:00Z",
+    ],
+)
+def test_utc_forms_give_the_same_string(ts: str) -> None:
+    assert _ts_to_utc_string(ts) == "2025-11-02 10:02:00"
+
+
+@pytest.mark.parametrize(
+    ("ts", "expected"),
+    [
+        ("2025-11-02T12:02:00+02:00", "2025-11-02 10:02:00"),
+        ("2025-11-02T12:02:00+0200", "2025-11-02 10:02:00"),
+        ("2025-11-02T04:32:00-05:30", "2025-11-02 10:02:00"),
+        # Offsets that cross midnight, a month end and a year end.
+        ("2025-11-02T01:00:00+02:00", "2025-11-01 23:00:00"),
+        ("2025-10-31T23:30:00-01:00", "2025-11-01 00:30:00"),
+        ("2026-01-01T00:15:00+01:00", "2025-12-31 23:15:00"),
+    ],
+)
+def test_offset_is_applied(ts: str, expected: str) -> None:
+    assert _ts_to_utc_string(ts) == expected
+
+
+@pytest.mark.parametrize(
+    "ts",
+    [
+        "2025-11-02T10:02:00.5Z",
+        "2025-11-02T10:02:00.999999+0000",
+        "2025-11-02T10:02:00.1234567890+00:00",
+    ],
+)
+def test_fractional_seconds_are_truncated(ts: str) -> None:
+    assert _ts_to_utc_string(ts) == "2025-11-02 10:02:00"
+
+
+@pytest.mark.parametrize(
+    ("ts", "expected"),
+    [
+        ("2025-11-02T10:02:00", "2025-11-02 10:02:00"),
+        ("2025-11-02 10:02:00.5", "2025-11-02 10:02:00"),
+    ],
+)
+def test_missing_offset_is_read_as_utc(ts: str, expected: str) -> None:
+    assert _ts_to_utc_string(ts) == expected
+
+
+@pytest.mark.parametrize(
+    ("ts", "expected"),
+    [
+        ("2025-11-02T12:02:00.5+02:00", "2025-11-02 10:02:00"),
+        ("2025-11-02 12:02:00+02:00", "2025-11-02 10:02:00"),
+        # Only the seconds field loses its fraction: an offset with seconds
+        # and a fraction is applied in full, then truncated (09:01:29.5).
+        ("2025-11-02T10:02:00+01:00:30.5", "2025-11-02 09:01:29"),
+    ],
+)
+def test_fraction_and_offset_combined(ts: str, expected: str) -> None:
+    assert _ts_to_utc_string(ts) == expected
+
+
+@pytest.mark.parametrize(
+    "ts",
+    [
+        "garbage",
+        "",
+        "2025-13-02T10:02:00Z",
+        "2025-11-02T25:02:00Z",
+        "2025-02-29T10:02:00Z",
+        "2025-07-09T23:59:60Z",  # leap second: not supported
+        "2025-11-02T10:02:00." + chr(0xFF15) + "Z",  # non-ASCII digit in fraction
+    ],
+)
+def test_invalid_date_time_raises(ts: str) -> None:
+    with pytest.raises(ValueError, match="not a valid RFC 3339 date-time"):
+        _ts_to_utc_string(ts)
+
+
+@pytest.mark.parametrize("ts", [1752055200, 1.5, b"2025-11-02T10:02:00Z"])
+def test_non_string_raises(ts: object) -> None:
+    with pytest.raises(ValueError, match="is not a string"):
+        _ts_to_utc_string(ts)  # type: ignore[arg-type]
+
+
+@pytest.mark.parametrize(
+    "ts", ["9999-12-31T23:59:59-01:00", "0001-01-01T00:00:00+00:01"]
+)
+def test_out_of_range_after_conversion_raises(ts: str) -> None:
+    with pytest.raises(ValueError, match="outside years 1 to 9999"):
+        _ts_to_utc_string(ts)
+
+
+# ---------------------------------------------------------------------------
+# neats_json_to_flights
+# ---------------------------------------------------------------------------
+
+
+def test_json_offsets_reach_the_time_column() -> None:
+    ts_values = [
+        "2025-11-02T12:00:00+02:00",
+        "2025-11-02T10:01:00Z",
+        "2025-11-02T10:02:00",
+        None,
+    ]
+    (df,) = neats_json_to_flights([_flight(ts_values)])
+    assert df["time"].tolist() == [
+        "2025-11-02 10:00:00",
+        "2025-11-02 10:01:00",
+        "2025-11-02 10:02:00",
+        None,
+    ]
+
+
+@pytest.mark.parametrize("bad", ["garbage", 1752055200])
+def test_invalid_ts_error_names_flight_and_point(bad: object) -> None:
+    ts_values = ["2025-11-02T10:00:00Z", bad]
+    with pytest.raises(ValueError, match=r"flight 'BAD01', trajectory point 1: "):
+        neats_json_to_flights([_flight(ts_values, "BAD01")])  # type: ignore[arg-type]
+
+
+def test_offset_and_utc_inputs_parse_to_identical_times() -> None:
+    """End to end through NeatsTrajectoryParser: a +02:00 trajectory and the
+    same instants written in UTC give identical parsed times."""
+    utc = [f"2025-11-02T10:{m:02d}:00Z" for m in range(0, 30, 5)]
+    local = [f"2025-11-02T12:{m:02d}:00+02:00" for m in range(0, 30, 5)]
+    (df_utc, df_local) = neats_json_to_flights(
+        [_flight(utc, "UTC01"), _flight(local, "LOC01")]
+    )
+    parser = build(TrajectoryParser, "neats")
+    t_utc = pd.Series(parser(df_utc)["time"])
+    t_local = pd.Series(parser(df_local)["time"])
+    assert t_utc.tolist() == t_local.tolist()
+    assert t_utc.iloc[0] == pd.Timestamp("2025-11-02 10:00:00")