diff --git a/docs/internal/pyneats_units_audit.md b/docs/internal/pyneats_units_audit.md index 9896016..bf5e735 100644 --- a/docs/internal/pyneats_units_audit.md +++ b/docs/internal/pyneats_units_audit.md @@ -4,6 +4,15 @@ Reviewed revision: `f7502fe6344e5e085df1042ac817f02a05bd100d` (`origin/main`, 20 on branch `docs/units-audit`. Every `file:line` reference below points to that revision, before the comment patch was applied. The patch moves lines in the files it touches. +## Follow-up status + +The findings below are the audit as written. Each fix updates this table and adds a status +note under its item. Items not listed here are open. + +| Item | Status | Branch / PR | Date | +| --- | --- | --- | --- | +| D2 (trajectory `ts` offsets) | Fixed | `fix/d2-timestamp-utc` | 2026-10-20 | + ## 1. Outcome - **Two known interface differences.** `tas` (knots in the AO JSON, m/s in PyNEATS) and @@ -447,6 +456,13 @@ Each entry gives: document that the array form is required. **D2. Trajectory timestamp offsets discarded; some valid forms rejected.** +*Fixed on branch `fix/d2-timestamp-utc` (see the follow-up status table). The UTC offset +is now applied. Fractional seconds are truncated. A lowercase `t` or `z` and a space +separator are accepted. A `ts` without offset is read as UTC (maintainer decision). An +invalid `ts` still aborts the batch, now with an error naming the flight and point; isolating +it per flight is a separate change. No effect on existing data: on 148 NEATS JSON files on +the NEATS server (85,128 flights, 10,995,586 points, all `+0000`, `Z` or `+00:00`) the output +is identical.* 1. `ts` → `neats_io.py:61-68` → `neats_parser.py:233-242`. 2. Expected: every RFC 3339 `date-time` maps to the same instant in UTC. Observed: - `strptime('%Y-%m-%dT%H:%M:%S%z')` followed by `strftime('%Y-%m-%d %H:%M:%S')` **drops** diff --git a/src/pyneats/steps/parsing/neats_io.py b/src/pyneats/steps/parsing/neats_io.py index 6ddbdad..97b8552 100644 --- a/src/pyneats/steps/parsing/neats_io.py +++ b/src/pyneats/steps/parsing/neats_io.py @@ -6,8 +6,10 @@ from __future__ import annotations +import re from collections.abc import Mapping from collections.abc import Sequence +from datetime import UTC from datetime import datetime from typing import Any @@ -16,6 +18,53 @@ from pyneats.steps.parsing.views import Flight4D +# Fractional seconds of the seconds field (".5" in "2025-11-02T10:02:00.5Z"). +_FRACTIONAL_SECONDS_RE = re.compile( + r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\.\d+", re.ASCII +) + + +def _ts_to_utc_string(ts_raw: str) -> str: + """Convert an AO trajectory ``ts`` to a naive UTC ``"%Y-%m-%d %H:%M:%S"`` string. + + ``ts`` is an RFC 3339 date-time (AO schema ``"format": "date-time"``). + The UTC offset is applied, so ``"2025-11-02T12:02:00+02:00"`` becomes + ``"2025-11-02 10:02:00"``. A ``ts`` without offset is read as UTC, the + time scale the AO schema prescribes. Also accepted: a lowercase ``t`` or + ``z`` and a space instead of ``T``. Fractional seconds are truncated to + whole seconds. Leap seconds (``:60``) are not supported. UTC values + (``Z``, ``+0000``, ``+00:00``) give the same string as before this + function existed. + + Raises + ------ + ValueError + If ``ts`` is not a string or not a valid date-time, or if the UTC + result falls outside years 1 to 9999. + """ + if not isinstance(ts_raw, str): + raise ValueError(f"ts {ts_raw!r} is not a string") + s = ts_raw.upper() + if len(s) > 10 and s[10] == " ": + s = s[:10] + "T" + s[11:] + s = _FRACTIONAL_SECONDS_RE.sub(r"\1", s, count=1) + try: + dt = datetime.strptime(s, "%Y-%m-%dT%H:%M:%S%z") + except ValueError as err: + try: + dt = datetime.strptime(s, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + except ValueError: + raise ValueError( + f"ts {ts_raw!r} is not a valid RFC 3339 date-time ({err})" + ) from None + try: + return dt.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S") + except OverflowError: + raise ValueError( + f"ts {ts_raw!r} falls outside years 1 to 9999 in UTC" + ) from None + + def neats_json_to_flights( flights: Sequence[Mapping[str, Any]], *, @@ -53,14 +102,15 @@ def neats_json_to_flights( Per-point columns, from ``flight_information.trajectory.trajectory_data[*]``: - - ``ts`` -> ``time``: string ``"%Y-%m-%d %H:%M:%S"``. Parsed with - ``"%Y-%m-%dT%H:%M:%S%z"``; the UTC offset is parsed but discarded, not - applied, so a ``+02:00`` value keeps its local clock time and is later - read as UTC by ``NeatsTrajectoryParser`` (``utc=True``). Values must - already be UTC (``Z``, ``+0000`` or ``+00:00``), as the AO schema - requires. Fractional seconds, a lowercase ``z`` or a missing offset - raise ``ValueError`` (see docs/internal/pyneats_units_audit.md). Point - spacing is not assumed (the interpolation step resamples). + - ``ts`` -> ``time``: UTC string ``"%Y-%m-%d %H:%M:%S"``, read as UTC + by ``NeatsTrajectoryParser``. ``ts`` is an RFC 3339 date-time and its + UTC offset is applied (``"2025-11-02T12:02:00+02:00"`` -> + ``"2025-11-02 10:02:00"``); a ``ts`` without offset is read as UTC. A + lowercase ``t``/``z`` and a space separator are accepted; fractional + seconds are truncated to whole seconds, and points that then share a + second are reduced to one by the parser. An invalid ``ts`` raises + ``ValueError`` naming the flight and point. Point spacing is not + assumed (the interpolation step resamples). - ``lat`` / ``lon`` -> ``latitude`` / ``longitude``: decimal degrees (WGS 84), unchanged. - ``fl`` -> ``altitude``: still flight level (hundreds of ft). @@ -134,17 +184,21 @@ def neats_json_to_flights( tr = fi.get("trajectory", {}) or {} pts = tr.get("trajectory_data", []) or [] - for p in pts: + for i, p in enumerate(pts): ts_raw = p.get("ts") if ts_raw is None: time_over = None else: - # "2025-11-02T10:02:00+0000" → "2025-11-02 10:02:00" - # The UTC offset is parsed but discarded (not applied); the - # parser reads the result as UTC, so ts must already be UTC. - dt = datetime.strptime(ts_raw, "%Y-%m-%dT%H:%M:%S%z") - time_over = dt.strftime("%Y-%m-%d %H:%M:%S") + # "2025-11-02T12:02:00+02:00" → "2025-11-02 10:02:00" (UTC); + # the parser reads the result as UTC. + try: + time_over = _ts_to_utc_string(ts_raw) + except ValueError as exc: + raise ValueError( + f"flight {fi.get('flight_identification')!r}, " + f"trajectory point {i}: {exc}" + ) from exc rows.append( { diff --git a/tests/test_neats_io_timestamps.py b/tests/test_neats_io_timestamps.py new file mode 100644 index 0000000..00c351f --- /dev/null +++ b/tests/test_neats_io_timestamps.py @@ -0,0 +1,182 @@ +"""Trajectory ``ts`` parsing in ``neats_json_to_flights`` (units audit item D2). + +``ts`` is an RFC 3339 date-time. Its UTC offset must be applied, not +dropped, and every schema-valid form must be accepted. These tests need +neither BADA nor weather data. +""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from pyneats.core.steps_registry import build +from pyneats.steps.parsing import TrajectoryParser +from pyneats.steps.parsing.neats_io import _ts_to_utc_string +from pyneats.steps.parsing.neats_io import neats_json_to_flights + + +def _flight(ts_values: list[str | None], flight_id: str = "TEST01") -> dict: + """Minimal NEATS JSON flight at cruise level, one point per ``ts``.""" + return { + "flight_information": { + "flight_identification": flight_id, + "departure_airport": "LFPG", + "arrival_airport": "EDDF", + "departure_date_time": "2025-11-02T10:00:00+0000", + "aircraft_properties": {"aircraft_type": "A320"}, + "trajectory": { + "trajectory_data": [ + {"ts": ts, "lat": 49.0 + 0.1 * k, "lon": 2.5 + 0.1 * k, "fl": 350} + for k, ts in enumerate(ts_values) + ] + }, + } + } + + +# --------------------------------------------------------------------------- +# _ts_to_utc_string +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ts", + [ + "2025-11-02T10:02:00+0000", + "2025-11-02T10:02:00Z", + "2025-11-02T10:02:00+00:00", + "2025-11-02T10:02:00-00:00", + "2025-11-02t10:02:00z", + "2025-11-02 10:02:00Z", + ], +) +def test_utc_forms_give_the_same_string(ts: str) -> None: + assert _ts_to_utc_string(ts) == "2025-11-02 10:02:00" + + +@pytest.mark.parametrize( + ("ts", "expected"), + [ + ("2025-11-02T12:02:00+02:00", "2025-11-02 10:02:00"), + ("2025-11-02T12:02:00+0200", "2025-11-02 10:02:00"), + ("2025-11-02T04:32:00-05:30", "2025-11-02 10:02:00"), + # Offsets that cross midnight, a month end and a year end. + ("2025-11-02T01:00:00+02:00", "2025-11-01 23:00:00"), + ("2025-10-31T23:30:00-01:00", "2025-11-01 00:30:00"), + ("2026-01-01T00:15:00+01:00", "2025-12-31 23:15:00"), + ], +) +def test_offset_is_applied(ts: str, expected: str) -> None: + assert _ts_to_utc_string(ts) == expected + + +@pytest.mark.parametrize( + "ts", + [ + "2025-11-02T10:02:00.5Z", + "2025-11-02T10:02:00.999999+0000", + "2025-11-02T10:02:00.1234567890+00:00", + ], +) +def test_fractional_seconds_are_truncated(ts: str) -> None: + assert _ts_to_utc_string(ts) == "2025-11-02 10:02:00" + + +@pytest.mark.parametrize( + ("ts", "expected"), + [ + ("2025-11-02T10:02:00", "2025-11-02 10:02:00"), + ("2025-11-02 10:02:00.5", "2025-11-02 10:02:00"), + ], +) +def test_missing_offset_is_read_as_utc(ts: str, expected: str) -> None: + assert _ts_to_utc_string(ts) == expected + + +@pytest.mark.parametrize( + ("ts", "expected"), + [ + ("2025-11-02T12:02:00.5+02:00", "2025-11-02 10:02:00"), + ("2025-11-02 12:02:00+02:00", "2025-11-02 10:02:00"), + # Only the seconds field loses its fraction: an offset with seconds + # and a fraction is applied in full, then truncated (09:01:29.5). + ("2025-11-02T10:02:00+01:00:30.5", "2025-11-02 09:01:29"), + ], +) +def test_fraction_and_offset_combined(ts: str, expected: str) -> None: + assert _ts_to_utc_string(ts) == expected + + +@pytest.mark.parametrize( + "ts", + [ + "garbage", + "", + "2025-13-02T10:02:00Z", + "2025-11-02T25:02:00Z", + "2025-02-29T10:02:00Z", + "2025-07-09T23:59:60Z", # leap second: not supported + "2025-11-02T10:02:00." + chr(0xFF15) + "Z", # non-ASCII digit in fraction + ], +) +def test_invalid_date_time_raises(ts: str) -> None: + with pytest.raises(ValueError, match="not a valid RFC 3339 date-time"): + _ts_to_utc_string(ts) + + +@pytest.mark.parametrize("ts", [1752055200, 1.5, b"2025-11-02T10:02:00Z"]) +def test_non_string_raises(ts: object) -> None: + with pytest.raises(ValueError, match="is not a string"): + _ts_to_utc_string(ts) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "ts", ["9999-12-31T23:59:59-01:00", "0001-01-01T00:00:00+00:01"] +) +def test_out_of_range_after_conversion_raises(ts: str) -> None: + with pytest.raises(ValueError, match="outside years 1 to 9999"): + _ts_to_utc_string(ts) + + +# --------------------------------------------------------------------------- +# neats_json_to_flights +# --------------------------------------------------------------------------- + + +def test_json_offsets_reach_the_time_column() -> None: + ts_values = [ + "2025-11-02T12:00:00+02:00", + "2025-11-02T10:01:00Z", + "2025-11-02T10:02:00", + None, + ] + (df,) = neats_json_to_flights([_flight(ts_values)]) + assert df["time"].tolist() == [ + "2025-11-02 10:00:00", + "2025-11-02 10:01:00", + "2025-11-02 10:02:00", + None, + ] + + +@pytest.mark.parametrize("bad", ["garbage", 1752055200]) +def test_invalid_ts_error_names_flight_and_point(bad: object) -> None: + ts_values = ["2025-11-02T10:00:00Z", bad] + with pytest.raises(ValueError, match=r"flight 'BAD01', trajectory point 1: "): + neats_json_to_flights([_flight(ts_values, "BAD01")]) # type: ignore[arg-type] + + +def test_offset_and_utc_inputs_parse_to_identical_times() -> None: + """End to end through NeatsTrajectoryParser: a +02:00 trajectory and the + same instants written in UTC give identical parsed times.""" + utc = [f"2025-11-02T10:{m:02d}:00Z" for m in range(0, 30, 5)] + local = [f"2025-11-02T12:{m:02d}:00+02:00" for m in range(0, 30, 5)] + (df_utc, df_local) = neats_json_to_flights( + [_flight(utc, "UTC01"), _flight(local, "LOC01")] + ) + parser = build(TrajectoryParser, "neats") + t_utc = pd.Series(parser(df_utc)["time"]) + t_local = pd.Series(parser(df_local)["time"]) + assert t_utc.tolist() == t_local.tolist() + assert t_utc.iloc[0] == pd.Timestamp("2025-11-02 10:00:00")