Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions docs/internal/pyneats_units_audit.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ note under its item. Items not listed here are open.

| Item | Status | Branch / PR | Date |
| --- | --- | --- | --- |
| D2 (trajectory `ts` offsets) | Fixed | `fix/d2-timestamp-utc` | 2026-10-20 |
| D2 (trajectory `ts` offsets) | Fixed | #70 | 2026-10-20 |
| One flight that cannot be converted no longer stops the run (D2 follow-up) | Fixed | `fix/isolate-invalid-flights` | 2026-10-20 |
| D1 (bare single-flight object) | Open; now a clear `TypeError` on the `FleetRunner` JSON path | `fix/isolate-invalid-flights` | 2026-10-20 |

## 1. Outcome

Expand Down Expand Up @@ -444,6 +446,10 @@ Each entry gives:
### On the AO input path

**D1. Bare single-flight object rejected.**
*Still open. On the `FleetRunner` JSON path and in `scripts/run_json.py`, a file whose top
level is an object or a string now raises `TypeError("expected a sequence of flight objects")`
instead of an `AttributeError` (`fix/isolate-invalid-flights`). Accepting a bare object is not
decided.*
1. Top-level JSON → `neats_json_to_flights` (`neats_io.py:45-46`) → `FleetRunner._load_trajectories`
(`fleet.py:561-566`) and `scripts/run_json.py:238-240`.
2. Expected: the AO schema's `anyOf` allows one object. Observed: iterating a dict yields its
Expand All @@ -459,8 +465,11 @@ Each entry gives:
*Fixed on branch `fix/d2-timestamp-utc` (see the follow-up status table). The UTC offset
is now applied. Fractional seconds are truncated. A lowercase `t` or `z` and a space
separator are accepted. A `ts` without offset is read as UTC (maintainer decision). An
invalid `ts` still aborts the batch, now with an error naming the flight and point; isolating
it per flight is a separate change. No effect on existing data: on 148 NEATS JSON files on
invalid `ts` raises an error naming the flight and point. On the `FleetRunner` JSON path and
in `scripts/run_json.py`, such a flight (or any flight object that cannot be converted) becomes
a `Failed at parsing` error record and the other flights go on (`fix/isolate-invalid-flights`);
a direct call to `neats_json_to_flights` without a `failures` list still raises. If every
flight fails, the run returns only error records. No effect on existing data: on 148 NEATS JSON files on
the NEATS server (85,128 flights, 10,995,586 points, all `+0000`, `Z` or `+00:00`) the output
is identical.*
1. `ts` → `neats_io.py:61-68` → `neats_parser.py:233-242`.
Expand Down
28 changes: 22 additions & 6 deletions scripts/run_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from pyneats.runners.flight import RunnerConfig
from pyneats.runners.large_emitter import FleetRunnerLargeEmitter
from pyneats.runners.large_emitter import FlightRunnerLargeEmitter
from pyneats.steps.parsing.neats_io import FlightConversionFailure
from pyneats.steps.parsing.neats_io import neats_json_to_flights
from pyneats.steps.weather.weather_store import ZarrPaths
from pyneats.steps.weather.weather_store import get_weather_from_zarr
Expand Down Expand Up @@ -89,7 +90,10 @@ def _print_summary(
print("Failed flights:")
print("-" * 60)
for err in errors:
fid = err.get("flight_id", "unknown")
fid = str(
err.get("flight_id")
or err.get("flight_information", {}).get("flight_id", "unknown")
)
msg = err.get("error", err.get("error_msg", str(err)))
step = err.get("step_name", "")
detail = f"step={step:<20} {msg}" if step else msg
Expand Down Expand Up @@ -150,13 +154,25 @@ def run_flight_loop(
weather = get_weather_from_zarr(zarr_paths)
log.info("Weather loaded for flight-mode execution")

# 2. Convert JSON dicts → list[pd.DataFrame]
flight_dfs = neats_json_to_flights(input_flights)
log.info("Converted %d flights from JSON", len(flight_dfs))
# 2. Convert JSON dicts → list[pd.DataFrame]; a flight that cannot be
# converted is reported as an error and the others go on.
failures: list[FlightConversionFailure] = []
flight_dfs = neats_json_to_flights(input_flights, failures=failures)
log.info(
"Converted %d flights from JSON (%d could not be converted)",
len(flight_dfs),
len(failures),
)

# 3. Loop over flights
succeeded = 0
errors: list[dict[str, Any]] = []
errors: list[dict[str, Any]] = [
{
"flight_id": f.attrs.get("flight_id", f"input_flight_{f.index}"),
"error": str(f.error),
}
for f in failures
]

cfg = RunnerConfig(
params={
Expand Down Expand Up @@ -186,7 +202,7 @@ def run_flight_loop(
errors.append({"flight_id": fid, "error": str(exc)})

# 4. Print summary
_print_summary("FlightRunnerLargeEmitter", len(flight_dfs), succeeded, errors)
_print_summary("FlightRunnerLargeEmitter", len(input_flights), succeeded, errors)


# ---------------------------------------------------------------------------
Expand Down
35 changes: 30 additions & 5 deletions src/pyneats/runners/fleet.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
from pyneats.steps.emissions.protocol import EmissionModel
from pyneats.steps.emissions.views import FlightWithEmissions
from pyneats.steps.interpolation.protocol import TrajectoryInterpolator
from pyneats.steps.parsing.neats_io import FlightConversionFailure
from pyneats.steps.parsing.neats_io import neats_json_to_flights
from pyneats.steps.parsing.neats_io import split_df_into_flights
from pyneats.steps.parsing.protocol import TrajectoryParser
Expand Down Expand Up @@ -626,9 +627,12 @@ def _parser_params(self) -> dict[str, Any]:
# -------------------------------------------------------------------------

def _load_data(self) -> Self:
return (
self._load_trajectories()._load_weather() # pylint: disable=protected-access # pylint: disable=protected-access
)
self._load_trajectories()
# Every flight already failed at loading: the run stops here, so
# the weather is not needed.
if self._pipeline_aborted:
return self
return self._load_weather()

def _load_trajectories(self) -> Self:
# No unit conversion here: true_airspeed must already be m/s and q_fuel
Expand All @@ -638,8 +642,29 @@ def _load_trajectories(self) -> Self:
logger.info("Loading trajectories from %s", path)
with path.open("r", encoding="utf-8") as f:
json_flights: list[dict[str, Any]] = json.load(f)
self.source_fleet = neats_json_to_flights(json_flights)
logger.info("Loaded %d flights from JSON", len(self.source_fleet))
# A flight that cannot be converted (e.g. an invalid trajectory
# timestamp) becomes an error record; the other flights go on.
failures: list[FlightConversionFailure] = []
self.source_fleet = neats_json_to_flights(json_flights, failures=failures)
for failure in failures:
flight_id = failure.attrs.get(
"flight_id", f"input_flight_{failure.index}"
)
self.error_records.append(
_create_error_record(
failure, flight_id, "parsing", repr(failure.error)
)
)
logger.warning(
"Flight %s failed at parsing: %s", flight_id, failure.error
)
logger.info(
"Loaded %d flights from JSON (%d could not be converted)",
len(self.source_fleet),
len(failures),
)
if failures and not self.source_fleet:
self._pipeline_aborted = True
return self

if self.cfg.trajectory_dataframe is not None:
Expand Down
Loading
Loading