diff --git a/ARGO/oceancurrent/README.md b/ARGO/oceancurrent/README.md index 25e775ca..1078184b 100644 --- a/ARGO/oceancurrent/README.md +++ b/ARGO/oceancurrent/README.md @@ -219,3 +219,106 @@ Merging activates these configurations on the next scheduled run because the production job downloads the scanner from `master`. Gliders runs after existing products and writes up to six indexes under `gliders/`; check the existing runtime and memory logs after deployment. No cron or deployment changes are required. + +## Gliders mission catalogue + +`gliders_catalogue.py` independently converts the four mission tables in the +legacy catalogue into `gliders/gliders-catalogue.json`, relative to the website +root. This contains mission metadata, **not** the generic image inventory schema; +do not add it to `FILE_PATH_CONFIG` or the generic API image-index configuration. +API integration is a separate task. + +The source is `/mnt/oceancurrent/website/gliders/index.html`, published at +. A search of this repository +and its documented OceanCurrent pipeline found no maintained structured source +for this catalogue. Existing Gliders GIF/MP4 indexes describe assets, not mission +metadata. The parser uses the existing `beautifulsoup4` dependency with Python's +HTML parser; it does not fetch remote resources or inspect scientific files. + +### JSON schema (version 1) + +The top-level object contains `schema_version`, `source_url`, `missions` and +`orders`. `missions` is an array sorted by `id`, with one record per exact +published mission name and processing-version string. Optional scalar fields are +`null`; unavailable visualisations are empty arrays. + +| Field | Meaning | +| --- | --- | +| `id` | URL-quoted exact `mission_name`, then `@`, then URL-quoted `processing_version` (or `unknown` when absent). Stable for that published name/version; independent of catalogue row numbering. | +| `mission_name` | Exact catalogue name, including historical case, leg and `_nrt` suffixes. | +| `mission_id` | Name with only the terminal `_nrt` removed, to associate NRT and non-NRT publications. Other suffixes remain significant. This is a catalogue identity, not a scientific deployment identifier. | +| `catalogue_numbers` | All original `N` values for this record, as numerically sorted strings. Identical rows sometimes have different numbers. | +| `processing_version` | Exact processing text with surrounding whitespace removed, e.g. `5.00`, `3.0.` or `5.`. Never converted to a number. | +| `is_nrt` | Boolean based on the catalogue's terminal `_nrt` naming convention. False means a non-NRT catalogue publication; it does not independently certify scientific quality or delayed-mode status. | +| `dates_raw` | Original deployment date text with whitespace normalised. Always retained, even when parsing fails. | +| `start_date`, `end_date` | ISO dates for recognised `DD Mon - DD Mon YYYY` ranges. The trailing year is the end year; a wrapped month uses the previous start year. Unrecognised or invalid ranges leave both null. No dates are inferred from mission names. | +| `latitude`, `longitude` | Signed decimal degrees: the catalogue's degrees south become negative latitude; degrees east retain their sign. Header units are checked before conversion. | +| `duration_days`, `distance_travelled_km` | Nonnegative numeric values from `days` and `km`. The latter is travelled distance, **not** distance alongshore to Darwin. Nonfinite and out-of-range numbers become null. | +| `glider_id` | Published glider identifier. | +| `visualisations` | `details` and `animations` arrays of `{ "label": "4", "url": "https://..." }`. Labels and linked targets are preserved, including available 4/12/30-day pages and MP4s; no missing periods are invented. | +| `temperature_salinity_url` | Published T-S reference, which can be an HTML page rather than an image. | +| `scientific_data_url` | Published mission scientific data link only; null if absent. | + +`orders` maps `deployment_date`, `mission_name`, +`distance_alongshore_to_darwin` and `event_based_sampling` to ordered arrays of +record IDs. These retain the first occurrence in each respective source table, +including its membership omissions. Event membership and alongshore ordering +must use these lists; no numeric distance-to-Darwin is inferred. Year headings +and totals are excluded. Repeated occurrences do not duplicate a record or its +position within a view. These orders may have different lengths. + +### Local execution and tests + +Use an environment with the repository's `beautifulsoup4` dependency installed. +Both paths are required to prevent an accidental production run; the output's +parent directory must already exist. For example, using a local source copy: + +```bash +python3 ARGO/oceancurrent/gliders_catalogue.py \ + --input /tmp/gliders-index.html \ + --output /tmp/gliders-catalogue.json +python3 -m pytest -q ARGO/oceancurrent/test_gliders_catalogue.py +``` + +The standard library runner is also supported: + +```bash +cd ARGO/oceancurrent +python3 -m unittest test_gliders_catalogue +``` + +`--source-url` optionally changes the base used to resolve relative links; local +input paths never become public URLs. Output is deterministic for identical +input and base URL, with no generated timestamp. The generator parses and +validates before writing, uses a temporary file alongside the output, then +atomically replaces the destination. Parsing, validation or replacement failures +leave the previous catalogue intact, remove the temporary file and exit nonzero. + +### Refresh and limitations + +No cron, deployment, scanner or scientific processing changes are included. +A future separately reviewed scheduling change should run this generator after +the legacy catalogue has finished publishing, with explicit input and output +paths, in the existing OceanCurrent job environment. It should log the record +count and failures and alert on unexpected count reductions. Avoid reading a +partially written HTML file: use the publisher's completion signal or a completed +snapshot. Start with a manual comparison before enabling recurring refreshes; +refresh at the legacy catalogue's publication cadence rather than running a new +service. + +The four table anchors, column headers and coordinate units are deliberately +validated. Missing tables, empty main views and conflicting metadata for the +same name/version fail the run. The event table may be empty. Year/summary rows +and incomplete or unrecognised mission rows are skipped. This is a structural +sanity check, not proof that a partially published catalogue is complete: there +is no fixed mission-count threshold. Historical letter suffixes and `_1`/`_2` +legs are supported; new naming/header conventions need parser review. Scientific +and visualisation links are preserved but their availability is not checked. +The source's precision and errors remain authoritative; no scientific metadata, +annual-map hotspots or missing links are inferred. + +Future `ocean-current-api` work should define a dedicated catalogue response and +loader for this schema, retain the four order lists and mission/version identity, +and associate visualisation references with the existing Gliders asset indexes. +The catalogue must not be interpreted as an image index. Endpoints, frontend +browsing and caching/refresh behaviour remain separate integration work. diff --git a/ARGO/oceancurrent/fixtures/gliders_catalogue.html b/ARGO/oceancurrent/fixtures/gliders_catalogue.html new file mode 100644 index 00000000..5ec6354f --- /dev/null +++ b/ARGO/oceancurrent/fixtures/gliders_catalogue.html @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + +
Ndates lat lon dayskmT-Sdetails anim gliderprocmission name
459°S°E11348 256068 4/12/30day 4/12/30day vers(AODN thredds)
30202656411794
43827 Mar - 12 Apr 202632.16115.3917373T-S4 12 304 12 30SL248 5.01 TwoRocks20260327
39431 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
39531 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
25703 May - 27 May 201835.76152.1124855T-S4 12 304 12 30SL502 5.00 JervisBay20180503a
24906 Mar - 10 Mar 201828.17154.075403T-S4 12 304 12 30SG540 5.00 Brisbane20180306A
24806 Mar - 08 Mar 201827.85153.983143T-S4 12 304 12 30SG540 3.0. Brisbane20180306A_nrt
14801 Sep - 18 Nov 201432.06113.84791635T-S4 12 304 12 30SG540 5.00 Leeuwin20140901
55Jun - Aug 20114.00 Perth20110626_2
5426 Jun - 11 Oct 201130.33113.381082640T-S4 12 304 12 30SG514 5.00 Perth20110626_1
999incomplete mission row
+ + + + + + + + + + + +
Ndates lat lon dayskmT-Sdetails anim gliderprocmission name
459°S°E11348 256068 4/12/30day 4/12/30day vers(AODN thredds)
25703 May - 27 May 201835.76152.1124855T-S4 12 304 12 30SL502 5.00 JervisBay20180503a
14801 Sep - 18 Nov 201432.06113.84791635T-S4 12 304 12 30SG540 5.00 Leeuwin20140901
55Jun - Aug 20114.00 Perth20110626_2
5426 Jun - 11 Oct 201130.33113.381082640T-S4 12 304 12 30SG514 5.00 Perth20110626_1
39431 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
39531 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
43827 Mar - 12 Apr 202632.16115.3917373T-S4 12 304 12 30SL248 5.01 TwoRocks20260327
999incomplete mission row
+ + + + + + + + + + + + + + + +
Ndates lat lon dayskmT-Sdetails anim gliderprocmission name
459°S°E11348 256068 4/12/30day 4/12/30day vers(AODN thredds)
24806 Mar - 08 Mar 201827.85153.983143T-S4 12 304 12 30SG540 3.0. Brisbane20180306A_nrt
24906 Mar - 10 Mar 201828.17154.075403T-S4 12 304 12 30SG540 5.00 Brisbane20180306A
25703 May - 27 May 201835.76152.1124855T-S4 12 304 12 30SL502 5.00 JervisBay20180503a
39431 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
39531 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
39431 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
39531 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
55Jun - Aug 20114.00 Perth20110626_2
14801 Sep - 18 Nov 201432.06113.84791635T-S4 12 304 12 30SG540 5.00 Leeuwin20140901
43827 Mar - 12 Apr 202632.16115.3917373T-S4 12 304 12 30SL248 5.01 TwoRocks20260327
5426 Jun - 11 Oct 201130.33113.381082640T-S4 12 304 12 30SG514 5.00 Perth20110626_1
999incomplete mission row
+ + + + + + + +
Ndates lat lon dayskmT-Sdetails anim gliderprocmission name
459°S°E11348 256068 4/12/30day 4/12/30day vers(AODN thredds)
43827 Mar - 12 Apr 202632.16115.3917373T-S4 12 304 12 30SL248 5.01 TwoRocks20260327
39431 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
39531 Oct - 24 Nov 202442.14148.2825556T-S4 12 304 12 30SL248 5.01 TasEastCoast20241031
999incomplete mission row
diff --git a/ARGO/oceancurrent/gliders_catalogue.py b/ARGO/oceancurrent/gliders_catalogue.py new file mode 100644 index 00000000..85c8b3cb --- /dev/null +++ b/ARGO/oceancurrent/gliders_catalogue.py @@ -0,0 +1,193 @@ +"""Generate mission metadata from the legacy OceanCurrent Gliders catalogue.""" +import argparse +from datetime import date +import json +import math +import os +from pathlib import Path +import re +import tempfile +from urllib.parse import quote, urljoin, urlparse + +from bs4 import BeautifulSoup + +SOURCE_URL = "https://oceancurrent.aodn.org.au/gliders/index.html" +VIEWS = {"bydate": "deployment_date", "bymission": "mission_name", + "bydist": "distance_alongshore_to_darwin", "byebs": "event_based_sampling"} +HEADERS = ["N", "dates", "lat", "lon", "days", "km", "T-S", "details", + "anim", "glider", "proc", "mission name"] +MISSION = re.compile(r"[A-Za-z][A-Za-z0-9_-]*\d{8}[A-Za-z]?(?:_[12])?(?:_nrt)?$") +MONTHS = "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split() + + +def _text(cell): + return cell.get_text(" ", strip=True) + + +def _number(text, minimum, maximum=None): + try: + value = float(text) + except ValueError: + return None + if not math.isfinite(value) or value < minimum or (maximum is not None and value > maximum): + return None + return value + + +def _dates(raw): + """The catalogue gives the end year; a wrapped month belongs to the prior year.""" + match = re.fullmatch(r"(\d{2}) ([A-Z][a-z]{2}) - (\d{2}) ([A-Z][a-z]{2}) (\d{4})", raw) + if not match: + return None, None + day1, month1, day2, month2, year = match.groups() + try: + m1, m2 = MONTHS.index(month1) + 1, MONTHS.index(month2) + 1 + end = date(int(year), m2, int(day2)) + start = date(int(year) - (m1 > m2), m1, int(day1)) + if start > end: + return None, None + return start.isoformat(), end.isoformat() + except ValueError: + return None, None + + +def _links(cell, source_url): + links = [] + for anchor in cell.find_all("a", href=True): + href = anchor["href"].strip() + url = urljoin(source_url, href) + if href and urlparse(url).scheme in ("http", "https"): + link = {"label": _text(anchor), "url": url} + if link not in links: + links.append(link) + return links + + +def _record(cells, source_url): + values = [_text(cell) for cell in cells] + name = values[11] + if not values[0].isdigit() or not MISSION.fullmatch(name): + return None + processing = values[10] or None + start, end = _dates(values[1]) + # The header explicitly specifies degrees south and degrees east. + latitude = _number(values[2], -90, 90) + scientific = _links(cells[11], source_url) + ts = _links(cells[6], source_url) + return { + "id": quote(name, safe="") + "@" + quote(processing or "unknown", safe=""), + "mission_id": re.sub(r"_nrt$", "", name), + "mission_name": name, + "catalogue_numbers": [values[0]], + "processing_version": processing, + "is_nrt": name.endswith("_nrt"), + "dates_raw": values[1] or None, + "start_date": start, "end_date": end, + "latitude": -abs(latitude) if latitude is not None else None, + "longitude": _number(values[3], -180, 180), + "duration_days": _number(values[4], 0), + "distance_travelled_km": _number(values[5], 0), + "glider_id": values[9] or None, + "visualisations": {"details": _links(cells[7], source_url), + "animations": _links(cells[8], source_url)}, + "temperature_salinity_url": ts[0]["url"] if ts else None, + "scientific_data_url": scientific[0]["url"] if scientific else None, + } + + +def parse_catalogue(html, source_url=SOURCE_URL): + """Return a deterministic catalogue, rejecting missing views and conflicting rows. + + Optional values are null, unavailable visualisations are empty lists. Year and + summary rows have no mission identifier and are ignored, as are incomplete rows. + """ + soup = BeautifulSoup(html, "html.parser") + records, orders = {}, {} + tables = set() + for anchor_id, view in VIEWS.items(): + anchor = soup.find("a", id=anchor_id) + table = anchor.find_next("table") if anchor else None + if table is None or id(table) in tables: + raise ValueError("Missing catalogue table: " + anchor_id) + tables.add(id(table)) + rows = table.find_all("tr") + if not rows or [_text(c) for c in rows[0].find_all("th")] != HEADERS: + raise ValueError("Unexpected catalogue columns: " + anchor_id) + units = rows[1].get_text() if len(rows) > 1 else "" + if "°S" not in units or "°E" not in units: + raise ValueError("Unexpected coordinate units: " + anchor_id) + order = [] + for row in rows[1:]: + cells = row.find_all("td", recursive=False) + if len(cells) != len(HEADERS): + continue + record = _record(cells, source_url) + if record is None: + continue + key = record["id"] + if key in records: + previous = records[key] + for field, value in record.items(): + if field == "catalogue_numbers": + previous[field] = sorted(set(previous[field] + value), key=int) + elif field == "visualisations": + for kind, links in value.items(): + for link in links: + if link not in previous[field][kind]: + previous[field][kind].append(link) + elif previous[field] is None: + previous[field] = value + elif value is not None and previous[field] != value: + raise ValueError("Conflicting metadata for {}: {}".format(key, field)) + else: + records[key] = record + if key not in order: + order.append(key) + if not order and view != "event_based_sampling": + raise ValueError("Empty catalogue view: " + anchor_id) + orders[view] = order + if not records: + raise ValueError("Empty catalogue") + return {"schema_version": 1, "source_url": source_url, + "missions": [records[key] for key in sorted(records)], "orders": orders} + + +def generate_catalogue(input_path, output_path, source_url=SOURCE_URL): + """Parse fully before atomically replacing output; failures leave it untouched.""" + input_path, output_path = Path(input_path), Path(output_path) + if input_path.resolve() == output_path.resolve(): + raise ValueError("Input and output paths must differ") + catalogue = parse_catalogue(input_path.read_text(encoding="utf-8"), source_url) + payload = json.dumps(catalogue, indent=2, ensure_ascii=False, allow_nan=False) + "\n" + temporary = None + try: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=output_path.parent, + prefix="." + output_path.name, delete=False) as stream: + temporary = stream.name + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + # NamedTemporaryFile is private by default; the published catalogue is public. + os.chmod(temporary, 0o644) + os.replace(temporary, output_path) + finally: + if temporary and os.path.exists(temporary): + os.unlink(temporary) + return catalogue + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True, help="Local legacy index.html") + parser.add_argument("--output", type=Path, required=True, help="Destination JSON (parent must exist)") + parser.add_argument("--source-url", default=SOURCE_URL, help="Base URL for resolving catalogue links") + args = parser.parse_args() + try: + catalogue = generate_catalogue(args.input, args.output, args.source_url) + except (OSError, ValueError) as error: + parser.exit(1, "Catalogue generation failed: {}\n".format(error)) + print("Wrote {} mission versions to {}".format(len(catalogue["missions"]), args.output)) + + +if __name__ == "__main__": + main() diff --git a/ARGO/oceancurrent/test_gliders_catalogue.py b/ARGO/oceancurrent/test_gliders_catalogue.py new file mode 100644 index 00000000..ca7daf82 --- /dev/null +++ b/ARGO/oceancurrent/test_gliders_catalogue.py @@ -0,0 +1,169 @@ +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from bs4 import BeautifulSoup + +from gliders_catalogue import generate_catalogue, parse_catalogue + + +FIXTURE = Path(__file__).parent / "fixtures" / "gliders_catalogue.html" + + +class TestGlidersCatalogue(unittest.TestCase): + def setUp(self): + self.html = FIXTURE.read_text() + self.catalogue = parse_catalogue(self.html) + self.records = {r["mission_name"]: r for r in self.catalogue["missions"]} + + def test_unique_missions_and_versions(self): + self.assertEqual(len(self.records), 8) + normal = self.records["Brisbane20180306A"] + nrt = self.records["Brisbane20180306A_nrt"] + self.assertEqual(normal["mission_id"], nrt["mission_id"]) + self.assertNotEqual(normal["id"], nrt["id"]) + self.assertFalse(normal["is_nrt"]) + self.assertTrue(nrt["is_nrt"]) + self.assertEqual(nrt["processing_version"], "3.0.") + self.assertEqual(normal["processing_version"], "5.00") + self.assertEqual(self.records["TasEastCoast20241031"]["catalogue_numbers"], ["394", "395"]) + self.assertIn("JervisBay20180503a", self.records) + self.assertNotEqual(self.records["Perth20110626_1"]["mission_id"], + self.records["Perth20110626_2"]["mission_id"]) + + def test_metadata_and_links(self): + mission = self.records["Brisbane20180306A"] + self.assertEqual(mission["start_date"], "2018-03-06") + self.assertEqual(mission["end_date"], "2018-03-10") + self.assertLess(mission["latitude"], 0) + self.assertGreater(mission["longitude"], 0) + self.assertEqual(mission["glider_id"], "SG540") + self.assertTrue(mission["scientific_data_url"].endswith("Brisbane20180306A/catalog.html")) + self.assertTrue(mission["temperature_salinity_url"].endswith("missionTS/sg_Brisbane20180306A.html")) + for kind, ending in [("details", "/latest.html"), ("animations", ".mp4")]: + links = mission["visualisations"][kind] + self.assertEqual([link["label"] for link in links], ["4", "12", "30"]) + self.assertTrue(all(link["url"].endswith(ending) for link in links)) + # km is travelled distance, never the catalogue's alongshore sort metric. + self.assertEqual(mission["distance_travelled_km"], 403) + self.assertNotIn("distance_to_darwin", mission) + + def test_orders_and_event_membership(self): + orders = self.catalogue["orders"] + ids = {name: record["id"] for name, record in self.records.items()} + self.assertEqual(orders["event_based_sampling"], [ids["TwoRocks20260327"], ids["TasEastCoast20241031"]]) + self.assertEqual(orders["deployment_date"][0], ids["TwoRocks20260327"]) + self.assertEqual(orders["mission_name"][0], ids["JervisBay20180503a"]) + self.assertNotEqual(orders["distance_alongshore_to_darwin"], orders["deployment_date"]) + # The mission-name view in the real catalogue omits this historical NRT row. + self.assertNotIn(ids["Brisbane20180306A_nrt"], orders["mission_name"]) + for order in orders.values(): + self.assertEqual(len(order), len(set(order))) + self.assertTrue(set(order).issubset(ids.values())) + + def test_missing_optional_metadata(self): + mission = self.records["Perth20110626_2"] + self.assertEqual(mission["dates_raw"], "Jun - Aug 2011") + for field in ["start_date", "end_date", "latitude", "longitude", "duration_days", + "distance_travelled_km", "glider_id", "temperature_salinity_url", "scientific_data_url"]: + self.assertIsNone(mission[field], field) + self.assertEqual(mission["visualisations"], {"details": [], "animations": []}) + + def test_same_name_different_processing_versions(self): + soup = BeautifulSoup(self.html, "html.parser") + row = next(r for r in soup.find_all("tr") + if any(a.get_text() == "Brisbane20180306A" for a in r.find_all("a"))) + extra = copy.deepcopy(row) + extra.find_all("td")[10].string = "6.00" + row.insert_after(extra) + result = parse_catalogue(str(soup)) + versions = [r for r in result["missions"] if r["mission_name"] == "Brisbane20180306A"] + self.assertEqual(len(versions), 2) + self.assertEqual(len({r["id"] for r in versions}), 2) + + def test_dates(self): + html = self.html.replace("06 Mar - 10 Mar 2018", "20 Dec - 10 Jan 2018") + record = next(r for r in parse_catalogue(html)["missions"] if r["mission_name"] == "Brisbane20180306A") + self.assertEqual(record["start_date"], "2017-12-20") + self.assertEqual(record["end_date"], "2018-01-10") + html = self.html.replace("06 Mar - 10 Mar 2018", "31 Feb - 10 Mar 2018") + record = next(r for r in parse_catalogue(html)["missions"] if r["mission_name"] == "Brisbane20180306A") + self.assertIsNone(record["start_date"]) + + def test_conflicting_metadata_rejected(self): + # Change just one occurrence, leaving other catalogue views intact. + html = self.html.replace("06 Mar - 10 Mar 2018", "07 Mar - 10 Mar 2018", 1) + with self.assertRaisesRegex(ValueError, "Conflicting metadata"): + parse_catalogue(html) + + def test_empty_event_view_and_partial_visualisations(self): + soup = BeautifulSoup(self.html, "html.parser") + event_table = soup.find("a", id="byebs").find_next("table") + for row in event_table.find_all("tr")[2:]: + row.decompose() + date_table = soup.find("a", id="bydate").find_next("table") + for row in date_table.find_all("tr"): + cells = row.find_all("td", recursive=False) + if len(cells) == 12 and cells[-1].get_text() == "Brisbane20180306A": + cells[7].clear() + cells[8].clear() + result = parse_catalogue(str(soup)) + self.assertEqual(result["orders"]["event_based_sampling"], []) + record = next(r for r in result["missions"] if r["mission_name"] == "Brisbane20180306A") + self.assertEqual(len(record["visualisations"]["details"]), 3) + + def test_invalid_optional_numbers(self): + soup = BeautifulSoup(self.html, "html.parser") + for row in soup.find_all("tr"): + cells = row.find_all("td", recursive=False) + if len(cells) == 12 and cells[-1].get_text() == "Brisbane20180306A": + for index, value in [(2, "NaN"), (3, "999"), (4, "-2"), (5, "inf")]: + cells[index].string = value + record = next(r for r in parse_catalogue(str(soup))["missions"] + if r["mission_name"] == "Brisbane20180306A") + for field in ["latitude", "longitude", "duration_days", "distance_travelled_km"]: + self.assertIsNone(record[field]) + + def test_empty_and_incomplete_catalogue_rejected(self): + soup = BeautifulSoup(self.html, "html.parser") + for row in soup.find_all("tr"): + if row.find_all("td", recursive=False): + row.decompose() + with self.assertRaisesRegex(ValueError, "Empty catalogue view"): + parse_catalogue(str(soup)) + + def test_invalid_source_preserves_output(self): + with tempfile.TemporaryDirectory() as folder: + source, output = Path(folder) / "index.html", Path(folder) / "catalogue.json" + source.write_text(self.html) + generate_catalogue(source, output) + original = output.read_bytes() + for invalid in ["", "Maintenance", + self.html.replace('id="bydist"', 'id="missing"').replace('id=bydist', 'id=missing'), + self.html.replace("°S", "°N"), + self.html.replace("Brisbane20180306A", "invalid").replace("", "
")]: + source.write_text(invalid) + with self.assertRaises(ValueError): + generate_catalogue(source, output) + self.assertEqual(output.read_bytes(), original) + + def test_atomic_failure_and_determinism(self): + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) / "catalogue.json" + generate_catalogue(FIXTURE, output) + original = output.read_bytes() + self.assertEqual(json.loads(original), self.catalogue) + generate_catalogue(FIXTURE, output) + self.assertEqual(output.read_bytes(), original) + with patch("gliders_catalogue.os.replace", side_effect=OSError("write failed")): + with self.assertRaises(OSError): + generate_catalogue(FIXTURE, output) + self.assertEqual(output.read_bytes(), original) + self.assertEqual(list(Path(folder).iterdir()), [output]) + + +if __name__ == "__main__": + unittest.main()
changed