From 72d03a87753eacd1ecc8c89966b28ad88ca4399d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Germ=C3=A1n=20Rodr=C3=ADguez?= <36035218+grojeda@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:35:00 +1100 Subject: [PATCH] refactor(gliders): replace BeautifulSoup with stdlib HTMLParser --- ARGO/oceancurrent/README.md | 9 +- .../fixtures/gliders_catalogue.json | 415 ++++++++++++++++++ ARGO/oceancurrent/gliders_catalogue.py | 96 +++- .../gliders_catalogue_test_utils.py | 28 ++ ARGO/oceancurrent/test_gliders_catalogue.py | 90 ++-- 5 files changed, 591 insertions(+), 47 deletions(-) create mode 100644 ARGO/oceancurrent/fixtures/gliders_catalogue.json create mode 100644 ARGO/oceancurrent/gliders_catalogue_test_utils.py diff --git a/ARGO/oceancurrent/README.md b/ARGO/oceancurrent/README.md index 1078184b..bbc6e305 100644 --- a/ARGO/oceancurrent/README.md +++ b/ARGO/oceancurrent/README.md @@ -232,8 +232,8 @@ The source is `/mnt/oceancurrent/website/gliders/index.html`, published at . A search of this repository and its documented OceanCurrent pipeline found no maintained structured source for this catalogue. Existing Gliders GIF/MP4 indexes describe assets, not mission -metadata. The parser uses the existing `beautifulsoup4` dependency with Python's -HTML parser; it does not fetch remote resources or inspect scientific files. +metadata. The parser uses Python's standard library `html.parser.HTMLParser`; +it does not fetch remote resources or inspect scientific files. ### JSON schema (version 1) @@ -269,7 +269,7 @@ position within a view. These orders may have different lengths. ### Local execution and tests -Use an environment with the repository's `beautifulsoup4` dependency installed. +The generator requires only the Python 3 standard library. Both paths are required to prevent an accidental production run; the output's parent directory must already exist. For example, using a local source copy: @@ -280,6 +280,9 @@ python3 ARGO/oceancurrent/gliders_catalogue.py \ python3 -m pytest -q ARGO/oceancurrent/test_gliders_catalogue.py ``` +The JSON fixture records the previous BeautifulSoup implementation's output; tests +compare both the complete parsed catalogue and generated bytes against it. + The standard library runner is also supported: ```bash diff --git a/ARGO/oceancurrent/fixtures/gliders_catalogue.json b/ARGO/oceancurrent/fixtures/gliders_catalogue.json new file mode 100644 index 00000000..703bc808 --- /dev/null +++ b/ARGO/oceancurrent/fixtures/gliders_catalogue.json @@ -0,0 +1,415 @@ +{ + "schema_version": 1, + "source_url": "https://oceancurrent.aodn.org.au/gliders/index.html", + "missions": [ + { + "id": "Brisbane20180306A@5.00", + "mission_id": "Brisbane20180306A", + "mission_name": "Brisbane20180306A", + "catalogue_numbers": [ + "249" + ], + "processing_version": "5.00", + "is_nrt": false, + "dates_raw": "06 Mar - 10 Mar 2018", + "start_date": "2018-03-06", + "end_date": "2018-03-10", + "latitude": -28.17, + "longitude": 154.07, + "duration_days": 5.0, + "distance_travelled_km": 403.0, + "glider_id": "SG540", + "visualisations": { + "details": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_4d/latest.html" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_12d/latest.html" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_30d/latest.html" + } + ], + "animations": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_4d/Brisbane20180306A_4d.mp4" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_12d/Brisbane20180306A_12d.mp4" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_30d/Brisbane20180306A_30d.mp4" + } + ] + }, + "temperature_salinity_url": "https://oceancurrent.aodn.org.au/gliders/missionTS/sg_Brisbane20180306A.html", + "scientific_data_url": "http://thredds.aodn.org.au/thredds/catalog/IMOS/ANFOG/seaglider/Brisbane20180306A/catalog.html" + }, + { + "id": "Brisbane20180306A_nrt@3.0.", + "mission_id": "Brisbane20180306A", + "mission_name": "Brisbane20180306A_nrt", + "catalogue_numbers": [ + "248" + ], + "processing_version": "3.0.", + "is_nrt": true, + "dates_raw": "06 Mar - 08 Mar 2018", + "start_date": "2018-03-06", + "end_date": "2018-03-08", + "latitude": -27.85, + "longitude": 153.98, + "duration_days": 3.0, + "distance_travelled_km": 143.0, + "glider_id": "SG540", + "visualisations": { + "details": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_nrt_4d/latest.html" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_nrt_12d/latest.html" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_nrt_30d/latest.html" + } + ], + "animations": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_nrt_4d/Brisbane20180306A_nrt_4d.mp4" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_nrt_12d/Brisbane20180306A_nrt_12d.mp4" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/Brisbane20180306A_nrt_30d/Brisbane20180306A_nrt_30d.mp4" + } + ] + }, + "temperature_salinity_url": "https://oceancurrent.aodn.org.au/gliders/missionTS/sg_Brisbane20180306A_nrt.html", + "scientific_data_url": "http://thredds.aodn.org.au/thredds/catalog/IMOS/ANFOG/REALTIME/seaglider/Brisbane20180306A/catalog.html" + }, + { + "id": "JervisBay20180503a@5.00", + "mission_id": "JervisBay20180503a", + "mission_name": "JervisBay20180503a", + "catalogue_numbers": [ + "257" + ], + "processing_version": "5.00", + "is_nrt": false, + "dates_raw": "03 May - 27 May 2018", + "start_date": "2018-05-03", + "end_date": "2018-05-27", + "latitude": -35.76, + "longitude": 152.11, + "duration_days": 24.0, + "distance_travelled_km": 855.0, + "glider_id": "SL502", + "visualisations": { + "details": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/JervisBay20180503a_4d/latest.html" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/JervisBay20180503a_12d/latest.html" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/JervisBay20180503a_30d/latest.html" + } + ], + "animations": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/JervisBay20180503a_4d/JervisBay20180503a_4d.mp4" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/JervisBay20180503a_12d/JervisBay20180503a_12d.mp4" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/JervisBay20180503a_30d/JervisBay20180503a_30d.mp4" + } + ] + }, + "temperature_salinity_url": "https://oceancurrent.aodn.org.au/gliders/missionTS/sl_JervisBay20180503a.html", + "scientific_data_url": "http://thredds.aodn.org.au/thredds/catalog/IMOS/ANFOG/slocum_glider/JervisBay20180503a/catalog.html" + }, + { + "id": "Leeuwin20140901@5.00", + "mission_id": "Leeuwin20140901", + "mission_name": "Leeuwin20140901", + "catalogue_numbers": [ + "148" + ], + "processing_version": "5.00", + "is_nrt": false, + "dates_raw": "01 Sep - 18 Nov 2014", + "start_date": "2014-09-01", + "end_date": "2014-11-18", + "latitude": -32.06, + "longitude": 113.84, + "duration_days": 79.0, + "distance_travelled_km": 1635.0, + "glider_id": "SG540", + "visualisations": { + "details": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/Leeuwin20140901_4d/latest.html" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/Leeuwin20140901_12d/latest.html" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/Leeuwin20140901_30d/latest.html" + } + ], + "animations": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/Leeuwin20140901_4d/Leeuwin20140901_4d.mp4" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/Leeuwin20140901_12d/Leeuwin20140901_12d.mp4" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/Leeuwin20140901_30d/Leeuwin20140901_30d.mp4" + } + ] + }, + "temperature_salinity_url": "https://oceancurrent.aodn.org.au/gliders/missionTS/sg_Leeuwin20140901.html", + "scientific_data_url": "http://thredds.aodn.org.au/thredds/catalog/IMOS/ANFOG/seaglider/Leeuwin20140901/catalog.html" + }, + { + "id": "Perth20110626_1@5.00", + "mission_id": "Perth20110626_1", + "mission_name": "Perth20110626_1", + "catalogue_numbers": [ + "54" + ], + "processing_version": "5.00", + "is_nrt": false, + "dates_raw": "26 Jun - 11 Oct 2011", + "start_date": "2011-06-26", + "end_date": "2011-10-11", + "latitude": -30.33, + "longitude": 113.38, + "duration_days": 108.0, + "distance_travelled_km": 2640.0, + "glider_id": "SG514", + "visualisations": { + "details": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/Perth20110626_1_4d/latest.html" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/Perth20110626_1_12d/latest.html" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/Perth20110626_1_30d/latest.html" + } + ], + "animations": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/Perth20110626_1_4d/Perth20110626_1_4d.mp4" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/Perth20110626_1_12d/Perth20110626_1_12d.mp4" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/Perth20110626_1_30d/Perth20110626_1_30d.mp4" + } + ] + }, + "temperature_salinity_url": "https://oceancurrent.aodn.org.au/gliders/missionTS/sg_Perth20110626_1.html", + "scientific_data_url": "http://thredds.aodn.org.au/thredds/catalog/IMOS/ANFOG/seaglider/Perth20110626_1/catalog.html" + }, + { + "id": "Perth20110626_2@4.00", + "mission_id": "Perth20110626_2", + "mission_name": "Perth20110626_2", + "catalogue_numbers": [ + "55" + ], + "processing_version": "4.00", + "is_nrt": false, + "dates_raw": "Jun - Aug 2011", + "start_date": null, + "end_date": null, + "latitude": null, + "longitude": null, + "duration_days": null, + "distance_travelled_km": null, + "glider_id": null, + "visualisations": { + "details": [], + "animations": [] + }, + "temperature_salinity_url": null, + "scientific_data_url": null + }, + { + "id": "TasEastCoast20241031@5.01", + "mission_id": "TasEastCoast20241031", + "mission_name": "TasEastCoast20241031", + "catalogue_numbers": [ + "394", + "395" + ], + "processing_version": "5.01", + "is_nrt": false, + "dates_raw": "31 Oct - 24 Nov 2024", + "start_date": "2024-10-31", + "end_date": "2024-11-24", + "latitude": -42.14, + "longitude": 148.28, + "duration_days": 25.0, + "distance_travelled_km": 556.0, + "glider_id": "SL248", + "visualisations": { + "details": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/TasEastCoast20241031_4d/latest.html" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/TasEastCoast20241031_12d/latest.html" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/TasEastCoast20241031_30d/latest.html" + } + ], + "animations": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/TasEastCoast20241031_4d/TasEastCoast20241031_4d.mp4" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/TasEastCoast20241031_12d/TasEastCoast20241031_12d.mp4" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/TasEastCoast20241031_30d/TasEastCoast20241031_30d.mp4" + } + ] + }, + "temperature_salinity_url": "https://oceancurrent.aodn.org.au/gliders/missionTS/sl_TasEastCoast20241031.html", + "scientific_data_url": "http://thredds.aodn.org.au/thredds/catalog/IMOS/ANFOG/slocum_glider/TasEastCoast20241031/catalog.html" + }, + { + "id": "TwoRocks20260327@5.01", + "mission_id": "TwoRocks20260327", + "mission_name": "TwoRocks20260327", + "catalogue_numbers": [ + "438" + ], + "processing_version": "5.01", + "is_nrt": false, + "dates_raw": "27 Mar - 12 Apr 2026", + "start_date": "2026-03-27", + "end_date": "2026-04-12", + "latitude": -32.16, + "longitude": 115.39, + "duration_days": 17.0, + "distance_travelled_km": 373.0, + "glider_id": "SL248", + "visualisations": { + "details": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/TwoRocks20260327_4d/latest.html" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/TwoRocks20260327_12d/latest.html" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/TwoRocks20260327_30d/latest.html" + } + ], + "animations": [ + { + "label": "4", + "url": "https://oceancurrent.aodn.org.au/gliders/TwoRocks20260327_4d/TwoRocks20260327_4d.mp4" + }, + { + "label": "12", + "url": "https://oceancurrent.aodn.org.au/gliders/TwoRocks20260327_12d/TwoRocks20260327_12d.mp4" + }, + { + "label": "30", + "url": "https://oceancurrent.aodn.org.au/gliders/TwoRocks20260327_30d/TwoRocks20260327_30d.mp4" + } + ] + }, + "temperature_salinity_url": "https://oceancurrent.aodn.org.au/gliders/missionTS/sl_TwoRocks20260327.html", + "scientific_data_url": "http://thredds.aodn.org.au/thredds/catalog/IMOS/ANFOG/slocum_glider/TwoRocks20260327/catalog.html" + } + ], + "orders": { + "deployment_date": [ + "TwoRocks20260327@5.01", + "TasEastCoast20241031@5.01", + "JervisBay20180503a@5.00", + "Brisbane20180306A@5.00", + "Brisbane20180306A_nrt@3.0.", + "Leeuwin20140901@5.00", + "Perth20110626_2@4.00", + "Perth20110626_1@5.00" + ], + "mission_name": [ + "JervisBay20180503a@5.00", + "Leeuwin20140901@5.00", + "Perth20110626_2@4.00", + "Perth20110626_1@5.00", + "TasEastCoast20241031@5.01", + "TwoRocks20260327@5.01" + ], + "distance_alongshore_to_darwin": [ + "Brisbane20180306A_nrt@3.0.", + "Brisbane20180306A@5.00", + "JervisBay20180503a@5.00", + "TasEastCoast20241031@5.01", + "Perth20110626_2@4.00", + "Leeuwin20140901@5.00", + "TwoRocks20260327@5.01", + "Perth20110626_1@5.00" + ], + "event_based_sampling": [ + "TwoRocks20260327@5.01", + "TasEastCoast20241031@5.01" + ] + } +} diff --git a/ARGO/oceancurrent/gliders_catalogue.py b/ARGO/oceancurrent/gliders_catalogue.py index 85c8b3cb..b5fe5b1b 100644 --- a/ARGO/oceancurrent/gliders_catalogue.py +++ b/ARGO/oceancurrent/gliders_catalogue.py @@ -1,6 +1,7 @@ """Generate mission metadata from the legacy OceanCurrent Gliders catalogue.""" import argparse from datetime import date +from html.parser import HTMLParser import json import math import os @@ -9,8 +10,6 @@ import tempfile from urllib.parse import quote, urljoin, urlparse -from bs4 import BeautifulSoup - SOURCE_URL = "https://oceancurrent.aodn.org.au/gliders/index.html" VIEWS = {"bydate": "deployment_date", "bymission": "mission_name", "bydist": "distance_alongshore_to_darwin", "byebs": "event_based_sampling"} @@ -20,8 +19,78 @@ MONTHS = "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split() +class _Element: + """The small portion of an HTML tree needed for catalogue extraction.""" + + def __init__(self, tag, attrs=()): + self.tag = tag + self.attrs = dict(attrs) + self.children = [] + + def elements(self, tag, direct=False): + for child in self.children: + if isinstance(child, _Element): + if child.tag == tag: + yield child + if not direct: + yield from child.elements(tag) + + def strings(self): + for child in self.children: + if isinstance(child, str): + yield child + elif child.tag not in {"script", "style", "template"}: + yield from child.strings() + + +class _CatalogueParser(HTMLParser): + """Keep document order and nesting, without applying browser HTML repairs. + + Table selection uses the first table after each view anchor. A tree keeps + direct row cells distinct from nested markup and preserves text boundaries. + """ + + _VOID_TAGS = frozenset("area base br col embed hr img input link meta param source track wbr".split()) + + def __init__(self): + super().__init__(convert_charrefs=True) + self.elements = [] + self._stack = [_Element("")] + + def handle_starttag(self, tag, attrs): + element = _Element(tag, attrs) + self._stack[-1].children.append(element) + self.elements.append(element) + if tag not in self._VOID_TAGS: + self._stack.append(element) + + def handle_endtag(self, tag): + # Ignore unmatched closing tags; close any unclosed children as well. + for index in range(len(self._stack) - 1, 0, -1): + if self._stack[index].tag == tag: + del self._stack[index:] + break + + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + if tag not in self._VOID_TAGS: + self.handle_endtag(tag) + + def handle_data(self, data): + self._stack[-1].children.append(data) + + def table_after(self, anchor_id): + after_anchor = False + for element in self.elements: + if after_anchor and element.tag == "table": + return element + if element.tag == "a" and element.attrs.get("id") == anchor_id: + after_anchor = True + return None + + def _text(cell): - return cell.get_text(" ", strip=True) + return " ".join(text.strip() for text in cell.strings() if text.strip()) def _number(text, minimum, maximum=None): @@ -53,8 +122,10 @@ def _dates(raw): def _links(cell, source_url): links = [] - for anchor in cell.find_all("a", href=True): - href = anchor["href"].strip() + for anchor in cell.elements("a"): + if anchor.attrs.get("href") is None: + continue + href = anchor.attrs["href"].strip() url = urljoin(source_url, href) if href and urlparse(url).scheme in ("http", "https"): link = {"label": _text(anchor), "url": url} @@ -101,24 +172,25 @@ def parse_catalogue(html, source_url=SOURCE_URL): Optional values are null, unavailable visualisations are empty lists. Year and summary rows have no mission identifier and are ignored, as are incomplete rows. """ - soup = BeautifulSoup(html, "html.parser") + parser = _CatalogueParser() + parser.feed(html) + parser.close() records, orders = {}, {} tables = set() for anchor_id, view in VIEWS.items(): - anchor = soup.find("a", id=anchor_id) - table = anchor.find_next("table") if anchor else None + table = parser.table_after(anchor_id) if table is None or id(table) in tables: raise ValueError("Missing catalogue table: " + anchor_id) tables.add(id(table)) - rows = table.find_all("tr") - if not rows or [_text(c) for c in rows[0].find_all("th")] != HEADERS: + rows = list(table.elements("tr")) + if not rows or [_text(c) for c in rows[0].elements("th")] != HEADERS: raise ValueError("Unexpected catalogue columns: " + anchor_id) - units = rows[1].get_text() if len(rows) > 1 else "" + units = "".join(rows[1].strings()) if len(rows) > 1 else "" if "°S" not in units or "°E" not in units: raise ValueError("Unexpected coordinate units: " + anchor_id) order = [] for row in rows[1:]: - cells = row.find_all("td", recursive=False) + cells = list(row.elements("td", direct=True)) if len(cells) != len(HEADERS): continue record = _record(cells, source_url) diff --git a/ARGO/oceancurrent/gliders_catalogue_test_utils.py b/ARGO/oceancurrent/gliders_catalogue_test_utils.py new file mode 100644 index 00000000..277cd191 --- /dev/null +++ b/ARGO/oceancurrent/gliders_catalogue_test_utils.py @@ -0,0 +1,28 @@ +"""Edit the explicit, closed rows/cells in the local catalogue HTML fixture. + +These string edits deliberately avoid using the production parser to build test +inputs, so extraction regressions cannot also change the test setup. +""" +import re + + +def edit_rows(html, transform, view=None): + start = html.index('') if view else 0 + end = html.index('', start) if view else len(html) + return (html[:start] + + re.sub(r'.*?', lambda match: transform(match[0]), html[start:end], flags=re.S) + + html[end:]) + + +def has_mission(row, name): + return '>' + name + '' in row or '>' + name + '' in row + + +def edit_cells(row, updates): + cells = iter(enumerate(re.findall(r'.*?', row, flags=re.S))) + + def replace(match): + index, _ = next(cells) + return '' + updates[index] + '' if index in updates else match[0] + + return re.sub(r'.*?', replace, row, flags=re.S) diff --git a/ARGO/oceancurrent/test_gliders_catalogue.py b/ARGO/oceancurrent/test_gliders_catalogue.py index ca7daf82..c5d38b70 100644 --- a/ARGO/oceancurrent/test_gliders_catalogue.py +++ b/ARGO/oceancurrent/test_gliders_catalogue.py @@ -1,11 +1,10 @@ -import copy import json from pathlib import Path import tempfile import unittest from unittest.mock import patch -from bs4 import BeautifulSoup +from gliders_catalogue_test_utils import edit_cells, edit_rows, has_mission from gliders_catalogue import generate_catalogue, parse_catalogue @@ -19,6 +18,15 @@ def setUp(self): self.catalogue = parse_catalogue(self.html) self.records = {r["mission_name"]: r for r in self.catalogue["missions"]} + def test_matches_previous_implementation(self): + # Captured from the BeautifulSoup implementation using the same fixture. + expected = FIXTURE.with_suffix(".json").read_bytes() + self.assertEqual(self.catalogue, json.loads(expected)) + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) / "catalogue.json" + generate_catalogue(FIXTURE, output) + self.assertEqual(output.read_bytes(), expected) + def test_unique_missions_and_versions(self): self.assertEqual(len(self.records), 8) normal = self.records["Brisbane20180306A"] @@ -73,13 +81,12 @@ def test_missing_optional_metadata(self): self.assertEqual(mission["visualisations"], {"details": [], "animations": []}) def test_same_name_different_processing_versions(self): - soup = BeautifulSoup(self.html, "html.parser") - row = next(r for r in soup.find_all("tr") - if any(a.get_text() == "Brisbane20180306A" for a in r.find_all("a"))) - extra = copy.deepcopy(row) - extra.find_all("td")[10].string = "6.00" - row.insert_after(extra) - result = parse_catalogue(str(soup)) + def add_version(row): + if has_mission(row, "Brisbane20180306A"): + return row + edit_cells(row, {10: "6.00"}) + return row + + result = parse_catalogue(edit_rows(self.html, add_version, view="bydate")) versions = [r for r in result["missions"] if r["mission_name"] == "Brisbane20180306A"] self.assertEqual(len(versions), 2) self.assertEqual(len({r["id"] for r in versions}), 2) @@ -100,40 +107,59 @@ def test_conflicting_metadata_rejected(self): parse_catalogue(html) def test_empty_event_view_and_partial_visualisations(self): - soup = BeautifulSoup(self.html, "html.parser") - event_table = soup.find("a", id="byebs").find_next("table") - for row in event_table.find_all("tr")[2:]: - row.decompose() - date_table = soup.find("a", id="bydate").find_next("table") - for row in date_table.find_all("tr"): - cells = row.find_all("td", recursive=False) - if len(cells) == 12 and cells[-1].get_text() == "Brisbane20180306A": - cells[7].clear() - cells[8].clear() - result = parse_catalogue(str(soup)) + html = edit_rows(self.html, lambda row: "" if "" in row else row, view="byebs") + html = edit_rows(html, lambda row: edit_cells(row, {7: "", 8: ""}) + if has_mission(row, "Brisbane20180306A") else row, view="bydate") + result = parse_catalogue(html) self.assertEqual(result["orders"]["event_based_sampling"], []) record = next(r for r in result["missions"] if r["mission_name"] == "Brisbane20180306A") self.assertEqual(len(record["visualisations"]["details"]), 3) def test_invalid_optional_numbers(self): - soup = BeautifulSoup(self.html, "html.parser") - for row in soup.find_all("tr"): - cells = row.find_all("td", recursive=False) - if len(cells) == 12 and cells[-1].get_text() == "Brisbane20180306A": - for index, value in [(2, "NaN"), (3, "999"), (4, "-2"), (5, "inf")]: - cells[index].string = value - record = next(r for r in parse_catalogue(str(soup))["missions"] + html = edit_rows(self.html, lambda row: edit_cells(row, {2: "NaN", 3: "999", 4: "-2", 5: "inf"}) + if has_mission(row, "Brisbane20180306A") else row) + record = next(r for r in parse_catalogue(html)["missions"] if r["mission_name"] == "Brisbane20180306A") for field in ["latitude", "longitude", "duration_days", "distance_travelled_km"]: self.assertIsNone(record[field]) def test_empty_and_incomplete_catalogue_rejected(self): - soup = BeautifulSoup(self.html, "html.parser") - for row in soup.find_all("tr"): - if row.find_all("td", recursive=False): - row.decompose() + html = edit_rows(self.html, lambda row: "" if "" in row else row) with self.assertRaisesRegex(ValueError, "Empty catalogue view"): - parse_catalogue(str(soup)) + parse_catalogue(html) + + def test_nested_markup_entities_and_link_filtering(self): + link = '4 day' + details = (link + link + 'no hrefempty' + 'scriptmail') + html = edit_rows(self.html, lambda row: edit_cells(row, { + 7: details, 9: 'SG
540' + ''}) + if has_mission(row, "Brisbane20180306A") else row) + result = parse_catalogue(html, "https://example.org/gliders/index.html") + mission = next(r for r in result["missions"] if r["mission_name"] == "Brisbane20180306A") + self.assertEqual(mission["glider_id"], "SG 540") + self.assertEqual(mission["visualisations"]["details"], [ + {"label": "4 day", "url": "https://example.org/details.html?a=1&b=2"}]) + + def test_view_anchors_and_table_wrappers(self): + html = self.html.replace('', '
') + html = html.replace('
', '') + html = html.replace('', '
').replace('
', '') + self.assertEqual(parse_catalogue(html), self.catalogue) + + def test_two_views_cannot_share_a_table(self): + html = self.html.replace('', '', 1) + with self.assertRaisesRegex(ValueError, "Missing catalogue table: bymission"): + parse_catalogue(html) + + def test_columns_and_units_are_validated(self): + for html, message in [ + (self.html.replace('N', 'changed', 1), "Unexpected catalogue columns"), + (self.html.replace('°S', '°N', 1), "Unexpected coordinate units"), + (self.html.replace('°E', '°W', 1), "Unexpected coordinate units")]: + with self.subTest(message=message), self.assertRaisesRegex(ValueError, message): + parse_catalogue(html) def test_invalid_source_preserves_output(self): with tempfile.TemporaryDirectory() as folder: