From c857e8c1c9b0be6bea65ab2d604a0afbdd38f303 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Wed, 21 Oct 2026 14:12:38 +1300 Subject: [PATCH 1/3] Add a create-imdb stage collecting intensity measures into an IMDB create-imdb builds an IMDB (ucgmsim-imdb) from im-calc intensity measure files alone: simulated records per station and component, and gmm records per station, ground motion model and component. Observations are not included. Realisations named like clones (`_R`) are grouped into their event. Simulated records inside SW4's absorbing layer are left out. im-calc now records the component of each empirical estimate, always writes site parameters, and always writes the supergrid coordinates (NaN where the solver does not report them). Requires pandas 3 through ucgmsim-imdb, so velocity-modelling is raised to 2026.10.2, which drops its pandas<3 cap. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 5 +- tests/e2e/test_pipeline.py | 15 +- tests/test_create_imdb.py | 204 ++++++++++++ tests/test_im_calc.py | 4 + uv.lock | 199 +++++++++--- workflow/scripts/create_imdb.py | 482 +++++++++++++++++++++++++++++ workflow/scripts/extract_subset.py | 4 +- workflow/scripts/im_calc.py | 36 ++- workflow/sites.py | 4 + 9 files changed, 892 insertions(+), 61 deletions(-) create mode 100644 tests/test_create_imdb.py create mode 100644 workflow/scripts/create_imdb.py diff --git a/pyproject.toml b/pyproject.toml index 9a96bae4..5ac78bf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dynamic = ["version"] dependencies = [ # UCGMSim Dependencies "im-calculation>=2026.10.3", - "velocity-modelling>=2026.10.1", + "velocity-modelling>=2026.10.2", "nshmdb>=2026.10.1", "oq_wrapper>=2026.10.1", "qcore-utils[cli]>=2026.10.1", @@ -21,6 +21,7 @@ dependencies = [ "source_modelling>=2026.10.2", "nzcvm>=2026.10.7", "rupture-generator>=2026.10.1", + "ucgmsim-imdb>=2026.9.2", # Data Formats "geopandas", "fiona", # Imported, undeclared, by openquake's GMMs @@ -86,6 +87,7 @@ merge-ts = "workflow.scripts.merge_ts:app" hf-sim = "workflow.scripts.hf_sim:app" bb-sim = "workflow.scripts.bb_sim:app" im-calc = "workflow.scripts.im_calc:app" +create-imdb = "workflow.scripts.create_imdb:app" check-srf = "workflow.scripts.check_srf:app" check-domain = "workflow.scripts.check_domain:app" gcmt-auto-simulate = "workflow.scripts.gcmt_auto_simulate:app" @@ -157,6 +159,7 @@ known-first-party = [ "mera", "velocity_modelling", "site_calculation", + "imdb", ] [tool.ruff.lint.per-file-ignores] diff --git a/tests/e2e/test_pipeline.py b/tests/e2e/test_pipeline.py index 0162978a..677114a1 100644 --- a/tests/e2e/test_pipeline.py +++ b/tests/e2e/test_pipeline.py @@ -44,6 +44,7 @@ import pytest import xarray as xr +from imdb import IMDB from workflow.defaults import DefaultsVersion from workflow.realisations import ( DomainParameters, @@ -171,7 +172,7 @@ def test_tools_installed() -> None: "gcmt-to-realisation", "generate-domain", "realisation-to-srf", "check-srf", "srf-to-hdf5", "generate-stoch", "generate-station-coordinates", "create-nzvm-input", "nzcvm", "validate-sfile", "create-sw4-input", - "lf-to-xarray", "hf-sim", "bb-sim", "im-calc", "extract-subset", + "lf-to-xarray", "hf-sim", "bb-sim", "im-calc", "create-imdb", "extract-subset", ] # fmt: skip missing = [command for command in commands if shutil.which(command) is None] missing += [ @@ -374,6 +375,18 @@ def test_im_calc(realisation: Path, work: Path) -> None: assert any("/empirical/" in node.path for node in intensity_measures.subtree) +def test_create_imdb(work: Path) -> None: + """The intensity measures collected into a valid IMDB.""" + require(work / "intensity_measures.h5") + output = work / "imdb.duckdb" + run("create-imdb", output, work / "intensity_measures.h5") + + with IMDB(output) as db: + assert db.validate() == [] + assert set(db.get_sites().index) >= set(STATIONS) + assert set(db.get_records()["kind"]) == {"simulated", "gmm"} + + def test_extract_subset(work: Path) -> None: """The waveform and IM files cut down to a subset of stations.""" outputs = [ diff --git a/tests/test_create_imdb.py b/tests/test_create_imdb.py new file mode 100644 index 00000000..38d1dadc --- /dev/null +++ b/tests/test_create_imdb.py @@ -0,0 +1,204 @@ +"""Tests for collecting intensity measure files into an IMDB.""" + +from pathlib import Path + +import numpy as np +import pytest +import xarray as xr + +from imdb import IMDB +from workflow.scripts import create_imdb + +STATIONS = ["WEL", "AKL1", "0a1b2c3"] +PERIODS = [0.1, 1.0] +SIMULATED_COMPONENTS = ["000", "090", "ver", "geom", "rotd50"] + + +def write_im_file( + path: Path, + name: str, + frequencies: tuple[float, ...] = (0.5, 1.0), + supergrid_depth: tuple[float, ...] = (0.0, 0.0, 0.0), +) -> Path: + """Write a small IM file in the `im-calc` layout.""" + n_stations = len(STATIONS) + station_coords = { + "station": STATIONS, + "latitude": ("station", np.linspace(-41.0, -40.0, n_stations)), + "longitude": ("station", np.linspace(174.0, 175.0, n_stations)), + "vs30": ("station", np.linspace(200.0, 600.0, n_stations)), + "z1pt0": ("station", np.linspace(0.1, 0.3, n_stations)), + "z2pt5": ("station", np.linspace(1.0, 3.0, n_stations)), + "rrup": ("station", np.linspace(10.0, 30.0, n_stations)), + "rjb": ("station", np.linspace(5.0, 25.0, n_stations)), + "supergrid_depth": ("station", np.array(supergrid_depth)), + } + + def components( + shape: tuple[int, ...], dims: tuple[str, ...], value: float + ) -> dict[str, tuple]: + return {c: (dims, np.full(shape, value)) for c in SIMULATED_COMPONENTS} + + tree = xr.DataTree.from_dict( + { + "PGA": xr.Dataset( + components((n_stations,), ("station",), 0.2), coords=station_coords + ), + "CAV": xr.Dataset( + {"000": ("station", np.full(n_stations, 3.0))}, coords=station_coords + ), + "pSA": xr.Dataset( + components((n_stations, len(PERIODS)), ("station", "period"), 0.5), + coords=station_coords | {"period": PERIODS}, + ), + "FAS": xr.Dataset( + { + "eas": ( + ("station", "frequency"), + np.ones((n_stations, len(frequencies))), + ) + }, + coords=station_coords | {"frequency": list(frequencies)}, + ), + "pSA/empirical/NSHM2022": xr.Dataset( + { + "mean": ( + ("station", "period"), + np.zeros((n_stations, len(PERIODS))), + ), + "std_Total": ( + ("station", "period"), + np.full((n_stations, len(PERIODS)), 0.6), + ), + }, + coords=station_coords | {"period": PERIODS}, + attrs={ + "intensity_measure": "pSA", + "model": "NSHM2022", + "component": "rotd50", + }, + ), + "CAV/empirical/BCH_16": xr.Dataset( + { + "mean": ("station", np.full(n_stations, np.log(2.0))), + "std_Total": ("station", np.full(n_stations, 0.4)), + }, + coords=station_coords, + attrs={ + "intensity_measure": "CAV", + "model": "BCH_16", + "component": "geom", + }, + ), + } + ) + tree.attrs = { + "event": name, + "magnitude": 6.0, + "rake": 90.0, + "dip": 45.0, + "ztor": 1.0, + "zbot": 15.0, + "hypo_lat": -40.5, + "hypo_lon": 174.5, + "hypo_depth": 8.0, + "source": "POLYGON ((174 -41, 175 -41, 175 -40, 174 -41))", + "trace": "LINESTRING (174 -41, 175 -41)", + "domain": "POLYGON ((173 -42, 176 -42, 176 -39, 173 -42))", + "tect_type": "active_shallow", + } + tree.to_netcdf(path) + return path + + +@pytest.fixture +def imdb_ffp(tmp_path: Path) -> Path: + """An IMDB of two realisations of one event and one of another.""" + output_ffp = tmp_path / "imdb.duckdb" + create_imdb.create_imdb( + output_ffp, + [ + write_im_file(tmp_path / "b.h5", "Beta"), + write_im_file(tmp_path / "a2.h5", "Alpha_R2", frequencies=(0.5,)), + write_im_file( + tmp_path / "a1.h5", + "Alpha_R1", + supergrid_depth=(0.0, 0.0, 400.0), + ), + ], + ) + return output_ffp + + +def test_groups_realisations_into_events(imdb_ffp: Path) -> None: + with IMDB(imdb_ffp) as db: + assert db.validate() == [] + assert db.get_events().index.tolist() == ["Alpha", "Beta"] + assert db.get_realisations()["event_int_id"].to_dict() == { + "Alpha_R1": 0, + "Alpha_R2": 0, + "Beta": 1, + } + assert len(db.get_sites()) == len(STATIONS) + assert len(db.get_site_event()) == 2 * len(STATIONS) + + +def test_describes_the_source_and_sites(imdb_ffp: Path) -> None: + with IMDB(imdb_ffp) as db: + event = db.get_events().loc["Alpha"] + sites = db.get_sites() + assert event["tect_type"] == "ACTIVE_SHALLOW" + assert event["magnitude"] == pytest.approx(6.0) + assert sites["is_real"].to_dict() == {"WEL": True, "AKL1": True, "0a1b2c3": False} + assert sites.loc["AKL1", "z1p0"] == pytest.approx(0.2) + + +def test_leaves_out_simulations_in_the_absorbing_layer(imdb_ffp: Path) -> None: + with IMDB(imdb_ffp) as db: + simulated = db.get_records(rel_ids=["Alpha_R1"], kind="simulated") + empirical = db.get_records(rel_ids=["Alpha_R1"], kind="gmm") + assert "0a1b2c3" not in simulated["site_id"].values + assert "0a1b2c3" in empirical["site_id"].values + + +def test_stores_empirical_medians_on_their_components(imdb_ffp: Path) -> None: + with IMDB(imdb_ffp) as db: + psa = db.get_psa(sigma=True, rel_ids=["Beta"], gmm_key="NSHM2022") + cav = db.get_scalars(["CAV"], sigma=True, rel_ids=["Beta"], gmm_key="BCH_16") + components = db.get_records(rel_ids=["Beta"], kind="gmm").groupby("gmm_key")[ + "component" + ] + assert np.allclose(psa[["0.1", "1.0"]], 1.0) + assert np.allclose(psa[["0.1_sigma", "1.0_sigma"]], 0.6) + assert np.allclose(cav["CAV"], 2.0) + assert np.allclose(cav["CAV_sigma"], 0.4) + assert components.unique().to_dict() == {"BCH_16": ["geom"], "NSHM2022": ["rotd50"]} + + +def test_records_hold_only_the_ims_of_their_component(imdb_ffp: Path) -> None: + with IMDB(imdb_ffp) as db: + records = db.get_records(rel_ids=["Beta"], kind="simulated") + scalars = db.get_scalars(rel_ids=["Beta"], kind="simulated") + fas = db.get_fas(rel_ids=["Beta"], kind="simulated") + eas = records.index[records["component"] == "eas"] + assert set(fas.index) == set(eas) + assert scalars.index.intersection(eas).empty + cav = scalars["CAV"].dropna() + assert set(records.loc[cav.index, "component"]) == {"000"} + + +def test_places_spectra_on_a_shared_grid(imdb_ffp: Path) -> None: + with IMDB(imdb_ffp) as db: + fas = db.get_fas(rel_ids=["Alpha_R2"], kind="simulated") + assert np.allclose(fas["0.5"], 1.0) + assert fas["1.0"].isna().all() + + +def test_rejects_repeated_realisation_names(tmp_path: Path) -> None: + im_ffps = [ + write_im_file(tmp_path / "1.h5", "Alpha_R1"), + write_im_file(tmp_path / "2.h5", "Alpha_R1"), + ] + with pytest.raises(ValueError, match="realisation name"): + create_imdb.create_imdb(tmp_path / "imdb.duckdb", im_ffps) + assert not (tmp_path / "imdb.duckdb").exists() diff --git a/tests/test_im_calc.py b/tests/test_im_calc.py index cd7f1ac6..20c0f9b9 100644 --- a/tests/test_im_calc.py +++ b/tests/test_im_calc.py @@ -22,3 +22,7 @@ def test_cav5_uses_a_5_cm_per_s_squared_threshold() -> None: cav5_function = function_map[IM.CAV5] assert isinstance(cav5_function, functools.partial) assert cav5_function.keywords["threshold"] == 5.0 + + +def test_every_empirical_im_has_a_component() -> None: + assert set(im_calc.EMPIRICAL_COMPONENTS) == set(im_calc.EMPIRICAL_IM_NAMES) diff --git a/uv.lock b/uv.lock index 3509b8c9..237d9750 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.13" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -64,6 +64,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/1b/54f4ad77cd8a584fa70746c47df988e002cf1ee1eba43364d46f87803647/asgiref-3.12.1-py3-none-any.whl", hash = "sha256:fe386d1c2bff7259ea95929266d12a8cf9a8b5a1c2598402967d8792e7a7c094", size = 25478, upload-time = "2026-07-14T09:56:16.926Z" }, ] +[[package]] +name = "atpublic" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/3f/23b2643edfae61210baee60eec95873a4ad4fc6a7c096a725f240a0bf4db/atpublic-9.0.0.tar.gz", hash = "sha256:61ea62d8445d2aaa83b6dffaa3d90f99fcec10e16683ee9b13792cdcdafa0966", size = 27443, upload-time = "2026-10-13T01:49:05.987Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/d1/875c831006b60a9b93d8d5aba734fde33402d9136785d824fa0ba8765731/atpublic-9.0.0-py3-none-any.whl", hash = "sha256:449c3c4f0c74df79749d6fe225ba55e2a2fce34b303f0329211e4d6989ed6f6e", size = 11111, upload-time = "2026-10-13T01:49:05.07Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -782,6 +791,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, ] +[[package]] +name = "duckdb" +version = "1.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/0b/d65ea3be00ea79aa276a8388bec588a9cbf409ce637c6d306e5316210d15/duckdb-1.5.6.tar.gz", hash = "sha256:166a91dbfacfc0c9f08cc76c0243cb6d3d4296bfab5bad72a3cfb63140a5b7c8", size = 18032957, upload-time = "2026-09-28T13:38:37.978Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/5e/a476197fcba557738a588ec844747a19bc0a24b0e6f1809e308f29d68c0e/duckdb-1.5.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ae352646374cacf48e9981cf031191c494865192fc436d13667a2531fc5d1da3", size = 32810376, upload-time = "2026-09-28T13:38:05.148Z" }, + { url = "https://files.pythonhosted.org/packages/0c/6d/5466a2b53ddd557644dfa47a763f68748efccdf282e6ae7c4f1bcfb3da69/duckdb-1.5.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a1261e90785e9d29953293e44f60fa073bd1137098924e8de21a037a861b051", size = 17405385, upload-time = "2026-09-28T13:38:07.363Z" }, + { url = "https://files.pythonhosted.org/packages/d4/a0/bf87071170835ee4a34fe764fc11c1c6e7040a0e021b36c1b6f834a4c22f/duckdb-1.5.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97dd7a555b8f5298b76bc7d48a11cb2c64336e8de9bfde783cffb86ea9f54807", size = 15533132, upload-time = "2026-09-28T13:38:09.681Z" }, + { url = "https://files.pythonhosted.org/packages/31/e0/38095c8e140ecfbe847519ac07bcba94301b8fbb76b2870015e33e07f179/duckdb-1.5.6-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:364992ba1089a2b327391cfcb68fd0bd0ce9090cf293baef861a0ba6847abfee", size = 19454994, upload-time = "2026-09-28T13:38:11.836Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/61dd2876bbaa69cf77d7b5c620e52e8b25faae7096f4d2e4a812b52095d7/duckdb-1.5.6-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:644f54ce99b3b61844bc9a3fe80e0aecb1ea4084b1fffc4396d1569db6111679", size = 21568700, upload-time = "2026-09-28T13:38:14.258Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/100730e7785e85268be4d4d5bd62cfc8314e261d2f42efa208243eef35cb/duckdb-1.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:ced693d33ddcee2e5345f077d342c87d2aaa80e41c514e64c9ff2d4e5963c251", size = 13190707, upload-time = "2026-09-28T13:38:16.875Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2e/bc7f44eab4e89ee5c1cb427bb1168ad021d985042e6841ec0694c3d3d501/duckdb-1.5.6-cp313-cp313-win_arm64.whl", hash = "sha256:41ecc75bb9328d72d154a705c1a653d2c5c60f686a5c0c6578aa80020753c884", size = 14020962, upload-time = "2026-09-28T13:38:19.007Z" }, + { url = "https://files.pythonhosted.org/packages/fb/62/a8a30a4c6b94c0861d348ed5633b963f6745a5525527530f02f3c1a7c931/duckdb-1.5.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:aa21d2ad803b2524326e8622d7d96b2bb1ff1d5b60368e1978ee805df9c21fb3", size = 32828003, upload-time = "2026-09-28T13:38:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/71/b7/1dcca0005eb8c67adf9fc06bf0cbb1d2bf4ea1974cc89e7a7c2ad66aac28/duckdb-1.5.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8a1b2ad27d414068cbca06c55cfa802eece10f86ea4812ff082f8ab4cb25fc85", size = 17413912, upload-time = "2026-09-28T13:38:23.915Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/e3ac175443550f3464f2d95731a8b0aae9b4dc3875c3a186c352262b43c2/duckdb-1.5.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c79c6d222b1d015cde73b5139087186b00db65357fb4e2c94c2308fbbf465a72", size = 15543122, upload-time = "2026-09-28T13:38:26.317Z" }, + { url = "https://files.pythonhosted.org/packages/9d/08/cc510a7952aba69d5cdca17f3ef61c95713d86143f2ee9aa3e097d38f50b/duckdb-1.5.6-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1052b8050ef5696e2c0d8c836949c72f3dd11f0690466acbea739613e8e2750b", size = 19457946, upload-time = "2026-09-28T13:38:28.877Z" }, + { url = "https://files.pythonhosted.org/packages/ef/a5/6f8099d9a5a02ddff89e5c85875df3465054845b0920fb0703fbdf8dd2ec/duckdb-1.5.6-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19c5e485e59613b8878d1670bcaa7a010f53c5a4da5ae8e08863e5e529ca6182", size = 21575132, upload-time = "2026-09-28T13:38:31.231Z" }, + { url = "https://files.pythonhosted.org/packages/9f/58/762f7159662d7859e201fa05ca29f306795daeabf84f3e087215a966b001/duckdb-1.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:ebcbd09cd8578ab1093393e9b16289cda0e8f1791ac595bf00eb5bad75c3cf00", size = 13713963, upload-time = "2026-09-28T13:38:33.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/69/64d165db322de13f5c3e75d377b6b9694df1821155ad1fa4b14b04601abc/duckdb-1.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:820a8384faef11cd86068ea48c5da57ce2d8f1c7b3d2bdb9be3398317a7c3728", size = 14514368, upload-time = "2026-09-28T13:38:35.676Z" }, +] + [[package]] name = "fiona" version = "1.10.1" @@ -1170,6 +1201,35 @@ numpy = [ { name = "numpy" }, ] +[[package]] +name = "ibis-framework" +version = "12.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "atpublic" }, + { name = "parsy" }, + { name = "python-dateutil" }, + { name = "sqlglot" }, + { name = "toolz" }, + { name = "typing-extensions" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/8e/2e7ad9bdeaf45350da7beeb67a0d4317d400dac882825eb7c3bd4d3c6ae1/ibis_framework-12.0.0.tar.gz", hash = "sha256:238624f2c14fdab8382ca2f4f667c3cdb81e29844cd5f8db8a325d0743767c61", size = 1351369, upload-time = "2026-02-07T14:31:13.171Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/b3/11d406849715b47c9d69bb22f50874f80caee96bd1cbe7b61abbebbf5a05/ibis_framework-12.0.0-py3-none-any.whl", hash = "sha256:0bbd790f268da9cb87926d5eaad2b827a573927113c4ed3be5095efa89b9e512", size = 2079219, upload-time = "2026-02-07T14:31:10.646Z" }, +] + +[package.optional-dependencies] +duckdb = [ + { name = "duckdb" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyarrow-hotfix" }, + { name = "rich" }, +] + [[package]] name = "idna" version = "3.20" @@ -2177,42 +2237,55 @@ wheels = [ [[package]] name = "pandas" -version = "2.3.3" +version = "3.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/4b/18b035ee18f97c1040d94debd8f2e737000ad70ccc8f5513f4eefad75f4b/pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713", size = 11544671, upload-time = "2025-09-29T23:21:05.024Z" }, - { url = "https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8", size = 10680807, upload-time = "2025-09-29T23:21:15.979Z" }, - { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872, upload-time = "2025-09-29T23:21:27.165Z" }, - { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371, upload-time = "2025-09-29T23:21:40.532Z" }, - { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333, upload-time = "2025-09-29T23:21:55.77Z" }, - { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120, upload-time = "2025-09-29T23:22:10.109Z" }, - { url = "https://files.pythonhosted.org/packages/4f/c7/e54682c96a895d0c808453269e0b5928a07a127a15704fedb643e9b0a4c8/pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee", size = 10993991, upload-time = "2025-09-29T23:25:04.889Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ca/3f8d4f49740799189e1395812f3bf23b5e8fc7c190827d55a610da72ce55/pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5", size = 12048227, upload-time = "2025-09-29T23:22:24.343Z" }, - { url = "https://files.pythonhosted.org/packages/0e/5a/f43efec3e8c0cc92c4663ccad372dbdff72b60bdb56b2749f04aa1d07d7e/pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21", size = 11411056, upload-time = "2025-09-29T23:22:37.762Z" }, - { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189, upload-time = "2025-09-29T23:22:51.688Z" }, - { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912, upload-time = "2025-09-29T23:23:05.042Z" }, - { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160, upload-time = "2025-09-29T23:23:28.57Z" }, - { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, - { url = "https://files.pythonhosted.org/packages/04/fd/74903979833db8390b73b3a8a7d30d146d710bd32703724dd9083950386f/pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0", size = 11540635, upload-time = "2025-09-29T23:25:52.486Z" }, - { url = "https://files.pythonhosted.org/packages/21/00/266d6b357ad5e6d3ad55093a7e8efc7dd245f5a842b584db9f30b0f0a287/pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593", size = 10759079, upload-time = "2025-09-29T23:26:33.204Z" }, - { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" }, - { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" }, - { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" }, - { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" }, - { url = "https://files.pythonhosted.org/packages/a6/3d/124ac75fcd0ecc09b8fdccb0246ef65e35b012030defb0e0eba2cbbbe948/pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5", size = 11109071, upload-time = "2025-09-29T23:32:27.484Z" }, - { url = "https://files.pythonhosted.org/packages/89/9c/0e21c895c38a157e0faa1fb64587a9226d6dd46452cac4532d80c3c4a244/pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec", size = 12048504, upload-time = "2025-09-29T23:29:31.47Z" }, - { url = "https://files.pythonhosted.org/packages/d7/82/b69a1c95df796858777b68fbe6a81d37443a33319761d7c652ce77797475/pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7", size = 11410702, upload-time = "2025-09-29T23:29:54.591Z" }, - { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" }, - { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" }, - { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" }, - { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/17/d7b106e05bfa642e8694451e7d3d759c6a241c5386a5d962e4f66c047e06/pandas-3.0.6.tar.gz", hash = "sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10", size = 4667686, upload-time = "2026-09-17T23:23:18.345Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/1c/143605a1f6443ad50ebda78a31e5a3a10147fec2590e931584aaa5ff0a09/pandas-3.0.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ae8073aed8e21d1a7fe263dcdc6840743549722a6738198a0a46000fa9476f2", size = 10418900, upload-time = "2026-09-17T23:21:16.594Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ca/87f8548f73d452aab35e4a90f8b39ae303295e0f2ef0b4055c44d6b3f1be/pandas-3.0.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:60d81f9e1799b36f3739e7fff44d1fbb2e8fd5a271b3863e03de9715fccda0fa", size = 10064785, upload-time = "2026-09-17T23:21:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/43/1a/d951442e5607c6e3b2462eff8f420797d428aa74b87c6ecfe4f48553626e/pandas-3.0.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:097090508a1dd335013d39106fc10b20f4fd4a171638e47b77d55798ed9dab6c", size = 10245290, upload-time = "2026-09-17T23:21:22.797Z" }, + { url = "https://files.pythonhosted.org/packages/50/fa/96d50e1e6cd0b08b5e2b7c838f65ae644940f75a124063380b5ef73b6866/pandas-3.0.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e92d9fa834c7d877130027cddc0cad8dcff97c1f6cca26bd6310f847228b658", size = 10757657, upload-time = "2026-09-17T23:21:25.673Z" }, + { url = "https://files.pythonhosted.org/packages/7b/12/f82d13a2cb703e1a8acee7e01fdc2b898d9cd0c00f07d1dfce63af43e350/pandas-3.0.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b27c8d890e4aa2171437ae2a39de1d215e674158e4865c4023a8b31c932513b2", size = 11249114, upload-time = "2026-09-17T23:21:28.898Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ce/8aef2e561a2f2c8b38c913c67373c65ba6748174e763d27c80271b24bd17/pandas-3.0.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8029ec0f1f89e4f985929ce1f6626dabf3140d61a4e9c1215afdab34eaf9a5d", size = 11820511, upload-time = "2026-09-17T23:21:32.11Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bd/63cb67e6903ef6d9c2871916dbcbc09d254da0fe8b870cf62e16b21945f2/pandas-3.0.6-cp313-cp313-win_amd64.whl", hash = "sha256:f3ce8a6968045481e91a3990e797e348ce13db45ee164a7095bbc824e26c09dd", size = 9638092, upload-time = "2026-09-17T23:21:34.883Z" }, + { url = "https://files.pythonhosted.org/packages/75/2e/e7b35b712edb068d382ddc8b2bea8a04974100515ba2daa22b478b265842/pandas-3.0.6-cp313-cp313-win_arm64.whl", hash = "sha256:cc39303913e2ea129915670de5d1c9fbd647f543bb72e5543bac8baa94e9e42f", size = 8952032, upload-time = "2026-09-17T23:21:37.729Z" }, + { url = "https://files.pythonhosted.org/packages/75/55/1a8875395b05ccd572cbca0b9255dcd2db6e6508e632a558c1a6884b39ad/pandas-3.0.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ee913a91669056c1de1a6b733fbfeab711de9e54e3bee2dfa5fe79d9457247d1", size = 10492213, upload-time = "2026-09-17T23:21:40.746Z" }, + { url = "https://files.pythonhosted.org/packages/35/61/47ae13476995cc8a40cd609e93e7cf11f273d8692925c2903cb6d38aa0d1/pandas-3.0.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ff51a4459ed036e93d1eb1bb5e6e7b28685d3cb6b7c12b91c05b31024e234729", size = 10156618, upload-time = "2026-09-17T23:21:44.142Z" }, + { url = "https://files.pythonhosted.org/packages/bc/f2/cc5f2adb8d6e86a85d9fb5128f8cf205a61189336f70d1f7faf0d1b53ec9/pandas-3.0.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:654aae059295dbba6ecd2328ca12712a2cf1676214c8699f1c29213f7ccf9c34", size = 10375479, upload-time = "2026-09-17T23:21:47.159Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ba/ffdcb19be4ff6bfe7d969e7cef2c567c633df5a3a1cc1053394ad053bca8/pandas-3.0.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f51d7f651c8054c5e82a69265c98082e795d1442df7ca6edc3a545d61214b1", size = 10783244, upload-time = "2026-09-17T23:21:50.367Z" }, + { url = "https://files.pythonhosted.org/packages/77/5b/e150075b2c6eb69fae896f2d9239bc6ed07db97735971d53d66de6553460/pandas-3.0.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:22172a92e7ee678ec0140c7af4fc9366b55413834a1cd86af78b3caa0b0574de", size = 11382223, upload-time = "2026-09-17T23:21:53.355Z" }, + { url = "https://files.pythonhosted.org/packages/d6/8a/b441c587dc7355bf6e1f68a91b4f76a6c29740f0c23be3acc5d4ebbeea6d/pandas-3.0.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:583be68728a31d0d750d5b8d9e00f02b153df0d4655f858bde93cb84cfc4227c", size = 11854881, upload-time = "2026-09-17T23:21:56.342Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e9/f43410fada510b43fec09993c08f552086c3d247d3ee801a678f3cb10ea5/pandas-3.0.6-cp314-cp314-win_amd64.whl", hash = "sha256:77ccbe5057aece6fc172b9b77f19c04335af6882bc2e10c8f3ee4e6bfb3da553", size = 9791672, upload-time = "2026-09-17T23:21:59.332Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9e/db14c059c21f9baa1907d436f8bf30e0c76c6288225c5e8b79a08ba8b2c5/pandas-3.0.6-cp314-cp314-win_arm64.whl", hash = "sha256:fb625f426b375bcc96e3a04c5d5d266cd7be6ae5d6866e0e703382ab5164068c", size = 9121246, upload-time = "2026-09-17T23:22:02.123Z" }, + { url = "https://files.pythonhosted.org/packages/67/ba/bad0f8dac020ab38a8637fddab01a57a82da7a496a6e6f19590aad53ab62/pandas-3.0.6-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9e492cd4bdba6778de4fe0df7f4590c012161ebcf9902dce01b01dc683105514", size = 10920612, upload-time = "2026-09-17T23:22:05.404Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a9/b500982e9aac6d52a58da4ad3f11e14168a315b06906b3f397c427878065/pandas-3.0.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d7dcd21238cbb4828ff148481ba01cac8946dc5121457b5aeba28636f8f99a60", size = 10570213, upload-time = "2026-09-17T23:22:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fa/e6ecd0073c98be8f840ac3125b955272835d7d9fd69f5944383b164deb5e/pandas-3.0.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ff482fa91fa2bafd92e8fe66ce3645c851824310f295c1f0a2f96e928fc4541", size = 10252356, upload-time = "2026-09-17T23:22:11.302Z" }, + { url = "https://files.pythonhosted.org/packages/04/f5/001e230a7a7803590d9275a1a3f7e1bb605e3a495cfe5e8d3a532090621b/pandas-3.0.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:db7ec631f26223beee8e5c9e0b8f23c24d8197bbd1d982421d4e3188bea51965", size = 10655952, upload-time = "2026-09-17T23:22:14.283Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/bab587148a3852c96aae26c4b5f9e04ce2221801ad94e166b4fbf969ede0/pandas-3.0.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd75ed0c840f709fc2ae26ddd9534ac77ca1a48ac0cce521a74acaa85f3340a7", size = 11274177, upload-time = "2026-09-17T23:22:17.352Z" }, + { url = "https://files.pythonhosted.org/packages/f3/32/74b48d87df2b80892d713c149abfe36d5db4de41b4eccb042a2bc07dafc1/pandas-3.0.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ef738d71d1059245b6bb03e312be06d8b3821326a83486c1ad03b9aba3710e44", size = 11719041, upload-time = "2026-09-17T23:22:20.227Z" }, + { url = "https://files.pythonhosted.org/packages/f6/c6/d64b72d64d7eb0fad9fe424d34138e45dee70ddbea1360dcd0adf30e28f6/pandas-3.0.6-cp314-cp314t-win_amd64.whl", hash = "sha256:429d9df32731ab01383ed98f2baa7a60368090d1a94fc06019a12062510e8630", size = 10191388, upload-time = "2026-09-17T23:22:23.524Z" }, + { url = "https://files.pythonhosted.org/packages/7a/30/5e5b2ccabeca73ae2b03fc82bca3eabb7466cf43737f05ac08d591665d47/pandas-3.0.6-cp314-cp314t-win_arm64.whl", hash = "sha256:a4dbd4dc65cbe645b92b8785d0f96dd7311010dc6606cf620e51b07b8788a12a", size = 9387796, upload-time = "2026-09-17T23:22:26.64Z" }, + { url = "https://files.pythonhosted.org/packages/b6/77/47c5fb0be8bdd00116814c2c40d9ec42dbeb943865ef95130fe58a898226/pandas-3.0.6-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:50c44cbf5820b6b91a5f74aae04972472aefadd3cd9fbd1010409d85528bd570", size = 10483960, upload-time = "2026-09-17T23:22:30.071Z" }, + { url = "https://files.pythonhosted.org/packages/09/08/a310cb2fefe6d2b4623ab2150da818d93d4e73e33b58e4d163bb74243c5a/pandas-3.0.6-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:eb6900de08ac85f93ac4948aa6b80842eba555875337b8359035ac9c43e92d34", size = 10149432, upload-time = "2026-09-17T23:22:32.818Z" }, + { url = "https://files.pythonhosted.org/packages/54/36/6af478ec3a26d7754555cd62c3101c589c0931b1d85398e4fa5403910a1c/pandas-3.0.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e25e2e1adee99ddfada6f7206a79ae8e9c8a8861b0e3eaaba165006d3eef18e", size = 10360981, upload-time = "2026-09-17T23:22:35.621Z" }, + { url = "https://files.pythonhosted.org/packages/43/20/5ece0e9cb79a6473216620db6a90546f578001c2bd857b77c444b27acad1/pandas-3.0.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ff44b2cb51cbd691c91f92c4ea6c71e34003f239ebd67c2e857dc898466b49c", size = 10776585, upload-time = "2026-09-17T23:22:38.427Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b6/cd3038f31ade5e8d2b4e1c9549d4b31e4d598469562f47142b2ad9171c0a/pandas-3.0.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:5edd0a7abb0986ecce1ac81f56d99b6763f86aa6946dceb6c661224f90af5a19", size = 11385249, upload-time = "2026-09-17T23:22:41.18Z" }, + { url = "https://files.pythonhosted.org/packages/0a/87/05bb3003737f80375d7311774916a24d161e2e591abe8c672aed7813defc/pandas-3.0.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:1bcb3e9ed29e74a7439cedff9e2aefd3ea65de84d7de9ccb6c194192541bd60e", size = 11848595, upload-time = "2026-09-17T23:22:44.207Z" }, + { url = "https://files.pythonhosted.org/packages/c0/30/1c0d46acf236d19ef975e9cdd5d1e0dd54924b03f0ed8287096ad4a18152/pandas-3.0.6-cp315-cp315-win_amd64.whl", hash = "sha256:253e12cb9081b0afbac607920f6142975966bc315135e09de275fdbaa415d2de", size = 9783810, upload-time = "2026-09-17T23:22:47.097Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/df3304a9c2833c4810e7f1c887b04105b24731f9deef8b5d5d04522a375b/pandas-3.0.6-cp315-cp315-win_arm64.whl", hash = "sha256:97274c9adf6255bb48c620cd6959805efa7f09ea2167f0e0ae006a448cd2fca7", size = 9116820, upload-time = "2026-09-17T23:22:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/47/25/d5f6cce5efa17c38e4f752a875b4a26d66cfadd529cb8c81672f0376d6a6/pandas-3.0.6-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:265f562fdd1079f69f3de96dd425c3405224038c0af4f920c54bd240ee2c4640", size = 10894614, upload-time = "2026-09-17T23:22:53.223Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8b/e1876bfdc1df06bafc022d33202b5663bd86c81df2a6140aacafd0344669/pandas-3.0.6-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c6e4aae3e9bea26c6c9a20d88d96c86ec4a99b4db5fd516bcb4e829ab2c0ee36", size = 10546973, upload-time = "2026-09-17T23:22:56.155Z" }, + { url = "https://files.pythonhosted.org/packages/e9/27/e0a27a5a5c27f7db66657b44def9e93121fd0ad4f0808355b9898fcbf204/pandas-3.0.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a77a1a44e4d88f1c6a2a64d3eb12efec8420875722e14279800b173a7c7c2804", size = 10239717, upload-time = "2026-09-17T23:22:59.603Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4f/4eadb7d86a921c1e8bc70916cfe601ed667c4169118d17d69958611c21f8/pandas-3.0.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86fa853a12e0b70927e2b1ee00d56d2224ec9cbb4b9d58348b5ad52d2f21150e", size = 10641275, upload-time = "2026-09-17T23:23:02.81Z" }, + { url = "https://files.pythonhosted.org/packages/c7/d1/eba72e9d905e84e79aefeefdcc6bc1abe15d9077973566643f4211636966/pandas-3.0.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:c826e9babb7790142c399f58599d8de679bea059d7b39c5b6efa2096fac37266", size = 11262919, upload-time = "2026-09-17T23:23:06.038Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/1c5bd2642b450e374b6191189f47c49538fe41f82348a66de6f647e6ab59/pandas-3.0.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:8fe77b408d82e2615674dfed62533b95e18a03610573877422aada4f625d4947", size = 11709148, upload-time = "2026-09-17T23:23:09.082Z" }, + { url = "https://files.pythonhosted.org/packages/73/3d/1b142bd0d0f1326a5d98c91c955b923cd1e06b5eecb428cd9d8817fa01c0/pandas-3.0.6-cp315-cp315t-win_amd64.whl", hash = "sha256:83e91d15738d7783c050197cef2f2cf82fc6353dae9865aa87ed1fa16aa4d55a", size = 10161518, upload-time = "2026-09-17T23:23:12.365Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a3/6419c14da2adc1f09a6a183b8f91d7494d325b287f4ca984ac04f663638a/pandas-3.0.6-cp315-cp315t-win_arm64.whl", hash = "sha256:963ca21199097a84c7827c4678b04e30833084fbf8ef44fde3fa7180a29f8fa0", size = 9364529, upload-time = "2026-09-17T23:23:15.274Z" }, ] [package.optional-dependencies] @@ -2244,6 +2317,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/07/5c578992b965e9c5101c7ad377440ca250769b34542a1781ab06b25c56fc/parse-1.22.3-py2.py3-none-any.whl", hash = "sha256:2cd33a301b5a4b400ee79952f42364fe486e5f10701fbf819fbbab1eab478139", size = 21221, upload-time = "2026-10-10T17:29:39.676Z" }, ] +[[package]] +name = "parsy" +version = "2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/58/1e3f382eef9e50a2a115486b0c178d22bb97d2fbb85421ccbe5d3a783530/parsy-2.2.tar.gz", hash = "sha256:e943147644a8cf0d82d1bcb5c5867dd517495254cea3e3eb058b1e421cb7561f", size = 47296, upload-time = "2025-09-12T11:39:26.783Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/fc/8cb9073bb1bee54eb49a1ae501a36402d01763812962ac811cdc1c81a9d7/parsy-2.2-py3-none-any.whl", hash = "sha256:5e981613d9d2d8b68012d1dd0afe928967bea2e4eefdb76c2f545af0dd02a9e7", size = 9538, upload-time = "2025-09-12T11:39:25.749Z" }, +] + [[package]] name = "partd" version = "1.4.2" @@ -2452,6 +2534,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/be/7b81a44d6a8e70581dcc1d6f01541f9000a973b1e5d75394aec91e7b179a/pyarrow-26.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:68cd662e9e2b00876a131950cf32336ace2d0865e1f9418763e3d3be8481dfa4", size = 29389953, upload-time = "2026-10-09T08:26:18.277Z" }, ] +[[package]] +name = "pyarrow-hotfix" +version = "0.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/ed/c3e8677f7abf3981838c2af7b5ac03e3589b3ef94fcb31d575426abae904/pyarrow_hotfix-0.7.tar.gz", hash = "sha256:59399cd58bdd978b2e42816a4183a55c6472d4e33d183351b6069f11ed42661d", size = 9910, upload-time = "2025-04-25T10:17:06.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/c3/94ade4906a2f88bc935772f59c934013b4205e773bcb4239db114a6da136/pyarrow_hotfix-0.7-py3-none-any.whl", hash = "sha256:3236f3b5f1260f0e2ac070a55c1a7b339c4bb7267839bd2015e283234e758100", size = 7923, upload-time = "2025-04-25T10:17:05.224Z" }, +] + [[package]] name = "pycparser" version = "3.11" @@ -2732,15 +2823,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] -[[package]] -name = "pytz" -version = "2026.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/14/21/d83d6ef28c4c912c4bb4d1dcf591f7b8c6bde87b9c66f9f454677314e16d/pytz-2026.5.tar.gz", hash = "sha256:fa23724b9c486543b9ff54a327ee7569ac83ade54bb9afd0fc18676620401c86", size = 318572, upload-time = "2026-10-04T02:37:58.719Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, -] - [[package]] name = "pyyaml" version = "6.0.3" @@ -3218,6 +3300,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/54/470f640665696656d336482d2c9492ef68c34e60e90a08aaf504ac1c8162/source_modelling-2026.10.3-cp315-cp315t-win_amd64.whl", hash = "sha256:a30ad1247bbfe7e88d6fb917cdefde4c6e2d0f362922cca6b9ed3508d29f89d6", size = 1018323, upload-time = "2026-10-15T23:35:28.989Z" }, ] +[[package]] +name = "sqlglot" +version = "30.24.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/7c/9fd907983ff041692399bf855d344e1fcc9a02c8d92a5fee6dbdf139d19e/sqlglot-30.24.0.tar.gz", hash = "sha256:6451aabd1c9994d0f73355c962a5eaf9258381b3f2554e53db916cd9a605552a", size = 6131857, upload-time = "2026-10-20T17:14:03.987Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/5f/d78f74306cc026c5f40f8808a47356dfd37461eb15eb759a20e5e9efe7d7/sqlglot-30.24.0-py3-none-any.whl", hash = "sha256:9488828cc018453b7a3ba01e65d2749c9a731314c14583695163ca1f358203fe", size = 792162, upload-time = "2026-10-20T17:14:02.053Z" }, +] + [[package]] name = "sqlparse" version = "0.6.0" @@ -3498,6 +3589,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, ] +[[package]] +name = "ucgmsim-imdb" +version = "2026.9.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ibis-framework", extra = ["duckdb"] }, + { name = "numpy" }, + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/d4/0c33d3795c5935731edb4d4c7a5c5aec32f3b2972a67e20a4505aa12874b/ucgmsim_imdb-2026.9.2.tar.gz", hash = "sha256:e6a239cdf16c18dc354d2f12dc74deeb263c04bacdc1b21cc6666c9f8e8a5a51", size = 93903, upload-time = "2026-09-18T03:05:40.622Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/92/fafb091d00cc7641b6dff813c54d64dc36d30dbfea9b9048cc73c78bee49/ucgmsim_imdb-2026.9.2-py3-none-any.whl", hash = "sha256:72efdf73397bc8292539882172e7d21a40be4976ba08b65af84a248afc4dda88", size = 14007, upload-time = "2026-09-18T03:05:39.52Z" }, +] + [[package]] name = "ucgmsim-srf" version = "2026.10.2" @@ -3563,6 +3668,7 @@ dependencies = [ { name = "tomli-w" }, { name = "tqdm" }, { name = "typer" }, + { name = "ucgmsim-imdb" }, { name = "velocity-modelling" }, { name = "xarray", extra = ["io"] }, ] @@ -3631,7 +3737,8 @@ requires-dist = [ { name = "typer" }, { name = "types-geopandas", marker = "extra == 'types'" }, { name = "types-requests", marker = "extra == 'types'" }, - { name = "velocity-modelling", specifier = ">=2026.10.1" }, + { name = "ucgmsim-imdb", specifier = ">=2026.9.2" }, + { name = "velocity-modelling", specifier = ">=2026.10.2" }, { name = "xarray", extras = ["io"] }, ] provides-extras = ["test", "types", "dev"] @@ -3659,7 +3766,7 @@ wheels = [ [[package]] name = "velocity-modelling" -version = "2026.10.1" +version = "2026.10.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cartopy" }, @@ -3677,9 +3784,9 @@ dependencies = [ { name = "typer" }, { name = "xarray", extra = ["io"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/59/2d/9a5dddc1486e4d75103a0affa96789eb39adb2e68bec8d9b9aa0f0db50c2/velocity_modelling-2026.10.1.tar.gz", hash = "sha256:232a271be90b76364700e6c3370f0178e709cdc382bd98c447b58d11b5d3218c", size = 111615, upload-time = "2026-10-08T20:24:22.444Z" } +sdist = { url = "https://files.pythonhosted.org/packages/35/08/2cef51c512e2ac5a2130a24d1b4f00ede0b7324f17540e14a9bdc425fc84/velocity_modelling-2026.10.2.tar.gz", hash = "sha256:642941addb8524ea64bb976065a8265cf266cd53bc93415f15cc28f9b8a25307", size = 111634, upload-time = "2026-10-20T21:11:23.728Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/af/0d1082bb31a9d5f57b5346a6087ab384468672ad810ca0e28377d059e98e/velocity_modelling-2026.10.1-py3-none-any.whl", hash = "sha256:9e9cad585a9466c7f189b8210a010ec6ae781e6532bae2f8e7d82b8774981ada", size = 131419, upload-time = "2026-10-08T20:24:21.094Z" }, + { url = "https://files.pythonhosted.org/packages/ac/6f/ad7afb737594a72b135d7541c297033f834044e4a3b216504e57c84f5a00/velocity_modelling-2026.10.2-py3-none-any.whl", hash = "sha256:2e2a8b60d8033e8dc80ac496c836c2702a3318bb27be3f7618a622be05958802", size = 131416, upload-time = "2026-10-20T21:11:22.208Z" }, ] [[package]] diff --git a/workflow/scripts/create_imdb.py b/workflow/scripts/create_imdb.py new file mode 100644 index 00000000..04cba685 --- /dev/null +++ b/workflow/scripts/create_imdb.py @@ -0,0 +1,482 @@ +"""IMDB Creation. + +Description +----------- +Collect the simulated and empirical intensity measures of many realisations +into one intensity measure database (IMDB). + +Inputs +------ +Intensity measure files from [Intensity Measure Calculation](#intensity-measure-calculation), +one per realisation, each identified by the realisation's name. Realisations +named like clones of one event (`_R`, see `clone-realisations`) are +grouped into that event; any other realisation is the only one of its event. + +Outputs +------- +A DuckDB database in the [IMDB](https://github.com/ucgmsim/imdb) schema, +holding a `simulated` record per station and component, and a `gmm` record +per station, ground motion model and component for the empirical estimates. +Simulated records are left out for stations inside the solver's absorbing +layer, where the waveforms are not ground-motion predictions. + +Environment +----------- +Can be run in the cybershake container. Can also be run from your own computer using the `create-imdb` command which is installed after running `pip install ucgmsim-workflow@git+https://github.com/ucgmsim/workflow`. + +Usage +----- +`create-imdb [OPTIONS] OUTPUT_FFP INTENSITY_MEASURE_FFPS...` + +For More Help +------------- +See the output of `create-imdb --help`. +""" + +from collections import Counter, defaultdict +from collections.abc import Iterator +from importlib import metadata +from pathlib import Path +from typing import Annotated, Any + +import numpy as np +import numpy.typing as npt +import pandas as pd +import parse +import typer +import xarray as xr + +from imdb import IMDB, schema +from qcore import cli +from workflow import log_utils, sw4 +from workflow.sites import OBSERVATION_STATION_PATTERN + +app = typer.Typer() + +REALISATION_NAME_TEMPLATE = "{event}_R{realisation:d}" +"""How realisations cloned from one event are named, as a `parse` template +(the `clone-realisations` default).""" + +SITE_COLUMNS = { + "latitude": "lat", + "longitude": "lon", + "vs30": "vs30", + "z1pt0": "z1p0", + "z2pt5": "z2p5", +} +"""Map from the station coordinates `im-calc` writes to IMDB site columns.""" + +SUPERGRID_DEPTH = sw4.SUPERGRID_DEPTH_COORDINATES["SGDEPTH"] +"""The station coordinate holding penetration into the absorbing layer.""" + + +def event_name(realisation_name: str) -> str: + """Identify the event a realisation belongs to. + + Parameters + ---------- + realisation_name : str + The realisation's name, which `im-calc` records as `event`. + + Returns + ------- + str + The event the realisation was cloned from, if its name follows + `REALISATION_NAME_TEMPLATE`, otherwise the realisation's own name. + """ + parsed = parse.parse(REALISATION_NAME_TEMPLATE, realisation_name) + return parsed["event"] if isinstance(parsed, parse.Result) else realisation_name + + +def survey( + intensity_measure_ffps: list[Path], +) -> tuple[ + list[tuple[str, str, Path]], npt.NDArray[np.float64], npt.NDArray[np.float64] +]: + """Identify the realisation in each intensity measure file, and the spectral grid covering them all. + + Parameters + ---------- + intensity_measure_ffps : list[Path] + The intensity measure files. + + Returns + ------- + realisations : list of (str, str, Path) + The event, realisation and file of each realisation, ordered by event. + periods : np.ndarray + The sorted union of every file's pSA periods. + frequencies : np.ndarray + The sorted union of every file's FAS frequencies. These differ + between realisations with different timesteps, because `im-calc` + stops at the Nyquist frequency. + + Raises + ------ + ValueError + If two files hold realisations of the same name. + """ + realisations = [] + periods = frequencies = np.array([], dtype=np.float64) + for intensity_measure_ffp in intensity_measure_ffps: + with xr.open_datatree(intensity_measure_ffp) as dtree: + rel_id = dtree.attrs["event"] + realisations.append((event_name(rel_id), rel_id, intensity_measure_ffp)) + if "pSA" in dtree: + periods = np.union1d(periods, dtree["pSA"].period.values) + if "FAS" in dtree: + frequencies = np.union1d(frequencies, dtree["FAS"].frequency.values) + + counts = Counter(rel_id for _, rel_id, _ in realisations) + if duplicates := [rel_id for rel_id, count in counts.items() if count > 1]: + raise ValueError( + "Intensity measure files share a realisation name (give each " + f"realisation its own `metadata.name`): {duplicates}" + ) + return sorted(realisations), periods, frequencies + + +def read_stations(dtree: xr.DataTree) -> pd.DataFrame: + """Read the per-station parameters `im-calc` attaches to every IM. + + Parameters + ---------- + dtree : xr.DataTree + The intensity measure tree. + + Returns + ------- + pd.DataFrame + The station location, site parameters, distances and absorbing layer + penetration, indexed by station name. + """ + dataset = next(node.dataset for node in dtree.subtree if node.has_data) + return pd.DataFrame( + { + name: coordinate.values + for name, coordinate in dataset.coords.items() + if coordinate.dims == ("station",) and name != "station" + }, + index=pd.Index(dataset.station.values, name="station"), + ) + + +def event_row(dtree: xr.DataTree, event_id: str) -> dict[str, Any]: + """Describe an event from the source summary `im-calc` records. + + Parameters + ---------- + dtree : xr.DataTree + The intensity measure tree of any realisation of the event. + event_id : str + The event. + + Returns + ------- + dict + A row of the IMDB `events` table. Dip direction and length are left + empty because `im-calc` reduces multi-fault ruptures to a single + moment-averaged source, for which neither is defined. The magnitude + is BoldM, as `im-calc` records it. + """ + attrs = dtree.attrs + tect_type = attrs.get("tect_type") + return { + "event_id": event_id, + "magnitude": attrs["magnitude"], + "tect_type": tect_type.upper() if tect_type else None, + "dip": attrs["dip"], + "dtop": attrs["ztor"], + "dbottom": attrs["zbot"], + "source_wkt": attrs["source"], + "trace_wkt": attrs["trace"], + "domain_wkt": attrs["domain"], + } + + +def realisation_row(dtree: xr.DataTree, rel_id: str, event_id: str) -> dict[str, Any]: + """Describe a realisation from the source summary `im-calc` records. + + Parameters + ---------- + dtree : xr.DataTree + The intensity measure tree of the realisation. + rel_id : str + The realisation. + event_id : str + The event the realisation belongs to. + + Returns + ------- + dict + A row of the IMDB `realisations` table. The magnitude is BoldM, as + `im-calc` records it. + """ + attrs = dtree.attrs + return { + "rel_id": rel_id, + "event_id": event_id, + "magnitude": attrs["magnitude"], + "rake": attrs["rake"], + "hypo_lat": attrs["hypo_lat"], + "hypo_lon": attrs["hypo_lon"], + "hypo_depth": attrs["hypo_depth"], + } + + +def site_rows(stations: pd.DataFrame) -> pd.DataFrame: + """Describe stations as IMDB sites. + + Parameters + ---------- + stations : pd.DataFrame + The per-station parameters, indexed by station name. + + Returns + ------- + pd.DataFrame + Rows of the IMDB `sites` table. + """ + sites = stations[list(SITE_COLUMNS)].rename(columns=SITE_COLUMNS) + sites["is_real"] = sites.index.str.fullmatch(OBSERVATION_STATION_PATTERN) + return sites.rename_axis("site_id").reset_index() + + +def site_event_rows(stations: pd.DataFrame, event_id: str) -> pd.DataFrame: + """Describe the distances from stations to an event's rupture. + + Parameters + ---------- + stations : pd.DataFrame + The per-station parameters, indexed by station name. + event_id : str + The event. + + Returns + ------- + pd.DataFrame + Rows of the IMDB `site_event` table. `rx` and `ry` are empty for + ruptures with non-planar sources, as `im-calc` does not calculate them. + """ + site_events = stations.reindex(columns=["rrup", "rjb", "rx", "ry"]) + site_events["event_id"] = event_id + return site_events.rename_axis("site_id").reset_index() + + +def _spectrum( + dataset: xr.Dataset, component: str, dim: str +) -> npt.NDArray[np.float32] | None: + """Extract one component of a spectral IM, if it has that component.""" + if component not in dataset: + return None + return ( + dataset[component] + .transpose("station", dim) + .values.astype(np.float32, copy=False) + ) + + +def simulated_records( + dtree: xr.DataTree, + rel_id: str, + stations: pd.Index, + periods: npt.NDArray[np.float64], + frequencies: npt.NDArray[np.float64], +) -> Iterator[dict[str, Any]]: + """Read the simulated intensity measures of a realisation. + + Parameters + ---------- + dtree : xr.DataTree + The intensity measure tree. + rel_id : str + The realisation. + stations : pd.Index + The stations to read. + periods : np.ndarray + The database's periods, which pSA is placed on. + frequencies : np.ndarray + The database's frequencies, which FAS is placed on. + + Yields + ------ + dict + The `IMDB.add_records` arguments for the records of each component. + """ + + def read(im: str) -> xr.Dataset: + return dtree[im].dataset.sel(station=stations) if im in dtree else xr.Dataset() + + scalars = {im: read(im) for im in schema.SCALAR_IMS} + psa = read("pSA").reindex(period=periods) + fas = read("FAS").reindex(frequency=frequencies) + for component in schema.COMPONENTS: + scalar_values = { + im: dataset[component].values + for im, dataset in scalars.items() + if component in dataset + } + spectra = { + "pSA": _spectrum(psa, component, "period"), + "FAS": _spectrum(fas, component, "frequency"), + } + if scalar_values or any(value is not None for value in spectra.values()): + records = pd.DataFrame( + { + "rel_id": rel_id, + "site_id": stations, + "component": component, + "kind": "simulated", + } + | scalar_values + ) + yield {"df": records, **spectra} + + +def empirical_records( + dtree: xr.DataTree, + rel_id: str, + stations: pd.Index, + periods: npt.NDArray[np.float64], +) -> Iterator[dict[str, Any]]: + """Read the empirical intensity measure estimates of a realisation. + + `im-calc` stores the log-mean and log-standard deviation of each + estimate; the IMDB holds the median and the log-standard deviation. + + Parameters + ---------- + dtree : xr.DataTree + The intensity measure tree. + rel_id : str + The realisation. + stations : pd.Index + The stations to read. + periods : np.ndarray + The database's periods, which pSA is placed on. + + Yields + ------ + dict + The `IMDB.add_records` arguments for the records of each ground + motion model and component. + """ + estimates: dict[tuple[str, str], dict[str, xr.Dataset]] = defaultdict(dict) + for node in dtree.subtree: + if node.parent is not None and node.parent.name == "empirical": + attrs = node.attrs + estimates[attrs["model"], attrs["component"]][ + attrs["intensity_measure"] + ] = node.dataset.sel(station=stations) + + for (model, component), ims in estimates.items(): + records = pd.DataFrame( + { + "rel_id": rel_id, + "site_id": stations, + "component": component, + "kind": "gmm", + "gmm_key": model, + } + ) + spectra = {} + for im, estimate in ims.items(): + if im == "pSA": + estimate = estimate.reindex(period=periods).transpose( + "station", "period" + ) + spectra["pSA"] = np.exp(estimate["mean"].values).astype(np.float32) + spectra["pSA_sigma"] = estimate["std_Total"].values.astype(np.float32) + else: + records[im] = np.exp(estimate["mean"].values) + records[f"{im}_sigma"] = estimate["std_Total"].values + yield {"df": records, **spectra} + + +@cli.from_docstring(app) +@log_utils.log_call() +def create_imdb( + output_ffp: Annotated[Path, typer.Argument(dir_okay=False, writable=True)], + intensity_measure_ffps: Annotated[ + list[Path], typer.Argument(exists=True, dir_okay=False, readable=True) + ], +) -> None: + """Collect the intensity measures of many realisations into one IMDB. + + Parameters + ---------- + output_ffp : Path + The database to create. Replaced if it exists. + intensity_measure_ffps : list[Path] + Intensity measure files from `im-calc`, one per realisation. + """ + logger = log_utils.get_logger(__name__) + realisations, periods, frequencies = survey(intensity_measure_ffps) + + # Build beside the output and move it into place once complete, so an + # interrupted run never leaves a partial database at the output path. + partial_ffp = output_ffp.with_name(output_ffp.name + ".partial") + partial_ffp.unlink(missing_ok=True) + db = IMDB.create( + partial_ffp, + periods=periods.tolist(), + frequencies=frequencies.tolist(), + db_meta={ + "source": "ucgmsim workflow simulations", + "creator": f"create-imdb (ucgmsim-workflow {metadata.version('ucgmsim-workflow')})", + "sort_order": "event_id", + }, + ) + with db: + sites: set[str] = set() + event_sites: dict[str, set[str]] = {} + for event_id, rel_id, intensity_measure_ffp in realisations: + with xr.open_datatree(intensity_measure_ffp) as dtree: + stations = read_stations(dtree) + + new_sites = ~stations.index.isin(sites) + if new_sites.any(): + db.add_sites(site_rows(stations[new_sites])) + sites.update(stations.index[new_sites]) + + # Realisations of an event share its rupture, so the event and + # its site distances are written once, from the first of them. + if event_id not in event_sites: + db.add_events(pd.DataFrame([event_row(dtree, event_id)])) + event_sites[event_id] = set() + db.add_realisations( + pd.DataFrame([realisation_row(dtree, rel_id, event_id)]) + ) + + new_event_sites = ~stations.index.isin(event_sites[event_id]) + if new_event_sites.any(): + db.add_site_event( + site_event_rows(stations[new_event_sites], event_id) + ) + event_sites[event_id].update(stations.index[new_event_sites]) + + simulated = stations.index[~(stations[SUPERGRID_DEPTH] > 0)] + for records in simulated_records( + dtree, rel_id, simulated, periods, frequencies + ): + db.add_records(**records) + for records in empirical_records( + dtree, rel_id, stations.index, periods + ): + db.add_records(**records) + + logger.info( + "added realisation", + intensity_measure_ffp=intensity_measure_ffp, + rel_id=rel_id, + event_id=event_id, + n_simulated_stations=len(simulated), + ) + + if problems := db.validate(): + raise ValueError(f"Database failed validation: {problems}") + partial_ffp.replace(output_ffp) + + +if __name__ == "__main__": + app() diff --git a/workflow/scripts/extract_subset.py b/workflow/scripts/extract_subset.py index 083315f9..685b6877 100644 --- a/workflow/scripts/extract_subset.py +++ b/workflow/scripts/extract_subset.py @@ -40,12 +40,10 @@ from qcore import cli from workflow import log_utils +from workflow.sites import OBSERVATION_STATION_PATTERN app = typer.Typer() -OBSERVATION_STATION_PATTERN = r"^\w{3,4}$" -"""Matches observation station names, which are three or four characters long.""" - SUBSET_SUFFIX = "_subset" """Suffix added to the stem of each input file to name its subset.""" diff --git a/workflow/scripts/im_calc.py b/workflow/scripts/im_calc.py index 56a9383a..69968df6 100644 --- a/workflow/scripts/im_calc.py +++ b/workflow/scripts/im_calc.py @@ -154,6 +154,20 @@ } +EMPIRICAL_COMPONENTS = { + IM.PGA: "rotd50", + IM.PGV: "rotd50", + IM.pSA: "rotd50", + IM.CAV: "geom", + IM.AI: "geom", + IM.Ds575: "geom", + IM.Ds595: "geom", +} +"""The component each ground motion model's estimate of an IM describes. +Amplitude models predict RotD50; duration and energy models predict the +geometric mean.""" + + EMPIRICAL_STATISTIC_METADATA = { "mean": "Mean of the natural logarithm of {description}", "std_Total": "Total standard deviation of the natural logarithm of {description}", @@ -163,12 +177,15 @@ def _supergrid_coordinates(dataset: xr.Dataset) -> dict[str, xr.DataArray]: - """Extract the absorbing-layer penetration coordinates from a waveform file.""" + """Extract the absorbing-layer penetration coordinates from a waveform file, NaN where unreported.""" return { - name: (dataset.coords[name].astype(np.float32).compute()) + name: ( + dataset.coords[name].astype(np.float32).compute() + if name in dataset.coords + else xr.full_like(dataset.station, np.nan, dtype=np.float32) + ) for name in sw4.SUPERGRID_DEPTH_COORDINATES.values() - if name in dataset.coords } @@ -645,6 +662,7 @@ def annotate_empirical(dataset: xr.Dataset, im_name: IM, model_name: str) -> xr. dataset.attrs["intensity_measure"] = str(im_name) dataset.attrs["model"] = model_name + dataset.attrs["component"] = EMPIRICAL_COMPONENTS[im_name] dataset.attrs["description"] = ( f"{description} predicted by the {model_name} ground motion model" ) @@ -771,7 +789,8 @@ def calculate_intensity_measures( realisation_ffp : Path Path to the realisation file. broadband_simulation_ffp : Path - Path to the broadband simulation waveforms. + Path to the broadband simulation waveforms, carrying each station's + `vs30`, `z1pt0` and `z2pt5`, as `bb-sim` writes. output_path : Path Output directory for IM calc summary statistics. simulated_stations : bool, default True @@ -782,8 +801,7 @@ def calculate_intensity_measures( Intensity measures to calculate. If not set, reads from the realisation file. empirical : bool, default True If passed, additionally estimate intensity measures from the ground - motion models in the realisation file. Requires the broadband - waveforms to carry `vs30`, `z1pt0` and `z2pt5`, as `bb-sim` writes. + motion models in the realisation file. """ metadata = RealisationMetadata.read_from_realisation(realisation_ffp) intensity_measure_parameters = ( @@ -873,10 +891,8 @@ def calculate_intensity_measures( "zbot": source_parameters.avg_zbot, "hypo_depth": source_parameters.hypo_depth, } | _supergrid_attributes(broadband, supergrid) - site_parameters = None + site_parameters = read_site_parameters(broadband) if empirical: - site_parameters = read_site_parameters(broadband) - empirical_parameters = EmpiricalParameters.read_from_realisation_or_defaults( realisation_ffp, metadata.defaults_version ) @@ -897,7 +913,7 @@ def calculate_intensity_measures( dtree = add_station_parameters( dtree, distances.as_dict() - | ((site_parameters).as_dict() if site_parameters else {}) + | site_parameters.as_dict() | {"latitude": broadband["latitude"], "longitude": broadband["longitude"]} | supergrid, ) diff --git a/workflow/sites.py b/workflow/sites.py index 5a20dbe2..39b0a34f 100644 --- a/workflow/sites.py +++ b/workflow/sites.py @@ -29,6 +29,10 @@ INDEX = "station" """The name of the index column: the station name.""" +OBSERVATION_STATION_PATTERN = r"^\w{3,4}$" +"""Matches observation station names, which are three or four characters long. +Every other station is a virtual station on the simulation grid.""" + COLUMNS = ( "latitude", "longitude", From 4452e488be1d001f0749cfce2b54ccccedca6795 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Fri, 23 Oct 2026 21:20:15 +1300 Subject: [PATCH 2/3] Read station parameters with Dataset.to_dataframe and fix im_calc formatting Co-Authored-By: Claude Opus 5.5 --- workflow/scripts/create_imdb.py | 11 ++++------- workflow/scripts/im_calc.py | 4 ++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/workflow/scripts/create_imdb.py b/workflow/scripts/create_imdb.py index 04cba685..9d5d9b0e 100644 --- a/workflow/scripts/create_imdb.py +++ b/workflow/scripts/create_imdb.py @@ -151,13 +151,10 @@ def read_stations(dtree: xr.DataTree) -> pd.DataFrame: penetration, indexed by station name. """ dataset = next(node.dataset for node in dtree.subtree if node.has_data) - return pd.DataFrame( - { - name: coordinate.values - for name, coordinate in dataset.coords.items() - if coordinate.dims == ("station",) and name != "station" - }, - index=pd.Index(dataset.station.values, name="station"), + return ( + dataset.coords.to_dataset() + .drop_dims(set(dataset.dims) - {"station"}) + .to_dataframe() ) diff --git a/workflow/scripts/im_calc.py b/workflow/scripts/im_calc.py index bad1421e..4c76ba8e 100644 --- a/workflow/scripts/im_calc.py +++ b/workflow/scripts/im_calc.py @@ -894,11 +894,11 @@ def calculate_intensity_measures( "zbot": source_parameters.avg_zbot, "hypo_depth": source_parameters.hypo_depth, } | _supergrid_attributes(broadband, supergrid) - + site_parameters = read_site_parameters(broadband) if metadata.event_time is not None: attributes["event_time"] = metadata.event_time.isoformat() - + if empirical: empirical_parameters = EmpiricalParameters.read_from_realisation_or_defaults( realisation_ffp, metadata.defaults_version From 90211c3e769dde4104c246e62c25c0103090b76a Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Wed, 28 Oct 2026 07:35:21 +1300 Subject: [PATCH 3/3] Report e2e fixture stations outside the supergrid in fake SW4 fake_sw4 wrote every station's SGDEPTH as the full sponge width, marking all of them inside the absorbing layer. create-imdb rightly drops simulated records for such stations, so test_create_imdb found only gmm records. The fixture stations lie in the domain interior, where SW4 reports zero. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/fake_sw4.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e/fake_sw4.py b/tests/e2e/fake_sw4.py index 924773e3..8e9fcd53 100644 --- a/tests/e2e/fake_sw4.py +++ b/tests/e2e/fake_sw4.py @@ -89,8 +89,10 @@ def write_recording(commands: dict[str, list[dict[str, str]]]) -> Path: group.create_dataset("STLA,STLO,STDP", data=station["STLA,STLO,STDP"][:]) group.create_dataset("NPTS", data=np.array([npts], dtype=np.int32)) group.create_dataset("ISNSEW", data=np.array([1], dtype=np.int32)) - group.create_dataset("SGDEPTH", data=np.array([supergrid_width])) - group.create_dataset("SGDEPTHGP", data=np.array([1.0])) + # The fixture stations all lie inside the domain interior, so none + # penetrate the supergrid sponge. + group.create_dataset("SGDEPTH", data=np.array([0.0])) + group.create_dataset("SGDEPTHGP", data=np.array([0.0])) for j, component in enumerate(("EW", "NS", "UP")): phase = 0.3 * i + 0.7 * j waveform = 0.01 * np.exp(-time / 3) * np.sin(2 * np.pi * time + phase)