From faacb269293f7d188305b45b86aeb0ae4bffa162 Mon Sep 17 00:00:00 2001 From: Ian Harry Date: Fri, 4 Sep 2026 10:07:19 +0100 Subject: [PATCH] Make notebook data downloads resilient The "tutorial tests" CI job regularly fails while a notebook is fetching GW strain data. Two causes: * bare astropy download_file / !wget calls with no retry; * the data came from the Git-LFS media.githubusercontent.com mirror of gwastro/pycbc_data, which is subject to the LFS bandwidth quota and then serves truncated bodies / 404s. Route every gwastro/pycbc_data download through pycbc.io.get_file, passing the original GWOSC / DCC URL. get_file retries with bounded back-off and, inside GitHub Actions, transparently falls back to the pycbc_data release asset and then the LFS media copy (see the companion pycbc change that teaches _candidate_urls about dcc.ligo.org / losc.ligo.org URLs). The result is copied to its real basename with shutil.copy, because the .gwf filename matters to later cells / read_frame. Notebooks touched: tutorial/{1_CatalogData, inference_0_Overview, inference_1_ModelsAndPEByHand, inference_8_BHRingdown, inference_4_bbh_example/IntroToPyCBCInference} and examples/{gw150914_audio, gw151226_snr, gw170104_look, gw170817_mass_estimate}. Co-Authored-By: Claude Sonnet 5 --- examples/gw150914_audio.ipynb | 12 ++++++---- examples/gw151226_snr.ipynb | 12 +++++++--- examples/gw170104_look.ipynb | 17 +++++++++---- examples/gw170817_mass_estimate.ipynb | 17 ++++++++----- tutorial/1_CatalogData.ipynb | 9 ++++++- tutorial/inference_0_Overview.ipynb | 24 ++++++++++--------- tutorial/inference_1_ModelsAndPEByHand.ipynb | 22 ++++++++++------- .../IntroToPyCBCInference.ipynb | 19 ++++++++------- tutorial/inference_8_BHRingdown.ipynb | 16 ++++++++----- 9 files changed, 94 insertions(+), 54 deletions(-) diff --git a/examples/gw150914_audio.ipynb b/examples/gw150914_audio.ipynb index 1db5bb5..5c277cc 100644 --- a/examples/gw150914_audio.ipynb +++ b/examples/gw150914_audio.ipynb @@ -89,11 +89,15 @@ } ], "source": [ - "#Set up the libraries we need and download the data\n", + "# Set up the libraries we need and download the data\n", + "import os\n", + "import shutil\n", + "from pycbc.io import get_file\n", "\n", - "# Using GitHub instead of original LOSC location to improve reliability, especially in GitHub CI\n", - "# !wget https://losc.ligo.org/s/events/GW150914/H-H1_LOSC_4_V2-1126259446-32.gwf\n", - "!wget https://media.githubusercontent.com/media/gwastro/pycbc_data/master/H-H1_LOSC_4_V2-1126259446-32.gwf" + "url = \"https://gwosc.org/s/events/GW150914/H-H1_LOSC_4_V2-1126259446-32.gwf\"\n", + "fname = os.path.basename(url)\n", + "if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)" ] }, { diff --git a/examples/gw151226_snr.ipynb b/examples/gw151226_snr.ipynb index c87a8f4..36dc5dc 100644 --- a/examples/gw151226_snr.ipynb +++ b/examples/gw151226_snr.ipynb @@ -263,11 +263,17 @@ } ], "source": [ + "import os\n", + "import shutil\n", "from pycbc.frame import read_frame\n", + "from pycbc.io import get_file\n", + "\n", "# Now let's see what the SNR timeseries looks like in the other detector in Livingston.\n", - "# Get the data for L1. Using GitHub instead of original LOSC site for reliability\n", - "# !wget -O L-L1_LOSC_4_V2-1135136334-32.gwf https://losc.ligo.org/s/events/GW151226/L-L1_LOSC_4_V2-1135136334-32.gwf\n", - "!wget -O L-L1_LOSC_4_V2-1135136334-32.gwf https://media.githubusercontent.com/media/gwastro/pycbc_data/master/L-L1_LOSC_4_V2-1135136334-32.gwf\n", + "# Get the data for L1\n", + "url = \"https://gwosc.org/s/events/GW151226/L-L1_LOSC_4_V2-1135136334-32.gwf\"\n", + "fname = os.path.basename(url)\n", + "if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)\n", "\n", "# Read the Lvingston data and remove low frequency content\n", "l1 = read_frame('L-L1_LOSC_4_V2-1135136334-32.gwf', 'L1:LOSC-STRAIN')\n", diff --git a/examples/gw170104_look.ipynb b/examples/gw170104_look.ipynb index 7437eec..8947fbe 100644 --- a/examples/gw170104_look.ipynb +++ b/examples/gw170104_look.ipynb @@ -98,11 +98,18 @@ } ], "source": [ - "# Get the data for H1 and L1 redirect from LOSC to GitHub to improve reliability, especially in GitHub CI\n", - "# !wget https://losc.ligo.org/s/events/GW170104/H-H1_LOSC_4_V1-1167559920-32.gwf\n", - "# !wget https://losc.ligo.org/s/events/GW170104/L-L1_LOSC_4_V1-1167559920-32.gwf\n", - "!wget https://media.githubusercontent.com/media/gwastro/pycbc_data/master/H-H1_LOSC_4_V1-1167559920-32.gwf\n", - "!wget https://media.githubusercontent.com/media/gwastro/pycbc_data/master/L-L1_LOSC_4_V1-1167559920-32.gwf" + "import os\n", + "import shutil\n", + "from pycbc.io import get_file\n", + "\n", + "# Get the data for H1 and L1\n", + "for url in (\n", + " \"https://gwosc.org/s/events/GW170104/H-H1_LOSC_4_V1-1167559920-32.gwf\",\n", + " \"https://gwosc.org/s/events/GW170104/L-L1_LOSC_4_V1-1167559920-32.gwf\",\n", + "):\n", + " fname = os.path.basename(url)\n", + " if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)" ] }, { diff --git a/examples/gw170817_mass_estimate.ipynb b/examples/gw170817_mass_estimate.ipynb index 0edf3ec..de2826b 100644 --- a/examples/gw170817_mass_estimate.ipynb +++ b/examples/gw170817_mass_estimate.ipynb @@ -106,13 +106,18 @@ } ], "source": [ - "# Original links at \n", - "# https://dcc.ligo.org/public/0146/P1700349/001/H-H1_LOSC_CLN_4_V1-1187007040-2048.gwf\n", - "# https://dcc.ligo.org/public/0146/P1700349/001/L-L1_LOSC_CLN_4_V1-1187007040-2048.gwf\n", - "# Redirected to github for reliability\n", + "import os\n", + "import shutil\n", + "from pycbc.io import get_file\n", "\n", - "!wget -nc https://media.githubusercontent.com/media/gwastro/pycbc_data/master/H-H1_LOSC_CLN_4_V1-1187007040-2048.gwf\n", - "!wget -nc https://media.githubusercontent.com/media/gwastro/pycbc_data/master/L-L1_LOSC_CLN_4_V1-1187007040-2048.gwf" + "# Download the GW170817 data\n", + "for url in (\n", + " \"https://dcc.ligo.org/public/0146/P1700349/001/H-H1_LOSC_CLN_4_V1-1187007040-2048.gwf\",\n", + " \"https://dcc.ligo.org/public/0146/P1700349/001/L-L1_LOSC_CLN_4_V1-1187007040-2048.gwf\",\n", + "):\n", + " fname = os.path.basename(url)\n", + " if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)" ] }, { diff --git a/tutorial/1_CatalogData.ipynb b/tutorial/1_CatalogData.ipynb index 8874526..530be25 100644 --- a/tutorial/1_CatalogData.ipynb +++ b/tutorial/1_CatalogData.ipynb @@ -354,7 +354,14 @@ ], "source": [ "# We'll first download some data for this demonstration\n", - "!curl -O -J -L https://media.githubusercontent.com/media/gwastro/pycbc_data/master/H-H1_LOSC_4_V2-1128678884-32.gwf" + "import os\n", + "import shutil\n", + "from pycbc.io import get_file\n", + "\n", + "url = \"https://gwosc.org/s/events/LVT151012/H-H1_LOSC_4_V2-1128678884-32.gwf\"\n", + "fname = os.path.basename(url)\n", + "if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)" ] }, { diff --git a/tutorial/inference_0_Overview.ipynb b/tutorial/inference_0_Overview.ipynb index e9c7f0b..ead2a99 100644 --- a/tutorial/inference_0_Overview.ipynb +++ b/tutorial/inference_0_Overview.ipynb @@ -498,17 +498,20 @@ } ], "source": [ + "import os\n", + "import shutil\n", + "\n", "from pycbc.catalog import Merger\n", "from pycbc.psd import interpolate, inverse_spectrum_truncation\n", "from pycbc.frame import read_frame\n", "from pycbc.filter import highpass, resample_to_delta_t\n", - "from astropy.utils.data import download_file\n", + "from pycbc.io import get_file\n", "\n", "m = Merger(\"GW170817\")\n", "\n", "# List of observatories we'll analyze\n", - "ifos = ['H1', \n", - " 'V1', \n", + "ifos = ['H1',\n", + " 'V1',\n", " 'L1',\n", " ]\n", "\n", @@ -521,14 +524,13 @@ "\n", "for ifo in ifos:\n", " print(\"Processing {} data\".format(ifo))\n", - " \n", + "\n", " # Download the gravitational wave data for GW170817\n", - " \n", - " # Redirected to aid reliability of downloading, especially on the GitHub CI\n", - " # url = \"https://dcc.ligo.org/public/0146/P1700349/001/{}-{}1_LOSC_CLN_4_V1-1187007040-2048.gwf\"\n", - " url = \"https://media.githubusercontent.com/media/gwastro/pycbc_data/master/{}-{}1_LOSC_CLN_4_V1-1187007040-2048.gwf\"\n", - " \n", - " fname = download_file(url.format(ifo[0], ifo[0]), cache=True)\n", + " url = \"https://dcc.ligo.org/public/0146/P1700349/001/{}-{}1_LOSC_CLN_4_V1-1187007040-2048.gwf\"\n", + " url = url.format(ifo[0], ifo[0])\n", + " fname = os.path.basename(url)\n", + " if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)\n", " data_filenames[ifo] = fname\n", "\n", " # Read the gravitational wave data and do some minimal\n", @@ -543,7 +545,7 @@ "\n", " # Estimate the power spectral density of the data\n", " psd = interpolate(ts.psd(4), ts.delta_f)\n", - " psd = inverse_spectrum_truncation(psd, int(4 * psd.sample_rate), \n", + " psd = inverse_spectrum_truncation(psd, int(4 * psd.sample_rate),\n", " trunc_method='hann',\n", " low_frequency_cutoff=20.0)\n", " psds[ifo] = psd\n" diff --git a/tutorial/inference_1_ModelsAndPEByHand.ipynb b/tutorial/inference_1_ModelsAndPEByHand.ipynb index 981874a..81ff890 100644 --- a/tutorial/inference_1_ModelsAndPEByHand.ipynb +++ b/tutorial/inference_1_ModelsAndPEByHand.ipynb @@ -782,17 +782,20 @@ } ], "source": [ + "import os\n", + "import shutil\n", + "\n", "from pycbc.catalog import Merger\n", "from pycbc.psd import interpolate, inverse_spectrum_truncation\n", "from pycbc.frame import read_frame\n", "from pycbc.filter import highpass, resample_to_delta_t\n", - "from astropy.utils.data import download_file\n", + "from pycbc.io import get_file\n", "\n", "m = Merger(\"GW170817\")\n", "\n", "# List of observatories we'll analyze\n", - "ifos = ['H1', \n", - " 'V1', \n", + "ifos = ['H1',\n", + " 'V1',\n", " 'L1',\n", " ]\n", "\n", @@ -802,12 +805,13 @@ "\n", "for ifo in ifos:\n", " print(\"Processing {} data\".format(ifo))\n", - " \n", - " # Redirected to aid reliability of downloading, especially on the GitHub CI\n", - " # url = \"https://dcc.ligo.org/public/0146/P1700349/001/{}-{}1_LOSC_CLN_4_V1-1187007040-2048.gwf\"\n", - " url = \"https://media.githubusercontent.com/media/gwastro/pycbc_data/master/{}-{}1_LOSC_CLN_4_V1-1187007040-2048.gwf\"\n", "\n", - " fname = download_file(url.format(ifo[0], ifo[0]), cache=True) \n", + " # Download the gravitational wave data for GW170817\n", + " url = \"https://dcc.ligo.org/public/0146/P1700349/001/{}-{}1_LOSC_CLN_4_V1-1187007040-2048.gwf\"\n", + " url = url.format(ifo[0], ifo[0])\n", + " fname = os.path.basename(url)\n", + " if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)\n", "\n", " # Read the gravitational wave data and do some minimal\n", " # conditioning of the data.\n", @@ -821,7 +825,7 @@ "\n", " # Estimate the power spectral density of the data\n", " psd = interpolate(ts.psd(4), ts.delta_f)\n", - " psd = inverse_spectrum_truncation(psd, int(4 * psd.sample_rate), \n", + " psd = inverse_spectrum_truncation(psd, int(4 * psd.sample_rate),\n", " trunc_method='hann',\n", " low_frequency_cutoff=20.0)\n", " psds[ifo] = psd\n" diff --git a/tutorial/inference_4_bbh_example/IntroToPyCBCInference.ipynb b/tutorial/inference_4_bbh_example/IntroToPyCBCInference.ipynb index f2df556..52141cb 100644 --- a/tutorial/inference_4_bbh_example/IntroToPyCBCInference.ipynb +++ b/tutorial/inference_4_bbh_example/IntroToPyCBCInference.ipynb @@ -269,16 +269,17 @@ "metadata": {}, "outputs": [], "source": [ - "# Paths redirected from GWOSC to GitHub to improve reliability, especially in GitHub CI\n", - "#if not os.path.exists('H-H1_GWOSC_4KHZ_R1-1126257415-4096.gwf'):\n", - "# !wget https://www.gw-openscience.org/catalog/GWTC-1-confident/data/GW150914/H-H1_GWOSC_4KHZ_R1-1126257415-4096.gwf\n", - "#if not os.path.exists('L-L1_GWOSC_4KHZ_R1-1126257415-4096.gwf'):\n", - "# !wget https://www.gw-openscience.org/catalog/GWTC-1-confident/data/GW150914/L-L1_GWOSC_4KHZ_R1-1126257415-4096.gwf\n", + "# Download the GW150914 data\n", + "import shutil\n", + "from pycbc.io import get_file\n", "\n", - "if not os.path.exists('H-H1_GWOSC_4KHZ_R1-1126257415-4096.gwf'):\n", - " !wget https://media.githubusercontent.com/media/gwastro/pycbc_data/master/H-H1_GWOSC_4KHZ_R1-1126257415-4096.gwf\n", - "if not os.path.exists('L-L1_GWOSC_4KHZ_R1-1126257415-4096.gwf'):\n", - " !wget https://media.githubusercontent.com/media/gwastro/pycbc_data/master/L-L1_GWOSC_4KHZ_R1-1126257415-4096.gwf" + "for url in (\n", + " \"https://gwosc.org/eventapi/html/GWTC-1-confident/GW150914/v3/H-H1_GWOSC_4KHZ_R1-1126257415-4096.gwf\",\n", + " \"https://gwosc.org/eventapi/html/GWTC-1-confident/GW150914/v3/L-L1_GWOSC_4KHZ_R1-1126257415-4096.gwf\",\n", + "):\n", + " fname = os.path.basename(url)\n", + " if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)" ] }, { diff --git a/tutorial/inference_8_BHRingdown.ipynb b/tutorial/inference_8_BHRingdown.ipynb index e0e94df..1c72880 100644 --- a/tutorial/inference_8_BHRingdown.ipynb +++ b/tutorial/inference_8_BHRingdown.ipynb @@ -116,18 +116,22 @@ "metadata": {}, "outputs": [], "source": [ - "from astropy.utils.data import download_file\n", + "import os\n", + "import shutil\n", + "\n", + "from pycbc.io import get_file\n", "\n", "ifos = ['H1', 'L1', 'V1']\n", "data_filenames = {}\n", "for ifo in ifos:\n", " print(\"Downloading {} data\".format(ifo))\n", - " \n", + "\n", " # Download the gravitational wave data for GW190521\n", - " # URL changed from GWOSC to GitHub to improve reliability\n", - " #url = \"https://www.gw-openscience.org/eventapi/html/O3_Discovery_Papers/GW190521/v2/{}-{}1_GWOSC_4KHZ_R2-1242440920-4096.gwf\"\n", - " url = \"https://media.githubusercontent.com/media/gwastro/pycbc_data/master/{}-{}1_GWOSC_4KHZ_R2-1242440920-4096.gwf\"\n", - " fname = download_file(url.format(ifo[0], ifo[0]), cache=True)\n", + " url = \"https://gwosc.org/eventapi/html/O3_Discovery_Papers/GW190521/v2/{}-{}1_GWOSC_4KHZ_R2-1242440920-4096.gwf\"\n", + " url = url.format(ifo[0], ifo[0])\n", + " fname = os.path.basename(url)\n", + " if not os.path.exists(fname):\n", + " shutil.copy(get_file(url, cache=True), fname)\n", " data_filenames[ifo] = fname" ] },