The entries are organized into a concept network that connects the same concept appearing in different sources.
The team: Katja Berčič and Slobodan Stanojevikj.
diff --git a/web/concepts/views.py b/web/concepts/views.py index 2b067e7..565d8c5 100644 --- a/web/concepts/views.py +++ b/web/concepts/views.py @@ -57,6 +57,7 @@ def home(request): "agda_unimath": items.filter(source=Item.Source.AGDA_UNIMATH).count(), "lmfdb": items.filter(source=Item.Source.LMFDB).count(), "house_of_graphs": items.filter(source=Item.Source.HOUSE_OF_GRAPHS).count(), + "oeis": items.filter(source=Item.Source.OEIS).count(), }, } return render(request, "index.html", context) diff --git a/web/slurper/management/commands/clear_oeis.py b/web/slurper/management/commands/clear_oeis.py new file mode 100644 index 0000000..d775d09 --- /dev/null +++ b/web/slurper/management/commands/clear_oeis.py @@ -0,0 +1,46 @@ +import logging +import sys +from datetime import timedelta + +from concepts.models import Item +from django.core.management.base import BaseCommand +from slurper.models import SlurperRun + +MIN_INTERVAL = timedelta(days=7) + + +class Command(BaseCommand): + help = ( + "Delete all OEIS items. Guarded by a 7-day throttle; use --force to override." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Clear even if the OEIS slurper ran within the last 7 days.", + ) + + def handle(self, *args, force=False, **options): + source = Item.Source.OEIS + if not force and not SlurperRun.can_run(source, MIN_INTERVAL): + if sys.stdin.isatty(): + answer = ( + input( + f"OEIS slurper ran within the last {MIN_INTERVAL.days} days. " + f"Clear anyway? [y/N] " + ) + .strip() + .lower() + ) + if answer not in ("y", "yes"): + logging.info(f"[{source.label}] clear cancelled.") + return + else: + logging.info( + f"[{source.label}] clear skipped: ran less than " + f"{MIN_INTERVAL.days} days ago (use --force to override)." + ) + return + deleted, _ = Item.objects.filter(source=source).delete() + logging.info(f"[{source.label}] cleared {deleted} items.") diff --git a/web/slurper/management/commands/import_oeis.py b/web/slurper/management/commands/import_oeis.py new file mode 100644 index 0000000..23a5141 --- /dev/null +++ b/web/slurper/management/commands/import_oeis.py @@ -0,0 +1,14 @@ +from django.core.management.base import BaseCommand +from slurper import source_oeis + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Bypass the 7-day throttle and run anyway.", + ) + + def handle(self, *args, force=False, **options): + source_oeis.OEIS_SLURPER.save_items(force=force) diff --git a/web/slurper/source_oeis.py b/web/slurper/source_oeis.py new file mode 100644 index 0000000..100e1c6 --- /dev/null +++ b/web/slurper/source_oeis.py @@ -0,0 +1,103 @@ +""" +Slurper for the On-Line Encyclopedia of Integer Sequences (OEIS). + +Sequence names are fetched from the OEIS names.gz dump. OEIS content is +licensed under CC-BY-SA-4.0 by The On-Line Encyclopedia of Integer +Sequences, https://oeis.org/. +""" + +import gzip +import logging +from datetime import timedelta +from typing import Iterator, Optional, Tuple + +import requests +from concepts.models import Item, Link +from slurper.models import SlurperRun + +from web.settings import MATHSWITCH_CONTACT_EMAIL + + +class OeisSlurper: + NAMES_URL = "https://oeis.org/names.gz" + SEQUENCE_URL_PREFIX = "https://oeis.org/" + MIN_INTERVAL = timedelta(days=7) + + def __init__(self): + self.source = Item.Source.OEIS + + def fetch_names(self) -> bytes: + headers = {"User-Agent": f"MathSwitch/1.0 ({MATHSWITCH_CONTACT_EMAIL})"} + response = requests.get(self.NAMES_URL, headers=headers) + response.raise_for_status() + return gzip.decompress(response.content) + + def parse_names(self, raw: bytes) -> Iterator[Tuple[str, str]]: + for line in raw.decode("utf-8", errors="replace").splitlines(): + if not line or line.startswith("#"): + continue + identifier, _, description = line.partition(" ") + description = description.strip() + if not identifier or not description: + continue + yield identifier, description + + def line_to_item(self, identifier: str, description: str) -> Item: + return Item( + source=self.source, + identifier=identifier, + url=self.SEQUENCE_URL_PREFIX + identifier, + name=identifier, + description=description, + ) + + def extract_candidate_name(self, description: str) -> Optional[str]: + if ":" not in description: + return None + candidate = description.split(":", 1)[0].strip() + return candidate or None + + def save_items(self, force: bool = False): + if not force and not SlurperRun.can_run(self.source, self.MIN_INTERVAL): + logging.info( + f"[{self.source.label}] skipped: ran less than " + f"{self.MIN_INTERVAL.days} days ago (use --force to override)." + ) + return + total_filled = 0 + total_linked = 0 + for identifier, description in self.parse_names(self.fetch_names()): + existing = Item.objects.filter( + source=self.source, identifier=identifier + ).first() + if existing is not None: + existing.description = description + existing.save(update_fields=["description"]) + total_filled += 1 + continue + + candidate_name = self.extract_candidate_name(description) + if candidate_name is None: + continue + matches = list( + Item.objects.exclude(source=self.source).filter( + name__iexact=candidate_name + ) + ) + if not matches: + continue + + item = self.line_to_item(identifier, description) + item.save() + for match in matches: + Link.save_new(item, match, Link.Label.NAME_EQ) + total_linked += 1 + + SlurperRun.mark_ran(self.source) + logging.info( + f"[{self.source.label}] save_items finished: " + f"{total_filled} shells filled, {total_linked} NAME_EQ links created." + ) + + +OEIS_SLURPER = OeisSlurper() diff --git a/web/slurper/source_wikidata.py b/web/slurper/source_wikidata.py index 5864bc5..9bfe88f 100644 --- a/web/slurper/source_wikidata.py +++ b/web/slurper/source_wikidata.py @@ -10,8 +10,8 @@ from web.settings import ( IMPORT_MATH_ENTITIES, IMPORT_PHYSICS_ENTITIES, + MATHSWITCH_CONTACT_EMAIL, WIKIDATA_PAGE_SIZE, - WIKIPEDIA_CONTACT_EMAIL, ) # Wikipedia API contact email (required by Wikipedia API guidelines) @@ -233,7 +233,7 @@ def fetch_pages(self): def get_headers(self): return { - "User-Agent": f"MathSwitch/1.0 ({WIKIPEDIA_CONTACT_EMAIL})", + "User-Agent": f"MathSwitch/1.0 ({MATHSWITCH_CONTACT_EMAIL})", "Accept": "application/json", "Accept-Language": "en-US,en;q=0.9", } @@ -242,13 +242,13 @@ def fetch_article(self, json_item, index=None, total=None): global _missing_email_logged # Check if contact email is configured - if WIKIPEDIA_CONTACT_EMAIL is None: + if MATHSWITCH_CONTACT_EMAIL is None: if not _missing_email_logged: logging.log( logging.WARNING, - "WIKIPEDIA_CONTACT_EMAIL is not set. " + "MATHSWITCH_CONTACT_EMAIL is not set. " "Wikipedia article fetching is disabled. " - "Please set WIKIPEDIA_CONTACT_EMAIL at the top of " + "Please set MATHSWITCH_CONTACT_EMAIL at the top of " "source_wikidata.py to enable article fetching.", ) _missing_email_logged = True diff --git a/web/slurper/tests.py b/web/slurper/tests.py index 335ad0f..7fdc7d2 100644 --- a/web/slurper/tests.py +++ b/web/slurper/tests.py @@ -1,9 +1,12 @@ -from unittest.mock import patch +import gzip +from unittest.mock import Mock, patch -from concepts.models import Item +from concepts.models import Item, Link from django.core.management import call_command from django.test import TestCase -from slurper import source_house_of_graphs +from django.utils import timezone +from slurper import source_house_of_graphs, source_oeis +from slurper.models import SlurperRun # Run: ./venv/bin/python ./web/manage.py test slurper.tests @@ -56,3 +59,165 @@ def test_import_house_of_graphs_calls_slurper(self, mock_save_items): call_command("import_house_of_graphs") mock_save_items.assert_called_once_with() + + +# The following tests define what "done" means for the OEIS slurper's scoping +# rewrite (only import Wikidata-known A-numbers or NAME_EQ matches, fill in +# Wikidata-bridge shell items, credit the license, set a User-Agent). + + +class OeisSlurperTest(TestCase): + def test_line_to_item_maps_fields(self): + item = source_oeis.OEIS_SLURPER.line_to_item( + "A000045", + "Fibonacci numbers: a(0) = 0, a(1) = 1, a(n) = a(n-1) + a(n-2).", + ) + + self.assertEqual(item.identifier, "A000045") + self.assertEqual(item.url, "https://oeis.org/A000045") + self.assertEqual(item.name, "A000045") + self.assertEqual( + item.description, + "Fibonacci numbers: a(0) = 0, a(1) = 1, a(n) = a(n-1) + a(n-2).", + ) + + @patch("slurper.source_oeis.OeisSlurper.fetch_names") + @patch("slurper.source_oeis.OeisSlurper.parse_names") + def test_wikidata_shell_item_gets_description_filled( + self, mock_parse_names, mock_fetch_names + ): + Item.objects.create( + source=Item.Source.OEIS, + identifier="A000045", + url="https://oeis.org/A000045", + name="A000045", + description=None, + ) + mock_parse_names.return_value = [ + ( + "A000045", + "Fibonacci numbers: a(0) = 0, a(1) = 1, a(n) = a(n-1) + a(n-2).", + ) + ] + + source_oeis.OEIS_SLURPER.save_items(force=True) + + self.assertEqual(Item.objects.filter(source=Item.Source.OEIS).count(), 1) + item = Item.objects.get(source=Item.Source.OEIS, identifier="A000045") + self.assertEqual( + item.description, + "Fibonacci numbers: a(0) = 0, a(1) = 1, a(n) = a(n-1) + a(n-2).", + ) + + @patch("slurper.source_oeis.OeisSlurper.fetch_names") + @patch("slurper.source_oeis.OeisSlurper.parse_names") + def test_non_bridged_no_name_match_is_skipped( + self, mock_parse_names, mock_fetch_names + ): + mock_parse_names.return_value = [ + ("A999999", "Some obscure sequence description.") + ] + + source_oeis.OEIS_SLURPER.save_items(force=True) + + self.assertEqual(Item.objects.filter(source=Item.Source.OEIS).count(), 0) + + @patch("slurper.source_oeis.OeisSlurper.fetch_names") + @patch("slurper.source_oeis.OeisSlurper.parse_names") + def test_name_eq_match_creates_item_and_link( + self, mock_parse_names, mock_fetch_names + ): + nlab_item = Item.objects.create( + source=Item.Source.NLAB, + identifier="fibonacci-numbers", + url="https://ncatlab.org/nlab/show/fibonacci-numbers", + name="Fibonacci numbers", + ) + mock_parse_names.return_value = [ + ( + "A000045", + "fibonacci numbers: a(0) = 0, a(1) = 1, a(n) = a(n-1) + a(n-2).", + ) + ] + + source_oeis.OEIS_SLURPER.save_items(force=True) + + oeis_item = Item.objects.get(source=Item.Source.OEIS, identifier="A000045") + self.assertTrue( + Link.objects.filter( + label=Link.Label.NAME_EQ, source=oeis_item, destination=nlab_item + ).exists() + or Link.objects.filter( + label=Link.Label.NAME_EQ, source=nlab_item, destination=oeis_item + ).exists() + ) + + @patch("slurper.source_oeis.OeisSlurper.fetch_names") + def test_throttle_blocks_without_force(self, mock_fetch_names): + SlurperRun.objects.create( + source=Item.Source.OEIS, last_succeeded_at=timezone.now() + ) + + source_oeis.OEIS_SLURPER.save_items() + + mock_fetch_names.assert_not_called() + + @patch("slurper.source_oeis.OeisSlurper.fetch_names") + @patch("slurper.source_oeis.OeisSlurper.parse_names") + def test_force_bypasses_throttle(self, mock_parse_names, mock_fetch_names): + SlurperRun.objects.create( + source=Item.Source.OEIS, last_succeeded_at=timezone.now() + ) + mock_parse_names.return_value = [] + + source_oeis.OEIS_SLURPER.save_items(force=True) + + mock_fetch_names.assert_called_once() + + @patch("slurper.source_oeis.requests.get") + def test_fetch_names_sets_user_agent(self, mock_get): + mock_get.return_value = Mock(content=gzip.compress(b"A000045 test")) + + source_oeis.OEIS_SLURPER.fetch_names() + + headers = mock_get.call_args.kwargs.get("headers", {}) + self.assertIn("@", headers.get("User-Agent", "")) + + def test_module_docstring_credits_oeis(self): + docstring = source_oeis.__doc__ or "" + + self.assertIn("CC-BY-SA-4.0", docstring) + self.assertIn("oeis.org", docstring) + + +class OeisCommandsTest(TestCase): + @patch("slurper.source_oeis.OEIS_SLURPER.save_items") + def test_import_oeis_calls_slurper(self, mock_save_items): + call_command("import_oeis") + + mock_save_items.assert_called_once_with(force=False) + + @patch("slurper.source_oeis.OEIS_SLURPER.save_items") + def test_import_oeis_with_force(self, mock_save_items): + call_command("import_oeis", "--force") + + mock_save_items.assert_called_once_with(force=True) + + def test_clear_oeis_removes_only_oeis_items(self): + Item.objects.create( + source=Item.Source.WIKIDATA, + identifier="wd-1", + url="https://example.com/wd-1", + name="Wikidata item", + ) + Item.objects.create( + source=Item.Source.OEIS, + identifier="A000045", + url="https://oeis.org/A000045", + name="A000045", + ) + + call_command("clear_oeis", "--force") + + self.assertTrue(Item.objects.filter(source=Item.Source.WIKIDATA).exists()) + self.assertFalse(Item.objects.filter(source=Item.Source.OEIS).exists()) diff --git a/web/slurper/wd_raw_item.py b/web/slurper/wd_raw_item.py index 6c1e07e..61c4af5 100644 --- a/web/slurper/wd_raw_item.py +++ b/web/slurper/wd_raw_item.py @@ -20,6 +20,10 @@ "wd_property": "wdt:P7554", "json_key": "eomID", }, + Item.Source.OEIS: { + "wd_property": "wdt:P829", + "json_key": "oeisID", + }, } # Wikipedia is dealt with elsewhere @@ -123,6 +127,8 @@ def raw_item(source, json_item, domain=Item.Domain.MATHEMATICS): return PWRawItem(json_item, domain=domain) case Item.Source.ENCYCLOPEDIA_OF_MATHEMATICS: return EoMRawItem(json_item, domain=domain) + case Item.Source.OEIS: + return OEISRawItem(json_item, domain=domain) case Item.Source.WIKIPEDIA_EN: return WpENRawItem(json_item, domain=domain) @@ -219,3 +225,11 @@ def __init__(self, json_item, domain=Item.Domain.MATHEMATICS): def url(self): return "https://encyclopediaofmath.org/wiki/" + self.identifier() + + +class OEISRawItem(OtherWdRawItem): + def __init__(self, json_item, domain=Item.Domain.MATHEMATICS): + super().__init__(Item.Source.OEIS, json_item, domain=domain) + + def url(self): + return "https://oeis.org/" + self.identifier() diff --git a/web/web/settings.py b/web/web/settings.py index e2120ef..1b1cb1b 100644 --- a/web/web/settings.py +++ b/web/web/settings.py @@ -163,7 +163,7 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -WIKIPEDIA_CONTACT_EMAIL = config("WIKIPEDIA_CONTACT_EMAIL", default="my@email.com") +MATHSWITCH_CONTACT_EMAIL = config("MATHSWITCH_CONTACT_EMAIL", default="my@email.com") # Logging # NOTE SST: Change if needed, this is for visibility when running the commands