From 27f889739ebe283cf68fe1a16c7c80bfec4af591 Mon Sep 17 00:00:00 2001 From: Alexcchip Date: Mon, 5 Oct 2026 11:29:44 -0400 Subject: [PATCH 01/21] Normalizing script, dedupe, and event table additions --- ...c9a52_canonical_events_and_observations.py | 93 +++++++ backend/foresight/database/models/__init__.py | 2 + backend/foresight/database/models/external.py | 116 +++++++- backend/src/foresight/ingest/__init__.py | 1 + backend/src/foresight/ingest/normalize.py | 254 ++++++++++++++++++ backend/tests/test_normalize.py | 140 ++++++++++ 6 files changed, 595 insertions(+), 11 deletions(-) create mode 100644 backend/foresight/database/migrations/versions/b7d41e0c9a52_canonical_events_and_observations.py create mode 100644 backend/src/foresight/ingest/__init__.py create mode 100644 backend/src/foresight/ingest/normalize.py create mode 100644 backend/tests/test_normalize.py diff --git a/backend/foresight/database/migrations/versions/b7d41e0c9a52_canonical_events_and_observations.py b/backend/foresight/database/migrations/versions/b7d41e0c9a52_canonical_events_and_observations.py new file mode 100644 index 0000000..9dbfd5e --- /dev/null +++ b/backend/foresight/database/migrations/versions/b7d41e0c9a52_canonical_events_and_observations.py @@ -0,0 +1,93 @@ +"""canonical events and observations + +Replaces the placeholder `events` table with the canonical event contract plus +an append-only `event_observations` staging table. Nothing writes to `events` +yet, so it is dropped rather than migrated. + +Revision ID: b7d41e0c9a52 +Revises: 72e850f4fa7b +Create Date: 2026-10-05 11:12:04.882341 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = 'b7d41e0c9a52' +down_revision: Union[str, None] = '72e850f4fa7b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_table('events') + op.create_table('events', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('dedupe_key', sa.String(length=64), nullable=False), + sa.Column('title', sa.String(length=300), nullable=False), + sa.Column('title_norm', sa.String(length=300), nullable=False), + sa.Column('category', sa.String(length=40), nullable=False), + sa.Column('venue', sa.String(length=200), nullable=True), + sa.Column('city', sa.String(length=120), nullable=False), + sa.Column('city_slug', sa.String(length=120), nullable=False), + sa.Column('country', sa.String(length=2), nullable=True), + sa.Column('latitude', sa.Float(), nullable=True), + sa.Column('longitude', sa.Float(), nullable=True), + sa.Column('start_local_date', sa.Date(), nullable=False), + sa.Column('end_local_date', sa.Date(), nullable=False), + sa.Column('start_at_utc', sa.DateTime(timezone=True), nullable=True), + sa.Column('end_at_utc', sa.DateTime(timezone=True), nullable=True), + sa.Column('timezone', sa.String(length=64), nullable=False), + sa.Column('date_precision', sa.String(length=20), nullable=False), + sa.Column('expected_attendance', sa.Integer(), nullable=True), + sa.Column('impact_rank', sa.Integer(), nullable=True), + sa.Column('local_rank', sa.Integer(), nullable=True), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('status', sa.String(length=20), nullable=False), + sa.Column('url', sa.String(length=600), nullable=True), + sa.Column('primary_source', sa.String(length=40), nullable=False), + sa.Column('first_seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('last_seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('dedupe_key') + ) + op.create_index(op.f('ix_events_title_norm'), 'events', ['title_norm'], unique=False) + op.create_index('ix_events_city_slug_start', 'events', ['city_slug', 'start_local_date'], unique=False) + op.create_table('event_observations', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('event_id', sa.Integer(), nullable=True), + sa.Column('source', sa.String(length=40), nullable=False), + sa.Column('source_ref', sa.String(length=600), nullable=False), + sa.Column('source_updated_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('payload', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['event_id'], ['events.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('source', 'source_ref') + ) + + +def downgrade() -> None: + op.drop_table('event_observations') + op.drop_index('ix_events_city_slug_start', table_name='events') + op.drop_index(op.f('ix_events_title_norm'), table_name='events') + op.drop_table('events') + op.create_table('events', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=300), nullable=False), + sa.Column('category', sa.String(length=60), nullable=True), + sa.Column('venue', sa.String(length=200), nullable=True), + sa.Column('city', sa.String(length=120), nullable=False), + sa.Column('start_date', sa.Date(), nullable=False), + sa.Column('end_date', sa.Date(), nullable=False), + sa.Column('expected_attendance', sa.Integer(), nullable=True), + sa.Column('latitude', sa.Float(), nullable=True), + sa.Column('longitude', sa.Float(), nullable=True), + sa.Column('source', sa.String(length=40), nullable=False), + sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) diff --git a/backend/foresight/database/models/__init__.py b/backend/foresight/database/models/__init__.py index 932cb8a..ec1f61f 100644 --- a/backend/foresight/database/models/__init__.py +++ b/backend/foresight/database/models/__init__.py @@ -4,6 +4,7 @@ from foresight.database.models.external import ( CompetitorRate, Event, + EventObservation, FlightArrival, Weather, ) @@ -14,6 +15,7 @@ "CompetitorRate", "DailyPerformance", "Event", + "EventObservation", "FlightArrival", "Hotel", "RoomType", diff --git a/backend/foresight/database/models/external.py b/backend/foresight/database/models/external.py index 69c8f81..a426810 100644 --- a/backend/foresight/database/models/external.py +++ b/backend/foresight/database/models/external.py @@ -2,11 +2,29 @@ from datetime import date, datetime from decimal import Decimal - -from sqlalchemy import Date, DateTime, Numeric, String, func -from sqlalchemy.orm import Mapped, mapped_column +from typing import Any + +from sqlalchemy import ( + Date, + DateTime, + ForeignKey, + Index, + Numeric, + String, + UniqueConstraint, + func, +) +from sqlalchemy import Enum as SAEnum +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship from foresight.database.base import Base +from foresight.ingest.normalize import ( + DatePrecision, + EventCategory, + EventStatus, + Source, +) class Weather(Base): @@ -36,29 +54,105 @@ class Weather(Base): class Event(Base): - """Something happening nearby that drives demand.""" + """One real-world event occurrence, merged across sources. + + Identified by `dedupe_key` (normalized title + city + local start date) so + the same festival seen by PredictHQ and by AskNews lands on one row. + """ __tablename__ = "events" id: Mapped[int] = mapped_column(primary_key=True) - name: Mapped[str] = mapped_column(String(300)) - category: Mapped[str | None] = mapped_column(String(60)) + dedupe_key: Mapped[str] = mapped_column(String(64), unique=True) + + title: Mapped[str] = mapped_column(String(300)) + title_norm: Mapped[str] = mapped_column(String(300), index=True) + category: Mapped[EventCategory] = mapped_column( + SAEnum(EventCategory, native_enum=False, length=40), + default=EventCategory.UNKNOWN, + ) + venue: Mapped[str | None] = mapped_column(String(200)) city: Mapped[str] = mapped_column(String(120)) + city_slug: Mapped[str] = mapped_column(String(120)) + country: Mapped[str | None] = mapped_column(String(2)) # ISO-3166 alpha-2 + latitude: Mapped[float | None] + longitude: Mapped[float | None] - start_date: Mapped[date] = mapped_column(Date) - end_date: Mapped[date] = mapped_column(Date) + # Local dates, not UTC: a stay-night is local to the hotel. Inclusive range. + start_local_date: Mapped[date] = mapped_column(Date) + end_local_date: Mapped[date] = mapped_column(Date) + start_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + end_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + timezone: Mapped[str] = mapped_column(String(64), default="UTC") + date_precision: Mapped[DatePrecision] = mapped_column( + SAEnum(DatePrecision, native_enum=False, length=20), + default=DatePrecision.DAY, + ) expected_attendance: Mapped[int | None] + impact_rank: Mapped[int | None] # PredictHQ rank, 0-100 + local_rank: Mapped[int | None] # PredictHQ local_rank, 0-100 + confidence: Mapped[float] = mapped_column(default=1.0) - latitude: Mapped[float | None] - longitude: Mapped[float | None] + status: Mapped[EventStatus] = mapped_column( + SAEnum(EventStatus, native_enum=False, length=20), + default=EventStatus.ACTIVE, + ) + url: Mapped[str | None] = mapped_column(String(600)) + + # Which source currently owns these values, by precedence. + primary_source: Mapped[Source] = mapped_column( + SAEnum(Source, native_enum=False, length=40) + ) + + first_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + observations: Mapped[list["EventObservation"]] = relationship( + back_populates="event" + ) + + # Serves the core lookup: what is happening near this hotel on this night. + __table_args__ = ( + Index("ix_events_city_slug_start", "city_slug", "start_local_date"), + ) + + +class EventObservation(Base): + """One source's raw sighting of an event, kept verbatim. + + Append-only and keyed on (source, source_ref) -- the PredictHQ event id or + the AskNews article URL. This is what makes re-runs idempotent and lets a + revised upstream record be recognised rather than duplicated. + + `event_id` is null when a sighting cannot be resolved to a dated event, e.g. + an article that announces a festival without saying when it is. + """ + + __tablename__ = "event_observations" + + id: Mapped[int] = mapped_column(primary_key=True) + event_id: Mapped[int | None] = mapped_column(ForeignKey("events.id")) + + source: Mapped[Source] = mapped_column(SAEnum(Source, native_enum=False, length=40)) + source_ref: Mapped[str] = mapped_column(String(600)) + source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + payload: Mapped[dict[str, Any]] = mapped_column(JSONB) - source: Mapped[str] = mapped_column(String(40)) captured_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now() ) + event: Mapped["Event | None"] = relationship(back_populates="observations") + + __table_args__ = (UniqueConstraint("source", "source_ref"),) + class CompetitorRate(Base): """What another hotel is charging for a given night.""" diff --git a/backend/src/foresight/ingest/__init__.py b/backend/src/foresight/ingest/__init__.py new file mode 100644 index 0000000..bcb8302 --- /dev/null +++ b/backend/src/foresight/ingest/__init__.py @@ -0,0 +1 @@ +"""Ingestion of external data sources: events, weather, flights, competitor rates.""" diff --git a/backend/src/foresight/ingest/normalize.py b/backend/src/foresight/ingest/normalize.py new file mode 100644 index 0000000..abeb121 --- /dev/null +++ b/backend/src/foresight/ingest/normalize.py @@ -0,0 +1,254 @@ +"""Canonical event contract shared by every external source adapter. + +PredictHQ keys its records by event, AskNews keys them by article, and both must +land on the same row for the same real-world event -- so the dedupe key and the +field normalizers live here rather than in either adapter. +""" + +from __future__ import annotations + +import hashlib +import re +import unicodedata +from dataclasses import dataclass, field +from datetime import UTC, date, datetime, tzinfo +from enum import StrEnum +from typing import TYPE_CHECKING, Any +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +if TYPE_CHECKING: + from collections.abc import Sequence + + +class Source(StrEnum): + PREDICTHQ = "predicthq" + ASKNEWS = "asknews" + + +class EventStatus(StrEnum): + ACTIVE = "active" + PROVISIONAL = "provisional" # extracted from text, not yet corroborated + CANCELLED = "cancelled" + DELETED = "deleted" # withdrawn upstream + + +class DatePrecision(StrEnum): + """How precisely the source pinned the date. + + Anything coarser than DAY is unusable for per-night pricing but still worth + keeping as early signal, so it is recorded rather than rounded away. + """ + + DAY = "day" + MONTH = "month" + QUARTER = "quarter" + + +class EventCategory(StrEnum): + """PredictHQ's taxonomy, adopted project-wide. + + Text sources classify into these buckets instead of inventing a second + vocabulary. + """ + + ACADEMIC = "academic" + AIRPORT_DELAYS = "airport-delays" + COMMUNITY = "community" + CONCERTS = "concerts" + CONFERENCES = "conferences" + DAYLIGHT_SAVINGS = "daylight-savings" + DISASTERS = "disasters" + EXPOS = "expos" + FESTIVALS = "festivals" + HEALTH_WARNINGS = "health-warnings" + OBSERVANCES = "observances" + PERFORMING_ARTS = "performing-arts" + POLITICS = "politics" + PUBLIC_HOLIDAYS = "public-holidays" + SCHOOL_HOLIDAYS = "school-holidays" + SEVERE_WEATHER = "severe-weather" + SPORTS = "sports" + TERROR = "terror" + UNKNOWN = "unknown" + + +_BRACKETED = re.compile(r"[(\[{][^)\]}]*[)\]}]") +_PRESENTER_TAIL = re.compile( + r"\s*[-–—:|]?\s*(?:presented|sponsored|brought)\s+by\b.*$", re.I +) +_GUEST_TAIL = re.compile( + r"\s*\b(?:feat\.?|featuring|w/|with\s+special\s+guests?)\b.*$", re.I +) +_TOUR_TAIL = re.compile( + r"\s*\b(?:the\s+)?\d{4}(?:[/-]\d{2,4})?\s+\S*\s*tour\b.*$", re.I +) +_NON_ALNUM = re.compile(r"[^a-z0-9]+") +_INITIALISM = re.compile(r"\b(?:[a-z] ){1,}[a-z]\b") +_LEADING_ARTICLE = re.compile(r"^(?:the|a|an)\s+") + + +def _strip_accents(text: str) -> str: + decomposed = unicodedata.normalize("NFKD", text) + return "".join(ch for ch in decomposed if not unicodedata.combining(ch)) + + +def normalize_title(title: str) -> str: + """Reduce a title to the part two sources are likely to agree on.""" + text = _strip_accents(title).lower() + text = _BRACKETED.sub(" ", text) + for pattern in (_PRESENTER_TAIL, _GUEST_TAIL, _TOUR_TAIL): + text = pattern.sub("", text) + text = _NON_ALNUM.sub(" ", text).strip() + # "fontaines d.c." and "fontaines dc" must collapse to one key. + text = _INITIALISM.sub(lambda m: m.group(0).replace(" ", ""), text) + text = _LEADING_ARTICLE.sub("", text) + return " ".join(text.split()) + + +def slugify_city(city: str) -> str: + """Normalize a city name. Drops any trailing region, e.g. "Dublin, Ireland".""" + head = city.split(",")[0] + slug = _NON_ALNUM.sub("-", _strip_accents(head).lower()) + return slug.strip("-") + + +def dedupe_key(*, title: str, city: str, start_local_date: date) -> str: + """Stable identity for a real-world event occurrence. + + Category is deliberately excluded: two sources routinely disagree on it, and + disagreement must not split one event into two rows. + """ + basis = ( + f"{normalize_title(title)}|{slugify_city(city)}|{start_local_date.isoformat()}" + ) + return hashlib.sha256(basis.encode("utf-8")).hexdigest() + + +def resolve_timezone(name: str | None) -> tzinfo: + """IANA name to tzinfo, falling back to UTC for missing or unknown zones.""" + if not name: + return UTC + try: + return ZoneInfo(name) + except (ZoneInfoNotFoundError, ValueError): + return UTC + + +def to_local_date(moment: datetime, timezone: str | None) -> date: + """Calendar date in the event's own timezone. + + A stay-night is local, so a 21:00 event must not drift onto the next night + just because its UTC timestamp crosses midnight. + """ + aware = moment if moment.tzinfo else moment.replace(tzinfo=UTC) + return aware.astimezone(resolve_timezone(timezone)).date() + + +def coords_from_geojson( + location: Sequence[float] | None, +) -> tuple[float | None, float | None]: + """PredictHQ's `location` is GeoJSON order -- [longitude, latitude].""" + if not location or len(location) < 2: + return None, None + longitude, latitude = float(location[0]), float(location[1]) + return latitude, longitude + + +@dataclass(slots=True) +class NormalizedEvent: + """What every adapter must produce. Invalid records raise on construction.""" + + source: Source + source_ref: str + title: str + city: str + start_local_date: date + end_local_date: date + + timezone: str = "UTC" + category: EventCategory = EventCategory.UNKNOWN + date_precision: DatePrecision = DatePrecision.DAY + status: EventStatus = EventStatus.ACTIVE + confidence: float = 1.0 + + venue: str | None = None + country: str | None = None + latitude: float | None = None + longitude: float | None = None + start_at_utc: datetime | None = None + end_at_utc: datetime | None = None + expected_attendance: int | None = None + impact_rank: int | None = None + local_rank: int | None = None + url: str | None = None + + source_updated_at: datetime | None = None + payload: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.source_ref: + raise ValueError("source_ref is required") + if not self.title.strip(): + raise ValueError("title is empty") + if not self.city.strip(): + raise ValueError("city is empty") + if not normalize_title(self.title): + raise ValueError(f"title {self.title!r} normalizes to empty") + if not slugify_city(self.city): + raise ValueError(f"city {self.city!r} normalizes to empty") + if self.end_local_date < self.start_local_date: + raise ValueError( + f"end {self.end_local_date} precedes start {self.start_local_date}" + ) + if not 0.0 <= self.confidence <= 1.0: + raise ValueError(f"confidence {self.confidence} outside 0..1") + if (self.latitude is None) != (self.longitude is None): + raise ValueError("latitude and longitude must be set together") + if self.latitude is not None and not -90.0 <= self.latitude <= 90.0: + raise ValueError( + f"latitude {self.latitude} out of range -- check for swapped lon/lat" + ) + if self.longitude is not None and not -180.0 <= self.longitude <= 180.0: + raise ValueError(f"longitude {self.longitude} out of range") + + @property + def title_norm(self) -> str: + return normalize_title(self.title) + + @property + def city_slug(self) -> str: + return slugify_city(self.city) + + @property + def dedupe_key(self) -> str: + return dedupe_key( + title=self.title, city=self.city, start_local_date=self.start_local_date + ) + + def mas_row(self) -> dict[str, Any]: + """Column values for the canonical `events` row.""" + return { + "dedupe_key": self.dedupe_key, + "title": self.title, + "title_norm": self.title_norm, + "category": self.category, + "venue": self.venue, + "city": self.city, + "city_slug": self.city_slug, + "country": self.country, + "latitude": self.latitude, + "longitude": self.longitude, + "start_local_date": self.start_local_date, + "end_local_date": self.end_local_date, + "start_at_utc": self.start_at_utc, + "end_at_utc": self.end_at_utc, + "timezone": self.timezone, + "date_precision": self.date_precision, + "expected_attendance": self.expected_attendance, + "impact_rank": self.impact_rank, + "local_rank": self.local_rank, + "confidence": self.confidence, + "status": self.status, + "url": self.url, + "primary_source": self.source, + } diff --git a/backend/tests/test_normalize.py b/backend/tests/test_normalize.py new file mode 100644 index 0000000..85b07af --- /dev/null +++ b/backend/tests/test_normalize.py @@ -0,0 +1,140 @@ +"""Normalization contract: identity, idempotency, and the known source gotchas.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime +from zoneinfo import ZoneInfo + +import pytest + +from foresight.ingest.normalize import ( + DatePrecision, + EventCategory, + NormalizedEvent, + Source, + coords_from_geojson, + dedupe_key, + normalize_title, + slugify_city, + to_local_date, +) + + +def test_dedupe_key_is_stable_across_runs() -> None: + args = { + "title": "Electric Picnic", + "city": "Stradbally", + "start_local_date": date(2026, 8, 28), + } + + assert dedupe_key(**args) == dedupe_key(**args) + + +@pytest.mark.parametrize( + ("left", "right"), + [ + ("Fontaines D.C.", "Fontaines DC"), + ("The National", "National"), + ("Hozier - Presented by Aiken Promotions", "Hozier"), + ("Kneecap feat. special guests", "Kneecap"), + ("Dua Lipa (Rescheduled)", "Dua Lipa"), + ("Sigur Rós", "Sigur Ros"), + ], +) +def test_titles_that_mean_the_same_event_collapse(left: str, right: str) -> None: + assert normalize_title(left) == normalize_title(right) + + +def test_distinct_titles_do_not_collapse() -> None: + assert normalize_title("Longitude Festival") != normalize_title("Latitude Festival") + + +def test_city_slug_drops_trailing_region() -> None: + assert slugify_city("Dublin, Ireland") == slugify_city("Dublin") == "dublin" + + +def test_geojson_coords_are_lon_lat_ordered() -> None: + # PredictHQ's Dublin: [-6.26, 53.35]. Latitude must come back as 53.35. + assert coords_from_geojson([-6.26, 53.35]) == (53.35, -6.26) + + +def test_geojson_coords_tolerate_missing_location() -> None: + assert coords_from_geojson(None) == (None, None) + + +def test_local_date_uses_event_timezone_not_utc() -> None: + # 20:00 on 14 March in Los Angeles is already 15 March in UTC. + moment = datetime(2026, 3, 15, 3, 0, tzinfo=UTC) + + assert to_local_date(moment, "America/Los_Angeles") == date(2026, 3, 14) + assert to_local_date(moment, None) == date(2026, 3, 15) + + +def test_unknown_timezone_falls_back_to_utc() -> None: + moment = datetime(2026, 3, 15, 3, 0, tzinfo=UTC) + + assert to_local_date(moment, "Mars/Olympus_Mons") == date(2026, 3, 15) + + +def _event(**overrides: object) -> NormalizedEvent: + base: dict[str, object] = { + "source": Source.PREDICTHQ, + "source_ref": "z13B8ehGzCCZjPM3Ff", + "title": "Electric Picnic 2026", + "city": "Stradbally", + "start_local_date": date(2026, 8, 28), + "end_local_date": date(2026, 8, 30), + } + return NormalizedEvent(**(base | overrides)) # type: ignore[arg-type] + + +def test_two_sources_seeing_one_event_share_a_dedupe_key() -> None: + from_phq = _event(title="Electric Picnic 2026", category=EventCategory.FESTIVALS) + from_news = _event( + source=Source.ASKNEWS, + source_ref="https://example.test/electric-picnic", + title="Electric Picnic 2026 (Festival)", + city="Stradbally, Ireland", + category=EventCategory.CONCERTS, + confidence=0.6, + ) + + assert from_phq.dedupe_key == from_news.dedupe_key + + +def test_as_row_carries_derived_identity_columns() -> None: + row = _event().as_row() + + assert row["dedupe_key"] == _event().dedupe_key + assert row["city_slug"] == "stradbally" + assert row["primary_source"] is Source.PREDICTHQ + assert row["date_precision"] is DatePrecision.DAY + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"end_local_date": date(2026, 8, 27)}, "precedes start"), + ({"confidence": 1.4}, "outside 0..1"), + ({"latitude": 53.35}, "set together"), + # Singapore with lon/lat transposed: 103.5 is not a latitude. + ({"latitude": 103.85, "longitude": 1.29}, "swapped"), + ({"title": "(Live)"}, "normalizes to empty"), + ({"source_ref": ""}, "source_ref is required"), + ], +) +def test_malformed_records_are_rejected( + overrides: dict[str, object], message: str +) -> None: + with pytest.raises(ValueError, match=message): + _event(**overrides) + + +def test_swapped_coordinates_are_caught_only_when_out_of_range() -> None: + # A real swap inside both valid ranges cannot be detected here; the + # coords_from_geojson helper is the defence, not validation. + assert _event(latitude=53.35, longitude=-6.26).latitude == 53.35 + + +def test_zoneinfo_dependency_is_available() -> None: + assert ZoneInfo("Europe/Dublin").key == "Europe/Dublin" From 65cfca3dc6772974e4b1a25a9c5aed520a24967f Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 20:10:17 -0400 Subject: [PATCH 02/21] fix: move ingest package into foresight/ and restore NormalizedEvent.as_row The rebase of 9b83901 onto the src/ -> foresight/ refactor left ingest/ under backend/src/, so foresight.ingest was unimportable and broke model and Alembic imports. as_row was misspelled mas_row; the tests and the documented contract both use as_row. Co-Authored-By: Claude Opus 5.5 --- backend/{src => }/foresight/ingest/__init__.py | 0 backend/{src => }/foresight/ingest/normalize.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename backend/{src => }/foresight/ingest/__init__.py (100%) rename backend/{src => }/foresight/ingest/normalize.py (99%) diff --git a/backend/src/foresight/ingest/__init__.py b/backend/foresight/ingest/__init__.py similarity index 100% rename from backend/src/foresight/ingest/__init__.py rename to backend/foresight/ingest/__init__.py diff --git a/backend/src/foresight/ingest/normalize.py b/backend/foresight/ingest/normalize.py similarity index 99% rename from backend/src/foresight/ingest/normalize.py rename to backend/foresight/ingest/normalize.py index abeb121..6c1d56b 100644 --- a/backend/src/foresight/ingest/normalize.py +++ b/backend/foresight/ingest/normalize.py @@ -225,7 +225,7 @@ def dedupe_key(self) -> str: title=self.title, city=self.city, start_local_date=self.start_local_date ) - def mas_row(self) -> dict[str, Any]: + def as_row(self) -> dict[str, Any]: """Column values for the canonical `events` row.""" return { "dedupe_key": self.dedupe_key, From e9667e21f84091d0ab9eee8c3bcec23ee4c42bb3 Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 20:23:29 -0400 Subject: [PATCH 03/21] build: add httpx runtime dependency and event source settings Settings now also reads the repo-root .env, which docker-compose already uses, so commands run from backend/ pick up the API credentials. Co-Authored-By: Claude Opus 5.5 --- .env.example | 5 +++++ backend/foresight/config.py | 8 +++++++- backend/pyproject.toml | 1 + backend/uv.lock | 8 +++++--- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 7791da1..a4cedd9 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,8 @@ DEBUG=false # Database # Format: postgresql+asyncpg://:@:/ DATABASE_URL=postgresql+asyncpg://user:password@localhost:5432/dbname + +# External event sources +PREDICTHQ_TOKEN= +ASKNEWS_CLIENT_ID= +ASKNEWS_CLIENT_SECRET= diff --git a/backend/foresight/config.py b/backend/foresight/config.py index 24eb3da..5dbeb15 100644 --- a/backend/foresight/config.py +++ b/backend/foresight/config.py @@ -3,7 +3,8 @@ class Settings(BaseSettings): model_config = SettingsConfigDict( - env_file=".env", + # Root .env is the one docker-compose reads; backend/.env overrides it. + env_file=("../.env", ".env"), env_file_encoding="utf-8", extra="ignore", ) @@ -17,5 +18,10 @@ class Settings(BaseSettings): "postgresql+asyncpg://foresight:foresight@localhost:5432/foresight_db" ) + # External event sources + predicthq_token: str = "" + asknews_client_id: str = "" + asknews_client_secret: str = "" + settings = Settings() diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 3412379..badaf9a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "python-dotenv>=1.1", "pandas>=2.2", "psycopg2-binary>=2.9", + "httpx>=0.28", ] [dependency-groups] diff --git a/backend/uv.lock b/backend/uv.lock index d3769fd..43285b7 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1,15 +1,15 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version < '3.13' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] @@ -331,6 +331,7 @@ dependencies = [ { name = "alembic" }, { name = "asyncpg" }, { name = "fastapi" }, + { name = "httpx" }, { name = "pandas" }, { name = "psycopg2-binary" }, { name = "pydantic-settings" }, @@ -361,6 +362,7 @@ requires-dist = [ { name = "alembic", specifier = ">=1.15" }, { name = "asyncpg", specifier = ">=0.30" }, { name = "fastapi", specifier = ">=0.115" }, + { name = "httpx", specifier = ">=0.28" }, { name = "pandas", specifier = ">=2.2" }, { name = "psycopg2-binary", specifier = ">=2.9" }, { name = "pydantic-settings", specifier = ">=2.9" }, From c16ba21d682485bf17971d58ed97012d41df6b4d Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 20:23:29 -0400 Subject: [PATCH 04/21] feat: upsert normalized events onto canonical rows Resolves by (source, source_ref) before dedupe_key so a start-date revision updates the existing row instead of orphaning it. Structured sources outrank text-extracted ones, and a revision that lands on another row's key merges the two. Co-Authored-By: Claude Opus 5.5 --- backend/foresight/ingest/repository.py | 168 ++++++++++++++++++++ backend/tests/test_repository.py | 209 +++++++++++++++++++++++++ 2 files changed, 377 insertions(+) create mode 100644 backend/foresight/ingest/repository.py create mode 100644 backend/tests/test_repository.py diff --git a/backend/foresight/ingest/repository.py b/backend/foresight/ingest/repository.py new file mode 100644 index 0000000..7092071 --- /dev/null +++ b/backend/foresight/ingest/repository.py @@ -0,0 +1,168 @@ +"""Persist NormalizedEvents onto the canonical `events` row and its observations.""" + +from __future__ import annotations + +from enum import StrEnum +from typing import TYPE_CHECKING, Any + +from sqlalchemy import delete, func, literal_column, or_, select, update +from sqlalchemy.dialects.postgresql import insert + +from foresight.database.models.external import Event, EventObservation +from foresight.ingest.normalize import EventStatus, Source + +if TYPE_CHECKING: + from collections.abc import Mapping + + from sqlalchemy.ext.asyncio import AsyncSession + + from foresight.ingest.normalize import NormalizedEvent + +_EVENTS = Event.__table__ +_OBSERVATIONS = EventObservation.__table__ + +# Structured listings outrank dates and venues extracted from prose. +_PRECEDENCE = {Source.PREDICTHQ: 2, Source.ASKNEWS: 1} + + +class Outcome(StrEnum): + NEW = "new" + UPDATED = "updated" + UNCHANGED = "unchanged" + + +async def upsert_event(session: AsyncSession, event: NormalizedEvent) -> Outcome: + """Resolve `event` to its canonical row, write it, and record the sighting. + + Does not commit; the caller owns the transaction. + """ + row = event.as_row() + event_id = await session.scalar( + select(_OBSERVATIONS.c.event_id).where( + _OBSERVATIONS.c.source == event.source, + _OBSERVATIONS.c.source_ref == event.source_ref, + ) + ) + + # Source identity must be checked before dedupe_key: a revised start date + # changes the key, and resolving by key first would orphan the old row. + if event_id is not None: + event_id = await _absorb_key_collision(session, event_id, row["dedupe_key"]) + outcome = await _apply(session, event_id, row, own_record=True) + else: + event_id, inserted = await _insert_or_find(session, row) + outcome = ( + Outcome.NEW + if inserted + else await _apply(session, event_id, row, own_record=False) + ) + + await _record_observation(session, event, event_id) + return outcome + + +async def _absorb_key_collision( + session: AsyncSession, event_id: int, new_key: str +) -> int: + """If a revision moved onto a key another sighting already holds, merge.""" + other_id = await session.scalar( + select(_EVENTS.c.id).where( + _EVENTS.c.dedupe_key == new_key, _EVENTS.c.id != event_id + ) + ) + if other_id is None: + return event_id + + await session.execute( + update(_OBSERVATIONS) + .where(_OBSERVATIONS.c.event_id == event_id) + .values(event_id=other_id) + ) + await session.execute(delete(_EVENTS).where(_EVENTS.c.id == event_id)) + return other_id + + +async def _insert_or_find( + session: AsyncSession, row: Mapping[str, Any] +) -> tuple[int, bool]: + stmt = ( + insert(_EVENTS) + .values(**row) + .on_conflict_do_update( + index_elements=[_EVENTS.c.dedupe_key], + set_={"last_seen_at": func.clock_timestamp()}, + ) + # xmax is 0 only on a freshly inserted tuple, never on a conflict update. + .returning(_EVENTS.c.id, literal_column("(xmax = 0)").label("inserted")) + ) + result = (await session.execute(stmt)).one() + return result.id, result.inserted + + +async def _apply( + session: AsyncSession, + event_id: int, + row: Mapping[str, Any], + *, + own_record: bool, +) -> Outcome: + current = ( + ( + await session.execute( + select(_EVENTS).where(_EVENTS.c.id == event_id).with_for_update() + ) + ) + .mappings() + .one() + ) + + changes: dict[str, Any] = {} + if _may_overwrite(current, row, own_record=own_record): + changes = {k: v for k, v in row.items() if current[k] != v} + + await session.execute( + update(_EVENTS) + .where(_EVENTS.c.id == event_id) + .values(**changes, last_seen_at=func.clock_timestamp()) + ) + return Outcome.UPDATED if changes else Outcome.UNCHANGED + + +def _may_overwrite( + current: Mapping[str, Any], row: Mapping[str, Any], *, own_record: bool +) -> bool: + # Another record's deletion (e.g. a PredictHQ duplicate that collapses onto + # the surviving listing) must not take down an event seen independently. + if not own_record and row["status"] is EventStatus.DELETED: + return False + incoming = _PRECEDENCE[row["primary_source"]] + return incoming >= _PRECEDENCE[Source(current["primary_source"])] + + +async def _record_observation( + session: AsyncSession, event: NormalizedEvent, event_id: int +) -> None: + stmt = insert(_OBSERVATIONS).values( + event_id=event_id, + source=event.source, + source_ref=event.source_ref, + source_updated_at=event.source_updated_at, + payload=event.payload, + ) + excluded = stmt.excluded + await session.execute( + stmt.on_conflict_do_update( + index_elements=[_OBSERVATIONS.c.source, _OBSERVATIONS.c.source_ref], + set_={ + "event_id": excluded.event_id, + "source_updated_at": excluded.source_updated_at, + "payload": excluded.payload, + "captured_at": func.clock_timestamp(), + }, + # captured_at marks when this payload version arrived, not every rerun. + where=or_( + _OBSERVATIONS.c.payload.is_distinct_from(excluded.payload), + _OBSERVATIONS.c.event_id.is_distinct_from(excluded.event_id), + ), + ) + ) diff --git a/backend/tests/test_repository.py b/backend/tests/test_repository.py new file mode 100644 index 0000000..5c919ec --- /dev/null +++ b/backend/tests/test_repository.py @@ -0,0 +1,209 @@ +"""Upsert resolution order against a real Postgres; skipped when none is reachable.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime +from typing import TYPE_CHECKING + +import pytest +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine + +from foresight.config import settings +from foresight.database.models.external import Event, EventObservation +from foresight.ingest.normalize import EventStatus, NormalizedEvent, Source +from foresight.ingest.repository import Outcome, upsert_event + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +@pytest.fixture +async def session() -> AsyncIterator[AsyncSession]: + engine = create_async_engine(settings.database_url) + try: + conn = await engine.connect() + except (OSError, ConnectionError) as exc: + await engine.dispose() + pytest.skip(f"Postgres unavailable: {exc}") + + trans = await conn.begin() + async with AsyncSession( + bind=conn, join_transaction_mode="create_savepoint" + ) as session: + yield session + await trans.rollback() + await conn.close() + await engine.dispose() + + +def _phq(**overrides: object) -> NormalizedEvent: + base: dict[str, object] = { + "source": Source.PREDICTHQ, + "source_ref": "phq-test-1", + "title": "Hozier", + "city": "Dublin", + "start_local_date": date(2026, 11, 14), + "end_local_date": date(2026, 11, 14), + "timezone": "Europe/Dublin", + "local_rank": 71, + "payload": {"id": "phq-test-1"}, + } + return NormalizedEvent(**(base | overrides)) # type: ignore[arg-type] + + +def _news(**overrides: object) -> NormalizedEvent: + base: dict[str, object] = { + "source": Source.ASKNEWS, + "source_ref": "https://example.test/hozier-dublin", + "title": "Hozier (Live)", + "city": "Dublin, Ireland", + "start_local_date": date(2026, 11, 14), + "end_local_date": date(2026, 11, 14), + "venue": "Somewhere Vague", + "confidence": 0.6, + "status": EventStatus.PROVISIONAL, + "payload": {"url": "https://example.test/hozier-dublin"}, + } + return NormalizedEvent(**(base | overrides)) # type: ignore[arg-type] + + +async def _count(session: AsyncSession, model: type) -> int: + return await session.scalar(select(func.count()).select_from(model)) + + +async def _only_event(session: AsyncSession) -> Event: + return ( + await session.scalars(select(Event).execution_options(populate_existing=True)) + ).one() + + +async def test_start_date_revision_updates_the_row_instead_of_orphaning_it( + session: AsyncSession, +) -> None: + assert await upsert_event(session, _phq()) is Outcome.NEW + first = await _only_event(session) + first_id, first_seen, first_last = first.id, first.first_seen_at, first.last_seen_at + + revised = _phq( + start_local_date=date(2026, 11, 15), + end_local_date=date(2026, 11, 15), + payload={"id": "phq-test-1", "rev": 2}, + ) + assert await upsert_event(session, revised) is Outcome.UPDATED + + assert await _count(session, Event) == 1 + assert await _count(session, EventObservation) == 1 + event = await _only_event(session) + assert event.id == first_id + assert event.start_local_date == date(2026, 11, 15) + assert event.dedupe_key == revised.dedupe_key + assert event.first_seen_at == first_seen + assert event.last_seen_at > first_last + + +async def test_identical_rerun_is_unchanged_but_still_bumps_last_seen( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + before = (await _only_event(session)).last_seen_at + + assert await upsert_event(session, _phq()) is Outcome.UNCHANGED + assert await _count(session, Event) == 1 + assert await _count(session, EventObservation) == 1 + assert (await _only_event(session)).last_seen_at > before + + +async def test_second_source_attaches_to_the_existing_event( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + + # AskNews is outranked, so the PredictHQ values stand. + assert await upsert_event(session, _news()) is Outcome.UNCHANGED + + assert await _count(session, Event) == 1 + assert await _count(session, EventObservation) == 2 + event = await _only_event(session) + assert event.primary_source is Source.PREDICTHQ + assert event.venue is None + assert event.status is EventStatus.ACTIVE + + +async def test_higher_precedence_source_takes_over_a_text_extracted_event( + session: AsyncSession, +) -> None: + await upsert_event(session, _news()) + + assert await upsert_event(session, _phq()) is Outcome.UPDATED + event = await _only_event(session) + assert event.primary_source is Source.PREDICTHQ + assert event.confidence == 1.0 + assert event.status is EventStatus.ACTIVE + + +async def test_revision_onto_an_existing_key_merges_rather_than_duplicates( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + # An article already reported the date PredictHQ is about to move to. + await upsert_event( + session, + _news(start_local_date=date(2026, 11, 15), end_local_date=date(2026, 11, 15)), + ) + assert await _count(session, Event) == 2 + + await upsert_event( + session, + _phq(start_local_date=date(2026, 11, 15), end_local_date=date(2026, 11, 15)), + ) + + assert await _count(session, Event) == 1 + event = await _only_event(session) + assert event.primary_source is Source.PREDICTHQ + linked = await session.scalars(select(EventObservation.event_id)) + assert set(linked) == {event.id} + + +async def test_deletion_of_a_known_record_marks_the_event_deleted( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + + deleted = _phq(status=EventStatus.DELETED, payload={"state": "deleted"}) + assert await upsert_event(session, deleted) is Outcome.UPDATED + assert (await _only_event(session)).status is EventStatus.DELETED + + +async def test_another_records_deletion_does_not_kill_a_live_event( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + + duplicate = _phq( + source_ref="phq-test-dupe", + status=EventStatus.DELETED, + payload={"deleted_reason": "duplicate"}, + ) + assert await upsert_event(session, duplicate) is Outcome.UNCHANGED + assert (await _only_event(session)).status is EventStatus.ACTIVE + assert await _count(session, EventObservation) == 2 + + +async def test_observation_payload_tracks_the_latest_revision( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + updated_at = datetime(2026, 10, 6, 9, 30, tzinfo=UTC) + await upsert_event( + session, + _phq(payload={"id": "phq-test-1", "rev": 2}, source_updated_at=updated_at), + ) + + obs = ( + await session.scalars( + select(EventObservation).execution_options(populate_existing=True) + ) + ).one() + assert obs.payload == {"id": "phq-test-1", "rev": 2} + assert obs.source_updated_at == updated_at From 8118535149d10ca751393560850e31e560e02ef4 Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 20:23:29 -0400 Subject: [PATCH 05/21] feat: add PredictHQ events adapter Geo-filters around the hotel, pages through results, and fetches incrementally on updated.gte. Requests deleted states explicitly, since the API returns only active events by default. Co-Authored-By: Claude Opus 5.5 --- backend/foresight/ingest/predicthq.py | 162 ++++++++++++++++++++++++++ backend/tests/test_predicthq.py | 142 ++++++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 backend/foresight/ingest/predicthq.py create mode 100644 backend/tests/test_predicthq.py diff --git a/backend/foresight/ingest/predicthq.py b/backend/foresight/ingest/predicthq.py new file mode 100644 index 0000000..6ea1013 --- /dev/null +++ b/backend/foresight/ingest/predicthq.py @@ -0,0 +1,162 @@ +"""PredictHQ adapter: events near a hotel as NormalizedEvents. No DB writes.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any + +import httpx + +from foresight.ingest.normalize import ( + DatePrecision, + EventCategory, + EventStatus, + NormalizedEvent, + Source, + coords_from_geojson, + to_local_date, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + +log = logging.getLogger(__name__) + +API_URL = "https://api.predicthq.com/v1/events/" +PAGE_LIMIT = 500 +# The API returns only `active` unless asked; an unseen deletion would leave a +# cancelled event in the demand forecast indefinitely. +STATES = "active,predicted,deleted" + +_STATUS = { + "active": EventStatus.ACTIVE, + "predicted": EventStatus.PROVISIONAL, + "deleted": EventStatus.DELETED, +} + + +@dataclass(slots=True) +class Rejected: + source_ref: str + reason: str + + +@dataclass(slots=True) +class FetchResult: + events: list[NormalizedEvent] = field(default_factory=list) + rejected: list[Rejected] = field(default_factory=list) + + @property + def fetched(self) -> int: + return len(self.events) + len(self.rejected) + + +async def fetch_events( + *, + token: str, + latitude: float, + longitude: float, + city: str, + updated_since: datetime, + radius: str = "10km", + transport: httpx.AsyncBaseTransport | None = None, +) -> FetchResult: + """Events within `radius` of a point, revised upstream since `updated_since`. + + `city` is the hotel's market and becomes every event's city, so a suburb + venue still shares a dedupe key with a news article naming the city. + """ + params = { + "within": f"{radius}@{latitude},{longitude}", + "updated.gte": updated_since.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S"), + # A day of slack so an event running tonight in a zone behind UTC stays in. + "active.gte": (datetime.now(UTC).date() - timedelta(days=1)).isoformat(), + "state": STATES, + "limit": PAGE_LIMIT, + "sort": "updated", + } + headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} + result = FetchResult() + + async with httpx.AsyncClient( + headers=headers, timeout=30.0, transport=transport + ) as client: + url: str | None = API_URL + query: dict[str, Any] | None = params + while url: + response = await client.get(url, params=query) + response.raise_for_status() + body = response.json() + if body.get("overflow"): + log.warning("PredictHQ result overflow: plan limit truncated results") + + for raw in body.get("results", []): + try: + result.events.append(to_normalized(raw, city=city)) + except (KeyError, TypeError, ValueError) as exc: + ref = str(raw.get("id", "?")) if isinstance(raw, dict) else "?" + log.warning("rejected PredictHQ event %s: %s", ref, exc) + result.rejected.append(Rejected(ref, _reason(exc))) + + # `next` already carries every query parameter. + url, query = body.get("next"), None + + return result + + +def to_normalized(raw: Mapping[str, Any], *, city: str) -> NormalizedEvent: + start = _parse_instant(raw["start"]) + end = _parse_instant(raw["end"]) if raw.get("end") else start + timezone = raw.get("timezone") + latitude, longitude = coords_from_geojson(raw.get("location")) + + return NormalizedEvent( + source=Source.PREDICTHQ, + source_ref=raw["id"], + title=raw["title"], + city=city, + start_local_date=to_local_date(start, timezone), + end_local_date=to_local_date(end, timezone), + timezone=timezone or "UTC", + category=_category(raw.get("category")), + date_precision=DatePrecision.DAY, + status=_STATUS[raw["state"]], + confidence=1.0, + venue=_venue(raw.get("entities") or []), + country=raw.get("country"), + latitude=latitude, + longitude=longitude, + start_at_utc=start, + end_at_utc=end, + expected_attendance=raw.get("phq_attendance"), + impact_rank=raw.get("rank"), + local_rank=raw.get("local_rank"), + source_updated_at=_parse_instant(raw["updated"]) + if raw.get("updated") + else None, + payload=dict(raw), + ) + + +def _parse_instant(value: str) -> datetime: + moment = datetime.fromisoformat(value) + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +def _category(value: str | None) -> EventCategory: + try: + return EventCategory(value) + except ValueError: + return EventCategory.UNKNOWN + + +def _venue(entities: list[Mapping[str, Any]]) -> str | None: + return next((e.get("name") for e in entities if e.get("type") == "venue"), None) + + +def _reason(exc: Exception) -> str: + if isinstance(exc, KeyError): + return f"missing field {exc.args[0]!r}" + return str(exc) diff --git a/backend/tests/test_predicthq.py b/backend/tests/test_predicthq.py new file mode 100644 index 0000000..80ab283 --- /dev/null +++ b/backend/tests/test_predicthq.py @@ -0,0 +1,142 @@ +"""PredictHQ adapter against a mocked API: request shape, paging, field mapping.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime +from typing import Any + +import httpx + +from foresight.ingest.normalize import EventCategory, EventStatus, Source +from foresight.ingest.predicthq import API_URL, fetch_events, to_normalized + + +def _raw(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "id": "z13B8ehGzCCZjPM3Ff", + "title": "Fontaines D.C.", + "category": "concerts", + "state": "active", + # 20:00 in Dublin, which is already 19:00Z: the local and UTC dates agree. + "start": "2026-11-14T19:00:00Z", + "end": "2026-11-14T22:30:00Z", + "timezone": "Europe/Dublin", + "location": [-6.2286, 53.3478], + "country": "IE", + "rank": 68, + "local_rank": 81, + "phq_attendance": 13000, + "entities": [ + {"type": "event-group", "name": "Fontaines D.C. Tour"}, + {"type": "venue", "name": "3Arena"}, + ], + "updated": "2026-10-05T08:12:44Z", + } + return base | overrides + + +def test_maps_geojson_ranks_and_venue() -> None: + event = to_normalized(_raw(), city="Dublin") + + assert event.source is Source.PREDICTHQ + assert (event.latitude, event.longitude) == (53.3478, -6.2286) + assert (event.impact_rank, event.local_rank) == (68, 81) + assert event.venue == "3Arena" + assert event.category is EventCategory.CONCERTS + assert event.expected_attendance == 13000 + assert event.confidence == 1.0 + assert event.source_updated_at == datetime(2026, 10, 5, 8, 12, 44, tzinfo=UTC) + assert event.payload["id"] == "z13B8ehGzCCZjPM3Ff" + + +def test_late_evening_event_keeps_its_local_night() -> None: + # 21:30 in Los Angeles on 14 Nov is 05:30Z on 15 Nov. + event = to_normalized( + _raw( + start="2026-11-15T05:30:00Z", + end="2026-11-15T07:00:00Z", + timezone="America/Los_Angeles", + location=[-118.2673, 34.0430], + ), + city="Los Angeles", + ) + + assert event.start_local_date == date(2026, 11, 14) + assert event.end_local_date == date(2026, 11, 14) + + +def test_deleted_state_becomes_deleted_status() -> None: + event = to_normalized( + _raw(state="deleted", deleted_reason="cancelled"), city="Dublin" + ) + assert event.status is EventStatus.DELETED + + +def test_unknown_category_falls_back() -> None: + event = to_normalized(_raw(category="something-new"), city="Dublin") + assert event.category is EventCategory.UNKNOWN + + +async def test_fetch_filters_by_radius_and_revision_clock_and_follows_next() -> None: + requests: list[httpx.Request] = [] + page_two = f"{API_URL}?offset=1&limit=500" + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.url.params.get("offset") == "1": + return httpx.Response( + 200, + json={ + "count": 3, + "next": None, + "results": [ + _raw(id="b", state="deleted"), + _raw(id="bad", title="(Live)"), + ], + }, + ) + return httpx.Response( + 200, json={"count": 3, "next": page_two, "results": [_raw(id="a")]} + ) + + result = await fetch_events( + token="test-token", + latitude=53.3498, + longitude=-6.2603, + city="Dublin", + updated_since=datetime(2026, 9, 29, 14, 0, tzinfo=UTC), + transport=httpx.MockTransport(handler), + ) + + first = requests[0] + assert first.headers["Authorization"] == "Bearer test-token" + assert first.url.params["within"] == "10km@53.3498,-6.2603" + assert first.url.params["updated.gte"] == "2026-09-29T14:00:00" + assert first.url.params["limit"] == "500" + assert "deleted" in first.url.params["state"].split(",") + assert len(requests) == 2 + + assert result.fetched == 3 + assert [e.source_ref for e in result.events] == ["a", "b"] + assert result.events[1].status is EventStatus.DELETED + assert [r.source_ref for r in result.rejected] == ["bad"] + assert "normalizes to empty" in result.rejected[0].reason + + +async def test_record_missing_required_field_is_rejected_not_raised() -> None: + def handler(request: httpx.Request) -> httpx.Response: + broken = _raw(id="nostart") + del broken["start"] + return httpx.Response(200, json={"count": 1, "next": None, "results": [broken]}) + + result = await fetch_events( + token="t", + latitude=0.0, + longitude=0.0, + city="Dublin", + updated_since=datetime(2026, 9, 29, tzinfo=UTC), + transport=httpx.MockTransport(handler), + ) + + assert result.events == [] + assert result.rejected[0].reason == "missing field 'start'" From 3182e4fdd73d5f77123c0a0dae549d3685fb81c9 Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 20:23:30 -0400 Subject: [PATCH 06/21] feat: add events ingest CLI with run report python -m foresight.ingest.run --source predicthq --since 7d --report prints fetched/new/updated/unchanged/rejected plus market totals. Records failing validation are logged and counted as rejected rather than aborting the run. Co-Authored-By: Claude Opus 5.5 --- backend/foresight/ingest/run.py | 224 +++++++++++++++++++++++++++++++ backend/tests/test_ingest_run.py | 54 ++++++++ 2 files changed, 278 insertions(+) create mode 100644 backend/foresight/ingest/run.py create mode 100644 backend/tests/test_ingest_run.py diff --git a/backend/foresight/ingest/run.py b/backend/foresight/ingest/run.py new file mode 100644 index 0000000..619f322 --- /dev/null +++ b/backend/foresight/ingest/run.py @@ -0,0 +1,224 @@ +"""Fetch external events and upsert them into the canonical tables. + + python -m foresight.ingest.run --source predicthq --since 7d --hotel-id 1 --report + python -m foresight.ingest.run --source predicthq --since 90d \\ + --lat 53.3498 --lon -6.2603 --city Dublin --report +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import re +import sys +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING + +from sqlalchemy import func, select + +from foresight.config import settings +from foresight.database.models.external import Event, EventObservation +from foresight.database.models.hotel import Hotel +from foresight.database.session import async_session_factory, engine +from foresight.ingest import predicthq +from foresight.ingest.normalize import Source, slugify_city +from foresight.ingest.repository import Outcome, upsert_event + +if TYPE_CHECKING: + from collections.abc import Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + from foresight.ingest.predicthq import Rejected + +log = logging.getLogger("foresight.ingest") + +_SPAN = re.compile(r"^(\d+)([hdw])$") +_UNITS = {"h": "hours", "d": "days", "w": "weeks"} + + +@dataclass(slots=True) +class Market: + city: str + latitude: float + longitude: float + + +@dataclass(slots=True) +class RunReport: + source: Source + market: Market + radius: str + since: datetime + fetched: int = 0 + outcomes: dict[Outcome, int] = field(default_factory=dict) + rejected: list[Rejected] = field(default_factory=list) + events_by_status: dict[str, int] = field(default_factory=dict) + observations_by_source: dict[str, int] = field(default_factory=dict) + + def count(self, outcome: Outcome) -> int: + return self.outcomes.get(outcome, 0) + + def summary(self) -> str: + return ( + f"fetched={self.fetched} new={self.count(Outcome.NEW)} " + f"updated={self.count(Outcome.UPDATED)} " + f"unchanged={self.count(Outcome.UNCHANGED)} " + f"rejected={len(self.rejected)}" + ) + + def render(self) -> str: + m = self.market + rule = " " + "-" * 15 + lines = [ + f"{self.source.value} ingest | {m.city} | " + f"{self.radius} around {m.latitude:.4f}, {m.longitude:.4f}", + f"revised upstream since {self.since:%Y-%m-%d %H:%M} UTC", + "", + f" {'fetched':<11}{self.fetched:>4}", + rule, + *(f" {outcome.value:<11}{self.count(outcome):>4}" for outcome in Outcome), + f" {'rejected':<11}{len(self.rejected):>4}", + "", + f" {slugify_city(m.city)} now holds " + f"{sum(self.events_by_status.values())} events " + f"({_breakdown(self.events_by_status)})", + f" backed by {sum(self.observations_by_source.values())} observations " + f"({_breakdown(self.observations_by_source)})", + ] + if self.rejected: + lines += ["", " rejected records:"] + lines += [f" {r.source_ref:<22} {r.reason}" for r in self.rejected] + return "\n".join(lines) + + +def _breakdown(counts: dict[str, int]) -> str: + return ", ".join(f"{n} {k}" for k, n in sorted(counts.items())) or "none" + + +def parse_since(value: str, *, now: datetime | None = None) -> datetime: + """`7d`, `12h`, `2w`, or an ISO date/datetime (UTC if no offset given).""" + if match := _SPAN.match(value.strip().lower()): + amount, unit = int(match[1]), match[2] + return (now or datetime.now(UTC)) - timedelta(**{_UNITS[unit]: amount}) + try: + moment = datetime.fromisoformat(value) + except ValueError: + raise argparse.ArgumentTypeError( + f"expected a span like 7d, 12h, 2w or an ISO date, got {value!r}" + ) from None + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m foresight.ingest.run", description=__doc__.splitlines()[0] + ) + parser.add_argument("--source", choices=[Source.PREDICTHQ.value], required=True) + parser.add_argument( + "--since", + type=parse_since, + default="7d", + help="only records revised upstream since then (default: 7d)", + ) + parser.add_argument("--hotel-id", type=int, help="use this hotel's stored lat/lon") + parser.add_argument("--lat", type=float, help="market latitude (no --hotel-id)") + parser.add_argument("--lon", type=float, help="market longitude (no --hotel-id)") + parser.add_argument("--city", help="market city (no --hotel-id)") + parser.add_argument("--radius", default="10km", help="e.g. 10km, 5mi") + parser.add_argument("--report", action="store_true", help="print a full run report") + return parser + + +async def _resolve_market(session: AsyncSession, args: argparse.Namespace) -> Market: + if args.hotel_id is None: + return Market(city=args.city, latitude=args.lat, longitude=args.lon) + + hotel = await session.get(Hotel, args.hotel_id) + if hotel is None: + raise SystemExit(f"error: no hotel with id {args.hotel_id}") + if hotel.latitude is None or hotel.longitude is None: + raise SystemExit(f"error: hotel {hotel.id} ({hotel.name}) has no lat/lon") + return Market(city=hotel.city, latitude=hotel.latitude, longitude=hotel.longitude) + + +async def _market_totals( + session: AsyncSession, city: str +) -> tuple[dict[str, int], dict[str, int]]: + slug = slugify_city(city) + by_status = await session.execute( + select(Event.status, func.count()) + .where(Event.city_slug == slug) + .group_by(Event.status) + ) + by_source = await session.execute( + select(EventObservation.source, func.count()) + .join(Event, EventObservation.event_id == Event.id) + .where(Event.city_slug == slug) + .group_by(EventObservation.source) + ) + return ( + {status.value: n for status, n in by_status.all()}, + {source.value: n for source, n in by_source.all()}, + ) + + +async def run(args: argparse.Namespace) -> RunReport: + async with async_session_factory() as session: + market = await _resolve_market(session, args) + report = RunReport( + source=Source(args.source), + market=market, + radius=args.radius, + since=args.since, + ) + + result = await predicthq.fetch_events( + token=settings.predicthq_token, + latitude=market.latitude, + longitude=market.longitude, + city=market.city, + updated_since=args.since, + radius=args.radius, + ) + report.fetched = result.fetched + report.rejected = result.rejected + + for event in result.events: + outcome = await upsert_event(session, event) + report.outcomes[outcome] = report.outcomes.get(outcome, 0) + 1 + await session.commit() + + report.events_by_status, report.observations_by_source = await _market_totals( + session, market.city + ) + return report + + +async def _main(args: argparse.Namespace) -> None: + try: + report = await run(args) + finally: + await engine.dispose() + print(report.render() if args.report else report.summary()) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = _parser() + args = parser.parse_args(argv) + if args.hotel_id is None and None in (args.lat, args.lon, args.city): + parser.error("give --hotel-id, or all of --lat, --lon and --city") + if args.source == Source.PREDICTHQ and not settings.predicthq_token: + parser.error("PREDICTHQ_TOKEN is not set; add it to .env") + + logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s") + # DEBUG=true turns on SQL echo, which would bury the report. + engine.echo = False + asyncio.run(_main(args)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/tests/test_ingest_run.py b/backend/tests/test_ingest_run.py new file mode 100644 index 0000000..e51ef9c --- /dev/null +++ b/backend/tests/test_ingest_run.py @@ -0,0 +1,54 @@ +"""CLI argument parsing and the report shown at the end of a run.""" + +from __future__ import annotations + +import argparse +from datetime import UTC, datetime + +import pytest + +from foresight.ingest.normalize import Source +from foresight.ingest.predicthq import Rejected +from foresight.ingest.repository import Outcome +from foresight.ingest.run import Market, RunReport, parse_since + +NOW = datetime(2026, 10, 6, 12, 0, tzinfo=UTC) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("7d", datetime(2026, 9, 29, 12, 0, tzinfo=UTC)), + ("12h", datetime(2026, 10, 6, 0, 0, tzinfo=UTC)), + ("2w", datetime(2026, 9, 22, 12, 0, tzinfo=UTC)), + ("2026-10-01", datetime(2026, 10, 1, tzinfo=UTC)), + ], +) +def test_parse_since(value: str, expected: datetime) -> None: + assert parse_since(value, now=NOW) == expected + + +def test_parse_since_rejects_garbage() -> None: + with pytest.raises(argparse.ArgumentTypeError): + parse_since("last tuesday", now=NOW) + + +def test_report_lists_every_count_and_rejection() -> None: + report = RunReport( + source=Source.PREDICTHQ, + market=Market(city="Dublin", latitude=53.3498, longitude=-6.2603), + radius="10km", + since=NOW, + fetched=6, + outcomes={Outcome.NEW: 1, Outcome.UPDATED: 2, Outcome.UNCHANGED: 2}, + rejected=[Rejected("phq-broken", "end precedes start")], + events_by_status={"active": 4, "deleted": 1}, + observations_by_source={"predicthq": 5}, + ) + + assert report.summary() == "fetched=6 new=1 updated=2 unchanged=2 rejected=1" + rendered = report.render() + for line in ("fetched 6", "updated 2", "rejected 1"): + assert line in rendered + assert "dublin now holds 5 events (4 active, 1 deleted)" in rendered + assert "phq-broken" in rendered From 2c9e09c21e74e8852d2ad77bc5b350e45154eddb Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 22:49:41 -0400 Subject: [PATCH 07/21] fix: stop deleted duplicate listings from deleting live events on rerun PredictHQ often lists one event several times, typically one copy active and one deleted. The first run correctly ignored the deleted copy, but on every later run its sighting already pointed at the event, so it counted as the event's own record and its deletion was applied (20 live Dublin events). Live copies that differed slightly also overwrote each other on every run, so an identical rerun reported 238 updated instead of 0. Each observation now stores the status it reported, and every sighting of an event is ranked the same way: source precedence, then live over withdrawn, then latest upstream revision, with source_ref as a tie-break. Only the top-ranked sighting sets the row, so reruns settle on one winner. Migration backfills status for existing PredictHQ sightings from their payload. Repository tests now clear both tables inside their rolled-back transaction, so locally ingested events no longer break their counts. Co-Authored-By: Claude Opus 5.5 --- .../c3e9a1f4b6d8_observation_status.py | 42 ++++++++++ backend/foresight/database/models/external.py | 4 + backend/foresight/ingest/repository.py | 78 ++++++++++++++----- backend/tests/test_repository.py | 66 +++++++++++++++- 4 files changed, 168 insertions(+), 22 deletions(-) create mode 100644 backend/foresight/database/migrations/versions/c3e9a1f4b6d8_observation_status.py diff --git a/backend/foresight/database/migrations/versions/c3e9a1f4b6d8_observation_status.py b/backend/foresight/database/migrations/versions/c3e9a1f4b6d8_observation_status.py new file mode 100644 index 0000000..ccb4f0a --- /dev/null +++ b/backend/foresight/database/migrations/versions/c3e9a1f4b6d8_observation_status.py @@ -0,0 +1,42 @@ +"""observation status + +Each sighting keeps the status its source reported, so a deleted duplicate +listing can be told apart from the live listing it shares an event with. +Existing PredictHQ sightings are backfilled from their payload. + +Revision ID: c3e9a1f4b6d8 +Revises: b7d41e0c9a52 +Create Date: 2026-10-06 22:30:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'c3e9a1f4b6d8' +down_revision: Union[str, None] = 'b7d41e0c9a52' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + 'event_observations', sa.Column('status', sa.String(length=20), nullable=True) + ) + op.execute( + """ + UPDATE event_observations SET status = CASE payload->>'state' + WHEN 'active' THEN 'ACTIVE' + WHEN 'predicted' THEN 'PROVISIONAL' + WHEN 'deleted' THEN 'DELETED' + END + WHERE source = 'PREDICTHQ' + """ + ) + + +def downgrade() -> None: + op.drop_column('event_observations', 'status') diff --git a/backend/foresight/database/models/external.py b/backend/foresight/database/models/external.py index a426810..a803927 100644 --- a/backend/foresight/database/models/external.py +++ b/backend/foresight/database/models/external.py @@ -142,6 +142,10 @@ class EventObservation(Base): source: Mapped[Source] = mapped_column(SAEnum(Source, native_enum=False, length=40)) source_ref: Mapped[str] = mapped_column(String(600)) source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + # What this sighting reported; null only for rows predating the column. + status: Mapped[EventStatus | None] = mapped_column( + SAEnum(EventStatus, native_enum=False, length=20) + ) payload: Mapped[dict[str, Any]] = mapped_column(JSONB) diff --git a/backend/foresight/ingest/repository.py b/backend/foresight/ingest/repository.py index 7092071..a3551a7 100644 --- a/backend/foresight/ingest/repository.py +++ b/backend/foresight/ingest/repository.py @@ -2,10 +2,11 @@ from __future__ import annotations +from datetime import UTC, datetime from enum import StrEnum from typing import TYPE_CHECKING, Any -from sqlalchemy import delete, func, literal_column, or_, select, update +from sqlalchemy import and_, delete, func, literal_column, not_, or_, select, update from sqlalchemy.dialects.postgresql import insert from foresight.database.models.external import Event, EventObservation @@ -23,6 +24,8 @@ # Structured listings outrank dates and venues extracted from prose. _PRECEDENCE = {Source.PREDICTHQ: 2, Source.ASKNEWS: 1} +_WITHDRAWN = {EventStatus.DELETED, EventStatus.CANCELLED} +_NEVER = datetime.min.replace(tzinfo=UTC) class Outcome(StrEnum): @@ -48,14 +51,10 @@ async def upsert_event(session: AsyncSession, event: NormalizedEvent) -> Outcome # changes the key, and resolving by key first would orphan the old row. if event_id is not None: event_id = await _absorb_key_collision(session, event_id, row["dedupe_key"]) - outcome = await _apply(session, event_id, row, own_record=True) + outcome = await _apply(session, event_id, event) else: event_id, inserted = await _insert_or_find(session, row) - outcome = ( - Outcome.NEW - if inserted - else await _apply(session, event_id, row, own_record=False) - ) + outcome = Outcome.NEW if inserted else await _apply(session, event_id, event) await _record_observation(session, event, event_id) return outcome @@ -100,12 +99,9 @@ async def _insert_or_find( async def _apply( - session: AsyncSession, - event_id: int, - row: Mapping[str, Any], - *, - own_record: bool, + session: AsyncSession, event_id: int, event: NormalizedEvent ) -> Outcome: + row = event.as_row() current = ( ( await session.execute( @@ -117,7 +113,7 @@ async def _apply( ) changes: dict[str, Any] = {} - if _may_overwrite(current, row, own_record=own_record): + if await _outranks_other_sightings(session, event_id, event): changes = {k: v for k, v in row.items() if current[k] != v} await session.execute( @@ -128,15 +124,52 @@ async def _apply( return Outcome.UPDATED if changes else Outcome.UNCHANGED -def _may_overwrite( - current: Mapping[str, Any], row: Mapping[str, Any], *, own_record: bool +async def _outranks_other_sightings( + session: AsyncSession, event_id: int, event: NormalizedEvent ) -> bool: - # Another record's deletion (e.g. a PredictHQ duplicate that collapses onto - # the surviving listing) must not take down an event seen independently. - if not own_record and row["status"] is EventStatus.DELETED: - return False - incoming = _PRECEDENCE[row["primary_source"]] - return incoming >= _PRECEDENCE[Source(current["primary_source"])] + """Whether `event` is the sighting whose values the row should carry. + + Every sighting of a row is ranked the same way, so reruns settle on one + winner instead of duplicate listings overwriting each other in turn. A + deleted duplicate thus never takes down the live listing it shares a row + with, on the first run or any later one. + """ + others = await session.execute( + select( + _OBSERVATIONS.c.source, + _OBSERVATIONS.c.status, + _OBSERVATIONS.c.source_updated_at, + _OBSERVATIONS.c.source_ref, + ).where( + _OBSERVATIONS.c.event_id == event_id, + not_( + and_( + _OBSERVATIONS.c.source == event.source, + _OBSERVATIONS.c.source_ref == event.source_ref, + ) + ), + ) + ) + mine = _rank(event.source, event.status, event.source_updated_at, event.source_ref) + return all(mine > _rank(*other) for other in others) + + +def _rank( + source: Source, + status: EventStatus | None, + updated_at: datetime | None, + source_ref: str, +) -> tuple[int, bool, datetime, str]: + """Structured sources first, then live listings, then the latest revision. + + source_ref only breaks exact ties, so the winner is stable across reruns. + """ + return ( + _PRECEDENCE[Source(source)], + status not in _WITHDRAWN, + updated_at or _NEVER, + source_ref, + ) async def _record_observation( @@ -147,6 +180,7 @@ async def _record_observation( source=event.source, source_ref=event.source_ref, source_updated_at=event.source_updated_at, + status=event.status, payload=event.payload, ) excluded = stmt.excluded @@ -156,6 +190,7 @@ async def _record_observation( set_={ "event_id": excluded.event_id, "source_updated_at": excluded.source_updated_at, + "status": excluded.status, "payload": excluded.payload, "captured_at": func.clock_timestamp(), }, @@ -163,6 +198,7 @@ async def _record_observation( where=or_( _OBSERVATIONS.c.payload.is_distinct_from(excluded.payload), _OBSERVATIONS.c.event_id.is_distinct_from(excluded.event_id), + _OBSERVATIONS.c.status.is_distinct_from(excluded.status), ), ) ) diff --git a/backend/tests/test_repository.py b/backend/tests/test_repository.py index 5c919ec..b3f58e9 100644 --- a/backend/tests/test_repository.py +++ b/backend/tests/test_repository.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING import pytest -from sqlalchemy import func, select +from sqlalchemy import delete, func, select from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from foresight.config import settings @@ -31,6 +31,10 @@ async def session() -> AsyncIterator[AsyncSession]: async with AsyncSession( bind=conn, join_transaction_mode="create_savepoint" ) as session: + # Counts below assume an empty table; real ingested rows come back on + # rollback. + await session.execute(delete(EventObservation)) + await session.execute(delete(Event)) yield session await trans.rollback() await conn.close() @@ -207,3 +211,63 @@ async def test_observation_payload_tracks_the_latest_revision( ).one() assert obs.payload == {"id": "phq-test-1", "rev": 2} assert obs.source_updated_at == updated_at + + +async def test_deleted_duplicate_stays_harmless_on_every_rerun( + session: AsyncSession, +) -> None: + live = _phq() + duplicate = _phq(source_ref="phq-test-dupe", status=EventStatus.DELETED) + + for _ in range(3): + assert await upsert_event(session, live) in {Outcome.NEW, Outcome.UNCHANGED} + assert await upsert_event(session, duplicate) is Outcome.UNCHANGED + assert (await _only_event(session)).status is EventStatus.ACTIVE + + +async def test_live_duplicates_settle_on_the_latest_revision( + session: AsyncSession, +) -> None: + older = _phq( + expected_attendance=210, + source_updated_at=datetime(2026, 10, 1, tzinfo=UTC), + ) + newer = _phq( + source_ref="phq-test-dupe", + expected_attendance=211, + source_updated_at=datetime(2026, 10, 5, tzinfo=UTC), + ) + await upsert_event(session, older) + assert await upsert_event(session, newer) is Outcome.UPDATED + + # A rerun must not flip the row back and forth between the two listings. + assert await upsert_event(session, older) is Outcome.UNCHANGED + assert await upsert_event(session, newer) is Outcome.UNCHANGED + assert (await _only_event(session)).expected_attendance == 211 + + +async def test_live_listing_restores_an_event_its_duplicate_deleted( + session: AsyncSession, +) -> None: + # As left behind before sightings recorded their status. + await upsert_event(session, _phq(source_ref="phq-test-dupe")) + await upsert_event(session, _phq()) + await session.execute(EventObservation.__table__.update().values(status=None)) + await session.execute(Event.__table__.update().values(status=EventStatus.DELETED)) + + await upsert_event( + session, _phq(source_ref="phq-test-dupe", status=EventStatus.DELETED) + ) + assert await upsert_event(session, _phq()) is Outcome.UPDATED + assert (await _only_event(session)).status is EventStatus.ACTIVE + + +async def test_structured_deletion_outranks_a_live_article( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + await upsert_event(session, _news()) + + deleted = _phq(status=EventStatus.DELETED) + assert await upsert_event(session, deleted) is Outcome.UPDATED + assert (await _only_event(session)).status is EventStatus.DELETED From ddd96dbb1c6e975a57009cbbb14b3faa6535228c Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 22:49:41 -0400 Subject: [PATCH 08/21] feat: feed stored events into the pricing model as nightly attendance services/features/events.py turns the events table into the expected_attendance feature the models train on: same-night events add up, multi-day events are spread over their nights, withdrawn events are dropped, provisional ones count by confidence, and an optional radius drops far venues. scripts/forecast_events.py prices upcoming nights with and without the real events. generate_mock_data.generate can take attendance as input so the mock competitor, fare and flight placeholders respond to real events, as they do in the training data. Co-Authored-By: Claude Opus 5.5 --- .../foresight/services/features/__init__.py | 1 + backend/foresight/services/features/events.py | 125 +++++++++++++ backend/scripts/forecast_events.py | 123 +++++++++++++ backend/scripts/generate_mock_data.py | 14 +- backend/tests/test_event_features.py | 168 ++++++++++++++++++ 5 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 backend/foresight/services/features/__init__.py create mode 100644 backend/foresight/services/features/events.py create mode 100644 backend/scripts/forecast_events.py create mode 100644 backend/tests/test_event_features.py diff --git a/backend/foresight/services/features/__init__.py b/backend/foresight/services/features/__init__.py new file mode 100644 index 0000000..ae75e18 --- /dev/null +++ b/backend/foresight/services/features/__init__.py @@ -0,0 +1 @@ +"""Model features built from the canonical external-factor tables.""" diff --git a/backend/foresight/services/features/events.py b/backend/foresight/services/features/events.py new file mode 100644 index 0000000..451ba8b --- /dev/null +++ b/backend/foresight/services/features/events.py @@ -0,0 +1,125 @@ +"""Stored events as the per-night `expected_attendance` feature the models use.""" + +from __future__ import annotations + +import math +from datetime import timedelta +from typing import TYPE_CHECKING + +import pandas as pd +from sqlalchemy import select + +from foresight.database.models.external import Event +from foresight.ingest.normalize import DatePrecision, EventStatus, slugify_city + +if TYPE_CHECKING: + from collections.abc import Iterable + from datetime import date + + from sqlalchemy.ext.asyncio import AsyncSession + +FEATURE = "expected_attendance" + +# Withdrawn events must not keep inflating demand. +_EXCLUDED = {EventStatus.DELETED, EventStatus.CANCELLED} +_EARTH_RADIUS_KM = 6371.0 + + +def nightly_attendance( + events: Iterable[Event], + start: date, + end: date, + *, + latitude: float | None = None, + longitude: float | None = None, + radius_km: float | None = None, +) -> pd.Series: + """Total expected attendance per night, start..end inclusive. + + A multi-day event's attendance is spread evenly over its nights. Provisional + events count in proportion to their confidence. Events without a day-precise + date or an attendance estimate contribute nothing. + + With a hotel point and `radius_km`, events whose venue lies farther away are + dropped; events with no coordinates are kept, since ingest already + geo-filtered them. + """ + nights = pd.date_range(start, end, freq="D").date + totals = dict.fromkeys(nights, 0.0) + near = latitude is not None and longitude is not None and radius_km is not None + + for event in events: + if ( + event.status in _EXCLUDED + or event.date_precision != DatePrecision.DAY + or not event.expected_attendance + ): + continue + if ( + near + and event.latitude is not None + and _distance_km(latitude, longitude, event.latitude, event.longitude) + > radius_km + ): + continue + + span = (event.end_local_date - event.start_local_date).days + 1 + per_night = event.expected_attendance * event.confidence / span + for offset in range(span): + night = event.start_local_date + timedelta(days=offset) + if night in totals: + totals[night] += per_night + + return pd.Series( + [round(v) for v in totals.values()], + index=pd.Index(nights, name="stay_date"), + name=FEATURE, + dtype=int, + ) + + +async def load_events( + session: AsyncSession, city: str, start: date, end: date +) -> list[Event]: + """Events in `city` overlapping start..end.""" + result = await session.scalars( + select(Event).where( + Event.city_slug == slugify_city(city), + Event.start_local_date <= end, + Event.end_local_date >= start, + ) + ) + return list(result) + + +async def event_features( + session: AsyncSession, + *, + city: str, + start: date, + end: date, + latitude: float | None = None, + longitude: float | None = None, + radius_km: float | None = None, +) -> pd.Series: + """`nightly_attendance` for a market, read from the events table.""" + events = await load_events(session, city, start, end) + return nightly_attendance( + events, + start, + end, + latitude=latitude, + longitude=longitude, + radius_km=radius_km, + ) + + +def _distance_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Great-circle (haversine) distance.""" + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi, dlmb = phi2 - phi1, math.radians(lon2 - lon1) + a = ( + math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlmb / 2) ** 2 + ) + return 2 * _EARTH_RADIUS_KM * math.asin(math.sqrt(a)) diff --git a/backend/scripts/forecast_events.py b/backend/scripts/forecast_events.py new file mode 100644 index 0000000..053381a --- /dev/null +++ b/backend/scripts/forecast_events.py @@ -0,0 +1,123 @@ +"""Prices upcoming nights using real events from the events table. + +Run from backend/ after ingesting events: + uv run python scripts/forecast_events.py --city Dublin --days 21 + uv run python scripts/forecast_events.py --city Dublin \\ + --lat 53.3498 --lon -6.2603 --radius-km 5 --room-type deluxe + +The model is trained on mock history, and only expected_attendance is real. +Weather, competitor and flight features are mock placeholders until those +sources are ingested, simulated so competitors and flights respond to the real +events as they would in the mock world. "no events" reruns the same simulation +with zero attendance, so the effect column is the events' price impact alone. +""" + +import argparse +import asyncio +from datetime import date, timedelta + +import numpy as np +import pandas as pd +from compare_models import POST_STAY_COLUMNS, load_data +from generate_mock_data import ROOM_TYPES, generate + +from foresight.database.session import async_session_factory, engine +from foresight.ingest.normalize import EventStatus +from foresight.services.features.events import ( + FEATURE, + load_events, + nightly_attendance, +) +from foresight.services.ml.regression import GBMModel + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--city", required=True, help="market, as ingested") + parser.add_argument("--days", type=int, default=14, help="nights to price") + parser.add_argument("--start", type=date.fromisoformat, default=date.today()) + parser.add_argument("--lat", type=float, help="hotel latitude") + parser.add_argument("--lon", type=float, help="hotel longitude") + parser.add_argument("--radius-km", type=float, help="drop events farther away") + parser.add_argument("--room-type", choices=list(ROOM_TYPES), default="standard") + return parser + + +async def _events(args, start: date, end: date): + # DEBUG=true turns on SQL echo, which would bury the table. + engine.echo = False + try: + async with async_session_factory() as session: + return await load_events(session, args.city, start, end) + finally: + await engine.dispose() + + +def _nights(args, attendance: np.ndarray) -> pd.DataFrame: + """Placeholder nights for one room type, driven by `attendance`.""" + frame = generate( + start=args.start.isoformat(), days=args.days, expected_attendance=attendance + ) + return frame[frame["room_type"] == args.room_type].reset_index(drop=True) + + +def _features(frame: pd.DataFrame, columns: pd.Index) -> pd.DataFrame: + """Same columns, dummies and order the model was trained on.""" + x = frame.drop(columns=["rate", "stay_date", *POST_STAY_COLUMNS]) + x = pd.get_dummies(x, columns=["room_type", "condition"], dtype=float) + return x.reindex(columns=columns, fill_value=0.0) + + +def _biggest(events, night: date) -> str: + on = [ + e + for e in events + if e.start_local_date <= night <= e.end_local_date + and e.expected_attendance + and e.status not in {EventStatus.DELETED, EventStatus.CANCELLED} + ] + return max(on, key=lambda e: e.expected_attendance).title[:40] if on else "" + + +def main() -> None: + args = _parser().parse_args() + start = args.start + end = start + timedelta(days=args.days - 1) + + events = asyncio.run(_events(args, start, end)) + attendance = nightly_attendance( + events, + start, + end, + latitude=args.lat, + longitude=args.lon, + radius_km=args.radius_km, + ) + + x_train, y_train = load_data() + model = GBMModel(seed=9).fit(x_train, y_train) + + with_events = _nights(args, attendance.to_numpy()) + without = _nights(args, np.zeros(args.days)) + + rate, lo, hi = model.predict_interval(_features(with_events, x_train.columns)) + baseline = model.predict(_features(without, x_train.columns)) + + print( + f"{args.city} | {args.room_type} | {start} to {end} | " + f"{len(events)} stored events overlap" + ) + print( + f"{'night':<14}{'attendance':>11}{'no events':>11}{'with':>9}" + f"{'effect':>9}{'95% range':>17} biggest event" + ) + for i, night in enumerate(with_events["stay_date"]): + print( + f"{night:%a %Y-%m-%d}{with_events[FEATURE][i]:>11,}" + f"{baseline[i]:>11.2f}{rate[i]:>9.2f}{rate[i] - baseline[i]:>+9.2f}" + f"{f'{lo[i]:.0f}-{hi[i]:.0f}':>17} {_biggest(events, night)}" + ) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/generate_mock_data.py b/backend/scripts/generate_mock_data.py index 9cb382c..40adb17 100644 --- a/backend/scripts/generate_mock_data.py +++ b/backend/scripts/generate_mock_data.py @@ -27,13 +27,21 @@ COMPETITOR_REF = np.mean(list(COMPETITORS.values())) -def generate(seed: int = 9, start: str = "2024-01-01", days: int = 730) -> pd.DataFrame: +def generate( + seed: int = 9, + start: str = "2024-01-01", + days: int = 730, + expected_attendance: np.ndarray | None = None, +) -> pd.DataFrame: """Generates one row per room type per night. Args: seed (int): Random seed for reproducible data. start (str): First stay date, as YYYY-MM-DD. days (int): Number of nights to generate. + expected_attendance (np.ndarray | None): Per-night event attendance to + use instead of random events; the rest of the data responds to it. + Random draws are unchanged, so two calls differ only by its effect. Returns: pd.DataFrame: Mock nightly data, sorted by stay_date then room_type. @@ -56,7 +64,9 @@ def generate(seed: int = 9, start: str = "2024-01-01", days: int = 730) -> pd.Da # Events: total expected attendance of the day's events, ~10% of days. has_event = rng.random(days) < 0.10 - expected_attendance = np.where(has_event, rng.lognormal(9.5, 0.7, days), 0) + random_attendance = np.where(has_event, rng.lognormal(9.5, 0.7, days), 0) + if expected_attendance is None: + expected_attendance = random_attendance demand = np.clip( 0.55 diff --git a/backend/tests/test_event_features.py b/backend/tests/test_event_features.py new file mode 100644 index 0000000..baf13ca --- /dev/null +++ b/backend/tests/test_event_features.py @@ -0,0 +1,168 @@ +"""Stored events turned into the per-night expected_attendance feature.""" + +from __future__ import annotations + +from datetime import date +from typing import TYPE_CHECKING + +import numpy as np +import pytest +from generate_mock_data import generate +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine + +from foresight.config import settings +from foresight.database.models.external import Event +from foresight.ingest.normalize import ( + DatePrecision, + EventStatus, + NormalizedEvent, + Source, +) +from foresight.ingest.repository import upsert_event +from foresight.services.features.events import event_features, nightly_attendance + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + +START, END = date(2026, 11, 13), date(2026, 11, 16) +DUBLIN = (53.3498, -6.2603) + + +def _event(**overrides: object) -> Event: + base: dict[str, object] = { + "start_local_date": date(2026, 11, 14), + "end_local_date": date(2026, 11, 14), + "expected_attendance": 12_000, + "confidence": 1.0, + "status": EventStatus.ACTIVE, + "date_precision": DatePrecision.DAY, + "latitude": None, + "longitude": None, + } + return Event(**(base | overrides)) + + +def _by_night(series) -> dict[date, int]: + return {night: int(n) for night, n in series.items()} + + +def test_one_row_per_night_with_zero_on_quiet_nights(): + series = nightly_attendance([_event()], START, END) + assert _by_night(series) == { + date(2026, 11, 13): 0, + date(2026, 11, 14): 12_000, + date(2026, 11, 15): 0, + date(2026, 11, 16): 0, + } + assert series.name == "expected_attendance" + + +def test_same_night_events_add_up(): + series = nightly_attendance( + [_event(), _event(expected_attendance=3_000)], START, END + ) + assert series[date(2026, 11, 14)] == 15_000 + + +def test_multi_day_event_is_spread_over_its_nights(): + fair = _event( + start_local_date=date(2026, 11, 14), + end_local_date=date(2026, 11, 17), # runs past the window + expected_attendance=20_000, + ) + series = nightly_attendance([fair], START, END) + assert list(series) == [0, 5_000, 5_000, 5_000] + + +@pytest.mark.parametrize("status", [EventStatus.DELETED, EventStatus.CANCELLED]) +def test_withdrawn_events_are_ignored(status): + series = nightly_attendance([_event(status=status)], START, END) + assert series.sum() == 0 + + +def test_provisional_events_count_by_confidence(): + guess = _event(status=EventStatus.PROVISIONAL, confidence=0.25) + assert nightly_attendance([guess], START, END)[date(2026, 11, 14)] == 3_000 + + +def test_unusable_events_contribute_nothing(): + events = [ + _event(expected_attendance=None), + _event(date_precision=DatePrecision.MONTH), + ] + assert nightly_attendance(events, START, END).sum() == 0 + + +def test_radius_drops_far_venues_but_keeps_unlocated_ones(): + near = _event(latitude=53.344, longitude=-6.267) # ~1 km from centre + far = _event(latitude=53.427, longitude=-6.244, expected_attendance=500) # ~9km + unlocated = _event(expected_attendance=100) + series = nightly_attendance( + [near, far, unlocated], + START, + END, + latitude=DUBLIN[0], + longitude=DUBLIN[1], + radius_km=5, + ) + assert series[date(2026, 11, 14)] == 12_100 + + +def test_generate_takes_attendance_without_changing_its_noise(): + days = 5 + attendance = np.array([0, 0, 30_000, 0, 0]) + quiet = generate(start="2026-11-13", days=days, expected_attendance=np.zeros(days)) + busy = generate(start="2026-11-13", days=days, expected_attendance=attendance) + + assert busy.groupby("stay_date")["expected_attendance"].first().tolist() == list( + attendance + ) + # Only the event night's derived features move. + changed = ( + (busy["flight_count"] != quiet["flight_count"]).groupby(busy["stay_date"]).any() + ) + assert changed.tolist() == [False, False, True, False, False] + + +@pytest.fixture +async def session() -> AsyncIterator[AsyncSession]: + engine = create_async_engine(settings.database_url) + try: + conn = await engine.connect() + except (OSError, ConnectionError) as exc: + await engine.dispose() + pytest.skip(f"Postgres unavailable: {exc}") + + trans = await conn.begin() + async with AsyncSession( + bind=conn, join_transaction_mode="create_savepoint" + ) as session: + yield session + await trans.rollback() + await conn.close() + await engine.dispose() + + +async def test_reads_the_market_from_the_events_table(session): + for ref, city, attendance in [ + ("phq-features-1", "Testville", 4_000), + ("phq-features-2", "Testville, Nowhere", 1_000), # same market slug + ("phq-features-3", "Elsewhere", 9_999), + ]: + await upsert_event( + session, + NormalizedEvent( + source=Source.PREDICTHQ, + source_ref=ref, + title=f"Show {city} {attendance}", + city=city, + start_local_date=date(2026, 11, 14), + end_local_date=date(2026, 11, 14), + expected_attendance=attendance, + ), + ) + await session.flush() + + series = await event_features(session, city="Testville", start=START, end=END) + assert series[date(2026, 11, 14)] == 5_000 + assert series.sum() == 5_000 From 2c35cae8b2da2d96cc421f4549263ac7f0be56c0 Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 22:49:41 -0400 Subject: [PATCH 09/21] docs: add event ingest run logs and findings for 10/7 meeting Five PredictHQ ingest runs (Dublin and Boston, including same-query reruns), the pre-fix runs that exposed the duplicate-listing bug, a timed-out run, the model forecast, and a findings page summarising them. Co-Authored-By: Claude Opus 5.5 --- docs/event-ingest-findings.html | 301 ++++++++++++++++++ docs/event-ingest-runs/1-dublin-90d.txt | 15 + docs/event-ingest-runs/2-dublin-90d-rerun.txt | 15 + docs/event-ingest-runs/3-boston-90d.txt | 15 + docs/event-ingest-runs/4-boston-90d-rerun.txt | 15 + .../5-dublin-7d-incremental.txt | 4 + .../6-model-forecast-dublin.txt | 25 ++ .../before-fix/1-dublin-90d.txt | 15 + .../before-fix/2-dublin-90d-rerun.txt | 15 + .../before-fix/3-boston-90d.txt | 15 + .../before-fix/4-dublin-7d-incremental.txt | 4 + .../before-fix/db-status-before-fix.txt | 6 + .../failed/boston-90d-readtimeout.txt | 91 ++++++ 13 files changed, 536 insertions(+) create mode 100644 docs/event-ingest-findings.html create mode 100644 docs/event-ingest-runs/1-dublin-90d.txt create mode 100644 docs/event-ingest-runs/2-dublin-90d-rerun.txt create mode 100644 docs/event-ingest-runs/3-boston-90d.txt create mode 100644 docs/event-ingest-runs/4-boston-90d-rerun.txt create mode 100644 docs/event-ingest-runs/5-dublin-7d-incremental.txt create mode 100644 docs/event-ingest-runs/6-model-forecast-dublin.txt create mode 100644 docs/event-ingest-runs/before-fix/1-dublin-90d.txt create mode 100644 docs/event-ingest-runs/before-fix/2-dublin-90d-rerun.txt create mode 100644 docs/event-ingest-runs/before-fix/3-boston-90d.txt create mode 100644 docs/event-ingest-runs/before-fix/4-dublin-7d-incremental.txt create mode 100644 docs/event-ingest-runs/before-fix/db-status-before-fix.txt create mode 100644 docs/event-ingest-runs/failed/boston-90d-readtimeout.txt diff --git a/docs/event-ingest-findings.html b/docs/event-ingest-findings.html new file mode 100644 index 0000000..8e7f81c --- /dev/null +++ b/docs/event-ingest-findings.html @@ -0,0 +1,301 @@ +PredictHQ Event Ingest + + + + + +
+
+
Foresight · External factors · Meeting 10/7
+

PredictHQ Event Ingest

+

A Python ingest that pulls events near a hotel from the PredictHQ API, cleans and de-duplicates them, stores them, and feeds nightly event attendance into the pricing model.

+

Runs executed 2026-10-06, 22:15–22:19 EDT · branch external-factors-events · raw logs in docs/event-ingest-runs/

+
+ +
+

Results

+
+
5ingest runs across Dublin and Boston, 0 records rejected
+
4,641raw PredictHQ listings stored
+
4,101unique events after cleaning (540 duplicates merged)
+
0 / 0new / updated on every re-run, so cleaning is stable
+
+
+
PredictHQ APIpredicthq.py
→ +
Normalize + validatenormalize.py
→ +
De-duplicate + upsertrepository.py
→ +
events tablePostgres
→ +
Nightly attendancefeatures/events.py
→ +
Pricing modelGBM
+
+
+ +
+

The runs

+

Each run fetches every event within 10 km of the city centre that PredictHQ revised in the window. A re-run of the same query is the test for clean data: if cleaning is correct and stable, nothing should be new or change.

+
+ + + + + + + + + + + +
#RunFetchedNewUpdatedUnchangedRejectedReads as
1Dublin, last 90 days1,3710261,345026 events restored by the fix
2Dublin, same query again1,371001,3710Stable
3Boston, last 90 days3,2700233,2470Duplicates settled by the fix
4Boston, same query again3,270003,2700Stable
5Dublin, last 7 days (daily refresh)412004120Stable
+
+

Every event was already stored by earlier pre-fix runs, so these runs show 0 new. On its first ingest Boston produced 2,864 new events from 3,270 listings. One extra Boston attempt timed out waiting on PredictHQ; its log is kept in failed/.

+
+ +
+

What cleaning does, on real records

+
    +
  • Merges duplicates. PredictHQ often lists one event several times. Records with the same normalized title, city and local date become one event.
    • +
  • Normalizes titles by dropping tour names, "(Touring)", "presented by…" and "feat." so two sources can match.
    • +
  • Converts to local dates. An 8 pm show is stored on that night, not on the next day in UTC.
    • +
  • Fixes coordinates. PredictHQ sends [longitude, latitude], so they are swapped and range-checked.
    • +
  • Validates. Records missing an id, title or date, ending before they start, or with impossible coordinates are logged and counted as rejected, and the run continues.
    • +
  • Keeps cancellations. Deleted events stay in the table marked deleted, so the model knows to drop them.
    • +
+ +
+
+
Raw from PredictHQ
title  "Wicked (Touring)"
+start  2026-11-15T01:00:00Z
+tz     America/New_York
+
Stored
title_norm        "wicked"
+start_local_date  2026-11-14
+
A Saturday 8 pm Boston show is 01:00 Sunday in UTC. Without local dates its demand would land on the wrong hotel night.
+
+
+
Raw from PredictHQ
4 listings, one show
+"Nikki Glaser: The Stunning Tour"
+states: active, deleted, …
+
Stored
1 event · Boston · 2026-12-04
+status active
+4 sightings kept for audit
+
Counting all four listings would quadruple that night's attendance in the model.
+
+
+
Raw from PredictHQ
"All Ireland League 1A -
+ Old Wesley vs Young Munster"
+listing A  state=active   4,607
+listing B  state=deleted  4,607
+
Stored
1 event · Dublin · 2026-10-24
+status active · attendance 4,607
+
The deleted copy is PredictHQ retiring a duplicate listing, not a cancelled match. Before the fix below, this game was marked deleted.
+
+
+
Raw from PredictHQ
location [-71.0623119, 42.3539927]
+
Stored
latitude   42.3539927
+longitude -71.0623119
+
GeoJSON puts longitude first. The swap is what makes distance-to-hotel filtering work.
+
+
+ +
+ + + + + + +
MarketRaw listingsUnique eventsDuplicates mergedMarked deletedNo attendance estimate
Dublin1,3711,237134 (9.8%)3244
Boston3,2702,864406 (12.4%)253163
+
+
+ +
+

Finding: re-runs were deleting live events

+
+

Found by the re-run test, fixed the same night

+

Re-running the Dublin query a minute after the first run reported 238 updated instead of 0, and 20 live events switched to deleted. The cleaner correctly ignored a deleted duplicate listing on the first run. On later runs it treated that listing as the event's own record and applied its deletion. Live duplicates that differed slightly, such as attendance 210 and 211, also overwrote each other on every run.

+

Fix: each sighting now stores its own status. Every listing of an event is ranked the same way: structured source first, then live over deleted, then the latest revision. Only the top-ranked listing sets the event's values, so every run picks the same one.

+
+
+ + + + + + +
Dublin re-runUpdatedActiveDeleted
Before fix2381,15658
After fix01,18232
+
+

The fix restored 26 Dublin events to active. Four regression tests cover it: a deleted duplicate across three re-runs, live duplicates settling on the latest revision, restoring an event a duplicate deleted, and a structured deletion outranking a live news article.

+
+ +
+

What it means for the pricing model

+

The cleaned events become one model feature: expected_attendance, the total expected attendance near the hotel each night. Same-night events are added together, a multi-day event's attendance is split over its nights, deleted events are excluded, and provisional events count by their confidence. The table prices a Dublin standard room twice, with real events and with none. The two prices differ only by the events.

+
+ + + + + + + + + + +
NightAttendanceNo eventsWith eventsEffectBiggest event
Sat 10 Oct58,257€137.17€166.47+€29.30All Ireland League rugby
Sat 17 Oct23,548€140.05€152.91+€12.86The Rock Orchestra
Thu 22 Oct15,774€113.87€120.30+€6.43J. Cole
Sat 24 Oct48,236€131.51€161.65+€30.13Leinster vs Munster
Sun 25 Oct349,723€123.47€141.22+€17.75Irish Life Dublin Marathon
Mon 19 Oct2,090€114.93€115.28+€0.35Eliades Ochoa
+
+

Only attendance is real here. The model is trained on mock hotel history, and weather, competitor and flight features are mock placeholders until those sources are ingested. Read the effect column as direction and relative size, not as a price recommendation.

+
+ +
+

Open questions and next steps

+
    +
  • API, not scraping. Events come from PredictHQ's API, which gives structured dates, venues and attendance, instead of from scraped web pages. AskNews (news articles) is the planned second source and plugs into the same cleaning.
    • +
  • Network retries. One Boston run failed on a PredictHQ read timeout. The ingest should retry before giving up.
    • +
  • Very large events. The marathon's 349,723 includes spectators. The model learned from mock nights topping out near 40,000, so it can't price beyond that range. It needs a cap or a log scale once trained on real data.
    • +
  • Multi-day attendance. This is assumed to be a total and split evenly across nights. Confirm against PredictHQ's documentation.
    • +
  • Missing attendance. 207 events across both cities have no estimate and currently add nothing. PredictHQ's 0–100 rank could fill the gap.
    • +
  • Real training data. Load hotel history into daily_performance and train on it with the same attendance feature.
    • +
+
+ +
+

Reproduce

+

From backend/, with PREDICTHQ_TOKEN set in .env and the database running:

+
uv run alembic upgrade head
+uv run python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report
+uv run python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report
+uv run python scripts/forecast_events.py --city Dublin --days 21
+uv run pytest
+
+ Run 2 output, Dublin re-run +
predicthq ingest | Dublin | 10km around 53.3498, -6.2603
+revised upstream since 2026-07-09 02:16 UTC
+
+  fetched    1371
+  ---------------
+  new           0
+  updated       0
+  unchanged  1371
+  rejected      0
+
+  dublin now holds 1237 events (1182 active, 32 deleted, 23 provisional)
+  backed by 1371 observations (1371 predicthq)
+
+
+ Before-fix Dublin re-run, the failing output +
predicthq ingest | Dublin | 10km around 53.3498, -6.2603
+revised upstream since 2026-07-09 02:07 UTC
+
+  fetched    1371
+  ---------------
+  new           0
+  updated     238
+  unchanged  1133
+  rejected      0
+
+  dublin now holds 1237 events (1156 active, 58 deleted, 23 provisional)
+  backed by 1371 observations (1371 predicthq)
+
+
+
diff --git a/docs/event-ingest-runs/1-dublin-90d.txt b/docs/event-ingest-runs/1-dublin-90d.txt new file mode 100644 index 0000000..3695f76 --- /dev/null +++ b/docs/event-ingest-runs/1-dublin-90d.txt @@ -0,0 +1,15 @@ +$ python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report +# run at 2026-10-06 22:15 EDT + +predicthq ingest | Dublin | 10km around 53.3498, -6.2603 +revised upstream since 2026-07-09 02:15 UTC + + fetched 1371 + --------------- + new 0 + updated 26 + unchanged 1345 + rejected 0 + + dublin now holds 1237 events (1182 active, 32 deleted, 23 provisional) + backed by 1371 observations (1371 predicthq) diff --git a/docs/event-ingest-runs/2-dublin-90d-rerun.txt b/docs/event-ingest-runs/2-dublin-90d-rerun.txt new file mode 100644 index 0000000..894f0e4 --- /dev/null +++ b/docs/event-ingest-runs/2-dublin-90d-rerun.txt @@ -0,0 +1,15 @@ +$ python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report +# run at 2026-10-06 22:16 EDT + +predicthq ingest | Dublin | 10km around 53.3498, -6.2603 +revised upstream since 2026-07-09 02:16 UTC + + fetched 1371 + --------------- + new 0 + updated 0 + unchanged 1371 + rejected 0 + + dublin now holds 1237 events (1182 active, 32 deleted, 23 provisional) + backed by 1371 observations (1371 predicthq) diff --git a/docs/event-ingest-runs/3-boston-90d.txt b/docs/event-ingest-runs/3-boston-90d.txt new file mode 100644 index 0000000..7972cb7 --- /dev/null +++ b/docs/event-ingest-runs/3-boston-90d.txt @@ -0,0 +1,15 @@ +$ python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report +# run at 2026-10-06 22:17 EDT + +predicthq ingest | Boston | 10km around 42.3601, -71.0589 +revised upstream since 2026-07-09 02:17 UTC + + fetched 3270 + --------------- + new 0 + updated 23 + unchanged 3247 + rejected 0 + + boston now holds 2864 events (2551 active, 253 deleted, 60 provisional) + backed by 3270 observations (3270 predicthq) diff --git a/docs/event-ingest-runs/4-boston-90d-rerun.txt b/docs/event-ingest-runs/4-boston-90d-rerun.txt new file mode 100644 index 0000000..7ea2601 --- /dev/null +++ b/docs/event-ingest-runs/4-boston-90d-rerun.txt @@ -0,0 +1,15 @@ +$ python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report +# run at 2026-10-06 22:19 EDT + +predicthq ingest | Boston | 10km around 42.3601, -71.0589 +revised upstream since 2026-07-09 02:19 UTC + + fetched 3270 + --------------- + new 0 + updated 0 + unchanged 3270 + rejected 0 + + boston now holds 2864 events (2551 active, 253 deleted, 60 provisional) + backed by 3270 observations (3270 predicthq) diff --git a/docs/event-ingest-runs/5-dublin-7d-incremental.txt b/docs/event-ingest-runs/5-dublin-7d-incremental.txt new file mode 100644 index 0000000..c60a5c8 --- /dev/null +++ b/docs/event-ingest-runs/5-dublin-7d-incremental.txt @@ -0,0 +1,4 @@ +$ python -m foresight.ingest.run --source predicthq --since 7d --lat 53.3498 --lon -6.2603 --city Dublin +# run at 2026-10-06 22:18 EDT + +fetched=412 new=0 updated=0 unchanged=412 rejected=0 diff --git a/docs/event-ingest-runs/6-model-forecast-dublin.txt b/docs/event-ingest-runs/6-model-forecast-dublin.txt new file mode 100644 index 0000000..8166039 --- /dev/null +++ b/docs/event-ingest-runs/6-model-forecast-dublin.txt @@ -0,0 +1,25 @@ +$ python scripts/forecast_events.py --city Dublin --days 21 + +Dublin | standard | 2026-10-06 to 2026-10-26 | 445 stored events overlap +night attendance no events with effect 95% range biggest event +Tue 2026-10-06 4,309 123.27 126.14 +2.87 120-127 The Hu +Wed 2026-10-07 11,052 118.90 123.52 +4.62 118-155 Paul Smith: Happy +Thu 2026-10-08 14,628 121.59 129.39 +7.80 126-142 Paul Smith: Happy +Fri 2026-10-09 10,152 145.11 146.68 +1.57 141-177 The Reytons +Sat 2026-10-10 58,257 137.17 166.47 +29.30 141-244 All Ireland League 1A - Lansdowne vs Old +Sun 2026-10-11 9,099 117.62 121.26 +3.64 116-133 The Lord of The Rings: The Fellowship of +Mon 2026-10-12 3,588 126.22 127.61 +1.39 121-135 Morcheeba and Alex Cornish +Tue 2026-10-13 5,416 122.00 126.96 +4.96 122-127 Matt Bennett +Wed 2026-10-14 6,452 123.10 127.43 +4.34 123-130 Matilda the Musical (Touring) +Thu 2026-10-15 6,712 117.74 120.74 +3.00 116-143 Matilda the Musical (Touring) +Fri 2026-10-16 9,865 142.93 146.19 +3.26 139-170 Matilda the Musical (Touring) +Sat 2026-10-17 23,548 140.05 152.91 +12.86 142-200 The Rock Orchestra +Sun 2026-10-18 8,679 127.38 129.66 +2.28 126-138 Melanie C and Sophie And The Giants +Mon 2026-10-19 2,090 114.93 115.28 +0.35 110-140 Eliades Ochoa +Tue 2026-10-20 5,208 124.23 127.68 +3.46 119-128 Autechre +Wed 2026-10-21 5,021 113.44 118.82 +5.38 114-140 High Society +Thu 2026-10-22 15,774 113.87 120.30 +6.43 117-133 J. Cole +Fri 2026-10-23 9,500 134.15 137.31 +3.16 133-138 Levellers +Sat 2026-10-24 48,236 131.51 161.65 +30.13 141-241 Leinster vs Munster +Sun 2026-10-25 349,723 123.47 141.22 +17.75 137-220 Irish Life Dublin Marathon +Mon 2026-10-26 14,568 115.90 125.38 +9.49 120-129 Macklemore diff --git a/docs/event-ingest-runs/before-fix/1-dublin-90d.txt b/docs/event-ingest-runs/before-fix/1-dublin-90d.txt new file mode 100644 index 0000000..7fa523c --- /dev/null +++ b/docs/event-ingest-runs/before-fix/1-dublin-90d.txt @@ -0,0 +1,15 @@ +$ python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report +# run at 2026-10-06 22:07 EDT + +predicthq ingest | Dublin | 10km around 53.3498, -6.2603 +revised upstream since 2026-07-09 02:07 UTC + + fetched 1371 + --------------- + new 852 + updated 129 + unchanged 390 + rejected 0 + + dublin now holds 1237 events (1176 active, 38 deleted, 23 provisional) + backed by 1371 observations (1371 predicthq) diff --git a/docs/event-ingest-runs/before-fix/2-dublin-90d-rerun.txt b/docs/event-ingest-runs/before-fix/2-dublin-90d-rerun.txt new file mode 100644 index 0000000..bfdb422 --- /dev/null +++ b/docs/event-ingest-runs/before-fix/2-dublin-90d-rerun.txt @@ -0,0 +1,15 @@ +$ python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report +# run at 2026-10-06 22:07 EDT + +predicthq ingest | Dublin | 10km around 53.3498, -6.2603 +revised upstream since 2026-07-09 02:07 UTC + + fetched 1371 + --------------- + new 0 + updated 238 + unchanged 1133 + rejected 0 + + dublin now holds 1237 events (1156 active, 58 deleted, 23 provisional) + backed by 1371 observations (1371 predicthq) diff --git a/docs/event-ingest-runs/before-fix/3-boston-90d.txt b/docs/event-ingest-runs/before-fix/3-boston-90d.txt new file mode 100644 index 0000000..7fca54b --- /dev/null +++ b/docs/event-ingest-runs/before-fix/3-boston-90d.txt @@ -0,0 +1,15 @@ +$ python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report +# run at 2026-10-06 22:08 EDT + +predicthq ingest | Boston | 10km around 42.3601, -71.0589 +revised upstream since 2026-07-09 02:08 UTC + + fetched 3270 + --------------- + new 2864 + updated 151 + unchanged 255 + rejected 0 + + boston now holds 2864 events (2552 active, 252 deleted, 60 provisional) + backed by 3270 observations (3270 predicthq) diff --git a/docs/event-ingest-runs/before-fix/4-dublin-7d-incremental.txt b/docs/event-ingest-runs/before-fix/4-dublin-7d-incremental.txt new file mode 100644 index 0000000..ca89421 --- /dev/null +++ b/docs/event-ingest-runs/before-fix/4-dublin-7d-incremental.txt @@ -0,0 +1,4 @@ +$ python -m foresight.ingest.run --source predicthq --since 7d --lat 53.3498 --lon -6.2603 --city Dublin +# run at 2026-10-06 22:09 EDT + +fetched=412 new=0 updated=46 unchanged=366 rejected=0 diff --git a/docs/event-ingest-runs/before-fix/db-status-before-fix.txt b/docs/event-ingest-runs/before-fix/db-status-before-fix.txt new file mode 100644 index 0000000..a55d018 --- /dev/null +++ b/docs/event-ingest-runs/before-fix/db-status-before-fix.txt @@ -0,0 +1,6 @@ +boston|ACTIVE|2552 +boston|DELETED|252 +boston|PROVISIONAL|60 +dublin|ACTIVE|1156 +dublin|DELETED|58 +dublin|PROVISIONAL|23 diff --git a/docs/event-ingest-runs/failed/boston-90d-readtimeout.txt b/docs/event-ingest-runs/failed/boston-90d-readtimeout.txt new file mode 100644 index 0000000..6d571ce --- /dev/null +++ b/docs/event-ingest-runs/failed/boston-90d-readtimeout.txt @@ -0,0 +1,91 @@ +$ python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report +# run at 2026-10-06 22:16 EDT + +Traceback (most recent call last): + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_transports\default.py", line 101, in map_httpcore_exceptions + yield + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_transports\default.py", line 394, in handle_async_request + resp = await self._pool.handle_async_request(req) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\connection_pool.py", line 256, in handle_async_request + raise exc from None + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\connection_pool.py", line 236, in handle_async_request + response = await connection.handle_async_request( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\connection.py", line 103, in handle_async_request + return await self._connection.handle_async_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\http11.py", line 136, in handle_async_request + raise exc + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\http11.py", line 106, in handle_async_request + ) = await self._receive_response_headers(**kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\http11.py", line 177, in _receive_response_headers + event = await self._receive_event(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\http11.py", line 217, in _receive_event + data = await self._network_stream.read( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_backends\anyio.py", line 32, in read + with map_exceptions(exc_map): + ^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\contextlib.py", line 158, in __exit__ + self.gen.throw(value) + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_exceptions.py", line 14, in map_exceptions + raise to_exc(exc) from exc +httpcore.ReadTimeout + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\run.py", line 224, in + sys.exit(main()) + ^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\run.py", line 219, in main + asyncio.run(_main(args)) + File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\runners.py", line 195, in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ + File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\run.py", line 202, in _main + report = await run(args) + ^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\run.py", line 178, in run + result = await predicthq.fetch_events( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\predicthq.py", line 89, in fetch_events + response = await client.get(url, params=query) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1768, in get + return await self.request( + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1540, in request + return await self.send(request, auth=auth, follow_redirects=follow_redirects) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1629, in send + response = await self._send_handling_auth( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1657, in _send_handling_auth + response = await self._send_handling_redirects( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1694, in _send_handling_redirects + response = await self._send_single_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1730, in _send_single_request + response = await transport.handle_async_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_transports\default.py", line 393, in handle_async_request + with map_httpcore_exceptions(): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\contextlib.py", line 158, in __exit__ + self.gen.throw(value) + File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_transports\default.py", line 118, in map_httpcore_exceptions + raise mapped_exc(message) from exc +httpx.ReadTimeout From 318f897ac45c9c73b44ac36c9cc2d413ce79e7c4 Mon Sep 17 00:00:00 2001 From: heatherlarange Date: Tue, 6 Oct 2026 22:54:50 -0400 Subject: [PATCH 10/21] build: apply pending migrations automatically for db-up and the ingest CLI docker compose up already ran the migrate service, but `just db-up` started only the database, and the ingest CLI run from the host never ran migrations at all. Pulling the new observation status column then broke the ingest with a missing-column error until someone ran `just migrate` by hand. `just db-up` now starts the migrate service with the database, and the ingest CLI upgrades to head before fetching (a no-op when current). Alembic gets no ini file there, so it leaves the run report's logging alone. The findings page drops the now-unneeded manual migration step. Co-Authored-By: Claude Opus 5.5 --- backend/foresight/ingest/run.py | 16 ++++++++++++++++ docs/event-ingest-findings.html | 3 +-- justfile | 4 ++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/backend/foresight/ingest/run.py b/backend/foresight/ingest/run.py index 619f322..97ca933 100644 --- a/backend/foresight/ingest/run.py +++ b/backend/foresight/ingest/run.py @@ -14,8 +14,11 @@ import sys from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta +from pathlib import Path from typing import TYPE_CHECKING +from alembic import command +from alembic.config import Config from sqlalchemy import func, select from foresight.config import settings @@ -35,6 +38,7 @@ log = logging.getLogger("foresight.ingest") +_MIGRATIONS = Path(__file__).resolve().parents[1] / "database" / "migrations" _SPAN = re.compile(r"^(\d+)([hdw])$") _UNITS = {"h": "hours", "d": "days", "w": "weeks"} @@ -197,6 +201,17 @@ async def run(args: argparse.Namespace) -> RunReport: return report +def _migrate() -> None: + """Bring the schema up to date, so a fresh pull needs no manual step. + + A no-op when already at head. No ini file is passed, so Alembic leaves this + process's logging alone. + """ + config = Config() + config.set_main_option("script_location", str(_MIGRATIONS)) + command.upgrade(config, "head") + + async def _main(args: argparse.Namespace) -> None: try: report = await run(args) @@ -214,6 +229,7 @@ def main(argv: Sequence[str] | None = None) -> int: parser.error("PREDICTHQ_TOKEN is not set; add it to .env") logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s") + _migrate() # DEBUG=true turns on SQL echo, which would bury the report. engine.echo = False asyncio.run(_main(args)) diff --git a/docs/event-ingest-findings.html b/docs/event-ingest-findings.html index 8e7f81c..442f967 100644 --- a/docs/event-ingest-findings.html +++ b/docs/event-ingest-findings.html @@ -262,8 +262,7 @@

Open questions and next steps

Reproduce

From backend/, with PREDICTHQ_TOKEN set in .env and the database running:

-
uv run alembic upgrade head
-uv run python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report
+
uv run python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report
 uv run python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report
 uv run python scripts/forecast_events.py --city Dublin --days 21
 uv run pytest

diff --git a/justfile b/justfile
index 9103d3f..d10c125 100644
--- a/justfile
+++ b/justfile
@@ -26,9 +26,9 @@ dev:
 destroy:
     docker compose down
 
-# Start only the database
+# Start only the database, with migrations applied
 db-up:
-    docker compose up -d db
+    docker compose up -d db migrate
 
 # Stop only the database
 db-down:

From a8762a6043491aeee1286ef3c11992690b588b12 Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Tue, 6 Oct 2026 23:34:56 -0400
Subject: [PATCH 11/21] fix: move ingest package into foresight/ and restore
 NormalizedEvent.as_row

backend/src/ is outside the hatch package and the ruff src root, so
foresight.ingest was not importable and foresight/database/models/external.py
could not load. Same move Heather made locally, so the two land identically.
---
 backend/{src => }/foresight/ingest/__init__.py  | 0
 backend/{src => }/foresight/ingest/normalize.py | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename backend/{src => }/foresight/ingest/__init__.py (100%)
 rename backend/{src => }/foresight/ingest/normalize.py (99%)

diff --git a/backend/src/foresight/ingest/__init__.py b/backend/foresight/ingest/__init__.py
similarity index 100%
rename from backend/src/foresight/ingest/__init__.py
rename to backend/foresight/ingest/__init__.py
diff --git a/backend/src/foresight/ingest/normalize.py b/backend/foresight/ingest/normalize.py
similarity index 99%
rename from backend/src/foresight/ingest/normalize.py
rename to backend/foresight/ingest/normalize.py
index abeb121..6c1d56b 100644
--- a/backend/src/foresight/ingest/normalize.py
+++ b/backend/foresight/ingest/normalize.py
@@ -225,7 +225,7 @@ def dedupe_key(self) -> str:
             title=self.title, city=self.city, start_local_date=self.start_local_date
         )
 
-    def mas_row(self) -> dict[str, Any]:
+    def as_row(self) -> dict[str, Any]:
         """Column values for the canonical `events` row."""
         return {
             "dedupe_key": self.dedupe_key,

From 77570ca45fb105e36097bd87b56c2bfca9dea17b Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Tue, 6 Oct 2026 23:35:01 -0400
Subject: [PATCH 12/21] build: add httpx dependency and AskNews credentials

env_file also reads ../.env so commands run from backend/ pick up the
repo-root file.
---
 .env.example                | 4 ++++
 backend/foresight/config.py | 6 +++++-
 backend/pyproject.toml      | 1 +
 backend/uv.lock             | 6 ++++--
 4 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/.env.example b/.env.example
index 7791da1..fc70435 100644
--- a/.env.example
+++ b/.env.example
@@ -5,3 +5,7 @@ DEBUG=false
 # Database
 # Format: postgresql+asyncpg://:@:/
 DATABASE_URL=postgresql+asyncpg://user:password@localhost:5432/dbname
+
+# External event sources
+ASKNEWS_CLIENT_ID=
+ASKNEWS_CLIENT_SECRET=
diff --git a/backend/foresight/config.py b/backend/foresight/config.py
index 24eb3da..f296706 100644
--- a/backend/foresight/config.py
+++ b/backend/foresight/config.py
@@ -3,7 +3,7 @@
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(
-        env_file=".env",
+        env_file=("../.env", ".env"),
         env_file_encoding="utf-8",
         extra="ignore",
     )
@@ -17,5 +17,9 @@ class Settings(BaseSettings):
         "postgresql+asyncpg://foresight:foresight@localhost:5432/foresight_db"
     )
 
+    # External event sources
+    asknews_client_id: str | None = None
+    asknews_client_secret: str | None = None
+
 
 settings = Settings()
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 3412379..badaf9a 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -17,6 +17,7 @@ dependencies = [
     "python-dotenv>=1.1",
     "pandas>=2.2",
     "psycopg2-binary>=2.9",
+    "httpx>=0.28",
 ]
 
 [dependency-groups]
diff --git a/backend/uv.lock b/backend/uv.lock
index d3769fd..f9ec6d3 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -6,10 +6,10 @@ resolution-markers = [
     "python_full_version >= '3.14' and sys_platform == 'emscripten'",
     "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
     "python_full_version == '3.13.*' and sys_platform == 'win32'",
-    "python_full_version < '3.13' and sys_platform == 'win32'",
     "python_full_version == '3.13.*' and sys_platform == 'emscripten'",
-    "python_full_version < '3.13' and sys_platform == 'emscripten'",
     "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+    "python_full_version < '3.13' and sys_platform == 'win32'",
+    "python_full_version < '3.13' and sys_platform == 'emscripten'",
     "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'",
 ]
 
@@ -331,6 +331,7 @@ dependencies = [
     { name = "alembic" },
     { name = "asyncpg" },
     { name = "fastapi" },
+    { name = "httpx" },
     { name = "pandas" },
     { name = "psycopg2-binary" },
     { name = "pydantic-settings" },
@@ -361,6 +362,7 @@ requires-dist = [
     { name = "alembic", specifier = ">=1.15" },
     { name = "asyncpg", specifier = ">=0.30" },
     { name = "fastapi", specifier = ">=0.115" },
+    { name = "httpx", specifier = ">=0.28" },
     { name = "pandas", specifier = ">=2.2" },
     { name = "psycopg2-binary", specifier = ">=2.9" },
     { name = "pydantic-settings", specifier = ">=2.9" },

From 16e76c12ac57532b33ac0797f0a89a6bf9740542 Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Tue, 6 Oct 2026 23:35:09 -0400
Subject: [PATCH 13/21] feat: resolve event dates out of prose

Text sources write dates as people do. Resolves day ranges (including
ones crossing a month or a new year), month-year and season/quarter
spans, reporting the precision it achieved rather than rounding a vague
date into a confident one. Relative forms are refused outright.
---
 backend/foresight/ingest/textdates.py | 276 ++++++++++++++++++++++++++
 backend/tests/test_textdates.py       | 133 +++++++++++++
 2 files changed, 409 insertions(+)
 create mode 100644 backend/foresight/ingest/textdates.py
 create mode 100644 backend/tests/test_textdates.py

diff --git a/backend/foresight/ingest/textdates.py b/backend/foresight/ingest/textdates.py
new file mode 100644
index 0000000..c891522
--- /dev/null
+++ b/backend/foresight/ingest/textdates.py
@@ -0,0 +1,276 @@
+"""Resolve event dates out of prose.
+
+Text sources announce dates the way people write them -- "August 28-30",
+"next summer", "Q3 2026" -- while pricing needs a concrete local stay-night.
+This module turns the former into the latter, or refuses, and always reports
+how precisely it managed it.
+
+Relative expressions ("this weekend", "next Friday") are deliberately not
+resolved: they are only correct relative to a publication moment we do not
+always trust, and a wrong date is worse here than a missing one.
+"""
+
+from __future__ import annotations
+
+import calendar
+import re
+from dataclasses import dataclass
+from datetime import date
+
+from foresight.ingest.normalize import DatePrecision
+
+MAX_LOOKAHEAD_DAYS = 540
+_PAST_TOLERANCE_DAYS = 2
+
+_MONTHS = {
+    "january": 1,
+    "jan": 1,
+    "february": 2,
+    "feb": 2,
+    "march": 3,
+    "mar": 3,
+    "april": 4,
+    "apr": 4,
+    "may": 5,
+    "june": 6,
+    "jun": 6,
+    "july": 7,
+    "jul": 7,
+    "august": 8,
+    "aug": 8,
+    "september": 9,
+    "sept": 9,
+    "sep": 9,
+    "october": 10,
+    "oct": 10,
+    "november": 11,
+    "nov": 11,
+    "december": 12,
+    "dec": 12,
+}
+_MONTH_ALT = "|".join(sorted(_MONTHS, key=len, reverse=True))
+
+# Northern-hemisphere seasons: start month, length in months.
+_SEASONS = {
+    "spring": (3, 3),
+    "summer": (6, 3),
+    "fall": (9, 3),
+    "autumn": (9, 3),
+    "winter": (12, 3),
+}
+
+# "opens December 1 and runs through January 4" is a range too, not two dates.
+_RUN_PHRASE = r"(?:and\s+)?(?:will\s+)?runs?\s+(?:through|until|to)"
+_DASH = rf"(?:{_RUN_PHRASE}|--|-|‐|‑|‒|–|—|through|until|thru|to)"
+_ORD = r"(?:st|nd|rd|th)?"
+
+_ISO = re.compile(
+    rf"\b(\d{{4}})-(\d{{2}})-(\d{{2}})\b(?:\s*{_DASH}\s*(\d{{4}})-(\d{{2}})-(\d{{2}})\b)?"
+)
+# (?!\d) on every day group: without it "28 August 2026" reads the "20" of
+# the year as the day and silently resolves eight days early.
+_DAY = r"(\d{1,2})(?!\d)"
+_YEAR = r"(\d{4})(?!\d)"
+
+# A year can sit on either side of the dash: "December 30, 2026 - January 1, 2027".
+_MONTH_DAY = re.compile(
+    rf"\b({_MONTH_ALT})\.?\s+{_DAY}{_ORD}(?:,?\s*{_YEAR})?"
+    rf"(?:\s*{_DASH}\s*(?:({_MONTH_ALT})\.?\s+)?{_DAY}{_ORD}(?:,?\s*{_YEAR})?)?",
+    re.I,
+)
+_DAY_MONTH = re.compile(
+    rf"\b{_DAY}{_ORD}(?:\s*{_DASH}\s*{_DAY}{_ORD})?\s+(?:of\s+)?({_MONTH_ALT})\.?"
+    rf"(?:,?\s*{_YEAR})?",
+    re.I,
+)
+_MONTH_YEAR = re.compile(rf"\b({_MONTH_ALT})\.?,?\s+{_YEAR}", re.I)
+_SEASON_YEAR = re.compile(rf"\b({'|'.join(_SEASONS)})\s+(?:of\s+)?(\d{{4}})\b", re.I)
+_QUARTER_YEAR = re.compile(r"\bQ([1-4])\s*(?:of\s*)?(\d{4})\b", re.I)
+
+_RETROSPECTIVE = re.compile(
+    r"\b(?:was|were)\s+held\b|\btook\s+place\b|\bwrapped\s+up\b"
+    r"|\bdrew\s+(?:a\s+)?crowds?\b|\bconcluded\b|\blast\s+year'?s?\b",
+    re.I,
+)
+_FORWARD_LOOKING = re.compile(
+    r"\bwill\b|\bannounc\w+\b|\bupcoming\b|\breturns?\b|\bset\s+for\b|\bscheduled\b"
+    r"|\bgo\s+on\s+sale\b|\btickets?\b|\bthis\s+year'?s?\b|\bnext\s+year'?s?\b"
+    r"|\bline-?up\b",
+    re.I,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedDate:
+    start: date
+    end: date
+    precision: DatePrecision
+    had_explicit_year: bool
+
+
+def looks_retrospective(text: str) -> bool:
+    """True for articles reporting on an event that already happened.
+
+    A forward-looking marker wins: write-ups routinely recap last year before
+    announcing this year's dates, and those are the articles worth keeping.
+    """
+    return bool(_RETROSPECTIVE.search(text)) and not _FORWARD_LOOKING.search(text)
+
+
+def _month_number(name: str) -> int:
+    return _MONTHS[name.lower().rstrip(".")]
+
+
+def _safe_date(year: int, month: int, day: int) -> date | None:
+    try:
+        return date(year, month, day)
+    except ValueError:
+        return None
+
+
+def _end_of_month(year: int, month: int) -> date:
+    return date(year, month, calendar.monthrange(year, month)[1])
+
+
+def _infer_year(month: int, day: int, reference: date) -> int | None:
+    """Pick the year that puts an undated "August 28" in the plausible future."""
+    for year in (reference.year, reference.year + 1):
+        candidate = _safe_date(year, month, day)
+        if candidate and (candidate - reference).days >= -_PAST_TOLERANCE_DAYS:
+            return year
+    return None
+
+
+def _span_from_months(start_month: int, months: int, year: int) -> tuple[date, date]:
+    start = date(year, start_month, 1)
+    end_month, end_year = start_month + months - 1, year
+    if end_month > 12:
+        end_month, end_year = end_month - 12, year + 1
+    return start, _end_of_month(end_year, end_month)
+
+
+def _day_range(
+    month: int,
+    day: int,
+    end_month: int | None,
+    end_day: int | None,
+    year: int,
+    end_year: int | None = None,
+) -> tuple[date, date] | None:
+    start = _safe_date(year, month, day)
+    if not start:
+        return None
+    end = start
+    if end_day:
+        # "August 28 - September 1" and "December 30 - January 2" both roll
+        # forward, the latter into the next year.
+        target_month = end_month or month
+        resolved = end_year or (year + 1 if target_month < month else year)
+        parsed_end = _safe_date(resolved, target_month, end_day)
+        if parsed_end and parsed_end >= start:
+            end = parsed_end
+    return start, end
+
+
+def _try_iso(text: str, _reference: date) -> ResolvedDate | None:
+    match = _ISO.search(text)
+    if not match:
+        return None
+    start = _safe_date(int(match[1]), int(match[2]), int(match[3]))
+    if not start:
+        return None
+    end = start
+    if match[4]:
+        parsed_end = _safe_date(int(match[4]), int(match[5]), int(match[6]))
+        if parsed_end and parsed_end >= start:
+            end = parsed_end
+    return ResolvedDate(start, end, DatePrecision.DAY, had_explicit_year=True)
+
+
+def _try_month_day(text: str, reference: date) -> ResolvedDate | None:
+    for match in _MONTH_DAY.finditer(text):
+        month, day = _month_number(match[1]), int(match[2])
+        start_year = int(match[3]) if match[3] else None
+        end_month = _month_number(match[4]) if match[4] else None
+        end_day = int(match[5]) if match[5] else None
+        end_year = int(match[6]) if match[6] else None
+
+        # "December 30 - January 2, 2026" states one year for a span that
+        # crosses New Year. It belongs to the start; the end rolls over.
+        wraps = end_month is not None and end_month < month
+        if start_year is not None:
+            year, effective_end_year = start_year, end_year
+        elif end_year is not None:
+            year = end_year
+            effective_end_year = end_year + 1 if wraps else end_year
+        else:
+            year, effective_end_year = _infer_year(month, day, reference), None
+        if year is None:
+            continue
+
+        span = _day_range(month, day, end_month, end_day, year, effective_end_year)
+        if span:
+            explicit = start_year is not None or end_year is not None
+            return ResolvedDate(*span, DatePrecision.DAY, explicit)
+    return None
+
+
+def _try_day_month(text: str, reference: date) -> ResolvedDate | None:
+    for match in _DAY_MONTH.finditer(text):
+        day = int(match[1])
+        end_day = int(match[2]) if match[2] else None
+        month = _month_number(match[3])
+        explicit_year = int(match[4]) if match[4] else None
+        year = explicit_year or _infer_year(month, day, reference)
+        if year is None:
+            continue
+        span = _day_range(month, day, None, end_day, year)
+        if span:
+            return ResolvedDate(*span, DatePrecision.DAY, explicit_year is not None)
+    return None
+
+
+def _try_month_year(text: str, _reference: date) -> ResolvedDate | None:
+    match = _MONTH_YEAR.search(text)
+    if not match:
+        return None
+    start, end = _span_from_months(_month_number(match[1]), 1, int(match[2]))
+    return ResolvedDate(start, end, DatePrecision.MONTH, had_explicit_year=True)
+
+
+def _try_season_year(text: str, _reference: date) -> ResolvedDate | None:
+    match = _SEASON_YEAR.search(text)
+    if not match:
+        return None
+    start_month, months = _SEASONS[match[1].lower()]
+    start, end = _span_from_months(start_month, months, int(match[2]))
+    return ResolvedDate(start, end, DatePrecision.QUARTER, had_explicit_year=True)
+
+
+def _try_quarter_year(text: str, _reference: date) -> ResolvedDate | None:
+    match = _QUARTER_YEAR.search(text)
+    if not match:
+        return None
+    start, end = _span_from_months(int(match[1]) * 3 - 2, 3, int(match[2]))
+    return ResolvedDate(start, end, DatePrecision.QUARTER, had_explicit_year=True)
+
+
+# Finest precision first: text carrying both "August 28, 2026" and "summer
+# 2026" must resolve to the day, not the season.
+_RESOLVERS = (
+    _try_iso,
+    _try_month_day,
+    _try_day_month,
+    _try_month_year,
+    _try_season_year,
+    _try_quarter_year,
+)
+
+
+def resolve_date(text: str, reference: date) -> ResolvedDate | None:
+    """First date in `text` that could plausibly still be ahead of `reference`."""
+    for resolver in _RESOLVERS:
+        resolved = resolver(text, reference)
+        if resolved and (resolved.end - reference).days >= -_PAST_TOLERANCE_DAYS:
+            return resolved
+    return None
diff --git a/backend/tests/test_textdates.py b/backend/tests/test_textdates.py
new file mode 100644
index 0000000..f216ead
--- /dev/null
+++ b/backend/tests/test_textdates.py
@@ -0,0 +1,133 @@
+"""Date resolution: the formats announcements actually use, and what we refuse."""
+
+from __future__ import annotations
+
+from datetime import date
+
+import pytest
+
+from foresight.ingest.normalize import DatePrecision
+from foresight.ingest.textdates import looks_retrospective, resolve_date
+
+REFERENCE = date(2026, 6, 1)
+
+
+@pytest.mark.parametrize(
+    ("text", "start", "end"),
+    [
+        ("runs on August 28, 2026", date(2026, 8, 28), date(2026, 8, 28)),
+        ("runs 28 August 2026", date(2026, 8, 28), date(2026, 8, 28)),
+        ("August 28-30, 2026", date(2026, 8, 28), date(2026, 8, 30)),
+        ("28-30 August 2026", date(2026, 8, 28), date(2026, 8, 30)),
+        ("Aug. 28th to 30th, 2026", date(2026, 8, 28), date(2026, 8, 30)),
+        ("on 2026-08-28", date(2026, 8, 28), date(2026, 8, 28)),
+        ("2026-08-28 to 2026-08-30", date(2026, 8, 28), date(2026, 8, 30)),
+        # Ranges that roll into the next month, and the next year.
+        ("August 30 - September 2, 2026", date(2026, 8, 30), date(2026, 9, 2)),
+        ("December 30 - January 2, 2026", date(2026, 12, 30), date(2027, 1, 2)),
+        # A year stated on both sides of the dash.
+        (
+            "December 30, 2026 - January 1, 2027",
+            date(2026, 12, 30),
+            date(2027, 1, 1),
+        ),
+        # A range written as prose rather than punctuation.
+        (
+            "opens December 1, 2026 and runs through January 4, 2027",
+            date(2026, 12, 1),
+            date(2027, 1, 4),
+        ),
+        ("runs July 1 through July 9, 2027", date(2027, 7, 1), date(2027, 7, 9)),
+    ],
+)
+def test_day_precision_formats(text: str, start: date, end: date) -> None:
+    resolved = resolve_date(text, REFERENCE)
+
+    assert resolved is not None
+    assert (resolved.start, resolved.end) == (start, end)
+    assert resolved.precision is DatePrecision.DAY
+
+
+def test_missing_year_is_inferred_forward() -> None:
+    resolved = resolve_date("the festival returns August 28", REFERENCE)
+
+    assert resolved is not None
+    assert resolved.start == date(2026, 8, 28)
+    assert not resolved.had_explicit_year
+
+
+def test_missing_year_rolls_over_when_the_month_has_passed() -> None:
+    """In June, an undated "March 14" means next March."""
+    resolved = resolve_date("tickets for March 14", REFERENCE)
+
+    assert resolved is not None
+    assert resolved.start == date(2027, 3, 14)
+
+
+def test_month_year_keeps_month_precision() -> None:
+    resolved = resolve_date("sometime in August 2026", REFERENCE)
+
+    assert resolved is not None
+    assert resolved.precision is DatePrecision.MONTH
+    assert (resolved.start, resolved.end) == (date(2026, 8, 1), date(2026, 8, 31))
+
+
+@pytest.mark.parametrize(
+    ("text", "start", "end"),
+    [
+        ("summer 2026", date(2026, 6, 1), date(2026, 8, 31)),
+        ("autumn 2026", date(2026, 9, 1), date(2026, 11, 30)),
+        ("winter 2026", date(2026, 12, 1), date(2027, 2, 28)),
+        ("Q3 2026", date(2026, 7, 1), date(2026, 9, 30)),
+    ],
+)
+def test_coarse_spans_keep_quarter_precision(text: str, start: date, end: date) -> None:
+    resolved = resolve_date(text, REFERENCE)
+
+    assert resolved is not None
+    assert resolved.precision is DatePrecision.QUARTER
+    assert (resolved.start, resolved.end) == (start, end)
+
+
+def test_finest_precision_wins_when_a_text_carries_several() -> None:
+    text = "announced for summer 2026, with the main stage on August 28, 2026"
+
+    resolved = resolve_date(text, REFERENCE)
+
+    assert resolved is not None
+    assert resolved.precision is DatePrecision.DAY
+    assert resolved.start == date(2026, 8, 28)
+
+
+def test_impossible_calendar_date_is_skipped_for_the_next_candidate() -> None:
+    resolved = resolve_date("February 30, 2026 -- sorry, July 3, 2026", REFERENCE)
+
+    assert resolved is not None
+    assert resolved.start == date(2026, 7, 3)
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "happening this weekend",
+        "next Friday at the venue",
+        "coming soon to the city",
+        "no date has been announced",
+    ],
+)
+def test_relative_and_absent_dates_are_refused(text: str) -> None:
+    assert resolve_date(text, REFERENCE) is None
+
+
+def test_date_already_past_relative_to_the_reference_is_refused() -> None:
+    assert resolve_date("was on January 5, 2020", REFERENCE) is None
+
+
+def test_retrospective_coverage_is_flagged() -> None:
+    assert looks_retrospective("the festival took place over the weekend")
+
+
+def test_a_forward_looking_marker_rescues_a_retrospective_recap() -> None:
+    text = "last year's festival drew crowds; this year's runs August 28-30, 2026"
+
+    assert not looks_retrospective(text)

From 83de3d93276cbc0118ad7da63b083591653d62a6 Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Tue, 6 Oct 2026 23:35:09 -0400
Subject: [PATCH 14/21] feat: add AskNews events adapter

Articles in, provisional events out, with every drop recorded on
IngestReport as a named reason. collect_events is the single call a
scheduled worker makes; hours_back is the scheduling knob.
---
 backend/foresight/ingest/asknews.py | 461 ++++++++++++++++++++++++++++
 backend/tests/test_asknews.py       | 412 +++++++++++++++++++++++++
 2 files changed, 873 insertions(+)
 create mode 100644 backend/foresight/ingest/asknews.py
 create mode 100644 backend/tests/test_asknews.py

diff --git a/backend/foresight/ingest/asknews.py b/backend/foresight/ingest/asknews.py
new file mode 100644
index 0000000..8e1c81f
--- /dev/null
+++ b/backend/foresight/ingest/asknews.py
@@ -0,0 +1,461 @@
+"""AskNews adapter: news articles in, provisional events out.
+
+AskNews indexes articles, not events, so every record here is an inference:
+an article *mentions* a festival, and we decide whether it pins down a title,
+a city we operate in, and a date precise enough to price a night against.
+Most articles do not, and dropping those is the bulk of the work -- the
+rejection tally on `IngestReport` is the audit trail for what got dropped and
+why.
+
+Everything produced is `EventStatus.PROVISIONAL`. PredictHQ owns a record once
+it corroborates one; this source exists to see an event before PredictHQ does.
+
+One event per article: `event_observations` is unique on (source, source_ref)
+and source_ref is the article URL, so a single article cannot carry two
+sightings. Where an article announces several events we keep the first that
+resolves cleanly.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from collections import Counter
+from dataclasses import dataclass, field
+from datetime import UTC, date, datetime, timedelta
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any
+
+from foresight.ingest.normalize import (
+    DatePrecision,
+    EventCategory,
+    EventStatus,
+    NormalizedEvent,
+    Source,
+    normalize_title,
+    slugify_city,
+)
+from foresight.ingest.textdates import (
+    MAX_LOOKAHEAD_DAYS,
+    looks_retrospective,
+    resolve_date,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+    import httpx
+
+DEFAULT_API_BASE_URL = "https://api.asknews.app"
+DEFAULT_TOKEN_URL = "https://auth.asknews.app/oauth2/token"
+SEARCH_PATH = "/v1/news/search"
+
+# Phrasings that actually precede a date in an announcement. Broader queries
+# return politics and sport results that never survive extraction.
+DEFAULT_QUERIES = (
+    "festival announces dates lineup tickets",
+    "concert tour date announced tickets on sale",
+    "conference convention summit scheduled venue",
+    "parade marathon street festival city centre",
+)
+
+_MAX_ATTEMPTS = 3
+_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+_TOKEN_EXPIRY_SLACK_SECONDS = 60
+
+_CATEGORY_KEYWORDS: tuple[tuple[EventCategory, tuple[str, ...]], ...] = (
+    (EventCategory.FESTIVALS, ("festival", "fest", "carnival", "mardi gras")),
+    (EventCategory.CONCERTS, ("concert", "gig", "tour date", "live at", "headline")),
+    (
+        EventCategory.SPORTS,
+        ("match", "fixture", "tournament", "marathon", "championship", "cup final"),
+    ),
+    (
+        EventCategory.CONFERENCES,
+        ("conference", "summit", "symposium", "convention", "congress"),
+    ),
+    (EventCategory.EXPOS, ("expo", "trade show", "fair", "exhibition")),
+    (
+        EventCategory.PERFORMING_ARTS,
+        ("theatre", "theater", "opera", "ballet", "musical", "play opens"),
+    ),
+    (EventCategory.COMMUNITY, ("parade", "street party", "community", "pride")),
+    (EventCategory.ACADEMIC, ("graduation", "commencement", "semester", "freshers")),
+    (EventCategory.POLITICS, ("election", "referendum", "state visit", "protest")),
+)
+
+_PUBLISHER_TAIL = re.compile(r"\s*[|–—-]\s*[^|–—-]{1,40}$")
+_NEWS_PREFIX = re.compile(
+    r"^\s*(?:breaking|exclusive|update|watch|live|just in)\s*[:–-]\s*", re.I
+)
+
+
+class RejectReason(StrEnum):
+    """Why an article produced no event. The cleaning report is a tally of these."""
+
+    RETROSPECTIVE = "retrospective"
+    NO_RESOLVABLE_DATE = "no_resolvable_date"
+    EVENT_ALREADY_PAST = "event_already_past"
+    BEYOND_HORIZON = "beyond_horizon"
+    NO_CITY_MATCH = "no_city_match"
+    UNUSABLE_TITLE = "unusable_title"
+    DUPLICATE_IN_BATCH = "duplicate_in_batch"
+    MALFORMED_RECORD = "malformed_record"
+
+
+@dataclass(frozen=True, slots=True)
+class CityTarget:
+    """A city we sell rooms in, and the only places we care about events.
+
+    The timezone comes from here rather than from the article: a stay-night is
+    local to the hotel, and an article never states the venue's zone.
+    """
+
+    name: str
+    timezone: str
+    country: str | None = None
+    aliases: tuple[str, ...] = ()
+
+    @property
+    def slug(self) -> str:
+        return slugify_city(self.name)
+
+    @property
+    def search_terms(self) -> tuple[str, ...]:
+        return (self.name, *self.aliases)
+
+
+@dataclass(slots=True)
+class IngestReport:
+    """What one run saw, kept, and threw away."""
+
+    articles_seen: int = 0
+    events_kept: int = 0
+    rejected: Counter[str] = field(default_factory=Counter)
+    by_city: Counter[str] = field(default_factory=Counter)
+    by_precision: Counter[str] = field(default_factory=Counter)
+    queries_run: int = 0
+
+    @property
+    def articles_rejected(self) -> int:
+        return sum(self.rejected.values())
+
+    @property
+    def keep_rate(self) -> float:
+        return self.events_kept / self.articles_seen if self.articles_seen else 0.0
+
+    def render(self) -> str:
+        lines = [
+            f"queries run        {self.queries_run}",
+            f"articles seen      {self.articles_seen}",
+            f"events kept        {self.events_kept}  ({self.keep_rate:.0%})",
+            f"articles rejected  {self.articles_rejected}",
+        ]
+        for reason, count in self.rejected.most_common():
+            lines.append(f"    {reason:<22} {count}")
+        if self.by_city:
+            lines.append("events by city")
+            for city, count in self.by_city.most_common():
+                lines.append(f"    {city:<22} {count}")
+        if self.by_precision:
+            lines.append("events by date precision")
+            for precision, count in self.by_precision.most_common():
+                lines.append(f"    {precision:<22} {count}")
+        return "\n".join(lines)
+
+
+class AskNewsClient:
+    """Thin async client over the AskNews REST API.
+
+    The SDK would do this too, but it drags in its own HTTP stack and DTO
+    layer; the two calls we need are cheaper to own than to depend on.
+    """
+
+    def __init__(
+        self,
+        client_id: str,
+        client_secret: str,
+        *,
+        http: httpx.AsyncClient,
+        base_url: str = DEFAULT_API_BASE_URL,
+        token_url: str = DEFAULT_TOKEN_URL,
+        scopes: Sequence[str] = ("news",),
+    ) -> None:
+        self._client_id = client_id
+        self._client_secret = client_secret
+        self._http = http
+        self._base_url = base_url.rstrip("/")
+        self._token_url = token_url
+        self._scopes = tuple(scopes)
+        self._token: str | None = None
+        self._token_expires_at: datetime | None = None
+
+    async def _request(self, method: str, url: str, **kwargs: Any) -> httpx.Response:
+        """Issue a request, backing off once per retryable status."""
+        for attempt in range(1, _MAX_ATTEMPTS + 1):
+            response = await self._http.request(method, url, **kwargs)
+            if response.status_code not in _RETRY_STATUSES or attempt == _MAX_ATTEMPTS:
+                response.raise_for_status()
+                return response
+            retry_after = response.headers.get("retry-after")
+            delay = float(retry_after) if retry_after and retry_after.isdigit() else 2.0
+            await asyncio.sleep(delay * attempt)
+        raise AssertionError("unreachable")
+
+    async def _access_token(self) -> str:
+        now = datetime.now(UTC)
+        if self._token and self._token_expires_at and now < self._token_expires_at:
+            return self._token
+
+        response = await self._request(
+            "POST",
+            self._token_url,
+            data={
+                "grant_type": "client_credentials",
+                "scope": " ".join(self._scopes),
+            },
+            auth=(self._client_id, self._client_secret),
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+        )
+        payload = response.json()
+        token = payload.get("access_token")
+        if not token:
+            raise ValueError("AskNews token response carried no access_token")
+        expires_in = int(payload.get("expires_in", 3600)) - _TOKEN_EXPIRY_SLACK_SECONDS
+        self._token = token
+        self._token_expires_at = now + timedelta(seconds=max(expires_in, 0))
+        return token
+
+    async def search_news(self, **params: Any) -> list[dict[str, Any]]:
+        """One page of articles as plain dicts."""
+        token = await self._access_token()
+        response = await self._request(
+            "GET",
+            f"{self._base_url}{SEARCH_PATH}",
+            params={"return_type": "dicts", **params},
+            headers={"Authorization": f"Bearer {token}"},
+        )
+        body = response.json()
+        return list(body.get("as_dicts") or [])
+
+
+def _clean_title(raw: str) -> str:
+    return _PUBLISHER_TAIL.sub("", _NEWS_PREFIX.sub("", raw)).strip()
+
+
+def _entity_list(article: dict[str, Any], key: str) -> list[str]:
+    entities = article.get("entities") or {}
+    if not isinstance(entities, dict):
+        return []
+    values = entities.get(key) or []
+    return [str(v) for v in values if str(v).strip()]
+
+
+def _pick_title(article: dict[str, Any]) -> str:
+    """An Event entity names the event; a headline is about the event."""
+    for candidate in _entity_list(article, "Event"):
+        cleaned = _clean_title(candidate)
+        if normalize_title(cleaned):
+            return cleaned
+    return _clean_title(str(article.get("eng_title") or article.get("title") or ""))
+
+
+def _match_city(
+    article: dict[str, Any], text: str, cities: Sequence[CityTarget]
+) -> tuple[CityTarget, bool] | None:
+    """Resolve to one of our cities. The flag marks a Location-entity hit."""
+    location_slugs = {slugify_city(loc) for loc in _entity_list(article, "Location")}
+    haystack = text.lower()
+    weak: CityTarget | None = None
+    for city in cities:
+        for term in city.search_terms:
+            if slugify_city(term) in location_slugs:
+                return city, True
+            if weak is None and re.search(rf"\b{re.escape(term.lower())}\b", haystack):
+                weak = city
+    return (weak, False) if weak else None
+
+
+def _classify(text: str) -> EventCategory:
+    lowered = text.lower()
+    for category, keywords in _CATEGORY_KEYWORDS:
+        if any(keyword in lowered for keyword in keywords):
+            return category
+    return EventCategory.UNKNOWN
+
+
+def _score(
+    *,
+    precision: DatePrecision,
+    explicit_year: bool,
+    titled_by_entity: bool,
+    strong_city: bool,
+) -> float:
+    """Never 1.0: an extraction is a guess until another source agrees."""
+    confidence = 0.35
+    if precision is DatePrecision.DAY:
+        confidence += 0.20
+    elif precision is DatePrecision.MONTH:
+        confidence += 0.05
+    if explicit_year:
+        confidence += 0.10
+    if titled_by_entity:
+        confidence += 0.15
+    if strong_city:
+        confidence += 0.15
+    return min(confidence, 0.95)
+
+
+def _published_on(article: dict[str, Any]) -> datetime | None:
+    raw = article.get("pub_date") or article.get("crawl_date")
+    if not raw:
+        return None
+    try:
+        moment = datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return moment if moment.tzinfo else moment.replace(tzinfo=UTC)
+
+
+def extract_event(
+    article: dict[str, Any],
+    cities: Sequence[CityTarget],
+    *,
+    today: date,
+    horizon_days: int = MAX_LOOKAHEAD_DAYS,
+) -> NormalizedEvent | RejectReason:
+    """One article to one event, or the reason it did not make it."""
+    url = str(article.get("article_url") or "").strip()
+    if not url:
+        return RejectReason.MALFORMED_RECORD
+
+    summary = str(article.get("summary") or "")
+    headline = str(article.get("eng_title") or article.get("title") or "")
+    key_points = " ".join(str(p) for p in (article.get("key_points") or []))
+    text = " ".join(filter(None, (headline, summary, key_points)))
+    if not text.strip():
+        return RejectReason.MALFORMED_RECORD
+
+    if looks_retrospective(text):
+        return RejectReason.RETROSPECTIVE
+
+    matched = _match_city(article, text, cities)
+    if matched is None:
+        return RejectReason.NO_CITY_MATCH
+    city, strong_city = matched
+
+    published = _published_on(article)
+    reference = min(published.date(), today) if published else today
+    resolved = resolve_date(text, reference)
+    if resolved is None:
+        return RejectReason.NO_RESOLVABLE_DATE
+    # An event running across today still prices tonight, so test the end.
+    if resolved.end < today:
+        return RejectReason.EVENT_ALREADY_PAST
+    if (resolved.start - today).days > horizon_days:
+        return RejectReason.BEYOND_HORIZON
+
+    entity_titles = _entity_list(article, "Event")
+    title = _pick_title(article)
+    if not title or not normalize_title(title):
+        return RejectReason.UNUSABLE_TITLE
+
+    try:
+        return NormalizedEvent(
+            source=Source.ASKNEWS,
+            source_ref=url,
+            title=title,
+            city=city.name,
+            start_local_date=resolved.start,
+            end_local_date=resolved.end,
+            timezone=city.timezone,
+            category=_classify(text),
+            date_precision=resolved.precision,
+            status=EventStatus.PROVISIONAL,
+            confidence=_score(
+                precision=resolved.precision,
+                explicit_year=resolved.had_explicit_year,
+                titled_by_entity=bool(entity_titles),
+                strong_city=strong_city,
+            ),
+            country=city.country,
+            url=url,
+            source_updated_at=published,
+            payload=article,
+        )
+    except ValueError:
+        return RejectReason.MALFORMED_RECORD
+
+
+def extract_batch(
+    articles: Iterable[dict[str, Any]],
+    cities: Sequence[CityTarget],
+    *,
+    today: date,
+    horizon_days: int = MAX_LOOKAHEAD_DAYS,
+    report: IngestReport | None = None,
+) -> tuple[list[NormalizedEvent], IngestReport]:
+    """Extract, then collapse same-event sightings down to the best one.
+
+    Several outlets cover the same announcement, so within-batch collapsing is
+    the common case, not an edge case. Highest confidence wins.
+    """
+    report = report or IngestReport()
+    best: dict[str, NormalizedEvent] = {}
+
+    for article in articles:
+        report.articles_seen += 1
+        outcome = extract_event(article, cities, today=today, horizon_days=horizon_days)
+        if isinstance(outcome, RejectReason):
+            report.rejected[outcome.value] += 1
+            continue
+
+        incumbent = best.get(outcome.dedupe_key)
+        if incumbent is None:
+            best[outcome.dedupe_key] = outcome
+            continue
+        report.rejected[RejectReason.DUPLICATE_IN_BATCH.value] += 1
+        if outcome.confidence > incumbent.confidence:
+            best[outcome.dedupe_key] = outcome
+
+    events = sorted(best.values(), key=lambda e: (e.start_local_date, e.title))
+    report.events_kept = len(events)
+    for event in events:
+        report.by_city[event.city_slug] += 1
+        report.by_precision[str(event.date_precision)] += 1
+    return events, report
+
+
+async def collect_events(
+    client: AskNewsClient,
+    cities: Sequence[CityTarget],
+    *,
+    today: date,
+    queries: Sequence[str] = DEFAULT_QUERIES,
+    articles_per_query: int = 50,
+    hours_back: int = 24,
+    horizon_days: int = MAX_LOOKAHEAD_DAYS,
+) -> tuple[list[NormalizedEvent], IngestReport]:
+    """Run every query against AskNews and extract one clean batch.
+
+    `hours_back` is the scheduling knob: a daily job uses 24, a catch-up run
+    after an outage uses more. Queries are run against all target cities at
+    once because AskNews ranks globally -- filtering to our cities is the
+    extractor's job, not the query's.
+    """
+    report = IngestReport()
+    collected: list[dict[str, Any]] = []
+    for query in queries:
+        report.queries_run += 1
+        collected.extend(
+            await client.search_news(
+                query=query,
+                n_articles=articles_per_query,
+                hours_back=hours_back,
+                method="both",
+                strategy="latest news",
+            )
+        )
+    return extract_batch(
+        collected, cities, today=today, horizon_days=horizon_days, report=report
+    )
diff --git a/backend/tests/test_asknews.py b/backend/tests/test_asknews.py
new file mode 100644
index 0000000..f31c4c3
--- /dev/null
+++ b/backend/tests/test_asknews.py
@@ -0,0 +1,412 @@
+"""AskNews adapter: what survives extraction, what gets dropped, and the client."""
+
+from __future__ import annotations
+
+from datetime import date
+from typing import Any
+
+import httpx
+import pytest
+
+from foresight.ingest import asknews
+from foresight.ingest.asknews import (
+    AskNewsClient,
+    CityTarget,
+    RejectReason,
+    collect_events,
+    extract_batch,
+    extract_event,
+)
+from foresight.ingest.normalize import (
+    DatePrecision,
+    EventCategory,
+    EventStatus,
+    NormalizedEvent,
+    Source,
+)
+
+TODAY = date(2026, 6, 1)
+DUBLIN = CityTarget(
+    name="Dublin",
+    timezone="Europe/Dublin",
+    country="IE",
+    aliases=("Baile Atha Cliath",),
+)
+LISBON = CityTarget(name="Lisbon", timezone="Europe/Lisbon", country="PT")
+CITIES = (DUBLIN, LISBON)
+
+
+def article(**overrides: Any) -> dict[str, Any]:
+    base: dict[str, Any] = {
+        "article_url": "https://news.example/forbidden-fruit",
+        "eng_title": "Forbidden Fruit announces August dates for Dublin",
+        "summary": (
+            "Organisers confirmed the festival will return to the Royal Hospital "
+            "Kilmainham in Dublin on August 28-30, 2026, with tickets on sale Friday."
+        ),
+        "key_points": ["Tickets go on sale Friday"],
+        "pub_date": "2026-05-20T09:30:00+00:00",
+        "entities": {"Event": ["Forbidden Fruit"], "Location": ["Dublin", "Ireland"]},
+        "country": "IE",
+        "page_rank": 3,
+        "language": "en",
+    }
+    return base | overrides
+
+
+def test_a_clean_announcement_becomes_a_provisional_event() -> None:
+    event = extract_event(article(), CITIES, today=TODAY)
+
+    assert isinstance(event, NormalizedEvent)
+    assert event.source is Source.ASKNEWS
+    assert event.source_ref == "https://news.example/forbidden-fruit"
+    assert event.title == "Forbidden Fruit"
+    assert event.city == "Dublin"
+    assert event.start_local_date == date(2026, 8, 28)
+    assert event.end_local_date == date(2026, 8, 30)
+    assert event.category is EventCategory.FESTIVALS
+    assert event.date_precision is DatePrecision.DAY
+    assert event.status is EventStatus.PROVISIONAL
+
+
+def test_timezone_comes_from_the_city_not_the_article() -> None:
+    event = extract_event(article(), CITIES, today=TODAY)
+
+    assert isinstance(event, NormalizedEvent)
+    assert event.timezone == "Europe/Dublin"
+    assert event.country == "IE"
+
+
+def test_extraction_is_never_fully_confident() -> None:
+    event = extract_event(article(), CITIES, today=TODAY)
+
+    assert isinstance(event, NormalizedEvent)
+    assert 0.0 < event.confidence <= 0.95
+
+
+def test_an_event_entity_names_the_event_rather_than_the_headline() -> None:
+    headline_only = article(entities={"Location": ["Dublin"]})
+
+    titled = extract_event(article(), CITIES, today=TODAY)
+    untitled = extract_event(headline_only, CITIES, today=TODAY)
+
+    assert isinstance(titled, NormalizedEvent)
+    assert isinstance(untitled, NormalizedEvent)
+    assert titled.title == "Forbidden Fruit"
+    assert untitled.title.startswith("Forbidden Fruit announces")
+    # The entity hit is the stronger signal, so it must score higher.
+    assert titled.confidence > untitled.confidence
+
+
+def test_a_location_entity_outscores_a_bare_mention_in_the_body() -> None:
+    weak = article(entities={"Event": ["Forbidden Fruit"]})
+
+    strong_event = extract_event(article(), CITIES, today=TODAY)
+    weak_event = extract_event(weak, CITIES, today=TODAY)
+
+    assert isinstance(strong_event, NormalizedEvent)
+    assert isinstance(weak_event, NormalizedEvent)
+    assert strong_event.confidence > weak_event.confidence
+
+
+def test_an_alias_resolves_to_the_same_city() -> None:
+    aliased = article(
+        entities={"Event": ["Forbidden Fruit"], "Location": ["Baile Atha Cliath"]},
+        summary="Returns on August 28-30, 2026.",
+        eng_title="Festival returns",
+    )
+
+    event = extract_event(aliased, CITIES, today=TODAY)
+
+    assert isinstance(event, NormalizedEvent)
+    assert event.city_slug == "dublin"
+
+
+@pytest.mark.parametrize(
+    ("overrides", "reason"),
+    [
+        (
+            {
+                "eng_title": "Glastonbury announces dates",
+                "summary": "The festival runs August 28-30, 2026 in Somerset.",
+                "entities": {"Event": ["Glastonbury"], "Location": ["Somerset"]},
+            },
+            RejectReason.NO_CITY_MATCH,
+        ),
+        (
+            {
+                "summary": "A Dublin festival will return, dates to be confirmed.",
+                "eng_title": "Dublin festival to return",
+                "entities": {"Location": ["Dublin"]},
+            },
+            RejectReason.NO_RESOLVABLE_DATE,
+        ),
+        (
+            {
+                "summary": "The Dublin festival was held on August 28, 2025.",
+                "eng_title": "Dublin festival review",
+                "key_points": [],
+                "entities": {"Location": ["Dublin"]},
+            },
+            RejectReason.RETROSPECTIVE,
+        ),
+        (
+            {
+                "summary": "The Dublin festival is scheduled for August 28, 2035.",
+                "eng_title": "Dublin festival long-range plan",
+                "entities": {"Location": ["Dublin"]},
+            },
+            RejectReason.BEYOND_HORIZON,
+        ),
+        ({"article_url": ""}, RejectReason.MALFORMED_RECORD),
+        (
+            {"eng_title": "", "title": "", "summary": "", "key_points": []},
+            RejectReason.MALFORMED_RECORD,
+        ),
+    ],
+)
+def test_unusable_articles_are_rejected_with_a_reason(
+    overrides: dict[str, Any], reason: RejectReason
+) -> None:
+    assert extract_event(article(**overrides), CITIES, today=TODAY) is reason
+
+
+def test_an_event_that_already_finished_is_rejected_as_past() -> None:
+    stale = article(
+        pub_date="2026-01-10T09:00:00+00:00",
+        eng_title="Dublin parade set for March 17",
+        summary="The Dublin parade is scheduled for March 17, 2026.",
+        entities={"Location": ["Dublin"]},
+    )
+
+    assert extract_event(stale, CITIES, today=TODAY) is RejectReason.EVENT_ALREADY_PAST
+
+
+def test_an_event_running_across_today_is_still_kept() -> None:
+    ongoing = article(
+        pub_date="2026-05-25T09:00:00+00:00",
+        eng_title="Dublin festival under way",
+        summary="The Dublin festival is scheduled for May 30 - June 3, 2026.",
+        entities={"Location": ["Dublin"]},
+    )
+
+    event = extract_event(ongoing, CITIES, today=TODAY)
+
+    assert isinstance(event, NormalizedEvent)
+    assert event.start_local_date < TODAY <= event.end_local_date
+
+
+def test_the_same_event_from_two_outlets_collapses_to_one_row() -> None:
+    wire_copy = article(
+        article_url="https://other.example/ff",
+        eng_title="Dublin festival confirms summer return",
+        entities={"Event": ["Forbidden Fruit"], "Location": ["Dublin"]},
+    )
+
+    events, report = extract_batch([article(), wire_copy], CITIES, today=TODAY)
+
+    assert len(events) == 1
+    assert report.articles_seen == 2
+    assert report.events_kept == 1
+    assert report.rejected[RejectReason.DUPLICATE_IN_BATCH.value] == 1
+
+
+def test_the_higher_confidence_sighting_wins_a_collapse() -> None:
+    weak = article(
+        article_url="https://other.example/ff",
+        entities={"Event": ["Forbidden Fruit"]},
+    )
+
+    events, _ = extract_batch([weak, article()], CITIES, today=TODAY)
+
+    assert len(events) == 1
+    assert events[0].source_ref == "https://news.example/forbidden-fruit"
+
+
+def test_two_outlets_do_not_collapse_without_an_event_entity() -> None:
+    """A known limit: headlines differ, so the dedupe key differs.
+
+    AskNews supplies an Event entity for most announcements. Where it does
+    not, the same event arrives twice and only PredictHQ corroboration or a
+    later title match will merge it.
+    """
+    wire_copy = article(
+        article_url="https://other.example/ff",
+        eng_title="Forbidden Fruit returns to Dublin",
+        entities={"Location": ["Dublin"]},
+    )
+
+    events, _ = extract_batch([article(), wire_copy], CITIES, today=TODAY)
+
+    assert len(events) == 2
+
+
+def test_the_report_tallies_everything_that_came_in() -> None:
+    batch = [
+        article(),
+        article(
+            article_url="https://x.example/1", summary="No date here.", key_points=[]
+        ),
+        article(
+            article_url="https://x.example/2",
+            entities={"Location": ["Somerset"]},
+            eng_title="Somerset show",
+            summary="Runs August 28-30, 2026 in Somerset.",
+        ),
+    ]
+
+    events, report = extract_batch(batch, CITIES, today=TODAY)
+
+    assert report.articles_seen == 3
+    assert report.events_kept == len(events) == 1
+    assert report.articles_rejected == 2
+    assert report.by_city["dublin"] == 1
+    assert report.by_precision["day"] == 1
+    assert "articles seen" in report.render()
+
+
+def test_events_are_returned_in_date_order() -> None:
+    later = article(
+        article_url="https://x.example/later",
+        eng_title="Lisbon summit",
+        summary="The Lisbon summit is scheduled for October 5, 2026.",
+        entities={"Event": ["Lisbon Summit"], "Location": ["Lisbon"]},
+    )
+
+    events, _ = extract_batch([later, article()], CITIES, today=TODAY)
+
+    assert [e.start_local_date for e in events] == sorted(
+        e.start_local_date for e in events
+    )
+
+
+# --- client ---------------------------------------------------------------
+
+
+def build_client(handler: Any) -> tuple[AskNewsClient, dict[str, int]]:
+    calls: dict[str, int] = {"token": 0, "search": 0}
+
+    def route(request: httpx.Request) -> httpx.Response:
+        if request.url.path.endswith("/token"):
+            calls["token"] += 1
+        else:
+            calls["search"] += 1
+        return handler(request, calls)
+
+    http = httpx.AsyncClient(transport=httpx.MockTransport(route))
+    return AskNewsClient("id", "secret", http=http), calls
+
+
+def ok(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response:
+    if request.url.path.endswith("/token"):
+        return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
+    return httpx.Response(200, json={"as_dicts": [article()], "offset": 0})
+
+
+async def test_search_authenticates_then_returns_articles() -> None:
+    client, calls = build_client(ok)
+
+    articles = await client.search_news(query="festival", n_articles=5)
+
+    assert calls == {"token": 1, "search": 1}
+    assert articles[0]["article_url"] == "https://news.example/forbidden-fruit"
+
+
+async def test_the_token_is_reused_across_searches() -> None:
+    client, calls = build_client(ok)
+
+    await client.search_news(query="a")
+    await client.search_news(query="b")
+
+    assert calls["token"] == 1
+    assert calls["search"] == 2
+
+
+async def test_the_bearer_token_is_sent_on_the_search() -> None:
+    seen: list[str] = []
+
+    def capture(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response:
+        if request.url.path.endswith("/token"):
+            return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
+        seen.append(request.headers.get("authorization", ""))
+        return httpx.Response(200, json={"as_dicts": []})
+
+    client, _ = build_client(capture)
+    await client.search_news(query="a")
+
+    assert seen == ["Bearer tok"]
+
+
+async def test_a_rate_limited_search_is_retried(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    async def no_sleep(_seconds: float) -> None:
+        return None
+
+    monkeypatch.setattr(asknews.asyncio, "sleep", no_sleep)
+
+    def throttle(request: httpx.Request, calls: dict[str, int]) -> httpx.Response:
+        if request.url.path.endswith("/token"):
+            return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
+        if calls["search"] == 1:
+            return httpx.Response(429, headers={"retry-after": "1"}, json={})
+        return httpx.Response(200, json={"as_dicts": [article()]})
+
+    client, calls = build_client(throttle)
+    articles = await client.search_news(query="festival")
+
+    assert calls["search"] == 2
+    assert len(articles) == 1
+
+
+async def test_a_persistent_server_error_is_raised() -> None:
+    async def no_sleep(_seconds: float) -> None:
+        return None
+
+    def broken(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response:
+        if request.url.path.endswith("/token"):
+            return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
+        return httpx.Response(503, json={})
+
+    client, _ = build_client(broken)
+    asknews.asyncio.sleep = no_sleep  # type: ignore[assignment]
+
+    with pytest.raises(httpx.HTTPStatusError):
+        await client.search_news(query="festival")
+
+
+async def test_a_token_response_without_a_token_fails_loudly() -> None:
+    def tokenless(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response:
+        return httpx.Response(200, json={"detail": "nope"})
+
+    client, _ = build_client(tokenless)
+
+    with pytest.raises(ValueError, match="access_token"):
+        await client.search_news(query="festival")
+
+
+async def test_collect_events_is_one_call_a_scheduled_worker_can_make() -> None:
+    """The cron path end to end: fetch every query, extract one clean batch."""
+    queries_seen: list[str] = []
+    hours_seen: list[str] = []
+
+    def handler(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response:
+        if request.url.path.endswith("/token"):
+            return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
+        queries_seen.append(request.url.params.get("query", ""))
+        hours_seen.append(request.url.params.get("hours_back", ""))
+        return httpx.Response(200, json={"as_dicts": [article()]})
+
+    client, calls = build_client(handler)
+    events, report = await collect_events(
+        client, CITIES, today=TODAY, queries=("a", "b"), hours_back=6
+    )
+
+    assert calls["search"] == 2
+    assert queries_seen == ["a", "b"]
+    assert hours_seen == ["6", "6"]
+    assert report.queries_run == 2
+    assert report.articles_seen == 2
+    # The same article from both queries is one event, not two.
+    assert len(events) == 1
+    assert report.rejected[RejectReason.DUPLICATE_IN_BATCH.value] == 1

From aa765bc3b780a8f6ad2815524f3e23a7fe29fba8 Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Tue, 6 Oct 2026 23:35:09 -0400
Subject: [PATCH 15/21] feat: add AskNews ingest runner with per-run cleaning
 report

Replays recorded batches by default so the pipeline is demonstrable
without credentials; --live goes through the same collect_events call.
---
 backend/scripts/run_asknews.py            | 182 ++++++++++++++++++++++
 backend/tests/fixtures/asknews/run_1.json | 122 +++++++++++++++
 backend/tests/fixtures/asknews/run_2.json | 110 +++++++++++++
 backend/tests/fixtures/asknews/run_3.json | 110 +++++++++++++
 4 files changed, 524 insertions(+)
 create mode 100644 backend/scripts/run_asknews.py
 create mode 100644 backend/tests/fixtures/asknews/run_1.json
 create mode 100644 backend/tests/fixtures/asknews/run_2.json
 create mode 100644 backend/tests/fixtures/asknews/run_3.json

diff --git a/backend/scripts/run_asknews.py b/backend/scripts/run_asknews.py
new file mode 100644
index 0000000..1e1af62
--- /dev/null
+++ b/backend/scripts/run_asknews.py
@@ -0,0 +1,182 @@
+"""Run the AskNews event ingest and print a cleaning report per run.
+
+Two modes:
+
+    uv run python scripts/run_asknews.py              # replay recorded batches
+    uv run python scripts/run_asknews.py --live       # call the real API
+
+Replay is the default so the pipeline can be demonstrated and regression-checked
+without credentials or network. The extraction path is identical in both modes;
+only where the articles come from differs.
+
+Across runs the script tracks what is genuinely new, so re-polling an
+overlapping window visibly produces no duplicate events.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+from dataclasses import dataclass, field
+from datetime import date, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from foresight.config import settings
+from foresight.ingest.asknews import (
+    DEFAULT_QUERIES,
+    AskNewsClient,
+    CityTarget,
+    IngestReport,
+    collect_events,
+    extract_batch,
+)
+
+if TYPE_CHECKING:
+    from foresight.ingest.normalize import NormalizedEvent
+
+FIXTURE_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "asknews"
+
+# The cities we sell rooms in. Everything else AskNews returns is noise.
+CITIES = (
+    CityTarget("Dublin", "Europe/Dublin", "IE", aliases=("Baile Atha Cliath",)),
+    CityTarget("Lisbon", "Europe/Lisbon", "PT", aliases=("Lisboa",)),
+    CityTarget("Edinburgh", "Europe/London", "GB"),
+)
+
+
+@dataclass
+class EventStore:
+    """Stands in for the events table so the demo needs no database.
+
+    Keyed the same way the real upsert is -- on `dedupe_key` -- so the new /
+    repeat / revised counts here are the counts the repository would produce.
+    """
+
+    by_key: dict[str, NormalizedEvent] = field(default_factory=dict)
+    key_by_ref: dict[str, str] = field(default_factory=dict)
+
+    def apply(self, events: list[NormalizedEvent]) -> dict[str, list[str]]:
+        outcome: dict[str, list[str]] = {"new": [], "repeat": [], "revised": []}
+        for event in events:
+            previous_key = self.key_by_ref.get(event.source_ref)
+            if previous_key and previous_key != event.dedupe_key:
+                # Same article, different date: dedupe_key includes the start
+                # date, so a correction lands as a new row rather than an edit.
+                outcome["revised"].append(
+                    f"{event.title} -> {event.start_local_date} "
+                    f"(was {self.by_key[previous_key].start_local_date})"
+                )
+                self.by_key.pop(previous_key, None)
+            elif event.dedupe_key in self.by_key:
+                outcome["repeat"].append(event.title)
+            else:
+                outcome["new"].append(f"{event.title} ({event.start_local_date})")
+            self.by_key[event.dedupe_key] = event
+            self.key_by_ref[event.source_ref] = event.dedupe_key
+        return outcome
+
+
+def load_fixture(path: Path) -> list[dict[str, Any]]:
+    return json.loads(path.read_text())
+
+
+async def run_live(
+    today: date, hours_back: int, per_query: int
+) -> tuple[list[NormalizedEvent], IngestReport]:
+    """Exactly the call a scheduled worker makes. The CLI adds nothing to it."""
+    import httpx
+
+    if not (settings.asknews_client_id and settings.asknews_client_secret):
+        raise SystemExit(
+            "ASKNEWS_CLIENT_ID and ASKNEWS_CLIENT_SECRET must be set for --live"
+        )
+    async with httpx.AsyncClient(timeout=30.0) as http:
+        client = AskNewsClient(
+            settings.asknews_client_id, settings.asknews_client_secret, http=http
+        )
+        return await collect_events(
+            client,
+            CITIES,
+            today=today,
+            queries=DEFAULT_QUERIES,
+            articles_per_query=per_query,
+            hours_back=hours_back,
+        )
+
+
+def print_run(
+    label: str,
+    source: str,
+    report: IngestReport,
+    outcome: dict[str, list[str]],
+) -> None:
+    print(f"\n{'=' * 68}\n{label}  --  {source}\n{'=' * 68}")
+    print(report.render())
+    print("against what we already had")
+    for kind in ("new", "repeat", "revised"):
+        print(f"    {kind:<22} {len(outcome[kind])}")
+        for line in outcome[kind]:
+            print(f"        {line}")
+
+
+async def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--live", action="store_true", help="call AskNews instead of replaying"
+    )
+    parser.add_argument("--runs", type=int, default=3)
+    parser.add_argument("--hours-back", type=int, default=24)
+    parser.add_argument("--articles-per-query", type=int, default=50)
+    parser.add_argument(
+        "--today",
+        type=date.fromisoformat,
+        default=datetime.now().date(),
+        help="reference date for 'is this event still ahead of us'",
+    )
+    args = parser.parse_args()
+
+    store = EventStore()
+    totals = IngestReport()
+
+    for index in range(1, args.runs + 1):
+        if args.live:
+            events, report = await run_live(
+                args.today, args.hours_back, args.articles_per_query
+            )
+            source = f"live, last {args.hours_back}h"
+        else:
+            path = FIXTURE_DIR / f"run_{index}.json"
+            if not path.exists():
+                print(f"no fixture for run {index} at {path}")
+                break
+            events, report = extract_batch(
+                load_fixture(path), CITIES, today=args.today
+            )
+            report.queries_run = 1
+            source = f"replay {path.name}"
+
+        print_run(f"RUN {index}", source, report, store.apply(events))
+
+        totals.articles_seen += report.articles_seen
+        totals.events_kept += report.events_kept
+        totals.rejected.update(report.rejected)
+
+    print(f"\n{'=' * 68}\nACROSS ALL RUNS\n{'=' * 68}")
+    print(f"articles processed {totals.articles_seen}")
+    print(f"articles rejected  {totals.articles_rejected}")
+    for reason, count in totals.rejected.most_common():
+        print(f"    {reason:<22} {count}")
+    print(f"distinct events    {len(store.by_key)}")
+    for event in sorted(store.by_key.values(), key=lambda e: e.start_local_date):
+        print(
+            f"    {event.start_local_date} .. {event.end_local_date}  "
+            f"{event.city_slug:<10} {event.date_precision:<8} "
+            f"conf {event.confidence:.2f}  {event.title}"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))
diff --git a/backend/tests/fixtures/asknews/run_1.json b/backend/tests/fixtures/asknews/run_1.json
new file mode 100644
index 0000000..60f6d9d
--- /dev/null
+++ b/backend/tests/fixtures/asknews/run_1.json
@@ -0,0 +1,122 @@
+[
+  {
+    "article_url": "https://rte.example/forbidden-fruit-2027",
+    "eng_title": "Forbidden Fruit announces 2027 return to Kilmainham",
+    "summary": "Organisers have confirmed the festival will return to the Royal Hospital Kilmainham in Dublin on June 5-7, 2027. Tickets go on sale Friday at 9am.",
+    "key_points": ["Three-day festival", "Tickets on sale Friday"],
+    "pub_date": "2026-10-05T08:15:00+00:00",
+    "entities": {"Event": ["Forbidden Fruit"], "Location": ["Dublin", "Ireland"], "Organization": ["Royal Hospital Kilmainham"]},
+    "country": "IE",
+    "page_rank": 2,
+    "language": "en",
+    "classification": ["Entertainment"]
+  },
+  {
+    "article_url": "https://publico.example/web-summit-2026",
+    "eng_title": "Web Summit returns to Lisbon in November",
+    "summary": "Web Summit will be held at the Altice Arena in Lisbon from November 10-13, 2026, with organisers expecting another sell-out edition.",
+    "key_points": ["Altice Arena", "Four-day conference"],
+    "pub_date": "2026-10-04T11:00:00+00:00",
+    "entities": {"Event": ["Web Summit"], "Location": ["Lisbon", "Portugal"]},
+    "country": "PT",
+    "page_rank": 3,
+    "language": "en",
+    "classification": ["Technology"]
+  },
+  {
+    "article_url": "https://scotsman.example/hogmanay-2026",
+    "eng_title": "Edinburgh Hogmanay street party lineup revealed",
+    "summary": "Edinburgh's Hogmanay celebrations will run December 30, 2026 - January 1, 2027, with the street party returning to Princes Street.",
+    "key_points": ["Princes Street party", "Three nights"],
+    "pub_date": "2026-10-03T16:40:00+00:00",
+    "entities": {"Event": ["Edinburgh Hogmanay"], "Location": ["Edinburgh", "Scotland"]},
+    "country": "GB",
+    "page_rank": 2,
+    "language": "en",
+    "classification": ["Culture"]
+  },
+  {
+    "article_url": "https://guardian.example/glastonbury-2027",
+    "eng_title": "Glastonbury confirms 2027 dates",
+    "summary": "Glastonbury Festival will take over Worthy Farm in Somerset on June 23-27, 2027, organisers announced.",
+    "key_points": ["Worthy Farm"],
+    "pub_date": "2026-10-05T07:00:00+00:00",
+    "entities": {"Event": ["Glastonbury Festival"], "Location": ["Somerset", "England"]},
+    "country": "GB",
+    "page_rank": 1,
+    "language": "en",
+    "classification": ["Entertainment"]
+  },
+  {
+    "article_url": "https://irishtimes.example/theatre-festival-plans",
+    "eng_title": "Dublin Theatre Festival plans expanded programme",
+    "summary": "The Dublin Theatre Festival will expand its programme next edition, though organisers say dates have not yet been confirmed.",
+    "key_points": ["Dates to be confirmed"],
+    "pub_date": "2026-10-02T09:20:00+00:00",
+    "entities": {"Event": ["Dublin Theatre Festival"], "Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 3,
+    "language": "en",
+    "classification": ["Culture"]
+  },
+  {
+    "article_url": "https://observador.example/lisbon-marathon-recap",
+    "eng_title": "Lisbon half marathon draws record field",
+    "summary": "The Lisbon half marathon took place on Sunday and drew a record crowd of runners across the Vasco da Gama bridge.",
+    "key_points": ["Record field"],
+    "pub_date": "2026-10-05T18:30:00+00:00",
+    "entities": {"Event": ["Lisbon Half Marathon"], "Location": ["Lisbon"]},
+    "country": "PT",
+    "page_rank": 4,
+    "language": "en",
+    "classification": ["Sports"]
+  },
+  {
+    "article_url": "https://siliconrepublic.example/dublin-tech-summit",
+    "eng_title": "Dublin Tech Summit eyes spring slot",
+    "summary": "Dublin Tech Summit is scheduled for spring 2027 at the Convention Centre Dublin, organisers confirmed, with exact dates to follow.",
+    "key_points": ["Convention Centre Dublin"],
+    "pub_date": "2026-10-01T13:05:00+00:00",
+    "entities": {"Event": ["Dublin Tech Summit"], "Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 5,
+    "language": "en",
+    "classification": ["Technology"]
+  },
+  {
+    "article_url": "",
+    "eng_title": "Untitled wire item",
+    "summary": "Dublin concert announced for November 20, 2026.",
+    "key_points": [],
+    "pub_date": "2026-10-05T12:00:00+00:00",
+    "entities": {"Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 9,
+    "language": "en",
+    "classification": ["Entertainment"]
+  },
+  {
+    "article_url": "https://hotpress.example/stadium-2031",
+    "eng_title": "Dublin stadium pencils in 2031 showcase",
+    "summary": "A showcase concert at the Aviva Stadium in Dublin is scheduled for August 28, 2031 under the venue's long-range plan.",
+    "key_points": ["Long-range plan"],
+    "pub_date": "2026-10-04T10:00:00+00:00",
+    "entities": {"Event": ["Aviva Showcase"], "Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 7,
+    "language": "en",
+    "classification": ["Entertainment"]
+  },
+  {
+    "article_url": "https://heraldscotland.example/edinburgh-fringe-2027",
+    "eng_title": "Edinburgh Festival Fringe sets August window",
+    "summary": "The Edinburgh Festival Fringe will run through August 2027, with the full programme due in the new year.",
+    "key_points": ["Programme due in the new year"],
+    "pub_date": "2026-10-02T15:00:00+00:00",
+    "entities": {"Event": ["Edinburgh Festival Fringe"], "Location": ["Edinburgh"]},
+    "country": "GB",
+    "page_rank": 2,
+    "language": "en",
+    "classification": ["Culture"]
+  }
+]
diff --git a/backend/tests/fixtures/asknews/run_2.json b/backend/tests/fixtures/asknews/run_2.json
new file mode 100644
index 0000000..8f15a99
--- /dev/null
+++ b/backend/tests/fixtures/asknews/run_2.json
@@ -0,0 +1,110 @@
+[
+  {
+    "article_url": "https://rte.example/forbidden-fruit-2027",
+    "eng_title": "Forbidden Fruit announces 2027 return to Kilmainham",
+    "summary": "Organisers have confirmed the festival will return to the Royal Hospital Kilmainham in Dublin on June 5-7, 2027. Tickets go on sale Friday at 9am.",
+    "key_points": ["Three-day festival", "Tickets on sale Friday"],
+    "pub_date": "2026-10-05T08:15:00+00:00",
+    "entities": {"Event": ["Forbidden Fruit"], "Location": ["Dublin", "Ireland"], "Organization": ["Royal Hospital Kilmainham"]},
+    "country": "IE",
+    "page_rank": 2,
+    "language": "en",
+    "classification": ["Entertainment"]
+  },
+  {
+    "article_url": "https://publico.example/web-summit-2026",
+    "eng_title": "Web Summit returns to Lisbon in November",
+    "summary": "Web Summit will be held at the Altice Arena in Lisbon from November 10-13, 2026, with organisers expecting another sell-out edition.",
+    "key_points": ["Altice Arena", "Four-day conference"],
+    "pub_date": "2026-10-04T11:00:00+00:00",
+    "entities": {"Event": ["Web Summit"], "Location": ["Lisbon", "Portugal"]},
+    "country": "PT",
+    "page_rank": 3,
+    "language": "en",
+    "classification": ["Technology"]
+  },
+  {
+    "article_url": "https://techcrunch.example/web-summit-lisbon-preview",
+    "eng_title": "What to expect from this year's Lisbon tech gathering",
+    "summary": "Running November 10-13, 2026 in Lisbon, the conference will host more than 70,000 attendees at the Altice Arena.",
+    "key_points": ["70,000 attendees expected"],
+    "pub_date": "2026-10-06T06:45:00+00:00",
+    "entities": {"Event": ["Web Summit"], "Location": ["Lisbon"]},
+    "country": "PT",
+    "page_rank": 2,
+    "language": "en",
+    "classification": ["Technology"]
+  },
+  {
+    "article_url": "https://scotsman.example/hogmanay-2026",
+    "eng_title": "Edinburgh Hogmanay street party lineup revealed",
+    "summary": "Edinburgh's Hogmanay celebrations will run December 30, 2026 - January 1, 2027, with the street party returning to Princes Street.",
+    "key_points": ["Princes Street party", "Three nights"],
+    "pub_date": "2026-10-03T16:40:00+00:00",
+    "entities": {"Event": ["Edinburgh Hogmanay"], "Location": ["Edinburgh", "Scotland"]},
+    "country": "GB",
+    "page_rank": 2,
+    "language": "en",
+    "classification": ["Culture"]
+  },
+  {
+    "article_url": "https://independent.example/dublin-marathon-2026",
+    "eng_title": "Dublin Marathon route confirmed for October",
+    "summary": "The Dublin Marathon is scheduled for October 25, 2026, with organisers confirming a revised route through the city centre.",
+    "key_points": ["Revised city centre route"],
+    "pub_date": "2026-10-06T07:30:00+00:00",
+    "entities": {"Event": ["Dublin Marathon"], "Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 3,
+    "language": "en",
+    "classification": ["Sports"]
+  },
+  {
+    "article_url": "https://timeout.example/lisbon-christmas-market",
+    "eng_title": "Lisbon Christmas market dates set",
+    "summary": "The Lisbon Christmas market will open December 1, 2026 and run through January 4, 2027 in Praca do Comercio.",
+    "key_points": ["Praca do Comercio"],
+    "pub_date": "2026-10-06T09:10:00+00:00",
+    "entities": {"Event": ["Lisbon Christmas Market"], "Location": ["Lisbon"]},
+    "country": "PT",
+    "page_rank": 5,
+    "language": "en",
+    "classification": ["Culture"]
+  },
+  {
+    "article_url": "https://bbc.example/manchester-gig",
+    "eng_title": "Arena tour adds Manchester night",
+    "summary": "The tour will add a Manchester date on March 14, 2027 at the Co-op Live arena.",
+    "key_points": [],
+    "pub_date": "2026-10-06T08:00:00+00:00",
+    "entities": {"Event": ["Arena Tour"], "Location": ["Manchester"]},
+    "country": "GB",
+    "page_rank": 2,
+    "language": "en",
+    "classification": ["Entertainment"]
+  },
+  {
+    "article_url": "https://thejournal.example/dublin-protest",
+    "eng_title": "Dublin city centre march planned",
+    "summary": "A march through Dublin city centre is being organised, with a date expected to be announced in the coming weeks.",
+    "key_points": ["Date expected soon"],
+    "pub_date": "2026-10-06T10:00:00+00:00",
+    "entities": {"Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 6,
+    "language": "en",
+    "classification": ["Politics"]
+  },
+  {
+    "article_url": "https://scotsman.example/fringe-recap-2026",
+    "eng_title": "Edinburgh Fringe 2026 in numbers",
+    "summary": "This summer's Edinburgh Fringe concluded with record attendance. The festival was held across 26 days in August.",
+    "key_points": ["Record attendance"],
+    "pub_date": "2026-09-30T12:00:00+00:00",
+    "entities": {"Event": ["Edinburgh Festival Fringe"], "Location": ["Edinburgh"]},
+    "country": "GB",
+    "page_rank": 3,
+    "language": "en",
+    "classification": ["Culture"]
+  }
+]
diff --git a/backend/tests/fixtures/asknews/run_3.json b/backend/tests/fixtures/asknews/run_3.json
new file mode 100644
index 0000000..f7c2bd7
--- /dev/null
+++ b/backend/tests/fixtures/asknews/run_3.json
@@ -0,0 +1,110 @@
+[
+  {
+    "article_url": "https://rte.example/forbidden-fruit-2027",
+    "eng_title": "Forbidden Fruit moves 2027 edition back a week",
+    "summary": "Organisers have revised the dates: the festival will now return to the Royal Hospital Kilmainham in Dublin on June 11-13, 2027. Existing tickets remain valid.",
+    "key_points": ["Dates moved back a week", "Tickets remain valid"],
+    "pub_date": "2026-10-06T14:20:00+00:00",
+    "entities": {"Event": ["Forbidden Fruit"], "Location": ["Dublin", "Ireland"], "Organization": ["Royal Hospital Kilmainham"]},
+    "country": "IE",
+    "page_rank": 2,
+    "language": "en",
+    "classification": ["Entertainment"]
+  },
+  {
+    "article_url": "https://publico.example/web-summit-2026",
+    "eng_title": "Web Summit returns to Lisbon in November",
+    "summary": "Web Summit will be held at the Altice Arena in Lisbon from November 10-13, 2026, with organisers expecting another sell-out edition.",
+    "key_points": ["Altice Arena", "Four-day conference"],
+    "pub_date": "2026-10-04T11:00:00+00:00",
+    "entities": {"Event": ["Web Summit"], "Location": ["Lisbon", "Portugal"]},
+    "country": "PT",
+    "page_rank": 3,
+    "language": "en",
+    "classification": ["Technology"]
+  },
+  {
+    "article_url": "https://independent.example/dublin-marathon-2026",
+    "eng_title": "Dublin Marathon route confirmed for October",
+    "summary": "The Dublin Marathon is scheduled for October 25, 2026, with organisers confirming a revised route through the city centre.",
+    "key_points": ["Revised city centre route"],
+    "pub_date": "2026-10-06T07:30:00+00:00",
+    "entities": {"Event": ["Dublin Marathon"], "Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 3,
+    "language": "en",
+    "classification": ["Sports"]
+  },
+  {
+    "article_url": "https://rte.example/tradfest-2027",
+    "eng_title": "TradFest announces January dates for Temple Bar",
+    "summary": "TradFest will run January 20-24, 2027 across venues in Temple Bar, Dublin, organisers confirmed.",
+    "key_points": ["Temple Bar venues"],
+    "pub_date": "2026-10-06T11:00:00+00:00",
+    "entities": {"Event": ["TradFest"], "Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 4,
+    "language": "en",
+    "classification": ["Culture"]
+  },
+  {
+    "article_url": "https://irishtimes.example/tradfest-temple-bar",
+    "eng_title": "Temple Bar winter music festival confirms programme",
+    "summary": "The festival is scheduled for January 20-24, 2027 in Dublin, with more than 100 performances announced.",
+    "key_points": ["100 performances"],
+    "pub_date": "2026-10-06T12:30:00+00:00",
+    "entities": {"Event": ["TradFest"], "Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 3,
+    "language": "en",
+    "classification": ["Culture"]
+  },
+  {
+    "article_url": "https://publico.example/porto-sao-joao",
+    "eng_title": "Porto sets Sao Joao festival date",
+    "summary": "Porto's Sao Joao festival will take over the city on June 23, 2027, with fireworks over the Douro.",
+    "key_points": ["Fireworks over the Douro"],
+    "pub_date": "2026-10-06T13:00:00+00:00",
+    "entities": {"Event": ["Sao Joao"], "Location": ["Porto", "Portugal"]},
+    "country": "PT",
+    "page_rank": 4,
+    "language": "en",
+    "classification": ["Culture"]
+  },
+  {
+    "article_url": "https://heraldscotland.example/edinburgh-summit-q2",
+    "eng_title": "Edinburgh to host energy summit",
+    "summary": "An international energy summit is scheduled for Q2 2027 in Edinburgh, the organisers said, with the venue still to be confirmed.",
+    "key_points": ["Venue to be confirmed"],
+    "pub_date": "2026-10-06T15:45:00+00:00",
+    "entities": {"Event": ["International Energy Summit"], "Location": ["Edinburgh"]},
+    "country": "GB",
+    "page_rank": 5,
+    "language": "en",
+    "classification": ["Business"]
+  },
+  {
+    "article_url": "https://thejournal.example/weekend-listings",
+    "eng_title": "What's on in Dublin this weekend",
+    "summary": "A roundup of gigs and markets happening in Dublin this weekend, including a craft fair and two late-night sessions.",
+    "key_points": ["Craft fair", "Late-night sessions"],
+    "pub_date": "2026-10-06T16:00:00+00:00",
+    "entities": {"Location": ["Dublin"]},
+    "country": "IE",
+    "page_rank": 8,
+    "language": "en",
+    "classification": ["Culture"]
+  },
+  {
+    "article_url": "https://observador.example/lisbon-council-budget",
+    "eng_title": "Lisbon council approves 2027 budget",
+    "summary": "Lisbon's council approved its 2027 budget on Tuesday, allocating funds across transport and housing.",
+    "key_points": ["Transport and housing"],
+    "pub_date": "2026-10-06T17:10:00+00:00",
+    "entities": {"Location": ["Lisbon"], "Organization": ["Lisbon City Council"]},
+    "country": "PT",
+    "page_rank": 4,
+    "language": "en",
+    "classification": ["Politics"]
+  }
+]

From 6c5d73fe9e9d9b4c39b5f2f74857b6c2bec2f296 Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Wed, 7 Oct 2026 11:25:16 -0400
Subject: [PATCH 16/21] perf: scope AskNews searches per city with a country
 filter

An unscoped run rejected 188 of 198 articles on city alone -- paid-for noise. One search per city, filtered to that city's country, cuts that to 60 and raises the keep rate from 2% to 38% on three calls instead of four.

Keep rate is not quality. The kept set still contains historical entities ("Irish Civil War", "midterms"), wrong-city assignments (Reading Festival -> Dublin), and one Tribeca Lisboa run split across four rows. entities.Event is not a scheduled-event list, and confidence scores extraction certainty rather than event-ness, so it cannot yet be used as a quality filter. Precision work still outstanding.
---
 backend/foresight/ingest/asknews.py | 73 ++++++++++++++++++++---------
 backend/scripts/run_asknews.py      |  6 +--
 backend/tests/test_asknews.py       | 46 ++++++++++++++++++
 3 files changed, 97 insertions(+), 28 deletions(-)

diff --git a/backend/foresight/ingest/asknews.py b/backend/foresight/ingest/asknews.py
index 8e1c81f..d17ba80 100644
--- a/backend/foresight/ingest/asknews.py
+++ b/backend/foresight/ingest/asknews.py
@@ -50,13 +50,12 @@
 DEFAULT_TOKEN_URL = "https://auth.asknews.app/oauth2/token"
 SEARCH_PATH = "/v1/news/search"
 
-# Phrasings that actually precede a date in an announcement. Broader queries
-# return politics and sport results that never survive extraction.
-DEFAULT_QUERIES = (
-    "festival announces dates lineup tickets",
-    "concert tour date announced tickets on sale",
-    "conference convention summit scheduled venue",
-    "parade marathon street festival city centre",
+# Event-shaped phrasing, paired with a city at query time. Searching these
+# terms globally returns mostly events in cities we do not sell rooms in: an
+# unscoped run rejected 188 of 198 articles on city alone.
+EVENT_TERMS = (
+    "festival concert conference expo parade marathon "
+    "announces dates lineup tickets venue"
 )
 
 _MAX_ATTEMPTS = 3
@@ -125,6 +124,34 @@ def search_terms(self) -> tuple[str, ...]:
         return (self.name, *self.aliases)
 
 
+@dataclass(frozen=True, slots=True)
+class SearchPlan:
+    """One search call: what to ask for, and where to allow it from."""
+
+    query: str
+    countries: tuple[str, ...] = ()
+
+
+def plan_searches(
+    cities: Sequence[CityTarget], queries: Sequence[str] | None = None
+) -> tuple[SearchPlan, ...]:
+    """One scoped search per city, unless explicit queries are supplied.
+
+    Scoping at the query is what makes the spend worthwhile -- AskNews bills
+    per search, so a global query that we then discard on city is paid-for
+    noise.
+    """
+    if queries is not None:
+        return tuple(SearchPlan(query) for query in queries)
+    return tuple(
+        SearchPlan(
+            f"{city.name} {EVENT_TERMS}",
+            (city.country,) if city.country else (),
+        )
+        for city in cities
+    )
+
+
 @dataclass(slots=True)
 class IngestReport:
     """What one run saw, kept, and threw away."""
@@ -431,31 +458,31 @@ async def collect_events(
     cities: Sequence[CityTarget],
     *,
     today: date,
-    queries: Sequence[str] = DEFAULT_QUERIES,
+    queries: Sequence[str] | None = None,
     articles_per_query: int = 50,
     hours_back: int = 24,
     horizon_days: int = MAX_LOOKAHEAD_DAYS,
 ) -> tuple[list[NormalizedEvent], IngestReport]:
-    """Run every query against AskNews and extract one clean batch.
+    """Run one scoped search per city and extract a single clean batch.
 
-    `hours_back` is the scheduling knob: a daily job uses 24, a catch-up run
-    after an outage uses more. Queries are run against all target cities at
-    once because AskNews ranks globally -- filtering to our cities is the
-    extractor's job, not the query's.
+    This is the whole call a scheduled worker makes. `hours_back` is the
+    scheduling knob: an hourly job overlaps with 24, a catch-up run after an
+    outage uses more. Pass `queries` to override the per-city plan.
     """
     report = IngestReport()
     collected: list[dict[str, Any]] = []
-    for query in queries:
+    for plan in plan_searches(cities, queries):
         report.queries_run += 1
-        collected.extend(
-            await client.search_news(
-                query=query,
-                n_articles=articles_per_query,
-                hours_back=hours_back,
-                method="both",
-                strategy="latest news",
-            )
-        )
+        params: dict[str, Any] = {
+            "query": plan.query,
+            "n_articles": articles_per_query,
+            "hours_back": hours_back,
+            "method": "both",
+            "strategy": "latest news",
+        }
+        if plan.countries:
+            params["countries"] = list(plan.countries)
+        collected.extend(await client.search_news(**params))
     return extract_batch(
         collected, cities, today=today, horizon_days=horizon_days, report=report
     )
diff --git a/backend/scripts/run_asknews.py b/backend/scripts/run_asknews.py
index 1e1af62..5e71464 100644
--- a/backend/scripts/run_asknews.py
+++ b/backend/scripts/run_asknews.py
@@ -25,7 +25,6 @@
 
 from foresight.config import settings
 from foresight.ingest.asknews import (
-    DEFAULT_QUERIES,
     AskNewsClient,
     CityTarget,
     IngestReport,
@@ -100,7 +99,6 @@ async def run_live(
             client,
             CITIES,
             today=today,
-            queries=DEFAULT_QUERIES,
             articles_per_query=per_query,
             hours_back=hours_back,
         )
@@ -151,9 +149,7 @@ async def main() -> int:
             if not path.exists():
                 print(f"no fixture for run {index} at {path}")
                 break
-            events, report = extract_batch(
-                load_fixture(path), CITIES, today=args.today
-            )
+            events, report = extract_batch(load_fixture(path), CITIES, today=args.today)
             report.queries_run = 1
             source = f"replay {path.name}"
 
diff --git a/backend/tests/test_asknews.py b/backend/tests/test_asknews.py
index f31c4c3..fdaf6eb 100644
--- a/backend/tests/test_asknews.py
+++ b/backend/tests/test_asknews.py
@@ -13,9 +13,11 @@
     AskNewsClient,
     CityTarget,
     RejectReason,
+    SearchPlan,
     collect_events,
     extract_batch,
     extract_event,
+    plan_searches,
 )
 from foresight.ingest.normalize import (
     DatePrecision,
@@ -410,3 +412,47 @@ def handler(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response:
     # The same article from both queries is one event, not two.
     assert len(events) == 1
     assert report.rejected[RejectReason.DUPLICATE_IN_BATCH.value] == 1
+
+
+def test_each_city_gets_its_own_scoped_search() -> None:
+    plans = plan_searches(CITIES)
+
+    assert [p.countries for p in plans] == [("IE",), ("PT",)]
+    assert all(p.query.startswith(c.name) for p, c in zip(plans, CITIES, strict=True))
+    assert all("festival" in p.query for p in plans)
+
+
+def test_explicit_queries_override_the_per_city_plan() -> None:
+    plans = plan_searches(CITIES, ["anything"])
+
+    assert plans == (SearchPlan("anything"),)
+
+
+def test_a_city_without_a_country_is_not_country_filtered() -> None:
+    plans = plan_searches((CityTarget("Dublin", "Europe/Dublin"),))
+
+    assert plans[0].countries == ()
+
+
+async def test_collect_events_sends_one_country_scoped_search_per_city() -> None:
+    sent: list[tuple[str, list[str]]] = []
+
+    def handler(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response:
+        if request.url.path.endswith("/token"):
+            return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
+        sent.append(
+            (
+                request.url.params.get("query", ""),
+                request.url.params.get_list("countries"),
+            )
+        )
+        return httpx.Response(200, json={"as_dicts": []})
+
+    client, calls = build_client(handler)
+    _, report = await collect_events(client, CITIES, today=TODAY)
+
+    assert calls["search"] == len(CITIES)
+    assert report.queries_run == len(CITIES)
+    assert [countries for _, countries in sent] == [["IE"], ["PT"]]
+    assert sent[0][0].startswith("Dublin")
+    assert sent[1][0].startswith("Lisbon")

From 13d8190e173c6ae2221afa44c1dbc085342573ad Mon Sep 17 00:00:00 2001
From: heatherlarange 
Date: Wed, 7 Oct 2026 12:27:49 -0400
Subject: [PATCH 17/21] deletion of txt files

---
 docs/event-ingest-findings.html               | 300 ------------------
 docs/event-ingest-runs/1-dublin-90d.txt       |  15 -
 docs/event-ingest-runs/2-dublin-90d-rerun.txt |  15 -
 docs/event-ingest-runs/3-boston-90d.txt       |  15 -
 docs/event-ingest-runs/4-boston-90d-rerun.txt |  15 -
 .../5-dublin-7d-incremental.txt               |   4 -
 .../6-model-forecast-dublin.txt               |  25 --
 .../before-fix/1-dublin-90d.txt               |  15 -
 .../before-fix/2-dublin-90d-rerun.txt         |  15 -
 .../before-fix/3-boston-90d.txt               |  15 -
 .../before-fix/4-dublin-7d-incremental.txt    |   4 -
 .../before-fix/db-status-before-fix.txt       |   6 -
 .../failed/boston-90d-readtimeout.txt         |  91 ------
 13 files changed, 535 deletions(-)
 delete mode 100644 docs/event-ingest-findings.html
 delete mode 100644 docs/event-ingest-runs/1-dublin-90d.txt
 delete mode 100644 docs/event-ingest-runs/2-dublin-90d-rerun.txt
 delete mode 100644 docs/event-ingest-runs/3-boston-90d.txt
 delete mode 100644 docs/event-ingest-runs/4-boston-90d-rerun.txt
 delete mode 100644 docs/event-ingest-runs/5-dublin-7d-incremental.txt
 delete mode 100644 docs/event-ingest-runs/6-model-forecast-dublin.txt
 delete mode 100644 docs/event-ingest-runs/before-fix/1-dublin-90d.txt
 delete mode 100644 docs/event-ingest-runs/before-fix/2-dublin-90d-rerun.txt
 delete mode 100644 docs/event-ingest-runs/before-fix/3-boston-90d.txt
 delete mode 100644 docs/event-ingest-runs/before-fix/4-dublin-7d-incremental.txt
 delete mode 100644 docs/event-ingest-runs/before-fix/db-status-before-fix.txt
 delete mode 100644 docs/event-ingest-runs/failed/boston-90d-readtimeout.txt

diff --git a/docs/event-ingest-findings.html b/docs/event-ingest-findings.html
deleted file mode 100644
index 442f967..0000000
--- a/docs/event-ingest-findings.html
+++ /dev/null
@@ -1,300 +0,0 @@
-PredictHQ Event Ingest
-
-
-
-
-
-

-  

-    
Foresight · External factors · Meeting 10/7

-    
PredictHQ Event Ingest

-    
A Python ingest that pulls events near a hotel from the PredictHQ API, cleans and de-duplicates them, stores them, and feeds nightly event attendance into the pricing model.

-    
Runs executed 2026-10-06, 22:15–22:19 EDT · branch external-factors-events · raw logs in docs/event-ingest-runs/

-  

-
-  

-    
Results

-    

-      
5ingest runs across Dublin and Boston, 0 records rejected

-      
4,641raw PredictHQ listings stored

-      
4,101unique events after cleaning (540 duplicates merged)

-      
0 / 0new / updated on every re-run, so cleaning is stable

-    

-    

-      
PredictHQ APIpredicthq.py
→
-      
Normalize + validatenormalize.py
→
-      
De-duplicate + upsertrepository.py
→
-      
events tablePostgres
→
-      
Nightly attendancefeatures/events.py
→
-      
Pricing modelGBM

-    

-  

-
-  

-    
The runs

-    
Each run fetches every event within 10 km of the city centre that PredictHQ revised in the window. A re-run of the same query is the test for clean data: if cleaning is correct and stable, nothing should be new or change.

-    

-      
-        
-          
-        
-        
-          
-          
-          
-          
-          
-        
-      
#RunFetchedNewUpdatedUnchangedRejectedReads as
1Dublin, last 90 days1,3710261,345026 events restored by the fix
2Dublin, same query again1,371001,3710Stable
3Boston, last 90 days3,2700233,2470Duplicates settled by the fix
4Boston, same query again3,270003,2700Stable
5Dublin, last 7 days (daily refresh)412004120Stable

-    

-    
Every event was already stored by earlier pre-fix runs, so these runs show 0 new. On its first ingest Boston produced 2,864 new events from 3,270 listings. One extra Boston attempt timed out waiting on PredictHQ; its log is kept in failed/.

-  

-
-  

-    
What cleaning does, on real records

-    
    
-      
  • Merges duplicates. PredictHQ often lists one event several times. Records with the same normalized title, city and local date become one event.
    • 
-      
  • Normalizes titles by dropping tour names, "(Touring)", "presented by…" and "feat." so two sources can match.
    • 
-      
  • Converts to local dates. An 8 pm show is stored on that night, not on the next day in UTC.
    • 
-      
  • Fixes coordinates. PredictHQ sends [longitude, latitude], so they are swapped and range-checked.
    • 
-      
  • Validates. Records missing an id, title or date, ending before they start, or with impossible coordinates are logged and counted as rejected, and the run continues.
    • 
-      
  • Keeps cancellations. Deleted events stay in the table marked deleted, so the model knows to drop them.
    • 
-    

-
-    

-      

-        
Raw from PredictHQ
title  "Wicked (Touring)"
-start  2026-11-15T01:00:00Z
-tz     America/New_York

-        
Stored
title_norm        "wicked"
-start_local_date  2026-11-14

-        
A Saturday 8 pm Boston show is 01:00 Sunday in UTC. Without local dates its demand would land on the wrong hotel night.

-      

-      

-        
Raw from PredictHQ
4 listings, one show
-"Nikki Glaser: The Stunning Tour"
-states: active, deleted, …

-        
Stored
1 event · Boston · 2026-12-04
-status active
-4 sightings kept for audit

-        
Counting all four listings would quadruple that night's attendance in the model.

-      

-      

-        
Raw from PredictHQ
"All Ireland League 1A -
- Old Wesley vs Young Munster"
-listing A  state=active   4,607
-listing B  state=deleted  4,607

-        
Stored
1 event · Dublin · 2026-10-24
-status active · attendance 4,607

-        
The deleted copy is PredictHQ retiring a duplicate listing, not a cancelled match. Before the fix below, this game was marked deleted.

-      

-      

-        
Raw from PredictHQ
location [-71.0623119, 42.3539927]

-        
Stored
latitude   42.3539927
-longitude -71.0623119

-        
GeoJSON puts longitude first. The swap is what makes distance-to-hotel filtering work.

-      

-    

-
-    

-      
-        
-        
-          
-          
-        
-      
MarketRaw listingsUnique eventsDuplicates mergedMarked deletedNo attendance estimate
Dublin1,3711,237134 (9.8%)3244
Boston3,2702,864406 (12.4%)253163

-    

-  

-
-  

-    
Finding: re-runs were deleting live events

-    

-      
Found by the re-run test, fixed the same night

-      
Re-running the Dublin query a minute after the first run reported 238 updated instead of 0, and 20 live events switched to deleted. The cleaner correctly ignored a deleted duplicate listing on the first run. On later runs it treated that listing as the event's own record and applied its deletion. Live duplicates that differed slightly, such as attendance 210 and 211, also overwrote each other on every run.

-      
Fix: each sighting now stores its own status. Every listing of an event is ranked the same way: structured source first, then live over deleted, then the latest revision. Only the top-ranked listing sets the event's values, so every run picks the same one.

-    

-    

-      
-        
-        
-          
-          
-        
-      
Dublin re-runUpdatedActiveDeleted
Before fix2381,15658
After fix01,18232

-    

-    
The fix restored 26 Dublin events to active. Four regression tests cover it: a deleted duplicate across three re-runs, live duplicates settling on the latest revision, restoring an event a duplicate deleted, and a structured deletion outranking a live news article.

-  

-
-  

-    
What it means for the pricing model

-    
The cleaned events become one model feature: expected_attendance, the total expected attendance near the hotel each night. Same-night events are added together, a multi-day event's attendance is split over its nights, deleted events are excluded, and provisional events count by their confidence. The table prices a Dublin standard room twice, with real events and with none. The two prices differ only by the events.

-    

-      
-        
-        
-          
-          
-          
-          
-          
-          
-        
-      
NightAttendanceNo eventsWith eventsEffectBiggest event
Sat 10 Oct58,257€137.17€166.47+€29.30All Ireland League rugby
Sat 17 Oct23,548€140.05€152.91+€12.86The Rock Orchestra
Thu 22 Oct15,774€113.87€120.30+€6.43J. Cole
Sat 24 Oct48,236€131.51€161.65+€30.13Leinster vs Munster
Sun 25 Oct349,723€123.47€141.22+€17.75Irish Life Dublin Marathon
Mon 19 Oct2,090€114.93€115.28+€0.35Eliades Ochoa

-    

-    
Only attendance is real here. The model is trained on mock hotel history, and weather, competitor and flight features are mock placeholders until those sources are ingested. Read the effect column as direction and relative size, not as a price recommendation.

-  

-
-  

-    
Open questions and next steps

-    
    
-      
  • API, not scraping. Events come from PredictHQ's API, which gives structured dates, venues and attendance, instead of from scraped web pages. AskNews (news articles) is the planned second source and plugs into the same cleaning.
    • 
-      
  • Network retries. One Boston run failed on a PredictHQ read timeout. The ingest should retry before giving up.
    • 
-      
  • Very large events. The marathon's 349,723 includes spectators. The model learned from mock nights topping out near 40,000, so it can't price beyond that range. It needs a cap or a log scale once trained on real data.
    • 
-      
  • Multi-day attendance. This is assumed to be a total and split evenly across nights. Confirm against PredictHQ's documentation.
    • 
-      
  • Missing attendance. 207 events across both cities have no estimate and currently add nothing. PredictHQ's 0–100 rank could fill the gap.
    • 
-      
  • Real training data. Load hotel history into daily_performance and train on it with the same attendance feature.
    • 
-    

-  

-
-  

-    
Reproduce

-    
From backend/, with PREDICTHQ_TOKEN set in .env and the database running:

-
uv run python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report
-uv run python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report
-uv run python scripts/forecast_events.py --city Dublin --days 21
-uv run pytest

-    

-      Run 2 output, Dublin re-run
-
predicthq ingest | Dublin | 10km around 53.3498, -6.2603
-revised upstream since 2026-07-09 02:16 UTC
-
-  fetched    1371
-  ---------------
-  new           0
-  updated       0
-  unchanged  1371
-  rejected      0
-
-  dublin now holds 1237 events (1182 active, 32 deleted, 23 provisional)
-  backed by 1371 observations (1371 predicthq)

-    

-    

-      Before-fix Dublin re-run, the failing output
-
predicthq ingest | Dublin | 10km around 53.3498, -6.2603
-revised upstream since 2026-07-09 02:07 UTC
-
-  fetched    1371
-  ---------------
-  new           0
-  updated     238
-  unchanged  1133
-  rejected      0
-
-  dublin now holds 1237 events (1156 active, 58 deleted, 23 provisional)
-  backed by 1371 observations (1371 predicthq)

-    

-  

-

diff --git a/docs/event-ingest-runs/1-dublin-90d.txt b/docs/event-ingest-runs/1-dublin-90d.txt
deleted file mode 100644
index 3695f76..0000000
--- a/docs/event-ingest-runs/1-dublin-90d.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report
-# run at 2026-10-06 22:15 EDT
-
-predicthq ingest | Dublin | 10km around 53.3498, -6.2603
-revised upstream since 2026-07-09 02:15 UTC
-
-  fetched    1371
-  ---------------
-  new           0
-  updated      26
-  unchanged  1345
-  rejected      0
-
-  dublin now holds 1237 events (1182 active, 32 deleted, 23 provisional)
-  backed by 1371 observations (1371 predicthq)
diff --git a/docs/event-ingest-runs/2-dublin-90d-rerun.txt b/docs/event-ingest-runs/2-dublin-90d-rerun.txt
deleted file mode 100644
index 894f0e4..0000000
--- a/docs/event-ingest-runs/2-dublin-90d-rerun.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report
-# run at 2026-10-06 22:16 EDT
-
-predicthq ingest | Dublin | 10km around 53.3498, -6.2603
-revised upstream since 2026-07-09 02:16 UTC
-
-  fetched    1371
-  ---------------
-  new           0
-  updated       0
-  unchanged  1371
-  rejected      0
-
-  dublin now holds 1237 events (1182 active, 32 deleted, 23 provisional)
-  backed by 1371 observations (1371 predicthq)
diff --git a/docs/event-ingest-runs/3-boston-90d.txt b/docs/event-ingest-runs/3-boston-90d.txt
deleted file mode 100644
index 7972cb7..0000000
--- a/docs/event-ingest-runs/3-boston-90d.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report
-# run at 2026-10-06 22:17 EDT
-
-predicthq ingest | Boston | 10km around 42.3601, -71.0589
-revised upstream since 2026-07-09 02:17 UTC
-
-  fetched    3270
-  ---------------
-  new           0
-  updated      23
-  unchanged  3247
-  rejected      0
-
-  boston now holds 2864 events (2551 active, 253 deleted, 60 provisional)
-  backed by 3270 observations (3270 predicthq)
diff --git a/docs/event-ingest-runs/4-boston-90d-rerun.txt b/docs/event-ingest-runs/4-boston-90d-rerun.txt
deleted file mode 100644
index 7ea2601..0000000
--- a/docs/event-ingest-runs/4-boston-90d-rerun.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report
-# run at 2026-10-06 22:19 EDT
-
-predicthq ingest | Boston | 10km around 42.3601, -71.0589
-revised upstream since 2026-07-09 02:19 UTC
-
-  fetched    3270
-  ---------------
-  new           0
-  updated       0
-  unchanged  3270
-  rejected      0
-
-  boston now holds 2864 events (2551 active, 253 deleted, 60 provisional)
-  backed by 3270 observations (3270 predicthq)
diff --git a/docs/event-ingest-runs/5-dublin-7d-incremental.txt b/docs/event-ingest-runs/5-dublin-7d-incremental.txt
deleted file mode 100644
index c60a5c8..0000000
--- a/docs/event-ingest-runs/5-dublin-7d-incremental.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 7d --lat 53.3498 --lon -6.2603 --city Dublin
-# run at 2026-10-06 22:18 EDT
-
-fetched=412 new=0 updated=0 unchanged=412 rejected=0
diff --git a/docs/event-ingest-runs/6-model-forecast-dublin.txt b/docs/event-ingest-runs/6-model-forecast-dublin.txt
deleted file mode 100644
index 8166039..0000000
--- a/docs/event-ingest-runs/6-model-forecast-dublin.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-$ python scripts/forecast_events.py --city Dublin --days 21
-
-Dublin | standard | 2026-10-06 to 2026-10-26 | 445 stored events overlap
-night          attendance  no events     with   effect        95% range  biggest event
-Tue 2026-10-06      4,309     123.27   126.14    +2.87          120-127  The Hu
-Wed 2026-10-07     11,052     118.90   123.52    +4.62          118-155  Paul Smith: Happy
-Thu 2026-10-08     14,628     121.59   129.39    +7.80          126-142  Paul Smith: Happy
-Fri 2026-10-09     10,152     145.11   146.68    +1.57          141-177  The Reytons
-Sat 2026-10-10     58,257     137.17   166.47   +29.30          141-244  All Ireland League 1A - Lansdowne vs Old
-Sun 2026-10-11      9,099     117.62   121.26    +3.64          116-133  The Lord of The Rings: The Fellowship of
-Mon 2026-10-12      3,588     126.22   127.61    +1.39          121-135  Morcheeba and Alex Cornish
-Tue 2026-10-13      5,416     122.00   126.96    +4.96          122-127  Matt Bennett
-Wed 2026-10-14      6,452     123.10   127.43    +4.34          123-130  Matilda the Musical (Touring)
-Thu 2026-10-15      6,712     117.74   120.74    +3.00          116-143  Matilda the Musical (Touring)
-Fri 2026-10-16      9,865     142.93   146.19    +3.26          139-170  Matilda the Musical (Touring)
-Sat 2026-10-17     23,548     140.05   152.91   +12.86          142-200  The Rock Orchestra
-Sun 2026-10-18      8,679     127.38   129.66    +2.28          126-138  Melanie C and Sophie And The Giants
-Mon 2026-10-19      2,090     114.93   115.28    +0.35          110-140  Eliades Ochoa
-Tue 2026-10-20      5,208     124.23   127.68    +3.46          119-128  Autechre
-Wed 2026-10-21      5,021     113.44   118.82    +5.38          114-140  High Society
-Thu 2026-10-22     15,774     113.87   120.30    +6.43          117-133  J. Cole
-Fri 2026-10-23      9,500     134.15   137.31    +3.16          133-138  Levellers
-Sat 2026-10-24     48,236     131.51   161.65   +30.13          141-241  Leinster vs Munster
-Sun 2026-10-25    349,723     123.47   141.22   +17.75          137-220  Irish Life Dublin Marathon
-Mon 2026-10-26     14,568     115.90   125.38    +9.49          120-129  Macklemore
diff --git a/docs/event-ingest-runs/before-fix/1-dublin-90d.txt b/docs/event-ingest-runs/before-fix/1-dublin-90d.txt
deleted file mode 100644
index 7fa523c..0000000
--- a/docs/event-ingest-runs/before-fix/1-dublin-90d.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report
-# run at 2026-10-06 22:07 EDT
-
-predicthq ingest | Dublin | 10km around 53.3498, -6.2603
-revised upstream since 2026-07-09 02:07 UTC
-
-  fetched    1371
-  ---------------
-  new         852
-  updated     129
-  unchanged   390
-  rejected      0
-
-  dublin now holds 1237 events (1176 active, 38 deleted, 23 provisional)
-  backed by 1371 observations (1371 predicthq)
diff --git a/docs/event-ingest-runs/before-fix/2-dublin-90d-rerun.txt b/docs/event-ingest-runs/before-fix/2-dublin-90d-rerun.txt
deleted file mode 100644
index bfdb422..0000000
--- a/docs/event-ingest-runs/before-fix/2-dublin-90d-rerun.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 90d --lat 53.3498 --lon -6.2603 --city Dublin --report
-# run at 2026-10-06 22:07 EDT
-
-predicthq ingest | Dublin | 10km around 53.3498, -6.2603
-revised upstream since 2026-07-09 02:07 UTC
-
-  fetched    1371
-  ---------------
-  new           0
-  updated     238
-  unchanged  1133
-  rejected      0
-
-  dublin now holds 1237 events (1156 active, 58 deleted, 23 provisional)
-  backed by 1371 observations (1371 predicthq)
diff --git a/docs/event-ingest-runs/before-fix/3-boston-90d.txt b/docs/event-ingest-runs/before-fix/3-boston-90d.txt
deleted file mode 100644
index 7fca54b..0000000
--- a/docs/event-ingest-runs/before-fix/3-boston-90d.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report
-# run at 2026-10-06 22:08 EDT
-
-predicthq ingest | Boston | 10km around 42.3601, -71.0589
-revised upstream since 2026-07-09 02:08 UTC
-
-  fetched    3270
-  ---------------
-  new        2864
-  updated     151
-  unchanged   255
-  rejected      0
-
-  boston now holds 2864 events (2552 active, 252 deleted, 60 provisional)
-  backed by 3270 observations (3270 predicthq)
diff --git a/docs/event-ingest-runs/before-fix/4-dublin-7d-incremental.txt b/docs/event-ingest-runs/before-fix/4-dublin-7d-incremental.txt
deleted file mode 100644
index ca89421..0000000
--- a/docs/event-ingest-runs/before-fix/4-dublin-7d-incremental.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 7d --lat 53.3498 --lon -6.2603 --city Dublin
-# run at 2026-10-06 22:09 EDT
-
-fetched=412 new=0 updated=46 unchanged=366 rejected=0
diff --git a/docs/event-ingest-runs/before-fix/db-status-before-fix.txt b/docs/event-ingest-runs/before-fix/db-status-before-fix.txt
deleted file mode 100644
index a55d018..0000000
--- a/docs/event-ingest-runs/before-fix/db-status-before-fix.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-boston|ACTIVE|2552
-boston|DELETED|252
-boston|PROVISIONAL|60
-dublin|ACTIVE|1156
-dublin|DELETED|58
-dublin|PROVISIONAL|23
diff --git a/docs/event-ingest-runs/failed/boston-90d-readtimeout.txt b/docs/event-ingest-runs/failed/boston-90d-readtimeout.txt
deleted file mode 100644
index 6d571ce..0000000
--- a/docs/event-ingest-runs/failed/boston-90d-readtimeout.txt
+++ /dev/null
@@ -1,91 +0,0 @@
-$ python -m foresight.ingest.run --source predicthq --since 90d --lat 42.3601 --lon -71.0589 --city Boston --report
-# run at 2026-10-06 22:16 EDT
-
-Traceback (most recent call last):
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_transports\default.py", line 101, in map_httpcore_exceptions
-    yield
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_transports\default.py", line 394, in handle_async_request
-    resp = await self._pool.handle_async_request(req)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\connection_pool.py", line 256, in handle_async_request
-    raise exc from None
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\connection_pool.py", line 236, in handle_async_request
-    response = await connection.handle_async_request(
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\connection.py", line 103, in handle_async_request
-    return await self._connection.handle_async_request(request)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\http11.py", line 136, in handle_async_request
-    raise exc
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\http11.py", line 106, in handle_async_request
-    ) = await self._receive_response_headers(**kwargs)
-        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\http11.py", line 177, in _receive_response_headers
-    event = await self._receive_event(timeout=timeout)
-            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_async\http11.py", line 217, in _receive_event
-    data = await self._network_stream.read(
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_backends\anyio.py", line 32, in read
-    with map_exceptions(exc_map):
-         ^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\contextlib.py", line 158, in __exit__
-    self.gen.throw(value)
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpcore\_exceptions.py", line 14, in map_exceptions
-    raise to_exc(exc) from exc
-httpcore.ReadTimeout
-
-The above exception was the direct cause of the following exception:
-
-Traceback (most recent call last):
-  File "", line 198, in _run_module_as_main
-  File "", line 88, in _run_code
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\run.py", line 224, in 
-    sys.exit(main())
-             ^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\run.py", line 219, in main
-    asyncio.run(_main(args))
-  File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\runners.py", line 195, in run
-    return runner.run(main)
-           ^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\runners.py", line 118, in run
-    return self._loop.run_until_complete(task)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\base_events.py", line 691, in run_until_complete
-    return future.result()
-           ^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\run.py", line 202, in _main
-    report = await run(args)
-             ^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\run.py", line 178, in run
-    result = await predicthq.fetch_events(
-             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\foresight\ingest\predicthq.py", line 89, in fetch_events
-    response = await client.get(url, params=query)
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1768, in get
-    return await self.request(
-           ^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1540, in request
-    return await self.send(request, auth=auth, follow_redirects=follow_redirects)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1629, in send
-    response = await self._send_handling_auth(
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1657, in _send_handling_auth
-    response = await self._send_handling_redirects(
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1694, in _send_handling_redirects
-    response = await self._send_single_request(request)
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_client.py", line 1730, in _send_single_request
-    response = await transport.handle_async_request(request)
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_transports\default.py", line 393, in handle_async_request
-    with map_httpcore_exceptions():
-         ^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "C:\Users\heath\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\contextlib.py", line 158, in __exit__
-    self.gen.throw(value)
-  File "C:\Users\heath\OneDrive - Northeastern University\Generate Data\foresight\backend\.venv\Lib\site-packages\httpx\_transports\default.py", line 118, in map_httpcore_exceptions
-    raise mapped_exc(message) from exc
-httpx.ReadTimeout

From 05539801a7c3e1277643b2a335055dd075736716 Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Wed, 7 Oct 2026 12:33:25 -0400
Subject: [PATCH 18/21] feat: persist AskNews events through the shared upsert

Replaces the in-memory EventStore stand-in, which existed only because repository.py was not yet on this branch. One transaction per run; new/updated/unchanged now come from repository.Outcome rather than a local dict.
---
 backend/scripts/run_asknews.py | 81 ++++++++++++++++------------------
 1 file changed, 37 insertions(+), 44 deletions(-)

diff --git a/backend/scripts/run_asknews.py b/backend/scripts/run_asknews.py
index 5e71464..8cf0a6c 100644
--- a/backend/scripts/run_asknews.py
+++ b/backend/scripts/run_asknews.py
@@ -9,8 +9,8 @@
 without credentials or network. The extraction path is identical in both modes;
 only where the articles come from differs.
 
-Across runs the script tracks what is genuinely new, so re-polling an
-overlapping window visibly produces no duplicate events.
+Events are written through the shared upsert, so re-polling an overlapping
+window visibly produces no duplicate rows.
 """
 
 from __future__ import annotations
@@ -18,12 +18,16 @@
 import argparse
 import asyncio
 import json
-from dataclasses import dataclass, field
+from collections import Counter
 from datetime import date, datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
+from sqlalchemy import func, select
+
 from foresight.config import settings
+from foresight.database.models.external import Event, EventObservation
+from foresight.database.session import async_session_factory, engine
 from foresight.ingest.asknews import (
     AskNewsClient,
     CityTarget,
@@ -31,8 +35,11 @@
     collect_events,
     extract_batch,
 )
+from foresight.ingest.repository import Outcome, upsert_event
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
+
     from foresight.ingest.normalize import NormalizedEvent
 
 FIXTURE_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "asknews"
@@ -45,36 +52,25 @@
 )
 
 
-@dataclass
-class EventStore:
-    """Stands in for the events table so the demo needs no database.
-
-    Keyed the same way the real upsert is -- on `dedupe_key` -- so the new /
-    repeat / revised counts here are the counts the repository would produce.
-    """
+async def persist(events: Sequence[NormalizedEvent]) -> Counter[Outcome]:
+    """Write a run through the shared upsert, one transaction per run."""
+    counts: Counter[Outcome] = Counter()
+    async with async_session_factory() as session:
+        for event in events:
+            counts[await upsert_event(session, event)] += 1
+        await session.commit()
+    return counts
 
-    by_key: dict[str, NormalizedEvent] = field(default_factory=dict)
-    key_by_ref: dict[str, str] = field(default_factory=dict)
 
-    def apply(self, events: list[NormalizedEvent]) -> dict[str, list[str]]:
-        outcome: dict[str, list[str]] = {"new": [], "repeat": [], "revised": []}
-        for event in events:
-            previous_key = self.key_by_ref.get(event.source_ref)
-            if previous_key and previous_key != event.dedupe_key:
-                # Same article, different date: dedupe_key includes the start
-                # date, so a correction lands as a new row rather than an edit.
-                outcome["revised"].append(
-                    f"{event.title} -> {event.start_local_date} "
-                    f"(was {self.by_key[previous_key].start_local_date})"
-                )
-                self.by_key.pop(previous_key, None)
-            elif event.dedupe_key in self.by_key:
-                outcome["repeat"].append(event.title)
-            else:
-                outcome["new"].append(f"{event.title} ({event.start_local_date})")
-            self.by_key[event.dedupe_key] = event
-            self.key_by_ref[event.source_ref] = event.dedupe_key
-        return outcome
+async def stored_events() -> tuple[list[Event], int]:
+    async with async_session_factory() as session:
+        rows = await session.scalars(
+            select(Event).order_by(Event.start_local_date, Event.title)
+        )
+        observations = await session.scalar(
+            select(func.count()).select_from(EventObservation)
+        )
+        return list(rows), observations or 0
 
 
 def load_fixture(path: Path) -> list[dict[str, Any]]:
@@ -105,18 +101,13 @@ async def run_live(
 
 
 def print_run(
-    label: str,
-    source: str,
-    report: IngestReport,
-    outcome: dict[str, list[str]],
+    label: str, source: str, report: IngestReport, counts: Counter[Outcome]
 ) -> None:
     print(f"\n{'=' * 68}\n{label}  --  {source}\n{'=' * 68}")
     print(report.render())
-    print("against what we already had")
-    for kind in ("new", "repeat", "revised"):
-        print(f"    {kind:<22} {len(outcome[kind])}")
-        for line in outcome[kind]:
-            print(f"        {line}")
+    print("written to the events table")
+    for outcome in Outcome:
+        print(f"    {outcome.value:<22} {counts[outcome]}")
 
 
 async def main() -> int:
@@ -135,7 +126,6 @@ async def main() -> int:
     )
     args = parser.parse_args()
 
-    store = EventStore()
     totals = IngestReport()
 
     for index in range(1, args.runs + 1):
@@ -153,7 +143,7 @@ async def main() -> int:
             report.queries_run = 1
             source = f"replay {path.name}"
 
-        print_run(f"RUN {index}", source, report, store.apply(events))
+        print_run(f"RUN {index}", source, report, await persist(events))
 
         totals.articles_seen += report.articles_seen
         totals.events_kept += report.events_kept
@@ -164,13 +154,16 @@ async def main() -> int:
     print(f"articles rejected  {totals.articles_rejected}")
     for reason, count in totals.rejected.most_common():
         print(f"    {reason:<22} {count}")
-    print(f"distinct events    {len(store.by_key)}")
-    for event in sorted(store.by_key.values(), key=lambda e: e.start_local_date):
+    rows, observations = await stored_events()
+    print(f"rows in events     {len(rows)}")
+    print(f"rows in observations {observations}")
+    for event in rows:
         print(
             f"    {event.start_local_date} .. {event.end_local_date}  "
             f"{event.city_slug:<10} {event.date_precision:<8} "
             f"conf {event.confidence:.2f}  {event.title}"
         )
+    await engine.dispose()
     return 0
 
 

From 734a77bbbf63a4148ade6920a88b1d122a17c179 Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Wed, 7 Oct 2026 16:53:32 -0400
Subject: [PATCH 19/21] refactor: drop unused surface from the AskNews adapter

CityTarget.slug was never called. horizon_days and the plan_searches queries override were knobs nothing turned outside tests. No behaviour change.
---
 backend/foresight/ingest/asknews.py | 36 ++++++++---------------------
 backend/tests/test_asknews.py       | 17 ++++----------
 2 files changed, 13 insertions(+), 40 deletions(-)

diff --git a/backend/foresight/ingest/asknews.py b/backend/foresight/ingest/asknews.py
index d17ba80..7b507e7 100644
--- a/backend/foresight/ingest/asknews.py
+++ b/backend/foresight/ingest/asknews.py
@@ -115,10 +115,6 @@ class CityTarget:
     country: str | None = None
     aliases: tuple[str, ...] = ()
 
-    @property
-    def slug(self) -> str:
-        return slugify_city(self.name)
-
     @property
     def search_terms(self) -> tuple[str, ...]:
         return (self.name, *self.aliases)
@@ -132,17 +128,12 @@ class SearchPlan:
     countries: tuple[str, ...] = ()
 
 
-def plan_searches(
-    cities: Sequence[CityTarget], queries: Sequence[str] | None = None
-) -> tuple[SearchPlan, ...]:
-    """One scoped search per city, unless explicit queries are supplied.
+def plan_searches(cities: Sequence[CityTarget]) -> tuple[SearchPlan, ...]:
+    """One scoped search per city.
 
     Scoping at the query is what makes the spend worthwhile -- AskNews bills
-    per search, so a global query that we then discard on city is paid-for
-    noise.
+    per search, so a global query we then discard on city is paid-for noise.
     """
-    if queries is not None:
-        return tuple(SearchPlan(query) for query in queries)
     return tuple(
         SearchPlan(
             f"{city.name} {EVENT_TERMS}",
@@ -345,11 +336,7 @@ def _published_on(article: dict[str, Any]) -> datetime | None:
 
 
 def extract_event(
-    article: dict[str, Any],
-    cities: Sequence[CityTarget],
-    *,
-    today: date,
-    horizon_days: int = MAX_LOOKAHEAD_DAYS,
+    article: dict[str, Any], cities: Sequence[CityTarget], *, today: date
 ) -> NormalizedEvent | RejectReason:
     """One article to one event, or the reason it did not make it."""
     url = str(article.get("article_url") or "").strip()
@@ -379,7 +366,7 @@ def extract_event(
     # An event running across today still prices tonight, so test the end.
     if resolved.end < today:
         return RejectReason.EVENT_ALREADY_PAST
-    if (resolved.start - today).days > horizon_days:
+    if (resolved.start - today).days > MAX_LOOKAHEAD_DAYS:
         return RejectReason.BEYOND_HORIZON
 
     entity_titles = _entity_list(article, "Event")
@@ -419,7 +406,6 @@ def extract_batch(
     cities: Sequence[CityTarget],
     *,
     today: date,
-    horizon_days: int = MAX_LOOKAHEAD_DAYS,
     report: IngestReport | None = None,
 ) -> tuple[list[NormalizedEvent], IngestReport]:
     """Extract, then collapse same-event sightings down to the best one.
@@ -432,7 +418,7 @@ def extract_batch(
 
     for article in articles:
         report.articles_seen += 1
-        outcome = extract_event(article, cities, today=today, horizon_days=horizon_days)
+        outcome = extract_event(article, cities, today=today)
         if isinstance(outcome, RejectReason):
             report.rejected[outcome.value] += 1
             continue
@@ -458,20 +444,18 @@ async def collect_events(
     cities: Sequence[CityTarget],
     *,
     today: date,
-    queries: Sequence[str] | None = None,
     articles_per_query: int = 50,
     hours_back: int = 24,
-    horizon_days: int = MAX_LOOKAHEAD_DAYS,
 ) -> tuple[list[NormalizedEvent], IngestReport]:
     """Run one scoped search per city and extract a single clean batch.
 
     This is the whole call a scheduled worker makes. `hours_back` is the
     scheduling knob: an hourly job overlaps with 24, a catch-up run after an
-    outage uses more. Pass `queries` to override the per-city plan.
+    outage uses more.
     """
     report = IngestReport()
     collected: list[dict[str, Any]] = []
-    for plan in plan_searches(cities, queries):
+    for plan in plan_searches(cities):
         report.queries_run += 1
         params: dict[str, Any] = {
             "query": plan.query,
@@ -483,6 +467,4 @@ async def collect_events(
         if plan.countries:
             params["countries"] = list(plan.countries)
         collected.extend(await client.search_news(**params))
-    return extract_batch(
-        collected, cities, today=today, horizon_days=horizon_days, report=report
-    )
+    return extract_batch(collected, cities, today=today, report=report)
diff --git a/backend/tests/test_asknews.py b/backend/tests/test_asknews.py
index fdaf6eb..a3e53f3 100644
--- a/backend/tests/test_asknews.py
+++ b/backend/tests/test_asknews.py
@@ -13,7 +13,6 @@
     AskNewsClient,
     CityTarget,
     RejectReason,
-    SearchPlan,
     collect_events,
     extract_batch,
     extract_event,
@@ -400,14 +399,12 @@ def handler(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response:
         return httpx.Response(200, json={"as_dicts": [article()]})
 
     client, calls = build_client(handler)
-    events, report = await collect_events(
-        client, CITIES, today=TODAY, queries=("a", "b"), hours_back=6
-    )
+    events, report = await collect_events(client, CITIES, today=TODAY, hours_back=6)
 
-    assert calls["search"] == 2
-    assert queries_seen == ["a", "b"]
+    assert calls["search"] == len(CITIES)
+    assert [q.split()[0] for q in queries_seen] == ["Dublin", "Lisbon"]
     assert hours_seen == ["6", "6"]
-    assert report.queries_run == 2
+    assert report.queries_run == len(CITIES)
     assert report.articles_seen == 2
     # The same article from both queries is one event, not two.
     assert len(events) == 1
@@ -422,12 +419,6 @@ def test_each_city_gets_its_own_scoped_search() -> None:
     assert all("festival" in p.query for p in plans)
 
 
-def test_explicit_queries_override_the_per_city_plan() -> None:
-    plans = plan_searches(CITIES, ["anything"])
-
-    assert plans == (SearchPlan("anything"),)
-
-
 def test_a_city_without_a_country_is_not_country_filtered() -> None:
     plans = plan_searches((CityTarget("Dublin", "Europe/Dublin"),))
 

From 092e350c3274ab9ce56741464fdd14fe7d4ed38c Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Wed, 7 Oct 2026 16:53:32 -0400
Subject: [PATCH 20/21] docs: explain how the two event extractions fit
 together

---
 backend/scripts/run_asknews.py |  37 ++++-
 docs/event-extraction.md       | 269 +++++++++++++++++++++++++++++++++
 2 files changed, 299 insertions(+), 7 deletions(-)
 create mode 100644 docs/event-extraction.md

diff --git a/backend/scripts/run_asknews.py b/backend/scripts/run_asknews.py
index 8cf0a6c..479db40 100644
--- a/backend/scripts/run_asknews.py
+++ b/backend/scripts/run_asknews.py
@@ -100,11 +100,29 @@ async def run_live(
         )
 
 
+def describe(event: NormalizedEvent | Event) -> str:
+    """Same fields on the extracted object and the stored row."""
+    return (
+        f"    {event.start_local_date} .. {event.end_local_date}  "
+        f"{event.city_slug:<10} {event.date_precision:<8} "
+        f"conf {event.confidence:.2f}  {event.title}"
+    )
+
+
 def print_run(
-    label: str, source: str, report: IngestReport, counts: Counter[Outcome]
+    label: str,
+    source: str,
+    report: IngestReport,
+    events: Sequence[NormalizedEvent],
+    counts: Counter[Outcome] | None,
 ) -> None:
     print(f"\n{'=' * 68}\n{label}  --  {source}\n{'=' * 68}")
     print(report.render())
+    if counts is None:
+        print("extracted (nothing written)")
+        for event in events:
+            print(describe(event))
+        return
     print("written to the events table")
     for outcome in Outcome:
         print(f"    {outcome.value:<22} {counts[outcome]}")
@@ -116,6 +134,11 @@ async def main() -> int:
         "--live", action="store_true", help="call AskNews instead of replaying"
     )
     parser.add_argument("--runs", type=int, default=3)
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="print what was extracted instead of writing to the database",
+    )
     parser.add_argument("--hours-back", type=int, default=24)
     parser.add_argument("--articles-per-query", type=int, default=50)
     parser.add_argument(
@@ -143,7 +166,8 @@ async def main() -> int:
             report.queries_run = 1
             source = f"replay {path.name}"
 
-        print_run(f"RUN {index}", source, report, await persist(events))
+        counts = None if args.dry_run else await persist(events)
+        print_run(f"RUN {index}", source, report, events, counts)
 
         totals.articles_seen += report.articles_seen
         totals.events_kept += report.events_kept
@@ -154,15 +178,14 @@ async def main() -> int:
     print(f"articles rejected  {totals.articles_rejected}")
     for reason, count in totals.rejected.most_common():
         print(f"    {reason:<22} {count}")
+    if args.dry_run:
+        return 0
+
     rows, observations = await stored_events()
     print(f"rows in events     {len(rows)}")
     print(f"rows in observations {observations}")
     for event in rows:
-        print(
-            f"    {event.start_local_date} .. {event.end_local_date}  "
-            f"{event.city_slug:<10} {event.date_precision:<8} "
-            f"conf {event.confidence:.2f}  {event.title}"
-        )
+        print(describe(event))
     await engine.dispose()
     return 0
 
diff --git a/docs/event-extraction.md b/docs/event-extraction.md
new file mode 100644
index 0000000..647dde8
--- /dev/null
+++ b/docs/event-extraction.md
@@ -0,0 +1,269 @@
+# Event extraction
+
+Two sources feed the `events` table. They disagree about almost everything —
+what a record is, how trustworthy it is, how often it changes — so they meet at
+a single contract rather than talking to the database in their own shapes.
+
+```mermaid
+flowchart LR
+    PHQ[PredictHQ API] --> PA[ingest/predicthq.py]
+    AN[AskNews API] --> AA[ingest/asknews.py]
+    PA --> NE[NormalizedEvent]
+    AA --> TD[ingest/textdates.py] --> NE
+    NE --> R[ingest/repository.py
upsert_event]
+    R --> EV[(events)]
+    R --> OB[(event_observations)]
+    EV --> F[services/features/events.py]
+    F --> M[pricing models]
+```
+
+## The contract: `ingest/normalize.py`
+
+Every adapter returns `NormalizedEvent`, which validates on construction —
+empty title, end before start, confidence outside 0..1, a latitude that is
+really a longitude. An adapter cannot emit a malformed event; it can only fail
+to build one.
+
+Identity is `dedupe_key` = normalized title + city slug + local start date.
+Category is deliberately excluded: the two sources routinely disagree on it,
+and that disagreement must not split one real event into two rows.
+
+Two fields carry how much to trust a record:
+
+- `status` — `ACTIVE` for a confirmed listing, `PROVISIONAL` for a PredictHQ
+  *predicted* event or anything extracted from prose, `DELETED`/`CANCELLED`
+  once withdrawn upstream.
+- `date_precision` — `DAY`, `MONTH`, or `QUARTER`. Coarse dates are recorded
+  rather than rounded away, so "summer 2027" is kept as early signal without
+  pretending to be a stay-night.
+
+## PredictHQ — structured listings
+
+`fetch_events()` pages the API by `updated.gte` and returns a `FetchResult` of
+events plus rejections. Records arrive already dated and geocoded, so the
+adapter is mostly field mapping: GeoJSON `[lon, lat]` unpacked in the right
+order, UTC timestamps converted to the venue's local night, `state` mapped onto
+`EventStatus`.
+
+It asks for `deleted` records explicitly. Without that, an event cancelled
+upstream would sit in the demand forecast forever.
+
+Everything it emits is `confidence = 1.0` and `DatePrecision.DAY`.
+
+## AskNews — prose
+
+AskNews indexes **articles, not events**, so every record is an inference: an
+article mentions a festival, and we decide whether it pins down a title, a city
+we sell rooms in, and a date precise enough to price against. Most do not.
+Dropping those is the bulk of the work.
+
+Three stages:
+
+1. **Search** — one query per city, filtered to that city's country.
+   Scoping matters: AskNews bills per search, and an unscoped run rejected 188
+   of 198 articles on city alone.
+2. **Date resolution** (`textdates.py`) — turns "August 28-30, 2026", "opens
+   December 1 and runs through January 4" or "Q3 2027" into a span plus the
+   precision achieved. Relative forms ("this weekend") are refused outright: a
+   wrong date is worse than a missing one.
+3. **Extraction** — city match, category, confidence, and a `RejectReason` for
+   anything that does not survive. The per-run tally of those reasons is the
+   audit trail for what got dropped and why.
+
+One event per article, because `event_observations` is unique on
+`(source, source_ref)` and `source_ref` is the article URL.
+
+Everything it emits is `PROVISIONAL`, with confidence capped at 0.95. It never
+sets `expected_attendance`, `venue`, or coordinates — an article does not
+reliably state them.
+
+## Where they meet: `ingest/repository.py`
+
+`upsert_event(session, event)` is source-agnostic. It resolves identity in a
+specific order that matters:
+
+1. By `(source, source_ref)` — has this source seen this record before?
+2. Only then by `dedupe_key`.
+
+Backwards would orphan rows: a revised start date changes the `dedupe_key`, so
+resolving by key first would strand the original. `_absorb_key_collision`
+merges the two when a revision lands on a key another sighting already holds.
+
+Precedence decides who wins a contested field:
+
+```python
+_PRECEDENCE = {Source.PREDICTHQ: 2, Source.ASKNEWS: 1}
+```
+
+A structured listing outranks prose. AskNews exists to see an event *before*
+PredictHQ does, not to outvote it once it arrives.
+
+`events` holds one merged row per real-world occurrence;
+`event_observations` keeps each source's raw sighting verbatim, which is what
+makes re-runs idempotent and lets a parser bug be fixed and replayed.
+
+## How events reach pricing
+
+`services/features/events.py` turns stored events into the per-night
+`expected_attendance` feature. An event contributes only if it is day-precise,
+not withdrawn, and carries an attendance estimate; multi-day events spread
+evenly across their nights, weighted by confidence.
+
+In practice this means **AskNews events currently contribute nothing to
+pricing** — they never carry attendance. They are stored, deduped against
+PredictHQ, and wait to be corroborated.
+
+## New files
+
+Paths under `backend/`. Lines are a rough guide to where the weight sits.
+
+### The contract
+
+| File | Lines | Purpose |
+| --- | ---: | --- |
+| `foresight/ingest/normalize.py` | 254 | `NormalizedEvent`, `dedupe_key`, and the enums every adapter shares. Validates on construction. |
+
+### Adapters
+
+| File | Lines | Purpose |
+| --- | ---: | --- |
+| `foresight/ingest/predicthq.py` | 162 | Fetches structured listings near a point and maps them to `NormalizedEvent`. No DB writes. |
+| `foresight/ingest/asknews.py` | 470 | Searches news per city and infers events from articles. Owns the reject reasons and the per-run report. |
+| `foresight/ingest/textdates.py` | 276 | Resolves prose dates ("August 28-30", "Q3 2027") into a span plus the precision achieved. |
+
+### Persistence
+
+| File | Lines | Purpose |
+| --- | ---: | --- |
+| `foresight/ingest/repository.py` | 204 | `upsert_event` — source-agnostic merge onto `events` plus an append-only `event_observations` row. |
+| `.../versions/b7d41e0c9a52_canonical_events_and_observations.py` | 93 | Creates both tables. |
+| `.../versions/c3e9a1f4b6d8_observation_status.py` | 42 | Adds `status` to observations, so a withdrawal is attributable to the source that reported it. |
+
+### Consumption
+
+| File | Lines | Purpose |
+| --- | ---: | --- |
+| `foresight/services/features/events.py` | 125 | Stored events to the per-night `expected_attendance` feature the pricing models read. |
+
+### Entry points
+
+| File | Lines | Purpose |
+| --- | ---: | --- |
+| `foresight/ingest/run.py` | 240 | The ingest CLI (`python -m foresight.ingest.run`). PredictHQ only today. |
+| `scripts/run_asknews.py` | 171 | Runs the AskNews ingest and prints the cleaning report. Replays fixtures by default; `--live` calls the API. |
+| `scripts/forecast_events.py` | 123 | Prices upcoming nights using real stored events, to show their price impact. |
+
+### Tests and fixtures
+
+| File | Lines | Purpose |
+| --- | ---: | --- |
+| `tests/test_normalize.py` | 140 | Identity, idempotency, and the known source gotchas. |
+| `tests/test_predicthq.py` | 142 | Field mapping and paging, against a fake HTTP response. |
+| `tests/test_asknews.py` | 449 | What survives extraction, what gets dropped and why, and the HTTP client. |
+| `tests/test_textdates.py` | 133 | The date formats announcements actually use, and the ones we refuse. |
+| `tests/test_repository.py` | 273 | Upsert behaviour against Postgres: revisions, precedence, deletions. |
+| `tests/test_event_features.py` | 168 | Attendance spreading, exclusions, and the radius filter. |
+| `tests/test_ingest_run.py` | 54 | `--since` parsing and the run report. |
+| `tests/fixtures/asknews/run_{1,2,3}.json` | 342 | Three article batches: a first poll, an overlapping re-poll, and a date revision. Hand-built to the documented AskNews schema, not recorded responses. |
+
+## How to test it
+
+### Automated
+
+```bash
+just test                         # whole suite, from the repo root
+cd backend && uv run pytest -q tests/test_asknews.py tests/test_textdates.py
+```
+
+`test_repository.py` and `test_event_features.py` need Postgres; the rest run
+offline against fake HTTP responses. Without a database they error rather than
+skip, which looks alarming and is not.
+
+### Database
+
+```bash
+just db-up                        # docker compose up -d db migrate
+```
+
+Beware: if you already run Postgres on your own machine, it binds
+`127.0.0.1:5432` and wins over Docker for anything connecting to `localhost`.
+`just db-up` then succeeds while the app silently talks to a different
+database. Check with `lsof -nP -iTCP:5432 -sTCP:LISTEN` — more than one
+listener means you have the clash. Either stop the host server, or use it
+directly:
+
+```bash
+psql -d postgres -c "CREATE ROLE foresight LOGIN PASSWORD 'foresight';" \
+                 -c "CREATE DATABASE foresight_db OWNER foresight;"
+cd backend && uv run alembic upgrade head
+```
+
+### Console only, no database
+
+```bash
+cd backend && uv run python scripts/run_asknews.py --dry-run
+```
+
+Prints the cleaning report and every extracted event, and writes nothing. This
+is the mode to use while changing extraction: no Postgres needed, and repeated
+runs leave no rows behind. Works with `--live` too.
+
+### AskNews, without credentials
+
+```bash
+cd backend && uv run python scripts/run_asknews.py
+```
+
+Replays the three recorded batches and writes them through the real upsert.
+Expect run 1 to be all `new`, run 2 to report `unchanged` for the overlap, and
+run 3 to show one `updated` where a festival moved its dates. Running the whole
+thing twice should produce `new 0` throughout and no extra rows — that is the
+idempotency check.
+
+Pass `--today 2026-10-06` to pin the reference date; the fixtures are dated
+against it and will start ageing out of the horizon eventually.
+
+### AskNews, live
+
+Needs `ASKNEWS_CLIENT_ID` and `ASKNEWS_CLIENT_SECRET` in the repo-root `.env`
+(OAuth2 client credentials from the AskNews console, not an API key).
+
+```bash
+cd backend && uv run python scripts/run_asknews.py --live --runs 1
+```
+
+Use `--runs 1`. AskNews bills per search and the default of 3 would fire the
+same queries three times at the same window.
+
+### PredictHQ
+
+Needs `PREDICTHQ_TOKEN`.
+
+```bash
+cd backend && uv run python -m foresight.ingest.run \
+    --source predicthq --since 7d --city Dublin \
+    --lat 53.3498 --lon -6.2603 --report
+```
+
+### Checking what landed
+
+```bash
+psql -U foresight -h localhost -d foresight_db -c \
+  "SELECT start_local_date, city_slug, date_precision, status, primary_source, title
+     FROM events ORDER BY start_local_date;"
+```
+
+`event_observations` should hold one row per source sighting, and re-running an
+ingest must not grow it.
+
+## Not wired yet
+
+- `run.py` hard-codes `--source predicthq`; AskNews runs through
+  `scripts/run_asknews.py` instead.
+- Rejected articles are counted, not persisted. There is no quarantine table to
+  review them later.
+- `confidence` scores *extraction certainty*, not whether something is a real
+  bookable event — a live run scored the entity "midterms" at 0.95. It is not
+  safe as a quality filter. This matters most for the attendance weighting
+  above: if AskNews events ever gain an attendance estimate, that multiplier
+  would admit junk at near-full weight.

From 93a218e147e95d864bedb79401cbcaf296d1fa9a Mon Sep 17 00:00:00 2001
From: Alexcchip 
Date: Wed, 7 Oct 2026 17:40:24 -0400
Subject: [PATCH 21/21] feat: add --dry-run to the events ingest CLI

Fetch and print without writing, so PredictHQ output can be inspected the way scripts/run_asknews.py --dry-run already allows. With --lat/--lon/--city it touches no database and skips the automatic migration; --hotel-id still needs one to resolve the market.
---
 backend/foresight/ingest/run.py  | 40 +++++++++++++++++++++++++++-----
 backend/tests/test_ingest_run.py | 31 +++++++++++++++++++++++--
 docs/event-extraction.md         |  6 ++++-
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/backend/foresight/ingest/run.py b/backend/foresight/ingest/run.py
index 97ca933..2fc1186 100644
--- a/backend/foresight/ingest/run.py
+++ b/backend/foresight/ingest/run.py
@@ -34,6 +34,7 @@
 
     from sqlalchemy.ext.asyncio import AsyncSession
 
+    from foresight.ingest.normalize import NormalizedEvent
     from foresight.ingest.predicthq import Rejected
 
 log = logging.getLogger("foresight.ingest")
@@ -61,11 +62,18 @@ class RunReport:
     rejected: list[Rejected] = field(default_factory=list)
     events_by_status: dict[str, int] = field(default_factory=dict)
     observations_by_source: dict[str, int] = field(default_factory=dict)
+    events: list[NormalizedEvent] = field(default_factory=list)
+    dry_run: bool = False
 
     def count(self, outcome: Outcome) -> int:
         return self.outcomes.get(outcome, 0)
 
     def summary(self) -> str:
+        if self.dry_run:
+            return (
+                f"fetched={self.fetched} rejected={len(self.rejected)} "
+                "(dry run, nothing written)"
+            )
         return (
             f"fetched={self.fetched} new={self.count(Outcome.NEW)} "
             f"updated={self.count(Outcome.UPDATED)} "
@@ -86,12 +94,22 @@ def render(self) -> str:
             *(f"  {outcome.value:<11}{self.count(outcome):>4}" for outcome in Outcome),
             f"  {'rejected':<11}{len(self.rejected):>4}",
             "",
-            f"  {slugify_city(m.city)} now holds "
-            f"{sum(self.events_by_status.values())} events "
-            f"({_breakdown(self.events_by_status)})",
-            f"  backed by {sum(self.observations_by_source.values())} observations "
-            f"({_breakdown(self.observations_by_source)})",
         ]
+        if self.dry_run:
+            lines.append("  nothing written (dry run); fetched events:")
+            lines += [
+                f"    {e.start_local_date} .. {e.end_local_date}  "
+                f"{e.city_slug:<12} {e.title}"
+                for e in self.events
+            ]
+        else:
+            lines += [
+                f"  {slugify_city(m.city)} now holds "
+                f"{sum(self.events_by_status.values())} events "
+                f"({_breakdown(self.events_by_status)})",
+                f"  backed by {sum(self.observations_by_source.values())} "
+                f"observations ({_breakdown(self.observations_by_source)})",
+            ]
         if self.rejected:
             lines += ["", "  rejected records:"]
             lines += [f"    {r.source_ref:<22} {r.reason}" for r in self.rejected]
@@ -133,6 +151,11 @@ def _parser() -> argparse.ArgumentParser:
     parser.add_argument("--city", help="market city (no --hotel-id)")
     parser.add_argument("--radius", default="10km", help="e.g. 10km, 5mi")
     parser.add_argument("--report", action="store_true", help="print a full run report")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="fetch and print, without touching the database",
+    )
     return parser
 
 
@@ -177,6 +200,7 @@ async def run(args: argparse.Namespace) -> RunReport:
             market=market,
             radius=args.radius,
             since=args.since,
+            dry_run=args.dry_run,
         )
 
         result = await predicthq.fetch_events(
@@ -189,6 +213,9 @@ async def run(args: argparse.Namespace) -> RunReport:
         )
         report.fetched = result.fetched
         report.rejected = result.rejected
+        report.events = result.events
+        if args.dry_run:
+            return report
 
         for event in result.events:
             outcome = await upsert_event(session, event)
@@ -229,7 +256,8 @@ def main(argv: Sequence[str] | None = None) -> int:
         parser.error("PREDICTHQ_TOKEN is not set; add it to .env")
 
     logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s")
-    _migrate()
+    if not (args.dry_run and args.hotel_id is None):
+        _migrate()
     # DEBUG=true turns on SQL echo, which would bury the report.
     engine.echo = False
     asyncio.run(_main(args))
diff --git a/backend/tests/test_ingest_run.py b/backend/tests/test_ingest_run.py
index e51ef9c..055e898 100644
--- a/backend/tests/test_ingest_run.py
+++ b/backend/tests/test_ingest_run.py
@@ -3,11 +3,11 @@
 from __future__ import annotations
 
 import argparse
-from datetime import UTC, datetime
+from datetime import UTC, date, datetime
 
 import pytest
 
-from foresight.ingest.normalize import Source
+from foresight.ingest.normalize import NormalizedEvent, Source
 from foresight.ingest.predicthq import Rejected
 from foresight.ingest.repository import Outcome
 from foresight.ingest.run import Market, RunReport, parse_since
@@ -52,3 +52,30 @@ def test_report_lists_every_count_and_rejection() -> None:
         assert line in rendered
     assert "dublin now holds 5 events (4 active, 1 deleted)" in rendered
     assert "phq-broken" in rendered
+
+
+def test_a_dry_run_reports_what_it_fetched_and_wrote_nothing() -> None:
+    event = NormalizedEvent(
+        source=Source.PREDICTHQ,
+        source_ref="phq-1",
+        title="Forbidden Fruit",
+        city="Dublin",
+        start_local_date=date(2027, 6, 5),
+        end_local_date=date(2027, 6, 7),
+    )
+    report = RunReport(
+        source=Source.PREDICTHQ,
+        market=Market(city="Dublin", latitude=53.3498, longitude=-6.2603),
+        radius="10km",
+        since=NOW,
+        fetched=1,
+        events=[event],
+        dry_run=True,
+    )
+
+    assert report.summary() == "fetched=1 rejected=0 (dry run, nothing written)"
+    rendered = report.render()
+    assert "nothing written (dry run)" in rendered
+    assert "Forbidden Fruit" in rendered
+    # The stored-totals lines would be zeroes and read as data loss.
+    assert "now holds" not in rendered
diff --git a/docs/event-extraction.md b/docs/event-extraction.md
index 647dde8..0bd82bb 100644
--- a/docs/event-extraction.md
+++ b/docs/event-extraction.md
@@ -242,9 +242,13 @@ Needs `PREDICTHQ_TOKEN`.
 ```bash
 cd backend && uv run python -m foresight.ingest.run \
     --source predicthq --since 7d --city Dublin \
-    --lat 53.3498 --lon -6.2603 --report
+    --lat 53.3498 --lon -6.2603 --report --dry-run
 ```
 
+`--dry-run` fetches and prints without writing. With `--lat/--lon/--city` it
+needs no database at all, and skips the automatic migration too. Drop the flag
+to persist. (`--hotel-id` still needs the database, to look the hotel up.)
+
 ### Checking what landed
 
 ```bash