diff --git a/.env.example b/.env.example index 7791da1..a4cedd9 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,8 @@ DEBUG=false # Database # Format: postgresql+asyncpg://:@:/ DATABASE_URL=postgresql+asyncpg://user:password@localhost:5432/dbname + +# External event sources +PREDICTHQ_TOKEN= +ASKNEWS_CLIENT_ID= +ASKNEWS_CLIENT_SECRET= diff --git a/backend/foresight/config.py b/backend/foresight/config.py index 24eb3da..5dbeb15 100644 --- a/backend/foresight/config.py +++ b/backend/foresight/config.py @@ -3,7 +3,8 @@ class Settings(BaseSettings): model_config = SettingsConfigDict( - env_file=".env", + # Root .env is the one docker-compose reads; backend/.env overrides it. + env_file=("../.env", ".env"), env_file_encoding="utf-8", extra="ignore", ) @@ -17,5 +18,10 @@ class Settings(BaseSettings): "postgresql+asyncpg://foresight:foresight@localhost:5432/foresight_db" ) + # External event sources + predicthq_token: str = "" + asknews_client_id: str = "" + asknews_client_secret: str = "" + settings = Settings() diff --git a/backend/foresight/database/migrations/versions/b7d41e0c9a52_canonical_events_and_observations.py b/backend/foresight/database/migrations/versions/b7d41e0c9a52_canonical_events_and_observations.py new file mode 100644 index 0000000..9dbfd5e --- /dev/null +++ b/backend/foresight/database/migrations/versions/b7d41e0c9a52_canonical_events_and_observations.py @@ -0,0 +1,93 @@ +"""canonical events and observations + +Replaces the placeholder `events` table with the canonical event contract plus +an append-only `event_observations` staging table. Nothing writes to `events` +yet, so it is dropped rather than migrated. + +Revision ID: b7d41e0c9a52 +Revises: 72e850f4fa7b +Create Date: 2026-10-05 11:12:04.882341 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = 'b7d41e0c9a52' +down_revision: Union[str, None] = '72e850f4fa7b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_table('events') + op.create_table('events', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('dedupe_key', sa.String(length=64), nullable=False), + sa.Column('title', sa.String(length=300), nullable=False), + sa.Column('title_norm', sa.String(length=300), nullable=False), + sa.Column('category', sa.String(length=40), nullable=False), + sa.Column('venue', sa.String(length=200), nullable=True), + sa.Column('city', sa.String(length=120), nullable=False), + sa.Column('city_slug', sa.String(length=120), nullable=False), + sa.Column('country', sa.String(length=2), nullable=True), + sa.Column('latitude', sa.Float(), nullable=True), + sa.Column('longitude', sa.Float(), nullable=True), + sa.Column('start_local_date', sa.Date(), nullable=False), + sa.Column('end_local_date', sa.Date(), nullable=False), + sa.Column('start_at_utc', sa.DateTime(timezone=True), nullable=True), + sa.Column('end_at_utc', sa.DateTime(timezone=True), nullable=True), + sa.Column('timezone', sa.String(length=64), nullable=False), + sa.Column('date_precision', sa.String(length=20), nullable=False), + sa.Column('expected_attendance', sa.Integer(), nullable=True), + sa.Column('impact_rank', sa.Integer(), nullable=True), + sa.Column('local_rank', sa.Integer(), nullable=True), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('status', sa.String(length=20), nullable=False), + sa.Column('url', sa.String(length=600), nullable=True), + sa.Column('primary_source', sa.String(length=40), nullable=False), + sa.Column('first_seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('last_seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('dedupe_key') + ) + op.create_index(op.f('ix_events_title_norm'), 'events', ['title_norm'], unique=False) + op.create_index('ix_events_city_slug_start', 'events', ['city_slug', 'start_local_date'], unique=False) + op.create_table('event_observations', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('event_id', sa.Integer(), nullable=True), + sa.Column('source', sa.String(length=40), nullable=False), + sa.Column('source_ref', sa.String(length=600), nullable=False), + sa.Column('source_updated_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('payload', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['event_id'], ['events.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('source', 'source_ref') + ) + + +def downgrade() -> None: + op.drop_table('event_observations') + op.drop_index('ix_events_city_slug_start', table_name='events') + op.drop_index(op.f('ix_events_title_norm'), table_name='events') + op.drop_table('events') + op.create_table('events', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=300), nullable=False), + sa.Column('category', sa.String(length=60), nullable=True), + sa.Column('venue', sa.String(length=200), nullable=True), + sa.Column('city', sa.String(length=120), nullable=False), + sa.Column('start_date', sa.Date(), nullable=False), + sa.Column('end_date', sa.Date(), nullable=False), + sa.Column('expected_attendance', sa.Integer(), nullable=True), + sa.Column('latitude', sa.Float(), nullable=True), + sa.Column('longitude', sa.Float(), nullable=True), + sa.Column('source', sa.String(length=40), nullable=False), + sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) diff --git a/backend/foresight/database/migrations/versions/c3e9a1f4b6d8_observation_status.py b/backend/foresight/database/migrations/versions/c3e9a1f4b6d8_observation_status.py new file mode 100644 index 0000000..ccb4f0a --- /dev/null +++ b/backend/foresight/database/migrations/versions/c3e9a1f4b6d8_observation_status.py @@ -0,0 +1,42 @@ +"""observation status + +Each sighting keeps the status its source reported, so a deleted duplicate +listing can be told apart from the live listing it shares an event with. +Existing PredictHQ sightings are backfilled from their payload. + +Revision ID: c3e9a1f4b6d8 +Revises: b7d41e0c9a52 +Create Date: 2026-10-06 22:30:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'c3e9a1f4b6d8' +down_revision: Union[str, None] = 'b7d41e0c9a52' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + 'event_observations', sa.Column('status', sa.String(length=20), nullable=True) + ) + op.execute( + """ + UPDATE event_observations SET status = CASE payload->>'state' + WHEN 'active' THEN 'ACTIVE' + WHEN 'predicted' THEN 'PROVISIONAL' + WHEN 'deleted' THEN 'DELETED' + END + WHERE source = 'PREDICTHQ' + """ + ) + + +def downgrade() -> None: + op.drop_column('event_observations', 'status') diff --git a/backend/foresight/database/models/__init__.py b/backend/foresight/database/models/__init__.py index 932cb8a..ec1f61f 100644 --- a/backend/foresight/database/models/__init__.py +++ b/backend/foresight/database/models/__init__.py @@ -4,6 +4,7 @@ from foresight.database.models.external import ( CompetitorRate, Event, + EventObservation, FlightArrival, Weather, ) @@ -14,6 +15,7 @@ "CompetitorRate", "DailyPerformance", "Event", + "EventObservation", "FlightArrival", "Hotel", "RoomType", diff --git a/backend/foresight/database/models/external.py b/backend/foresight/database/models/external.py index 69c8f81..a803927 100644 --- a/backend/foresight/database/models/external.py +++ b/backend/foresight/database/models/external.py @@ -2,11 +2,29 @@ from datetime import date, datetime from decimal import Decimal - -from sqlalchemy import Date, DateTime, Numeric, String, func -from sqlalchemy.orm import Mapped, mapped_column +from typing import Any + +from sqlalchemy import ( + Date, + DateTime, + ForeignKey, + Index, + Numeric, + String, + UniqueConstraint, + func, +) +from sqlalchemy import Enum as SAEnum +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship from foresight.database.base import Base +from foresight.ingest.normalize import ( + DatePrecision, + EventCategory, + EventStatus, + Source, +) class Weather(Base): @@ -36,29 +54,109 @@ class Weather(Base): class Event(Base): - """Something happening nearby that drives demand.""" + """One real-world event occurrence, merged across sources. + + Identified by `dedupe_key` (normalized title + city + local start date) so + the same festival seen by PredictHQ and by AskNews lands on one row. + """ __tablename__ = "events" id: Mapped[int] = mapped_column(primary_key=True) - name: Mapped[str] = mapped_column(String(300)) - category: Mapped[str | None] = mapped_column(String(60)) + dedupe_key: Mapped[str] = mapped_column(String(64), unique=True) + + title: Mapped[str] = mapped_column(String(300)) + title_norm: Mapped[str] = mapped_column(String(300), index=True) + category: Mapped[EventCategory] = mapped_column( + SAEnum(EventCategory, native_enum=False, length=40), + default=EventCategory.UNKNOWN, + ) + venue: Mapped[str | None] = mapped_column(String(200)) city: Mapped[str] = mapped_column(String(120)) + city_slug: Mapped[str] = mapped_column(String(120)) + country: Mapped[str | None] = mapped_column(String(2)) # ISO-3166 alpha-2 + latitude: Mapped[float | None] + longitude: Mapped[float | None] - start_date: Mapped[date] = mapped_column(Date) - end_date: Mapped[date] = mapped_column(Date) + # Local dates, not UTC: a stay-night is local to the hotel. Inclusive range. + start_local_date: Mapped[date] = mapped_column(Date) + end_local_date: Mapped[date] = mapped_column(Date) + start_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + end_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + timezone: Mapped[str] = mapped_column(String(64), default="UTC") + date_precision: Mapped[DatePrecision] = mapped_column( + SAEnum(DatePrecision, native_enum=False, length=20), + default=DatePrecision.DAY, + ) expected_attendance: Mapped[int | None] + impact_rank: Mapped[int | None] # PredictHQ rank, 0-100 + local_rank: Mapped[int | None] # PredictHQ local_rank, 0-100 + confidence: Mapped[float] = mapped_column(default=1.0) - latitude: Mapped[float | None] - longitude: Mapped[float | None] + status: Mapped[EventStatus] = mapped_column( + SAEnum(EventStatus, native_enum=False, length=20), + default=EventStatus.ACTIVE, + ) + url: Mapped[str | None] = mapped_column(String(600)) + + # Which source currently owns these values, by precedence. + primary_source: Mapped[Source] = mapped_column( + SAEnum(Source, native_enum=False, length=40) + ) + + first_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + observations: Mapped[list["EventObservation"]] = relationship( + back_populates="event" + ) + + # Serves the core lookup: what is happening near this hotel on this night. + __table_args__ = ( + Index("ix_events_city_slug_start", "city_slug", "start_local_date"), + ) + + +class EventObservation(Base): + """One source's raw sighting of an event, kept verbatim. + + Append-only and keyed on (source, source_ref) -- the PredictHQ event id or + the AskNews article URL. This is what makes re-runs idempotent and lets a + revised upstream record be recognised rather than duplicated. + + `event_id` is null when a sighting cannot be resolved to a dated event, e.g. + an article that announces a festival without saying when it is. + """ + + __tablename__ = "event_observations" + + id: Mapped[int] = mapped_column(primary_key=True) + event_id: Mapped[int | None] = mapped_column(ForeignKey("events.id")) + + source: Mapped[Source] = mapped_column(SAEnum(Source, native_enum=False, length=40)) + source_ref: Mapped[str] = mapped_column(String(600)) + source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + # What this sighting reported; null only for rows predating the column. + status: Mapped[EventStatus | None] = mapped_column( + SAEnum(EventStatus, native_enum=False, length=20) + ) + + payload: Mapped[dict[str, Any]] = mapped_column(JSONB) - source: Mapped[str] = mapped_column(String(40)) captured_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now() ) + event: Mapped["Event | None"] = relationship(back_populates="observations") + + __table_args__ = (UniqueConstraint("source", "source_ref"),) + class CompetitorRate(Base): """What another hotel is charging for a given night.""" diff --git a/backend/foresight/ingest/__init__.py b/backend/foresight/ingest/__init__.py new file mode 100644 index 0000000..bcb8302 --- /dev/null +++ b/backend/foresight/ingest/__init__.py @@ -0,0 +1 @@ +"""Ingestion of external data sources: events, weather, flights, competitor rates.""" diff --git a/backend/foresight/ingest/asknews.py b/backend/foresight/ingest/asknews.py new file mode 100644 index 0000000..7b507e7 --- /dev/null +++ b/backend/foresight/ingest/asknews.py @@ -0,0 +1,470 @@ +"""AskNews adapter: news articles in, provisional events out. + +AskNews indexes articles, not events, so every record here is an inference: +an article *mentions* a festival, and we decide whether it pins down a title, +a city we operate in, and a date precise enough to price a night against. +Most articles do not, and dropping those is the bulk of the work -- the +rejection tally on `IngestReport` is the audit trail for what got dropped and +why. + +Everything produced is `EventStatus.PROVISIONAL`. PredictHQ owns a record once +it corroborates one; this source exists to see an event before PredictHQ does. + +One event per article: `event_observations` is unique on (source, source_ref) +and source_ref is the article URL, so a single article cannot carry two +sightings. Where an article announces several events we keep the first that +resolves cleanly. +""" + +from __future__ import annotations + +import asyncio +import re +from collections import Counter +from dataclasses import dataclass, field +from datetime import UTC, date, datetime, timedelta +from enum import StrEnum +from typing import TYPE_CHECKING, Any + +from foresight.ingest.normalize import ( + DatePrecision, + EventCategory, + EventStatus, + NormalizedEvent, + Source, + normalize_title, + slugify_city, +) +from foresight.ingest.textdates import ( + MAX_LOOKAHEAD_DAYS, + looks_retrospective, + resolve_date, +) + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + + import httpx + +DEFAULT_API_BASE_URL = "https://api.asknews.app" +DEFAULT_TOKEN_URL = "https://auth.asknews.app/oauth2/token" +SEARCH_PATH = "/v1/news/search" + +# Event-shaped phrasing, paired with a city at query time. Searching these +# terms globally returns mostly events in cities we do not sell rooms in: an +# unscoped run rejected 188 of 198 articles on city alone. +EVENT_TERMS = ( + "festival concert conference expo parade marathon " + "announces dates lineup tickets venue" +) + +_MAX_ATTEMPTS = 3 +_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504}) +_TOKEN_EXPIRY_SLACK_SECONDS = 60 + +_CATEGORY_KEYWORDS: tuple[tuple[EventCategory, tuple[str, ...]], ...] = ( + (EventCategory.FESTIVALS, ("festival", "fest", "carnival", "mardi gras")), + (EventCategory.CONCERTS, ("concert", "gig", "tour date", "live at", "headline")), + ( + EventCategory.SPORTS, + ("match", "fixture", "tournament", "marathon", "championship", "cup final"), + ), + ( + EventCategory.CONFERENCES, + ("conference", "summit", "symposium", "convention", "congress"), + ), + (EventCategory.EXPOS, ("expo", "trade show", "fair", "exhibition")), + ( + EventCategory.PERFORMING_ARTS, + ("theatre", "theater", "opera", "ballet", "musical", "play opens"), + ), + (EventCategory.COMMUNITY, ("parade", "street party", "community", "pride")), + (EventCategory.ACADEMIC, ("graduation", "commencement", "semester", "freshers")), + (EventCategory.POLITICS, ("election", "referendum", "state visit", "protest")), +) + +_PUBLISHER_TAIL = re.compile(r"\s*[|–—-]\s*[^|–—-]{1,40}$") +_NEWS_PREFIX = re.compile( + r"^\s*(?:breaking|exclusive|update|watch|live|just in)\s*[:–-]\s*", re.I +) + + +class RejectReason(StrEnum): + """Why an article produced no event. The cleaning report is a tally of these.""" + + RETROSPECTIVE = "retrospective" + NO_RESOLVABLE_DATE = "no_resolvable_date" + EVENT_ALREADY_PAST = "event_already_past" + BEYOND_HORIZON = "beyond_horizon" + NO_CITY_MATCH = "no_city_match" + UNUSABLE_TITLE = "unusable_title" + DUPLICATE_IN_BATCH = "duplicate_in_batch" + MALFORMED_RECORD = "malformed_record" + + +@dataclass(frozen=True, slots=True) +class CityTarget: + """A city we sell rooms in, and the only places we care about events. + + The timezone comes from here rather than from the article: a stay-night is + local to the hotel, and an article never states the venue's zone. + """ + + name: str + timezone: str + country: str | None = None + aliases: tuple[str, ...] = () + + @property + def search_terms(self) -> tuple[str, ...]: + return (self.name, *self.aliases) + + +@dataclass(frozen=True, slots=True) +class SearchPlan: + """One search call: what to ask for, and where to allow it from.""" + + query: str + countries: tuple[str, ...] = () + + +def plan_searches(cities: Sequence[CityTarget]) -> tuple[SearchPlan, ...]: + """One scoped search per city. + + Scoping at the query is what makes the spend worthwhile -- AskNews bills + per search, so a global query we then discard on city is paid-for noise. + """ + return tuple( + SearchPlan( + f"{city.name} {EVENT_TERMS}", + (city.country,) if city.country else (), + ) + for city in cities + ) + + +@dataclass(slots=True) +class IngestReport: + """What one run saw, kept, and threw away.""" + + articles_seen: int = 0 + events_kept: int = 0 + rejected: Counter[str] = field(default_factory=Counter) + by_city: Counter[str] = field(default_factory=Counter) + by_precision: Counter[str] = field(default_factory=Counter) + queries_run: int = 0 + + @property + def articles_rejected(self) -> int: + return sum(self.rejected.values()) + + @property + def keep_rate(self) -> float: + return self.events_kept / self.articles_seen if self.articles_seen else 0.0 + + def render(self) -> str: + lines = [ + f"queries run {self.queries_run}", + f"articles seen {self.articles_seen}", + f"events kept {self.events_kept} ({self.keep_rate:.0%})", + f"articles rejected {self.articles_rejected}", + ] + for reason, count in self.rejected.most_common(): + lines.append(f" {reason:<22} {count}") + if self.by_city: + lines.append("events by city") + for city, count in self.by_city.most_common(): + lines.append(f" {city:<22} {count}") + if self.by_precision: + lines.append("events by date precision") + for precision, count in self.by_precision.most_common(): + lines.append(f" {precision:<22} {count}") + return "\n".join(lines) + + +class AskNewsClient: + """Thin async client over the AskNews REST API. + + The SDK would do this too, but it drags in its own HTTP stack and DTO + layer; the two calls we need are cheaper to own than to depend on. + """ + + def __init__( + self, + client_id: str, + client_secret: str, + *, + http: httpx.AsyncClient, + base_url: str = DEFAULT_API_BASE_URL, + token_url: str = DEFAULT_TOKEN_URL, + scopes: Sequence[str] = ("news",), + ) -> None: + self._client_id = client_id + self._client_secret = client_secret + self._http = http + self._base_url = base_url.rstrip("/") + self._token_url = token_url + self._scopes = tuple(scopes) + self._token: str | None = None + self._token_expires_at: datetime | None = None + + async def _request(self, method: str, url: str, **kwargs: Any) -> httpx.Response: + """Issue a request, backing off once per retryable status.""" + for attempt in range(1, _MAX_ATTEMPTS + 1): + response = await self._http.request(method, url, **kwargs) + if response.status_code not in _RETRY_STATUSES or attempt == _MAX_ATTEMPTS: + response.raise_for_status() + return response + retry_after = response.headers.get("retry-after") + delay = float(retry_after) if retry_after and retry_after.isdigit() else 2.0 + await asyncio.sleep(delay * attempt) + raise AssertionError("unreachable") + + async def _access_token(self) -> str: + now = datetime.now(UTC) + if self._token and self._token_expires_at and now < self._token_expires_at: + return self._token + + response = await self._request( + "POST", + self._token_url, + data={ + "grant_type": "client_credentials", + "scope": " ".join(self._scopes), + }, + auth=(self._client_id, self._client_secret), + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + payload = response.json() + token = payload.get("access_token") + if not token: + raise ValueError("AskNews token response carried no access_token") + expires_in = int(payload.get("expires_in", 3600)) - _TOKEN_EXPIRY_SLACK_SECONDS + self._token = token + self._token_expires_at = now + timedelta(seconds=max(expires_in, 0)) + return token + + async def search_news(self, **params: Any) -> list[dict[str, Any]]: + """One page of articles as plain dicts.""" + token = await self._access_token() + response = await self._request( + "GET", + f"{self._base_url}{SEARCH_PATH}", + params={"return_type": "dicts", **params}, + headers={"Authorization": f"Bearer {token}"}, + ) + body = response.json() + return list(body.get("as_dicts") or []) + + +def _clean_title(raw: str) -> str: + return _PUBLISHER_TAIL.sub("", _NEWS_PREFIX.sub("", raw)).strip() + + +def _entity_list(article: dict[str, Any], key: str) -> list[str]: + entities = article.get("entities") or {} + if not isinstance(entities, dict): + return [] + values = entities.get(key) or [] + return [str(v) for v in values if str(v).strip()] + + +def _pick_title(article: dict[str, Any]) -> str: + """An Event entity names the event; a headline is about the event.""" + for candidate in _entity_list(article, "Event"): + cleaned = _clean_title(candidate) + if normalize_title(cleaned): + return cleaned + return _clean_title(str(article.get("eng_title") or article.get("title") or "")) + + +def _match_city( + article: dict[str, Any], text: str, cities: Sequence[CityTarget] +) -> tuple[CityTarget, bool] | None: + """Resolve to one of our cities. The flag marks a Location-entity hit.""" + location_slugs = {slugify_city(loc) for loc in _entity_list(article, "Location")} + haystack = text.lower() + weak: CityTarget | None = None + for city in cities: + for term in city.search_terms: + if slugify_city(term) in location_slugs: + return city, True + if weak is None and re.search(rf"\b{re.escape(term.lower())}\b", haystack): + weak = city + return (weak, False) if weak else None + + +def _classify(text: str) -> EventCategory: + lowered = text.lower() + for category, keywords in _CATEGORY_KEYWORDS: + if any(keyword in lowered for keyword in keywords): + return category + return EventCategory.UNKNOWN + + +def _score( + *, + precision: DatePrecision, + explicit_year: bool, + titled_by_entity: bool, + strong_city: bool, +) -> float: + """Never 1.0: an extraction is a guess until another source agrees.""" + confidence = 0.35 + if precision is DatePrecision.DAY: + confidence += 0.20 + elif precision is DatePrecision.MONTH: + confidence += 0.05 + if explicit_year: + confidence += 0.10 + if titled_by_entity: + confidence += 0.15 + if strong_city: + confidence += 0.15 + return min(confidence, 0.95) + + +def _published_on(article: dict[str, Any]) -> datetime | None: + raw = article.get("pub_date") or article.get("crawl_date") + if not raw: + return None + try: + moment = datetime.fromisoformat(str(raw).replace("Z", "+00:00")) + except ValueError: + return None + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +def extract_event( + article: dict[str, Any], cities: Sequence[CityTarget], *, today: date +) -> NormalizedEvent | RejectReason: + """One article to one event, or the reason it did not make it.""" + url = str(article.get("article_url") or "").strip() + if not url: + return RejectReason.MALFORMED_RECORD + + summary = str(article.get("summary") or "") + headline = str(article.get("eng_title") or article.get("title") or "") + key_points = " ".join(str(p) for p in (article.get("key_points") or [])) + text = " ".join(filter(None, (headline, summary, key_points))) + if not text.strip(): + return RejectReason.MALFORMED_RECORD + + if looks_retrospective(text): + return RejectReason.RETROSPECTIVE + + matched = _match_city(article, text, cities) + if matched is None: + return RejectReason.NO_CITY_MATCH + city, strong_city = matched + + published = _published_on(article) + reference = min(published.date(), today) if published else today + resolved = resolve_date(text, reference) + if resolved is None: + return RejectReason.NO_RESOLVABLE_DATE + # An event running across today still prices tonight, so test the end. + if resolved.end < today: + return RejectReason.EVENT_ALREADY_PAST + if (resolved.start - today).days > MAX_LOOKAHEAD_DAYS: + return RejectReason.BEYOND_HORIZON + + entity_titles = _entity_list(article, "Event") + title = _pick_title(article) + if not title or not normalize_title(title): + return RejectReason.UNUSABLE_TITLE + + try: + return NormalizedEvent( + source=Source.ASKNEWS, + source_ref=url, + title=title, + city=city.name, + start_local_date=resolved.start, + end_local_date=resolved.end, + timezone=city.timezone, + category=_classify(text), + date_precision=resolved.precision, + status=EventStatus.PROVISIONAL, + confidence=_score( + precision=resolved.precision, + explicit_year=resolved.had_explicit_year, + titled_by_entity=bool(entity_titles), + strong_city=strong_city, + ), + country=city.country, + url=url, + source_updated_at=published, + payload=article, + ) + except ValueError: + return RejectReason.MALFORMED_RECORD + + +def extract_batch( + articles: Iterable[dict[str, Any]], + cities: Sequence[CityTarget], + *, + today: date, + report: IngestReport | None = None, +) -> tuple[list[NormalizedEvent], IngestReport]: + """Extract, then collapse same-event sightings down to the best one. + + Several outlets cover the same announcement, so within-batch collapsing is + the common case, not an edge case. Highest confidence wins. + """ + report = report or IngestReport() + best: dict[str, NormalizedEvent] = {} + + for article in articles: + report.articles_seen += 1 + outcome = extract_event(article, cities, today=today) + if isinstance(outcome, RejectReason): + report.rejected[outcome.value] += 1 + continue + + incumbent = best.get(outcome.dedupe_key) + if incumbent is None: + best[outcome.dedupe_key] = outcome + continue + report.rejected[RejectReason.DUPLICATE_IN_BATCH.value] += 1 + if outcome.confidence > incumbent.confidence: + best[outcome.dedupe_key] = outcome + + events = sorted(best.values(), key=lambda e: (e.start_local_date, e.title)) + report.events_kept = len(events) + for event in events: + report.by_city[event.city_slug] += 1 + report.by_precision[str(event.date_precision)] += 1 + return events, report + + +async def collect_events( + client: AskNewsClient, + cities: Sequence[CityTarget], + *, + today: date, + articles_per_query: int = 50, + hours_back: int = 24, +) -> tuple[list[NormalizedEvent], IngestReport]: + """Run one scoped search per city and extract a single clean batch. + + This is the whole call a scheduled worker makes. `hours_back` is the + scheduling knob: an hourly job overlaps with 24, a catch-up run after an + outage uses more. + """ + report = IngestReport() + collected: list[dict[str, Any]] = [] + for plan in plan_searches(cities): + report.queries_run += 1 + params: dict[str, Any] = { + "query": plan.query, + "n_articles": articles_per_query, + "hours_back": hours_back, + "method": "both", + "strategy": "latest news", + } + if plan.countries: + params["countries"] = list(plan.countries) + collected.extend(await client.search_news(**params)) + return extract_batch(collected, cities, today=today, report=report) diff --git a/backend/foresight/ingest/normalize.py b/backend/foresight/ingest/normalize.py new file mode 100644 index 0000000..6c1d56b --- /dev/null +++ b/backend/foresight/ingest/normalize.py @@ -0,0 +1,254 @@ +"""Canonical event contract shared by every external source adapter. + +PredictHQ keys its records by event, AskNews keys them by article, and both must +land on the same row for the same real-world event -- so the dedupe key and the +field normalizers live here rather than in either adapter. +""" + +from __future__ import annotations + +import hashlib +import re +import unicodedata +from dataclasses import dataclass, field +from datetime import UTC, date, datetime, tzinfo +from enum import StrEnum +from typing import TYPE_CHECKING, Any +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +if TYPE_CHECKING: + from collections.abc import Sequence + + +class Source(StrEnum): + PREDICTHQ = "predicthq" + ASKNEWS = "asknews" + + +class EventStatus(StrEnum): + ACTIVE = "active" + PROVISIONAL = "provisional" # extracted from text, not yet corroborated + CANCELLED = "cancelled" + DELETED = "deleted" # withdrawn upstream + + +class DatePrecision(StrEnum): + """How precisely the source pinned the date. + + Anything coarser than DAY is unusable for per-night pricing but still worth + keeping as early signal, so it is recorded rather than rounded away. + """ + + DAY = "day" + MONTH = "month" + QUARTER = "quarter" + + +class EventCategory(StrEnum): + """PredictHQ's taxonomy, adopted project-wide. + + Text sources classify into these buckets instead of inventing a second + vocabulary. + """ + + ACADEMIC = "academic" + AIRPORT_DELAYS = "airport-delays" + COMMUNITY = "community" + CONCERTS = "concerts" + CONFERENCES = "conferences" + DAYLIGHT_SAVINGS = "daylight-savings" + DISASTERS = "disasters" + EXPOS = "expos" + FESTIVALS = "festivals" + HEALTH_WARNINGS = "health-warnings" + OBSERVANCES = "observances" + PERFORMING_ARTS = "performing-arts" + POLITICS = "politics" + PUBLIC_HOLIDAYS = "public-holidays" + SCHOOL_HOLIDAYS = "school-holidays" + SEVERE_WEATHER = "severe-weather" + SPORTS = "sports" + TERROR = "terror" + UNKNOWN = "unknown" + + +_BRACKETED = re.compile(r"[(\[{][^)\]}]*[)\]}]") +_PRESENTER_TAIL = re.compile( + r"\s*[-–—:|]?\s*(?:presented|sponsored|brought)\s+by\b.*$", re.I +) +_GUEST_TAIL = re.compile( + r"\s*\b(?:feat\.?|featuring|w/|with\s+special\s+guests?)\b.*$", re.I +) +_TOUR_TAIL = re.compile( + r"\s*\b(?:the\s+)?\d{4}(?:[/-]\d{2,4})?\s+\S*\s*tour\b.*$", re.I +) +_NON_ALNUM = re.compile(r"[^a-z0-9]+") +_INITIALISM = re.compile(r"\b(?:[a-z] ){1,}[a-z]\b") +_LEADING_ARTICLE = re.compile(r"^(?:the|a|an)\s+") + + +def _strip_accents(text: str) -> str: + decomposed = unicodedata.normalize("NFKD", text) + return "".join(ch for ch in decomposed if not unicodedata.combining(ch)) + + +def normalize_title(title: str) -> str: + """Reduce a title to the part two sources are likely to agree on.""" + text = _strip_accents(title).lower() + text = _BRACKETED.sub(" ", text) + for pattern in (_PRESENTER_TAIL, _GUEST_TAIL, _TOUR_TAIL): + text = pattern.sub("", text) + text = _NON_ALNUM.sub(" ", text).strip() + # "fontaines d.c." and "fontaines dc" must collapse to one key. + text = _INITIALISM.sub(lambda m: m.group(0).replace(" ", ""), text) + text = _LEADING_ARTICLE.sub("", text) + return " ".join(text.split()) + + +def slugify_city(city: str) -> str: + """Normalize a city name. Drops any trailing region, e.g. "Dublin, Ireland".""" + head = city.split(",")[0] + slug = _NON_ALNUM.sub("-", _strip_accents(head).lower()) + return slug.strip("-") + + +def dedupe_key(*, title: str, city: str, start_local_date: date) -> str: + """Stable identity for a real-world event occurrence. + + Category is deliberately excluded: two sources routinely disagree on it, and + disagreement must not split one event into two rows. + """ + basis = ( + f"{normalize_title(title)}|{slugify_city(city)}|{start_local_date.isoformat()}" + ) + return hashlib.sha256(basis.encode("utf-8")).hexdigest() + + +def resolve_timezone(name: str | None) -> tzinfo: + """IANA name to tzinfo, falling back to UTC for missing or unknown zones.""" + if not name: + return UTC + try: + return ZoneInfo(name) + except (ZoneInfoNotFoundError, ValueError): + return UTC + + +def to_local_date(moment: datetime, timezone: str | None) -> date: + """Calendar date in the event's own timezone. + + A stay-night is local, so a 21:00 event must not drift onto the next night + just because its UTC timestamp crosses midnight. + """ + aware = moment if moment.tzinfo else moment.replace(tzinfo=UTC) + return aware.astimezone(resolve_timezone(timezone)).date() + + +def coords_from_geojson( + location: Sequence[float] | None, +) -> tuple[float | None, float | None]: + """PredictHQ's `location` is GeoJSON order -- [longitude, latitude].""" + if not location or len(location) < 2: + return None, None + longitude, latitude = float(location[0]), float(location[1]) + return latitude, longitude + + +@dataclass(slots=True) +class NormalizedEvent: + """What every adapter must produce. Invalid records raise on construction.""" + + source: Source + source_ref: str + title: str + city: str + start_local_date: date + end_local_date: date + + timezone: str = "UTC" + category: EventCategory = EventCategory.UNKNOWN + date_precision: DatePrecision = DatePrecision.DAY + status: EventStatus = EventStatus.ACTIVE + confidence: float = 1.0 + + venue: str | None = None + country: str | None = None + latitude: float | None = None + longitude: float | None = None + start_at_utc: datetime | None = None + end_at_utc: datetime | None = None + expected_attendance: int | None = None + impact_rank: int | None = None + local_rank: int | None = None + url: str | None = None + + source_updated_at: datetime | None = None + payload: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.source_ref: + raise ValueError("source_ref is required") + if not self.title.strip(): + raise ValueError("title is empty") + if not self.city.strip(): + raise ValueError("city is empty") + if not normalize_title(self.title): + raise ValueError(f"title {self.title!r} normalizes to empty") + if not slugify_city(self.city): + raise ValueError(f"city {self.city!r} normalizes to empty") + if self.end_local_date < self.start_local_date: + raise ValueError( + f"end {self.end_local_date} precedes start {self.start_local_date}" + ) + if not 0.0 <= self.confidence <= 1.0: + raise ValueError(f"confidence {self.confidence} outside 0..1") + if (self.latitude is None) != (self.longitude is None): + raise ValueError("latitude and longitude must be set together") + if self.latitude is not None and not -90.0 <= self.latitude <= 90.0: + raise ValueError( + f"latitude {self.latitude} out of range -- check for swapped lon/lat" + ) + if self.longitude is not None and not -180.0 <= self.longitude <= 180.0: + raise ValueError(f"longitude {self.longitude} out of range") + + @property + def title_norm(self) -> str: + return normalize_title(self.title) + + @property + def city_slug(self) -> str: + return slugify_city(self.city) + + @property + def dedupe_key(self) -> str: + return dedupe_key( + title=self.title, city=self.city, start_local_date=self.start_local_date + ) + + def as_row(self) -> dict[str, Any]: + """Column values for the canonical `events` row.""" + return { + "dedupe_key": self.dedupe_key, + "title": self.title, + "title_norm": self.title_norm, + "category": self.category, + "venue": self.venue, + "city": self.city, + "city_slug": self.city_slug, + "country": self.country, + "latitude": self.latitude, + "longitude": self.longitude, + "start_local_date": self.start_local_date, + "end_local_date": self.end_local_date, + "start_at_utc": self.start_at_utc, + "end_at_utc": self.end_at_utc, + "timezone": self.timezone, + "date_precision": self.date_precision, + "expected_attendance": self.expected_attendance, + "impact_rank": self.impact_rank, + "local_rank": self.local_rank, + "confidence": self.confidence, + "status": self.status, + "url": self.url, + "primary_source": self.source, + } diff --git a/backend/foresight/ingest/predicthq.py b/backend/foresight/ingest/predicthq.py new file mode 100644 index 0000000..6ea1013 --- /dev/null +++ b/backend/foresight/ingest/predicthq.py @@ -0,0 +1,162 @@ +"""PredictHQ adapter: events near a hotel as NormalizedEvents. No DB writes.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any + +import httpx + +from foresight.ingest.normalize import ( + DatePrecision, + EventCategory, + EventStatus, + NormalizedEvent, + Source, + coords_from_geojson, + to_local_date, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + +log = logging.getLogger(__name__) + +API_URL = "https://api.predicthq.com/v1/events/" +PAGE_LIMIT = 500 +# The API returns only `active` unless asked; an unseen deletion would leave a +# cancelled event in the demand forecast indefinitely. +STATES = "active,predicted,deleted" + +_STATUS = { + "active": EventStatus.ACTIVE, + "predicted": EventStatus.PROVISIONAL, + "deleted": EventStatus.DELETED, +} + + +@dataclass(slots=True) +class Rejected: + source_ref: str + reason: str + + +@dataclass(slots=True) +class FetchResult: + events: list[NormalizedEvent] = field(default_factory=list) + rejected: list[Rejected] = field(default_factory=list) + + @property + def fetched(self) -> int: + return len(self.events) + len(self.rejected) + + +async def fetch_events( + *, + token: str, + latitude: float, + longitude: float, + city: str, + updated_since: datetime, + radius: str = "10km", + transport: httpx.AsyncBaseTransport | None = None, +) -> FetchResult: + """Events within `radius` of a point, revised upstream since `updated_since`. + + `city` is the hotel's market and becomes every event's city, so a suburb + venue still shares a dedupe key with a news article naming the city. + """ + params = { + "within": f"{radius}@{latitude},{longitude}", + "updated.gte": updated_since.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S"), + # A day of slack so an event running tonight in a zone behind UTC stays in. + "active.gte": (datetime.now(UTC).date() - timedelta(days=1)).isoformat(), + "state": STATES, + "limit": PAGE_LIMIT, + "sort": "updated", + } + headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} + result = FetchResult() + + async with httpx.AsyncClient( + headers=headers, timeout=30.0, transport=transport + ) as client: + url: str | None = API_URL + query: dict[str, Any] | None = params + while url: + response = await client.get(url, params=query) + response.raise_for_status() + body = response.json() + if body.get("overflow"): + log.warning("PredictHQ result overflow: plan limit truncated results") + + for raw in body.get("results", []): + try: + result.events.append(to_normalized(raw, city=city)) + except (KeyError, TypeError, ValueError) as exc: + ref = str(raw.get("id", "?")) if isinstance(raw, dict) else "?" + log.warning("rejected PredictHQ event %s: %s", ref, exc) + result.rejected.append(Rejected(ref, _reason(exc))) + + # `next` already carries every query parameter. + url, query = body.get("next"), None + + return result + + +def to_normalized(raw: Mapping[str, Any], *, city: str) -> NormalizedEvent: + start = _parse_instant(raw["start"]) + end = _parse_instant(raw["end"]) if raw.get("end") else start + timezone = raw.get("timezone") + latitude, longitude = coords_from_geojson(raw.get("location")) + + return NormalizedEvent( + source=Source.PREDICTHQ, + source_ref=raw["id"], + title=raw["title"], + city=city, + start_local_date=to_local_date(start, timezone), + end_local_date=to_local_date(end, timezone), + timezone=timezone or "UTC", + category=_category(raw.get("category")), + date_precision=DatePrecision.DAY, + status=_STATUS[raw["state"]], + confidence=1.0, + venue=_venue(raw.get("entities") or []), + country=raw.get("country"), + latitude=latitude, + longitude=longitude, + start_at_utc=start, + end_at_utc=end, + expected_attendance=raw.get("phq_attendance"), + impact_rank=raw.get("rank"), + local_rank=raw.get("local_rank"), + source_updated_at=_parse_instant(raw["updated"]) + if raw.get("updated") + else None, + payload=dict(raw), + ) + + +def _parse_instant(value: str) -> datetime: + moment = datetime.fromisoformat(value) + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +def _category(value: str | None) -> EventCategory: + try: + return EventCategory(value) + except ValueError: + return EventCategory.UNKNOWN + + +def _venue(entities: list[Mapping[str, Any]]) -> str | None: + return next((e.get("name") for e in entities if e.get("type") == "venue"), None) + + +def _reason(exc: Exception) -> str: + if isinstance(exc, KeyError): + return f"missing field {exc.args[0]!r}" + return str(exc) diff --git a/backend/foresight/ingest/repository.py b/backend/foresight/ingest/repository.py new file mode 100644 index 0000000..a3551a7 --- /dev/null +++ b/backend/foresight/ingest/repository.py @@ -0,0 +1,204 @@ +"""Persist NormalizedEvents onto the canonical `events` row and its observations.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from enum import StrEnum +from typing import TYPE_CHECKING, Any + +from sqlalchemy import and_, delete, func, literal_column, not_, or_, select, update +from sqlalchemy.dialects.postgresql import insert + +from foresight.database.models.external import Event, EventObservation +from foresight.ingest.normalize import EventStatus, Source + +if TYPE_CHECKING: + from collections.abc import Mapping + + from sqlalchemy.ext.asyncio import AsyncSession + + from foresight.ingest.normalize import NormalizedEvent + +_EVENTS = Event.__table__ +_OBSERVATIONS = EventObservation.__table__ + +# Structured listings outrank dates and venues extracted from prose. +_PRECEDENCE = {Source.PREDICTHQ: 2, Source.ASKNEWS: 1} +_WITHDRAWN = {EventStatus.DELETED, EventStatus.CANCELLED} +_NEVER = datetime.min.replace(tzinfo=UTC) + + +class Outcome(StrEnum): + NEW = "new" + UPDATED = "updated" + UNCHANGED = "unchanged" + + +async def upsert_event(session: AsyncSession, event: NormalizedEvent) -> Outcome: + """Resolve `event` to its canonical row, write it, and record the sighting. + + Does not commit; the caller owns the transaction. + """ + row = event.as_row() + event_id = await session.scalar( + select(_OBSERVATIONS.c.event_id).where( + _OBSERVATIONS.c.source == event.source, + _OBSERVATIONS.c.source_ref == event.source_ref, + ) + ) + + # Source identity must be checked before dedupe_key: a revised start date + # changes the key, and resolving by key first would orphan the old row. + if event_id is not None: + event_id = await _absorb_key_collision(session, event_id, row["dedupe_key"]) + outcome = await _apply(session, event_id, event) + else: + event_id, inserted = await _insert_or_find(session, row) + outcome = Outcome.NEW if inserted else await _apply(session, event_id, event) + + await _record_observation(session, event, event_id) + return outcome + + +async def _absorb_key_collision( + session: AsyncSession, event_id: int, new_key: str +) -> int: + """If a revision moved onto a key another sighting already holds, merge.""" + other_id = await session.scalar( + select(_EVENTS.c.id).where( + _EVENTS.c.dedupe_key == new_key, _EVENTS.c.id != event_id + ) + ) + if other_id is None: + return event_id + + await session.execute( + update(_OBSERVATIONS) + .where(_OBSERVATIONS.c.event_id == event_id) + .values(event_id=other_id) + ) + await session.execute(delete(_EVENTS).where(_EVENTS.c.id == event_id)) + return other_id + + +async def _insert_or_find( + session: AsyncSession, row: Mapping[str, Any] +) -> tuple[int, bool]: + stmt = ( + insert(_EVENTS) + .values(**row) + .on_conflict_do_update( + index_elements=[_EVENTS.c.dedupe_key], + set_={"last_seen_at": func.clock_timestamp()}, + ) + # xmax is 0 only on a freshly inserted tuple, never on a conflict update. + .returning(_EVENTS.c.id, literal_column("(xmax = 0)").label("inserted")) + ) + result = (await session.execute(stmt)).one() + return result.id, result.inserted + + +async def _apply( + session: AsyncSession, event_id: int, event: NormalizedEvent +) -> Outcome: + row = event.as_row() + current = ( + ( + await session.execute( + select(_EVENTS).where(_EVENTS.c.id == event_id).with_for_update() + ) + ) + .mappings() + .one() + ) + + changes: dict[str, Any] = {} + if await _outranks_other_sightings(session, event_id, event): + changes = {k: v for k, v in row.items() if current[k] != v} + + await session.execute( + update(_EVENTS) + .where(_EVENTS.c.id == event_id) + .values(**changes, last_seen_at=func.clock_timestamp()) + ) + return Outcome.UPDATED if changes else Outcome.UNCHANGED + + +async def _outranks_other_sightings( + session: AsyncSession, event_id: int, event: NormalizedEvent +) -> bool: + """Whether `event` is the sighting whose values the row should carry. + + Every sighting of a row is ranked the same way, so reruns settle on one + winner instead of duplicate listings overwriting each other in turn. A + deleted duplicate thus never takes down the live listing it shares a row + with, on the first run or any later one. + """ + others = await session.execute( + select( + _OBSERVATIONS.c.source, + _OBSERVATIONS.c.status, + _OBSERVATIONS.c.source_updated_at, + _OBSERVATIONS.c.source_ref, + ).where( + _OBSERVATIONS.c.event_id == event_id, + not_( + and_( + _OBSERVATIONS.c.source == event.source, + _OBSERVATIONS.c.source_ref == event.source_ref, + ) + ), + ) + ) + mine = _rank(event.source, event.status, event.source_updated_at, event.source_ref) + return all(mine > _rank(*other) for other in others) + + +def _rank( + source: Source, + status: EventStatus | None, + updated_at: datetime | None, + source_ref: str, +) -> tuple[int, bool, datetime, str]: + """Structured sources first, then live listings, then the latest revision. + + source_ref only breaks exact ties, so the winner is stable across reruns. + """ + return ( + _PRECEDENCE[Source(source)], + status not in _WITHDRAWN, + updated_at or _NEVER, + source_ref, + ) + + +async def _record_observation( + session: AsyncSession, event: NormalizedEvent, event_id: int +) -> None: + stmt = insert(_OBSERVATIONS).values( + event_id=event_id, + source=event.source, + source_ref=event.source_ref, + source_updated_at=event.source_updated_at, + status=event.status, + payload=event.payload, + ) + excluded = stmt.excluded + await session.execute( + stmt.on_conflict_do_update( + index_elements=[_OBSERVATIONS.c.source, _OBSERVATIONS.c.source_ref], + set_={ + "event_id": excluded.event_id, + "source_updated_at": excluded.source_updated_at, + "status": excluded.status, + "payload": excluded.payload, + "captured_at": func.clock_timestamp(), + }, + # captured_at marks when this payload version arrived, not every rerun. + where=or_( + _OBSERVATIONS.c.payload.is_distinct_from(excluded.payload), + _OBSERVATIONS.c.event_id.is_distinct_from(excluded.event_id), + _OBSERVATIONS.c.status.is_distinct_from(excluded.status), + ), + ) + ) diff --git a/backend/foresight/ingest/run.py b/backend/foresight/ingest/run.py new file mode 100644 index 0000000..2fc1186 --- /dev/null +++ b/backend/foresight/ingest/run.py @@ -0,0 +1,268 @@ +"""Fetch external events and upsert them into the canonical tables. + + python -m foresight.ingest.run --source predicthq --since 7d --hotel-id 1 --report + python -m foresight.ingest.run --source predicthq --since 90d \\ + --lat 53.3498 --lon -6.2603 --city Dublin --report +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import re +import sys +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +from alembic import command +from alembic.config import Config +from sqlalchemy import func, select + +from foresight.config import settings +from foresight.database.models.external import Event, EventObservation +from foresight.database.models.hotel import Hotel +from foresight.database.session import async_session_factory, engine +from foresight.ingest import predicthq +from foresight.ingest.normalize import Source, slugify_city +from foresight.ingest.repository import Outcome, upsert_event + +if TYPE_CHECKING: + from collections.abc import Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + from foresight.ingest.normalize import NormalizedEvent + from foresight.ingest.predicthq import Rejected + +log = logging.getLogger("foresight.ingest") + +_MIGRATIONS = Path(__file__).resolve().parents[1] / "database" / "migrations" +_SPAN = re.compile(r"^(\d+)([hdw])$") +_UNITS = {"h": "hours", "d": "days", "w": "weeks"} + + +@dataclass(slots=True) +class Market: + city: str + latitude: float + longitude: float + + +@dataclass(slots=True) +class RunReport: + source: Source + market: Market + radius: str + since: datetime + fetched: int = 0 + outcomes: dict[Outcome, int] = field(default_factory=dict) + rejected: list[Rejected] = field(default_factory=list) + events_by_status: dict[str, int] = field(default_factory=dict) + observations_by_source: dict[str, int] = field(default_factory=dict) + events: list[NormalizedEvent] = field(default_factory=list) + dry_run: bool = False + + def count(self, outcome: Outcome) -> int: + return self.outcomes.get(outcome, 0) + + def summary(self) -> str: + if self.dry_run: + return ( + f"fetched={self.fetched} rejected={len(self.rejected)} " + "(dry run, nothing written)" + ) + return ( + f"fetched={self.fetched} new={self.count(Outcome.NEW)} " + f"updated={self.count(Outcome.UPDATED)} " + f"unchanged={self.count(Outcome.UNCHANGED)} " + f"rejected={len(self.rejected)}" + ) + + def render(self) -> str: + m = self.market + rule = " " + "-" * 15 + lines = [ + f"{self.source.value} ingest | {m.city} | " + f"{self.radius} around {m.latitude:.4f}, {m.longitude:.4f}", + f"revised upstream since {self.since:%Y-%m-%d %H:%M} UTC", + "", + f" {'fetched':<11}{self.fetched:>4}", + rule, + *(f" {outcome.value:<11}{self.count(outcome):>4}" for outcome in Outcome), + f" {'rejected':<11}{len(self.rejected):>4}", + "", + ] + if self.dry_run: + lines.append(" nothing written (dry run); fetched events:") + lines += [ + f" {e.start_local_date} .. {e.end_local_date} " + f"{e.city_slug:<12} {e.title}" + for e in self.events + ] + else: + lines += [ + f" {slugify_city(m.city)} now holds " + f"{sum(self.events_by_status.values())} events " + f"({_breakdown(self.events_by_status)})", + f" backed by {sum(self.observations_by_source.values())} " + f"observations ({_breakdown(self.observations_by_source)})", + ] + if self.rejected: + lines += ["", " rejected records:"] + lines += [f" {r.source_ref:<22} {r.reason}" for r in self.rejected] + return "\n".join(lines) + + +def _breakdown(counts: dict[str, int]) -> str: + return ", ".join(f"{n} {k}" for k, n in sorted(counts.items())) or "none" + + +def parse_since(value: str, *, now: datetime | None = None) -> datetime: + """`7d`, `12h`, `2w`, or an ISO date/datetime (UTC if no offset given).""" + if match := _SPAN.match(value.strip().lower()): + amount, unit = int(match[1]), match[2] + return (now or datetime.now(UTC)) - timedelta(**{_UNITS[unit]: amount}) + try: + moment = datetime.fromisoformat(value) + except ValueError: + raise argparse.ArgumentTypeError( + f"expected a span like 7d, 12h, 2w or an ISO date, got {value!r}" + ) from None + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m foresight.ingest.run", description=__doc__.splitlines()[0] + ) + parser.add_argument("--source", choices=[Source.PREDICTHQ.value], required=True) + parser.add_argument( + "--since", + type=parse_since, + default="7d", + help="only records revised upstream since then (default: 7d)", + ) + parser.add_argument("--hotel-id", type=int, help="use this hotel's stored lat/lon") + parser.add_argument("--lat", type=float, help="market latitude (no --hotel-id)") + parser.add_argument("--lon", type=float, help="market longitude (no --hotel-id)") + parser.add_argument("--city", help="market city (no --hotel-id)") + parser.add_argument("--radius", default="10km", help="e.g. 10km, 5mi") + parser.add_argument("--report", action="store_true", help="print a full run report") + parser.add_argument( + "--dry-run", + action="store_true", + help="fetch and print, without touching the database", + ) + return parser + + +async def _resolve_market(session: AsyncSession, args: argparse.Namespace) -> Market: + if args.hotel_id is None: + return Market(city=args.city, latitude=args.lat, longitude=args.lon) + + hotel = await session.get(Hotel, args.hotel_id) + if hotel is None: + raise SystemExit(f"error: no hotel with id {args.hotel_id}") + if hotel.latitude is None or hotel.longitude is None: + raise SystemExit(f"error: hotel {hotel.id} ({hotel.name}) has no lat/lon") + return Market(city=hotel.city, latitude=hotel.latitude, longitude=hotel.longitude) + + +async def _market_totals( + session: AsyncSession, city: str +) -> tuple[dict[str, int], dict[str, int]]: + slug = slugify_city(city) + by_status = await session.execute( + select(Event.status, func.count()) + .where(Event.city_slug == slug) + .group_by(Event.status) + ) + by_source = await session.execute( + select(EventObservation.source, func.count()) + .join(Event, EventObservation.event_id == Event.id) + .where(Event.city_slug == slug) + .group_by(EventObservation.source) + ) + return ( + {status.value: n for status, n in by_status.all()}, + {source.value: n for source, n in by_source.all()}, + ) + + +async def run(args: argparse.Namespace) -> RunReport: + async with async_session_factory() as session: + market = await _resolve_market(session, args) + report = RunReport( + source=Source(args.source), + market=market, + radius=args.radius, + since=args.since, + dry_run=args.dry_run, + ) + + result = await predicthq.fetch_events( + token=settings.predicthq_token, + latitude=market.latitude, + longitude=market.longitude, + city=market.city, + updated_since=args.since, + radius=args.radius, + ) + report.fetched = result.fetched + report.rejected = result.rejected + report.events = result.events + if args.dry_run: + return report + + for event in result.events: + outcome = await upsert_event(session, event) + report.outcomes[outcome] = report.outcomes.get(outcome, 0) + 1 + await session.commit() + + report.events_by_status, report.observations_by_source = await _market_totals( + session, market.city + ) + return report + + +def _migrate() -> None: + """Bring the schema up to date, so a fresh pull needs no manual step. + + A no-op when already at head. No ini file is passed, so Alembic leaves this + process's logging alone. + """ + config = Config() + config.set_main_option("script_location", str(_MIGRATIONS)) + command.upgrade(config, "head") + + +async def _main(args: argparse.Namespace) -> None: + try: + report = await run(args) + finally: + await engine.dispose() + print(report.render() if args.report else report.summary()) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = _parser() + args = parser.parse_args(argv) + if args.hotel_id is None and None in (args.lat, args.lon, args.city): + parser.error("give --hotel-id, or all of --lat, --lon and --city") + if args.source == Source.PREDICTHQ and not settings.predicthq_token: + parser.error("PREDICTHQ_TOKEN is not set; add it to .env") + + logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s") + if not (args.dry_run and args.hotel_id is None): + _migrate() + # DEBUG=true turns on SQL echo, which would bury the report. + engine.echo = False + asyncio.run(_main(args)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/foresight/ingest/textdates.py b/backend/foresight/ingest/textdates.py new file mode 100644 index 0000000..c891522 --- /dev/null +++ b/backend/foresight/ingest/textdates.py @@ -0,0 +1,276 @@ +"""Resolve event dates out of prose. + +Text sources announce dates the way people write them -- "August 28-30", +"next summer", "Q3 2026" -- while pricing needs a concrete local stay-night. +This module turns the former into the latter, or refuses, and always reports +how precisely it managed it. + +Relative expressions ("this weekend", "next Friday") are deliberately not +resolved: they are only correct relative to a publication moment we do not +always trust, and a wrong date is worse here than a missing one. +""" + +from __future__ import annotations + +import calendar +import re +from dataclasses import dataclass +from datetime import date + +from foresight.ingest.normalize import DatePrecision + +MAX_LOOKAHEAD_DAYS = 540 +_PAST_TOLERANCE_DAYS = 2 + +_MONTHS = { + "january": 1, + "jan": 1, + "february": 2, + "feb": 2, + "march": 3, + "mar": 3, + "april": 4, + "apr": 4, + "may": 5, + "june": 6, + "jun": 6, + "july": 7, + "jul": 7, + "august": 8, + "aug": 8, + "september": 9, + "sept": 9, + "sep": 9, + "october": 10, + "oct": 10, + "november": 11, + "nov": 11, + "december": 12, + "dec": 12, +} +_MONTH_ALT = "|".join(sorted(_MONTHS, key=len, reverse=True)) + +# Northern-hemisphere seasons: start month, length in months. +_SEASONS = { + "spring": (3, 3), + "summer": (6, 3), + "fall": (9, 3), + "autumn": (9, 3), + "winter": (12, 3), +} + +# "opens December 1 and runs through January 4" is a range too, not two dates. +_RUN_PHRASE = r"(?:and\s+)?(?:will\s+)?runs?\s+(?:through|until|to)" +_DASH = rf"(?:{_RUN_PHRASE}|--|-|‐|‑|‒|–|—|through|until|thru|to)" +_ORD = r"(?:st|nd|rd|th)?" + +_ISO = re.compile( + rf"\b(\d{{4}})-(\d{{2}})-(\d{{2}})\b(?:\s*{_DASH}\s*(\d{{4}})-(\d{{2}})-(\d{{2}})\b)?" +) +# (?!\d) on every day group: without it "28 August 2026" reads the "20" of +# the year as the day and silently resolves eight days early. +_DAY = r"(\d{1,2})(?!\d)" +_YEAR = r"(\d{4})(?!\d)" + +# A year can sit on either side of the dash: "December 30, 2026 - January 1, 2027". +_MONTH_DAY = re.compile( + rf"\b({_MONTH_ALT})\.?\s+{_DAY}{_ORD}(?:,?\s*{_YEAR})?" + rf"(?:\s*{_DASH}\s*(?:({_MONTH_ALT})\.?\s+)?{_DAY}{_ORD}(?:,?\s*{_YEAR})?)?", + re.I, +) +_DAY_MONTH = re.compile( + rf"\b{_DAY}{_ORD}(?:\s*{_DASH}\s*{_DAY}{_ORD})?\s+(?:of\s+)?({_MONTH_ALT})\.?" + rf"(?:,?\s*{_YEAR})?", + re.I, +) +_MONTH_YEAR = re.compile(rf"\b({_MONTH_ALT})\.?,?\s+{_YEAR}", re.I) +_SEASON_YEAR = re.compile(rf"\b({'|'.join(_SEASONS)})\s+(?:of\s+)?(\d{{4}})\b", re.I) +_QUARTER_YEAR = re.compile(r"\bQ([1-4])\s*(?:of\s*)?(\d{4})\b", re.I) + +_RETROSPECTIVE = re.compile( + r"\b(?:was|were)\s+held\b|\btook\s+place\b|\bwrapped\s+up\b" + r"|\bdrew\s+(?:a\s+)?crowds?\b|\bconcluded\b|\blast\s+year'?s?\b", + re.I, +) +_FORWARD_LOOKING = re.compile( + r"\bwill\b|\bannounc\w+\b|\bupcoming\b|\breturns?\b|\bset\s+for\b|\bscheduled\b" + r"|\bgo\s+on\s+sale\b|\btickets?\b|\bthis\s+year'?s?\b|\bnext\s+year'?s?\b" + r"|\bline-?up\b", + re.I, +) + + +@dataclass(frozen=True, slots=True) +class ResolvedDate: + start: date + end: date + precision: DatePrecision + had_explicit_year: bool + + +def looks_retrospective(text: str) -> bool: + """True for articles reporting on an event that already happened. + + A forward-looking marker wins: write-ups routinely recap last year before + announcing this year's dates, and those are the articles worth keeping. + """ + return bool(_RETROSPECTIVE.search(text)) and not _FORWARD_LOOKING.search(text) + + +def _month_number(name: str) -> int: + return _MONTHS[name.lower().rstrip(".")] + + +def _safe_date(year: int, month: int, day: int) -> date | None: + try: + return date(year, month, day) + except ValueError: + return None + + +def _end_of_month(year: int, month: int) -> date: + return date(year, month, calendar.monthrange(year, month)[1]) + + +def _infer_year(month: int, day: int, reference: date) -> int | None: + """Pick the year that puts an undated "August 28" in the plausible future.""" + for year in (reference.year, reference.year + 1): + candidate = _safe_date(year, month, day) + if candidate and (candidate - reference).days >= -_PAST_TOLERANCE_DAYS: + return year + return None + + +def _span_from_months(start_month: int, months: int, year: int) -> tuple[date, date]: + start = date(year, start_month, 1) + end_month, end_year = start_month + months - 1, year + if end_month > 12: + end_month, end_year = end_month - 12, year + 1 + return start, _end_of_month(end_year, end_month) + + +def _day_range( + month: int, + day: int, + end_month: int | None, + end_day: int | None, + year: int, + end_year: int | None = None, +) -> tuple[date, date] | None: + start = _safe_date(year, month, day) + if not start: + return None + end = start + if end_day: + # "August 28 - September 1" and "December 30 - January 2" both roll + # forward, the latter into the next year. + target_month = end_month or month + resolved = end_year or (year + 1 if target_month < month else year) + parsed_end = _safe_date(resolved, target_month, end_day) + if parsed_end and parsed_end >= start: + end = parsed_end + return start, end + + +def _try_iso(text: str, _reference: date) -> ResolvedDate | None: + match = _ISO.search(text) + if not match: + return None + start = _safe_date(int(match[1]), int(match[2]), int(match[3])) + if not start: + return None + end = start + if match[4]: + parsed_end = _safe_date(int(match[4]), int(match[5]), int(match[6])) + if parsed_end and parsed_end >= start: + end = parsed_end + return ResolvedDate(start, end, DatePrecision.DAY, had_explicit_year=True) + + +def _try_month_day(text: str, reference: date) -> ResolvedDate | None: + for match in _MONTH_DAY.finditer(text): + month, day = _month_number(match[1]), int(match[2]) + start_year = int(match[3]) if match[3] else None + end_month = _month_number(match[4]) if match[4] else None + end_day = int(match[5]) if match[5] else None + end_year = int(match[6]) if match[6] else None + + # "December 30 - January 2, 2026" states one year for a span that + # crosses New Year. It belongs to the start; the end rolls over. + wraps = end_month is not None and end_month < month + if start_year is not None: + year, effective_end_year = start_year, end_year + elif end_year is not None: + year = end_year + effective_end_year = end_year + 1 if wraps else end_year + else: + year, effective_end_year = _infer_year(month, day, reference), None + if year is None: + continue + + span = _day_range(month, day, end_month, end_day, year, effective_end_year) + if span: + explicit = start_year is not None or end_year is not None + return ResolvedDate(*span, DatePrecision.DAY, explicit) + return None + + +def _try_day_month(text: str, reference: date) -> ResolvedDate | None: + for match in _DAY_MONTH.finditer(text): + day = int(match[1]) + end_day = int(match[2]) if match[2] else None + month = _month_number(match[3]) + explicit_year = int(match[4]) if match[4] else None + year = explicit_year or _infer_year(month, day, reference) + if year is None: + continue + span = _day_range(month, day, None, end_day, year) + if span: + return ResolvedDate(*span, DatePrecision.DAY, explicit_year is not None) + return None + + +def _try_month_year(text: str, _reference: date) -> ResolvedDate | None: + match = _MONTH_YEAR.search(text) + if not match: + return None + start, end = _span_from_months(_month_number(match[1]), 1, int(match[2])) + return ResolvedDate(start, end, DatePrecision.MONTH, had_explicit_year=True) + + +def _try_season_year(text: str, _reference: date) -> ResolvedDate | None: + match = _SEASON_YEAR.search(text) + if not match: + return None + start_month, months = _SEASONS[match[1].lower()] + start, end = _span_from_months(start_month, months, int(match[2])) + return ResolvedDate(start, end, DatePrecision.QUARTER, had_explicit_year=True) + + +def _try_quarter_year(text: str, _reference: date) -> ResolvedDate | None: + match = _QUARTER_YEAR.search(text) + if not match: + return None + start, end = _span_from_months(int(match[1]) * 3 - 2, 3, int(match[2])) + return ResolvedDate(start, end, DatePrecision.QUARTER, had_explicit_year=True) + + +# Finest precision first: text carrying both "August 28, 2026" and "summer +# 2026" must resolve to the day, not the season. +_RESOLVERS = ( + _try_iso, + _try_month_day, + _try_day_month, + _try_month_year, + _try_season_year, + _try_quarter_year, +) + + +def resolve_date(text: str, reference: date) -> ResolvedDate | None: + """First date in `text` that could plausibly still be ahead of `reference`.""" + for resolver in _RESOLVERS: + resolved = resolver(text, reference) + if resolved and (resolved.end - reference).days >= -_PAST_TOLERANCE_DAYS: + return resolved + return None diff --git a/backend/foresight/services/features/__init__.py b/backend/foresight/services/features/__init__.py new file mode 100644 index 0000000..ae75e18 --- /dev/null +++ b/backend/foresight/services/features/__init__.py @@ -0,0 +1 @@ +"""Model features built from the canonical external-factor tables.""" diff --git a/backend/foresight/services/features/events.py b/backend/foresight/services/features/events.py new file mode 100644 index 0000000..451ba8b --- /dev/null +++ b/backend/foresight/services/features/events.py @@ -0,0 +1,125 @@ +"""Stored events as the per-night `expected_attendance` feature the models use.""" + +from __future__ import annotations + +import math +from datetime import timedelta +from typing import TYPE_CHECKING + +import pandas as pd +from sqlalchemy import select + +from foresight.database.models.external import Event +from foresight.ingest.normalize import DatePrecision, EventStatus, slugify_city + +if TYPE_CHECKING: + from collections.abc import Iterable + from datetime import date + + from sqlalchemy.ext.asyncio import AsyncSession + +FEATURE = "expected_attendance" + +# Withdrawn events must not keep inflating demand. +_EXCLUDED = {EventStatus.DELETED, EventStatus.CANCELLED} +_EARTH_RADIUS_KM = 6371.0 + + +def nightly_attendance( + events: Iterable[Event], + start: date, + end: date, + *, + latitude: float | None = None, + longitude: float | None = None, + radius_km: float | None = None, +) -> pd.Series: + """Total expected attendance per night, start..end inclusive. + + A multi-day event's attendance is spread evenly over its nights. Provisional + events count in proportion to their confidence. Events without a day-precise + date or an attendance estimate contribute nothing. + + With a hotel point and `radius_km`, events whose venue lies farther away are + dropped; events with no coordinates are kept, since ingest already + geo-filtered them. + """ + nights = pd.date_range(start, end, freq="D").date + totals = dict.fromkeys(nights, 0.0) + near = latitude is not None and longitude is not None and radius_km is not None + + for event in events: + if ( + event.status in _EXCLUDED + or event.date_precision != DatePrecision.DAY + or not event.expected_attendance + ): + continue + if ( + near + and event.latitude is not None + and _distance_km(latitude, longitude, event.latitude, event.longitude) + > radius_km + ): + continue + + span = (event.end_local_date - event.start_local_date).days + 1 + per_night = event.expected_attendance * event.confidence / span + for offset in range(span): + night = event.start_local_date + timedelta(days=offset) + if night in totals: + totals[night] += per_night + + return pd.Series( + [round(v) for v in totals.values()], + index=pd.Index(nights, name="stay_date"), + name=FEATURE, + dtype=int, + ) + + +async def load_events( + session: AsyncSession, city: str, start: date, end: date +) -> list[Event]: + """Events in `city` overlapping start..end.""" + result = await session.scalars( + select(Event).where( + Event.city_slug == slugify_city(city), + Event.start_local_date <= end, + Event.end_local_date >= start, + ) + ) + return list(result) + + +async def event_features( + session: AsyncSession, + *, + city: str, + start: date, + end: date, + latitude: float | None = None, + longitude: float | None = None, + radius_km: float | None = None, +) -> pd.Series: + """`nightly_attendance` for a market, read from the events table.""" + events = await load_events(session, city, start, end) + return nightly_attendance( + events, + start, + end, + latitude=latitude, + longitude=longitude, + radius_km=radius_km, + ) + + +def _distance_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Great-circle (haversine) distance.""" + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi, dlmb = phi2 - phi1, math.radians(lon2 - lon1) + a = ( + math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlmb / 2) ** 2 + ) + return 2 * _EARTH_RADIUS_KM * math.asin(math.sqrt(a)) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 14bcc7f..7c0fab8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "python-dotenv>=1.1", "pandas>=2.2", "psycopg2-binary>=2.9", + "httpx>=0.28", "scikit-learn>=1.9.1", ] diff --git a/backend/scripts/forecast_events.py b/backend/scripts/forecast_events.py new file mode 100644 index 0000000..053381a --- /dev/null +++ b/backend/scripts/forecast_events.py @@ -0,0 +1,123 @@ +"""Prices upcoming nights using real events from the events table. + +Run from backend/ after ingesting events: + uv run python scripts/forecast_events.py --city Dublin --days 21 + uv run python scripts/forecast_events.py --city Dublin \\ + --lat 53.3498 --lon -6.2603 --radius-km 5 --room-type deluxe + +The model is trained on mock history, and only expected_attendance is real. +Weather, competitor and flight features are mock placeholders until those +sources are ingested, simulated so competitors and flights respond to the real +events as they would in the mock world. "no events" reruns the same simulation +with zero attendance, so the effect column is the events' price impact alone. +""" + +import argparse +import asyncio +from datetime import date, timedelta + +import numpy as np +import pandas as pd +from compare_models import POST_STAY_COLUMNS, load_data +from generate_mock_data import ROOM_TYPES, generate + +from foresight.database.session import async_session_factory, engine +from foresight.ingest.normalize import EventStatus +from foresight.services.features.events import ( + FEATURE, + load_events, + nightly_attendance, +) +from foresight.services.ml.regression import GBMModel + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--city", required=True, help="market, as ingested") + parser.add_argument("--days", type=int, default=14, help="nights to price") + parser.add_argument("--start", type=date.fromisoformat, default=date.today()) + parser.add_argument("--lat", type=float, help="hotel latitude") + parser.add_argument("--lon", type=float, help="hotel longitude") + parser.add_argument("--radius-km", type=float, help="drop events farther away") + parser.add_argument("--room-type", choices=list(ROOM_TYPES), default="standard") + return parser + + +async def _events(args, start: date, end: date): + # DEBUG=true turns on SQL echo, which would bury the table. + engine.echo = False + try: + async with async_session_factory() as session: + return await load_events(session, args.city, start, end) + finally: + await engine.dispose() + + +def _nights(args, attendance: np.ndarray) -> pd.DataFrame: + """Placeholder nights for one room type, driven by `attendance`.""" + frame = generate( + start=args.start.isoformat(), days=args.days, expected_attendance=attendance + ) + return frame[frame["room_type"] == args.room_type].reset_index(drop=True) + + +def _features(frame: pd.DataFrame, columns: pd.Index) -> pd.DataFrame: + """Same columns, dummies and order the model was trained on.""" + x = frame.drop(columns=["rate", "stay_date", *POST_STAY_COLUMNS]) + x = pd.get_dummies(x, columns=["room_type", "condition"], dtype=float) + return x.reindex(columns=columns, fill_value=0.0) + + +def _biggest(events, night: date) -> str: + on = [ + e + for e in events + if e.start_local_date <= night <= e.end_local_date + and e.expected_attendance + and e.status not in {EventStatus.DELETED, EventStatus.CANCELLED} + ] + return max(on, key=lambda e: e.expected_attendance).title[:40] if on else "" + + +def main() -> None: + args = _parser().parse_args() + start = args.start + end = start + timedelta(days=args.days - 1) + + events = asyncio.run(_events(args, start, end)) + attendance = nightly_attendance( + events, + start, + end, + latitude=args.lat, + longitude=args.lon, + radius_km=args.radius_km, + ) + + x_train, y_train = load_data() + model = GBMModel(seed=9).fit(x_train, y_train) + + with_events = _nights(args, attendance.to_numpy()) + without = _nights(args, np.zeros(args.days)) + + rate, lo, hi = model.predict_interval(_features(with_events, x_train.columns)) + baseline = model.predict(_features(without, x_train.columns)) + + print( + f"{args.city} | {args.room_type} | {start} to {end} | " + f"{len(events)} stored events overlap" + ) + print( + f"{'night':<14}{'attendance':>11}{'no events':>11}{'with':>9}" + f"{'effect':>9}{'95% range':>17} biggest event" + ) + for i, night in enumerate(with_events["stay_date"]): + print( + f"{night:%a %Y-%m-%d}{with_events[FEATURE][i]:>11,}" + f"{baseline[i]:>11.2f}{rate[i]:>9.2f}{rate[i] - baseline[i]:>+9.2f}" + f"{f'{lo[i]:.0f}-{hi[i]:.0f}':>17} {_biggest(events, night)}" + ) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/generate_mock_data.py b/backend/scripts/generate_mock_data.py index 9cb382c..40adb17 100644 --- a/backend/scripts/generate_mock_data.py +++ b/backend/scripts/generate_mock_data.py @@ -27,13 +27,21 @@ COMPETITOR_REF = np.mean(list(COMPETITORS.values())) -def generate(seed: int = 9, start: str = "2024-01-01", days: int = 730) -> pd.DataFrame: +def generate( + seed: int = 9, + start: str = "2024-01-01", + days: int = 730, + expected_attendance: np.ndarray | None = None, +) -> pd.DataFrame: """Generates one row per room type per night. Args: seed (int): Random seed for reproducible data. start (str): First stay date, as YYYY-MM-DD. days (int): Number of nights to generate. + expected_attendance (np.ndarray | None): Per-night event attendance to + use instead of random events; the rest of the data responds to it. + Random draws are unchanged, so two calls differ only by its effect. Returns: pd.DataFrame: Mock nightly data, sorted by stay_date then room_type. @@ -56,7 +64,9 @@ def generate(seed: int = 9, start: str = "2024-01-01", days: int = 730) -> pd.Da # Events: total expected attendance of the day's events, ~10% of days. has_event = rng.random(days) < 0.10 - expected_attendance = np.where(has_event, rng.lognormal(9.5, 0.7, days), 0) + random_attendance = np.where(has_event, rng.lognormal(9.5, 0.7, days), 0) + if expected_attendance is None: + expected_attendance = random_attendance demand = np.clip( 0.55 diff --git a/backend/scripts/run_asknews.py b/backend/scripts/run_asknews.py new file mode 100644 index 0000000..479db40 --- /dev/null +++ b/backend/scripts/run_asknews.py @@ -0,0 +1,194 @@ +"""Run the AskNews event ingest and print a cleaning report per run. + +Two modes: + + uv run python scripts/run_asknews.py # replay recorded batches + uv run python scripts/run_asknews.py --live # call the real API + +Replay is the default so the pipeline can be demonstrated and regression-checked +without credentials or network. The extraction path is identical in both modes; +only where the articles come from differs. + +Events are written through the shared upsert, so re-polling an overlapping +window visibly produces no duplicate rows. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from collections import Counter +from datetime import date, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, select + +from foresight.config import settings +from foresight.database.models.external import Event, EventObservation +from foresight.database.session import async_session_factory, engine +from foresight.ingest.asknews import ( + AskNewsClient, + CityTarget, + IngestReport, + collect_events, + extract_batch, +) +from foresight.ingest.repository import Outcome, upsert_event + +if TYPE_CHECKING: + from collections.abc import Sequence + + from foresight.ingest.normalize import NormalizedEvent + +FIXTURE_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "asknews" + +# The cities we sell rooms in. Everything else AskNews returns is noise. +CITIES = ( + CityTarget("Dublin", "Europe/Dublin", "IE", aliases=("Baile Atha Cliath",)), + CityTarget("Lisbon", "Europe/Lisbon", "PT", aliases=("Lisboa",)), + CityTarget("Edinburgh", "Europe/London", "GB"), +) + + +async def persist(events: Sequence[NormalizedEvent]) -> Counter[Outcome]: + """Write a run through the shared upsert, one transaction per run.""" + counts: Counter[Outcome] = Counter() + async with async_session_factory() as session: + for event in events: + counts[await upsert_event(session, event)] += 1 + await session.commit() + return counts + + +async def stored_events() -> tuple[list[Event], int]: + async with async_session_factory() as session: + rows = await session.scalars( + select(Event).order_by(Event.start_local_date, Event.title) + ) + observations = await session.scalar( + select(func.count()).select_from(EventObservation) + ) + return list(rows), observations or 0 + + +def load_fixture(path: Path) -> list[dict[str, Any]]: + return json.loads(path.read_text()) + + +async def run_live( + today: date, hours_back: int, per_query: int +) -> tuple[list[NormalizedEvent], IngestReport]: + """Exactly the call a scheduled worker makes. The CLI adds nothing to it.""" + import httpx + + if not (settings.asknews_client_id and settings.asknews_client_secret): + raise SystemExit( + "ASKNEWS_CLIENT_ID and ASKNEWS_CLIENT_SECRET must be set for --live" + ) + async with httpx.AsyncClient(timeout=30.0) as http: + client = AskNewsClient( + settings.asknews_client_id, settings.asknews_client_secret, http=http + ) + return await collect_events( + client, + CITIES, + today=today, + articles_per_query=per_query, + hours_back=hours_back, + ) + + +def describe(event: NormalizedEvent | Event) -> str: + """Same fields on the extracted object and the stored row.""" + return ( + f" {event.start_local_date} .. {event.end_local_date} " + f"{event.city_slug:<10} {event.date_precision:<8} " + f"conf {event.confidence:.2f} {event.title}" + ) + + +def print_run( + label: str, + source: str, + report: IngestReport, + events: Sequence[NormalizedEvent], + counts: Counter[Outcome] | None, +) -> None: + print(f"\n{'=' * 68}\n{label} -- {source}\n{'=' * 68}") + print(report.render()) + if counts is None: + print("extracted (nothing written)") + for event in events: + print(describe(event)) + return + print("written to the events table") + for outcome in Outcome: + print(f" {outcome.value:<22} {counts[outcome]}") + + +async def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--live", action="store_true", help="call AskNews instead of replaying" + ) + parser.add_argument("--runs", type=int, default=3) + parser.add_argument( + "--dry-run", + action="store_true", + help="print what was extracted instead of writing to the database", + ) + parser.add_argument("--hours-back", type=int, default=24) + parser.add_argument("--articles-per-query", type=int, default=50) + parser.add_argument( + "--today", + type=date.fromisoformat, + default=datetime.now().date(), + help="reference date for 'is this event still ahead of us'", + ) + args = parser.parse_args() + + totals = IngestReport() + + for index in range(1, args.runs + 1): + if args.live: + events, report = await run_live( + args.today, args.hours_back, args.articles_per_query + ) + source = f"live, last {args.hours_back}h" + else: + path = FIXTURE_DIR / f"run_{index}.json" + if not path.exists(): + print(f"no fixture for run {index} at {path}") + break + events, report = extract_batch(load_fixture(path), CITIES, today=args.today) + report.queries_run = 1 + source = f"replay {path.name}" + + counts = None if args.dry_run else await persist(events) + print_run(f"RUN {index}", source, report, events, counts) + + totals.articles_seen += report.articles_seen + totals.events_kept += report.events_kept + totals.rejected.update(report.rejected) + + print(f"\n{'=' * 68}\nACROSS ALL RUNS\n{'=' * 68}") + print(f"articles processed {totals.articles_seen}") + print(f"articles rejected {totals.articles_rejected}") + for reason, count in totals.rejected.most_common(): + print(f" {reason:<22} {count}") + if args.dry_run: + return 0 + + rows, observations = await stored_events() + print(f"rows in events {len(rows)}") + print(f"rows in observations {observations}") + for event in rows: + print(describe(event)) + await engine.dispose() + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/backend/tests/fixtures/asknews/run_1.json b/backend/tests/fixtures/asknews/run_1.json new file mode 100644 index 0000000..60f6d9d --- /dev/null +++ b/backend/tests/fixtures/asknews/run_1.json @@ -0,0 +1,122 @@ +[ + { + "article_url": "https://rte.example/forbidden-fruit-2027", + "eng_title": "Forbidden Fruit announces 2027 return to Kilmainham", + "summary": "Organisers have confirmed the festival will return to the Royal Hospital Kilmainham in Dublin on June 5-7, 2027. Tickets go on sale Friday at 9am.", + "key_points": ["Three-day festival", "Tickets on sale Friday"], + "pub_date": "2026-10-05T08:15:00+00:00", + "entities": {"Event": ["Forbidden Fruit"], "Location": ["Dublin", "Ireland"], "Organization": ["Royal Hospital Kilmainham"]}, + "country": "IE", + "page_rank": 2, + "language": "en", + "classification": ["Entertainment"] + }, + { + "article_url": "https://publico.example/web-summit-2026", + "eng_title": "Web Summit returns to Lisbon in November", + "summary": "Web Summit will be held at the Altice Arena in Lisbon from November 10-13, 2026, with organisers expecting another sell-out edition.", + "key_points": ["Altice Arena", "Four-day conference"], + "pub_date": "2026-10-04T11:00:00+00:00", + "entities": {"Event": ["Web Summit"], "Location": ["Lisbon", "Portugal"]}, + "country": "PT", + "page_rank": 3, + "language": "en", + "classification": ["Technology"] + }, + { + "article_url": "https://scotsman.example/hogmanay-2026", + "eng_title": "Edinburgh Hogmanay street party lineup revealed", + "summary": "Edinburgh's Hogmanay celebrations will run December 30, 2026 - January 1, 2027, with the street party returning to Princes Street.", + "key_points": ["Princes Street party", "Three nights"], + "pub_date": "2026-10-03T16:40:00+00:00", + "entities": {"Event": ["Edinburgh Hogmanay"], "Location": ["Edinburgh", "Scotland"]}, + "country": "GB", + "page_rank": 2, + "language": "en", + "classification": ["Culture"] + }, + { + "article_url": "https://guardian.example/glastonbury-2027", + "eng_title": "Glastonbury confirms 2027 dates", + "summary": "Glastonbury Festival will take over Worthy Farm in Somerset on June 23-27, 2027, organisers announced.", + "key_points": ["Worthy Farm"], + "pub_date": "2026-10-05T07:00:00+00:00", + "entities": {"Event": ["Glastonbury Festival"], "Location": ["Somerset", "England"]}, + "country": "GB", + "page_rank": 1, + "language": "en", + "classification": ["Entertainment"] + }, + { + "article_url": "https://irishtimes.example/theatre-festival-plans", + "eng_title": "Dublin Theatre Festival plans expanded programme", + "summary": "The Dublin Theatre Festival will expand its programme next edition, though organisers say dates have not yet been confirmed.", + "key_points": ["Dates to be confirmed"], + "pub_date": "2026-10-02T09:20:00+00:00", + "entities": {"Event": ["Dublin Theatre Festival"], "Location": ["Dublin"]}, + "country": "IE", + "page_rank": 3, + "language": "en", + "classification": ["Culture"] + }, + { + "article_url": "https://observador.example/lisbon-marathon-recap", + "eng_title": "Lisbon half marathon draws record field", + "summary": "The Lisbon half marathon took place on Sunday and drew a record crowd of runners across the Vasco da Gama bridge.", + "key_points": ["Record field"], + "pub_date": "2026-10-05T18:30:00+00:00", + "entities": {"Event": ["Lisbon Half Marathon"], "Location": ["Lisbon"]}, + "country": "PT", + "page_rank": 4, + "language": "en", + "classification": ["Sports"] + }, + { + "article_url": "https://siliconrepublic.example/dublin-tech-summit", + "eng_title": "Dublin Tech Summit eyes spring slot", + "summary": "Dublin Tech Summit is scheduled for spring 2027 at the Convention Centre Dublin, organisers confirmed, with exact dates to follow.", + "key_points": ["Convention Centre Dublin"], + "pub_date": "2026-10-01T13:05:00+00:00", + "entities": {"Event": ["Dublin Tech Summit"], "Location": ["Dublin"]}, + "country": "IE", + "page_rank": 5, + "language": "en", + "classification": ["Technology"] + }, + { + "article_url": "", + "eng_title": "Untitled wire item", + "summary": "Dublin concert announced for November 20, 2026.", + "key_points": [], + "pub_date": "2026-10-05T12:00:00+00:00", + "entities": {"Location": ["Dublin"]}, + "country": "IE", + "page_rank": 9, + "language": "en", + "classification": ["Entertainment"] + }, + { + "article_url": "https://hotpress.example/stadium-2031", + "eng_title": "Dublin stadium pencils in 2031 showcase", + "summary": "A showcase concert at the Aviva Stadium in Dublin is scheduled for August 28, 2031 under the venue's long-range plan.", + "key_points": ["Long-range plan"], + "pub_date": "2026-10-04T10:00:00+00:00", + "entities": {"Event": ["Aviva Showcase"], "Location": ["Dublin"]}, + "country": "IE", + "page_rank": 7, + "language": "en", + "classification": ["Entertainment"] + }, + { + "article_url": "https://heraldscotland.example/edinburgh-fringe-2027", + "eng_title": "Edinburgh Festival Fringe sets August window", + "summary": "The Edinburgh Festival Fringe will run through August 2027, with the full programme due in the new year.", + "key_points": ["Programme due in the new year"], + "pub_date": "2026-10-02T15:00:00+00:00", + "entities": {"Event": ["Edinburgh Festival Fringe"], "Location": ["Edinburgh"]}, + "country": "GB", + "page_rank": 2, + "language": "en", + "classification": ["Culture"] + } +] diff --git a/backend/tests/fixtures/asknews/run_2.json b/backend/tests/fixtures/asknews/run_2.json new file mode 100644 index 0000000..8f15a99 --- /dev/null +++ b/backend/tests/fixtures/asknews/run_2.json @@ -0,0 +1,110 @@ +[ + { + "article_url": "https://rte.example/forbidden-fruit-2027", + "eng_title": "Forbidden Fruit announces 2027 return to Kilmainham", + "summary": "Organisers have confirmed the festival will return to the Royal Hospital Kilmainham in Dublin on June 5-7, 2027. Tickets go on sale Friday at 9am.", + "key_points": ["Three-day festival", "Tickets on sale Friday"], + "pub_date": "2026-10-05T08:15:00+00:00", + "entities": {"Event": ["Forbidden Fruit"], "Location": ["Dublin", "Ireland"], "Organization": ["Royal Hospital Kilmainham"]}, + "country": "IE", + "page_rank": 2, + "language": "en", + "classification": ["Entertainment"] + }, + { + "article_url": "https://publico.example/web-summit-2026", + "eng_title": "Web Summit returns to Lisbon in November", + "summary": "Web Summit will be held at the Altice Arena in Lisbon from November 10-13, 2026, with organisers expecting another sell-out edition.", + "key_points": ["Altice Arena", "Four-day conference"], + "pub_date": "2026-10-04T11:00:00+00:00", + "entities": {"Event": ["Web Summit"], "Location": ["Lisbon", "Portugal"]}, + "country": "PT", + "page_rank": 3, + "language": "en", + "classification": ["Technology"] + }, + { + "article_url": "https://techcrunch.example/web-summit-lisbon-preview", + "eng_title": "What to expect from this year's Lisbon tech gathering", + "summary": "Running November 10-13, 2026 in Lisbon, the conference will host more than 70,000 attendees at the Altice Arena.", + "key_points": ["70,000 attendees expected"], + "pub_date": "2026-10-06T06:45:00+00:00", + "entities": {"Event": ["Web Summit"], "Location": ["Lisbon"]}, + "country": "PT", + "page_rank": 2, + "language": "en", + "classification": ["Technology"] + }, + { + "article_url": "https://scotsman.example/hogmanay-2026", + "eng_title": "Edinburgh Hogmanay street party lineup revealed", + "summary": "Edinburgh's Hogmanay celebrations will run December 30, 2026 - January 1, 2027, with the street party returning to Princes Street.", + "key_points": ["Princes Street party", "Three nights"], + "pub_date": "2026-10-03T16:40:00+00:00", + "entities": {"Event": ["Edinburgh Hogmanay"], "Location": ["Edinburgh", "Scotland"]}, + "country": "GB", + "page_rank": 2, + "language": "en", + "classification": ["Culture"] + }, + { + "article_url": "https://independent.example/dublin-marathon-2026", + "eng_title": "Dublin Marathon route confirmed for October", + "summary": "The Dublin Marathon is scheduled for October 25, 2026, with organisers confirming a revised route through the city centre.", + "key_points": ["Revised city centre route"], + "pub_date": "2026-10-06T07:30:00+00:00", + "entities": {"Event": ["Dublin Marathon"], "Location": ["Dublin"]}, + "country": "IE", + "page_rank": 3, + "language": "en", + "classification": ["Sports"] + }, + { + "article_url": "https://timeout.example/lisbon-christmas-market", + "eng_title": "Lisbon Christmas market dates set", + "summary": "The Lisbon Christmas market will open December 1, 2026 and run through January 4, 2027 in Praca do Comercio.", + "key_points": ["Praca do Comercio"], + "pub_date": "2026-10-06T09:10:00+00:00", + "entities": {"Event": ["Lisbon Christmas Market"], "Location": ["Lisbon"]}, + "country": "PT", + "page_rank": 5, + "language": "en", + "classification": ["Culture"] + }, + { + "article_url": "https://bbc.example/manchester-gig", + "eng_title": "Arena tour adds Manchester night", + "summary": "The tour will add a Manchester date on March 14, 2027 at the Co-op Live arena.", + "key_points": [], + "pub_date": "2026-10-06T08:00:00+00:00", + "entities": {"Event": ["Arena Tour"], "Location": ["Manchester"]}, + "country": "GB", + "page_rank": 2, + "language": "en", + "classification": ["Entertainment"] + }, + { + "article_url": "https://thejournal.example/dublin-protest", + "eng_title": "Dublin city centre march planned", + "summary": "A march through Dublin city centre is being organised, with a date expected to be announced in the coming weeks.", + "key_points": ["Date expected soon"], + "pub_date": "2026-10-06T10:00:00+00:00", + "entities": {"Location": ["Dublin"]}, + "country": "IE", + "page_rank": 6, + "language": "en", + "classification": ["Politics"] + }, + { + "article_url": "https://scotsman.example/fringe-recap-2026", + "eng_title": "Edinburgh Fringe 2026 in numbers", + "summary": "This summer's Edinburgh Fringe concluded with record attendance. The festival was held across 26 days in August.", + "key_points": ["Record attendance"], + "pub_date": "2026-09-30T12:00:00+00:00", + "entities": {"Event": ["Edinburgh Festival Fringe"], "Location": ["Edinburgh"]}, + "country": "GB", + "page_rank": 3, + "language": "en", + "classification": ["Culture"] + } +] diff --git a/backend/tests/fixtures/asknews/run_3.json b/backend/tests/fixtures/asknews/run_3.json new file mode 100644 index 0000000..f7c2bd7 --- /dev/null +++ b/backend/tests/fixtures/asknews/run_3.json @@ -0,0 +1,110 @@ +[ + { + "article_url": "https://rte.example/forbidden-fruit-2027", + "eng_title": "Forbidden Fruit moves 2027 edition back a week", + "summary": "Organisers have revised the dates: the festival will now return to the Royal Hospital Kilmainham in Dublin on June 11-13, 2027. Existing tickets remain valid.", + "key_points": ["Dates moved back a week", "Tickets remain valid"], + "pub_date": "2026-10-06T14:20:00+00:00", + "entities": {"Event": ["Forbidden Fruit"], "Location": ["Dublin", "Ireland"], "Organization": ["Royal Hospital Kilmainham"]}, + "country": "IE", + "page_rank": 2, + "language": "en", + "classification": ["Entertainment"] + }, + { + "article_url": "https://publico.example/web-summit-2026", + "eng_title": "Web Summit returns to Lisbon in November", + "summary": "Web Summit will be held at the Altice Arena in Lisbon from November 10-13, 2026, with organisers expecting another sell-out edition.", + "key_points": ["Altice Arena", "Four-day conference"], + "pub_date": "2026-10-04T11:00:00+00:00", + "entities": {"Event": ["Web Summit"], "Location": ["Lisbon", "Portugal"]}, + "country": "PT", + "page_rank": 3, + "language": "en", + "classification": ["Technology"] + }, + { + "article_url": "https://independent.example/dublin-marathon-2026", + "eng_title": "Dublin Marathon route confirmed for October", + "summary": "The Dublin Marathon is scheduled for October 25, 2026, with organisers confirming a revised route through the city centre.", + "key_points": ["Revised city centre route"], + "pub_date": "2026-10-06T07:30:00+00:00", + "entities": {"Event": ["Dublin Marathon"], "Location": ["Dublin"]}, + "country": "IE", + "page_rank": 3, + "language": "en", + "classification": ["Sports"] + }, + { + "article_url": "https://rte.example/tradfest-2027", + "eng_title": "TradFest announces January dates for Temple Bar", + "summary": "TradFest will run January 20-24, 2027 across venues in Temple Bar, Dublin, organisers confirmed.", + "key_points": ["Temple Bar venues"], + "pub_date": "2026-10-06T11:00:00+00:00", + "entities": {"Event": ["TradFest"], "Location": ["Dublin"]}, + "country": "IE", + "page_rank": 4, + "language": "en", + "classification": ["Culture"] + }, + { + "article_url": "https://irishtimes.example/tradfest-temple-bar", + "eng_title": "Temple Bar winter music festival confirms programme", + "summary": "The festival is scheduled for January 20-24, 2027 in Dublin, with more than 100 performances announced.", + "key_points": ["100 performances"], + "pub_date": "2026-10-06T12:30:00+00:00", + "entities": {"Event": ["TradFest"], "Location": ["Dublin"]}, + "country": "IE", + "page_rank": 3, + "language": "en", + "classification": ["Culture"] + }, + { + "article_url": "https://publico.example/porto-sao-joao", + "eng_title": "Porto sets Sao Joao festival date", + "summary": "Porto's Sao Joao festival will take over the city on June 23, 2027, with fireworks over the Douro.", + "key_points": ["Fireworks over the Douro"], + "pub_date": "2026-10-06T13:00:00+00:00", + "entities": {"Event": ["Sao Joao"], "Location": ["Porto", "Portugal"]}, + "country": "PT", + "page_rank": 4, + "language": "en", + "classification": ["Culture"] + }, + { + "article_url": "https://heraldscotland.example/edinburgh-summit-q2", + "eng_title": "Edinburgh to host energy summit", + "summary": "An international energy summit is scheduled for Q2 2027 in Edinburgh, the organisers said, with the venue still to be confirmed.", + "key_points": ["Venue to be confirmed"], + "pub_date": "2026-10-06T15:45:00+00:00", + "entities": {"Event": ["International Energy Summit"], "Location": ["Edinburgh"]}, + "country": "GB", + "page_rank": 5, + "language": "en", + "classification": ["Business"] + }, + { + "article_url": "https://thejournal.example/weekend-listings", + "eng_title": "What's on in Dublin this weekend", + "summary": "A roundup of gigs and markets happening in Dublin this weekend, including a craft fair and two late-night sessions.", + "key_points": ["Craft fair", "Late-night sessions"], + "pub_date": "2026-10-06T16:00:00+00:00", + "entities": {"Location": ["Dublin"]}, + "country": "IE", + "page_rank": 8, + "language": "en", + "classification": ["Culture"] + }, + { + "article_url": "https://observador.example/lisbon-council-budget", + "eng_title": "Lisbon council approves 2027 budget", + "summary": "Lisbon's council approved its 2027 budget on Tuesday, allocating funds across transport and housing.", + "key_points": ["Transport and housing"], + "pub_date": "2026-10-06T17:10:00+00:00", + "entities": {"Location": ["Lisbon"], "Organization": ["Lisbon City Council"]}, + "country": "PT", + "page_rank": 4, + "language": "en", + "classification": ["Politics"] + } +] diff --git a/backend/tests/test_asknews.py b/backend/tests/test_asknews.py new file mode 100644 index 0000000..a3e53f3 --- /dev/null +++ b/backend/tests/test_asknews.py @@ -0,0 +1,449 @@ +"""AskNews adapter: what survives extraction, what gets dropped, and the client.""" + +from __future__ import annotations + +from datetime import date +from typing import Any + +import httpx +import pytest + +from foresight.ingest import asknews +from foresight.ingest.asknews import ( + AskNewsClient, + CityTarget, + RejectReason, + collect_events, + extract_batch, + extract_event, + plan_searches, +) +from foresight.ingest.normalize import ( + DatePrecision, + EventCategory, + EventStatus, + NormalizedEvent, + Source, +) + +TODAY = date(2026, 6, 1) +DUBLIN = CityTarget( + name="Dublin", + timezone="Europe/Dublin", + country="IE", + aliases=("Baile Atha Cliath",), +) +LISBON = CityTarget(name="Lisbon", timezone="Europe/Lisbon", country="PT") +CITIES = (DUBLIN, LISBON) + + +def article(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "article_url": "https://news.example/forbidden-fruit", + "eng_title": "Forbidden Fruit announces August dates for Dublin", + "summary": ( + "Organisers confirmed the festival will return to the Royal Hospital " + "Kilmainham in Dublin on August 28-30, 2026, with tickets on sale Friday." + ), + "key_points": ["Tickets go on sale Friday"], + "pub_date": "2026-05-20T09:30:00+00:00", + "entities": {"Event": ["Forbidden Fruit"], "Location": ["Dublin", "Ireland"]}, + "country": "IE", + "page_rank": 3, + "language": "en", + } + return base | overrides + + +def test_a_clean_announcement_becomes_a_provisional_event() -> None: + event = extract_event(article(), CITIES, today=TODAY) + + assert isinstance(event, NormalizedEvent) + assert event.source is Source.ASKNEWS + assert event.source_ref == "https://news.example/forbidden-fruit" + assert event.title == "Forbidden Fruit" + assert event.city == "Dublin" + assert event.start_local_date == date(2026, 8, 28) + assert event.end_local_date == date(2026, 8, 30) + assert event.category is EventCategory.FESTIVALS + assert event.date_precision is DatePrecision.DAY + assert event.status is EventStatus.PROVISIONAL + + +def test_timezone_comes_from_the_city_not_the_article() -> None: + event = extract_event(article(), CITIES, today=TODAY) + + assert isinstance(event, NormalizedEvent) + assert event.timezone == "Europe/Dublin" + assert event.country == "IE" + + +def test_extraction_is_never_fully_confident() -> None: + event = extract_event(article(), CITIES, today=TODAY) + + assert isinstance(event, NormalizedEvent) + assert 0.0 < event.confidence <= 0.95 + + +def test_an_event_entity_names_the_event_rather_than_the_headline() -> None: + headline_only = article(entities={"Location": ["Dublin"]}) + + titled = extract_event(article(), CITIES, today=TODAY) + untitled = extract_event(headline_only, CITIES, today=TODAY) + + assert isinstance(titled, NormalizedEvent) + assert isinstance(untitled, NormalizedEvent) + assert titled.title == "Forbidden Fruit" + assert untitled.title.startswith("Forbidden Fruit announces") + # The entity hit is the stronger signal, so it must score higher. + assert titled.confidence > untitled.confidence + + +def test_a_location_entity_outscores_a_bare_mention_in_the_body() -> None: + weak = article(entities={"Event": ["Forbidden Fruit"]}) + + strong_event = extract_event(article(), CITIES, today=TODAY) + weak_event = extract_event(weak, CITIES, today=TODAY) + + assert isinstance(strong_event, NormalizedEvent) + assert isinstance(weak_event, NormalizedEvent) + assert strong_event.confidence > weak_event.confidence + + +def test_an_alias_resolves_to_the_same_city() -> None: + aliased = article( + entities={"Event": ["Forbidden Fruit"], "Location": ["Baile Atha Cliath"]}, + summary="Returns on August 28-30, 2026.", + eng_title="Festival returns", + ) + + event = extract_event(aliased, CITIES, today=TODAY) + + assert isinstance(event, NormalizedEvent) + assert event.city_slug == "dublin" + + +@pytest.mark.parametrize( + ("overrides", "reason"), + [ + ( + { + "eng_title": "Glastonbury announces dates", + "summary": "The festival runs August 28-30, 2026 in Somerset.", + "entities": {"Event": ["Glastonbury"], "Location": ["Somerset"]}, + }, + RejectReason.NO_CITY_MATCH, + ), + ( + { + "summary": "A Dublin festival will return, dates to be confirmed.", + "eng_title": "Dublin festival to return", + "entities": {"Location": ["Dublin"]}, + }, + RejectReason.NO_RESOLVABLE_DATE, + ), + ( + { + "summary": "The Dublin festival was held on August 28, 2025.", + "eng_title": "Dublin festival review", + "key_points": [], + "entities": {"Location": ["Dublin"]}, + }, + RejectReason.RETROSPECTIVE, + ), + ( + { + "summary": "The Dublin festival is scheduled for August 28, 2035.", + "eng_title": "Dublin festival long-range plan", + "entities": {"Location": ["Dublin"]}, + }, + RejectReason.BEYOND_HORIZON, + ), + ({"article_url": ""}, RejectReason.MALFORMED_RECORD), + ( + {"eng_title": "", "title": "", "summary": "", "key_points": []}, + RejectReason.MALFORMED_RECORD, + ), + ], +) +def test_unusable_articles_are_rejected_with_a_reason( + overrides: dict[str, Any], reason: RejectReason +) -> None: + assert extract_event(article(**overrides), CITIES, today=TODAY) is reason + + +def test_an_event_that_already_finished_is_rejected_as_past() -> None: + stale = article( + pub_date="2026-01-10T09:00:00+00:00", + eng_title="Dublin parade set for March 17", + summary="The Dublin parade is scheduled for March 17, 2026.", + entities={"Location": ["Dublin"]}, + ) + + assert extract_event(stale, CITIES, today=TODAY) is RejectReason.EVENT_ALREADY_PAST + + +def test_an_event_running_across_today_is_still_kept() -> None: + ongoing = article( + pub_date="2026-05-25T09:00:00+00:00", + eng_title="Dublin festival under way", + summary="The Dublin festival is scheduled for May 30 - June 3, 2026.", + entities={"Location": ["Dublin"]}, + ) + + event = extract_event(ongoing, CITIES, today=TODAY) + + assert isinstance(event, NormalizedEvent) + assert event.start_local_date < TODAY <= event.end_local_date + + +def test_the_same_event_from_two_outlets_collapses_to_one_row() -> None: + wire_copy = article( + article_url="https://other.example/ff", + eng_title="Dublin festival confirms summer return", + entities={"Event": ["Forbidden Fruit"], "Location": ["Dublin"]}, + ) + + events, report = extract_batch([article(), wire_copy], CITIES, today=TODAY) + + assert len(events) == 1 + assert report.articles_seen == 2 + assert report.events_kept == 1 + assert report.rejected[RejectReason.DUPLICATE_IN_BATCH.value] == 1 + + +def test_the_higher_confidence_sighting_wins_a_collapse() -> None: + weak = article( + article_url="https://other.example/ff", + entities={"Event": ["Forbidden Fruit"]}, + ) + + events, _ = extract_batch([weak, article()], CITIES, today=TODAY) + + assert len(events) == 1 + assert events[0].source_ref == "https://news.example/forbidden-fruit" + + +def test_two_outlets_do_not_collapse_without_an_event_entity() -> None: + """A known limit: headlines differ, so the dedupe key differs. + + AskNews supplies an Event entity for most announcements. Where it does + not, the same event arrives twice and only PredictHQ corroboration or a + later title match will merge it. + """ + wire_copy = article( + article_url="https://other.example/ff", + eng_title="Forbidden Fruit returns to Dublin", + entities={"Location": ["Dublin"]}, + ) + + events, _ = extract_batch([article(), wire_copy], CITIES, today=TODAY) + + assert len(events) == 2 + + +def test_the_report_tallies_everything_that_came_in() -> None: + batch = [ + article(), + article( + article_url="https://x.example/1", summary="No date here.", key_points=[] + ), + article( + article_url="https://x.example/2", + entities={"Location": ["Somerset"]}, + eng_title="Somerset show", + summary="Runs August 28-30, 2026 in Somerset.", + ), + ] + + events, report = extract_batch(batch, CITIES, today=TODAY) + + assert report.articles_seen == 3 + assert report.events_kept == len(events) == 1 + assert report.articles_rejected == 2 + assert report.by_city["dublin"] == 1 + assert report.by_precision["day"] == 1 + assert "articles seen" in report.render() + + +def test_events_are_returned_in_date_order() -> None: + later = article( + article_url="https://x.example/later", + eng_title="Lisbon summit", + summary="The Lisbon summit is scheduled for October 5, 2026.", + entities={"Event": ["Lisbon Summit"], "Location": ["Lisbon"]}, + ) + + events, _ = extract_batch([later, article()], CITIES, today=TODAY) + + assert [e.start_local_date for e in events] == sorted( + e.start_local_date for e in events + ) + + +# --- client --------------------------------------------------------------- + + +def build_client(handler: Any) -> tuple[AskNewsClient, dict[str, int]]: + calls: dict[str, int] = {"token": 0, "search": 0} + + def route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/token"): + calls["token"] += 1 + else: + calls["search"] += 1 + return handler(request, calls) + + http = httpx.AsyncClient(transport=httpx.MockTransport(route)) + return AskNewsClient("id", "secret", http=http), calls + + +def ok(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response: + if request.url.path.endswith("/token"): + return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600}) + return httpx.Response(200, json={"as_dicts": [article()], "offset": 0}) + + +async def test_search_authenticates_then_returns_articles() -> None: + client, calls = build_client(ok) + + articles = await client.search_news(query="festival", n_articles=5) + + assert calls == {"token": 1, "search": 1} + assert articles[0]["article_url"] == "https://news.example/forbidden-fruit" + + +async def test_the_token_is_reused_across_searches() -> None: + client, calls = build_client(ok) + + await client.search_news(query="a") + await client.search_news(query="b") + + assert calls["token"] == 1 + assert calls["search"] == 2 + + +async def test_the_bearer_token_is_sent_on_the_search() -> None: + seen: list[str] = [] + + def capture(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response: + if request.url.path.endswith("/token"): + return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600}) + seen.append(request.headers.get("authorization", "")) + return httpx.Response(200, json={"as_dicts": []}) + + client, _ = build_client(capture) + await client.search_news(query="a") + + assert seen == ["Bearer tok"] + + +async def test_a_rate_limited_search_is_retried( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def no_sleep(_seconds: float) -> None: + return None + + monkeypatch.setattr(asknews.asyncio, "sleep", no_sleep) + + def throttle(request: httpx.Request, calls: dict[str, int]) -> httpx.Response: + if request.url.path.endswith("/token"): + return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600}) + if calls["search"] == 1: + return httpx.Response(429, headers={"retry-after": "1"}, json={}) + return httpx.Response(200, json={"as_dicts": [article()]}) + + client, calls = build_client(throttle) + articles = await client.search_news(query="festival") + + assert calls["search"] == 2 + assert len(articles) == 1 + + +async def test_a_persistent_server_error_is_raised() -> None: + async def no_sleep(_seconds: float) -> None: + return None + + def broken(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response: + if request.url.path.endswith("/token"): + return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600}) + return httpx.Response(503, json={}) + + client, _ = build_client(broken) + asknews.asyncio.sleep = no_sleep # type: ignore[assignment] + + with pytest.raises(httpx.HTTPStatusError): + await client.search_news(query="festival") + + +async def test_a_token_response_without_a_token_fails_loudly() -> None: + def tokenless(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response: + return httpx.Response(200, json={"detail": "nope"}) + + client, _ = build_client(tokenless) + + with pytest.raises(ValueError, match="access_token"): + await client.search_news(query="festival") + + +async def test_collect_events_is_one_call_a_scheduled_worker_can_make() -> None: + """The cron path end to end: fetch every query, extract one clean batch.""" + queries_seen: list[str] = [] + hours_seen: list[str] = [] + + def handler(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response: + if request.url.path.endswith("/token"): + return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600}) + queries_seen.append(request.url.params.get("query", "")) + hours_seen.append(request.url.params.get("hours_back", "")) + return httpx.Response(200, json={"as_dicts": [article()]}) + + client, calls = build_client(handler) + events, report = await collect_events(client, CITIES, today=TODAY, hours_back=6) + + assert calls["search"] == len(CITIES) + assert [q.split()[0] for q in queries_seen] == ["Dublin", "Lisbon"] + assert hours_seen == ["6", "6"] + assert report.queries_run == len(CITIES) + assert report.articles_seen == 2 + # The same article from both queries is one event, not two. + assert len(events) == 1 + assert report.rejected[RejectReason.DUPLICATE_IN_BATCH.value] == 1 + + +def test_each_city_gets_its_own_scoped_search() -> None: + plans = plan_searches(CITIES) + + assert [p.countries for p in plans] == [("IE",), ("PT",)] + assert all(p.query.startswith(c.name) for p, c in zip(plans, CITIES, strict=True)) + assert all("festival" in p.query for p in plans) + + +def test_a_city_without_a_country_is_not_country_filtered() -> None: + plans = plan_searches((CityTarget("Dublin", "Europe/Dublin"),)) + + assert plans[0].countries == () + + +async def test_collect_events_sends_one_country_scoped_search_per_city() -> None: + sent: list[tuple[str, list[str]]] = [] + + def handler(request: httpx.Request, _calls: dict[str, int]) -> httpx.Response: + if request.url.path.endswith("/token"): + return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600}) + sent.append( + ( + request.url.params.get("query", ""), + request.url.params.get_list("countries"), + ) + ) + return httpx.Response(200, json={"as_dicts": []}) + + client, calls = build_client(handler) + _, report = await collect_events(client, CITIES, today=TODAY) + + assert calls["search"] == len(CITIES) + assert report.queries_run == len(CITIES) + assert [countries for _, countries in sent] == [["IE"], ["PT"]] + assert sent[0][0].startswith("Dublin") + assert sent[1][0].startswith("Lisbon") diff --git a/backend/tests/test_event_features.py b/backend/tests/test_event_features.py new file mode 100644 index 0000000..baf13ca --- /dev/null +++ b/backend/tests/test_event_features.py @@ -0,0 +1,168 @@ +"""Stored events turned into the per-night expected_attendance feature.""" + +from __future__ import annotations + +from datetime import date +from typing import TYPE_CHECKING + +import numpy as np +import pytest +from generate_mock_data import generate +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine + +from foresight.config import settings +from foresight.database.models.external import Event +from foresight.ingest.normalize import ( + DatePrecision, + EventStatus, + NormalizedEvent, + Source, +) +from foresight.ingest.repository import upsert_event +from foresight.services.features.events import event_features, nightly_attendance + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + +START, END = date(2026, 11, 13), date(2026, 11, 16) +DUBLIN = (53.3498, -6.2603) + + +def _event(**overrides: object) -> Event: + base: dict[str, object] = { + "start_local_date": date(2026, 11, 14), + "end_local_date": date(2026, 11, 14), + "expected_attendance": 12_000, + "confidence": 1.0, + "status": EventStatus.ACTIVE, + "date_precision": DatePrecision.DAY, + "latitude": None, + "longitude": None, + } + return Event(**(base | overrides)) + + +def _by_night(series) -> dict[date, int]: + return {night: int(n) for night, n in series.items()} + + +def test_one_row_per_night_with_zero_on_quiet_nights(): + series = nightly_attendance([_event()], START, END) + assert _by_night(series) == { + date(2026, 11, 13): 0, + date(2026, 11, 14): 12_000, + date(2026, 11, 15): 0, + date(2026, 11, 16): 0, + } + assert series.name == "expected_attendance" + + +def test_same_night_events_add_up(): + series = nightly_attendance( + [_event(), _event(expected_attendance=3_000)], START, END + ) + assert series[date(2026, 11, 14)] == 15_000 + + +def test_multi_day_event_is_spread_over_its_nights(): + fair = _event( + start_local_date=date(2026, 11, 14), + end_local_date=date(2026, 11, 17), # runs past the window + expected_attendance=20_000, + ) + series = nightly_attendance([fair], START, END) + assert list(series) == [0, 5_000, 5_000, 5_000] + + +@pytest.mark.parametrize("status", [EventStatus.DELETED, EventStatus.CANCELLED]) +def test_withdrawn_events_are_ignored(status): + series = nightly_attendance([_event(status=status)], START, END) + assert series.sum() == 0 + + +def test_provisional_events_count_by_confidence(): + guess = _event(status=EventStatus.PROVISIONAL, confidence=0.25) + assert nightly_attendance([guess], START, END)[date(2026, 11, 14)] == 3_000 + + +def test_unusable_events_contribute_nothing(): + events = [ + _event(expected_attendance=None), + _event(date_precision=DatePrecision.MONTH), + ] + assert nightly_attendance(events, START, END).sum() == 0 + + +def test_radius_drops_far_venues_but_keeps_unlocated_ones(): + near = _event(latitude=53.344, longitude=-6.267) # ~1 km from centre + far = _event(latitude=53.427, longitude=-6.244, expected_attendance=500) # ~9km + unlocated = _event(expected_attendance=100) + series = nightly_attendance( + [near, far, unlocated], + START, + END, + latitude=DUBLIN[0], + longitude=DUBLIN[1], + radius_km=5, + ) + assert series[date(2026, 11, 14)] == 12_100 + + +def test_generate_takes_attendance_without_changing_its_noise(): + days = 5 + attendance = np.array([0, 0, 30_000, 0, 0]) + quiet = generate(start="2026-11-13", days=days, expected_attendance=np.zeros(days)) + busy = generate(start="2026-11-13", days=days, expected_attendance=attendance) + + assert busy.groupby("stay_date")["expected_attendance"].first().tolist() == list( + attendance + ) + # Only the event night's derived features move. + changed = ( + (busy["flight_count"] != quiet["flight_count"]).groupby(busy["stay_date"]).any() + ) + assert changed.tolist() == [False, False, True, False, False] + + +@pytest.fixture +async def session() -> AsyncIterator[AsyncSession]: + engine = create_async_engine(settings.database_url) + try: + conn = await engine.connect() + except (OSError, ConnectionError) as exc: + await engine.dispose() + pytest.skip(f"Postgres unavailable: {exc}") + + trans = await conn.begin() + async with AsyncSession( + bind=conn, join_transaction_mode="create_savepoint" + ) as session: + yield session + await trans.rollback() + await conn.close() + await engine.dispose() + + +async def test_reads_the_market_from_the_events_table(session): + for ref, city, attendance in [ + ("phq-features-1", "Testville", 4_000), + ("phq-features-2", "Testville, Nowhere", 1_000), # same market slug + ("phq-features-3", "Elsewhere", 9_999), + ]: + await upsert_event( + session, + NormalizedEvent( + source=Source.PREDICTHQ, + source_ref=ref, + title=f"Show {city} {attendance}", + city=city, + start_local_date=date(2026, 11, 14), + end_local_date=date(2026, 11, 14), + expected_attendance=attendance, + ), + ) + await session.flush() + + series = await event_features(session, city="Testville", start=START, end=END) + assert series[date(2026, 11, 14)] == 5_000 + assert series.sum() == 5_000 diff --git a/backend/tests/test_ingest_run.py b/backend/tests/test_ingest_run.py new file mode 100644 index 0000000..055e898 --- /dev/null +++ b/backend/tests/test_ingest_run.py @@ -0,0 +1,81 @@ +"""CLI argument parsing and the report shown at the end of a run.""" + +from __future__ import annotations + +import argparse +from datetime import UTC, date, datetime + +import pytest + +from foresight.ingest.normalize import NormalizedEvent, Source +from foresight.ingest.predicthq import Rejected +from foresight.ingest.repository import Outcome +from foresight.ingest.run import Market, RunReport, parse_since + +NOW = datetime(2026, 10, 6, 12, 0, tzinfo=UTC) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("7d", datetime(2026, 9, 29, 12, 0, tzinfo=UTC)), + ("12h", datetime(2026, 10, 6, 0, 0, tzinfo=UTC)), + ("2w", datetime(2026, 9, 22, 12, 0, tzinfo=UTC)), + ("2026-10-01", datetime(2026, 10, 1, tzinfo=UTC)), + ], +) +def test_parse_since(value: str, expected: datetime) -> None: + assert parse_since(value, now=NOW) == expected + + +def test_parse_since_rejects_garbage() -> None: + with pytest.raises(argparse.ArgumentTypeError): + parse_since("last tuesday", now=NOW) + + +def test_report_lists_every_count_and_rejection() -> None: + report = RunReport( + source=Source.PREDICTHQ, + market=Market(city="Dublin", latitude=53.3498, longitude=-6.2603), + radius="10km", + since=NOW, + fetched=6, + outcomes={Outcome.NEW: 1, Outcome.UPDATED: 2, Outcome.UNCHANGED: 2}, + rejected=[Rejected("phq-broken", "end precedes start")], + events_by_status={"active": 4, "deleted": 1}, + observations_by_source={"predicthq": 5}, + ) + + assert report.summary() == "fetched=6 new=1 updated=2 unchanged=2 rejected=1" + rendered = report.render() + for line in ("fetched 6", "updated 2", "rejected 1"): + assert line in rendered + assert "dublin now holds 5 events (4 active, 1 deleted)" in rendered + assert "phq-broken" in rendered + + +def test_a_dry_run_reports_what_it_fetched_and_wrote_nothing() -> None: + event = NormalizedEvent( + source=Source.PREDICTHQ, + source_ref="phq-1", + title="Forbidden Fruit", + city="Dublin", + start_local_date=date(2027, 6, 5), + end_local_date=date(2027, 6, 7), + ) + report = RunReport( + source=Source.PREDICTHQ, + market=Market(city="Dublin", latitude=53.3498, longitude=-6.2603), + radius="10km", + since=NOW, + fetched=1, + events=[event], + dry_run=True, + ) + + assert report.summary() == "fetched=1 rejected=0 (dry run, nothing written)" + rendered = report.render() + assert "nothing written (dry run)" in rendered + assert "Forbidden Fruit" in rendered + # The stored-totals lines would be zeroes and read as data loss. + assert "now holds" not in rendered diff --git a/backend/tests/test_normalize.py b/backend/tests/test_normalize.py new file mode 100644 index 0000000..85b07af --- /dev/null +++ b/backend/tests/test_normalize.py @@ -0,0 +1,140 @@ +"""Normalization contract: identity, idempotency, and the known source gotchas.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime +from zoneinfo import ZoneInfo + +import pytest + +from foresight.ingest.normalize import ( + DatePrecision, + EventCategory, + NormalizedEvent, + Source, + coords_from_geojson, + dedupe_key, + normalize_title, + slugify_city, + to_local_date, +) + + +def test_dedupe_key_is_stable_across_runs() -> None: + args = { + "title": "Electric Picnic", + "city": "Stradbally", + "start_local_date": date(2026, 8, 28), + } + + assert dedupe_key(**args) == dedupe_key(**args) + + +@pytest.mark.parametrize( + ("left", "right"), + [ + ("Fontaines D.C.", "Fontaines DC"), + ("The National", "National"), + ("Hozier - Presented by Aiken Promotions", "Hozier"), + ("Kneecap feat. special guests", "Kneecap"), + ("Dua Lipa (Rescheduled)", "Dua Lipa"), + ("Sigur Rós", "Sigur Ros"), + ], +) +def test_titles_that_mean_the_same_event_collapse(left: str, right: str) -> None: + assert normalize_title(left) == normalize_title(right) + + +def test_distinct_titles_do_not_collapse() -> None: + assert normalize_title("Longitude Festival") != normalize_title("Latitude Festival") + + +def test_city_slug_drops_trailing_region() -> None: + assert slugify_city("Dublin, Ireland") == slugify_city("Dublin") == "dublin" + + +def test_geojson_coords_are_lon_lat_ordered() -> None: + # PredictHQ's Dublin: [-6.26, 53.35]. Latitude must come back as 53.35. + assert coords_from_geojson([-6.26, 53.35]) == (53.35, -6.26) + + +def test_geojson_coords_tolerate_missing_location() -> None: + assert coords_from_geojson(None) == (None, None) + + +def test_local_date_uses_event_timezone_not_utc() -> None: + # 20:00 on 14 March in Los Angeles is already 15 March in UTC. + moment = datetime(2026, 3, 15, 3, 0, tzinfo=UTC) + + assert to_local_date(moment, "America/Los_Angeles") == date(2026, 3, 14) + assert to_local_date(moment, None) == date(2026, 3, 15) + + +def test_unknown_timezone_falls_back_to_utc() -> None: + moment = datetime(2026, 3, 15, 3, 0, tzinfo=UTC) + + assert to_local_date(moment, "Mars/Olympus_Mons") == date(2026, 3, 15) + + +def _event(**overrides: object) -> NormalizedEvent: + base: dict[str, object] = { + "source": Source.PREDICTHQ, + "source_ref": "z13B8ehGzCCZjPM3Ff", + "title": "Electric Picnic 2026", + "city": "Stradbally", + "start_local_date": date(2026, 8, 28), + "end_local_date": date(2026, 8, 30), + } + return NormalizedEvent(**(base | overrides)) # type: ignore[arg-type] + + +def test_two_sources_seeing_one_event_share_a_dedupe_key() -> None: + from_phq = _event(title="Electric Picnic 2026", category=EventCategory.FESTIVALS) + from_news = _event( + source=Source.ASKNEWS, + source_ref="https://example.test/electric-picnic", + title="Electric Picnic 2026 (Festival)", + city="Stradbally, Ireland", + category=EventCategory.CONCERTS, + confidence=0.6, + ) + + assert from_phq.dedupe_key == from_news.dedupe_key + + +def test_as_row_carries_derived_identity_columns() -> None: + row = _event().as_row() + + assert row["dedupe_key"] == _event().dedupe_key + assert row["city_slug"] == "stradbally" + assert row["primary_source"] is Source.PREDICTHQ + assert row["date_precision"] is DatePrecision.DAY + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"end_local_date": date(2026, 8, 27)}, "precedes start"), + ({"confidence": 1.4}, "outside 0..1"), + ({"latitude": 53.35}, "set together"), + # Singapore with lon/lat transposed: 103.5 is not a latitude. + ({"latitude": 103.85, "longitude": 1.29}, "swapped"), + ({"title": "(Live)"}, "normalizes to empty"), + ({"source_ref": ""}, "source_ref is required"), + ], +) +def test_malformed_records_are_rejected( + overrides: dict[str, object], message: str +) -> None: + with pytest.raises(ValueError, match=message): + _event(**overrides) + + +def test_swapped_coordinates_are_caught_only_when_out_of_range() -> None: + # A real swap inside both valid ranges cannot be detected here; the + # coords_from_geojson helper is the defence, not validation. + assert _event(latitude=53.35, longitude=-6.26).latitude == 53.35 + + +def test_zoneinfo_dependency_is_available() -> None: + assert ZoneInfo("Europe/Dublin").key == "Europe/Dublin" diff --git a/backend/tests/test_predicthq.py b/backend/tests/test_predicthq.py new file mode 100644 index 0000000..80ab283 --- /dev/null +++ b/backend/tests/test_predicthq.py @@ -0,0 +1,142 @@ +"""PredictHQ adapter against a mocked API: request shape, paging, field mapping.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime +from typing import Any + +import httpx + +from foresight.ingest.normalize import EventCategory, EventStatus, Source +from foresight.ingest.predicthq import API_URL, fetch_events, to_normalized + + +def _raw(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "id": "z13B8ehGzCCZjPM3Ff", + "title": "Fontaines D.C.", + "category": "concerts", + "state": "active", + # 20:00 in Dublin, which is already 19:00Z: the local and UTC dates agree. + "start": "2026-11-14T19:00:00Z", + "end": "2026-11-14T22:30:00Z", + "timezone": "Europe/Dublin", + "location": [-6.2286, 53.3478], + "country": "IE", + "rank": 68, + "local_rank": 81, + "phq_attendance": 13000, + "entities": [ + {"type": "event-group", "name": "Fontaines D.C. Tour"}, + {"type": "venue", "name": "3Arena"}, + ], + "updated": "2026-10-05T08:12:44Z", + } + return base | overrides + + +def test_maps_geojson_ranks_and_venue() -> None: + event = to_normalized(_raw(), city="Dublin") + + assert event.source is Source.PREDICTHQ + assert (event.latitude, event.longitude) == (53.3478, -6.2286) + assert (event.impact_rank, event.local_rank) == (68, 81) + assert event.venue == "3Arena" + assert event.category is EventCategory.CONCERTS + assert event.expected_attendance == 13000 + assert event.confidence == 1.0 + assert event.source_updated_at == datetime(2026, 10, 5, 8, 12, 44, tzinfo=UTC) + assert event.payload["id"] == "z13B8ehGzCCZjPM3Ff" + + +def test_late_evening_event_keeps_its_local_night() -> None: + # 21:30 in Los Angeles on 14 Nov is 05:30Z on 15 Nov. + event = to_normalized( + _raw( + start="2026-11-15T05:30:00Z", + end="2026-11-15T07:00:00Z", + timezone="America/Los_Angeles", + location=[-118.2673, 34.0430], + ), + city="Los Angeles", + ) + + assert event.start_local_date == date(2026, 11, 14) + assert event.end_local_date == date(2026, 11, 14) + + +def test_deleted_state_becomes_deleted_status() -> None: + event = to_normalized( + _raw(state="deleted", deleted_reason="cancelled"), city="Dublin" + ) + assert event.status is EventStatus.DELETED + + +def test_unknown_category_falls_back() -> None: + event = to_normalized(_raw(category="something-new"), city="Dublin") + assert event.category is EventCategory.UNKNOWN + + +async def test_fetch_filters_by_radius_and_revision_clock_and_follows_next() -> None: + requests: list[httpx.Request] = [] + page_two = f"{API_URL}?offset=1&limit=500" + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.url.params.get("offset") == "1": + return httpx.Response( + 200, + json={ + "count": 3, + "next": None, + "results": [ + _raw(id="b", state="deleted"), + _raw(id="bad", title="(Live)"), + ], + }, + ) + return httpx.Response( + 200, json={"count": 3, "next": page_two, "results": [_raw(id="a")]} + ) + + result = await fetch_events( + token="test-token", + latitude=53.3498, + longitude=-6.2603, + city="Dublin", + updated_since=datetime(2026, 9, 29, 14, 0, tzinfo=UTC), + transport=httpx.MockTransport(handler), + ) + + first = requests[0] + assert first.headers["Authorization"] == "Bearer test-token" + assert first.url.params["within"] == "10km@53.3498,-6.2603" + assert first.url.params["updated.gte"] == "2026-09-29T14:00:00" + assert first.url.params["limit"] == "500" + assert "deleted" in first.url.params["state"].split(",") + assert len(requests) == 2 + + assert result.fetched == 3 + assert [e.source_ref for e in result.events] == ["a", "b"] + assert result.events[1].status is EventStatus.DELETED + assert [r.source_ref for r in result.rejected] == ["bad"] + assert "normalizes to empty" in result.rejected[0].reason + + +async def test_record_missing_required_field_is_rejected_not_raised() -> None: + def handler(request: httpx.Request) -> httpx.Response: + broken = _raw(id="nostart") + del broken["start"] + return httpx.Response(200, json={"count": 1, "next": None, "results": [broken]}) + + result = await fetch_events( + token="t", + latitude=0.0, + longitude=0.0, + city="Dublin", + updated_since=datetime(2026, 9, 29, tzinfo=UTC), + transport=httpx.MockTransport(handler), + ) + + assert result.events == [] + assert result.rejected[0].reason == "missing field 'start'" diff --git a/backend/tests/test_repository.py b/backend/tests/test_repository.py new file mode 100644 index 0000000..b3f58e9 --- /dev/null +++ b/backend/tests/test_repository.py @@ -0,0 +1,273 @@ +"""Upsert resolution order against a real Postgres; skipped when none is reachable.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime +from typing import TYPE_CHECKING + +import pytest +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine + +from foresight.config import settings +from foresight.database.models.external import Event, EventObservation +from foresight.ingest.normalize import EventStatus, NormalizedEvent, Source +from foresight.ingest.repository import Outcome, upsert_event + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +@pytest.fixture +async def session() -> AsyncIterator[AsyncSession]: + engine = create_async_engine(settings.database_url) + try: + conn = await engine.connect() + except (OSError, ConnectionError) as exc: + await engine.dispose() + pytest.skip(f"Postgres unavailable: {exc}") + + trans = await conn.begin() + async with AsyncSession( + bind=conn, join_transaction_mode="create_savepoint" + ) as session: + # Counts below assume an empty table; real ingested rows come back on + # rollback. + await session.execute(delete(EventObservation)) + await session.execute(delete(Event)) + yield session + await trans.rollback() + await conn.close() + await engine.dispose() + + +def _phq(**overrides: object) -> NormalizedEvent: + base: dict[str, object] = { + "source": Source.PREDICTHQ, + "source_ref": "phq-test-1", + "title": "Hozier", + "city": "Dublin", + "start_local_date": date(2026, 11, 14), + "end_local_date": date(2026, 11, 14), + "timezone": "Europe/Dublin", + "local_rank": 71, + "payload": {"id": "phq-test-1"}, + } + return NormalizedEvent(**(base | overrides)) # type: ignore[arg-type] + + +def _news(**overrides: object) -> NormalizedEvent: + base: dict[str, object] = { + "source": Source.ASKNEWS, + "source_ref": "https://example.test/hozier-dublin", + "title": "Hozier (Live)", + "city": "Dublin, Ireland", + "start_local_date": date(2026, 11, 14), + "end_local_date": date(2026, 11, 14), + "venue": "Somewhere Vague", + "confidence": 0.6, + "status": EventStatus.PROVISIONAL, + "payload": {"url": "https://example.test/hozier-dublin"}, + } + return NormalizedEvent(**(base | overrides)) # type: ignore[arg-type] + + +async def _count(session: AsyncSession, model: type) -> int: + return await session.scalar(select(func.count()).select_from(model)) + + +async def _only_event(session: AsyncSession) -> Event: + return ( + await session.scalars(select(Event).execution_options(populate_existing=True)) + ).one() + + +async def test_start_date_revision_updates_the_row_instead_of_orphaning_it( + session: AsyncSession, +) -> None: + assert await upsert_event(session, _phq()) is Outcome.NEW + first = await _only_event(session) + first_id, first_seen, first_last = first.id, first.first_seen_at, first.last_seen_at + + revised = _phq( + start_local_date=date(2026, 11, 15), + end_local_date=date(2026, 11, 15), + payload={"id": "phq-test-1", "rev": 2}, + ) + assert await upsert_event(session, revised) is Outcome.UPDATED + + assert await _count(session, Event) == 1 + assert await _count(session, EventObservation) == 1 + event = await _only_event(session) + assert event.id == first_id + assert event.start_local_date == date(2026, 11, 15) + assert event.dedupe_key == revised.dedupe_key + assert event.first_seen_at == first_seen + assert event.last_seen_at > first_last + + +async def test_identical_rerun_is_unchanged_but_still_bumps_last_seen( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + before = (await _only_event(session)).last_seen_at + + assert await upsert_event(session, _phq()) is Outcome.UNCHANGED + assert await _count(session, Event) == 1 + assert await _count(session, EventObservation) == 1 + assert (await _only_event(session)).last_seen_at > before + + +async def test_second_source_attaches_to_the_existing_event( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + + # AskNews is outranked, so the PredictHQ values stand. + assert await upsert_event(session, _news()) is Outcome.UNCHANGED + + assert await _count(session, Event) == 1 + assert await _count(session, EventObservation) == 2 + event = await _only_event(session) + assert event.primary_source is Source.PREDICTHQ + assert event.venue is None + assert event.status is EventStatus.ACTIVE + + +async def test_higher_precedence_source_takes_over_a_text_extracted_event( + session: AsyncSession, +) -> None: + await upsert_event(session, _news()) + + assert await upsert_event(session, _phq()) is Outcome.UPDATED + event = await _only_event(session) + assert event.primary_source is Source.PREDICTHQ + assert event.confidence == 1.0 + assert event.status is EventStatus.ACTIVE + + +async def test_revision_onto_an_existing_key_merges_rather_than_duplicates( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + # An article already reported the date PredictHQ is about to move to. + await upsert_event( + session, + _news(start_local_date=date(2026, 11, 15), end_local_date=date(2026, 11, 15)), + ) + assert await _count(session, Event) == 2 + + await upsert_event( + session, + _phq(start_local_date=date(2026, 11, 15), end_local_date=date(2026, 11, 15)), + ) + + assert await _count(session, Event) == 1 + event = await _only_event(session) + assert event.primary_source is Source.PREDICTHQ + linked = await session.scalars(select(EventObservation.event_id)) + assert set(linked) == {event.id} + + +async def test_deletion_of_a_known_record_marks_the_event_deleted( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + + deleted = _phq(status=EventStatus.DELETED, payload={"state": "deleted"}) + assert await upsert_event(session, deleted) is Outcome.UPDATED + assert (await _only_event(session)).status is EventStatus.DELETED + + +async def test_another_records_deletion_does_not_kill_a_live_event( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + + duplicate = _phq( + source_ref="phq-test-dupe", + status=EventStatus.DELETED, + payload={"deleted_reason": "duplicate"}, + ) + assert await upsert_event(session, duplicate) is Outcome.UNCHANGED + assert (await _only_event(session)).status is EventStatus.ACTIVE + assert await _count(session, EventObservation) == 2 + + +async def test_observation_payload_tracks_the_latest_revision( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + updated_at = datetime(2026, 10, 6, 9, 30, tzinfo=UTC) + await upsert_event( + session, + _phq(payload={"id": "phq-test-1", "rev": 2}, source_updated_at=updated_at), + ) + + obs = ( + await session.scalars( + select(EventObservation).execution_options(populate_existing=True) + ) + ).one() + assert obs.payload == {"id": "phq-test-1", "rev": 2} + assert obs.source_updated_at == updated_at + + +async def test_deleted_duplicate_stays_harmless_on_every_rerun( + session: AsyncSession, +) -> None: + live = _phq() + duplicate = _phq(source_ref="phq-test-dupe", status=EventStatus.DELETED) + + for _ in range(3): + assert await upsert_event(session, live) in {Outcome.NEW, Outcome.UNCHANGED} + assert await upsert_event(session, duplicate) is Outcome.UNCHANGED + assert (await _only_event(session)).status is EventStatus.ACTIVE + + +async def test_live_duplicates_settle_on_the_latest_revision( + session: AsyncSession, +) -> None: + older = _phq( + expected_attendance=210, + source_updated_at=datetime(2026, 10, 1, tzinfo=UTC), + ) + newer = _phq( + source_ref="phq-test-dupe", + expected_attendance=211, + source_updated_at=datetime(2026, 10, 5, tzinfo=UTC), + ) + await upsert_event(session, older) + assert await upsert_event(session, newer) is Outcome.UPDATED + + # A rerun must not flip the row back and forth between the two listings. + assert await upsert_event(session, older) is Outcome.UNCHANGED + assert await upsert_event(session, newer) is Outcome.UNCHANGED + assert (await _only_event(session)).expected_attendance == 211 + + +async def test_live_listing_restores_an_event_its_duplicate_deleted( + session: AsyncSession, +) -> None: + # As left behind before sightings recorded their status. + await upsert_event(session, _phq(source_ref="phq-test-dupe")) + await upsert_event(session, _phq()) + await session.execute(EventObservation.__table__.update().values(status=None)) + await session.execute(Event.__table__.update().values(status=EventStatus.DELETED)) + + await upsert_event( + session, _phq(source_ref="phq-test-dupe", status=EventStatus.DELETED) + ) + assert await upsert_event(session, _phq()) is Outcome.UPDATED + assert (await _only_event(session)).status is EventStatus.ACTIVE + + +async def test_structured_deletion_outranks_a_live_article( + session: AsyncSession, +) -> None: + await upsert_event(session, _phq()) + await upsert_event(session, _news()) + + deleted = _phq(status=EventStatus.DELETED) + assert await upsert_event(session, deleted) is Outcome.UPDATED + assert (await _only_event(session)).status is EventStatus.DELETED diff --git a/backend/tests/test_textdates.py b/backend/tests/test_textdates.py new file mode 100644 index 0000000..f216ead --- /dev/null +++ b/backend/tests/test_textdates.py @@ -0,0 +1,133 @@ +"""Date resolution: the formats announcements actually use, and what we refuse.""" + +from __future__ import annotations + +from datetime import date + +import pytest + +from foresight.ingest.normalize import DatePrecision +from foresight.ingest.textdates import looks_retrospective, resolve_date + +REFERENCE = date(2026, 6, 1) + + +@pytest.mark.parametrize( + ("text", "start", "end"), + [ + ("runs on August 28, 2026", date(2026, 8, 28), date(2026, 8, 28)), + ("runs 28 August 2026", date(2026, 8, 28), date(2026, 8, 28)), + ("August 28-30, 2026", date(2026, 8, 28), date(2026, 8, 30)), + ("28-30 August 2026", date(2026, 8, 28), date(2026, 8, 30)), + ("Aug. 28th to 30th, 2026", date(2026, 8, 28), date(2026, 8, 30)), + ("on 2026-08-28", date(2026, 8, 28), date(2026, 8, 28)), + ("2026-08-28 to 2026-08-30", date(2026, 8, 28), date(2026, 8, 30)), + # Ranges that roll into the next month, and the next year. + ("August 30 - September 2, 2026", date(2026, 8, 30), date(2026, 9, 2)), + ("December 30 - January 2, 2026", date(2026, 12, 30), date(2027, 1, 2)), + # A year stated on both sides of the dash. + ( + "December 30, 2026 - January 1, 2027", + date(2026, 12, 30), + date(2027, 1, 1), + ), + # A range written as prose rather than punctuation. + ( + "opens December 1, 2026 and runs through January 4, 2027", + date(2026, 12, 1), + date(2027, 1, 4), + ), + ("runs July 1 through July 9, 2027", date(2027, 7, 1), date(2027, 7, 9)), + ], +) +def test_day_precision_formats(text: str, start: date, end: date) -> None: + resolved = resolve_date(text, REFERENCE) + + assert resolved is not None + assert (resolved.start, resolved.end) == (start, end) + assert resolved.precision is DatePrecision.DAY + + +def test_missing_year_is_inferred_forward() -> None: + resolved = resolve_date("the festival returns August 28", REFERENCE) + + assert resolved is not None + assert resolved.start == date(2026, 8, 28) + assert not resolved.had_explicit_year + + +def test_missing_year_rolls_over_when_the_month_has_passed() -> None: + """In June, an undated "March 14" means next March.""" + resolved = resolve_date("tickets for March 14", REFERENCE) + + assert resolved is not None + assert resolved.start == date(2027, 3, 14) + + +def test_month_year_keeps_month_precision() -> None: + resolved = resolve_date("sometime in August 2026", REFERENCE) + + assert resolved is not None + assert resolved.precision is DatePrecision.MONTH + assert (resolved.start, resolved.end) == (date(2026, 8, 1), date(2026, 8, 31)) + + +@pytest.mark.parametrize( + ("text", "start", "end"), + [ + ("summer 2026", date(2026, 6, 1), date(2026, 8, 31)), + ("autumn 2026", date(2026, 9, 1), date(2026, 11, 30)), + ("winter 2026", date(2026, 12, 1), date(2027, 2, 28)), + ("Q3 2026", date(2026, 7, 1), date(2026, 9, 30)), + ], +) +def test_coarse_spans_keep_quarter_precision(text: str, start: date, end: date) -> None: + resolved = resolve_date(text, REFERENCE) + + assert resolved is not None + assert resolved.precision is DatePrecision.QUARTER + assert (resolved.start, resolved.end) == (start, end) + + +def test_finest_precision_wins_when_a_text_carries_several() -> None: + text = "announced for summer 2026, with the main stage on August 28, 2026" + + resolved = resolve_date(text, REFERENCE) + + assert resolved is not None + assert resolved.precision is DatePrecision.DAY + assert resolved.start == date(2026, 8, 28) + + +def test_impossible_calendar_date_is_skipped_for_the_next_candidate() -> None: + resolved = resolve_date("February 30, 2026 -- sorry, July 3, 2026", REFERENCE) + + assert resolved is not None + assert resolved.start == date(2026, 7, 3) + + +@pytest.mark.parametrize( + "text", + [ + "happening this weekend", + "next Friday at the venue", + "coming soon to the city", + "no date has been announced", + ], +) +def test_relative_and_absent_dates_are_refused(text: str) -> None: + assert resolve_date(text, REFERENCE) is None + + +def test_date_already_past_relative_to_the_reference_is_refused() -> None: + assert resolve_date("was on January 5, 2020", REFERENCE) is None + + +def test_retrospective_coverage_is_flagged() -> None: + assert looks_retrospective("the festival took place over the weekend") + + +def test_a_forward_looking_marker_rescues_a_retrospective_recap() -> None: + text = "last year's festival drew crowds; this year's runs August 28-30, 2026" + + assert not looks_retrospective(text) diff --git a/backend/uv.lock b/backend/uv.lock index 87cadef..cfbc3fc 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1,15 +1,15 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version < '3.13' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] @@ -331,6 +331,7 @@ dependencies = [ { name = "alembic" }, { name = "asyncpg" }, { name = "fastapi" }, + { name = "httpx" }, { name = "pandas" }, { name = "psycopg2-binary" }, { name = "pydantic-settings" }, @@ -362,6 +363,7 @@ requires-dist = [ { name = "alembic", specifier = ">=1.15" }, { name = "asyncpg", specifier = ">=0.30" }, { name = "fastapi", specifier = ">=0.115" }, + { name = "httpx", specifier = ">=0.28" }, { name = "pandas", specifier = ">=2.2" }, { name = "psycopg2-binary", specifier = ">=2.9" }, { name = "pydantic-settings", specifier = ">=2.9" }, diff --git a/docs/event-extraction.md b/docs/event-extraction.md new file mode 100644 index 0000000..0bd82bb --- /dev/null +++ b/docs/event-extraction.md @@ -0,0 +1,273 @@ +# Event extraction + +Two sources feed the `events` table. They disagree about almost everything — +what a record is, how trustworthy it is, how often it changes — so they meet at +a single contract rather than talking to the database in their own shapes. + +```mermaid +flowchart LR + PHQ[PredictHQ API] --> PA[ingest/predicthq.py] + AN[AskNews API] --> AA[ingest/asknews.py] + PA --> NE[NormalizedEvent] + AA --> TD[ingest/textdates.py] --> NE + NE --> R[ingest/repository.py
upsert_event] + R --> EV[(events)] + R --> OB[(event_observations)] + EV --> F[services/features/events.py] + F --> M[pricing models] +``` + +## The contract: `ingest/normalize.py` + +Every adapter returns `NormalizedEvent`, which validates on construction — +empty title, end before start, confidence outside 0..1, a latitude that is +really a longitude. An adapter cannot emit a malformed event; it can only fail +to build one. + +Identity is `dedupe_key` = normalized title + city slug + local start date. +Category is deliberately excluded: the two sources routinely disagree on it, +and that disagreement must not split one real event into two rows. + +Two fields carry how much to trust a record: + +- `status` — `ACTIVE` for a confirmed listing, `PROVISIONAL` for a PredictHQ + *predicted* event or anything extracted from prose, `DELETED`/`CANCELLED` + once withdrawn upstream. +- `date_precision` — `DAY`, `MONTH`, or `QUARTER`. Coarse dates are recorded + rather than rounded away, so "summer 2027" is kept as early signal without + pretending to be a stay-night. + +## PredictHQ — structured listings + +`fetch_events()` pages the API by `updated.gte` and returns a `FetchResult` of +events plus rejections. Records arrive already dated and geocoded, so the +adapter is mostly field mapping: GeoJSON `[lon, lat]` unpacked in the right +order, UTC timestamps converted to the venue's local night, `state` mapped onto +`EventStatus`. + +It asks for `deleted` records explicitly. Without that, an event cancelled +upstream would sit in the demand forecast forever. + +Everything it emits is `confidence = 1.0` and `DatePrecision.DAY`. + +## AskNews — prose + +AskNews indexes **articles, not events**, so every record is an inference: an +article mentions a festival, and we decide whether it pins down a title, a city +we sell rooms in, and a date precise enough to price against. Most do not. +Dropping those is the bulk of the work. + +Three stages: + +1. **Search** — one query per city, filtered to that city's country. + Scoping matters: AskNews bills per search, and an unscoped run rejected 188 + of 198 articles on city alone. +2. **Date resolution** (`textdates.py`) — turns "August 28-30, 2026", "opens + December 1 and runs through January 4" or "Q3 2027" into a span plus the + precision achieved. Relative forms ("this weekend") are refused outright: a + wrong date is worse than a missing one. +3. **Extraction** — city match, category, confidence, and a `RejectReason` for + anything that does not survive. The per-run tally of those reasons is the + audit trail for what got dropped and why. + +One event per article, because `event_observations` is unique on +`(source, source_ref)` and `source_ref` is the article URL. + +Everything it emits is `PROVISIONAL`, with confidence capped at 0.95. It never +sets `expected_attendance`, `venue`, or coordinates — an article does not +reliably state them. + +## Where they meet: `ingest/repository.py` + +`upsert_event(session, event)` is source-agnostic. It resolves identity in a +specific order that matters: + +1. By `(source, source_ref)` — has this source seen this record before? +2. Only then by `dedupe_key`. + +Backwards would orphan rows: a revised start date changes the `dedupe_key`, so +resolving by key first would strand the original. `_absorb_key_collision` +merges the two when a revision lands on a key another sighting already holds. + +Precedence decides who wins a contested field: + +```python +_PRECEDENCE = {Source.PREDICTHQ: 2, Source.ASKNEWS: 1} +``` + +A structured listing outranks prose. AskNews exists to see an event *before* +PredictHQ does, not to outvote it once it arrives. + +`events` holds one merged row per real-world occurrence; +`event_observations` keeps each source's raw sighting verbatim, which is what +makes re-runs idempotent and lets a parser bug be fixed and replayed. + +## How events reach pricing + +`services/features/events.py` turns stored events into the per-night +`expected_attendance` feature. An event contributes only if it is day-precise, +not withdrawn, and carries an attendance estimate; multi-day events spread +evenly across their nights, weighted by confidence. + +In practice this means **AskNews events currently contribute nothing to +pricing** — they never carry attendance. They are stored, deduped against +PredictHQ, and wait to be corroborated. + +## New files + +Paths under `backend/`. Lines are a rough guide to where the weight sits. + +### The contract + +| File | Lines | Purpose | +| --- | ---: | --- | +| `foresight/ingest/normalize.py` | 254 | `NormalizedEvent`, `dedupe_key`, and the enums every adapter shares. Validates on construction. | + +### Adapters + +| File | Lines | Purpose | +| --- | ---: | --- | +| `foresight/ingest/predicthq.py` | 162 | Fetches structured listings near a point and maps them to `NormalizedEvent`. No DB writes. | +| `foresight/ingest/asknews.py` | 470 | Searches news per city and infers events from articles. Owns the reject reasons and the per-run report. | +| `foresight/ingest/textdates.py` | 276 | Resolves prose dates ("August 28-30", "Q3 2027") into a span plus the precision achieved. | + +### Persistence + +| File | Lines | Purpose | +| --- | ---: | --- | +| `foresight/ingest/repository.py` | 204 | `upsert_event` — source-agnostic merge onto `events` plus an append-only `event_observations` row. | +| `.../versions/b7d41e0c9a52_canonical_events_and_observations.py` | 93 | Creates both tables. | +| `.../versions/c3e9a1f4b6d8_observation_status.py` | 42 | Adds `status` to observations, so a withdrawal is attributable to the source that reported it. | + +### Consumption + +| File | Lines | Purpose | +| --- | ---: | --- | +| `foresight/services/features/events.py` | 125 | Stored events to the per-night `expected_attendance` feature the pricing models read. | + +### Entry points + +| File | Lines | Purpose | +| --- | ---: | --- | +| `foresight/ingest/run.py` | 240 | The ingest CLI (`python -m foresight.ingest.run`). PredictHQ only today. | +| `scripts/run_asknews.py` | 171 | Runs the AskNews ingest and prints the cleaning report. Replays fixtures by default; `--live` calls the API. | +| `scripts/forecast_events.py` | 123 | Prices upcoming nights using real stored events, to show their price impact. | + +### Tests and fixtures + +| File | Lines | Purpose | +| --- | ---: | --- | +| `tests/test_normalize.py` | 140 | Identity, idempotency, and the known source gotchas. | +| `tests/test_predicthq.py` | 142 | Field mapping and paging, against a fake HTTP response. | +| `tests/test_asknews.py` | 449 | What survives extraction, what gets dropped and why, and the HTTP client. | +| `tests/test_textdates.py` | 133 | The date formats announcements actually use, and the ones we refuse. | +| `tests/test_repository.py` | 273 | Upsert behaviour against Postgres: revisions, precedence, deletions. | +| `tests/test_event_features.py` | 168 | Attendance spreading, exclusions, and the radius filter. | +| `tests/test_ingest_run.py` | 54 | `--since` parsing and the run report. | +| `tests/fixtures/asknews/run_{1,2,3}.json` | 342 | Three article batches: a first poll, an overlapping re-poll, and a date revision. Hand-built to the documented AskNews schema, not recorded responses. | + +## How to test it + +### Automated + +```bash +just test # whole suite, from the repo root +cd backend && uv run pytest -q tests/test_asknews.py tests/test_textdates.py +``` + +`test_repository.py` and `test_event_features.py` need Postgres; the rest run +offline against fake HTTP responses. Without a database they error rather than +skip, which looks alarming and is not. + +### Database + +```bash +just db-up # docker compose up -d db migrate +``` + +Beware: if you already run Postgres on your own machine, it binds +`127.0.0.1:5432` and wins over Docker for anything connecting to `localhost`. +`just db-up` then succeeds while the app silently talks to a different +database. Check with `lsof -nP -iTCP:5432 -sTCP:LISTEN` — more than one +listener means you have the clash. Either stop the host server, or use it +directly: + +```bash +psql -d postgres -c "CREATE ROLE foresight LOGIN PASSWORD 'foresight';" \ + -c "CREATE DATABASE foresight_db OWNER foresight;" +cd backend && uv run alembic upgrade head +``` + +### Console only, no database + +```bash +cd backend && uv run python scripts/run_asknews.py --dry-run +``` + +Prints the cleaning report and every extracted event, and writes nothing. This +is the mode to use while changing extraction: no Postgres needed, and repeated +runs leave no rows behind. Works with `--live` too. + +### AskNews, without credentials + +```bash +cd backend && uv run python scripts/run_asknews.py +``` + +Replays the three recorded batches and writes them through the real upsert. +Expect run 1 to be all `new`, run 2 to report `unchanged` for the overlap, and +run 3 to show one `updated` where a festival moved its dates. Running the whole +thing twice should produce `new 0` throughout and no extra rows — that is the +idempotency check. + +Pass `--today 2026-10-06` to pin the reference date; the fixtures are dated +against it and will start ageing out of the horizon eventually. + +### AskNews, live + +Needs `ASKNEWS_CLIENT_ID` and `ASKNEWS_CLIENT_SECRET` in the repo-root `.env` +(OAuth2 client credentials from the AskNews console, not an API key). + +```bash +cd backend && uv run python scripts/run_asknews.py --live --runs 1 +``` + +Use `--runs 1`. AskNews bills per search and the default of 3 would fire the +same queries three times at the same window. + +### PredictHQ + +Needs `PREDICTHQ_TOKEN`. + +```bash +cd backend && uv run python -m foresight.ingest.run \ + --source predicthq --since 7d --city Dublin \ + --lat 53.3498 --lon -6.2603 --report --dry-run +``` + +`--dry-run` fetches and prints without writing. With `--lat/--lon/--city` it +needs no database at all, and skips the automatic migration too. Drop the flag +to persist. (`--hotel-id` still needs the database, to look the hotel up.) + +### Checking what landed + +```bash +psql -U foresight -h localhost -d foresight_db -c \ + "SELECT start_local_date, city_slug, date_precision, status, primary_source, title + FROM events ORDER BY start_local_date;" +``` + +`event_observations` should hold one row per source sighting, and re-running an +ingest must not grow it. + +## Not wired yet + +- `run.py` hard-codes `--source predicthq`; AskNews runs through + `scripts/run_asknews.py` instead. +- Rejected articles are counted, not persisted. There is no quarantine table to + review them later. +- `confidence` scores *extraction certainty*, not whether something is a real + bookable event — a live run scored the entity "midterms" at 0.95. It is not + safe as a quality filter. This matters most for the attendance weighting + above: if AskNews events ever gain an attendance estimate, that multiplier + would admit junk at near-full weight. diff --git a/justfile b/justfile index 42bdb46..a027aef 100644 --- a/justfile +++ b/justfile @@ -28,9 +28,9 @@ dev: destroy: docker compose down -# Start only the database +# Start only the database, with migrations applied db-up: - docker compose up -d db + docker compose up -d db migrate # Stop only the database db-down: