From 050f38834489201ab37d8d5c9f5b8e51d8e727d7 Mon Sep 17 00:00:00 2001 From: Adrian Immer Date: Tue, 31 Mar 2026 13:47:57 +0200 Subject: [PATCH] fix: update logging levels and improve job URL handling in daily task --- .gitignore | 1 + README.md | 2 +- daily_task.py | 43 ++++++++++++++++--------------------- tests/test_app_ui.py | 2 ++ tests/test_bundesagentur.py | 3 +++ 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index d96a373..0f2987a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ __pycache__/ .mypy_cache/ .ruff_cache/ .pytest_cache/ +pytest-of-*/ htmlcov/ build/ develop-eggs/ diff --git a/README.md b/README.md index eb86a90..575f2b6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Immermatch -[![CI](https://github.com/TheTrueAI/immermatch/actions/workflows/ci.yml/badge.svg)](https://github.com/TheTrueAI/immermatch/actions/workflows/ci.yml) +[![Daily Digest to Users](https://github.com/TheTrueAI/immermatch/actions/workflows/daily-digest.yml/badge.svg)](https://github.com/TheTrueAI/immermatch/actions/workflows/daily-digest.yml) [![License: AGPL-3.0](https://img.shields.io/badge/License-AGPL--3.0-blue.svg)](LICENSE) [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-green.svg)](https://www.python.org/) diff --git a/daily_task.py b/daily_task.py index 883f88e..3e50903 100644 --- a/daily_task.py +++ b/daily_task.py @@ -53,6 +53,8 @@ level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", ) +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) log = logging.getLogger("daily_task") @@ -111,10 +113,10 @@ def main() -> int: ) # ── 5. Search once per unique (query-set, location) ────────────────── - # Collect all jobs keyed by title|company_name|location to avoid duplication - all_jobs: dict[str, JobListing] = {} - # Track which URLs belong to which normalized location + # URL is the source of truth for DB upserts and per-subscriber processing. + # Track URLs per normalized location and keep one JobListing per URL. location_urls: dict[str, set[str]] = defaultdict(set) + url_to_job: dict[str, JobListing] = {} for loc, queries in location_queries.items(): query_list = sorted(queries) # deterministic order @@ -125,35 +127,28 @@ def main() -> int: location=loc, ) for job in found: - key = f"{job.title}|{job.company_name}|{job.location}" - if key not in all_jobs: - all_jobs[key] = job url = _listing_url(job) if url: location_urls[loc].add(url) + url_to_job.setdefault(url, job) - log.info("Found %d unique jobs total", len(all_jobs)) - if not all_jobs: + log.info("Found %d unique job URLs total", len(url_to_job)) + if not url_to_job: log.info("No jobs found — exiting.") return 0 # ── 6. Upsert all jobs into DB (with descriptions) ─────────────────── - jobs_list = list(all_jobs.values()) - job_dicts = [] - url_to_job: dict[str, JobListing] = {} - for job in jobs_list: - url = _listing_url(job) - if url: - url_to_job[url] = job - job_dicts.append( - { - "title": job.title, - "company": job.company_name, - "url": url, - "location": job.location, - "description": job.description, - } - ) + + job_dicts = [ + { + "title": job.title, + "company": job.company_name, + "url": url, + "location": job.location, + "description": job.description, + } + for url, job in url_to_job.items() + ] if job_dicts: upsert_jobs(db, job_dicts) diff --git a/tests/test_app_ui.py b/tests/test_app_ui.py index 3b08229..51ac3c1 100644 --- a/tests/test_app_ui.py +++ b/tests/test_app_ui.py @@ -144,12 +144,14 @@ def test_sidebar_shows_pipeline_activity(self, _mock_purge: MagicMock, _mock_db: @patch.dict("os.environ", _FAKE_ENV, clear=False) @patch("streamlit.rerun", return_value=None) + @patch("immermatch.llm.create_client", return_value=MagicMock()) @patch("immermatch.db.get_admin_client", return_value=MagicMock()) @patch("immermatch.db.purge_inactive_subscribers", return_value=0) def test_queries_render_in_status_during_active_run( self, _mock_purge: MagicMock, _mock_db: MagicMock, + _mock_llm: MagicMock, _mock_rerun: MagicMock, tmp_path: Path, monkeypatch, diff --git a/tests/test_bundesagentur.py b/tests/test_bundesagentur.py index e259da2..1ec1b3e 100644 --- a/tests/test_bundesagentur.py +++ b/tests/test_bundesagentur.py @@ -334,6 +334,7 @@ def test_search_returns_listings(self) -> None: with ( patch.object(provider, "_get_with_retry", return_value=mock_resp), patch.object(provider, "_enrich", side_effect=lambda it: [_parse_listing(i) for i in it]), + patch("immermatch.search_api.bundesagentur.httpx.Client"), ): jobs = provider.search("Python", "Berlin", max_results=10) @@ -351,6 +352,7 @@ def test_search_empty_results(self) -> None: with ( patch.object(provider, "_get_with_retry", return_value=mock_resp), patch.object(provider, "_enrich", side_effect=lambda it: [_parse_listing(i) for i in it]), + patch("immermatch.search_api.bundesagentur.httpx.Client"), ): jobs = provider.search("Niche Job", "Berlin") assert jobs == [] @@ -374,6 +376,7 @@ def test_search_respects_max_results(self) -> None: with ( patch.object(provider, "_get_with_retry", return_value=mock_resp), patch.object(provider, "_enrich", side_effect=lambda it: [_parse_listing(i) for i in it]), + patch("immermatch.search_api.bundesagentur.httpx.Client"), ): jobs = provider.search("Dev", "Berlin", max_results=3)