fix: update logging levels and improve job URL handling in daily task

.gitignore

@@ -15,6 +15,7 @@ __pycache__/
 .mypy_cache/
 .ruff_cache/
 .pytest_cache/
+pytest-of-*/
 htmlcov/
 build/
 develop-eggs/

README.md

@@ -1,6 +1,6 @@
 # Immermatch
 
-[![CI](https://github.com/TheTrueAI/immermatch/actions/workflows/ci.yml/badge.svg)](https://github.com/TheTrueAI/immermatch/actions/workflows/ci.yml)
+[![Daily Digest to Users](https://github.com/TheTrueAI/immermatch/actions/workflows/daily-digest.yml/badge.svg)](https://github.com/TheTrueAI/immermatch/actions/workflows/daily-digest.yml)
 [![License: AGPL-3.0](https://img.shields.io/badge/License-AGPL--3.0-blue.svg)](LICENSE)
 [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-green.svg)](https://www.python.org/)
 

daily_task.py

@@ -53,6 +53,8 @@
     level=logging.INFO,
     format="%(asctime)s  %(levelname)-8s  %(message)s",
 )
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
 log = logging.getLogger("daily_task")
 
 
@@ -111,10 +113,10 @@ def main() -> int:
     )
 
     # ── 5. Search once per unique (query-set, location) ──────────────────
-    # Collect all jobs keyed by title|company_name|location to avoid duplication
-    all_jobs: dict[str, JobListing] = {}
-    # Track which URLs belong to which normalized location
+    # URL is the source of truth for DB upserts and per-subscriber processing.
+    # Track URLs per normalized location and keep one JobListing per URL.
     location_urls: dict[str, set[str]] = defaultdict(set)
+    url_to_job: dict[str, JobListing] = {}
 
     for loc, queries in location_queries.items():
         query_list = sorted(queries)  # deterministic order
@@ -125,35 +127,28 @@ def main() -> int:
             location=loc,
         )
         for job in found:
-            key = f"{job.title}|{job.company_name}|{job.location}"
-            if key not in all_jobs:
-                all_jobs[key] = job
             url = _listing_url(job)
             if url:
                 location_urls[loc].add(url)
+                url_to_job.setdefault(url, job)
 
-    log.info("Found %d unique jobs total", len(all_jobs))
-    if not all_jobs:
+    log.info("Found %d unique job URLs total", len(url_to_job))
+    if not url_to_job:
         log.info("No jobs found — exiting.")
         return 0
 
     # ── 6. Upsert all jobs into DB (with descriptions) ───────────────────
-    jobs_list = list(all_jobs.values())
-    job_dicts = []
-    url_to_job: dict[str, JobListing] = {}
-    for job in jobs_list:
-        url = _listing_url(job)
-        if url:
-            url_to_job[url] = job
-            job_dicts.append(
-                {
-                    "title": job.title,
-                    "company": job.company_name,
-                    "url": url,
-                    "location": job.location,
-                    "description": job.description,
-                }
-            )
+
+    job_dicts = [
+        {
+            "title": job.title,
+            "company": job.company_name,
+            "url": url,
+            "location": job.location,
+            "description": job.description,
+        }
+        for url, job in url_to_job.items()
+    ]
 
     if job_dicts:
         upsert_jobs(db, job_dicts)

tests/test_app_ui.py

@@ -144,12 +144,14 @@ def test_sidebar_shows_pipeline_activity(self, _mock_purge: MagicMock, _mock_db:
 
     @patch.dict("os.environ", _FAKE_ENV, clear=False)
     @patch("streamlit.rerun", return_value=None)
+    @patch("immermatch.llm.create_client", return_value=MagicMock())
     @patch("immermatch.db.get_admin_client", return_value=MagicMock())
     @patch("immermatch.db.purge_inactive_subscribers", return_value=0)
     def test_queries_render_in_status_during_active_run(
         self,
         _mock_purge: MagicMock,
         _mock_db: MagicMock,
+        _mock_llm: MagicMock,
         _mock_rerun: MagicMock,
         tmp_path: Path,
         monkeypatch,

tests/test_bundesagentur.py

@@ -334,6 +334,7 @@ def test_search_returns_listings(self) -> None:
         with (
             patch.object(provider, "_get_with_retry", return_value=mock_resp),
             patch.object(provider, "_enrich", side_effect=lambda it: [_parse_listing(i) for i in it]),
+            patch("immermatch.search_api.bundesagentur.httpx.Client"),
         ):
             jobs = provider.search("Python", "Berlin", max_results=10)
 
@@ -351,6 +352,7 @@ def test_search_empty_results(self) -> None:
         with (
             patch.object(provider, "_get_with_retry", return_value=mock_resp),
             patch.object(provider, "_enrich", side_effect=lambda it: [_parse_listing(i) for i in it]),
+            patch("immermatch.search_api.bundesagentur.httpx.Client"),
         ):
             jobs = provider.search("Niche Job", "Berlin")
         assert jobs == []
@@ -374,6 +376,7 @@ def test_search_respects_max_results(self) -> None:
         with (
             patch.object(provider, "_get_with_retry", return_value=mock_resp),
             patch.object(provider, "_enrich", side_effect=lambda it: [_parse_listing(i) for i in it]),
+            patch("immermatch.search_api.bundesagentur.httpx.Client"),
         ):
             jobs = provider.search("Dev", "Berlin", max_results=3)