From b849cb2dbd1936d5faf06b2e2bdb7c0594e3a41b Mon Sep 17 00:00:00 2001 From: Mohammed Taha Khan Date: Fri, 11 Sep 2026 10:05:31 +0200 Subject: [PATCH] fix(harvester): skip harvest when metadata did not change --- site/cds_rdm/inspire_harvester/utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/site/cds_rdm/inspire_harvester/utils.py b/site/cds_rdm/inspire_harvester/utils.py index 47e42e05..e22e1fec 100644 --- a/site/cds_rdm/inspire_harvester/utils.py +++ b/site/cds_rdm/inspire_harvester/utils.py @@ -22,6 +22,11 @@ def retrieve_identifiers(identifiers, scheme): yield ident["identifier"] +def _keys_without_empty_values(value): + """Keys whose values are not None, [] or {} (empty dump placeholders).""" + return {k for k, v in value.items() if v not in (None, [], {})} + + def compare_metadata(a, b): """Compare metadata based on id key only.""" # If both are dicts @@ -30,11 +35,13 @@ def compare_metadata(a, b): if "id" in a and "id" in b: return a["id"] == b["id"] + keys_a, keys_b = _keys_without_empty_values(a), _keys_without_empty_values(b) + # Otherwise compare keys recursively - if a.keys() != b.keys(): + if keys_a != keys_b: return False - return all(compare_metadata(a[k], b[k]) for k in a) + return all(compare_metadata(a[k], b[k]) for k in keys_a) # If both are lists if isinstance(a, list) and isinstance(b, list):