From 47b5349bfee7bec7b0a4e5cf5318cf8a464c73ff Mon Sep 17 00:00:00 2001 From: kpsherva Date: Mon, 21 Sep 2026 17:08:54 +0200 Subject: [PATCH 1/5] add(subjects): more specific content types as subjects # Conflicts: # cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py --- .../records/transform/xml_processing/rules/research_committee.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py index 8f9ebf0b..6b79caaf 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py @@ -95,7 +95,6 @@ }, } - def _controlled_subject(term): """Build a subjects entry referencing a controlled-vocabulary term. From b2fa67efd540234429fc4ca2ba1b77788676bc96 Mon Sep 17 00:00:00 2001 From: kpsherva Date: Wed, 23 Sep 2026 18:04:04 +0200 Subject: [PATCH 2/5] change(transform): don't create separate versions for external DOIs * closes https://github.com/CERNDocumentServer/cds-migrator-kit/issues/609 --- .../records/transform/transform_versions.py | 36 ++++++++++++++ tests/cds-rdm/test_transform_versions.py | 47 +++++++++++++++++-- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/transform_versions.py b/cds_migrator_kit/rdm/records/transform/transform_versions.py index 05a1f64d..4298f9c5 100644 --- a/cds_migrator_kit/rdm/records/transform/transform_versions.py +++ b/cds_migrator_kit/rdm/records/transform/transform_versions.py @@ -23,6 +23,12 @@ class RecordVersionsTransform: file A gets uploaded, the later record version still needs to include file B too, so each version's file list is a cumulative snapshot, not just its own delta. + + Exception: when the record's DOI is external (not minted by our + DataCite prefix - see ``RecordEntry._pids()``), we don't own/manage + that DOI, so its legacy per-file version history is collapsed into a + single RDM version holding every file, instead of one RDM version per + legacy file revision - see ``_is_external_doi()``. """ def __init__(self, raw_dump_entry, record, files_dump_dir, plots, migration_logger): @@ -58,6 +64,25 @@ def build(self): own_file_dumps.setdefault(version_number, []).append(file_dump) representative_file.setdefault(version_number, file_dump) + if own_file_dumps and self._is_external_doi(): + # collapse every legacy file revision into a single version - + # its files (see the carry-forward below, still a no-op for one + # version) and its access/publication_date (from the latest + # legacy version's own representative file, i.e. the current + # state) instead of one RDM version per legacy revision. + latest_version_number = max(own_file_dumps) + own_file_dumps = OrderedDict( + [ + ( + latest_version_number, + [fd for fds in own_file_dumps.values() for fd in fds], + ) + ] + ) + representative_file = { + latest_version_number: representative_file[latest_version_number] + } + versions = OrderedDict( ( version_number, @@ -89,6 +114,17 @@ def build(self): return versions + def _is_external_doi(self): + """Return True if this record's DOI isn't minted through our prefix. + + Mirrors the ``provider`` set in ``RecordEntry._pids()`` + (``"external"`` when the DOI doesn't start with + ``current_app.config["DATACITE_PREFIX"]``); ``False`` (no + collapsing) when the record has no DOI at all. + """ + doi = self.record.body.get("pids", {}).get("doi", {}) + return doi.get("provider") == "external" + def _should_skip_file(self, file_dump): if file_dump["subformat"] in FILE_SUBFORMATS_TO_DROP: self.migration_logger.add_information( diff --git a/tests/cds-rdm/test_transform_versions.py b/tests/cds-rdm/test_transform_versions.py index 493e40b3..584c9ca0 100644 --- a/tests/cds-rdm/test_transform_versions.py +++ b/tests/cds-rdm/test_transform_versions.py @@ -67,11 +67,18 @@ def _file_dump( } -def _record(): +def _record(pids=None): """Build a minimal RecordEntry-shaped test double.""" - return SimpleNamespace( - access_status="public", - body={"metadata": {"publication_date": "2020-01-01"}}, + body = {"metadata": {"publication_date": "2020-01-01"}} + if pids: + body["pids"] = pids + return SimpleNamespace(access_status="public", body=body) + + +def _external_doi_record(): + """A record double whose DOI provider is "external" (not our prefix).""" + return _record( + pids={"doi": {"identifier": "10.1234/external", "provider": "external"}} ) @@ -181,6 +188,38 @@ def test_versions_no_files_falls_back_to_metadata_only_version(transform): } +def test_versions_external_doi_collapses_into_single_version(transform): + """An external DOI record gets one version with every file, not one per revision.""" + raw_dump_entry = { + "recid": 123, + "files": [ + _file_dump(file_version=1, checksum="checksum-v1"), + _file_dump(file_version=2, checksum="checksum-v2"), + _file_dump(full_name="test.pdf", file_version=1, checksum="checksum-v3"), + ], + } + + versions = transform._versions(raw_dump_entry, _external_doi_record()) + + assert list(versions.keys()) == [2] + assert set(versions[2]["files"]) == {"draft.pdf", "test.pdf"} + # the latest revision of a same-named file wins, same as the regular + # cross-version carry-forward would produce + assert versions[2]["files"]["draft.pdf"]["version"] == 2 + assert versions[2]["files"]["draft.pdf"]["checksum"] == "checksum-v2" + assert versions[2]["files"]["test.pdf"]["version"] == 1 + + +def test_versions_external_doi_with_no_files_falls_back_to_metadata_only(transform): + """An external DOI record with no files still gets the metadata-only fallback.""" + raw_dump_entry = {"recid": 123, "files": []} + + versions = transform._versions(raw_dump_entry, _external_doi_record()) + + assert list(versions.keys()) == [1] + assert versions[1]["files"] == {} + + def test_versions_individual_file_restriction_sets_access_meta(transform): """A file with its own restriction status flags that version as restricted.""" status = ( From 331a82193ffaa63a2ff3ab10fed6d10d5fb4faae Mon Sep 17 00:00:00 2001 From: kpsherva Date: Wed, 23 Sep 2026 18:10:10 +0200 Subject: [PATCH 3/5] change(config): add dirac and experiment mapping --- cds_migrator_kit/rdm/migration_config.py | 15 ++++++++++- .../rdm/records/transform/config.py | 8 +++++- cds_migrator_kit/rdm/streams.yaml | 25 +---------------- cds_migrator_kit/rdm/streams_shelved.yaml | 27 ++++++++++++++----- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/cds_migrator_kit/rdm/migration_config.py b/cds_migrator_kit/rdm/migration_config.py index 4be35615..6735fc48 100644 --- a/cds_migrator_kit/rdm/migration_config.py +++ b/cds_migrator_kit/rdm/migration_config.py @@ -535,7 +535,10 @@ def resolve_record_pid(pid): ### EP Approval configuration only needed for local, it should use cds-rdm config for de/sandbox/prod # =========================== -CDS_CERN_SCIENTIFIC_COMMUNITY_ID = "78b3c4aa-c4e6-4502-8226-67ba2d347afe" +# ATTENTION: please don't modify this local value - the community is created +# via cds-rdm fixtures with this id - if you have another ID locally +# change it in your local db +CDS_CERN_SCIENTIFIC_COMMUNITY_ID = "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" """The id of the CERN Scientific community. This is only a local-dev default: on other instances (sandbox/prod), set the @@ -616,4 +619,14 @@ def resolve_record_pid(pid): "counter_digits": 3, # zero-padding width, e.g. 3 → "001" }, }, + "6a289642-5378-4daf-87b5-bb58af00487a": { + # DIRAC + "label": "EP approval", # shown in UI buttons/headings + "referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug + "report_number": { + "prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP" + "include_year": True, # append the current year after prefix + "counter_digits": 3, # zero-padding width, e.g. 3 → "001" + }, + }, } diff --git a/cds_migrator_kit/rdm/records/transform/config.py b/cds_migrator_kit/rdm/records/transform/config.py index 2bca4a47..38e30519 100644 --- a/cds_migrator_kit/rdm/records/transform/config.py +++ b/cds_migrator_kit/rdm/records/transform/config.py @@ -161,5 +161,11 @@ # Legacy experiment names remapped to vocabulary ids before lookup EXPERIMENT_ALIASES = { - "t2k": "re13", + "t2k": "RE13", + "antares": "RE6", + "dirac": "PS212", + "dirac ps212": "PS212", + "harp ps214": "PS214", + "harp": "PS214", + "dampe": "RE29", } diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index 4603be42..2d677fcd 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -237,33 +237,10 @@ records: communities_ids: - "88a5cdf4-974a-44d4-b145-d19ee6346bb8" - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - lcd_restr: - plots: true - data_dir: cds_migrator_kit/rdm/data/former_exp/lcd_restr - restricted: "True" - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/lcd_restr - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "adc02716-780b-4bba-8e89-6316c9a11cf0" - lcd: - plots: true - create_inclusion_request: true - data_dir: cds_migrator_kit/rdm/data/former_exp/lcd - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/lcd - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "adc02716-780b-4bba-8e89-6316c9a11cf0" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" re29: data_dir: cds_migrator_kit/rdm/data/former_exp/re29 plots: true + create_inclusion_request: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/re29 transform: diff --git a/cds_migrator_kit/rdm/streams_shelved.yaml b/cds_migrator_kit/rdm/streams_shelved.yaml index bd155cc1..371e50a6 100644 --- a/cds_migrator_kit/rdm/streams_shelved.yaml +++ b/cds_migrator_kit/rdm/streams_shelved.yaml @@ -1,16 +1,29 @@ db_uri: postgresql://cds-rdm:cds-rdm@localhost:5432/cds-rdm records: - thesis: - data_dir: cds_migrator_kit/rdm/data/thesis + lcd_restr: + plots: true + data_dir: cds_migrator_kit/rdm/data/former_exp/lcd_restr + restricted: "True" + create_inclusion_request: true extract: - dirpath: cds_migrator_kit/rdm/data/thesis/dump/ + dirpath: cds_migrator_kit/rdm/data/former_exp/lcd_restr transform: - files_dump_dir: cds_migrator_kit/rdm/data/thesis/files/ + files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f - load: - legacy_pids_to_redirect: cds_migrator_kit/rdm/data/thesis/duplicated_pids.json + - "adc02716-780b-4bba-8e89-6316c9a11cf0" + lcd: + plots: true + create_inclusion_request: true + data_dir: cds_migrator_kit/rdm/data/former_exp/lcd + extract: + dirpath: cds_migrator_kit/rdm/data/former_exp/lcd + transform: + files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "adc02716-780b-4bba-8e89-6316c9a11cf0" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" mous: data_dir: cds_migrator_kit/rdm/data/mous extract: From 83b511aa70878bd41f35304c5005c35aabba42bc Mon Sep 17 00:00:00 2001 From: kpsherva Date: Wed, 23 Sep 2026 18:11:00 +0200 Subject: [PATCH 4/5] change(users): add resolving accounts stored in 506 --- cds_migration_progress.html | 862 ++++++++++++++++++ .../xml_processing/models/submitter.py | 8 +- .../xml_processing/rules/access_grants.py | 54 ++ cds_migrator_kit/users/load.py | 13 + cds_migrator_kit/users/transform.py | 7 +- tests/cds-rdm/test_access_grant_emails.py | 93 ++ 6 files changed, 1035 insertions(+), 2 deletions(-) create mode 100644 cds_migration_progress.html create mode 100644 cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py create mode 100644 tests/cds-rdm/test_access_grant_emails.py diff --git a/cds_migration_progress.html b/cds_migration_progress.html new file mode 100644 index 00000000..e9046ed8 --- /dev/null +++ b/cds_migration_progress.html @@ -0,0 +1,862 @@ +CDS Migration Progress, 2015–2026 + + +
+
+
+

CDS migration progress, 2015–2026

+

Legacy CDS (cds.cern.ch) total record count (estimated including restricted records), plotted on the same scale as cumulative records migrated into repository.cern, cumulative books held in catalogue.library.cern since its 21 April 2021 launch, and cumulative videos held on videos.cern.ch since the 2017 migration.

+
+ +
+
+ Legacy CDS total records (est. incl. restricted) + repository.cern cumulative records + catalogue.library.cern cumulative records + videos.cern.ch cumulative records + Migration milestone +
+ +
+ +
+ +
+ View underlying data (Wayback Machine snapshots & migration events) + + + + +
Legacy CDS (cds.cern.ch) — Wayback Machine snapshots
Snapshot datePublic count (scraped)Retroactive adjustmentEst. total incl. restricted (plotted)repository.cern migrated to dateNet of repository.cern
+ + + + +
repository.cern — migration events
DateContent migratedRecords addedCumulative total
+ + + + +
catalogue.library.cern — books created per year (via /api/documents)
PeriodBooks createdCumulative total
+ + + + +
videos.cern.ch — videos created per year (via /api/records)
PeriodVideos createdCumulative total
+
+ +
+ Legacy CDS source: record counts scraped from the CDS homepage as captured by the Internet Archive Wayback + Machine (web.archive.org, cds.cern.ch), which reflects only publicly visible records. The + confirmed current total including restricted records is 592,089 (2 August 2026); comparing that to the + closest scraped snapshot (566,063, 18 July 2026) implies roughly 26,026 restricted records are not shown by + the public counter. Since we have no historical breakdown of restricted records, that offset is extrapolated + as a constant and added to every earlier scraped snapshot to estimate a total-including-restricted series + (shown in the data table); this assumes the restricted share has stayed roughly flat in absolute terms, which + is a simplification. The final point uses the confirmed live total (592,089, 2 August 2026) directly as its + total, rather than a scraped-plus-offset estimate. Every other plotted point carries a + retroactive adjustment, summing three components, so the total line depicts the drop each + repository.cern migration should produce even though the scraped public counter itself never dips at those + dates (that content isn't purged from legacy CDS on migration). Component A covers CERN Bulletin & + Courier, IT department, and HR content (Nov 2025 – Jan 2026): every point from 2015 through Jan 2024 + carries the same flat allowance (~111,000–114,000, an approximate reconstruction of that content's + size) since none of it had migrated yet; it tapers to +63,000 after CERN Bulletin and Courier moved (Nov + 2025), +3,000 after IT department content moved (Dec 2025), and 0 once HR content had also moved (Jan + 2026). Component B covers Staff Association content and the ALEPH/DELPHI/L3/OPAL research records (Jul + 2026): a flat +13,000 on every point up to 29 July 2026, then 0. Component C covers theses (12,000): a flat + +12,000 on every point up to the 15 May 2025 migration, then 0. The three sum, so points before Nov 2024 + carry all three, points from Nov 2024 through early May 2025 carry A and C, and points from late May 2025 + through Jan 2026 carry only B (or A tapering alongside it from Nov 2025). Three points (Nov & Dec 2024, + Oct 2025) have no underlying wayback snapshot and are linearly interpolated between the nearest real ones + before any adjustment is applied; the data table below marks every reconstructed or boosted point. A + net-of-repository.cern figure — the total minus the cumulative amount migrated to + repository.cern by that date — is also computed and shown in the hover tooltip and the data table, for + readers who want legacy CDS's content net of what's already been migrated out. Neither retroactive + adjustment nor the net-of-repository.cern subtraction applies to the videos.cern.ch or catalogue.library.cern + migrations, since that content did not move to repository.cern. + repository.cern source: cumulative total (130,000) built from known migration batch sizes at each event date + (no continuous archive exists for this newer platform), including a ±1,500 reconciliation against the + confirmed live total to account for imprecise per-event estimates. catalogue.library.cern source: current + total holdings (211,323) queried live from the public /api/documents endpoint; yearly + books-created counts from the same endpoint's _created field, from 21 April 2021 onward, account + for 84,508 of that total, with the remaining 126,815 reconstructed as the baseline migrated on launch day (its + original bibliographic creation dates are preserved from the legacy system, so it isn't separately + timestamped as "migrated"). videos.cern.ch source: current total holdings (39,000, including restricted) + queried live; yearly videos-created counts from the public /api/records endpoint's + _created field, from 2017 onward, account for 17,357 of that total, with the remaining 21,643 + reconstructed as the baseline migrated during the 2017 bulk migration from legacy CDS. Both the + catalogue.library.cern and videos.cern.ch lines therefore step from 0 to their reconstructed baseline at + launch and grow to their live totals from there. All four series share a single record-count scale. Data + compiled 2 August 2026. +
+
+
+ + diff --git a/cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py b/cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py index a10886e2..c02cb438 100644 --- a/cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py +++ b/cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py @@ -146,7 +146,7 @@ class SubmitterModel(CdsOverdo): "542__u", # https://cds.cern.ch/record/2285212/export/hm?ln=en "560172", # https://cds.cern.ch/record/383486/export/hm?ln=en "56017a", # https://cds.cern.ch/record/383486/export/hm?ln=en wrong keyword subfield - "506__m", # mail + # "506__m", # e-group/reader's email, used to find/recreate their account "590__b", # abstract translation "590__a", # abstract translation TODO https://cds.cern.ch/record/1476067/export/hm?ln=en "594__a", # https://cds.cern.ch/record/466504/export/hm?ln=en, 455788 @@ -366,3 +366,9 @@ class SubmitterModel(CdsOverdo): bases=(base_model,), entry_point_group="cds_migrator_kit.migrator.rules.submitter", ) + +# Registers the 506 access-grant-emails rule directly on submitter_model, +# after it has been built above - see access_grants.py's module docstring +# for why this must be isolated to this instance instead of the shared +# base_model. +import cds_migrator_kit.rdm.users.transform.xml_processing.rules.access_grants # noqa: E402,F401 diff --git a/cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py b/cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py new file mode 100644 index 00000000..db87e021 --- /dev/null +++ b/cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM access grant accounts migration rules. + +Mirrors the 859__f "submitter" / 906__m "reviewer" rules (see +cds_migrator_kit/transform/xml_processing/rules/base.py and +cds_migrator_kit/rdm/users/transform/xml_processing/rules/reviewers.py): +506 access restriction fields (see the `access_grants` rule in +cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py) +can name a person directly by email, in subfields d/m/a, instead of an +e-group/role name. Those emails need an account pre-created too, so the +record's actual access grant can later be resolved to a `User` +(cds_migrator_kit/rdm/records/transform/entities/parent.py, `resolve_grants`). + +Registered directly on `submitter_model`, not on the shared `base_model`: +research.py/hr.py/it.py/faser_publication.py already register their own, +unrelated "^506[1_]_" rule on their own separate model instances, so this +rule must stay isolated to `submitter_model` to avoid clashing with those. +This is why it is imported at the bottom of +cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py, +after `submitter_model` has been constructed. +""" + +import re + +from dojson.errors import IgnoreKey + +from cds_migrator_kit.rdm.users.transform.xml_processing.models.submitter import ( + submitter_model, +) + +EMAIL_PATTERN = re.compile(r"[^@]+@[^@]+\.[^@]+") + + +@submitter_model.over("access_grant_emails", "^506[1_]_") +def record_access_grant_emails(self, key, value): + """Translate 506 access grant emails, ignoring e-group/role names.""" + emails = self.get("access_grant_emails", []) + for subfield in ("d", "m", "a"): + raw = value.get(subfield) + if isinstance(raw, tuple): + raw = raw[0] + if not raw: + continue + candidate = raw.strip().lower() + if EMAIL_PATTERN.match(candidate) and candidate not in emails: + emails.append(candidate) + self["access_grant_emails"] = emails + raise IgnoreKey("access_grant_emails") diff --git a/cds_migrator_kit/users/load.py b/cds_migrator_kit/users/load.py index 736c8a52..2dc5aa05 100644 --- a/cds_migrator_kit/users/load.py +++ b/cds_migrator_kit/users/load.py @@ -46,6 +46,7 @@ def _load(self, entry): """Load users.""" self._owner(entry) self._reviewers(entry) + self._access_grant_emails(entry) def _validate(self, entry): """Validate data before loading.""" @@ -74,6 +75,18 @@ def _reviewers(self, json_entry): else: self._find_or_create_reviewer_by_name(reviewer) + def _access_grant_emails(self, json_entry): + """Fetch or create accounts for direct emails in access grants. + + 506 access restriction fields can name a person directly by email + (see the `access_grants` rule in + cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py) + instead of an e-group/role name - those need an account too, so the + record's actual access grant can later be resolved to a User. + """ + for email in json_entry.get("access_grant_emails", []): + self._find_or_create_by_email(email) + def _find_or_create_by_email(self, email): """Fetch or create a user account by email.""" if not email: diff --git a/cds_migrator_kit/users/transform.py b/cds_migrator_kit/users/transform.py index ec38a06d..4e501fb0 100644 --- a/cds_migrator_kit/users/transform.py +++ b/cds_migrator_kit/users/transform.py @@ -38,7 +38,12 @@ def _transform(self, entry): timestamp, json_data = record_dump.latest_revision email = json_data.get("submitter") reviewers = json_data.get("reviewers", []) - return {"submitter": email, "reviewers": reviewers} + access_grant_emails = json_data.get("access_grant_emails", []) + return { + "submitter": email, + "reviewers": reviewers, + "access_grant_emails": access_grant_emails, + } except Exception as e: cli_logger.exception(e) diff --git a/tests/cds-rdm/test_access_grant_emails.py b/tests/cds-rdm/test_access_grant_emails.py new file mode 100644 index 00000000..583a1a99 --- /dev/null +++ b/tests/cds-rdm/test_access_grant_emails.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for pre-creating accounts for direct emails found in field 506. + +See cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py +and cds_migrator_kit/users/load.py::CDSSubmitterLoad._access_grant_emails. +""" + +from cds_dojson.marc21.utils import create_record +from invenio_accounts.testutils import create_test_user + +from cds_migrator_kit.rdm.users.transform.xml_processing.models.submitter import ( + submitter_model, +) +from cds_migrator_kit.users.load import CDSSubmitterLoad + + +def _do(marcxml): + return submitter_model.do(create_record(marcxml)) + + +class TestAccessGrantEmailsRule: + """Test the "^506[1_]_" -> "access_grant_emails" dojson rule.""" + + def test_extracts_email_from_subfield_d(self, base_app): + """A direct email in 506__d (e.g. record 2045640) is picked up.""" + with base_app.app_context(): + out = _do(""" + + + cds-edboard-dirac@cern.ch + + + """) + assert out["access_grant_emails"] == ["cds-edboard-dirac@cern.ch"] + + def test_ignores_egroup_names(self, base_app): + """E-group names (subfield m/a, no "@") are not treated as emails.""" + with base_app.app_context(): + out = _do(""" + + + cds-edboard-dirac [CERN] + + + cds-ph-ep-publications-referee-non-lhc [CERN] + + + """) + assert out.get("access_grant_emails", []) == [] + + def test_deduplicates_and_lowercases(self, base_app): + """Repeated/differently-cased emails across occurrences collapse to one.""" + with base_app.app_context(): + out = _do(""" + + + Jane.Doe@cern.ch + + + jane.doe@cern.ch + + + """) + assert out["access_grant_emails"] == ["jane.doe@cern.ch"] + + +class TestAccessGrantEmailsLoad: + """Test CDSSubmitterLoad._access_grant_emails().""" + + def test_finds_existing_account(self, app, db): + """An email matching an existing account is resolved, not recreated.""" + user = create_test_user(email="cds-edboard-dirac@cern.ch") + db.session.commit() + + load = CDSSubmitterLoad(dry_run=True) + load._access_grant_emails( + {"access_grant_emails": ["cds-edboard-dirac@cern.ch"]} + ) + + found = load._find_or_create_by_email("cds-edboard-dirac@cern.ch") + assert found == user.id + + def test_no_emails_is_a_noop(self, app, db): + """No access_grant_emails key/empty list does nothing.""" + load = CDSSubmitterLoad(dry_run=True) + load._access_grant_emails({}) + load._access_grant_emails({"access_grant_emails": []}) From 1e6f4de6be32e9eca1aeb32e9ddf72276c1f9bba Mon Sep 17 00:00:00 2001 From: kpsherva Date: Wed, 23 Sep 2026 18:12:29 +0200 Subject: [PATCH 5/5] change(transform): improve the metadata mapping, deduplicate descriptions --- .../transform/mappers/custom_fields.py | 2 +- .../rdm/records/transform/mappers/metadata.py | 23 +++++- .../rdm/records/transform/models/research.py | 2 +- .../transform/models/research_committee.py | 3 + .../transform/xml_processing/rules/base.py | 2 +- .../xml_processing/rules/research.py | 3 + tests/cds-rdm/test_metadata_mappers.py | 75 +++++++++++++++++++ 7 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 tests/cds-rdm/test_metadata_mappers.py diff --git a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py index 9e346a36..e223b6e5 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py @@ -139,7 +139,7 @@ def apply(self, ctx): "cern:accelerators", [] ) for accelerator in accelerators: - if accelerator.lower().strip() in ["not applicable", "xx", "fermi"]: + if accelerator.lower().strip() in ["not applicable", "xx", "fermi", "cern recognized expt."]: continue result = search_vocabulary(accelerator, "accelerators") if result and result not in accelerators_out: diff --git a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py index 17189b0e..cfd66f7d 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py @@ -183,12 +183,23 @@ def map_value(self, ctx): class TableOfContentsMapper(FieldMapper): - """Folds table_of_content into additional_descriptions.""" + """Folds table_of_content into additional_descriptions. + + Also the single place where the final ``additional_descriptions`` list + is deduplicated: many different dojson rules append to it (520/246/ + 035/500/210/... across base.py and the various collection-specific + rule modules), some legacy records repeat the very same MARC field + (identical text, sometimes only differing in a provenance subfield + nothing here reads), and not every one of those rules remembers to + guard against re-adding an entry already present. Deduplicating once + here, after every rule has run, doesn't depend on each of them getting + that guard right. + """ id = "additional_descriptions" def map_value(self, ctx): - """Move table_of_content into additional_descriptions and return it.""" + """Move table_of_content into additional_descriptions and dedupe.""" dojson_entry = ctx.dojson_entry toc = dojson_entry.get("table_of_content", []) additional_desc = dojson_entry.get("additional_descriptions", []) @@ -198,6 +209,14 @@ def map_value(self, ctx): ) dojson_entry["additional_descriptions"] = additional_desc dojson_entry.pop("table_of_content") + + deduped = [] + for description in dojson_entry.get("additional_descriptions", []): + if description not in deduped: + deduped.append(description) + if deduped: + dojson_entry["additional_descriptions"] = deduped + return dojson_entry.get("additional_descriptions") diff --git a/cds_migrator_kit/rdm/records/transform/models/research.py b/cds_migrator_kit/rdm/records/transform/models/research.py index 994bdc26..c1d4a249 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research.py +++ b/cds_migrator_kit/rdm/records/transform/models/research.py @@ -16,7 +16,7 @@ class ResearchModel(CdsOverdo): """Translation model for research.""" - __query__ = '693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693__:UA5 OR 693__:UA8 OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.a:DUMMY -690C_.a:SCICOM' + __query__ = '693__.e:"DAMPE RE29" OR 693__.e:RE29 OR 693__.e:DAMPE OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693__:UA5 OR 693__:UA8 OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.a:DUMMY -690C_.a:SCICOM -980:BULLETINNEWS' __ignore_keys__ = { "0248_a", diff --git a/cds_migrator_kit/rdm/records/transform/models/research_committee.py b/cds_migrator_kit/rdm/records/transform/models/research_committee.py index 3135e64b..13d08b02 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/models/research_committee.py @@ -42,12 +42,15 @@ class ResearchCommitteeModel(CdsOverdo): "340__a", # TODO ignore material? "540__3", # TODO still ignore the material of the license? "542__3", # TODO still ignore the material of the license? + "594__a", # ATN tag "595__i", # TODO ?? "695__e", # some inspire tag + "695__9", # some inspire tag "700__m", # email of contributor "700__q", # TODO ignore? aliteration of the name, used for searching "700__v", # TODO drop? "773__x", # INSPIRE publication note + "852__a", "8564_8", # file id "8564_s", # bibdoc id "8564_x", # icon thumbnails sizes diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 20b85620..b4120542 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -924,7 +924,7 @@ def related_identifiers_787(self, key, value): "resource_type": {"id": "publication-report"}, }, "complemented by": { - "relation_type": {"id": "issuplementedby"}, + "relation_type": {"id": "issupplementedby"}, "resource_type": {"id": "publication-report"}, }, "preprint": { diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index e1ac8b33..c9c444aa 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -748,6 +748,9 @@ def resource_type(self, key, value): "lhcf_proc": {"id": "publication-conferenceproceeding"}, "lhcf_reports": {"id": "publication-report"}, "conferencepapers": {"id": "publication-conferencepaper"}, + "technical note": {"id": "publication-technicalnote"}, + "minutes": {"id": "publication-meetingminutes"}, + "presentation": {"id": "presentation"}, } try: diff --git a/tests/cds-rdm/test_metadata_mappers.py b/tests/cds-rdm/test_metadata_mappers.py new file mode 100644 index 00000000..223707e1 --- /dev/null +++ b/tests/cds-rdm/test_metadata_mappers.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for cds_migrator_kit/rdm/records/transform/mappers/metadata.py.""" + +from cds_migrator_kit.rdm.records.transform.mappers.base import ( + RecordTransformContext, +) +from cds_migrator_kit.rdm.records.transform.mappers.metadata import ( + TableOfContentsMapper, +) + + +def _ctx(dojson_entry): + return RecordTransformContext(dojson_entry=dojson_entry, raw_dump_entry={}) + + +class TestTableOfContentsMapper: + """Test TableOfContentsMapper.map_value().""" + + def test_deduplicates_identical_entries(self): + """Exact-duplicate descriptions (e.g. a MARC field repeated in the + legacy record) collapse to a single entry.""" + desc = {"description": "Same abstract text.", "type": {"id": "other"}} + dojson_entry = {"additional_descriptions": [desc, dict(desc), dict(desc)]} + + result = TableOfContentsMapper().map_value(_ctx(dojson_entry)) + + assert result == [desc] + + def test_keeps_distinct_entries(self): + """Descriptions that actually differ are all kept, in order.""" + desc_a = {"description": "Series info", "type": {"id": "series-information"}} + desc_b = {"description": "Other note", "type": {"id": "other"}} + dojson_entry = {"additional_descriptions": [desc_a, desc_b]} + + result = TableOfContentsMapper().map_value(_ctx(dojson_entry)) + + assert result == [desc_a, desc_b] + + def test_same_text_different_type_is_not_deduplicated(self): + """Same text under a different type is a distinct entry.""" + desc_a = {"description": "Same text", "type": {"id": "other"}} + desc_b = {"description": "Same text", "type": {"id": "series-information"}} + dojson_entry = {"additional_descriptions": [desc_a, desc_b]} + + result = TableOfContentsMapper().map_value(_ctx(dojson_entry)) + + assert result == [desc_a, desc_b] + + def test_folds_table_of_content_in_before_deduplicating(self): + """table_of_content is folded in, and still deduped against.""" + toc_entry = { + "description": "1. Intro\n2. Results", + "type": {"id": "table-of-contents"}, + } + dojson_entry = { + "table_of_content": "1. Intro\n2. Results", + "additional_descriptions": [dict(toc_entry)], + } + + result = TableOfContentsMapper().map_value(_ctx(dojson_entry)) + + assert result == [toc_entry] + assert "table_of_content" not in dojson_entry + + def test_no_descriptions_returns_falsy(self): + """No additional_descriptions/table_of_content at all is a no-op.""" + result = TableOfContentsMapper().map_value(_ctx({})) + + assert not result