diff --git a/cds_migration_progress.html b/cds_migration_progress.html new file mode 100644 index 00000000..e9046ed8 --- /dev/null +++ b/cds_migration_progress.html @@ -0,0 +1,862 @@ +CDS Migration Progress, 2015–2026 + + +
+
+
+

CDS migration progress, 2015–2026

+

Legacy CDS (cds.cern.ch) total record count (estimated including restricted records), plotted on the same scale as cumulative records migrated into repository.cern, cumulative books held in catalogue.library.cern since its 21 April 2021 launch, and cumulative videos held on videos.cern.ch since the 2017 migration.

+
+ +
+
+ Legacy CDS total records (est. incl. restricted) + repository.cern cumulative records + catalogue.library.cern cumulative records + videos.cern.ch cumulative records + Migration milestone +
+ +
+ +
+ +
+ View underlying data (Wayback Machine snapshots & migration events) + + + + +
Legacy CDS (cds.cern.ch) — Wayback Machine snapshots
Snapshot datePublic count (scraped)Retroactive adjustmentEst. total incl. restricted (plotted)repository.cern migrated to dateNet of repository.cern
+ + + + +
repository.cern — migration events
DateContent migratedRecords addedCumulative total
+ + + + +
catalogue.library.cern — books created per year (via /api/documents)
PeriodBooks createdCumulative total
+ + + + +
videos.cern.ch — videos created per year (via /api/records)
PeriodVideos createdCumulative total
+
+ + +
+
+ + diff --git a/cds_migrator_kit/rdm/migration_config.py b/cds_migrator_kit/rdm/migration_config.py index 4be35615..6735fc48 100644 --- a/cds_migrator_kit/rdm/migration_config.py +++ b/cds_migrator_kit/rdm/migration_config.py @@ -535,7 +535,10 @@ def resolve_record_pid(pid): ### EP Approval configuration only needed for local, it should use cds-rdm config for de/sandbox/prod # =========================== -CDS_CERN_SCIENTIFIC_COMMUNITY_ID = "78b3c4aa-c4e6-4502-8226-67ba2d347afe" +# ATTENTION: please don't modify this local value - the community is created +# via cds-rdm fixtures with this id - if you have another ID locally +# change it in your local db +CDS_CERN_SCIENTIFIC_COMMUNITY_ID = "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" """The id of the CERN Scientific community. This is only a local-dev default: on other instances (sandbox/prod), set the @@ -616,4 +619,14 @@ def resolve_record_pid(pid): "counter_digits": 3, # zero-padding width, e.g. 3 → "001" }, }, + "6a289642-5378-4daf-87b5-bb58af00487a": { + # DIRAC + "label": "EP approval", # shown in UI buttons/headings + "referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug + "report_number": { + "prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP" + "include_year": True, # append the current year after prefix + "counter_digits": 3, # zero-padding width, e.g. 3 → "001" + }, + }, } diff --git a/cds_migrator_kit/rdm/records/transform/config.py b/cds_migrator_kit/rdm/records/transform/config.py index 2bca4a47..38e30519 100644 --- a/cds_migrator_kit/rdm/records/transform/config.py +++ b/cds_migrator_kit/rdm/records/transform/config.py @@ -161,5 +161,11 @@ # Legacy experiment names remapped to vocabulary ids before lookup EXPERIMENT_ALIASES = { - "t2k": "re13", + "t2k": "RE13", + "antares": "RE6", + "dirac": "PS212", + "dirac ps212": "PS212", + "harp ps214": "PS214", + "harp": "PS214", + "dampe": "RE29", } diff --git a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py index 9e346a36..e223b6e5 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py @@ -139,7 +139,7 @@ def apply(self, ctx): "cern:accelerators", [] ) for accelerator in accelerators: - if accelerator.lower().strip() in ["not applicable", "xx", "fermi"]: + if accelerator.lower().strip() in ["not applicable", "xx", "fermi", "cern recognized expt."]: continue result = search_vocabulary(accelerator, "accelerators") if result and result not in accelerators_out: diff --git a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py index 17189b0e..cfd66f7d 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py @@ -183,12 +183,23 @@ def map_value(self, ctx): class TableOfContentsMapper(FieldMapper): - """Folds table_of_content into additional_descriptions.""" + """Folds table_of_content into additional_descriptions. + + Also the single place where the final ``additional_descriptions`` list + is deduplicated: many different dojson rules append to it (520/246/ + 035/500/210/... across base.py and the various collection-specific + rule modules), some legacy records repeat the very same MARC field + (identical text, sometimes only differing in a provenance subfield + nothing here reads), and not every one of those rules remembers to + guard against re-adding an entry already present. Deduplicating once + here, after every rule has run, doesn't depend on each of them getting + that guard right. + """ id = "additional_descriptions" def map_value(self, ctx): - """Move table_of_content into additional_descriptions and return it.""" + """Move table_of_content into additional_descriptions and dedupe.""" dojson_entry = ctx.dojson_entry toc = dojson_entry.get("table_of_content", []) additional_desc = dojson_entry.get("additional_descriptions", []) @@ -198,6 +209,14 @@ def map_value(self, ctx): ) dojson_entry["additional_descriptions"] = additional_desc dojson_entry.pop("table_of_content") + + deduped = [] + for description in dojson_entry.get("additional_descriptions", []): + if description not in deduped: + deduped.append(description) + if deduped: + dojson_entry["additional_descriptions"] = deduped + return dojson_entry.get("additional_descriptions") diff --git a/cds_migrator_kit/rdm/records/transform/models/research.py b/cds_migrator_kit/rdm/records/transform/models/research.py index 994bdc26..c1d4a249 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research.py +++ b/cds_migrator_kit/rdm/records/transform/models/research.py @@ -16,7 +16,7 @@ class ResearchModel(CdsOverdo): """Translation model for research.""" - __query__ = '693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693__:UA5 OR 693__:UA8 OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.a:DUMMY -690C_.a:SCICOM' + __query__ = '693__.e:"DAMPE RE29" OR 693__.e:RE29 OR 693__.e:DAMPE OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693__:UA5 OR 693__:UA8 OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.a:DUMMY -690C_.a:SCICOM -980:BULLETINNEWS' __ignore_keys__ = { "0248_a", diff --git a/cds_migrator_kit/rdm/records/transform/models/research_committee.py b/cds_migrator_kit/rdm/records/transform/models/research_committee.py index 3135e64b..13d08b02 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/models/research_committee.py @@ -42,12 +42,15 @@ class ResearchCommitteeModel(CdsOverdo): "340__a", # TODO ignore material? "540__3", # TODO still ignore the material of the license? "542__3", # TODO still ignore the material of the license? + "594__a", # ATN tag "595__i", # TODO ?? "695__e", # some inspire tag + "695__9", # some inspire tag "700__m", # email of contributor "700__q", # TODO ignore? aliteration of the name, used for searching "700__v", # TODO drop? "773__x", # INSPIRE publication note + "852__a", "8564_8", # file id "8564_s", # bibdoc id "8564_x", # icon thumbnails sizes diff --git a/cds_migrator_kit/rdm/records/transform/transform_versions.py b/cds_migrator_kit/rdm/records/transform/transform_versions.py index 05a1f64d..4298f9c5 100644 --- a/cds_migrator_kit/rdm/records/transform/transform_versions.py +++ b/cds_migrator_kit/rdm/records/transform/transform_versions.py @@ -23,6 +23,12 @@ class RecordVersionsTransform: file A gets uploaded, the later record version still needs to include file B too, so each version's file list is a cumulative snapshot, not just its own delta. + + Exception: when the record's DOI is external (not minted by our + DataCite prefix - see ``RecordEntry._pids()``), we don't own/manage + that DOI, so its legacy per-file version history is collapsed into a + single RDM version holding every file, instead of one RDM version per + legacy file revision - see ``_is_external_doi()``. """ def __init__(self, raw_dump_entry, record, files_dump_dir, plots, migration_logger): @@ -58,6 +64,25 @@ def build(self): own_file_dumps.setdefault(version_number, []).append(file_dump) representative_file.setdefault(version_number, file_dump) + if own_file_dumps and self._is_external_doi(): + # collapse every legacy file revision into a single version - + # its files (see the carry-forward below, still a no-op for one + # version) and its access/publication_date (from the latest + # legacy version's own representative file, i.e. the current + # state) instead of one RDM version per legacy revision. + latest_version_number = max(own_file_dumps) + own_file_dumps = OrderedDict( + [ + ( + latest_version_number, + [fd for fds in own_file_dumps.values() for fd in fds], + ) + ] + ) + representative_file = { + latest_version_number: representative_file[latest_version_number] + } + versions = OrderedDict( ( version_number, @@ -89,6 +114,17 @@ def build(self): return versions + def _is_external_doi(self): + """Return True if this record's DOI isn't minted through our prefix. + + Mirrors the ``provider`` set in ``RecordEntry._pids()`` + (``"external"`` when the DOI doesn't start with + ``current_app.config["DATACITE_PREFIX"]``); ``False`` (no + collapsing) when the record has no DOI at all. + """ + doi = self.record.body.get("pids", {}).get("doi", {}) + return doi.get("provider") == "external" + def _should_skip_file(self, file_dump): if file_dump["subformat"] in FILE_SUBFORMATS_TO_DROP: self.migration_logger.add_information( diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 20b85620..b4120542 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -924,7 +924,7 @@ def related_identifiers_787(self, key, value): "resource_type": {"id": "publication-report"}, }, "complemented by": { - "relation_type": {"id": "issuplementedby"}, + "relation_type": {"id": "issupplementedby"}, "resource_type": {"id": "publication-report"}, }, "preprint": { diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index e1ac8b33..c9c444aa 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -748,6 +748,9 @@ def resource_type(self, key, value): "lhcf_proc": {"id": "publication-conferenceproceeding"}, "lhcf_reports": {"id": "publication-report"}, "conferencepapers": {"id": "publication-conferencepaper"}, + "technical note": {"id": "publication-technicalnote"}, + "minutes": {"id": "publication-meetingminutes"}, + "presentation": {"id": "presentation"}, } try: diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py index 8f9ebf0b..6b79caaf 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py @@ -95,7 +95,6 @@ }, } - def _controlled_subject(term): """Build a subjects entry referencing a controlled-vocabulary term. diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index 4603be42..2d677fcd 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -237,33 +237,10 @@ records: communities_ids: - "88a5cdf4-974a-44d4-b145-d19ee6346bb8" - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - lcd_restr: - plots: true - data_dir: cds_migrator_kit/rdm/data/former_exp/lcd_restr - restricted: "True" - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/lcd_restr - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "adc02716-780b-4bba-8e89-6316c9a11cf0" - lcd: - plots: true - create_inclusion_request: true - data_dir: cds_migrator_kit/rdm/data/former_exp/lcd - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/lcd - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "adc02716-780b-4bba-8e89-6316c9a11cf0" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" re29: data_dir: cds_migrator_kit/rdm/data/former_exp/re29 plots: true + create_inclusion_request: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/re29 transform: diff --git a/cds_migrator_kit/rdm/streams_shelved.yaml b/cds_migrator_kit/rdm/streams_shelved.yaml index bd155cc1..371e50a6 100644 --- a/cds_migrator_kit/rdm/streams_shelved.yaml +++ b/cds_migrator_kit/rdm/streams_shelved.yaml @@ -1,16 +1,29 @@ db_uri: postgresql://cds-rdm:cds-rdm@localhost:5432/cds-rdm records: - thesis: - data_dir: cds_migrator_kit/rdm/data/thesis + lcd_restr: + plots: true + data_dir: cds_migrator_kit/rdm/data/former_exp/lcd_restr + restricted: "True" + create_inclusion_request: true extract: - dirpath: cds_migrator_kit/rdm/data/thesis/dump/ + dirpath: cds_migrator_kit/rdm/data/former_exp/lcd_restr transform: - files_dump_dir: cds_migrator_kit/rdm/data/thesis/files/ + files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f - load: - legacy_pids_to_redirect: cds_migrator_kit/rdm/data/thesis/duplicated_pids.json + - "adc02716-780b-4bba-8e89-6316c9a11cf0" + lcd: + plots: true + create_inclusion_request: true + data_dir: cds_migrator_kit/rdm/data/former_exp/lcd + extract: + dirpath: cds_migrator_kit/rdm/data/former_exp/lcd + transform: + files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "adc02716-780b-4bba-8e89-6316c9a11cf0" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" mous: data_dir: cds_migrator_kit/rdm/data/mous extract: diff --git a/cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py b/cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py index a10886e2..c02cb438 100644 --- a/cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py +++ b/cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py @@ -146,7 +146,7 @@ class SubmitterModel(CdsOverdo): "542__u", # https://cds.cern.ch/record/2285212/export/hm?ln=en "560172", # https://cds.cern.ch/record/383486/export/hm?ln=en "56017a", # https://cds.cern.ch/record/383486/export/hm?ln=en wrong keyword subfield - "506__m", # mail + # "506__m", # e-group/reader's email, used to find/recreate their account "590__b", # abstract translation "590__a", # abstract translation TODO https://cds.cern.ch/record/1476067/export/hm?ln=en "594__a", # https://cds.cern.ch/record/466504/export/hm?ln=en, 455788 @@ -366,3 +366,9 @@ class SubmitterModel(CdsOverdo): bases=(base_model,), entry_point_group="cds_migrator_kit.migrator.rules.submitter", ) + +# Registers the 506 access-grant-emails rule directly on submitter_model, +# after it has been built above - see access_grants.py's module docstring +# for why this must be isolated to this instance instead of the shared +# base_model. +import cds_migrator_kit.rdm.users.transform.xml_processing.rules.access_grants # noqa: E402,F401 diff --git a/cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py b/cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py new file mode 100644 index 00000000..db87e021 --- /dev/null +++ b/cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM access grant accounts migration rules. + +Mirrors the 859__f "submitter" / 906__m "reviewer" rules (see +cds_migrator_kit/transform/xml_processing/rules/base.py and +cds_migrator_kit/rdm/users/transform/xml_processing/rules/reviewers.py): +506 access restriction fields (see the `access_grants` rule in +cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py) +can name a person directly by email, in subfields d/m/a, instead of an +e-group/role name. Those emails need an account pre-created too, so the +record's actual access grant can later be resolved to a `User` +(cds_migrator_kit/rdm/records/transform/entities/parent.py, `resolve_grants`). + +Registered directly on `submitter_model`, not on the shared `base_model`: +research.py/hr.py/it.py/faser_publication.py already register their own, +unrelated "^506[1_]_" rule on their own separate model instances, so this +rule must stay isolated to `submitter_model` to avoid clashing with those. +This is why it is imported at the bottom of +cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py, +after `submitter_model` has been constructed. +""" + +import re + +from dojson.errors import IgnoreKey + +from cds_migrator_kit.rdm.users.transform.xml_processing.models.submitter import ( + submitter_model, +) + +EMAIL_PATTERN = re.compile(r"[^@]+@[^@]+\.[^@]+") + + +@submitter_model.over("access_grant_emails", "^506[1_]_") +def record_access_grant_emails(self, key, value): + """Translate 506 access grant emails, ignoring e-group/role names.""" + emails = self.get("access_grant_emails", []) + for subfield in ("d", "m", "a"): + raw = value.get(subfield) + if isinstance(raw, tuple): + raw = raw[0] + if not raw: + continue + candidate = raw.strip().lower() + if EMAIL_PATTERN.match(candidate) and candidate not in emails: + emails.append(candidate) + self["access_grant_emails"] = emails + raise IgnoreKey("access_grant_emails") diff --git a/cds_migrator_kit/users/load.py b/cds_migrator_kit/users/load.py index 736c8a52..2dc5aa05 100644 --- a/cds_migrator_kit/users/load.py +++ b/cds_migrator_kit/users/load.py @@ -46,6 +46,7 @@ def _load(self, entry): """Load users.""" self._owner(entry) self._reviewers(entry) + self._access_grant_emails(entry) def _validate(self, entry): """Validate data before loading.""" @@ -74,6 +75,18 @@ def _reviewers(self, json_entry): else: self._find_or_create_reviewer_by_name(reviewer) + def _access_grant_emails(self, json_entry): + """Fetch or create accounts for direct emails in access grants. + + 506 access restriction fields can name a person directly by email + (see the `access_grants` rule in + cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py) + instead of an e-group/role name - those need an account too, so the + record's actual access grant can later be resolved to a User. + """ + for email in json_entry.get("access_grant_emails", []): + self._find_or_create_by_email(email) + def _find_or_create_by_email(self, email): """Fetch or create a user account by email.""" if not email: diff --git a/cds_migrator_kit/users/transform.py b/cds_migrator_kit/users/transform.py index ec38a06d..4e501fb0 100644 --- a/cds_migrator_kit/users/transform.py +++ b/cds_migrator_kit/users/transform.py @@ -38,7 +38,12 @@ def _transform(self, entry): timestamp, json_data = record_dump.latest_revision email = json_data.get("submitter") reviewers = json_data.get("reviewers", []) - return {"submitter": email, "reviewers": reviewers} + access_grant_emails = json_data.get("access_grant_emails", []) + return { + "submitter": email, + "reviewers": reviewers, + "access_grant_emails": access_grant_emails, + } except Exception as e: cli_logger.exception(e) diff --git a/tests/cds-rdm/test_access_grant_emails.py b/tests/cds-rdm/test_access_grant_emails.py new file mode 100644 index 00000000..583a1a99 --- /dev/null +++ b/tests/cds-rdm/test_access_grant_emails.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for pre-creating accounts for direct emails found in field 506. + +See cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py +and cds_migrator_kit/users/load.py::CDSSubmitterLoad._access_grant_emails. +""" + +from cds_dojson.marc21.utils import create_record +from invenio_accounts.testutils import create_test_user + +from cds_migrator_kit.rdm.users.transform.xml_processing.models.submitter import ( + submitter_model, +) +from cds_migrator_kit.users.load import CDSSubmitterLoad + + +def _do(marcxml): + return submitter_model.do(create_record(marcxml)) + + +class TestAccessGrantEmailsRule: + """Test the "^506[1_]_" -> "access_grant_emails" dojson rule.""" + + def test_extracts_email_from_subfield_d(self, base_app): + """A direct email in 506__d (e.g. record 2045640) is picked up.""" + with base_app.app_context(): + out = _do(""" + + + cds-edboard-dirac@cern.ch + + + """) + assert out["access_grant_emails"] == ["cds-edboard-dirac@cern.ch"] + + def test_ignores_egroup_names(self, base_app): + """E-group names (subfield m/a, no "@") are not treated as emails.""" + with base_app.app_context(): + out = _do(""" + + + cds-edboard-dirac [CERN] + + + cds-ph-ep-publications-referee-non-lhc [CERN] + + + """) + assert out.get("access_grant_emails", []) == [] + + def test_deduplicates_and_lowercases(self, base_app): + """Repeated/differently-cased emails across occurrences collapse to one.""" + with base_app.app_context(): + out = _do(""" + + + Jane.Doe@cern.ch + + + jane.doe@cern.ch + + + """) + assert out["access_grant_emails"] == ["jane.doe@cern.ch"] + + +class TestAccessGrantEmailsLoad: + """Test CDSSubmitterLoad._access_grant_emails().""" + + def test_finds_existing_account(self, app, db): + """An email matching an existing account is resolved, not recreated.""" + user = create_test_user(email="cds-edboard-dirac@cern.ch") + db.session.commit() + + load = CDSSubmitterLoad(dry_run=True) + load._access_grant_emails( + {"access_grant_emails": ["cds-edboard-dirac@cern.ch"]} + ) + + found = load._find_or_create_by_email("cds-edboard-dirac@cern.ch") + assert found == user.id + + def test_no_emails_is_a_noop(self, app, db): + """No access_grant_emails key/empty list does nothing.""" + load = CDSSubmitterLoad(dry_run=True) + load._access_grant_emails({}) + load._access_grant_emails({"access_grant_emails": []}) diff --git a/tests/cds-rdm/test_metadata_mappers.py b/tests/cds-rdm/test_metadata_mappers.py new file mode 100644 index 00000000..223707e1 --- /dev/null +++ b/tests/cds-rdm/test_metadata_mappers.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for cds_migrator_kit/rdm/records/transform/mappers/metadata.py.""" + +from cds_migrator_kit.rdm.records.transform.mappers.base import ( + RecordTransformContext, +) +from cds_migrator_kit.rdm.records.transform.mappers.metadata import ( + TableOfContentsMapper, +) + + +def _ctx(dojson_entry): + return RecordTransformContext(dojson_entry=dojson_entry, raw_dump_entry={}) + + +class TestTableOfContentsMapper: + """Test TableOfContentsMapper.map_value().""" + + def test_deduplicates_identical_entries(self): + """Exact-duplicate descriptions (e.g. a MARC field repeated in the + legacy record) collapse to a single entry.""" + desc = {"description": "Same abstract text.", "type": {"id": "other"}} + dojson_entry = {"additional_descriptions": [desc, dict(desc), dict(desc)]} + + result = TableOfContentsMapper().map_value(_ctx(dojson_entry)) + + assert result == [desc] + + def test_keeps_distinct_entries(self): + """Descriptions that actually differ are all kept, in order.""" + desc_a = {"description": "Series info", "type": {"id": "series-information"}} + desc_b = {"description": "Other note", "type": {"id": "other"}} + dojson_entry = {"additional_descriptions": [desc_a, desc_b]} + + result = TableOfContentsMapper().map_value(_ctx(dojson_entry)) + + assert result == [desc_a, desc_b] + + def test_same_text_different_type_is_not_deduplicated(self): + """Same text under a different type is a distinct entry.""" + desc_a = {"description": "Same text", "type": {"id": "other"}} + desc_b = {"description": "Same text", "type": {"id": "series-information"}} + dojson_entry = {"additional_descriptions": [desc_a, desc_b]} + + result = TableOfContentsMapper().map_value(_ctx(dojson_entry)) + + assert result == [desc_a, desc_b] + + def test_folds_table_of_content_in_before_deduplicating(self): + """table_of_content is folded in, and still deduped against.""" + toc_entry = { + "description": "1. Intro\n2. Results", + "type": {"id": "table-of-contents"}, + } + dojson_entry = { + "table_of_content": "1. Intro\n2. Results", + "additional_descriptions": [dict(toc_entry)], + } + + result = TableOfContentsMapper().map_value(_ctx(dojson_entry)) + + assert result == [toc_entry] + assert "table_of_content" not in dojson_entry + + def test_no_descriptions_returns_falsy(self): + """No additional_descriptions/table_of_content at all is a no-op.""" + result = TableOfContentsMapper().map_value(_ctx({})) + + assert not result diff --git a/tests/cds-rdm/test_transform_versions.py b/tests/cds-rdm/test_transform_versions.py index 493e40b3..584c9ca0 100644 --- a/tests/cds-rdm/test_transform_versions.py +++ b/tests/cds-rdm/test_transform_versions.py @@ -67,11 +67,18 @@ def _file_dump( } -def _record(): +def _record(pids=None): """Build a minimal RecordEntry-shaped test double.""" - return SimpleNamespace( - access_status="public", - body={"metadata": {"publication_date": "2020-01-01"}}, + body = {"metadata": {"publication_date": "2020-01-01"}} + if pids: + body["pids"] = pids + return SimpleNamespace(access_status="public", body=body) + + +def _external_doi_record(): + """A record double whose DOI provider is "external" (not our prefix).""" + return _record( + pids={"doi": {"identifier": "10.1234/external", "provider": "external"}} ) @@ -181,6 +188,38 @@ def test_versions_no_files_falls_back_to_metadata_only_version(transform): } +def test_versions_external_doi_collapses_into_single_version(transform): + """An external DOI record gets one version with every file, not one per revision.""" + raw_dump_entry = { + "recid": 123, + "files": [ + _file_dump(file_version=1, checksum="checksum-v1"), + _file_dump(file_version=2, checksum="checksum-v2"), + _file_dump(full_name="test.pdf", file_version=1, checksum="checksum-v3"), + ], + } + + versions = transform._versions(raw_dump_entry, _external_doi_record()) + + assert list(versions.keys()) == [2] + assert set(versions[2]["files"]) == {"draft.pdf", "test.pdf"} + # the latest revision of a same-named file wins, same as the regular + # cross-version carry-forward would produce + assert versions[2]["files"]["draft.pdf"]["version"] == 2 + assert versions[2]["files"]["draft.pdf"]["checksum"] == "checksum-v2" + assert versions[2]["files"]["test.pdf"]["version"] == 1 + + +def test_versions_external_doi_with_no_files_falls_back_to_metadata_only(transform): + """An external DOI record with no files still gets the metadata-only fallback.""" + raw_dump_entry = {"recid": 123, "files": []} + + versions = transform._versions(raw_dump_entry, _external_doi_record()) + + assert list(versions.keys()) == [1] + assert versions[1]["files"] == {} + + def test_versions_individual_file_restriction_sets_access_meta(transform): """A file with its own restriction status flags that version as restricted.""" status = (