From 68c106381d76b3393c1438db786b1106579cf661 Mon Sep 17 00:00:00 2001 From: kpsherva Date: Mon, 21 Sep 2026 17:08:54 +0200 Subject: [PATCH 1/2] add(subjects): more specific content types as subjects # Conflicts: # cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py --- .../records/transform/xml_processing/rules/research_committee.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py index 8f9ebf0b..6b79caaf 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py @@ -95,7 +95,6 @@ }, } - def _controlled_subject(term): """Build a subjects entry referencing a controlled-vocabulary term. From df2f3a6ff074d0255b9b950ee5d7ba765a6f818f Mon Sep 17 00:00:00 2001 From: kpsherva Date: Wed, 23 Sep 2026 18:04:04 +0200 Subject: [PATCH 2/2] change(transform): don't create separate versions for external DOIs * closes https://github.com/CERNDocumentServer/cds-migrator-kit/issues/609 --- .../records/transform/transform_versions.py | 36 ++++++++++++++ tests/cds-rdm/test_transform_versions.py | 47 +++++++++++++++++-- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/transform_versions.py b/cds_migrator_kit/rdm/records/transform/transform_versions.py index 05a1f64d..4298f9c5 100644 --- a/cds_migrator_kit/rdm/records/transform/transform_versions.py +++ b/cds_migrator_kit/rdm/records/transform/transform_versions.py @@ -23,6 +23,12 @@ class RecordVersionsTransform: file A gets uploaded, the later record version still needs to include file B too, so each version's file list is a cumulative snapshot, not just its own delta. + + Exception: when the record's DOI is external (not minted by our + DataCite prefix - see ``RecordEntry._pids()``), we don't own/manage + that DOI, so its legacy per-file version history is collapsed into a + single RDM version holding every file, instead of one RDM version per + legacy file revision - see ``_is_external_doi()``. """ def __init__(self, raw_dump_entry, record, files_dump_dir, plots, migration_logger): @@ -58,6 +64,25 @@ def build(self): own_file_dumps.setdefault(version_number, []).append(file_dump) representative_file.setdefault(version_number, file_dump) + if own_file_dumps and self._is_external_doi(): + # collapse every legacy file revision into a single version - + # its files (see the carry-forward below, still a no-op for one + # version) and its access/publication_date (from the latest + # legacy version's own representative file, i.e. the current + # state) instead of one RDM version per legacy revision. + latest_version_number = max(own_file_dumps) + own_file_dumps = OrderedDict( + [ + ( + latest_version_number, + [fd for fds in own_file_dumps.values() for fd in fds], + ) + ] + ) + representative_file = { + latest_version_number: representative_file[latest_version_number] + } + versions = OrderedDict( ( version_number, @@ -89,6 +114,17 @@ def build(self): return versions + def _is_external_doi(self): + """Return True if this record's DOI isn't minted through our prefix. + + Mirrors the ``provider`` set in ``RecordEntry._pids()`` + (``"external"`` when the DOI doesn't start with + ``current_app.config["DATACITE_PREFIX"]``); ``False`` (no + collapsing) when the record has no DOI at all. + """ + doi = self.record.body.get("pids", {}).get("doi", {}) + return doi.get("provider") == "external" + def _should_skip_file(self, file_dump): if file_dump["subformat"] in FILE_SUBFORMATS_TO_DROP: self.migration_logger.add_information( diff --git a/tests/cds-rdm/test_transform_versions.py b/tests/cds-rdm/test_transform_versions.py index 493e40b3..584c9ca0 100644 --- a/tests/cds-rdm/test_transform_versions.py +++ b/tests/cds-rdm/test_transform_versions.py @@ -67,11 +67,18 @@ def _file_dump( } -def _record(): +def _record(pids=None): """Build a minimal RecordEntry-shaped test double.""" - return SimpleNamespace( - access_status="public", - body={"metadata": {"publication_date": "2020-01-01"}}, + body = {"metadata": {"publication_date": "2020-01-01"}} + if pids: + body["pids"] = pids + return SimpleNamespace(access_status="public", body=body) + + +def _external_doi_record(): + """A record double whose DOI provider is "external" (not our prefix).""" + return _record( + pids={"doi": {"identifier": "10.1234/external", "provider": "external"}} ) @@ -181,6 +188,38 @@ def test_versions_no_files_falls_back_to_metadata_only_version(transform): } +def test_versions_external_doi_collapses_into_single_version(transform): + """An external DOI record gets one version with every file, not one per revision.""" + raw_dump_entry = { + "recid": 123, + "files": [ + _file_dump(file_version=1, checksum="checksum-v1"), + _file_dump(file_version=2, checksum="checksum-v2"), + _file_dump(full_name="test.pdf", file_version=1, checksum="checksum-v3"), + ], + } + + versions = transform._versions(raw_dump_entry, _external_doi_record()) + + assert list(versions.keys()) == [2] + assert set(versions[2]["files"]) == {"draft.pdf", "test.pdf"} + # the latest revision of a same-named file wins, same as the regular + # cross-version carry-forward would produce + assert versions[2]["files"]["draft.pdf"]["version"] == 2 + assert versions[2]["files"]["draft.pdf"]["checksum"] == "checksum-v2" + assert versions[2]["files"]["test.pdf"]["version"] == 1 + + +def test_versions_external_doi_with_no_files_falls_back_to_metadata_only(transform): + """An external DOI record with no files still gets the metadata-only fallback.""" + raw_dump_entry = {"recid": 123, "files": []} + + versions = transform._versions(raw_dump_entry, _external_doi_record()) + + assert list(versions.keys()) == [1] + assert versions[1]["files"] == {} + + def test_versions_individual_file_restriction_sets_access_meta(transform): """A file with its own restriction status flags that version as restricted.""" status = (