Former experiments fixes

cds_migrator_kit/rdm/migration_config.py

@@ -1,3 +1,3 @@
 """
 InvenioRDM settings for CDS-RDM project.
 
@@ -535,7 +535,10 @@
 
 ### EP Approval configuration only needed for local, it should use cds-rdm config for de/sandbox/prod
 # ===========================
-CDS_CERN_SCIENTIFIC_COMMUNITY_ID = "78b3c4aa-c4e6-4502-8226-67ba2d347afe"
+# ATTENTION: please don't modify this local value - the community is created
+# via cds-rdm fixtures with this id - if you have another ID locally
+# change it in your local db
+CDS_CERN_SCIENTIFIC_COMMUNITY_ID = "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
 """The id of the CERN Scientific community.
 
 This is only a local-dev default: on other instances (sandbox/prod), set the
@@ -616,4 +619,14 @@
             "counter_digits": 3,  # zero-padding width, e.g. 3 → "001"
         },
     },
+    "6a289642-5378-4daf-87b5-bb58af00487a": {
+            # DIRAC
+            "label": "EP approval",  # shown in UI buttons/headings
+            "referee_group": "cds-ph-ep-publications-referee-non-lhc",  # CERN e-group slug
+            "report_number": {
+                "prefix": "CERN-EP",  # literal prefix, e.g. "CERN-EP"
+                "include_year": True,  # append the current year after prefix
+                "counter_digits": 3,  # zero-padding width, e.g. 3 → "001"
+            },
+    },
 }

cds_migrator_kit/rdm/records/transform/config.py

@@ -161,5 +161,11 @@
 
 # Legacy experiment names remapped to vocabulary ids before lookup
 EXPERIMENT_ALIASES = {
-    "t2k": "re13",
+    "t2k": "RE13",
+    "antares": "RE6",
+    "dirac": "PS212",
+    "dirac ps212": "PS212",
+    "harp ps214": "PS214",
+    "harp": "PS214",
+    "dampe": "RE29",
 }

cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2022-2026 CERN.
@@ -139,7 +139,7 @@
             "cern:accelerators", []
         )
         for accelerator in accelerators:
-            if accelerator.lower().strip() in ["not applicable", "xx", "fermi"]:
+            if accelerator.lower().strip() in ["not applicable", "xx", "fermi", "cern recognized expt."]:
                 continue
             result = search_vocabulary(accelerator, "accelerators")
             if result and result not in accelerators_out:

cds_migrator_kit/rdm/records/transform/mappers/metadata.py

@@ -183,12 +183,23 @@ def map_value(self, ctx):
 
 
 class TableOfContentsMapper(FieldMapper):
-    """Folds table_of_content into additional_descriptions."""
+    """Folds table_of_content into additional_descriptions.
+
+    Also the single place where the final ``additional_descriptions`` list
+    is deduplicated: many different dojson rules append to it (520/246/
+    035/500/210/... across base.py and the various collection-specific
+    rule modules), some legacy records repeat the very same MARC field
+    (identical text, sometimes only differing in a provenance subfield
+    nothing here reads), and not every one of those rules remembers to
+    guard against re-adding an entry already present. Deduplicating once
+    here, after every rule has run, doesn't depend on each of them getting
+    that guard right.
+    """
 
     id = "additional_descriptions"
 
     def map_value(self, ctx):
-        """Move table_of_content into additional_descriptions and return it."""
+        """Move table_of_content into additional_descriptions and dedupe."""
         dojson_entry = ctx.dojson_entry
         toc = dojson_entry.get("table_of_content", [])
         additional_desc = dojson_entry.get("additional_descriptions", [])
@@ -198,6 +209,14 @@ def map_value(self, ctx):
             )
             dojson_entry["additional_descriptions"] = additional_desc
             dojson_entry.pop("table_of_content")
+
+        deduped = []
+        for description in dojson_entry.get("additional_descriptions", []):
+            if description not in deduped:
+                deduped.append(description)
+        if deduped:
+            dojson_entry["additional_descriptions"] = deduped
+
         return dojson_entry.get("additional_descriptions")
 
 

cds_migrator_kit/rdm/records/transform/models/research.py

@@ -16,7 +16,7 @@
 class ResearchModel(CdsOverdo):
     """Translation model for research."""
 
-    __query__ = '693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693__:UA5 OR 693__:UA8 OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.a:DUMMY -690C_.a:SCICOM'
+    __query__ = '693__.e:"DAMPE RE29" OR 693__.e:RE29 OR 693__.e:DAMPE OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693__:UA5 OR 693__:UA8 OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.a:DUMMY -690C_.a:SCICOM -980:BULLETINNEWS'
 
     __ignore_keys__ = {
         "0248_a",

cds_migrator_kit/rdm/records/transform/models/research_committee.py

@@ -42,12 +42,15 @@ class ResearchCommitteeModel(CdsOverdo):
         "340__a",  # TODO ignore material?
         "540__3",  # TODO still ignore the material of the license?
         "542__3",  # TODO still ignore the material of the license?
+        "594__a",  # ATN tag
         "595__i",  # TODO ??
         "695__e",  # some inspire tag
+        "695__9",  # some inspire tag
         "700__m",  # email of contributor
         "700__q",  # TODO ignore? aliteration of the name, used for searching
         "700__v",  # TODO drop?
         "773__x",  # INSPIRE publication note
+        "852__a",
         "8564_8",  # file id
         "8564_s",  # bibdoc id
         "8564_x",  # icon thumbnails sizes

cds_migrator_kit/rdm/records/transform/transform_versions.py

@@ -23,6 +23,12 @@ class RecordVersionsTransform:
     file A gets uploaded, the later record version still needs to include
     file B too, so each version's file list is a cumulative snapshot, not
     just its own delta.
+
+    Exception: when the record's DOI is external (not minted by our
+    DataCite prefix - see ``RecordEntry._pids()``), we don't own/manage
+    that DOI, so its legacy per-file version history is collapsed into a
+    single RDM version holding every file, instead of one RDM version per
+    legacy file revision - see ``_is_external_doi()``.
     """
 
     def __init__(self, raw_dump_entry, record, files_dump_dir, plots, migration_logger):
@@ -58,6 +64,25 @@ def build(self):
             own_file_dumps.setdefault(version_number, []).append(file_dump)
             representative_file.setdefault(version_number, file_dump)
 
+        if own_file_dumps and self._is_external_doi():
+            # collapse every legacy file revision into a single version -
+            # its files (see the carry-forward below, still a no-op for one
+            # version) and its access/publication_date (from the latest
+            # legacy version's own representative file, i.e. the current
+            # state) instead of one RDM version per legacy revision.
+            latest_version_number = max(own_file_dumps)
+            own_file_dumps = OrderedDict(
+                [
+                    (
+                        latest_version_number,
+                        [fd for fds in own_file_dumps.values() for fd in fds],
+                    )
+                ]
+            )
+            representative_file = {
+                latest_version_number: representative_file[latest_version_number]
+            }
+
         versions = OrderedDict(
             (
                 version_number,
@@ -89,6 +114,17 @@ def build(self):
 
         return versions
 
+    def _is_external_doi(self):
+        """Return True if this record's DOI isn't minted through our prefix.
+
+        Mirrors the ``provider`` set in ``RecordEntry._pids()``
+        (``"external"`` when the DOI doesn't start with
+        ``current_app.config["DATACITE_PREFIX"]``); ``False`` (no
+        collapsing) when the record has no DOI at all.
+        """
+        doi = self.record.body.get("pids", {}).get("doi", {})
+        return doi.get("provider") == "external"
+
     def _should_skip_file(self, file_dump):
         if file_dump["subformat"] in FILE_SUBFORMATS_TO_DROP:
             self.migration_logger.add_information(

cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py

@@ -924,7 +924,7 @@ def related_identifiers_787(self, key, value):
             "resource_type": {"id": "publication-report"},
         },
         "complemented by": {
-            "relation_type": {"id": "issuplementedby"},
+            "relation_type": {"id": "issupplementedby"},
             "resource_type": {"id": "publication-report"},
         },
         "preprint": {

cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py

@@ -748,6 +748,9 @@ def resource_type(self, key, value):
         "lhcf_proc": {"id": "publication-conferenceproceeding"},
         "lhcf_reports": {"id": "publication-report"},
         "conferencepapers": {"id": "publication-conferencepaper"},
+        "technical note": {"id": "publication-technicalnote"},
+        "minutes": {"id": "publication-meetingminutes"},
+        "presentation": {"id": "presentation"},
     }
 
     try:

cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2026 CERN.
@@ -95,7 +95,6 @@
     },
 }
 
-
 def _controlled_subject(term):
     """Build a subjects entry referencing a controlled-vocabulary term.
 

cds_migrator_kit/rdm/streams.yaml

@@ -237,33 +237,10 @@ records:
       communities_ids:
         - "88a5cdf4-974a-44d4-b145-d19ee6346bb8"
         - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
-  lcd_restr:
-    plots: true
-    data_dir: cds_migrator_kit/rdm/data/former_exp/lcd_restr
-    restricted: "True"
-    create_inclusion_request: true
-    extract:
-      dirpath: cds_migrator_kit/rdm/data/former_exp/lcd_restr
-    transform:
-      files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/
-      missing_users: cds_migrator_kit/rdm/data/users
-      communities_ids:
-        - "adc02716-780b-4bba-8e89-6316c9a11cf0"
-  lcd:
-    plots: true
-    create_inclusion_request: true
-    data_dir: cds_migrator_kit/rdm/data/former_exp/lcd
-    extract:
-      dirpath: cds_migrator_kit/rdm/data/former_exp/lcd
-    transform:
-      files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/
-      missing_users: cds_migrator_kit/rdm/data/users
-      communities_ids:
-        - "adc02716-780b-4bba-8e89-6316c9a11cf0"
-        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
   re29:
     data_dir: cds_migrator_kit/rdm/data/former_exp/re29
     plots: true
+    create_inclusion_request: true
     extract:
       dirpath: cds_migrator_kit/rdm/data/former_exp/re29
     transform:

cds_migrator_kit/rdm/streams_shelved.yaml

@@ -1,16 +1,29 @@
 db_uri: postgresql://cds-rdm:cds-rdm@localhost:5432/cds-rdm
 records:
-  thesis:
-    data_dir: cds_migrator_kit/rdm/data/thesis
+  lcd_restr:
+    plots: true
+    data_dir: cds_migrator_kit/rdm/data/former_exp/lcd_restr
+    restricted: "True"
+    create_inclusion_request: true
     extract:
-      dirpath: cds_migrator_kit/rdm/data/thesis/dump/
+      dirpath: cds_migrator_kit/rdm/data/former_exp/lcd_restr
     transform:
-      files_dump_dir: cds_migrator_kit/rdm/data/thesis/files/
+      files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/
       missing_users: cds_migrator_kit/rdm/data/users
       communities_ids:
-        - c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f
-    load:
-      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/thesis/duplicated_pids.json
+        - "adc02716-780b-4bba-8e89-6316c9a11cf0"
+  lcd:
+    plots: true
+    create_inclusion_request: true
+    data_dir: cds_migrator_kit/rdm/data/former_exp/lcd
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/former_exp/lcd
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "adc02716-780b-4bba-8e89-6316c9a11cf0"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
   mous:
     data_dir: cds_migrator_kit/rdm/data/mous
     extract:

cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py

@@ -146,7 +146,7 @@ class SubmitterModel(CdsOverdo):
         "542__u",  # https://cds.cern.ch/record/2285212/export/hm?ln=en
         "560172",  # https://cds.cern.ch/record/383486/export/hm?ln=en
         "56017a",  # https://cds.cern.ch/record/383486/export/hm?ln=en wrong keyword subfield
-        "506__m",  # mail
+        # "506__m",  # e-group/reader's email, used to find/recreate their account
         "590__b",  # abstract translation
         "590__a",  # abstract translation TODO https://cds.cern.ch/record/1476067/export/hm?ln=en
         "594__a",  # https://cds.cern.ch/record/466504/export/hm?ln=en, 455788
@@ -366,3 +366,9 @@ class SubmitterModel(CdsOverdo):
     bases=(base_model,),
     entry_point_group="cds_migrator_kit.migrator.rules.submitter",
 )
+
+# Registers the 506 access-grant-emails rule directly on submitter_model,
+# after it has been built above - see access_grants.py's module docstring
+# for why this must be isolated to this instance instead of the shared
+# base_model.
+import cds_migrator_kit.rdm.users.transform.xml_processing.rules.access_grants  # noqa: E402,F401

cds_migrator_kit/rdm/users/transform/xml_processing/rules/access_grants.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""CDS-RDM access grant accounts migration rules.
+
+Mirrors the 859__f "submitter" / 906__m "reviewer" rules (see
+cds_migrator_kit/transform/xml_processing/rules/base.py and
+cds_migrator_kit/rdm/users/transform/xml_processing/rules/reviewers.py):
+506 access restriction fields (see the `access_grants` rule in
+cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py)
+can name a person directly by email, in subfields d/m/a, instead of an
+e-group/role name. Those emails need an account pre-created too, so the
+record's actual access grant can later be resolved to a `User`
+(cds_migrator_kit/rdm/records/transform/entities/parent.py, `resolve_grants`).
+
+Registered directly on `submitter_model`, not on the shared `base_model`:
+research.py/hr.py/it.py/faser_publication.py already register their own,
+unrelated "^506[1_]_" rule on their own separate model instances, so this
+rule must stay isolated to `submitter_model` to avoid clashing with those.
+This is why it is imported at the bottom of
+cds_migrator_kit/rdm/users/transform/xml_processing/models/submitter.py,
+after `submitter_model` has been constructed.
+"""
+
+import re
+
+from dojson.errors import IgnoreKey
+
+from cds_migrator_kit.rdm.users.transform.xml_processing.models.submitter import (
+    submitter_model,
+)
+
+EMAIL_PATTERN = re.compile(r"[^@]+@[^@]+\.[^@]+")
+
+
+@submitter_model.over("access_grant_emails", "^506[1_]_")
+def record_access_grant_emails(self, key, value):
+    """Translate 506 access grant emails, ignoring e-group/role names."""
+    emails = self.get("access_grant_emails", [])
+    for subfield in ("d", "m", "a"):
+        raw = value.get(subfield)
+        if isinstance(raw, tuple):
+            raw = raw[0]
+        if not raw:
+            continue
+        candidate = raw.strip().lower()
+        if EMAIL_PATTERN.match(candidate) and candidate not in emails:
+            emails.append(candidate)
+    self["access_grant_emails"] = emails
+    raise IgnoreKey("access_grant_emails")

cds_migrator_kit/users/load.py

@@ -46,6 +46,7 @@
         """Load users."""
         self._owner(entry)
         self._reviewers(entry)
+        self._access_grant_emails(entry)
 
     def _validate(self, entry):
         """Validate data before loading."""
@@ -74,6 +75,18 @@
             else:
                 self._find_or_create_reviewer_by_name(reviewer)
 
+    def _access_grant_emails(self, json_entry):
+        """Fetch or create accounts for direct emails in access grants.
+
+        506 access restriction fields can name a person directly by email
+        (see the `access_grants` rule in
+        cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py)
+        instead of an e-group/role name - those need an account too, so the
+        record's actual access grant can later be resolved to a User.
+        """
+        for email in json_entry.get("access_grant_emails", []):
+            self._find_or_create_by_email(email)
+
     def _find_or_create_by_email(self, email):
         """Fetch or create a user account by email."""
         if not email:
@@ -178,7 +191,7 @@
         later on. Only fills in missing values, never overwrites an
         existing (e.g. already CERN-synced) profile.
         """
         user = User.query.get(user_id)
         if user is None:
             return
         profile = dict(user.user_profile or {})

cds_migrator_kit/users/transform.py

@@ -38,7 +38,12 @@ def _transform(self, entry):
             timestamp, json_data = record_dump.latest_revision
             email = json_data.get("submitter")
             reviewers = json_data.get("reviewers", [])
-            return {"submitter": email, "reviewers": reviewers}
+            access_grant_emails = json_data.get("access_grant_emails", [])
+            return {
+                "submitter": email,
+                "reviewers": reviewers,
+                "access_grant_emails": access_grant_emails,
+            }
         except Exception as e:
             cli_logger.exception(e)