Fail the build if a schemas directory contains a file the glob does not match

build.py

@@ -4,7 +4,7 @@
 from openMINDS_pipeline.models import DirectoryStructure, Trigger
 from openMINDS_pipeline.resolver import resolve_extends, resolve_categories
 from openMINDS_pipeline.utils import clone_sources, find_schemas, evaluate_versions_to_be_built, clone_central, \
-    qualify_property_names, copy_to_target_directory, update_relevant_versions_from_repo
+    qualify_property_names, copy_to_target_directory, update_relevant_versions_from_repo, check_schema_file_names
 from openMINDS_pipeline.vocab import TypeExtractor, Types, PropertyExtractor, Property, enrich_with_types_and_properties
 from openMINDS_pipeline.schema_comparator import generate_changelogs_and_compatibility_resolution
 
@@ -37,6 +37,8 @@
     clone_sources(modules, version)
 
     # Step 3 - Find all involved schemas
+    # (Fail early if a module carries schema files the build would otherwise ignore silently)
+    check_schema_file_names(directory_structure, modules, version)
     all_schemas = find_schemas(directory_structure, modules, namespaces[version])
 
     # Step 4 - Resolve all "_extends" directives and save to target directory
@@ -51,7 +53,7 @@
     # Step 7 - Extract types from sources and update types.json
     extracted_types = TypeExtractor(directory_structure, version).extract_types(all_schemas)
 
-    # Step 8- Extract properties from source and update properties.json
+    # Step 8 - Extract properties from source and update properties.json
     extracted_properties = PropertyExtractor(directory_structure, version).extract_properties(all_schemas)
 
     # Step 9 - Enrich the schemas with central types and properties information

openMINDS_pipeline/constants.py

@@ -1,6 +1,13 @@
 SCHEMA_FILE_ENDING = ".schema.tpl.json"
 INSTANCE_FILE_ENDING = ".jsonld"
 FIRST_VERSION = "v1.0"
+# Files in a "schemas" directory which do not end in SCHEMA_FILE_ENDING but are tolerated anyway.
+# This prevents new types being added to old releases.
+# Do not add an entry here for a version whose content can still be corrected: just fix the file name.
+KNOWN_UNMATCHED_SCHEMA_FILES = {
+    ("v3.0", "core", "schemas/digitalIdentifier/genericIdentifier.tpl.json"),
+    ("v4.0", "core", "schemas/digitalIdentifier/genericIdentifier.tpl.json")
+}
 # Replacement mappings
 NAMESPACE_PATTERNS = {
     r"https://openminds.ebrains.eu/vocab/": "props:",

openMINDS_pipeline/utils.py

@@ -10,7 +10,7 @@
 from packaging.utils import canonicalize_version
 from packaging.version import Version
 
-from openMINDS_pipeline.constants import SCHEMA_FILE_ENDING
+from openMINDS_pipeline.constants import SCHEMA_FILE_ENDING, KNOWN_UNMATCHED_SCHEMA_FILES
 from openMINDS_pipeline.models import Trigger, OpenMINDSModule, DirectoryStructure, SchemaStructure
 from openMINDS_pipeline.resolver import TEMPLATE_PROPERTY_TYPE
 
@@ -142,6 +142,37 @@ def _evaluate_branch_and_commit_for_dynamic_instances(module_spec:OpenMINDSModul
     module_spec.commit = branch_to_commit[module_spec.branch]
 
 
+def check_schema_file_names(directory_structure: DirectoryStructure, modules: Dict[str, OpenMINDSModule], version: str) -> None:
+    """
+    Check that every file in the "schemas" directory of a module can actually be found by find_schemas.
+
+    find_schemas globs for SCHEMA_FILE_ENDING, so a file which does not end in it is never processed 
+    and the type will be silently missing from the built output.
+
+    Raises a ValueError if any such misnamed files are found, unless there is a specific exception in KNOWN_UNMATCHED_SCHEMA_FILES
+    """
+    unmatched = []
+    for module in sorted(modules):
+        absolute_schema_group_src_dir = directory_structure.source_schema_directory(module)
+        if not os.path.isdir(absolute_schema_group_src_dir):
+            # A module which does not define any schemas at all, e.g. because it is not released yet
+            continue
+        module_directory = os.path.join(directory_structure.source_directory, module)
+        for root, _, file_names in os.walk(absolute_schema_group_src_dir):
+            for file_name in sorted(file_names):
+                if file_name.endswith(SCHEMA_FILE_ENDING):
+                    continue
+                relative_path = os.path.relpath(os.path.join(root, file_name), module_directory)
+                if (version, module, relative_path) in KNOWN_UNMATCHED_SCHEMA_FILES:
+                    print(f"Tolerating the known unmatched file {module}/{relative_path} of {version}")
+                else:
+                    unmatched.append(f"{module}/{relative_path}")
+    if unmatched:
+        raise ValueError(f"The following {len(unmatched)} file(s) of {version} are in a 'schemas' directory but do not "
+                         f"end in '{SCHEMA_FILE_ENDING}', which means they would be excluded from the build without "
+                         f"any warning:\n  " + "\n  ".join(sorted(unmatched)))
+
+
 def find_schemas(directory_structure: DirectoryStructure, modules: Dict[str, OpenMINDSModule], namespaces: Dict[str, str]) -> List[SchemaStructure]:
     schema_information = []
     for schema_group in directory_structure.find_resource_directories(file_ending=SCHEMA_FILE_ENDING):