From 58546b69eff8219f1a96dc2f69208af28ce77776 Mon Sep 17 00:00:00 2001 From: Andrew Davison Date: Fri, 4 Sep 2026 16:01:48 +0200 Subject: [PATCH] Fail the build if a schemas directory contains a file the glob does not match. This prevents misnamed files from being silently ignored. --- build.py | 6 ++++-- openMINDS_pipeline/constants.py | 7 +++++++ openMINDS_pipeline/utils.py | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 41fb5293..b9cdbea5 100644 --- a/build.py +++ b/build.py @@ -4,7 +4,7 @@ from openMINDS_pipeline.models import DirectoryStructure, Trigger from openMINDS_pipeline.resolver import resolve_extends, resolve_categories from openMINDS_pipeline.utils import clone_sources, find_schemas, evaluate_versions_to_be_built, clone_central, \ - qualify_property_names, copy_to_target_directory, update_relevant_versions_from_repo + qualify_property_names, copy_to_target_directory, update_relevant_versions_from_repo, check_schema_file_names from openMINDS_pipeline.vocab import TypeExtractor, Types, PropertyExtractor, Property, enrich_with_types_and_properties from openMINDS_pipeline.schema_comparator import generate_changelogs_and_compatibility_resolution @@ -37,6 +37,8 @@ clone_sources(modules, version) # Step 3 - Find all involved schemas + # (Fail early if a module carries schema files the build would otherwise ignore silently) + check_schema_file_names(directory_structure, modules, version) all_schemas = find_schemas(directory_structure, modules, namespaces[version]) # Step 4 - Resolve all "_extends" directives and save to target directory @@ -51,7 +53,7 @@ # Step 7 - Extract types from sources and update types.json extracted_types = TypeExtractor(directory_structure, version).extract_types(all_schemas) - # Step 8- Extract properties from source and update properties.json + # Step 8 - Extract properties from source and update properties.json extracted_properties = PropertyExtractor(directory_structure, version).extract_properties(all_schemas) # Step 9 - Enrich the schemas with central types and properties information diff --git a/openMINDS_pipeline/constants.py b/openMINDS_pipeline/constants.py index c1511257..8363dc0a 100644 --- a/openMINDS_pipeline/constants.py +++ b/openMINDS_pipeline/constants.py @@ -1,6 +1,13 @@ SCHEMA_FILE_ENDING = ".schema.tpl.json" INSTANCE_FILE_ENDING = ".jsonld" FIRST_VERSION = "v1.0" +# Files in a "schemas" directory which do not end in SCHEMA_FILE_ENDING but are tolerated anyway. +# This prevents new types being added to old releases. +# Do not add an entry here for a version whose content can still be corrected: just fix the file name. +KNOWN_UNMATCHED_SCHEMA_FILES = { + ("v3.0", "core", "schemas/digitalIdentifier/genericIdentifier.tpl.json"), + ("v4.0", "core", "schemas/digitalIdentifier/genericIdentifier.tpl.json") +} # Replacement mappings NAMESPACE_PATTERNS = { r"https://openminds.ebrains.eu/vocab/": "props:", diff --git a/openMINDS_pipeline/utils.py b/openMINDS_pipeline/utils.py index b3141c07..e23d700d 100644 --- a/openMINDS_pipeline/utils.py +++ b/openMINDS_pipeline/utils.py @@ -10,7 +10,7 @@ from packaging.utils import canonicalize_version from packaging.version import Version -from openMINDS_pipeline.constants import SCHEMA_FILE_ENDING +from openMINDS_pipeline.constants import SCHEMA_FILE_ENDING, KNOWN_UNMATCHED_SCHEMA_FILES from openMINDS_pipeline.models import Trigger, OpenMINDSModule, DirectoryStructure, SchemaStructure from openMINDS_pipeline.resolver import TEMPLATE_PROPERTY_TYPE @@ -142,6 +142,37 @@ def _evaluate_branch_and_commit_for_dynamic_instances(module_spec:OpenMINDSModul module_spec.commit = branch_to_commit[module_spec.branch] +def check_schema_file_names(directory_structure: DirectoryStructure, modules: Dict[str, OpenMINDSModule], version: str) -> None: + """ + Check that every file in the "schemas" directory of a module can actually be found by find_schemas. + + find_schemas globs for SCHEMA_FILE_ENDING, so a file which does not end in it is never processed + and the type will be silently missing from the built output. + + Raises a ValueError if any such misnamed files are found, unless there is a specific exception in KNOWN_UNMATCHED_SCHEMA_FILES + """ + unmatched = [] + for module in sorted(modules): + absolute_schema_group_src_dir = directory_structure.source_schema_directory(module) + if not os.path.isdir(absolute_schema_group_src_dir): + # A module which does not define any schemas at all, e.g. because it is not released yet + continue + module_directory = os.path.join(directory_structure.source_directory, module) + for root, _, file_names in os.walk(absolute_schema_group_src_dir): + for file_name in sorted(file_names): + if file_name.endswith(SCHEMA_FILE_ENDING): + continue + relative_path = os.path.relpath(os.path.join(root, file_name), module_directory) + if (version, module, relative_path) in KNOWN_UNMATCHED_SCHEMA_FILES: + print(f"Tolerating the known unmatched file {module}/{relative_path} of {version}") + else: + unmatched.append(f"{module}/{relative_path}") + if unmatched: + raise ValueError(f"The following {len(unmatched)} file(s) of {version} are in a 'schemas' directory but do not " + f"end in '{SCHEMA_FILE_ENDING}', which means they would be excluded from the build without " + f"any warning:\n " + "\n ".join(sorted(unmatched))) + + def find_schemas(directory_structure: DirectoryStructure, modules: Dict[str, OpenMINDSModule], namespaces: Dict[str, str]) -> List[SchemaStructure]: schema_information = [] for schema_group in directory_structure.find_resource_directories(file_ending=SCHEMA_FILE_ENDING):