diff --git a/applications/adjoint_tests/Makefile b/applications/adjoint_tests/Makefile index fc5c26056b..c6eb1b9683 100644 --- a/applications/adjoint_tests/Makefile +++ b/applications/adjoint_tests/Makefile @@ -25,6 +25,10 @@ export PSYCLONE_CONFIG_FILE := $(PROJECT_DIR)/build/psyclone.cfg export BUILD_ADJ_TESTS := TRUE export PSYAD_CONFIG_FILE := $(PSYCLONE_CONFIG_FILE) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -123,6 +127,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) diff --git a/applications/gravity_wave/Makefile b/applications/gravity_wave/Makefile index 62f7c8731c..0fceddd6e5 100644 --- a/applications/gravity_wave/Makefile +++ b/applications/gravity_wave/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -46,6 +50,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -116,6 +121,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating namelist loaders for $(PROJECT)) $(call MESSAGE,=========================================================) @@ -157,8 +166,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run - +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/applications/gungho_model/Makefile b/applications/gungho_model/Makefile index 8d364462ee..b01b6e8c11 100644 --- a/applications/gungho_model/Makefile +++ b/applications/gungho_model/Makefile @@ -19,6 +19,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -87,6 +91,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) diff --git a/applications/jedi_lfric_tests/Makefile b/applications/jedi_lfric_tests/Makefile index c76919a8f2..2324011cc8 100644 --- a/applications/jedi_lfric_tests/Makefile +++ b/applications/jedi_lfric_tests/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -49,6 +53,7 @@ default: build integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) -include $(PROJECT_DIR)/build/project.mk @@ -123,6 +128,10 @@ build: ALWAYS $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source \ WORKING_DIR=$(WORKING_DIR) + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -175,8 +184,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run - +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests @@ -197,8 +206,8 @@ integration-tests/%: export PROJECT = $(PROJECT_NAME) integration-tests/%: export SOURCE_DIR = source integration-tests/%: export TEST_DIR = integration-test integration-tests/%: export WORKING_DIR := $(WORKING_DIR)/integration-tests -integration-tests: integration-tests/run - +integration-tests: extract_meta integration-tests/run +integration-tests/build: extract_meta integration-tests/build ############################################################################## # Clean diff --git a/applications/jules/Makefile b/applications/jules/Makefile index 698637e0c7..c3214a6f99 100644 --- a/applications/jules/Makefile +++ b/applications/jules/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + # Note - socrates_interface must come before jules_interface as there are dependencies on each other. export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ @@ -100,6 +104,10 @@ build: ALWAYS $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk + $(call MESSAGE,========================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/configuration.mk \ diff --git a/applications/lfric2lfric/Makefile b/applications/lfric2lfric/Makefile index 278036352f..f38be413ee 100644 --- a/applications/lfric2lfric/Makefile +++ b/applications/lfric2lfric/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/coupling \ $(CORE_ROOT_DIR)/components/driver \ @@ -47,6 +51,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -117,6 +122,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -153,8 +162,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source/kernel unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run - +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/applications/lfric_atm/Makefile b/applications/lfric_atm/Makefile index 15c770cc6a..b4ede497b6 100644 --- a/applications/lfric_atm/Makefile +++ b/applications/lfric_atm/Makefile @@ -32,6 +32,10 @@ export IGNORE_DEPENDENCIES = c_shum_byteswap \ f_shum_is_denormal_mod +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + # Note - socrates_interface must come before jules_interface as there are dependencies on each other. export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ @@ -115,6 +119,10 @@ build: ALWAYS $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk + $(call MESSAGE,========================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/configuration.mk \ diff --git a/applications/lfric_coupled/Makefile b/applications/lfric_coupled/Makefile index 65b50d5604..01ea26cb8b 100644 --- a/applications/lfric_coupled/Makefile +++ b/applications/lfric_coupled/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + # Note - socrates_interface must come before jules_interface as there are dependencies on each other. export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/coupling \ @@ -51,6 +55,7 @@ default: build test-suite documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) include build/project.mk @@ -141,6 +146,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -219,7 +228,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## diff --git a/applications/lfricinputs/Makefile b/applications/lfricinputs/Makefile index 271f4b6b92..e659b9d3bc 100644 --- a/applications/lfricinputs/Makefile +++ b/applications/lfricinputs/Makefile @@ -18,6 +18,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/science \ $(CORE_ROOT_DIR)/components/inventory \ @@ -86,6 +90,10 @@ build: ALWAYS $(call MESSAGE,Extracting $(PROJECT_NAME)) $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) diff --git a/applications/linear_model/Makefile b/applications/linear_model/Makefile index 5755c6a89c..7ad66520d0 100644 --- a/applications/linear_model/Makefile +++ b/applications/linear_model/Makefile @@ -19,6 +19,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -88,6 +92,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) diff --git a/applications/name_transport/Makefile b/applications/name_transport/Makefile index 7ddbee74c8..c7ea0be7a6 100644 --- a/applications/name_transport/Makefile +++ b/applications/name_transport/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -46,6 +50,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) -include $(PROJECT_DIR)/build/project.mk @@ -117,6 +122,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -160,7 +169,8 @@ unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests unit-tests/%: export UNIT_TEST_PRE_PROCESS_MACROS = "NO_MPI=no_mpi" -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/applications/ngarch/Makefile b/applications/ngarch/Makefile index c3ddc06dc3..eee95d32f3 100644 --- a/applications/ngarch/Makefile +++ b/applications/ngarch/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + # Note - socrates_interface must come before jules_interface as there are dependencies on each other. export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ @@ -136,6 +140,10 @@ build: ALWAYS $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk + $(call MESSAGE,========================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/configuration.mk \ diff --git a/applications/shallow_water/Makefile b/applications/shallow_water/Makefile index 13cb94ebd0..b52fc5d9f1 100644 --- a/applications/shallow_water/Makefile +++ b/applications/shallow_water/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -46,6 +50,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) -include $(PROJECT_DIR)/build/project.mk @@ -117,6 +122,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -159,7 +168,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## diff --git a/applications/solver/Makefile b/applications/solver/Makefile index 028af625aa..b3c86b49c3 100644 --- a/applications/solver/Makefile +++ b/applications/solver/Makefile @@ -20,6 +20,8 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ diff --git a/applications/transport/Makefile b/applications/transport/Makefile index b849928a52..ab8024aadb 100644 --- a/applications/transport/Makefile +++ b/applications/transport/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -46,6 +50,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) -include $(PROJECT_DIR)/build/project.mk @@ -117,6 +122,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -160,7 +169,8 @@ unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests unit-tests/%: export UNIT_TEST_PRE_PROCESS_MACROS = "NO_MPI=no_mpi" -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/build/extract/extract_meta.mk b/build/extract/extract_meta.mk new file mode 100644 index 0000000000..d165f87fbb --- /dev/null +++ b/build/extract/extract_meta.mk @@ -0,0 +1,26 @@ +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# +# Run this file to extract rose-metadata from external repositories +# +# The following environment variables are used for input: +# WORKING_DIR : Directory to hold working copies of source. +# META_DIR: Directory to save extracted external metadata +# APPS_ROOT_DIR: Location of apps clone +# EXTRA_ROSE_META: List of repos to extract metadata from +# +############################################################################### + +.PHONY: extract_meta + +extract_meta: + $(Q)for REPO in $(EXTRA_ROSE_META) ; do \ + python $(APPS_ROOT_DIR)/build/extract/extract_science.py \ + -r $$REPO \ + -d $(APPS_ROOT_DIR)/dependencies.yaml \ + -w $(WORKING_DIR) \ + -m $(META_DIR) \ + ; done diff --git a/build/extract/extract_science.py b/build/extract/extract_science.py index 845b87b012..207026c640 100755 --- a/build/extract/extract_science.py +++ b/build/extract/extract_science.py @@ -1,9 +1,16 @@ +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- + import argparse +import logging import os import yaml from pathlib import Path +from shutil import copytree from get_git_sources import clone_and_merge, run_command -import logging def load_yaml(fpath: Path) -> dict: @@ -17,7 +24,50 @@ def load_yaml(fpath: Path) -> dict: return sources -def extract_files(dependencies: dict, extract_lists: dict, working: Path) -> None: +def copy_rose_meta(rose_meta_dest: Path, clone_loc: Path) -> None: + """ + Copy rose-meta contents from extracted dependency to rose_meta_dest + """ + + rose_meta_orig = clone_loc / "rose-meta" + + for directory in rose_meta_orig.iterdir(): + copytree(directory, rose_meta_dest / directory.name, dirs_exist_ok=True) + + +def copy_extracted_files( + dependency: str, extract_lists: dict, working: Path, clone_loc: Path +) -> None: + """ + Copy extracted files to the working dir based on extract list + """ + + files = extract_lists[dependency] + + # make the working directory location + working_dir = working / dependency + working_dir.mkdir(parents=True, exist_ok=True) + + # rsync extract files from clone loc to the working directory + copy_command = "rsync --include='**/' " + for extract_file in files: + if not extract_file: + continue + if Path(clone_loc / extract_file).is_dir(): + extract_file = extract_file.rstrip("/") + extract_file += "/**" + copy_command += f"--include='{extract_file}' " + copy_command += f"--exclude='*' -avmq {clone_loc}/ {working_dir}" + run_command(copy_command) + + +def extract_files( + dependencies: dict, + working: Path, + meta: Path, + rose_meta: str = "", + extract_lists: dict = {}, +) -> None: """ Clone the dependency to a temporary location Then copy the desired files to the working directory @@ -29,9 +79,8 @@ def extract_files(dependencies: dict, extract_lists: dict, working: Path) -> Non mirror_loc = Path(mirror_loc) for dependency, sources in dependencies.items(): - if dependency not in extract_lists: + if dependency not in extract_lists and dependency != rose_meta: continue - files = extract_lists[dependency] # If the PHYSICS_ROOT environment variable is provided, then use sources there if "PHYSICS_ROOT" in os.environ and Path(os.environ["PHYSICS_ROOT"]).exists(): @@ -40,21 +89,10 @@ def extract_files(dependencies: dict, extract_lists: dict, working: Path) -> Non clone_loc = working.parent / "scratch" / dependency clone_and_merge(dependency, sources, clone_loc, use_mirrors, mirror_loc) - # make the working directory location - working_dir = working / dependency - working_dir.mkdir(parents=True, exist_ok=True) - - # rsync extract files from clone loc to the working directory - copy_command = "rsync --include='**/' " - for extract_file in files: - if not extract_file: - continue - if Path(clone_loc / extract_file).is_dir(): - extract_file = extract_file.rstrip("/") - extract_file += "/**" - copy_command += f"--include='{extract_file}' " - copy_command += f"--exclude='*' -avmq {clone_loc}/ {working_dir}" - run_command(copy_command) + if rose_meta: + copy_rose_meta(meta, clone_loc) + else: + copy_extracted_files(dependency, extract_lists, working, clone_loc) def parse_args() -> argparse.Namespace: @@ -70,15 +108,35 @@ def parse_args() -> argparse.Namespace: help="The dependencies file for the apps working copy", ) parser.add_argument("-w", "--working", default=".", help="Build location") + parser.add_argument( + "-m", + "--meta_dir", + default=None, + help="Path to store externally extracted rose-meta. Used if --rose-meta set. " + "Defaults to args.working/../rose-meta", + ) parser.add_argument( "-e", "--extract", default="./extract.yaml", help="Path to file containing extract lists", ) + parser.add_argument( + "-r", + "--rose_meta", + type=str, + default="", + help="Should be a repository in the dependencies file. If set, copy the " + "dependencies rose-meta directory contents to working/../rose-meta. " + "If set, the extract file will be ignored, and just rose-metadata copied", + ) args = parser.parse_args() args.working = Path(args.working) + if args.meta_dir is None: + args.meta_dir = args.working.parent / "rose-meta" + else: + args.meta_dir = Path(args.meta_dir) return args @@ -87,10 +145,21 @@ def main(): logging.basicConfig(level=logging.INFO) - extract_lists: dict = load_yaml(args.extract) dependencies: dict = load_yaml(args.dependencies) - extract_files(dependencies, extract_lists, args.working) + # If args.rose_meta is set then extract rose-meta contents from that repo + if args.rose_meta: + extract_files( + dependencies, args.working, args.meta_dir, rose_meta=args.rose_meta + ) + # If args.rose_meta is not set, then extract files based on the provided extract list + else: + extract_files( + dependencies, + args.working, + args.meta_dir, + extract_lists=load_yaml(args.extract), + ) if __name__ == "__main__": diff --git a/build/extract/get_git_sources.py b/build/extract/get_git_sources.py index 4bf5fd632f..59afdb0068 100644 --- a/build/extract/get_git_sources.py +++ b/build/extract/get_git_sources.py @@ -88,7 +88,11 @@ def datetime_str() -> str: def clone_and_merge( - dependency: str, opts: Union[list, dict], loc: Path, use_mirrors: bool, mirror_loc: Path + dependency: str, + opts: Union[list, dict], + loc: Path, + use_mirrors: bool, + mirror_loc: Path, ) -> None: """ Wrapper script for calling get_source and merge_source for a single dependency diff --git a/dependencies.yaml b/dependencies.yaml index 1a7aa4be9a..6536832e23 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -31,7 +31,7 @@ lfric_apps: lfric_core: source: git@github.com:MetOffice/lfric_core.git - ref: a963e38720b7ea0b01952176603f6f98767d6506 + ref: 3c8ccc6ea152cb03f0c5f27de5f1de01cc0c78b9 moci: source: git@github.com:MetOffice/moci.git diff --git a/interfaces/coupled_interface/Makefile b/interfaces/coupled_interface/Makefile index bd39248f47..74b920bdfd 100644 --- a/interfaces/coupled_interface/Makefile +++ b/interfaces/coupled_interface/Makefile @@ -20,12 +20,17 @@ export PSYCLONE_TRANSFORMATION ?= none export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/lfric-xios META_VN ?= HEAD META_FILE_DIR = $(PROJECT_DIR)/rose-meta/$(PROJECT_NAME)/$(META_VN) +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(PROJECT_DIR)/build/import.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -104,7 +109,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## diff --git a/interfaces/jedi_lfric_interface/Makefile b/interfaces/jedi_lfric_interface/Makefile index b2f843a18c..70120e8568 100644 --- a/interfaces/jedi_lfric_interface/Makefile +++ b/interfaces/jedi_lfric_interface/Makefile @@ -35,6 +35,10 @@ export EXTERNAL_DYNAMIC_LIBRARIES = netcdff netcdf yaxt yaxt_c export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(APPS_ROOT_DIR)/science/gungho \ $(CORE_ROOT_DIR)/components/lfric-xios \ @@ -51,6 +55,7 @@ default: unit-tests integration-tests documentation doc docs: document-uml document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(PROJECT_DIR)/build/import.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -92,7 +97,8 @@ unit-tests/%: export PROJECT = jedi_lfric unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## @@ -109,7 +115,8 @@ integration-tests/%: export PROJECT = $(PROJECT_NAME) integration-tests/%: export SOURCE_DIR = source integration-tests/%: export TEST_DIR = integration-test integration-tests/%: export WORKING_DIR := $(WORKING_DIR)/integration-tests -integration-tests: integration-tests/run +integration-tests: extract_meta integration-tests/run +integration-tests/build: extract_meta integration-tests/build ############################################################################## diff --git a/interfaces/physics_schemes_interface/Makefile b/interfaces/physics_schemes_interface/Makefile index 98fff36814..92c50a25d0 100644 --- a/interfaces/physics_schemes_interface/Makefile +++ b/interfaces/physics_schemes_interface/Makefile @@ -37,6 +37,8 @@ export EXTERNAL_STATIC_LIBRARIES = export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta + .PHONY: all all: default diff --git a/interfaces/socrates_interface/Makefile b/interfaces/socrates_interface/Makefile index af25a5a47d..09cd253d08 100644 --- a/interfaces/socrates_interface/Makefile +++ b/interfaces/socrates_interface/Makefile @@ -36,6 +36,8 @@ export EXTERNAL_DYNAMIC_LIBRARIES = yaxt yaxt_c netcdff netcdf hdf5 export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta + META_VN ?= HEAD META_FILE_DIR = $(PROJECT_DIR)/rose-meta/$(PROJECT_NAME)-radiation/$(META_VN) diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index f762b6e186..acb5091746 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -101,7 +101,6 @@ {% do LOG.debug("Not Running Physics Schemes") %} {% for item in ["casim", - "jules", "ukca", "socrates", "socrates-spectral", diff --git a/science/adjoint/Makefile b/science/adjoint/Makefile index 3120ac3e1f..1988e6d863 100644 --- a/science/adjoint/Makefile +++ b/science/adjoint/Makefile @@ -9,6 +9,10 @@ export PROFILE ?= fast-debug export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -28,6 +32,7 @@ default: unit-tests integration-tests documentation doc docs: document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -79,8 +84,8 @@ unit-tests/%: export PROJECT = adjoint unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests -unit-tests: unit-tests/run - +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/science/gungho/Makefile b/science/gungho/Makefile index 5a304b0cfc..b6f3646f49 100644 --- a/science/gungho/Makefile +++ b/science/gungho/Makefile @@ -33,6 +33,10 @@ export PROFILE ?= fast-debug export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -50,6 +54,7 @@ default: unit-tests integration-tests documentation doc docs: document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -105,7 +110,8 @@ unit-tests/%: export PROJECT = gungho unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################### # Integration tests @@ -119,7 +125,8 @@ integration-tests/%: export PROJECT = gungho integration-tests/%: export SOURCE_DIR = source integration-tests/%: export TEST_DIR = integration-test integration-tests/%: export WORKING_DIR := $(WORKING_DIR)/integration-tests -integration-tests: integration-tests/run +integration-tests: extrac_meta integration-tests/run +integration-tests/build: extract_meta integration-tests/build ############################################################################### # Clean diff --git a/science/linear/Makefile b/science/linear/Makefile index 91bc866946..61915baa48 100644 --- a/science/linear/Makefile +++ b/science/linear/Makefile @@ -9,6 +9,10 @@ export PROFILE ?= fast-debug export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -27,6 +31,7 @@ default: unit-tests integration-tests documentation doc docs: document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -79,8 +84,8 @@ unit-tests/%: export PROJECT = linear unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests - -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## @@ -99,7 +104,8 @@ integration-tests/%: export PROJECT = linear integration-tests/%: export SOURCE_DIR = source integration-tests/%: export TEST_DIR = integration-test integration-tests/%: export WORKING_DIR := $(WORKING_DIR)/integration-tests -integration-tests: integration-tests/run +integration-tests: extract_meta integration-tests/run +integration-tests/build: extract_meta integration-tests/build ############################################################################## # Clean