feat: run Julia make_data files in witch_translate_data (0.7.0)

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: witchtools
 Title: Data Management for Integrated Assessment Models
-Version: 0.6.0
+Version: 0.7.0
 Authors@R: 
     c(person(given = "Laurent",
            family = "Drouet",
@@ -59,7 +59,7 @@ Suggests:
     DBI,
     RSQLite,
     duckdb,
-    gdxtools (>= 1.0.0),
+    gdxtools (>= 1.2.0),
     ggplot2,
     knitr,
     remotes,

NEWS.md

@@ -1,3 +1,18 @@
+# witchtools 0.7.0
+
+## New features
+
+- `witch_translate_data()` also runs the Julia `input/make_data_*.jl`
+  files (after the R ones, before the GAMS ones) with
+  `julia --project=input`, instantiating the input Julia project first.
+  They take the same `-i`, `-w`, `-m` and `--noCheck` options as the R
+  scripts and write `input/build/data_*.gdx`.
+
+## Other changes
+
+- Requires gdxtools >= 1.2.0 and no longer calls the deprecated
+  `gdxtools::igdx()`.
+
 # witchtools 0.6.0
 
 ## New features

R/get_libraries.R

@@ -15,16 +15,16 @@
 #' }
 require_package <- function(pkgs, loading = TRUE) {
   # Special handling for gdxtools (installed from GitHub).
-  # witchtools requires gdxtools >= 1.0.0 (gamstransfer backend); install
+  # witchtools requires gdxtools >= 1.2.0 (gamstransfer backend); install
   # or upgrade from GitHub if the local copy is missing or too old.
   if ("gdxtools" %in% pkgs) {
-    if (!rlang::is_installed("gdxtools", version = "1.0.0")) {
+    if (!rlang::is_installed("gdxtools", version = "1.2.0")) {
       # First ensure remotes is available
       if (!rlang::is_installed("remotes")) {
         install_package_safe("remotes")
       }
 
-      message("Installing gdxtools (>= 1.0.0) from GitHub...")
+      message("Installing gdxtools (>= 1.2.0) from GitHub...")
       remotes::install_github("lolow/gdxtools", quiet = !interactive())
     }
   }

R/witch_make_files.R

@@ -65,3 +65,38 @@ make_data_R <- function(Rfile, idir, witch_dir, force = FALSE) {
 
   return(Rfile)
 }
+
+# Julia make_data files are run with the Julia project of the input folder
+# (input/Project.toml), which provides their package dependencies.
+make_data_jl <- function(jlfile, idir, witch_dir, force = FALSE) {
+
+  # generated gdx
+  gengdx <- file.path(
+    witch_dir, "input", "build",
+    paste0(stringr::str_sub(basename(jlfile), 6, -4), ".gdx")
+  )
+
+  # check if the generated gdx is older than the make_data julia file
+  if (file.exists(gengdx)) {
+    if (!force & (file.mtime(gengdx) > file.mtime(jlfile))) {
+      return()
+    }
+  }
+
+  cat(crayon::blue$bold(paste("Compiling", basename(jlfile), "\n")))
+  cmd <- paste0(
+    'julia --project="', normalizePath(file.path(witch_dir, "input")),
+    '" "', jlfile,
+    '" -i "', normalizePath(idir),
+    '" -w "', normalizePath(witch_dir),
+    '" -m ', getOption("witchtools.method")
+  )
+  if (getOption("witchtools.noCheck", TRUE)) {
+    cmd <- paste0(cmd, " --noCheck")
+  }
+
+  res <- system(cmd)
+  stopifnot(res == 0)
+
+  return(jlfile)
+}

R/witch_translate_data.R

@@ -1,7 +1,7 @@
 #' Translate WITCH data (run make_data files and convert data).
 #'
 #' \code{witch_translate_data} generates the input data for the WITCH model.
-#' First, it will run the R `make_data_files` and the gams `make_data_files`
+#' First, it will run the R, Julia and gams `make_data_files`
 #' usually located in the input folder of WITCH. These make_data_files produce
 #' gdx, sqlite, and duckdb files in the `build` folder. Time-series should be yearly.
 #' Spatial data should be preferably described at ISO3 level, but the script
@@ -49,11 +49,6 @@ witch_translate_data <- function(witch_dir = ".",
                                  force = FALSE) {
   cat(crayon::silver$bold("\U26AB Initialisation\n"))
 
-  # Check if gdxtool is available and working
-  if (requireNamespace("gdxtools", quietly = TRUE)) {
-    gdxtools::igdx(dirname(Sys.which("gams"))) # Please have gams in your PATH!
-  }
-
   # Region mapping
   reg_id <- stringr::str_replace(as.character(region), ".inc", "")
 
@@ -101,6 +96,21 @@ witch_translate_data <- function(witch_dir = ".",
     make_data_R(Rfile, idir, witch_dir)
   }
 
+  # Julia files make_data_*.jl
+  jlfiles <- Sys.glob(file.path(witch_dir, "input", "make_data_*.jl"))
+  if (length(jlfiles) > 0 && !nzchar(Sys.which("julia"))) {
+    stop("julia not found in the PATH, needed to run make_data_*.jl files")
+  }
+  if (length(jlfiles) > 0) {
+    cmd <- paste0('julia --project="',
+                  normalizePath(file.path(witch_dir, "input")),
+                  '" -e "using Pkg; Pkg.instantiate()"')
+    stopifnot(system(cmd) == 0)
+  }
+  res <- for (jlfile in jlfiles) {
+    make_data_jl(jlfile, idir, witch_dir)
+  }
+
   # GAMS files make_data_*.gms
   gamsfiles <- Sys.glob(file.path(witch_dir, "input", "make_data_*.gms"))
   gamsfiles <- gamsfiles[gamsfiles != "make_data_template.gms"]