Mesh files: one directory, atomic writes, and a quick tier that is quick (#563)

scripts/test_levels.sh

@@ -146,26 +146,34 @@ run_tests() {
 }
 
 # Functions for each test level using pytest markers
+#
+# A level selection EXCLUDES the levels above it, and has to: pytest MERGES
+# marks rather than overriding them, so a file whose module declares
+# `pytestmark = pytest.mark.level_1` and whose heavy test then carries
+# `@pytest.mark.level_2` leaves that test marked BOTH. A plain `-m level_1`
+# selects it, and the demotion the author wrote does nothing. Nine files rely
+# on that demotion; the heaviest of their tests is a 96-second homotopy solve
+# that was running in the "quick" tier because of it.
 run_level_1() {
     echo "⚡ Running LEVEL 1: Quick Tests (Core Functionality)"
-    echo "Using pytest marker: -m level_1"
+    echo "Using pytest marker: -m 'level_1 and not level_2 and not level_3'"
     echo "Expected runtime: ~2 minutes"
     echo ""
 
-    # Run all tests marked with level_1
+    # Tests marked level_1 and NOT demoted to a higher level (see above)
     run_tests "Level 1 tests (quick core functionality)" \
-        tests/ -m level_1
+        tests/ -m "level_1 and not level_2 and not level_3"
 }
 
 run_level_2() {
     echo "🔧 Running LEVEL 2: Intermediate Tests"
-    echo "Using pytest marker: -m level_2"
+    echo "Using pytest marker: -m 'level_2 and not level_3'"
     echo "Expected runtime: ~5 minutes"
     echo ""
 
-    # Run all tests marked with level_2
+    # Tests marked level_2 and NOT demoted to level_3 (see above)
     run_tests "Level 2 tests (units, integration, projections)" \
-        tests/ -m level_2
+        tests/ -m "level_2 and not level_3"
 
     # Parallel tests for global statistics (requires MPI)
     if [ $RUN_PARALLEL -eq 1 ]; then

src/underworld3/discretisation/discretisation_mesh.py

@@ -143,9 +143,20 @@ def _from_gmsh(filename, comm=None, markVertices=False, useRegions=True, useMult
             plex_0.setName("uw_mesh")
             plex_0.markBoundaryFaces("All_Boundaries", 1001)
 
-            viewer = PETSc.ViewerHDF5().create(filename + ".h5", "w", comm=PETSc.COMM_SELF)
+            # Write aside and rename, so ``filename + ".h5"`` never names a
+            # half-written file. The name comes from the mesh PARAMETERS, so a
+            # second process building the same geometry in this directory picks
+            # the same one and would otherwise read what this one is still
+            # writing (issue #563).
+            # Imported here, not at module scope: the meshing package imports
+            # this module, so a top-level import would close a cycle.
+            from underworld3.meshing._mesh_files import _scratch_name
+
+            scratch = _scratch_name(filename + ".h5")
+            viewer = PETSc.ViewerHDF5().create(str(scratch), "w", comm=PETSc.COMM_SELF)
             viewer(plex_0)
             viewer.destroy()
+            os.replace(scratch, filename + ".h5")
     finally:
         # The gmsh import options are import-time scratch — meaningful only for
         # the createFromFile above. Clear the whole namespace so a value set by
@@ -155,7 +166,10 @@ def _from_gmsh(filename, comm=None, markVertices=False, useRegions=True, useMult
         # read as 2-D). Runs on success or failure.
         _clear_gmsh_import_options()
 
-    # Now we have an h5 file and we can hand this to _from_plexh5
+    # Now we have an h5 file and we can hand this to _from_plexh5. The barrier
+    # is what the atomic write above makes necessary AND sufficient: the other
+    # ranks must not look for the file before rank 0 has renamed it into place.
+    uw.mpi.barrier()
 
     return _from_plexh5(filename + ".h5", comm, return_sf=True)
 

src/underworld3/meshing/_mesh_files.py

@@ -0,0 +1,77 @@
+r"""Where generated mesh files are written, and how they are written.
+
+Building a mesh hands gmsh a file to write and then reads it back through
+PETSc, so the pair — ``<name>.msh`` and the ``<name>.msh.h5`` PETSc converts
+it to — is scratch shared between those two steps. It is not a cache: every
+construction regenerates both, and nothing checks whether they already exist.
+
+The name is derived from the mesh PARAMETERS, so two processes building the
+same geometry in one working directory choose the same name, and one can read
+a file the other is still writing (issue #563). Identical geometry is exactly
+the colliding case, which is why a parameter sweep or a parallel test run hits
+it and ordinary use does not.
+
+Two mechanisms make that safe, and both are needed:
+
+* the directory is settable per process through ``UW_MESH_CACHE_DIR``, so
+  independent jobs can be given somewhere of their own;
+* every write lands atomically, so a reader that shares a directory anyway
+  sees a complete file or no file, never a half-written one.
+"""
+import os
+from pathlib import Path
+
+import underworld3 as uw
+
+DEFAULT_MESH_FILE_DIR = ".meshes"
+
+
+def mesh_file_dir():
+    """The directory generated mesh files are written to.
+
+    ``UW_MESH_CACHE_DIR`` overrides the default ``.meshes``. Every rank of one
+    job must agree on it, so this reads the environment — inherited identically
+    by every rank — and never anything process-local such as the pid.
+    """
+    return Path(os.environ.get("UW_MESH_CACHE_DIR", DEFAULT_MESH_FILE_DIR))
+
+
+def mesh_file_path(basename):
+    """Full path for a generated mesh file, with its directory created.
+
+    Parameters
+    ----------
+    basename : str
+        The file's name, conventionally ``uw_<generator>_<parameters>.msh``.
+    """
+    directory = mesh_file_dir()
+    if uw.mpi.rank == 0:
+        directory.mkdir(parents=True, exist_ok=True)
+    return str(directory / basename)
+
+
+def _scratch_name(final):
+    """A process-unique sibling of ``final`` KEEPING ITS EXTENSION.
+
+    The extension has to survive: gmsh chooses its output format from it, so
+    writing to ``mesh.msh.1234.tmp`` would silently produce something that is
+    not a gmsh mesh.
+    """
+    final = Path(final)
+    return final.with_name(f"{final.stem}.{os.getpid()}.tmp{final.suffix}")
+
+
+def write_gmsh(filename):
+    """``gmsh.write``, landing atomically at ``filename``.
+
+    gmsh writes in place, so a concurrent reader can open a file that is still
+    being filled. Writing under a process-unique name and renaming makes the
+    appearance of the final name atomic — :func:`os.replace` is atomic within a
+    filesystem — so a reader sees either the previous complete file or the new
+    one.
+    """
+    import gmsh
+
+    scratch = _scratch_name(filename)
+    gmsh.write(str(scratch))
+    os.replace(scratch, filename)

src/underworld3/meshing/annulus.py

@@ -15,6 +15,7 @@
 import math
 
 import underworld3 as uw
+from underworld3.meshing._mesh_files import mesh_file_dir, write_gmsh
 from underworld3.discretisation import Mesh
 from underworld3 import VarType
 from underworld3.coordinates import CoordinateSystemType
@@ -129,9 +130,9 @@ class boundaries(Enum):
 
     if filename is None:
         if uw.mpi.rank == 0:
-            os.makedirs(".meshes", exist_ok=True)
+            os.makedirs(mesh_file_dir(), exist_ok=True)
 
-        uw_filename = f"uw_QuarterAnnulus_ro{radiusOuter}_ri{radiusInner}_csize{cellSize}.msh"
+        uw_filename = f"{mesh_file_dir()}/uw_QuarterAnnulus_ro{radiusOuter}_ri{radiusInner}_csize{cellSize}.msh"
     else:
         uw_filename = filename
 
@@ -237,7 +238,7 @@ class boundaries(Enum):
 
         print("generate")
 
-        gmsh.write(uw_filename)
+        write_gmsh(uw_filename)
         gmsh.finalize()
 
     new_mesh = Mesh(
@@ -397,9 +398,9 @@ class boundaries(Enum):
 
     if filename is None:
         if uw.mpi.rank == 0:
-            os.makedirs(".meshes", exist_ok=True)
+            os.makedirs(mesh_file_dir(), exist_ok=True)
 
-        uw_filename = f".meshes/uw_annulus_ro{radiusOuter}_ri{radiusInner}_csize{cellSize}.msh"
+        uw_filename = f"{mesh_file_dir()}/uw_annulus_ro{radiusOuter}_ri{radiusInner}_csize{cellSize}.msh"
     else:
         uw_filename = filename
 
@@ -466,7 +467,7 @@ class boundaries(Enum):
         gmsh.model.geo.synchronize()
 
         gmsh.model.mesh.generate(2)
-        gmsh.write(uw_filename)
+        write_gmsh(uw_filename)
         gmsh.finalize()
 
     # Ensure boundaries conform (if refined)
@@ -625,9 +626,9 @@ class boundaries(Enum):
 
     if filename is None:
         if uw.mpi.rank == 0:
-            os.makedirs(".meshes", exist_ok=True)
+            os.makedirs(mesh_file_dir(), exist_ok=True)
 
-        uw_filename = f"uw_SegmentOfAnnulus_ro{radiusOuter}_ri{radiusInner}_extent{angleExtent}_csize{cellSize}.msh"
+        uw_filename = f"{mesh_file_dir()}/uw_SegmentOfAnnulus_ro{radiusOuter}_ri{radiusInner}_extent{angleExtent}_csize{cellSize}.msh"
     else:
         uw_filename = filename
 
@@ -720,7 +721,7 @@ class boundaries(Enum):
         gmsh.model.geo.synchronize()
 
         gmsh.model.mesh.generate(2)
-        gmsh.write(uw_filename)
+        write_gmsh(uw_filename)
         gmsh.finalize()
 
     # Ensure boundaries conform (if refined)
@@ -896,9 +897,9 @@ class boundaries(Enum):
 
     if filename is None:
         if uw.mpi.rank == 0:
-            os.makedirs(".meshes", exist_ok=True)
+            os.makedirs(mesh_file_dir(), exist_ok=True)
 
-        uw_filename = f".meshes/uw_annulus_ro{radiusOuter}_ri{radiusInner}_csize{cellSizeOuter}.msh"
+        uw_filename = f"{mesh_file_dir()}/uw_annulus_ro{radiusOuter}_ri{radiusInner}_csize{cellSizeOuter}.msh"
     else:
         uw_filename = filename
 
@@ -1004,7 +1005,7 @@ class boundaries(Enum):
 
         gmsh.model.mesh.generate(2)
 
-        gmsh.write(uw_filename)
+        write_gmsh(uw_filename)
 
         # We need to build the plex here in order to make some changes
         # before the mesh gets built
@@ -1249,9 +1250,9 @@ class regions(Enum):
 
     if filename is None:
         if uw.mpi.rank == 0:
-            os.makedirs(".meshes", exist_ok=True)
+            os.makedirs(mesh_file_dir(), exist_ok=True)
 
-        uw_filename = f".meshes/uw_annulus_internalBoundary_rO{radiusOuter}rInt{radiusInternal}_rI{radiusInner}_csize{cellSize}_csizefs{cellSize_Outer}.msh"
+        uw_filename = f"{mesh_file_dir()}/uw_annulus_internalBoundary_rO{radiusOuter}rInt{radiusInternal}_rI{radiusInner}_csize{cellSize}_csizefs{cellSize_Outer}.msh"
     else:
         uw_filename = filename
 
@@ -1345,7 +1346,7 @@ class regions(Enum):
         gmsh.model.geo.synchronize()
 
         gmsh.model.mesh.generate(2)
-        gmsh.write(uw_filename)
+        write_gmsh(uw_filename)
         gmsh.finalize()
 
     ## This is the same as the simple annulus
@@ -1558,9 +1559,9 @@ class boundaries(Enum):
 
     if filename is None:
         if uw.mpi.rank == 0:
-            os.makedirs(".meshes", exist_ok=True)
+            os.makedirs(mesh_file_dir(), exist_ok=True)
 
-        uw_filename = f".meshes/uw_disc_internalBoundaries_rO{radiusUpper}rInt{radiusInternal}_rI{radiusLower}_csize{cellSize}_csizefs{cellSize_Upper}.msh"
+        uw_filename = f"{mesh_file_dir()}/uw_disc_internalBoundaries_rO{radiusUpper}rInt{radiusInternal}_rI{radiusLower}_csize{cellSize}_csizefs{cellSize_Upper}.msh"
     else:
         uw_filename = filename
 
@@ -1643,7 +1644,7 @@ class boundaries(Enum):
         gmsh.model.geo.synchronize()
 
         gmsh.model.mesh.generate(2)
-        gmsh.write(uw_filename)
+        write_gmsh(uw_filename)
         gmsh.finalize()
 
     ## This is the same as the simple annulus