From 5934bd340ef6778913d56581bf2d7fa26215ad82 Mon Sep 17 00:00:00 2001 From: florian-simvia Date: Fri, 11 Sep 2026 15:35:48 +0200 Subject: [PATCH] fix: stop reporting crashed cases as RUNNING forever A run launched from the web UI that crashed before the solver started stayed RUNNING indefinitely. Two independent causes, either of which masked the other: - is_process_alive treated a zombie as alive. os.kill(pid, 0) succeeds on a process that exited but was never reaped, which is what every run launched by the long-lived server becomes, since run_cases spawns with Popen and never waits. Zombies are now dead, and reaped when we are the parent. This is process management, not solver knowledge, so it stays in runner.py. - CodeSaturneAdapter.detect_outcome now reads the run_status.failed and run_status.exceeded_time_limit markers when the logs give no verdict. A failure during preprocessing writes no run_solver.log at all, only a marker beside it. The names come from code_saturne's own cs_case.py state table; the other markers it writes are progress states. The check lives in the adapter, not in logs.py, because run_status.* is a code_saturne convention. --- CHANGELOG.md | 1 + csauto/runner.py | 29 ++++++++++++++++++- csauto/solvers/code_saturne.py | 24 +++++++++++++++- tests/runner/test_runner.py | 46 ++++++++++++++++++++++++++++++ tests/unit/test_solvers.py | 52 ++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8739a3b..d891c38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Make the dashboard follow what each solver can actually do: panels and action bu - `csauto doctor` reports the panels and capabilities derived for the configured solver ### Fixed +- A case launched from the web UI stayed `RUNNING` forever when the run crashed before the solver started (a missing mesh, say). Two causes, both fixed: `is_process_alive` reported a zombie as alive, because `os.kill(pid, 0)` succeeds on a process that exited but was never reaped, which is what every run launched by the long-lived server becomes; and `detect_run_outcome` only read `run_solver.log` and `listing`, so a failure that produces neither went undetected even though code_saturne writes an explicit `run_status.failed` marker next to them. Zombies are now reported as dead (and reaped when we are the parent), and the status markers are read when the logs give no verdict - Opening the solver GUI on a case whose shared dirs are symlinks (the default since `mesh_mode = "symlink"` became the default in 0.4.1) left those symlinks dangling inside the container: `build_gui_command` (docker) and the singularity branch of `build_runtime_gui_command` mounted only the runs dir, unlike their `run` counterparts which also bind the symlink targets. Both now bind them the same way, `MESH` read-only and `POST` writable - Requesting a restart on a solver without restart support returned HTTP 500 "Launch error", a client error reported as a server fault; it now returns HTTP 400 naming the solver - Live control on a solver declaring no control action reported "Invalid action (expected one of [])"; both the API and the CLI now name the solver diff --git a/csauto/runner.py b/csauto/runner.py index da622e9..e6e60f2 100644 --- a/csauto/runner.py +++ b/csauto/runner.py @@ -691,8 +691,29 @@ def _start_case_with_launch_slot( start_case() +def _is_zombie(pid: int) -> bool: + """True when the process has exited but its parent has not reaped it yet. + + Without procfs (non-Linux), report False and let the os.kill() answer stand. + """ + try: + stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8", errors="ignore") + except OSError: + return False + # The comm field is parenthesized and may itself contain spaces or + # parentheses, so the state is the first token after the last ")". + fields = stat.rpartition(")")[2].split() + return bool(fields) and fields[0] == "Z" + + def is_process_alive(pid: int) -> bool: - """Check if a PID is alive (best effort, POSIX-oriented).""" + """Check if a PID is alive (best effort, POSIX-oriented). + + A zombie counts as dead. `os.kill(pid, 0)` still succeeds for a process that + exited but was never reaped, which is what every run launched by the + long-lived web server becomes: `run_cases` spawns it with Popen and never + waits. Treating that as alive kept finished cases RUNNING forever. + """ try: os.kill(pid, 0) except ProcessLookupError: @@ -701,6 +722,12 @@ def is_process_alive(pid: int) -> bool: return True except OSError: return False + if _is_zombie(pid): + # Reap it when we are the parent, so the entry stops lingering in the + # process table for the lifetime of the server. + with contextlib.suppress(ChildProcessError, OSError): + os.waitpid(pid, os.WNOHANG) + return False return True diff --git a/csauto/solvers/code_saturne.py b/csauto/solvers/code_saturne.py index 744fdf8..db70989 100644 --- a/csauto/solvers/code_saturne.py +++ b/csauto/solvers/code_saturne.py @@ -18,6 +18,8 @@ ) from ..logs import ( ANOMALY_FILES_DEFAULT, + _is_recent, + _parse_start_time, detect_run_outcome, extract_last_iteration, extract_restart_origin, @@ -29,6 +31,7 @@ parse_performance_log, ) from ..probes import list_probe_files, list_profile_files, locate_probe_files +from ..registry import STATUS_FAILED from ..residuals import find_residuals_files, parse_residuals_from_log from ..template import find_run_cfg, find_setup_file from .base import CompareKind, PerfColumn, SolverAdapterBase @@ -40,6 +43,12 @@ CHECKPOINT_STATE_RE = re.compile(r"Checkpoint at iteration\s+(?P\d+),\s+physical time\s+(?P