From 565374dc24e60f5aa7334dd7919ed28154d93b17 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 18:32:26 +0000 Subject: [PATCH 01/56] feat(sandbox): add OPENSHELL as an additional SANDBOX_MODE OpenShell becomes a fourth selectable SANDBOX_MODE alongside FAKECHROOT / NAMESPACE / SECCOMP_ONLY. Nothing is retired: the existing modes and their launch paths are untouched, so a deployment opts in by setting one property and reverts the same way. - Constants: SANDBOX_MODE_OPENSHELL = "OPENSHELL", documented as additive. - OpenShellTransport: creates a named sandbox running the gaas worker as its canonical main process, forwards the worker port to a free local port over the gateway, and tears the sandbox down on close. Deterministic sandbox naming from a session key (SHA-256, DNS-1123 safe) so no registry lookup is needed. A sandbox that already exists is success, not an error. - ClientProcessWrapper: one new branch in the existing dispatch, before the NAMESPACE branch. Sets no udsPath, so the existing socket client takes its normal TCP path to 127.0.0.1: -- the socket layer is unchanged. Fails closed: never falls back to an in-Tomcat worker when the deployment asked for an external boundary. - shutdown(): closes the forward and deletes the sandbox, since the worker cannot outlive its sandbox. Verified: mvn compile clean. Existing modes behaviourally unchanged. --- src/prerna/om/ClientProcessWrapper.java | 37 +++- src/prerna/util/Constants.java | 4 + src/prerna/util/OpenShellTransport.java | 269 ++++++++++++++++++++++++ 3 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 src/prerna/util/OpenShellTransport.java diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index bde8082c19..c3a532f644 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -127,6 +127,8 @@ public class ClientProcessWrapper { // the same JSON protocol, so it reuses NativePySocketClient. private boolean nodeServer = false; private SymlinkHelper chrootSymlinkHelper; + // OPENSHELL mode: owns the sandbox + port-forward for this worker's lifetime + private prerna.util.OpenShellTransport openshell; private String classPath; private boolean debug; private String timeout; @@ -223,7 +225,27 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS boolean serverRunning = debug && port > 0; if (!serverRunning) { if (nativePyServer) { - if (this.chrootSymlinkHelper != null && this.chrootSymlinkHelper.isInjectMode()) { + // OPENSHELL: the worker is the canonical main process of its own + // sandbox, not a child of Tomcat. Create/attach the sandbox, take + // the forwarded local port, and let the existing socket client + // connect to it exactly as the non-sandbox path does. No udsPath + // is set, which is what selects the TCP branch below. + String openshellMode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); + if (Constants.SANDBOX_MODE_OPENSHELL.equalsIgnoreCase(openshellMode)) { + this.openshell = prerna.util.OpenShellTransport.forSession(this.serverDirectory != null + ? this.serverDirectory + : Integer.toHexString(System.identityHashCode(this))); + try { + this.port = this.openshell.start(); + } catch (Exception e) { + // Fail closed: never fall back to an in-Tomcat worker when the + // deployment asked for an external boundary. + throw new IllegalStateException( + "OpenShell executor sandbox unavailable and no fallback is permitted in " + + "SANDBOX_MODE=OPENSHELL", + e); + } + } else if (this.chrootSymlinkHelper != null && this.chrootSymlinkHelper.isInjectMode()) { String insightCache = Utility.getDIHelperProperty(Constants.INSIGHT_CACHE_DIR); Path serverDirectoryPath = Files.createTempDirectory(Paths.get(insightCache), "a"); this.serverDirectory = serverDirectoryPath.toString(); @@ -411,6 +433,19 @@ & deleteFolderWithRetries(this.sandboxJailDir) classLogger.error("Failed to destroy isolated analytics process for port {}", this.port, e); } } + // OPENSHELL: the worker lives in the sandbox, not in this JVM's process + // tree. Killing the local forward and deleting the sandbox is the whole + // teardown -- the gateway delete is authoritative and the worker cannot + // outlive its sandbox. There is no graceful-shutdown verb to send first. + if (this.openshell != null) { + try { + this.openshell.close(); + } catch (Exception e) { + classLogger.error("Failed to close OpenShell executor sandbox for port {}", this.port, e); + } finally { + this.openshell = null; + } + } } finally { lockDestroy.unlock(); } diff --git a/src/prerna/util/Constants.java b/src/prerna/util/Constants.java index 1faba0f61d..75b7c451f2 100644 --- a/src/prerna/util/Constants.java +++ b/src/prerna/util/Constants.java @@ -767,7 +767,11 @@ public class Constants { public static final String CHROOT_DIR = "CHROOT_DIR"; // OFF / FAKECHROOT / NAMESPACE / SECCOMP_ONLY (legacy NSJAIL is accepted) + // OPENSHELL is an additional mode, not a replacement for the others: the worker + // runs as the canonical main process of an OpenShell sandbox instead of a child + // process of Tomcat. See prerna.util.OpenShellTransport. public static final String SANDBOX_MODE = "SANDBOX_MODE"; + public static final String SANDBOX_MODE_OPENSHELL = "OPENSHELL"; public static final String SANDBOX_IO_DIR = "SANDBOX_IO_DIR"; // Colon/semicolon-separated executable directories exposed inside sandboxes public static final String SANDBOX_PATH = "SANDBOX_PATH"; diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java new file mode 100644 index 0000000000..af7ced6953 --- /dev/null +++ b/src/prerna/util/OpenShellTransport.java @@ -0,0 +1,269 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Transport to a SEMOSS python worker running inside an OpenShell sandbox. + * + *

+ * OpenShell ({@code SANDBOX_MODE=OPENSHELL}) makes the worker the canonical main + * process of its own sandbox rather than a child process of Tomcat. That changes + * three things and nothing else: + * + *

    + *
  1. Where the worker runs. A sandbox, created with + * {@code openshell sandbox create} driven from an admitted workload template.
    2. + *
  2. How bytes reach it. A local port is forwarded into the sandbox's + * loopback over the gateway ({@code openshell forward service}). The existing + * {@code NativePySocketClient} is unchanged: it connects to + * {@code 127.0.0.1:} exactly as it does for the non-sandbox path.
    3. + *
  3. What "the connection" means. The worker is single-connection + * ({@code --max_count 1}) and it is the sandbox's main process, so a client + * disconnect ends the worker and the sandbox. The connection must therefore + * be held for the session's life, and re-established by re-creating the sandbox, + * not merely by reconnecting a socket.
    4. + *
+ * + *

+ * What this class deliberately does not do: it does not choose the image, + * the policy, the runtime class, or the egress profile. Those are deployment + * properties admitted by the workload template, so that neither the user nor + * model-generated code can widen its own boundary. + */ +public final class OpenShellTransport { + + private static final Logger classLogger = LogManager.getLogger(OpenShellTransport.class); + + /** Property naming the OpenShell CLI. */ + public static final String CLI_PROP = "OPENSHELL_CLI"; + /** Property naming the sandbox workload template to create from. */ + public static final String TEMPLATE_PROP = "OPENSHELL_TEMPLATE"; + /** Property naming the port the worker listens on *inside* the sandbox. */ + public static final String WORKER_PORT_PROP = "OPENSHELL_WORKER_PORT"; + /** Property naming the label value used to group executor sandboxes. */ + public static final String LABEL_PROP = "OPENSHELL_LABEL"; + /** How long to wait for the sandbox to reach Ready. */ + public static final String READY_TIMEOUT_PROP = "OPENSHELL_READY_TIMEOUT_SECONDS"; + + private static final String DEFAULT_CLI = "openshell"; + private static final String DEFAULT_WORKER_PORT = "7100"; + private static final String DEFAULT_LABEL = "app=semoss-executor"; + private static final int DEFAULT_READY_TIMEOUT_S = 180; + + private final String sandboxName; + private final String workerPort; + private int forwardedPort = -1; + private Process forwardProcess; + + private OpenShellTransport(String sandboxName, String workerPort) { + this.sandboxName = sandboxName; + this.workerPort = workerPort; + } + + /** + * Deterministic sandbox name for a session key. + * + *

+ * Kept in Java so the name is a pure function of the session key and needs no + * lookup: the gateway's own object store is the registry. Mirrors the naming + * rule the prototype validated (DNS-1123 label: lowercase alphanumeric + '-', + * ≤63 chars). + */ + public static String sandboxNameFor(String sessionKey) { + if (sessionKey == null || sessionKey.trim().isEmpty()) { + throw new IllegalArgumentException("sessionKey is required to address an executor sandbox"); + } + String digest; + try { + java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256"); + byte[] hash = md.digest(sessionKey.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 8; i++) { + sb.append(String.format("%02x", hash[i])); + } + digest = sb.toString(); + } catch (java.security.NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable", e); + } + return "pyworker-" + digest; + } + + public static OpenShellTransport forSession(String sessionKey) { + String workerPort = Utility.getDIHelperProperty(WORKER_PORT_PROP); + if (workerPort == null || workerPort.trim().isEmpty()) { + workerPort = DEFAULT_WORKER_PORT; + } + return new OpenShellTransport(sandboxNameFor(sessionKey), workerPort.trim()); + } + + /** + * Create the sandbox if it does not exist, then forward its worker port. + * + *

+ * Create is idempotent from the caller's perspective: an "already exists" result + * is success, not an error. A sandbox that exists but has finished (phase + * {@code Completed}) is not reusable — the worker has exited and the + * next request must get a fresh sandbox, so callers should treat + * {@link #awaitReady()} failure as "delete and re-create". + * + * @return the local port to connect to + */ + public int start() throws IOException, InterruptedException { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + String template = Utility.getDIHelperProperty(TEMPLATE_PROP); + String label = propOr(LABEL_PROP, DEFAULT_LABEL); + int readyTimeout = intPropOr(READY_TIMEOUT_PROP, DEFAULT_READY_TIMEOUT_S); + + List create = new ArrayList<>(Arrays.asList(cli, "sandbox", "create", "--name", sandboxName, + "--keep", "--detach", "--label", label)); + if (template != null && !template.trim().isEmpty()) { + create.add("--template"); + create.add(template.trim()); + } + // The worker is the sandbox's canonical main process. It must be started with + // the insight folder it will write {insightFolder}/log.txt into. + create.add("--"); + create.add("python3"); + create.add("/opt/semosshome/py/gaas_tcp_socket_server.py"); + create.add("--port"); + create.add(workerPort); + create.add("--start"); + create.add("True"); + create.add("--py_folder"); + create.add("/opt/semosshome/py"); + create.add("--insight_folder"); + create.add("/opt/semosshome/insight"); + + classLogger.info("Starting executor sandbox {} (worker port {})", sandboxName, workerPort); + run(create, readyTimeout); + + this.forwardedPort = freeLocalPort(); + List forward = Arrays.asList(cli, "forward", "service", "--target-port", workerPort, "--local", + String.valueOf(this.forwardedPort), sandboxName); + classLogger.info("Forwarding localhost:{} -> sandbox {}:{}", this.forwardedPort, sandboxName, workerPort); + ProcessBuilder pb = new ProcessBuilder(forward); + pb.redirectErrorStream(true); + this.forwardProcess = pb.start(); + return this.forwardedPort; + } + + public int getForwardedPort() { + return forwardedPort; + } + + /** + * Stop forwarding and delete the sandbox. + * + *

+ * This is the whole teardown: the gateway delete is authoritative and handles + * process, cgroup and network cleanup. There is no "graceful worker shutdown" + * verb to call first, because the worker cannot outlive its sandbox. + */ + public void close() { + if (forwardProcess != null) { + forwardProcess.destroy(); + try { + if (!forwardProcess.waitFor(5, TimeUnit.SECONDS)) { + forwardProcess.destroyForcibly(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + forwardProcess = null; + } + try { + run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "delete", sandboxName), 60); + } catch (Exception e) { + classLogger.warn("Failed to delete executor sandbox {}", sandboxName, e); + } + } + + // ------------------------------------------------------------------ helpers + + private static String propOr(String key, String fallback) { + String v = Utility.getDIHelperProperty(key); + return (v == null || v.trim().isEmpty()) ? fallback : v.trim(); + } + + private static int intPropOr(String key, int fallback) { + String v = Utility.getDIHelperProperty(key); + if (v == null || v.trim().isEmpty()) { + return fallback; + } + try { + return Integer.parseInt(v.trim()); + } catch (NumberFormatException e) { + return fallback; + } + } + + /** + * Ask the OS for a free port rather than picking one: the forward binds + * host-local, and two sessions must not collide. + */ + private static int freeLocalPort() throws IOException { + try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { + return s.getLocalPort(); + } + } + + private static void run(List cmd, int timeoutSeconds) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.redirectErrorStream(true); + Process p = pb.start(); + StringBuilder out = new StringBuilder(); + try (java.io.BufferedReader r = new java.io.BufferedReader( + new java.io.InputStreamReader(p.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + out.append(line).append('\n'); + } + } + if (!p.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + p.destroyForcibly(); + throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); + } + if (p.exitValue() != 0) { + // A sandbox that already exists is the normal re-entry case, not a failure. + String lower = out.toString().toLowerCase(); + if (!lower.contains("already exists")) { + throw new IOException("command failed (" + p.exitValue() + "): " + String.join(" ", cmd) + "\n" + + out.toString().trim()); + } + } + } +} From d3050fba108e99bd7099f7da0cd8596c0989094e Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 18:34:21 +0000 Subject: [PATCH 02/56] feat(sandbox): wire image and policy properties into OpenShellTransport OPENSHELL_IMAGE (used when no OPENSHELL_TEMPLATE is set) and OPENSHELL_POLICY (a filesystem/egress policy YAML applied at creation). The policy has to be set at creation because Landlock settings are locked at sandbox startup and cannot be widened afterwards -- that immutability is the control, not an inconvenience. The stock OpenShell base image carries no python3, so an executor image is required in practice; this makes it a deployment property rather than a hard-coded default. --- src/prerna/util/OpenShellTransport.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index af7ced6953..a15661ed4a 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -72,6 +72,10 @@ public final class OpenShellTransport { public static final String CLI_PROP = "OPENSHELL_CLI"; /** Property naming the sandbox workload template to create from. */ public static final String TEMPLATE_PROP = "OPENSHELL_TEMPLATE"; + /** Property naming the sandbox image (used when no template is set). */ + public static final String IMAGE_PROP = "OPENSHELL_IMAGE"; + /** Property naming a filesystem-policy YAML applied at creation. */ + public static final String POLICY_PROP = "OPENSHELL_POLICY"; /** Property naming the port the worker listens on *inside* the sandbox. */ public static final String WORKER_PORT_PROP = "OPENSHELL_WORKER_PORT"; /** Property naming the label value used to group executor sandboxes. */ @@ -153,6 +157,21 @@ public int start() throws IOException, InterruptedException { if (template != null && !template.trim().isEmpty()) { create.add("--template"); create.add(template.trim()); + } else { + // No template: use an explicit image. The stock OpenShell base image + // carries no python3, so an executor image is mandatory in practice. + String image = Utility.getDIHelperProperty(IMAGE_PROP); + if (image != null && !image.trim().isEmpty()) { + create.add("--from"); + create.add(image.trim()); + } + } + // The filesystem/egress policy is applied at creation: Landlock settings are + // locked at startup and cannot be widened later, which is the point. + String policy = Utility.getDIHelperProperty(POLICY_PROP); + if (policy != null && !policy.trim().isEmpty()) { + create.add("--policy"); + create.add(policy.trim()); } // The worker is the sandbox's canonical main process. It must be started with // the insight folder it will write {insightFolder}/log.txt into. From a5808e10d823d016da99f33473a7ad80ffaf9a41 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 19:10:29 +0000 Subject: [PATCH 03/56] fix(sandbox): fail loudly when image/policy are missing; add transport harness Three fixes found by running the transport outside Tomcat: 1. FAIL LOUDLY on missing config. create() added --from/--policy only when the property was present, so an unset OPENSHELL_IMAGE silently fell back to the stock OpenShell base image -- which has no python3. The sandbox is still created and reaches Ready; only the workload fails to spawn, so the caller sees an unrelated null response several layers away. Both inputs now throw at construction. Same defect class as the AGENT_SANDBOX_ENABLE/ENFORCE mismatch: a control that is configured but never reaches enforcement, failing open. 2. Sandbox name must fit the gateway's 19-character limit (was 25). MAX_SANDBOX_NAME_LENGTH is now a named constant with its reason recorded; the digest is truncated to fit. 3. Wait for the port forward to actually accept before returning the port. The CLI binds lazily, so returning early produced a connection refused that surfaced as a null response. The readiness probe is a TCP connect, never a protocol exchange, because the worker is single-connection. Also adds OpenShellTransportHarness: the same classpath with a console attached, which is what made all three visible. It bootstraps the DIHelper from RDF_Map.prop because Utility.getDIHelperProperty reads that, not -D system properties -- a distinction that cost three debugging iterations. Known open issue recorded in the wiki: the worker exits ~0.5s after start when it is the sandbox's canonical main process, while the identical command run as a backgrounded child stays up and listens. Under investigation; the transport itself is correct up to and including the forward. --- src/prerna/util/OpenShellTransport.java | 135 +++++++++++++++--- .../util/OpenShellTransportHarness.java | 117 +++++++++++++++ 2 files changed, 235 insertions(+), 17 deletions(-) create mode 100644 src/prerna/util/OpenShellTransportHarness.java diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index a15661ed4a..7ffed154a2 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -92,38 +92,59 @@ public final class OpenShellTransport { private final String workerPort; private int forwardedPort = -1; private Process forwardProcess; + private StringBuilder forwardLog; private OpenShellTransport(String sandboxName, String workerPort) { this.sandboxName = sandboxName; this.workerPort = workerPort; } + /** + * Maximum sandbox name length the gateway accepts. + * + *

+ * Measured 2026-09-28: {@code openshell sandbox create} rejects longer names + * with Client specified an invalid argument: name exceeds maximum length + * (25 > 19). This is stricter than the DNS-1123 limit the name is + * otherwise modelled on, and it is not documented at the call site, so it is + * pinned here with its own regression check. + */ + public static final int MAX_SANDBOX_NAME_LENGTH = 19; + + private static final String NAME_PREFIX = "pyw-"; + /** * Deterministic sandbox name for a session key. * *

* Kept in Java so the name is a pure function of the session key and needs no - * lookup: the gateway's own object store is the registry. Mirrors the naming - * rule the prototype validated (DNS-1123 label: lowercase alphanumeric + '-', - * ≤63 chars). + * lookup: the gateway's own object store is the registry. The digest is + * truncated so the whole name fits {@link #MAX_SANDBOX_NAME_LENGTH}: with a + * 4-character prefix that leaves 15 hex characters (60 bits), which is still + * far beyond any realistic collision concern for concurrently active sessions. */ public static String sandboxNameFor(String sessionKey) { if (sessionKey == null || sessionKey.trim().isEmpty()) { throw new IllegalArgumentException("sessionKey is required to address an executor sandbox"); } + int digestChars = MAX_SANDBOX_NAME_LENGTH - NAME_PREFIX.length(); String digest; try { java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256"); byte[] hash = md.digest(sessionKey.getBytes(java.nio.charset.StandardCharsets.UTF_8)); StringBuilder sb = new StringBuilder(); - for (int i = 0; i < 8; i++) { + for (int i = 0; sb.length() < digestChars; i++) { sb.append(String.format("%02x", hash[i])); } - digest = sb.toString(); + digest = sb.substring(0, digestChars); } catch (java.security.NoSuchAlgorithmException e) { throw new IllegalStateException("SHA-256 unavailable", e); } - return "pyworker-" + digest; + String name = NAME_PREFIX + digest; + if (name.length() > MAX_SANDBOX_NAME_LENGTH) { + throw new IllegalStateException("sandbox name exceeds the gateway limit: " + name); + } + return name; } public static OpenShellTransport forSession(String sessionKey) { @@ -158,21 +179,31 @@ public int start() throws IOException, InterruptedException { create.add("--template"); create.add(template.trim()); } else { - // No template: use an explicit image. The stock OpenShell base image - // carries no python3, so an executor image is mandatory in practice. + // An explicit image is REQUIRED, not optional. The stock OpenShell base + // image carries no python3, and if --from is omitted the sandbox is still + // created and reaches Ready -- only the workload fails to spawn, which + // surfaces to the caller as an unrelated null response. Fail loudly here + // instead of silently provisioning an image that cannot run the worker. String image = Utility.getDIHelperProperty(IMAGE_PROP); - if (image != null && !image.trim().isEmpty()) { - create.add("--from"); - create.add(image.trim()); + if (image == null || image.trim().isEmpty()) { + throw new IOException("no sandbox image: set " + IMAGE_PROP + " (or " + TEMPLATE_PROP + + "). The stock OpenShell image has no python3, so omitting this produces a sandbox" + + " that starts and then cannot run the worker."); } + create.add("--from"); + create.add(image.trim()); } // The filesystem/egress policy is applied at creation: Landlock settings are - // locked at startup and cannot be widened later, which is the point. + // locked at startup and cannot be widened later, which is the point. Omitting + // it silently means NO filesystem policy, so require it explicitly too. String policy = Utility.getDIHelperProperty(POLICY_PROP); - if (policy != null && !policy.trim().isEmpty()) { - create.add("--policy"); - create.add(policy.trim()); + if (policy == null || policy.trim().isEmpty()) { + throw new IOException("no sandbox policy: set " + POLICY_PROP + + ". Creating a sandbox without a filesystem policy leaves the workload unfenced, which" + + " is never what a deployment intends."); } + create.add("--policy"); + create.add(policy.trim()); // The worker is the sandbox's canonical main process. It must be started with // the insight folder it will write {insightFolder}/log.txt into. create.add("--"); @@ -188,7 +219,8 @@ public int start() throws IOException, InterruptedException { create.add("/opt/semosshome/insight"); classLogger.info("Starting executor sandbox {} (worker port {})", sandboxName, workerPort); - run(create, readyTimeout); + String createOut = run(create, readyTimeout); + classLogger.info("Sandbox create output for {}: {}", sandboxName, createOut); this.forwardedPort = freeLocalPort(); List forward = Arrays.asList(cli, "forward", "service", "--target-port", workerPort, "--local", @@ -197,9 +229,77 @@ public int start() throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(forward); pb.redirectErrorStream(true); this.forwardProcess = pb.start(); + this.forwardLog = new StringBuilder(); + // A child whose pipe fills up blocks forever. The forward emits periodic + // output, so it must be drained continuously or the port never binds. + Thread drain = new Thread(() -> { + try (java.io.BufferedReader r = new java.io.BufferedReader(new java.io.InputStreamReader( + forwardProcess.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + synchronized (forwardLog) { + forwardLog.append(line).append('\n'); + } + } + } catch (IOException e) { + // process exited; the readiness check will report it + } + }, "openshell-forward-" + sandboxName); + drain.setDaemon(true); + drain.start(); + + // The forward binds lazily: the CLI starts, then opens the local listener. + // Returning before that produces a connection refused at the socket client + // and surfaces as "Received a null PayloadStruct response" -- measured + // 2026-09-28. Poll until the port actually accepts, or fail here with a + // clear message rather than downstream with a confusing one. + if (!awaitForwardReady(this.forwardedPort, readyTimeout)) { + String detail = readAvailable(forwardProcess); + close(); + throw new IOException("port forward for sandbox " + sandboxName + " did not become ready on 127.0.0.1:" + + this.forwardedPort + " within " + readyTimeout + "s" + (detail.isEmpty() ? "" : ": " + detail)); + } + classLogger.info("Executor sandbox {} ready; worker reachable on 127.0.0.1:{}", sandboxName, + this.forwardedPort); return this.forwardedPort; } + /** + * Poll the forwarded port until something accepts a connection. + * + *

+ * A TCP connect is the right probe here and deliberately not a protocol exchange: + * the worker is single-connection ({@code --max_count 1}), so sending anything to + * test readiness would consume its only slot and, on disconnect, end the worker. + */ + private static boolean awaitForwardReady(int port, int timeoutSeconds) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + while (System.nanoTime() < deadline) { + try (java.net.Socket probe = new java.net.Socket()) { + probe.connect(new java.net.InetSocketAddress("127.0.0.1", port), 1000); + return true; + } catch (IOException notYet) { + try { + Thread.sleep(250); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + } + return false; + } + + private String readAvailable(Process p) { + if (forwardLog == null) { + return ""; + } + synchronized (forwardLog) { + String s = forwardLog.toString().trim(); + return s.length() > 600 ? s.substring(s.length() - 600) : s; + } + } + public int getForwardedPort() { return forwardedPort; } @@ -260,7 +360,7 @@ private static int freeLocalPort() throws IOException { } } - private static void run(List cmd, int timeoutSeconds) throws IOException, InterruptedException { + private static String run(List cmd, int timeoutSeconds) throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(cmd); pb.redirectErrorStream(true); Process p = pb.start(); @@ -284,5 +384,6 @@ private static void run(List cmd, int timeoutSeconds) throws IOException + out.toString().trim()); } } + return out.toString().trim(); } } diff --git a/src/prerna/util/OpenShellTransportHarness.java b/src/prerna/util/OpenShellTransportHarness.java new file mode 100644 index 0000000000..41f2d1025d --- /dev/null +++ b/src/prerna/util/OpenShellTransportHarness.java @@ -0,0 +1,117 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +import java.net.InetSocketAddress; +import java.net.Socket; + +/** + * Standalone harness for {@link OpenShellTransport}, run outside Tomcat. + * + *

+ * Exists because debugging this inside a servlet was guesswork: the failure + * surfaced as "Received a null PayloadStruct response" several layers away from + * the cause, and the transport's own INFO lines were not visible in the Tomcat + * log. Running the same code with a console attached removes both problems. + * + *

+ * Usage: {@code java ... prerna.util.OpenShellTransportHarness } + */ +public final class OpenShellTransportHarness { + + private static byte[] readExactly(java.io.InputStream in, int n) throws java.io.IOException { + byte[] buf = new byte[n]; + int off = 0; + while (off < n) { + int read = in.read(buf, off, n - off); + if (read < 0) { + throw new java.io.IOException("peer closed after " + off + "/" + n + " bytes"); + } + off += read; + } + return buf; + } + + public static void main(String[] args) throws Exception { + // Bootstrap the DIHelper from the RDF map exactly as the webapp does. Without + // this, Utility.getDIHelperProperty() returns null for every property -- which + // is how an earlier round of debugging lost three iterations to a sandbox that + // was created from the stock image because OPENSHELL_IMAGE "was not set". + String rdf = System.getProperty("RDF_MAP"); + if (rdf == null || rdf.trim().isEmpty()) { + rdf = "/opt/semosshome/RDF_Map.prop"; + } + DIHelper.getInstance().loadCoreProp(rdf); + System.out.println("loaded RDF: " + rdf); + System.out.println("OPENSHELL_IMAGE=" + Utility.getDIHelperProperty(OpenShellTransport.IMAGE_PROP)); + System.out.println("OPENSHELL_POLICY=" + Utility.getDIHelperProperty(OpenShellTransport.POLICY_PROP)); + + String sessionKey = args.length > 0 ? args[0] : "harness-session-1"; + System.out.println("sandbox name would be: " + OpenShellTransport.sandboxNameFor(sessionKey)); + System.out.println("name length: " + OpenShellTransport.sandboxNameFor(sessionKey).length() + " (max " + + OpenShellTransport.MAX_SANDBOX_NAME_LENGTH + ")"); + + OpenShellTransport t = OpenShellTransport.forSession(sessionKey); + System.out.println("starting transport..."); + long t0 = System.currentTimeMillis(); + int port = t.start(); + System.out.println("start() returned port " + port + " after " + (System.currentTimeMillis() - t0) + "ms"); + + // Prove the worker is actually reachable, not merely that a port opened. + try (Socket s = new Socket()) { + s.connect(new InetSocketAddress("127.0.0.1", port), 5000); + // One framed request, exactly as the prototype does: [4-byte size][20-byte epoch][payload] + byte[] payload = ("{\"epoc\":\"00000000000000000001\",\"operation\":\"PYTHON\",\"methodName\":\"probe\"," + + "\"payload\":[\"import os; print('HARNESS_OK', os.getpid())\"],\"payloadClasses\":" + + "[\"java.lang.String\"],\"longRunning\":true,\"hasReturn\":true,\"response\":false," + + "\"interim\":false,\"insightId\":\"harness\",\"executionInsightId\":\"harness\"," + + "\"sessionId\":\"harness\",\"jobId\":\"harness\",\"mdc\":{},\"asset_paths\":[]," + + "\"runtime_vars\":{},\"append_vars\":{}}").getBytes(java.nio.charset.StandardCharsets.UTF_8); + byte[] epoc = "00000000000000000001".getBytes("UTF-8"); + java.io.OutputStream out = s.getOutputStream(); + out.write(new byte[] { (byte) (payload.length >>> 24), (byte) (payload.length >>> 16), + (byte) (payload.length >>> 8), (byte) payload.length }); + out.write(epoc); + out.write(payload); + out.flush(); + + java.io.InputStream in = s.getInputStream(); + // readNBytes() may return fewer bytes than asked on a stream (and throws on + // an empty read if the peer closed). Read exactly N, or fail with the count. + byte[] hdr = readExactly(in, 4); + int size = ((hdr[0] & 0xff) << 24) | ((hdr[1] & 0xff) << 16) | ((hdr[2] & 0xff) << 8) | (hdr[3] & 0xff); + System.out.println("reply declared size=" + size); + byte[] body = readExactly(in, size); + System.out.println("reply body=" + new String(body, java.nio.charset.StandardCharsets.UTF_8)); + } + + System.out.println("closing transport..."); + t.close(); + System.out.println("done"); + } +} From d293acc5969bf358c691a868c761f009d90ac3d2 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 19:28:35 +0000 Subject: [PATCH 04/56] fix(sandbox): readiness probe was killing every sandbox it checked awaitForwardReady() connected to the forwarded port. The forward proxies straight into the worker, the worker is single-connection (--max_count 1), so the probe consumed its only slot and its disconnect called stop_request() -- ending the worker and taking the sandbox to Completed. The symptom was indistinguishable from "the workload never started", which is why image, policy, argv, HOME, I/O pattern and environment breadth were all eliminated first. Readiness now comes from the CLI's own output: forward service prints a line containing "Forwarding" once its listener is bound. The method documents why it must not be "simplified" back into a socket connect. Also pins HOME for CLI children (OPENSHELL_CLI_HOME). The CLI resolves its gateway registration and mTLS client bundle from $HOME/.local/state/openshell/, so a caller with a thinner environment (Tomcat: 13 vars vs a shell's 124) could silently reach a different or missing gateway. Found by running the CLI with HOME unset, which reports "Or register one with: openshell gateway add". Adds OPENSHELL_KEEP_SANDBOX so a failed run leaves the sandbox in place for inspection -- deleting it destroyed the evidence every previous time. VERIFIED: a sandbox created by this class served a working persistent REPL through the protocol client (x=1 persists, mutates to 42, import math survives). --- src/prerna/util/OpenShellTransport.java | 176 +++++++++++++----- .../util/OpenShellTransportHarness.java | 15 ++ 2 files changed, 147 insertions(+), 44 deletions(-) diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 7ffed154a2..2f47c3ed1d 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -70,6 +70,29 @@ public final class OpenShellTransport { /** Property naming the OpenShell CLI. */ public static final String CLI_PROP = "OPENSHELL_CLI"; + /** + * Explicit {@code HOME} for the CLI child process. + * + *

+ * This is required, and it is the subtlest thing in this class. The + * {@code openshell} CLI resolves both its gateway registration and its mTLS client + * bundle from $HOME/.local/state/openshell/. A caller with a different + * (or unset) {@code HOME} therefore talks to a different, or non-existent, gateway -- + * and the failure is quiet: the sandbox is still created, but it is not the one the + * caller can reach, so the worker appears to die and the caller sees an unrelated + * null response several layers away. + * + *

+ * Measured 2026-09-28: the same argv stays {@code Ready} when launched from a shell + * with a full environment, and dies when launched from a process with Tomcat's much + * thinner one (13 variables versus 124). With {@code HOME} unset the CLI reports + * "Or register one with: openshell gateway add". + * + *

+ * Pinned explicitly rather than inherited, so behaviour does not depend on how the + * JVM happened to be started. + */ + public static final String CLI_HOME_PROP = "OPENSHELL_CLI_HOME"; /** Property naming the sandbox workload template to create from. */ public static final String TEMPLATE_PROP = "OPENSHELL_TEMPLATE"; /** Property naming the sandbox image (used when no template is set). */ @@ -88,8 +111,20 @@ public final class OpenShellTransport { private static final String DEFAULT_LABEL = "app=semoss-executor"; private static final int DEFAULT_READY_TIMEOUT_S = 180; + /** + * When true, {@link #close()} stops the forward but leaves the sandbox in place. + * + *

+ * Exists because the default teardown deletes the sandbox, which destroys the + * evidence needed to diagnose a startup failure: the worker's own output and the + * sandbox's final phase both vanish. Set with {@code -DOPENSHELL_KEEP_SANDBOX=true} + * or the {@link #KEEP_SANDBOX_PROP} property while investigating. + */ + public static final String KEEP_SANDBOX_PROP = "OPENSHELL_KEEP_SANDBOX"; + private final String sandboxName; private final String workerPort; + private final boolean keepSandbox; private int forwardedPort = -1; private Process forwardProcess; private StringBuilder forwardLog; @@ -97,6 +132,11 @@ public final class OpenShellTransport { private OpenShellTransport(String sandboxName, String workerPort) { this.sandboxName = sandboxName; this.workerPort = workerPort; + String keep = System.getProperty(KEEP_SANDBOX_PROP); + if (keep == null || keep.trim().isEmpty()) { + keep = Utility.getDIHelperProperty(KEEP_SANDBOX_PROP); + } + this.keepSandbox = "true".equalsIgnoreCase(keep == null ? "" : keep.trim()); } /** @@ -147,31 +187,25 @@ public static String sandboxNameFor(String sessionKey) { return name; } - public static OpenShellTransport forSession(String sessionKey) { - String workerPort = Utility.getDIHelperProperty(WORKER_PORT_PROP); - if (workerPort == null || workerPort.trim().isEmpty()) { - workerPort = DEFAULT_WORKER_PORT; + /** + * The exact argv {@link #start()} will run. Exposed so a hand-run command can be + * diffed against it byte for byte: every debugging round that failed here failed + * because the Java-side argv differed from the hand-run one in a way that was not + * visible by eye. + */ + public static String describeCreateCommand(String sessionKey) { + try { + return String.join(" ", createCommand(sandboxNameFor(sessionKey))); + } catch (Exception e) { + return "(cannot build: " + e.getMessage() + ")"; } - return new OpenShellTransport(sandboxNameFor(sessionKey), workerPort.trim()); } - /** - * Create the sandbox if it does not exist, then forward its worker port. - * - *

- * Create is idempotent from the caller's perspective: an "already exists" result - * is success, not an error. A sandbox that exists but has finished (phase - * {@code Completed}) is not reusable — the worker has exited and the - * next request must get a fresh sandbox, so callers should treat - * {@link #awaitReady()} failure as "delete and re-create". - * - * @return the local port to connect to - */ - public int start() throws IOException, InterruptedException { + private static List createCommand(String sandboxName) throws IOException { String cli = propOr(CLI_PROP, DEFAULT_CLI); String template = Utility.getDIHelperProperty(TEMPLATE_PROP); String label = propOr(LABEL_PROP, DEFAULT_LABEL); - int readyTimeout = intPropOr(READY_TIMEOUT_PROP, DEFAULT_READY_TIMEOUT_S); + String workerPort = propOr(WORKER_PORT_PROP, DEFAULT_WORKER_PORT); List create = new ArrayList<>(Arrays.asList(cli, "sandbox", "create", "--name", sandboxName, "--keep", "--detach", "--label", label)); @@ -179,11 +213,6 @@ public int start() throws IOException, InterruptedException { create.add("--template"); create.add(template.trim()); } else { - // An explicit image is REQUIRED, not optional. The stock OpenShell base - // image carries no python3, and if --from is omitted the sandbox is still - // created and reaches Ready -- only the workload fails to spawn, which - // surfaces to the caller as an unrelated null response. Fail loudly here - // instead of silently provisioning an image that cannot run the worker. String image = Utility.getDIHelperProperty(IMAGE_PROP); if (image == null || image.trim().isEmpty()) { throw new IOException("no sandbox image: set " + IMAGE_PROP + " (or " + TEMPLATE_PROP @@ -193,9 +222,6 @@ public int start() throws IOException, InterruptedException { create.add("--from"); create.add(image.trim()); } - // The filesystem/egress policy is applied at creation: Landlock settings are - // locked at startup and cannot be widened later, which is the point. Omitting - // it silently means NO filesystem policy, so require it explicitly too. String policy = Utility.getDIHelperProperty(POLICY_PROP); if (policy == null || policy.trim().isEmpty()) { throw new IOException("no sandbox policy: set " + POLICY_PROP @@ -204,8 +230,6 @@ public int start() throws IOException, InterruptedException { } create.add("--policy"); create.add(policy.trim()); - // The worker is the sandbox's canonical main process. It must be started with - // the insight folder it will write {insightFolder}/log.txt into. create.add("--"); create.add("python3"); create.add("/opt/semosshome/py/gaas_tcp_socket_server.py"); @@ -217,6 +241,35 @@ public int start() throws IOException, InterruptedException { create.add("/opt/semosshome/py"); create.add("--insight_folder"); create.add("/opt/semosshome/insight"); + return create; + } + + public static OpenShellTransport forSession(String sessionKey) { + String workerPort = Utility.getDIHelperProperty(WORKER_PORT_PROP); + if (workerPort == null || workerPort.trim().isEmpty()) { + workerPort = DEFAULT_WORKER_PORT; + } + return new OpenShellTransport(sandboxNameFor(sessionKey), workerPort.trim()); + } + + /** + * Create the sandbox if it does not exist, then forward its worker port. + * + *

+ * Create is idempotent from the caller's perspective: an "already exists" result + * is success, not an error. A sandbox that exists but has finished (phase + * {@code Completed}) is not reusable — the worker has exited and the + * next request must get a fresh sandbox, so callers should treat + * {@link #awaitReady()} failure as "delete and re-create". + * + * @return the local port to connect to + */ + public int start() throws IOException, InterruptedException { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + int readyTimeout = intPropOr(READY_TIMEOUT_PROP, DEFAULT_READY_TIMEOUT_S); + // Single source of truth: the argv used here is exactly the one + // describeCreateCommand() prints, so a hand-run diff is meaningful. + List create = createCommand(sandboxName); classLogger.info("Starting executor sandbox {} (worker port {})", sandboxName, workerPort); String createOut = run(create, readyTimeout); @@ -227,6 +280,7 @@ public int start() throws IOException, InterruptedException { String.valueOf(this.forwardedPort), sandboxName); classLogger.info("Forwarding localhost:{} -> sandbox {}:{}", this.forwardedPort, sandboxName, workerPort); ProcessBuilder pb = new ProcessBuilder(forward); + pb.environment().putAll(cliEnv()); pb.redirectErrorStream(true); this.forwardProcess = pb.start(); this.forwardLog = new StringBuilder(); @@ -253,7 +307,7 @@ public int start() throws IOException, InterruptedException { // and surfaces as "Received a null PayloadStruct response" -- measured // 2026-09-28. Poll until the port actually accepts, or fail here with a // clear message rather than downstream with a confusing one. - if (!awaitForwardReady(this.forwardedPort, readyTimeout)) { + if (!awaitForwardReady(readyTimeout)) { String detail = readAvailable(forwardProcess); close(); throw new IOException("port forward for sandbox " + sandboxName + " did not become ready on 127.0.0.1:" @@ -265,27 +319,40 @@ public int start() throws IOException, InterruptedException { } /** - * Poll the forwarded port until something accepts a connection. + * Poll until the forward is up, without ever connecting to the forwarded port. * *

- * A TCP connect is the right probe here and deliberately not a protocol exchange: - * the worker is single-connection ({@code --max_count 1}), so sending anything to - * test readiness would consume its only slot and, on disconnect, end the worker. + * Do not replace this with a TCP connect. An earlier version did exactly + * that and it killed every sandbox it was meant to check: the forward proxies + * straight into the worker, the worker is single-connection ({@code --max_count 1}), + * so the probe consumed the worker's only slot and its disconnect triggered + * {@code stop_request()} -- ending the worker and taking the sandbox to + * {@code Completed}. The symptom was indistinguishable from "the worker never + * started", which is why it took so long to find. See the wiki's + * "never open a second connection to the worker's port" rule; this method exists to + * honour it. + * + *

+ * Readiness is therefore taken from the CLI's own output: {@code forward service} + * prints a line containing {@code "Forwarding"} once the local listener is bound. */ - private static boolean awaitForwardReady(int port, int timeoutSeconds) { + private boolean awaitForwardReady(int timeoutSeconds) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); while (System.nanoTime() < deadline) { - try (java.net.Socket probe = new java.net.Socket()) { - probe.connect(new java.net.InetSocketAddress("127.0.0.1", port), 1000); - return true; - } catch (IOException notYet) { - try { - Thread.sleep(250); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; + if (forwardProcess == null || !forwardProcess.isAlive()) { + return false; + } + synchronized (forwardLog) { + if (forwardLog.indexOf("Forwarding") >= 0) { + return true; } } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } } return false; } @@ -324,6 +391,10 @@ public void close() { } forwardProcess = null; } + if (keepSandbox) { + classLogger.warn("KEEP_SANDBOX is set; leaving sandbox {} in place for inspection", sandboxName); + return; + } try { run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "delete", sandboxName), 60); } catch (Exception e) { @@ -333,6 +404,22 @@ public void close() { // ------------------------------------------------------------------ helpers + /** + * Environment for CLI children: inherit the caller's, but pin {@code HOME} so the + * CLI always finds the same gateway registration and client certificate bundle. + */ + private static java.util.Map cliEnv() { + java.util.Map env = new java.util.HashMap<>(System.getenv()); + String home = System.getProperty(CLI_HOME_PROP); + if (home == null || home.trim().isEmpty()) { + home = Utility.getDIHelperProperty(CLI_HOME_PROP); + } + if (home != null && !home.trim().isEmpty()) { + env.put("HOME", home.trim()); + } + return env; + } + private static String propOr(String key, String fallback) { String v = Utility.getDIHelperProperty(key); return (v == null || v.trim().isEmpty()) ? fallback : v.trim(); @@ -362,6 +449,7 @@ private static int freeLocalPort() throws IOException { private static String run(List cmd, int timeoutSeconds) throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(cmd); + pb.environment().putAll(cliEnv()); pb.redirectErrorStream(true); Process p = pb.start(); StringBuilder out = new StringBuilder(); diff --git a/src/prerna/util/OpenShellTransportHarness.java b/src/prerna/util/OpenShellTransportHarness.java index 41f2d1025d..7b97013cf6 100644 --- a/src/prerna/util/OpenShellTransportHarness.java +++ b/src/prerna/util/OpenShellTransportHarness.java @@ -76,6 +76,8 @@ public static void main(String[] args) throws Exception { System.out.println("name length: " + OpenShellTransport.sandboxNameFor(sessionKey).length() + " (max " + OpenShellTransport.MAX_SANDBOX_NAME_LENGTH + ")"); + System.out.println("EXPECTED ARGV:"); + System.out.println(" " + OpenShellTransport.describeCreateCommand(sessionKey)); OpenShellTransport t = OpenShellTransport.forSession(sessionKey); System.out.println("starting transport..."); long t0 = System.currentTimeMillis(); @@ -108,6 +110,19 @@ public static void main(String[] args) throws Exception { System.out.println("reply declared size=" + size); byte[] body = readExactly(in, size); System.out.println("reply body=" + new String(body, java.nio.charset.StandardCharsets.UTF_8)); + // Keep reading: the worker streams LOG frames before the terminal response. + for (int i = 0; i < 5; i++) { + try { + byte[] h2 = readExactly(in, 4); + int s2 = ((h2[0] & 0xff) << 24) | ((h2[1] & 0xff) << 16) | ((h2[2] & 0xff) << 8) | (h2[3] & 0xff); + byte[] b2 = readExactly(in, s2); + System.out.println("frame " + (i + 2) + " (" + s2 + " bytes): " + + new String(b2, java.nio.charset.StandardCharsets.UTF_8)); + } catch (Exception more) { + System.out.println("(no more frames: " + more.getMessage() + ")"); + break; + } + } } System.out.println("closing transport..."); From 8ce5db1655a7186846b1a8e014bd31c61d2c8c79 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 19:40:30 +0000 Subject: [PATCH 05/56] fix(sandbox): log forward failures instead of swallowing them The forward's exit reason was being discarded: awaitForwardReady returned false without recording why, and close() then deleted the sandbox, so the evidence went with it. Both paths now log the CLI output and the process exit code before cleanup. Also records two measured facts in comments so they are not re-derived: - The forwarded port IS listening once start() returns (verified), so a connection failure is not a readiness problem. - The worker's insight/log.txt captures every request it receives, which is the only reliable way to tell "never arrived" from "arrived and failed" -- the sandbox phase and the supervisor log both say nothing about the workload. No behavioural change to the transport path. --- src/prerna/util/OpenShellTransport.java | 9 ++++++++- src/prerna/util/OpenShellTransportHarness.java | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 2f47c3ed1d..3e2cbe5cab 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -309,9 +309,13 @@ public int start() throws IOException, InterruptedException { // clear message rather than downstream with a confusing one. if (!awaitForwardReady(readyTimeout)) { String detail = readAvailable(forwardProcess); + boolean alive = forwardProcess != null && forwardProcess.isAlive(); + classLogger.error("Forward for sandbox {} did not become ready on 127.0.0.1:{} (alive={}): {}", + sandboxName, this.forwardedPort, alive, detail); close(); throw new IOException("port forward for sandbox " + sandboxName + " did not become ready on 127.0.0.1:" - + this.forwardedPort + " within " + readyTimeout + "s" + (detail.isEmpty() ? "" : ": " + detail)); + + this.forwardedPort + " within " + readyTimeout + "s (process alive=" + alive + ")" + + (detail.isEmpty() ? "" : ": " + detail)); } classLogger.info("Executor sandbox {} ready; worker reachable on 127.0.0.1:{}", sandboxName, this.forwardedPort); @@ -340,6 +344,9 @@ private boolean awaitForwardReady(int timeoutSeconds) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); while (System.nanoTime() < deadline) { if (forwardProcess == null || !forwardProcess.isAlive()) { + // Died: record why before returning, or the reason is lost with the process. + classLogger.error("Forward process exited early (exit={}): {}", + forwardProcess == null ? "null" : forwardProcess.exitValue(), readAvailable(forwardProcess)); return false; } synchronized (forwardLog) { diff --git a/src/prerna/util/OpenShellTransportHarness.java b/src/prerna/util/OpenShellTransportHarness.java index 7b97013cf6..2fe39db6d8 100644 --- a/src/prerna/util/OpenShellTransportHarness.java +++ b/src/prerna/util/OpenShellTransportHarness.java @@ -44,6 +44,15 @@ */ public final class OpenShellTransportHarness { + private static String canConnect(int port) { + try (Socket s = new Socket()) { + s.connect(new InetSocketAddress("127.0.0.1", port), 2000); + return "yes"; + } catch (Exception e) { + return "no (" + e.getClass().getSimpleName() + ": " + e.getMessage() + ")"; + } + } + private static byte[] readExactly(java.io.InputStream in, int n) throws java.io.IOException { byte[] buf = new byte[n]; int off = 0; @@ -83,8 +92,15 @@ public static void main(String[] args) throws Exception { long t0 = System.currentTimeMillis(); int port = t.start(); System.out.println("start() returned port " + port + " after " + (System.currentTimeMillis() - t0) + "ms"); + System.out.println("is that port actually LISTENING? " + + canConnect(port)); - // Prove the worker is actually reachable, not merely that a port opened. + // Wait for the worker WITHOUT touching its port. "Forwarding" is printed when + // the CLI's listener binds, which precedes the worker's own readiness. A sleep + // is deliberate: probing the port would consume the worker's single connection + // slot and its disconnect would kill the worker -- see the wiki rule. + System.out.println("settling 4s before the single connection..."); + Thread.sleep(4000); try (Socket s = new Socket()) { s.connect(new InetSocketAddress("127.0.0.1", port), 5000); // One framed request, exactly as the prototype does: [4-byte size][20-byte epoch][payload] From 7707ebdb306a27225b4b085f362f7629c46e187e Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 19:52:38 +0000 Subject: [PATCH 06/56] fix(sandbox): read CLI output off the caller thread; add real-client harness run() read the CLI child with a blocking readLine() loop and no timeout. Because close() also calls run() to delete the sandbox, a wedged child parked the Tomcat request thread inside the failure path itself: the sandbox was never cleaned up and the call never returned. BufferedReader.readLine -> run(:459) -> close(:399) Now the read happens on a daemon thread while the caller waits on waitFor(timeout); on timeout the child is destroyed forcibly and the reader interrupted. A cleanup path must not be able to block indefinitely on the object it is cleaning up. Adds OpenShellNativeClientHarness, which wires the transport to NativePySocketClient -- the class that actually ships -- with no protocol code of its own. Verified: transport up on 38331, NativePySocketClient connected, ready=true. Debugging through a hand-rolled protocol harness had been mistaking a failing test client for a failing transport. --- .../util/OpenShellNativeClientHarness.java | 77 +++++++++++++++++++ src/prerna/util/OpenShellTransport.java | 41 +++++++--- 2 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 src/prerna/util/OpenShellNativeClientHarness.java diff --git a/src/prerna/util/OpenShellNativeClientHarness.java b/src/prerna/util/OpenShellNativeClientHarness.java new file mode 100644 index 0000000000..20ce61595e --- /dev/null +++ b/src/prerna/util/OpenShellNativeClientHarness.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +import prerna.tcp.client.NativePySocketClient; + +/** + * Drives the REAL client ({@link NativePySocketClient}) against a sandbox created by + * {@link OpenShellTransport}. + * + *

+ * This exists because the hand-rolled {@link OpenShellTransportHarness} proved the + * transport and then became the blocker: its own read path kept failing while an + * independent client succeeded against the same sandbox. Debugging through test + * scaffolding is how you spend hours on the wrong layer. This harness has no protocol + * code of its own -- it only wires the transport to the class that will actually ship. + * + *

+ * Usage: {@code java ... prerna.util.OpenShellNativeClientHarness } + */ +public final class OpenShellNativeClientHarness { + + public static void main(String[] args) throws Exception { + String rdf = System.getProperty("RDF_MAP", "/opt/semosshome/RDF_Map.prop"); + DIHelper.getInstance().loadCoreProp(rdf); + String sessionKey = args.length > 0 ? args[0] : "native-client-1"; + + OpenShellTransport t = OpenShellTransport.forSession(sessionKey); + int port = t.start(); + System.out.println("transport up on port " + port); + + // Wait for the worker WITHOUT touching its port: connecting to probe would + // consume the single connection slot and kill the worker. + System.out.println("settling 4s..."); + Thread.sleep(4000); + + NativePySocketClient client = new NativePySocketClient(java.util.Map.of()); + client.connect("127.0.0.1", port, false); + System.out.println("NativePySocketClient connected"); + + Thread reader = new Thread(client, "native-client-reader"); + reader.setDaemon(true); + reader.start(); + client.awaitReadyOrKill(30_000, 500); + System.out.println("client ready=" + client.isReady() + " connected=" + client.isConnected()); + + client.stopServer(); + client.close(); + t.close(); + System.out.println("done"); + } +} diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 3e2cbe5cab..64ec00f531 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -459,26 +459,49 @@ private static String run(List cmd, int timeoutSeconds) throws IOExcepti pb.environment().putAll(cliEnv()); pb.redirectErrorStream(true); Process p = pb.start(); + + // Read on a separate thread. A blocking readLine() here hangs the CALLER -- + // and because close() also calls run(), a wedged CLI child used to deadlock the + // failure path itself: the request thread sat in readLine() inside close() with + // no timeout, and the sandbox was never cleaned up. Measured 2026-09-28. StringBuilder out = new StringBuilder(); - try (java.io.BufferedReader r = new java.io.BufferedReader( - new java.io.InputStreamReader(p.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { - String line; - while ((line = r.readLine()) != null) { - out.append(line).append('\n'); + Thread reader = new Thread(() -> { + try (java.io.BufferedReader r = new java.io.BufferedReader(new java.io.InputStreamReader( + p.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + synchronized (out) { + out.append(line).append('\n'); + } + } + } catch (IOException ignored) { + // stream closed with the process; the exit code carries the outcome } - } + }, "openshell-cli-" + cmd.get(0)); + reader.setDaemon(true); + reader.start(); + if (!p.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { p.destroyForcibly(); + reader.interrupt(); throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); } + // Give the reader a moment to drain what the process wrote before it exited. + reader.join(2000); if (p.exitValue() != 0) { // A sandbox that already exists is the normal re-entry case, not a failure. - String lower = out.toString().toLowerCase(); + String collected; + synchronized (out) { + collected = out.toString(); + } + String lower = collected.toLowerCase(); if (!lower.contains("already exists")) { throw new IOException("command failed (" + p.exitValue() + "): " + String.join(" ", cmd) + "\n" - + out.toString().trim()); + + collected.trim()); } } - return out.toString().trim(); + synchronized (out) { + return out.toString().trim(); + } } } From 46652d9a4cef916033e2ca7207c62e69982f8fa4 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 19:54:15 +0000 Subject: [PATCH 07/56] test(sandbox): full PayloadStruct round trip through NativePySocketClient The harness previously only proved the handshake. It now sends OPERATION.PYTHON and reads the response, which is what PyTranslator does, so the chain is verified against shipping code rather than test scaffolding. Also corrects the PayloadStruct import (prerna.tcp, not prerna.om) and replaces a port probe with a plain sleep, since probing the forwarded port consumes the single-connection worker's only slot. --- .../util/OpenShellNativeClientHarness.java | 64 ++++++++++++++----- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/src/prerna/util/OpenShellNativeClientHarness.java b/src/prerna/util/OpenShellNativeClientHarness.java index 20ce61595e..cfc1d758c8 100644 --- a/src/prerna/util/OpenShellNativeClientHarness.java +++ b/src/prerna/util/OpenShellNativeClientHarness.java @@ -27,21 +27,23 @@ *******************************************************************************/ package prerna.util; +import prerna.tcp.PayloadStruct; import prerna.tcp.client.NativePySocketClient; /** - * Drives the REAL client ({@link NativePySocketClient}) against a sandbox created by - * {@link OpenShellTransport}. + * Drives the REAL client ({@link NativePySocketClient}) end to end against a sandbox + * created by {@link OpenShellTransport} -- connect, send, and read the reply. * *

- * This exists because the hand-rolled {@link OpenShellTransportHarness} proved the - * transport and then became the blocker: its own read path kept failing while an - * independent client succeeded against the same sandbox. Debugging through test - * scaffolding is how you spend hours on the wrong layer. This harness has no protocol - * code of its own -- it only wires the transport to the class that will actually ship. + * An earlier version only proved the handshake ({@code ready=true}). This one sends an + * actual {@link PayloadStruct} and waits for the response, which is the operation + * {@code PyTranslator} performs and the one the live pixel fails on. It carries no + * protocol code of its own: everything goes through the class that ships, because + * debugging through a hand-rolled protocol harness meant repeatedly mistaking a + * failing test client for a failing transport. * *

- * Usage: {@code java ... prerna.util.OpenShellNativeClientHarness } + * Usage: {@code java ... prerna.util.OpenShellNativeClientHarness [sessionKey]} */ public final class OpenShellNativeClientHarness { @@ -54,23 +56,51 @@ public static void main(String[] args) throws Exception { int port = t.start(); System.out.println("transport up on port " + port); - // Wait for the worker WITHOUT touching its port: connecting to probe would - // consume the single connection slot and kill the worker. - System.out.println("settling 4s..."); - Thread.sleep(4000); + // Wait for the worker WITHOUT touching its port: a probe connect would consume + // the single connection slot and its disconnect would kill the worker. + System.out.println("settling 5s..."); + Thread.sleep(5000); NativePySocketClient client = new NativePySocketClient(java.util.Map.of()); client.connect("127.0.0.1", port, false); - System.out.println("NativePySocketClient connected"); - Thread reader = new Thread(client, "native-client-reader"); reader.setDaemon(true); reader.start(); client.awaitReadyOrKill(30_000, 500); - System.out.println("client ready=" + client.isReady() + " connected=" + client.isConnected()); + System.out.println("connected=" + client.isConnected() + " ready=" + client.isReady()); + + System.out.println("--- sending a real PayloadStruct (what PyTranslator sends) ---"); + PayloadStruct ps = new PayloadStruct(); + ps.epoc = "00000000000000000001"; + ps.operation = PayloadStruct.OPERATION.PYTHON; + ps.methodName = "transportScriptWithExplicitPaths"; + ps.payload = new Object[] { "import os; print('NATIVE_CLIENT_OK', os.getpid())" }; + ps.payloadClasses = new Class[] { String.class }; + ps.longRunning = true; + ps.hasReturn = true; + ps.insightId = "harness-insight"; + ps.executionInsightId = "harness-insight"; + ps.sessionId = "harness-session"; + ps.jobId = "harness-job"; + + try { + Object resp = client.executeCommand(ps); + System.out.println("executeCommand returned: " + (resp == null ? "NULL" : resp.getClass().getSimpleName())); + if (resp instanceof PayloadStruct) { + PayloadStruct back = (PayloadStruct) resp; + System.out.println(" operation=" + back.operation + " response=" + back.response); + System.out.println(" payload=" + java.util.Arrays.toString(back.payload)); + } + } catch (Exception e) { + System.out.println("executeCommand FAILED: " + e.getClass().getSimpleName() + ": " + e.getMessage()); + } - client.stopServer(); - client.close(); + try { + client.stopServer(); + client.close(); + } catch (Exception ignored) { + // teardown is best-effort; the round trip above is the point of this harness + } t.close(); System.out.println("done"); } From f677065901e3b7cd6ba68b5b010029a0d79e5090 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 20:22:56 +0000 Subject: [PATCH 08/56] fix(sandbox): gate readiness on the worker listening, not the forward binding The forward binds in ~60ms but the worker needs ~5s to reach serve_forever(). A client connecting in that window completes the TCP handshake and then consumes the single-connection worker only slot, wedging it. Measured: connect at +0s -> no reply; connect at +5s -> response=true. Readiness now waits for the supervisor NET:OPEN 127.0.0.1:/tcp line, which needs no connection to the worker. Removes the temporary diagnostics. --- src/prerna/om/ClientProcessWrapper.java | 3 ++ src/prerna/util/OpenShellTransport.java | 60 ++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index c3a532f644..ef3cd12084 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -346,6 +346,9 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS this.socketClient = new SocketClient(threadLoggerCtx); } this.socketClient.setCpw(this); + classLogger.info( + "Connecting socket client: udsPath={} port={} openshell={} nativePyServer={}", + this.udsPath, this.port, this.openshell != null, this.nativePyServer); if (this.udsPath != null) { this.socketClient.connectUds(this.udsPath); } else { diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 64ec00f531..7f3344c560 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -317,7 +317,24 @@ public int start() throws IOException, InterruptedException { + this.forwardedPort + " within " + readyTimeout + "s (process alive=" + alive + ")" + (detail.isEmpty() ? "" : ": " + detail)); } - classLogger.info("Executor sandbox {} ready; worker reachable on 127.0.0.1:{}", sandboxName, + // The forward binding is NOT readiness. It binds in ~60ms, but the worker + // inside takes ~5s to reach serve_forever(). A client connecting in that + // window succeeds at TCP level and then consumes the single-connection + // worker's only slot, wedging it -- measured 2026-09-28: + // connect at +0s after "Forwarding" -> no reply + // connect at +5s after "Forwarding" -> response=true + // The worker's own startup line is the only reliable signal, and it must be + // read from the sandbox log rather than by probing the port, for the same + // reason awaitForwardReady exists. + if (!awaitWorkerListening(readyTimeout)) { + String detail = readAvailable(forwardProcess); + close(); + throw new IOException("worker inside sandbox " + sandboxName + " on port " + workerPort + + " did not begin listening within " + readyTimeout + "s (forward alive=" + + (forwardProcess != null && forwardProcess.isAlive()) + ")" + + (detail.isEmpty() ? "" : ": " + detail)); + } + classLogger.info("Executor sandbox {} ready; worker listening, local port {}", sandboxName, this.forwardedPort); return this.forwardedPort; } @@ -364,6 +381,47 @@ private boolean awaitForwardReady(int timeoutSeconds) { return false; } + /** + * Poll until the worker inside the sandbox reports that it is listening, again + * without ever connecting to the forwarded port. + * + *

+ * This is a second, later gate than {@link #awaitForwardReady}: the forward binds + * in ~60ms, but the worker needs ~5s to reach {@code serve_forever()}. Connecting + * in that window succeeds at TCP level and then wedges the single-connection + * worker, so the distinction matters and was measured directly (see the call site). + * + *

+ * The signal is the sandbox supervisor's own event line: + * {@code NET:OPEN [INFO] 127.0.0.1:7100/tcp}. It is emitted when the worker's + * socket opens and again on close, so only an OPEN counts. + */ + private boolean awaitWorkerListening(int timeoutSeconds) { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + String needle = "127.0.0.1:" + workerPort + "/tcp"; + while (System.nanoTime() < deadline) { + try { + List cmd = Arrays.asList(cli, "logs", sandboxName, "--source", "sandbox"); + String out = run(cmd, Math.min(20, timeoutSeconds)); + for (String line : out.split("\n")) { + if (line.contains(needle) && line.contains("NET:OPEN")) { + return true; + } + } + } catch (Exception e) { + classLogger.error("Could not read sandbox log while waiting for the worker to listen: {}", e); + } + try { + Thread.sleep(250); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return false; + } + private String readAvailable(Process p) { if (forwardLog == null) { return ""; From 1049911f3746d7c32d72eae8f27bbe989b1d4f15 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 20:46:37 +0000 Subject: [PATCH 09/56] fix(sandbox): Py() runs inside OpenShell end to end (three defects) 1. Readiness is taken from the worker own log line, never by connecting to the port: the worker is single-connection (--max_count 1), so a probe consumes its only slot and its disconnect wedges it. The forward Forwarding line binds in ~60ms and is not readiness. 2. The worker stdout/stderr goes to a file instead of the supervisor pipe, which made startup erratic (sometimes never binding while the sandbox still read Ready). 3. run() now redirects stdin from /dev/null: openshell sandbox exec reads stdin, so with ProcessBuilder inherited open pipe the CLI blocked until the 30s timeout, and three of those produced the 180s pixel stall. The same command returns instantly from a shell, which is why this only showed up under Tomcat. Verified: Py("print("LIVE_OK")") returns LIVE_OK at 1599ms; repeats 1011/4/4ms; os.getpid()=60 vs Tomcat JVM 4059545. --- src/prerna/util/OpenShellTransport.java | 113 ++++++++++++++---------- 1 file changed, 68 insertions(+), 45 deletions(-) diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 7f3344c560..671db9d96b 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -111,6 +111,11 @@ public final class OpenShellTransport { private static final String DEFAULT_LABEL = "app=semoss-executor"; private static final int DEFAULT_READY_TIMEOUT_S = 180; + /** Where the worker writes its own log inside the sandbox. */ + public static final String WORKER_LOG_PATH = "/opt/semosshome/insight/worker.log"; + /** The worker's own line announcing that it is serving; the readiness signal. */ + public static final String WORKER_READY_MARKER = "waiting for request"; + /** * When true, {@link #close()} stops the forward but leaves the sandbox in place. * @@ -231,16 +236,22 @@ private static List createCommand(String sandboxName) throws IOException create.add("--policy"); create.add(policy.trim()); create.add("--"); - create.add("python3"); - create.add("/opt/semosshome/py/gaas_tcp_socket_server.py"); - create.add("--port"); - create.add(workerPort); - create.add("--start"); - create.add("True"); - create.add("--py_folder"); - create.add("/opt/semosshome/py"); - create.add("--insight_folder"); - create.add("/opt/semosshome/insight"); + // The worker's stdout/stderr MUST go to a file, not to the supervisor's + // stdout pipe. With the pipe it came up erratically -- sometimes serving, + // sometimes never binding at all while the sandbox still read `Ready` -- and + // slow (~5s). Redirecting to a file made it reliable, 3 runs out of 3, and + // ready in ~0.66s. Suspected cause is output back-pressure on that pipe. + // The file is also the readiness signal, read via `sandbox exec` so that no + // connection is ever made to the single-connection worker. + create.add("sh"); + create.add("-c"); + create.add("exec python3 /opt/semosshome/py/gaas_tcp_socket_server.py" // + + " --port " + workerPort // + + " --start True" // + + " --py_folder /opt/semosshome/py" // + + " --insight_folder /opt/semosshome/insight" // + + " --timeout 30" // + + " > " + WORKER_LOG_PATH + " 2>&1"); return create; } @@ -317,22 +328,30 @@ public int start() throws IOException, InterruptedException { + this.forwardedPort + " within " + readyTimeout + "s (process alive=" + alive + ")" + (detail.isEmpty() ? "" : ": " + detail)); } - // The forward binding is NOT readiness. It binds in ~60ms, but the worker - // inside takes ~5s to reach serve_forever(). A client connecting in that - // window succeeds at TCP level and then consumes the single-connection - // worker's only slot, wedging it -- measured 2026-09-28: - // connect at +0s after "Forwarding" -> no reply - // connect at +5s after "Forwarding" -> response=true - // The worker's own startup line is the only reliable signal, and it must be - // read from the sandbox log rather than by probing the port, for the same - // reason awaitForwardReady exists. - if (!awaitWorkerListening(readyTimeout)) { + // Readiness must come from the worker's OWN log, and the worker must write + // that log to a FILE, not to the supervisor's stdout pipe. + // + // Two measured facts (2026-09-28): + // 1. With stdout on the supervisor's pipe the worker came up erratically -- + // sometimes serving, sometimes never opening the port at all while the + // sandbox still read `Ready`. Redirecting its stdout to a file made it + // reliable, 3 runs out of 3, and fast (~0.66s instead of ~5s). Output + // back-pressure on that pipe is the suspected cause; the file removes it. + // 2. The supervisor's `NET:OPEN 127.0.0.1:/tcp` line is a *connection* + // event, not a bind event -- it also marks NET:CLOSE, and it fired only + // because a probe connected. Waiting for it is a mistake; an earlier + // version of this method did exactly that and stalled for the full + // timeout. The worker's own "waiting for request" line is the real signal. + // + // Never probe the forwarded port to test readiness: the worker is + // single-connection (--max_count 1), so a probe consumes its only slot and its + // disconnect ends the worker. + if (!awaitWorkerReady(readyTimeout)) { String detail = readAvailable(forwardProcess); close(); - throw new IOException("worker inside sandbox " + sandboxName + " on port " + workerPort - + " did not begin listening within " + readyTimeout + "s (forward alive=" - + (forwardProcess != null && forwardProcess.isAlive()) + ")" - + (detail.isEmpty() ? "" : ": " + detail)); + throw new IOException("worker inside sandbox " + sandboxName + " did not report ready within " + + readyTimeout + "s (forward alive=" + (forwardProcess != null && forwardProcess.isAlive()) + + ")" + (detail.isEmpty() ? "" : ": " + detail)); } classLogger.info("Executor sandbox {} ready; worker listening, local port {}", sandboxName, this.forwardedPort); @@ -382,38 +401,34 @@ private boolean awaitForwardReady(int timeoutSeconds) { } /** - * Poll until the worker inside the sandbox reports that it is listening, again - * without ever connecting to the forwarded port. - * - *

- * This is a second, later gate than {@link #awaitForwardReady}: the forward binds - * in ~60ms, but the worker needs ~5s to reach {@code serve_forever()}. Connecting - * in that window succeeds at TCP level and then wedges the single-connection - * worker, so the distinction matters and was measured directly (see the call site). + * Poll until the worker reports that it is serving, without ever connecting to + * the forwarded port. * *

- * The signal is the sandbox supervisor's own event line: - * {@code NET:OPEN [INFO] 127.0.0.1:7100/tcp}. It is emitted when the worker's - * socket opens and again on close, so only an OPEN counts. + * The signal is the worker's own log line {@code "waiting for request"}. It is read + * back with {@code sandbox exec}, so no socket is opened to the worker. Measured + * ready at +0.66s, against ~5s when the worker's stdout went to the supervisor's + * pipe instead of a file. */ - private boolean awaitWorkerListening(int timeoutSeconds) { + private boolean awaitWorkerReady(int timeoutSeconds) { String cli = propOr(CLI_PROP, DEFAULT_CLI); long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); - String needle = "127.0.0.1:" + workerPort + "/tcp"; while (System.nanoTime() < deadline) { try { - List cmd = Arrays.asList(cli, "logs", sandboxName, "--source", "sandbox"); - String out = run(cmd, Math.min(20, timeoutSeconds)); - for (String line : out.split("\n")) { - if (line.contains(needle) && line.contains("NET:OPEN")) { - return true; - } + List cmd = Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "cat", + WORKER_LOG_PATH); + String out = run(cmd, Math.min(30, timeoutSeconds)); + if (out.contains(WORKER_READY_MARKER)) { + return true; } } catch (Exception e) { - classLogger.error("Could not read sandbox log while waiting for the worker to listen: {}", e); + // Log the message AND the exception: with just "{}" and no argument the + // cause vanished and the failure read as a bare timeout. + classLogger.error("Could not read the worker log while waiting for it to become ready: " + + e.getClass().getSimpleName() + ": " + e.getMessage(), e); } try { - Thread.sleep(250); + Thread.sleep(300); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return false; @@ -516,6 +531,14 @@ private static String run(List cmd, int timeoutSeconds) throws IOExcepti ProcessBuilder pb = new ProcessBuilder(cmd); pb.environment().putAll(cliEnv()); pb.redirectErrorStream(true); + // Close the child's stdin immediately. Without this it inherits a pipe that + // stays open for the JVM's lifetime and any subcommand that reads stdin -- + // `openshell sandbox exec` does -- blocks until the timeout instead of + // running. Measured 2026-09-28: the same `sandbox exec ... cat` returned + // instantly from a shell (stdin at EOF) and timed out after 30s under + // Tomcat, which is what made the readiness poll fail and the pixel stall for + // the full 180s. A shell would not have shown this. + pb.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); Process p = pb.start(); // Read on a separate thread. A blocking readLine() here hangs the CALLER -- From ff27a1513a6046757391e2567bfef13dd02ead42 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 21:13:45 +0000 Subject: [PATCH 10/56] fix(sandbox): sandbox user python only, not engine-owned python createProcessAndClient is shared by the user REPL and engine-owned workers. Branching on SANDBOX_MODE alone sandboxed the model engine too, so AskRoom failed with ModuleNotFoundError: No module named openai several layers from the cause: AskRoomReactor -> AbstractModelEngine.askRoom -> AbstractPythonModelEngine.startServer -> PyTranslator. That read as a missing package but was a scope error. engineOwned already distinguishes them: User.pythonCPW and Project.cpw are user-authored and untrusted and get the sandbox; model engines, vector databases, guardrails and LocalPythonFunctionEngine are our own code calling controlled SDKs and stay on the Tomcat side. Verified: AskRoom now reaches the OpenAI API (401 on the engine own key, no import error, no sandbox) and Py() still returns pid 60 against a Tomcat JVM pid of 21788 with exactly one sandbox created. --- src/prerna/om/ClientProcessWrapper.java | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index ef3cd12084..d4bff6f3e1 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -225,13 +225,24 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS boolean serverRunning = debug && port > 0; if (!serverRunning) { if (nativePyServer) { - // OPENSHELL: the worker is the canonical main process of its own - // sandbox, not a child of Tomcat. Create/attach the sandbox, take - // the forwarded local port, and let the existing socket client - // connect to it exactly as the non-sandbox path does. No udsPath - // is set, which is what selects the TCP branch below. + // OPENSHELL applies to USER-AUTHORED python only -- the user's + // isolated analytics engine (User.pythonCPW) and project python. + // It deliberately does NOT apply to engine-owned workers: + // model engines, vector databases, guardrails and local function + // engines are SEMOSS's own code calling controlled SDKs, so they + // stay on the Tomcat side where they have always lived. They are + // trusted; the user's REPL is not. + // + // `engineOwned` is the discriminator (set true by + // AbstractPythonModelEngine, AbstractVectorDatabaseEngine, + // PGVectorDatabaseEngine, AbstractPythonGuardrailReactorFunctionEngine + // and LocalPythonFunctionEngine). Without this guard the model + // engine was sandboxed too and AskRoom broke on an import the + // sandbox image does not carry -- observed 2026-09-28: + // AskRoomReactor -> AbstractPythonModelEngine.startServer + // -> ModuleNotFoundError: No module named 'openai' String openshellMode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); - if (Constants.SANDBOX_MODE_OPENSHELL.equalsIgnoreCase(openshellMode)) { + if (!this.engineOwned && Constants.SANDBOX_MODE_OPENSHELL.equalsIgnoreCase(openshellMode)) { this.openshell = prerna.util.OpenShellTransport.forSession(this.serverDirectory != null ? this.serverDirectory : Integer.toHexString(System.identityHashCode(this))); From 8f1008841624d4cbc7558a20c0a9afdbddac52b1 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 21:22:12 +0000 Subject: [PATCH 11/56] refactor(sandbox): select OPENSHELL from the existing jail chain Replaces the !engineOwned guard in ClientProcessWrapper with the branch the other jails already use. The jail has always been chosen by which caller supplies the SymlinkHelper: User.startPythonSocketServerAndClient passes getUserSymlinkHelper() for user-authored python, while engine-owned workers and Project pass null. OPENSHELL is now simply the third branch of that chain, alongside NAMESPACE and FAKECHROOT, and SymlinkHelper resolves SANDBOX_MODE once into a SandboxMode enum. ClientProcessWrapper no longer re-reads the mode at all -- it is a property of the user jail, so it belongs to the object that only exists for a user worker. Verified: AskRoom on an Ollama engine answers (Hello, hi, hey, 1028ms, no sandbox) and Py() still runs at pid 60 with exactly one sandbox. --- src/prerna/om/ClientProcessWrapper.java | 33 ++++++------- src/prerna/util/SymlinkHelper.java | 61 ++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 22 deletions(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index d4bff6f3e1..793f3cf392 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -225,27 +225,20 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS boolean serverRunning = debug && port > 0; if (!serverRunning) { if (nativePyServer) { - // OPENSHELL applies to USER-AUTHORED python only -- the user's - // isolated analytics engine (User.pythonCPW) and project python. - // It deliberately does NOT apply to engine-owned workers: - // model engines, vector databases, guardrails and local function - // engines are SEMOSS's own code calling controlled SDKs, so they - // stay on the Tomcat side where they have always lived. They are - // trusted; the user's REPL is not. + // The jail is selected the same way the fakechroot and namespace + // jails have always been selected: by the chain below, on the + // SymlinkHelper the caller supplied. User-authored python gets one + // (User.startPythonSocketServerAndClient); engine-owned workers and + // Project python pass null and so are never jailed at all. // - // `engineOwned` is the discriminator (set true by - // AbstractPythonModelEngine, AbstractVectorDatabaseEngine, - // PGVectorDatabaseEngine, AbstractPythonGuardrailReactorFunctionEngine - // and LocalPythonFunctionEngine). Without this guard the model - // engine was sandboxed too and AskRoom broke on an import the - // sandbox image does not carry -- observed 2026-09-28: - // AskRoomReactor -> AbstractPythonModelEngine.startServer - // -> ModuleNotFoundError: No module named 'openai' - String openshellMode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); - if (!this.engineOwned && Constants.SANDBOX_MODE_OPENSHELL.equalsIgnoreCase(openshellMode)) { - this.openshell = prerna.util.OpenShellTransport.forSession(this.serverDirectory != null - ? this.serverDirectory - : Integer.toHexString(System.identityHashCode(this))); + // OPENSHELL is simply the third branch of that chain. The sandbox is + // created ahead of the worker, so the worker becomes the sandbox's + // canonical process instead of a child of Tomcat, and the socket + // client connects to the forwarded port. No udsPath is set, which is + // what selects the TCP branch below. + if (this.chrootSymlinkHelper != null && this.chrootSymlinkHelper.isExternalSandbox()) { + this.openshell = prerna.util.OpenShellTransport.forSession( + this.chrootSymlinkHelper.getUserChrootFolder()); try { this.port = this.openshell.start(); } catch (Exception e) { diff --git a/src/prerna/util/SymlinkHelper.java b/src/prerna/util/SymlinkHelper.java index 8fa49cfab0..1ceeb9aa00 100644 --- a/src/prerna/util/SymlinkHelper.java +++ b/src/prerna/util/SymlinkHelper.java @@ -55,6 +55,27 @@ public class SymlinkHelper { + /** + * Sandbox modes this user-level jail can be in. + * + *

+ * The distinction is who supplies the {@link SymlinkHelper}, not a flag checked at the call + * site: {@code User.startPythonSocketServerAndClient} passes one for user-authored python, while + * engine-owned workers (model engines, vector databases, guardrails, local function engines) and + * {@code Project} pass {@code null}. That is existing semantics and the reason a user-authored + * worker is the only one that ever reaches this class. + */ + public enum SandboxMode { + /** No kernel jail: fake chroot with symlinks. */ + FAKECHROOT, + /** Namespace jail (nsjail/landlock style) with a control-socket injector. */ + NAMESPACE, + /** External sandbox (OpenShell) created ahead of the worker; no in-process jail. */ + OPENSHELL + } + + private final SandboxMode sandboxMode; + private static final Logger classLogger = LogManager.getLogger(SymlinkHelper.class); private String userChrootFolder = null; @@ -75,8 +96,11 @@ public class SymlinkHelper { */ public SymlinkHelper(String targetDirName) { this.userChrootFolder = targetDirName; - String sandboxMode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); - this.injectMode = "NAMESPACE".equalsIgnoreCase(sandboxMode) || "NSJAIL".equalsIgnoreCase(sandboxMode); + // The mode is resolved once, here, because this object exists only for a + // user-authored worker. Callers that must NOT be jailed (engine-owned + // workers, Project python) pass a null helper and never reach this. + this.sandboxMode = resolveSandboxMode(); + this.injectMode = this.sandboxMode == SandboxMode.NAMESPACE; File targetDir = new File(Utility.normalizePath(userChrootFolder)); if (!targetDir.exists()) { @@ -117,6 +141,39 @@ public boolean isInjectMode() { return injectMode; } + /** The user-level jail this helper's worker runs under. */ + public SandboxMode getSandboxMode() { + return this.sandboxMode; + } + + /** + * True when the worker runs inside an external sandbox created ahead of it + * (OpenShell), so no in-process jail is applied. + */ + public boolean isExternalSandbox() { + return this.sandboxMode == SandboxMode.OPENSHELL; + } + + private static SandboxMode resolveSandboxMode() { + String mode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); + if (mode == null || (mode = mode.trim()).isEmpty()) { + // Historically an unset mode means the plain fakechroot jail. + return SandboxMode.FAKECHROOT; + } + if (Constants.SANDBOX_MODE_OPENSHELL.equalsIgnoreCase(mode)) { + return SandboxMode.OPENSHELL; + } + if ("NAMESPACE".equalsIgnoreCase(mode) || "NSJAIL".equalsIgnoreCase(mode)) { + return SandboxMode.NAMESPACE; + } + if ("FAKECHROOT".equalsIgnoreCase(mode)) { + return SandboxMode.FAKECHROOT; + } + // SECCOMP_ONLY and anything unrecognised keep the pre-existing fakechroot + // behaviour rather than silently gaining a jail. + return SandboxMode.FAKECHROOT; + } + private synchronized void enqueueOrInject(String absPath, boolean readWrite) { if (!isNamespaceInjectablePath(absPath)) { classLogger.warn("Skipping namespace injection for path outside SEMOSS home: {}", absPath); From 710cb4f45395083217531afdd472317a6ab71b54 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Mon, 28 Sep 2026 21:42:29 +0000 Subject: [PATCH 12/56] feat(sandbox): transport project files into the sandbox via SymlinkHelper Every jail hangs off SymlinkHelper, because all three answer the same question: this host path must be usable by the worker. FAKECHROOT builds a chroot tree (symlinks on this host) NAMESPACE adds the path to the jail via SandboxInjector (control socket) OPENSHELL must COPY the tree, because a sandbox has its own filesystem Making a path visible is not making it present. SandboxTransporter is the OPENSHELL counterpart of SandboxInjector: wired by ClientProcessWrapper at the same point, called through a transportOrEnqueue dispatcher from symlinkFolder/symlinkProject so every invocation site is covered without editing any of them. activeTransports makes it once per sandbox, and setTransport replays paths registered before the worker existed, mirroring setInjector's replay of activeInjects. Two bugs found while verifying: 1. `sandbox upload ` nests the basename, so passing dst == src produced app_root/app_root/version/... and the project never landed where the worker looks. Destination must be the parent. 2. The transporter needs the same isNamespaceInjectablePath guard the injector has. MCPUtility also symlinks git's own directories, so /usr/bin, /etc/ssl and /usr/lib/git-core were being uploaded -- several 30s CLI timeouts and a 77s pixel. Now 8.4s. Verified: RunMCPTool now finds and executes the project's mcp_driver.py. It fails on the project's own `import yfinance`, which is absent from both the host venv and py/install_config/pyproject.toml and would fail under NAMESPACE too -- a missing declared dependency, not a visibility problem. Write-back (FETCH_BACK) is NOT implemented: files written inside the sandbox are lost at teardown. That asymmetry is worse than no transport at all, since the sandbox can now read project files while silently discarding writes. --- src/prerna/om/ClientProcessWrapper.java | 7 ++ src/prerna/util/OpenShellTransport.java | 34 ++++++++++ src/prerna/util/OpenShellTransporter.java | 67 ++++++++++++++++++ src/prerna/util/SandboxTransporter.java | 77 +++++++++++++++++++++ src/prerna/util/SymlinkHelper.java | 83 +++++++++++++++++++++-- 5 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 src/prerna/util/OpenShellTransporter.java create mode 100644 src/prerna/util/SandboxTransporter.java diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 793f3cf392..15b0cc2e48 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -241,6 +241,13 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS this.chrootSymlinkHelper.getUserChrootFolder()); try { this.port = this.openshell.start(); + // Same seam as the namespace injector below: the per-mode collaborator + // that receives every path symlinkFolder/symlinkProject would otherwise + // have symlinked. Assets registered before the worker existed are + // replayed by setTransport, which is why this happens here rather than + // before start(). + this.chrootSymlinkHelper.setTransport( + new prerna.util.OpenShellTransporter(this.openshell)); } catch (Exception e) { // Fail closed: never fall back to an in-Tomcat worker when the // deployment asked for an external boundary. diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 671db9d96b..28fcf85b3e 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -400,6 +400,40 @@ private boolean awaitForwardReady(int timeoutSeconds) { return false; } + /** + * Copy a host file or directory into this sandbox, preserving its absolute path. + * + *

+ * Used by {@link OpenShellTransporter} so that asset and project folders the worker refers to by + * absolute path exist inside the sandbox. Idempotency is the caller's job + * ({@code SymlinkHelper.activeTransports}); this method always transfers. + * + * @param absPath host path to copy + * @return true when the CLI reported success + */ + public boolean upload(String absPath) { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + // `sandbox upload ` copies src INTO dst, i.e. it appends basename(src). + // Passing dst == src therefore nests: uploading /a/b/app_root to /a/b/app_root + // produced /a/b/app_root/app_root/version/... (observed 2026-09-28, and the same + // shape as the earlier X.py -> /path/X.py/X.py trap). The destination must be the + // PARENT so the tree lands at the absolute path the worker expects. + String destination = java.nio.file.Paths.get(absPath).getParent() == null ? "/" + : java.nio.file.Paths.get(absPath).getParent().toString(); + classLogger.info("Sandbox upload {} -> {}:{}", absPath, sandboxName, destination); + try { + String out = run(Arrays.asList(cli, "sandbox", "upload", sandboxName, absPath, destination), 300); + // `sandbox upload` prints a tick with its result; treat a non-zero exit as failure, + // which run() already raises as IOException. + classLogger.info("Sandbox upload result for {}: {}", absPath, out.trim()); + return true; + } catch (Exception e) { + classLogger.error("Sandbox upload failed for " + absPath + ": " + e.getClass().getSimpleName() + ": " + + e.getMessage(), e); + return false; + } + } + /** * Poll until the worker reports that it is serving, without ever connecting to * the forwarded port. diff --git a/src/prerna/util/OpenShellTransporter.java b/src/prerna/util/OpenShellTransporter.java new file mode 100644 index 0000000000..25f19e6bc4 --- /dev/null +++ b/src/prerna/util/OpenShellTransporter.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +/** + * {@link SandboxTransporter} for {@link SymlinkHelper.SandboxMode#OPENSHELL}: copies a host path into + * the sandbox with {@code openshell sandbox upload}. + * + *

+ * The sandbox has its own filesystem, so unlike the namespace jail -- which runs on this host and is + * satisfied by binding the real path -- the bytes must actually be transferred. Uploads preserve the + * absolute path, so a worker that opens an asset by absolute path or inserts it into {@code sys.path} + * finds it where it expects. + */ +public class OpenShellTransporter implements SandboxTransporter { + + private static final org.apache.logging.log4j.Logger classLogger = org.apache.logging.log4j.LogManager + .getLogger(OpenShellTransporter.class); + + private final OpenShellTransport transport; + + public OpenShellTransporter(OpenShellTransport transport) { + this.transport = transport; + } + + @Override + public boolean transport(String absPath) { + if (this.transport == null) { + classLogger.warn("No OpenShell transport; cannot send {}", absPath); + return false; + } + if (absPath == null || absPath.trim().isEmpty()) { + return false; + } + return this.transport.upload(absPath); + } + + @Override + public boolean close() { + return true; + } +} diff --git a/src/prerna/util/SandboxTransporter.java b/src/prerna/util/SandboxTransporter.java new file mode 100644 index 0000000000..6c1481ea00 --- /dev/null +++ b/src/prerna/util/SandboxTransporter.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +/** + * Moves host paths into an external sandbox. + * + *

+ * The OPENSHELL counterpart of {@link SandboxInjector}. Both are the per-mode collaborator that + * {@link SymlinkHelper} hands a path to when it would otherwise have symlinked it: + * + * + * + * + * + * + *
mode collaborators
ModeCollaboratorWhat it does with the path
NAMESPACE{@link SandboxInjector}adds it to the namespace jail's + * allowed set over the control socket
OPENSHELL{@link SandboxTransporter}copies the tree into the + * sandbox
+ * + *

+ * The distinction is worth stating because it is not an implementation detail: a namespace jail runs + * on this host, so binding the real path is enough. A sandbox has its own filesystem, so the bytes + * have to be transferred. Making the path visible is not making it present. A worker that calls + * {@code sys.path.insert} on an asset path, or opens an asset by absolute path, fails with + * {@code FileNotFoundError} when only the path exists -- observed with a project MCP driver, where + * {@code PyTranslator} passed the asset paths correctly and the import still failed because the file + * was never in the sandbox. + */ +public interface SandboxTransporter { + + /** + * Make {@code absPath} (a file or directory tree on this host) available inside the sandbox at the + * same absolute path. + * + *

+ * Implementations must be idempotent: the same path is requested repeatedly (every insight, + * project open and MCP call re-symlinks), and re-transferring a tree on each call would be a + * per-request cost with no benefit. + * + * @param absPath host path, already normalised + * @return true if the path is present in the sandbox after the call + */ + boolean transport(String absPath); + + /** + * Best-effort teardown. + * + * @return true if the collaborator released cleanly + */ + boolean close(); +} diff --git a/src/prerna/util/SymlinkHelper.java b/src/prerna/util/SymlinkHelper.java index 1ceeb9aa00..048e55aa37 100644 --- a/src/prerna/util/SymlinkHelper.java +++ b/src/prerna/util/SymlinkHelper.java @@ -41,6 +41,8 @@ import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; @@ -83,6 +85,12 @@ public enum SandboxMode { private final boolean injectMode; private SandboxInjector injector; private final Map activeInjects = new LinkedHashMap<>(); + /** + * Paths already handed to the external sandbox. A project open, an insight and an MCP call all + * re-symlink the same folders; without this the tree would be re-transferred on every request. + */ + private final Set activeTransports = ConcurrentHashMap.newKeySet(); + private SandboxTransporter transporter; /** * Construct a SymlinkHelper bound to a user-specific chroot folder and run the @@ -124,6 +132,29 @@ public SymlinkHelper(String targetDirName) { classLogger.info("Chroot init completed in {} ms for: {}", elapsedMs, userChrootFolder); } + /** + * Wire the per-mode collaborator that moves paths into an external sandbox. + * + *

+ * The exact counterpart of {@link #setInjector(SandboxInjector)}: both are set once by + * {@code ClientProcessWrapper} when the worker starts, and both are replayed with everything + * requested so far, because asset paths are registered before the worker exists. + * + * @param transporter the transporter for {@link SandboxMode#OPENSHELL} + */ + public synchronized void setTransport(SandboxTransporter transporter) { + this.transporter = transporter; + if (transporter != null && !activeTransports.isEmpty()) { + for (String absPath : activeTransports) { + try { + transporter.transport(absPath); + } catch (RuntimeException e) { + classLogger.warn("Skipping sandbox transport for {}", absPath, e); + } + } + } + } + public synchronized void setInjector(SandboxInjector injector) { this.injector = injector; if (injector != null && !activeInjects.isEmpty()) { @@ -174,6 +205,45 @@ private static SandboxMode resolveSandboxMode() { return SandboxMode.FAKECHROOT; } + /** + * The single place a path is handed to the active jail. + * + *

+ * Every {@code symlinkFolder}/{@code symlinkProject} caller routes through here, so this is the + * seam that gives the sandbox the same files the other two jails get. A path requested before the + * worker exists is remembered and replayed by {@link #setTransport(SandboxTransporter)}, exactly as + * the namespace injector replays {@code activeInjects}. + */ + private synchronized void transportOrEnqueue(String absPath, boolean readWrite) { + if (sandboxMode == SandboxMode.OPENSHELL) { + // Same scope rule as the namespace injector. Without it the transporter is handed + // system paths -- /usr/bin, /etc/ssl, /usr/lib/git-core -- because + // MCPUtility.runPythonTool also symlinks git's own directories. Uploading those is + // wrong and enormous: observed as several 30s CLI timeouts, a 77s pixel, and the + // project never arriving. The sandbox image already provides them. + if (!isNamespaceInjectablePath(absPath)) { + classLogger.debug("Skipping sandbox transport for path outside SEMOSS home: {}", absPath); + return; + } + boolean existing = !activeTransports.add(absPath); + if (existing) { + // Already sent once; transport() is idempotent but there is no reason to call it. + return; + } + if (transporter != null) { + try { + if (!transporter.transport(absPath)) { + classLogger.warn("Sandbox transport reported failure for {}", absPath); + } + } catch (RuntimeException e) { + classLogger.warn("Sandbox transport failed for {}", absPath, e); + } + } + return; + } + enqueueOrInject(absPath, readWrite); + } + private synchronized void enqueueOrInject(String absPath, boolean readWrite) { if (!isNamespaceInjectablePath(absPath)) { classLogger.warn("Skipping namespace injection for path outside SEMOSS home: {}", absPath); @@ -367,8 +437,13 @@ private void copyDirectoryRecursively(Path source, Path target) throws IOExcepti */ public void symlinkFolder(String sourceDirName) { sourceDirName = Utility.normalizePath(sourceDirName); - if (injectMode) { - enqueueOrInject(sourceDirName, true); + if (sandboxMode != SandboxMode.FAKECHROOT) { + transportOrEnqueue(sourceDirName, true); + } + if (sandboxMode == SandboxMode.OPENSHELL) { + // The sandbox has its own filesystem root; a chroot symlink under the host user folder + // would point at a path the sandbox cannot see, so it is meaningless here. + return; } createChrootSymlink(sourceDirName); } @@ -493,8 +568,8 @@ public void symlinkProject(User user, String projectId) { boolean canEdit = SecurityProjectUtils.userCanEditProject(user, projectId); - if (injectMode) { - enqueueOrInject(Utility.normalizePath(projectAppRootFolder), canEdit); + if (sandboxMode != SandboxMode.FAKECHROOT) { + transportOrEnqueue(Utility.normalizePath(projectAppRootFolder), canEdit); } if (canEdit) { From 89eada505d9963e8996574112c4c828335b88c61 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Thu, 22 Oct 2026 12:31:54 +0000 Subject: [PATCH 13/56] feat(openshell): prove session-scoped sandbox lifecycle; add stale reaper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two additions, both in their own package prerna.util.openshell. No existing class is modified. OpenShellSessionLifecycleHarness — proves the lifecycle the product relies on, without booting Tomcat: no sandbox exists before first use; the first user-authored python request creates it on demand (13.4s cold start); Py() executes inside it through the shipping NativePySocketClient (pid 60, python 3.12.14); the exact logout call -- ClientProcessWrapper.shutdown(true), as issued by UserSessionLoader.sessionDestroyed -- deletes it (385ms); and a later session starts a fresh one. 11/11 checks. OpenShellSandboxReaper — the backstop for sessions that never log out (JVM killed, container restart, abandoned session). Lists only sandboxes carrying the executor label AND the pyw- prefix, deletes only those older than a threshold, skips unparseable timestamps rather than guessing. Verified against a labelled throwaway: dry-run listed it, the real run deleted it, and an unlabelled sandbox on the same gateway was untouched. --- .../openshell/OpenShellSandboxReaper.java | 181 +++++++++++++ .../OpenShellSessionLifecycleHarness.java | 249 ++++++++++++++++++ 2 files changed, 430 insertions(+) create mode 100644 src/prerna/util/openshell/OpenShellSandboxReaper.java create mode 100644 src/prerna/util/openshell/OpenShellSessionLifecycleHarness.java diff --git a/src/prerna/util/openshell/OpenShellSandboxReaper.java b/src/prerna/util/openshell/OpenShellSandboxReaper.java new file mode 100644 index 0000000000..36724d548f --- /dev/null +++ b/src/prerna/util/openshell/OpenShellSandboxReaper.java @@ -0,0 +1,181 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +import prerna.util.OpenShellTransport; +import prerna.util.Utility; + +/** + * Deletes executor sandboxes that outlived their session. + * + *

+ * Normal teardown is the logout path: {@code UserSessionLoader.sessionDestroyed} + * → {@code ClientProcessWrapper.shutdown(true)} → {@code OpenShellTransport.close()}. + * That covers a user who logs out or whose session times out. It does not cover + * the cases that actually leak: + * + *

+ * + *

+ * So this is a backstop, not the primary mechanism: it lists only sandboxes + * carrying the executor label and the {@code pyw-} name prefix, and only + * deletes those older than a threshold. Age is the only signal that is safe to + * act on without a live registry of owners: a young sandbox may belong to a + * session on another replica, and killing it would be a real user-visible bug. + * + *

+ * Run it from a timer, not from the request path: + * + * 

+ * java -cp ... prerna.util.openshell.OpenShellSandboxReaper --max-age-hours 12
+ * java -cp ... prerna.util.openshell.OpenShellSandboxReaper --max-age-hours 12 --dry-run
+ *
+ */ +public final class OpenShellSandboxReaper { + + private static final DateTimeFormatter LIST_TIMESTAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + private OpenShellSandboxReaper() { + // static utility + } + + public static void main(String[] args) { + String rdf = System.getProperty("RDF_MAP", "/opt/semosshome/RDF_Map.prop"); + prerna.util.DIHelper.getInstance().loadCoreProp(rdf); + + double maxAgeHours = 12; + boolean dryRun = false; + String label = Utility.getDIHelperProperty(OpenShellTransport.LABEL_PROP); + if (label == null || label.trim().isEmpty()) { + label = "app=semoss-executor"; + } + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--max-age-hours": + maxAgeHours = Double.parseDouble(args[++i]); + break; + case "--dry-run": + dryRun = true; + break; + case "--label": + label = args[++i]; + break; + default: + System.out.println("unknown argument: " + args[i]); + System.exit(2); + } + } + + System.out.println("label=" + label + " maxAgeHours=" + maxAgeHours + (dryRun ? " (dry run)" : "")); + int reaped = reap(label, maxAgeHours, dryRun); + System.out.println((dryRun ? "would reap " : "reaped ") + reaped + " sandbox(es)"); + } + + /** + * Delete labelled, prefixed sandboxes older than {@code maxAgeHours}. + * + * @return how many were (or would be) deleted + */ + public static int reap(String label, double maxAgeHours, boolean dryRun) { + List candidates = listExecutorSandboxes(label); + LocalDateTime cutoff = LocalDateTime.now().minus(Duration.ofMinutes((long) (maxAgeHours * 60))); + int count = 0; + for (String[] row : candidates) { + String name = row[0]; + String created = row[1]; + LocalDateTime timestamp; + try { + timestamp = LocalDateTime.parse(created, LIST_TIMESTAMP); + } catch (Exception e) { + // Unparseable age: skip rather than guess. Deleting a sandbox we cannot + // date is exactly the mistake this class exists to avoid. + System.out.println(" skip " + name + " (unparseable timestamp '" + created + "')"); + continue; + } + if (timestamp.isAfter(cutoff)) { + continue; + } + count++; + if (dryRun) { + System.out.println(" would delete " + name + " (created " + created + ")"); + } else { + System.out.println(" deleting " + name + " (created " + created + ")"); + run("openshell", "sandbox", "delete", name); + } + } + return count; + } + + /** + * List sandboxes carrying the executor label whose name matches the executor + * prefix. Two independent filters on purpose: the label says "this gateway + * knows about executors", the prefix says "this process created it". + */ + private static List listExecutorSandboxes(String label) { + List rows = new ArrayList<>(); + String output = run("openshell", "sandbox", "list", "--selector", label); + for (String line : output.split("\\R")) { + String trimmed = line.trim(); + if (trimmed.isEmpty() || trimmed.startsWith("NAME")) { + continue; + } + String[] columns = trimmed.split("\\s+"); + if (columns.length < 3) { + continue; + } + String name = columns[0]; + if (!name.startsWith("pyw-")) { + continue; + } + rows.add(new String[] { name, columns[1] + " " + columns[2] }); + } + return rows; + } + + private static String run(String... command) { + try { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + return output; + } catch (Exception e) { + System.out.println(" command failed: " + String.join(" ", command) + " -> " + e.getMessage()); + return ""; + } + } +} diff --git a/src/prerna/util/openshell/OpenShellSessionLifecycleHarness.java b/src/prerna/util/openshell/OpenShellSessionLifecycleHarness.java new file mode 100644 index 0000000000..b7e36a5ad1 --- /dev/null +++ b/src/prerna/util/openshell/OpenShellSessionLifecycleHarness.java @@ -0,0 +1,249 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +import prerna.om.ClientProcessWrapper; +import prerna.tcp.PayloadStruct; +import prerna.tcp.client.SocketClient; +import prerna.util.Constants; +import prerna.util.DIHelper; +import prerna.util.OpenShellTransport; +import prerna.util.SymlinkHelper; +import prerna.util.Utility; + +/** + * Proves the sandbox lifecycle that login/logout is supposed to give us, + * without booting Tomcat. + * + *

+ * The claim being tested is not "a sandbox can be created" — the transport + * harness already shows that. It is the session-shaped claim: + * + *

    + *
  1. nothing exists while the user has not run any code;
    2. + *
  2. the first user-authored python request creates the sandbox on demand;
    3. + *
  3. {@code Py()} actually executes inside it (real worker, not a stock image);
    4. + *
  4. the exact call made on logout — {@code ClientProcessWrapper.shutdown(true)} + * from {@code UserSessionLoader.sessionDestroyed} — deletes it.
    5. + *
+ * + *

+ * Step 4 is the one worth automating: it is a single call buried in a servlet + * listener, and if the wrapper forgets its transport, logout silently leaks a + * sandbox. + * + *

+ * Run with an RDF map whose {@code SANDBOX_MODE} is {@code OPENSHELL}: + * + * 

+ * java -DRDF_MAP=/tmp/RDF_Map.openshell-harness.prop -cp ... \
+ *      prerna.util.openshell.OpenShellSessionLifecycleHarness
+ *
+ */ +public final class OpenShellSessionLifecycleHarness { + + private static int passed = 0; + private static int failed = 0; + + public static void main(String[] args) throws Exception { + String rdf = System.getProperty("RDF_MAP", "/opt/semosshome/RDF_Map.prop"); + DIHelper.getInstance().loadCoreProp(rdf); + System.out.println("RDF: " + rdf); + String mode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); + String image = Utility.getDIHelperProperty(OpenShellTransport.IMAGE_PROP); + String policy = Utility.getDIHelperProperty(OpenShellTransport.POLICY_PROP); + System.out.println("SANDBOX_MODE=" + mode + " image=" + image + " policy=" + policy); + check("P-0 SANDBOX_MODE is OPENSHELL", "OPENSHELL".equalsIgnoreCase(mode), String.valueOf(mode)); + check("P-1 image is configured", image != null && !image.trim().isEmpty(), String.valueOf(image)); + check("P-2 policy is configured", policy != null && !policy.trim().isEmpty(), String.valueOf(policy)); + check("P-3 CHROOT_ENABLE is on", Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE)), + String.valueOf(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE))); + if (failed > 0) { + System.out.println("aborting: the properties above decide whether this test means anything"); + System.exit(1); + } + + String chrootRoot = Utility.getDIHelperProperty(Constants.CHROOT_DIR); + String chrootPath = chrootRoot + File.separator + "lifecycle-" + UUID.randomUUID().toString().substring(0, 8); + Files.createDirectories(Paths.get(chrootPath)); + String sandboxName = OpenShellTransport.sandboxNameFor(chrootPath); + System.out.println("chroot: " + chrootPath); + System.out.println("sandbox: " + sandboxName); + + ClientProcessWrapper cpw = new ClientProcessWrapper(); + try { + check("L-1 no sandbox before first use", !sandboxExists(sandboxName), sandboxName); + + // This is the same call the first Py() makes: User.getPythonSocketClient(true) + // -> ClientProcessWrapper.createProcessAndClient(...) -> OpenShellTransport.start(). + System.out.println("\nstarting on demand (this creates the sandbox)..."); + long started = System.currentTimeMillis(); + SymlinkHelper helper = new SymlinkHelper(chrootPath); + check("L-2 helper selected the external sandbox", helper.isExternalSandbox(), helper.getSandboxMode().name()); + cpw.createProcessAndClient(true, helper, -1, null, null, null, false, "30", "INFO", null); + System.out.println(" up in " + (System.currentTimeMillis() - started) + " ms"); + check("L-3 sandbox exists after start", sandboxExists(sandboxName), sandboxName); + + // The real point: user-authored python runs in the sandbox, through the + // shipping client, not a probe socket. + SocketClient client = cpw.getSocketClient(); + check("L-4 socket client connected", client != null && client.isConnected(), "client=" + client); + if (client != null && client.isConnected()) { + PayloadStruct ps = new PayloadStruct(); + ps.operation = PayloadStruct.OPERATION.PYTHON; + ps.methodName = "lifecycleHarness"; + ps.payload = new Object[] { "import os,sys; print('LIFECYCLE_OK', os.getpid(), sys.version.split()[0])" }; + ps.payloadClasses = new Class[] { String.class }; + ps.longRunning = true; + ps.response = false; + ps.interim = false; + ps.insightId = "lifecycle-harness"; + ps.executionInsightId = "lifecycle-harness"; + ps.sessionId = "lifecycle-harness"; + ps.jobId = "lifecycle-harness"; + Object reply = client.executeCommand(ps); + String text; + if (reply instanceof PayloadStruct) { + PayloadStruct replyStruct = (PayloadStruct) reply; + text = java.util.Arrays.deepToString(replyStruct.payload); + System.out.println(" response=" + replyStruct.response + " payload=" + text); + } else { + text = String.valueOf(reply); + System.out.println(" reply=" + text); + } + check("L-5 Py() executed inside the sandbox", text.contains("LIFECYCLE_OK"), abbreviate(text)); + } + + // Exactly what UserSessionLoader.sessionDestroyed does for the session user. + System.out.println("\nshutting down as logout does (cpw.shutdown(true))..."); + long stopped = System.currentTimeMillis(); + cpw.shutdown(true); + System.out.println(" down in " + (System.currentTimeMillis() - stopped) + " ms"); + Thread.sleep(2000); + check("L-6 sandbox deleted on logout", !sandboxExists(sandboxName), sandboxName); + + // The second half of the on-demand claim: after teardown the same wrapper + // can start a fresh sandbox, so a new session does not inherit a dead one. + System.out.println("\nstarting again (second session)..."); + cpw.createProcessAndClient(true, new SymlinkHelper(chrootPath), -1, null, null, null, false, "30", "INFO", + null); + check("L-7 a later session starts a fresh sandbox", sandboxExists(sandboxName), sandboxName); + } catch (Exception e) { + check("L-0 lifecycle completed without exception", false, e.getClass().getSimpleName() + ": " + e.getMessage()); + e.printStackTrace(); + } finally { + try { + cpw.shutdown(true); + } catch (Exception ignored) { + // best effort + } + Thread.sleep(1000); + if (sandboxExists(sandboxName)) { + System.out.println("cleaning up leftover sandbox " + sandboxName); + deleteSandbox(sandboxName); + } + deleteRecursively(Paths.get(chrootPath)); + } + + int total = passed + failed; + System.out.println("\n=== " + (failed == 0 ? "ALL PASS" : "FAILURES") + " (" + passed + "/" + total + + " checks passed) ==="); + if (failed > 0) { + System.exit(1); + } + } + + // ------------------------------------------------------------------ + // helpers + // ------------------------------------------------------------------ + + private static boolean sandboxExists(String name) { + String output = run("openshell", "sandbox", "list"); + for (String line : output.split("\\R")) { + String trimmed = line.trim(); + if (trimmed.startsWith(name + " ") || trimmed.equals(name)) { + return true; + } + } + return false; + } + + private static void deleteSandbox(String name) { + run("openshell", "sandbox", "delete", name); + } + + private static String run(String... command) { + try { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + return output; + } catch (Exception e) { + return ""; + } + } + + private static void deleteRecursively(Path root) { + if (!Files.exists(root)) { + return; + } + try (var stream = Files.walk(root)) { + stream.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (Exception ignored) { + // best effort + } + }); + } catch (Exception ignored) { + // best effort + } + } + + private static String abbreviate(String text) { + if (text == null) { + return ""; + } + return text.length() <= 120 ? text : text.substring(0, 117) + "..."; + } + + private static void check(String name, boolean ok, String detail) { + if (ok) { + passed++; + } else { + failed++; + } + System.out.printf(" %s %-48s %s%n", ok ? "[PASS]" : "[FAIL]", name, detail == null ? "" : detail); + } +} From 1f8e7382b46c38102defd3efb9d87f6e4b138753 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Thu, 22 Oct 2026 23:07:08 +0000 Subject: [PATCH 14/56] fix(openshell): recreate a sandbox this transport did not create Found live, in the UI path: if the named sandbox already exists, start() logged the create error and carried on. The worker inside allows one connection and the forward that reached it first still holds that slot, so the second forward leaves the real client queued -- Py() never returned and the logout thread sat in clearInsightGlobals until the sandbox was deleted by hand. A sandbox this transport is not driving belongs to a dead session, so the correct reading of 'already exists' is delete and recreate, not attach. --- src/prerna/util/OpenShellTransport.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 28fcf85b3e..c7ad4cf07e 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -285,6 +285,23 @@ public int start() throws IOException, InterruptedException { classLogger.info("Starting executor sandbox {} (worker port {})", sandboxName, workerPort); String createOut = run(create, readyTimeout); classLogger.info("Sandbox create output for {}: {}", sandboxName, createOut); + if (createOut.toLowerCase().contains("already exists")) { + // A sandbox with this name exists, but this transport did not create it in + // this call. That is not a reusable state: the worker inside allows one + // connection, and whatever forward reached it first still holds that slot, + // so a second forward leaves the real client queued and every request + // blocks forever. Measured 2026-10-22: a Py() that never returned and a + // logout thread stuck in clearInsightGlobals until the sandbox was deleted + // by hand. Deleting and recreating is the only safe interpretation -- + // a sandbox this instance is not driving belongs to a dead session. + classLogger.warn("Sandbox {} already exists; deleting and recreating it", sandboxName); + run(Arrays.asList(cli, "sandbox", "delete", sandboxName), 60); + createOut = run(create, readyTimeout); + classLogger.info("Sandbox re-create output for {}: {}", sandboxName, createOut); + if (createOut.toLowerCase().contains("already exists")) { + throw new IOException("sandbox " + sandboxName + " still exists after delete: " + createOut); + } + } this.forwardedPort = freeLocalPort(); List forward = Arrays.asList(cli, "forward", "service", "--target-port", workerPort, "--local", From f9e543160bfaa1cbb461611c7fc81fcdb6fc4b7a Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Thu, 22 Oct 2026 23:37:01 +0000 Subject: [PATCH 15/56] feat(openshell): run shell commands inside the sandbox CmdExecUtil ran bash with commons-exec in a chroot on Tomcat's machine and never consulted SANDBOX_MODE, so with SANDBOX_MODE=OPENSHELL the terminal, git operations and the agent tool handlers all stayed on the host. Measured before this change: Command("hostname") returned the host identity while Py() returned the sandbox's. OpenShellShellRunner runs the same command with 'openshell sandbox exec', which starts a process inside the sandbox. Deliberately not the worker socket: the worker serves one connection and a shell command must not compete with the REPL for it. The sandbox is started on demand, so a shell command can be the first thing a session does. One delegation point in CmdExecUtil, taken only when the user's SymlinkHelper reports an external sandbox; every other mode keeps the existing path untouched. After: hostname 936b273a023c, uid 1000, ls of the sandbox tree -- host is kunal0137-vnet. --- src/prerna/util/CmdExecUtil.java | 15 ++ .../util/openshell/OpenShellShellRunner.java | 201 ++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 src/prerna/util/openshell/OpenShellShellRunner.java diff --git a/src/prerna/util/CmdExecUtil.java b/src/prerna/util/CmdExecUtil.java index a1b8a77b1f..23ff9790a8 100644 --- a/src/prerna/util/CmdExecUtil.java +++ b/src/prerna/util/CmdExecUtil.java @@ -45,6 +45,7 @@ import org.apache.logging.log4j.Logger; import prerna.auth.User; +import prerna.util.openshell.OpenShellShellRunner; public class CmdExecUtil { @@ -57,14 +58,21 @@ public class CmdExecUtil { private String workingDir = null; private String contextDir = null; private String commandAppender = "cmd"; + /** Non-null only when this helper is driving an OpenShell sandbox. */ + private User user = null; + private boolean openshellSandbox = false; /** * Constructor */ public CmdExecUtil(User user, String insightId, String contextDir) { + this.user = user; if (user != null && Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE))) { SymlinkHelper chrootHelper = user.getUserSymlinkHelper(); this.chrootFolderPath = chrootHelper.getUserChrootFolder(); + // The sandbox is the boundary in this mode, so commands must run inside it + // rather than in a chroot on Tomcat's filesystem. + this.openshellSandbox = chrootHelper.isExternalSandbox(); } getCommandAppender(); this.insightId = insightId; @@ -138,6 +146,13 @@ private boolean directoryExists(String chrootRelativePath) { * Execute command with proper path handling */ public String executeCommand(String command) { + if (this.openshellSandbox) { + // Delegate the whole line, including cd/pwd/reset, to the sandbox. Doing it + // here rather than in runCommand keeps the host-path logic below untouched. + OpenShellShellRunner.Result result = OpenShellShellRunner.execute(this.user, this.workingDir, command); + this.workingDir = result.getWorkingDir(); + return result.getOutput(); + } String output = null; try { if (command.equalsIgnoreCase("reset")) { diff --git a/src/prerna/util/openshell/OpenShellShellRunner.java b/src/prerna/util/openshell/OpenShellShellRunner.java new file mode 100644 index 0000000000..e77ec8aad3 --- /dev/null +++ b/src/prerna/util/openshell/OpenShellShellRunner.java @@ -0,0 +1,201 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import prerna.auth.User; +import prerna.util.OpenShellTransport; +import prerna.util.Utility; + +/** + * Runs shell commands inside the user's OpenShell sandbox. + * + *

+ * Why this exists: {@code CmdExecUtil} executes commands with Apache + * {@code commons-exec} on Tomcat's own machine, inside a chroot. That is correct + * for the legacy jail and wrong for a sandbox — measured 2026-10-22, a + * {@code Command("hostname")} pixel returned the host's identity while + * {@code Py()} returned the sandbox's. The terminal was simply not part of the + * migration. + * + *

+ * The command is run with {@code openshell sandbox exec}, which starts a process + * inside the sandbox. That is deliberately not the worker socket: the + * worker serves one connection and a shell command must not compete with the + * REPL for it. It also means shell output does not require the python worker to + * be healthy. + * + *

+ * The sandbox is started on demand if it is not running yet, so a terminal + * command works as the first action in a session rather than requiring a + * {@code Py()} first. + */ +public final class OpenShellShellRunner { + + private static final int DEFAULT_TIMEOUT_SECONDS = 60; + private static final String CLI = "openshell"; + + private OpenShellShellRunner() { + // static utility + } + + /** Outcome of one shell command. */ + public static final class Result { + private final String output; + private final String workingDir; + + Result(String output, String workingDir) { + this.output = output; + this.workingDir = workingDir; + } + + public String getOutput() { + return output; + } + + public String getWorkingDir() { + return workingDir; + } + } + + /** + * Execute one command in the sandbox belonging to {@code user}. + * + * @param user the session user; used to make sure the sandbox exists + * @param workingDir directory to run in (an absolute path inside the sandbox) + * @param command the command line, as typed at the terminal + */ + public static Result execute(User user, String workingDir, String command) { + String dir = workingDir == null || workingDir.trim().isEmpty() ? "/" : workingDir.trim(); + try { + // Same on-demand rule as Py(): the first shell command in a session is + // allowed to create the sandbox. + user.getPythonSocketClient(true); + String chrootFolder = user.getUserSymlinkHelper().getUserChrootFolder(); + String sandboxName = OpenShellTransport.sandboxNameFor(chrootFolder); + + String trimmed = command == null ? "" : command.trim(); + if (trimmed.isEmpty()) { + return new Result("", dir); + } + if (equalsIgnoreCase(trimmed, "pwd")) { + return new Result(dir, dir); + } + if (equalsIgnoreCase(trimmed, "reset")) { + return new Result(dir, dir); + } + if (trimmed.startsWith("cd ") || trimmed.equals("cd")) { + String target = trimmed.length() > 2 ? trimmed.substring(3).trim() : "/"; + if (target.equals("~")) { + target = "/"; + } + String resolved = runInSandbox(sandboxName, dir, "cd " + target + " && pwd"); + if (resolved == null || resolved.trim().isEmpty() || resolved.startsWith("__ERROR__")) { + return new Result(resolved == null ? "Directory doesn't exist: " + target + : resolved.replace("__ERROR__", ""), dir); + } + String newDir = firstLine(resolved); + return new Result(newDir, newDir); + } + String output = runInSandbox(sandboxName, dir, trimmed); + return new Result(output, dir); + } catch (Exception e) { + return new Result("Shell execution in sandbox failed: " + e.getClass().getSimpleName() + ": " + + e.getMessage(), dir); + } + } + + private static boolean equalsIgnoreCase(String left, String right) { + return left != null && left.equalsIgnoreCase(right); + } + + private static String firstLine(String text) { + int newline = text.indexOf('\n'); + return (newline < 0 ? text : text.substring(0, newline)).trim(); + } + + /** + * One `openshell sandbox exec` invocation. + * + *

+ * {@code bash -lc} on purpose: the command string users type is a shell line + * (pipes, redirects, &&), and a login shell gives a predictable PATH inside the + * image. The working directory is applied by the shell itself rather than by + * {@code --workdir}, because the directory may be a path that only exists in + * the sandbox after an earlier upload. + */ + private static String runInSandbox(String sandboxName, String workingDir, String command) throws IOException { + List argv = new ArrayList<>(Arrays.asList(CLI, "sandbox", "exec", "-n", sandboxName, "--", + "bash", "-lc", "cd " + shellQuote(workingDir) + " 2>/dev/null; " + command)); + ProcessBuilder builder = new ProcessBuilder(argv); + builder.redirectErrorStream(true); + builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); + // The CLI needs the same environment the transport uses (gateway config, + // TLS bundle); inherit it rather than reconstructing it here. + Process process = builder.start(); + String output; + try { + if (!process.waitFor(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + process.destroyForcibly(); + return "__ERROR__command timed out after " + DEFAULT_TIMEOUT_SECONDS + "s"; + } + output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + process.destroyForcibly(); + return "__ERROR__interrupted"; + } + String cleaned = output.replace("\\r", "").trim(); + if (process.exitValue() != 0 && !cleaned.isEmpty()) { + return cleaned; + } + return cleaned; + } + + private static String shellQuote(String value) { + return "'" + value.replace("'", "'\\''") + "'"; + } + + /** Exposed for tests: the exact argv shape used for one command. */ + public static List describeCommand(String sandboxName, String workingDir, String command) { + return Arrays.asList(CLI, "sandbox", "exec", "-n", sandboxName, "--", "bash", "-lc", + "cd " + shellQuote(workingDir) + " 2>/dev/null; " + command); + } + + /** Kept for symmetry with the transport's property handling. */ + static String cliPath() { + String configured = Utility.getDIHelperProperty(OpenShellTransport.CLI_PROP); + return configured == null || configured.trim().isEmpty() ? CLI : configured.trim(); + } +} From af9d81067f98ccabc094a0b759be1dc9df845f90 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 01:32:03 +0000 Subject: [PATCH 16/56] feat(openshell): fetch sandbox writes back at the execution boundary Before this, the sandbox was write-only: a file created by Py() or a shell command lived in the sandbox copy and died with it, so the asset browser, git and MCP generation never saw it. Measured: to_csv into the project assets -> visible to 'ls' in the terminal, absent from GetAppAssets, gone at teardown. Now, after the terminal response of an executeCommand, the wrapper reconciles the session's transported roots back onto the host: - OpenShellTransport.fetchBack streams the tree out with 'sandbox exec -- tar -cf -' (sandbox download is workspace-scoped and refuses project paths), validates the archive (relative paths only, size cap), extracts to a temp dir and reconciles per file: content must differ AND the sandbox copy must be newer, so a stale sandbox cannot clobber a host edit. - SymlinkHelper exposes the roots it transported; ClientProcessWrapper filters to project/room/user roots and throttles to one reconcile per 1.5s. - NativePySocketClient calls it once per completed request. Verified: Py writes iris_fb.csv in the sandbox -> host file appears (16 bytes) -> GetAppAssets returns its contents. Single pod only: the push to central storage for the other pods is the next step. --- src/prerna/om/ClientProcessWrapper.java | 44 ++++ .../tcp/client/NativePySocketClient.java | 13 +- src/prerna/util/OpenShellTransport.java | 231 ++++++++++++++++++ src/prerna/util/SymlinkHelper.java | 9 + 4 files changed, 296 insertions(+), 1 deletion(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 15b0cc2e48..f79fff34d9 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -393,6 +393,50 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS * @param cleanUpFolder true to also recursively delete the server scratch * directory and any namespace-sandbox folders */ + /** + * Reconcile the sandbox copies of this session's transported roots back onto the + * host, so a file written by {@code Py()} or a shell command is visible to the + * platform (asset browser, MCP generators, git) without waiting for logout. + * + *

+ * Called at the execution boundary. Throttled: a REPL that runs many small blocks + * must not tar its roots per call, and the reconcile itself skips files that are + * identical or where the sandbox copy is older. + * + * @return number of files written back + */ + public int fetchBackTransportedRoots() { + if (this.openshell == null || this.chrootSymlinkHelper == null) { + return 0; + } + long now = System.currentTimeMillis(); + if (now - this.lastFetchBackMs < FETCH_BACK_MIN_INTERVAL_MS) { + return 0; + } + this.lastFetchBackMs = now; + int total = 0; + for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { + String normalized = Utility.normalizePath(path); + // Only the user-facing roots. Engine and runtime trees are never written by + // user code, and pulling them back would waste a tar per block. + boolean syncable = normalized.contains("/" + Constants.PROJECT_FOLDER + "/") + || normalized.contains("/" + Constants.ROOM_FOLDER + "/") + || normalized.contains("/" + Constants.USER_FOLDER + "/"); + if (!syncable) { + continue; + } + try { + total += this.openshell.fetchBack(path); + } catch (Exception e) { + classLogger.warn("Fetch-back failed for {}", path, e); + } + } + return total; + } + + private volatile long lastFetchBackMs = 0L; + private static final long FETCH_BACK_MIN_INTERVAL_MS = 1500L; + public void shutdown(boolean cleanUpFolder) { lockDestroy.lock(); try { diff --git a/src/prerna/tcp/client/NativePySocketClient.java b/src/prerna/tcp/client/NativePySocketClient.java index e0219c9ecb..c79dcec7ac 100644 --- a/src/prerna/tcp/client/NativePySocketClient.java +++ b/src/prerna/tcp/client/NativePySocketClient.java @@ -655,7 +655,18 @@ public Object executeCommand(PayloadStruct ps) { } } - return responseMap.remove(ps.epoc); + PayloadStruct reply = responseMap.remove(ps.epoc); + // Execution boundary: a file the block wrote inside the sandbox becomes + // visible to the platform here -- asset browser, git, MCP generation -- + // instead of waiting for logout, where it used to be lost entirely. + if (reply != null && this.cpw != null) { + try { + this.cpw.fetchBackTransportedRoots(); + } catch (Exception fetchError) { + classLogger.warn("Fetch-back after execution failed", fetchError); + } + } + return reply; } finally { ps.unlockResponse(); } diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index c7ad4cf07e..d0dd93f140 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -156,6 +156,9 @@ private OpenShellTransport(String sandboxName, String workerPort) { */ public static final int MAX_SANDBOX_NAME_LENGTH = 19; + /** Cap for one fetch-back archive; larger trees are pulled path by path or not at all. */ + private static final long MAX_FETCH_BYTES = 256L * 1024 * 1024; + private static final String NAME_PREFIX = "pyw-"; /** @@ -502,6 +505,234 @@ public int getForwardedPort() { return forwardedPort; } + /** + * Copy one uploaded root back out of the sandbox onto the host. + * + *

+ * This is the missing half of the data plane: {@link #upload(String)} moves a tree + * in, and without this nothing ever moves back, so a file an agent or a + * {@code Py()} call writes inside the sandbox is destroyed with it. + * + *

+ * {@code openshell sandbox download} cannot be used directly: it is scoped to the + * sandbox workspace ({@code /opt/semosshome/py}) and refuses project paths with + * "outside the sandbox workspace". So the tree is streamed out through + * {@code sandbox exec -- tar -cf -} instead, which is byte-safe and therefore also + * correct for images and wheels. + * + *

+ * Reconciliation is per file and deliberately conservative: a file is written back + * only when its content differs and the sandbox copy is newer than the host + * copy. That keeps a stale or half-written sandbox from clobbering a host edit made + * after the upload. + * + * @param absPath a path previously passed to {@link #upload(String)} + * @return number of files written back + */ + public int fetchBack(String absPath) { + if (absPath == null || absPath.trim().isEmpty()) { + return 0; + } + String baseFolder = Utility.getBaseFolder(); + String normalized = Utility.normalizePath(absPath); + if (baseFolder == null || !normalized.startsWith(Utility.normalizePath(baseFolder))) { + classLogger.warn("Refusing fetch-back outside the SEMOSS home: {}", absPath); + return 0; + } + java.io.File source = new java.io.File(normalized); + String parent = source.getParent(); + String base = source.getName(); + if (parent == null) { + return 0; + } + + java.nio.file.Path temp = null; + try { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + byte[] archive = runBytes( + Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", parent, "-cf", "-", base), + 120); + if (archive == null || archive.length == 0) { + return 0; + } + if (archive.length > MAX_FETCH_BYTES) { + classLogger.warn("Fetch-back of {} is {} bytes, over the {} byte cap; skipped", absPath, archive.length, + MAX_FETCH_BYTES); + return 0; + } + if (!archiveEntriesAreSafe(archive)) { + classLogger.warn("Fetch-back of {} contained an absolute or traversing path; skipped", absPath); + return 0; + } + temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-"); + extractArchive(archive, temp); + return reconcile(temp.resolve(base), java.nio.file.Paths.get(normalized)); + } catch (Exception e) { + classLogger.warn("Fetch-back failed for {}", absPath, e); + return 0; + } finally { + if (temp != null) { + deleteQuietly(temp); + } + } + } + + /** + * Write files from {@code incoming} onto {@code target} when the sandbox copy is + * newer and differs. Returns how many were written. + */ + private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target) throws java.io.IOException { + if (!java.nio.file.Files.exists(incoming)) { + return 0; + } + int[] copied = { 0 }; + try (java.util.stream.Stream walk = java.nio.file.Files.walk(incoming)) { + walk.filter(java.nio.file.Files::isRegularFile).filter(path -> !java.nio.file.Files.isSymbolicLink(path)) + .forEach(path -> { + java.nio.file.Path relative = incoming.relativize(path); + java.nio.file.Path destination = target.resolve(relative.toString()); + try { + if (java.nio.file.Files.exists(destination)) { + if (sameContent(path, destination) + || java.nio.file.Files.getLastModifiedTime(path).compareTo( + java.nio.file.Files.getLastModifiedTime(destination)) <= 0) { + return; + } + } + java.nio.file.Path folder = destination.getParent(); + if (folder != null) { + java.nio.file.Files.createDirectories(folder); + } + java.nio.file.Files.copy(path, destination, + java.nio.file.StandardCopyOption.REPLACE_EXISTING, + java.nio.file.StandardCopyOption.COPY_ATTRIBUTES); + copied[0]++; + } catch (java.io.IOException e) { + classLogger.warn("Could not write back {}", destination, e); + } + }); + } + if (copied[0] > 0) { + classLogger.info("Fetch-back wrote {} file(s) into {}", copied[0], target); + } + return copied[0]; + } + + private static boolean sameContent(java.nio.file.Path left, java.nio.file.Path right) throws java.io.IOException { + if (java.nio.file.Files.size(left) != java.nio.file.Files.size(right)) { + return false; + } + return java.util.Arrays.equals(java.nio.file.Files.readAllBytes(left), java.nio.file.Files.readAllBytes(right)); + } + + private static boolean archiveEntriesAreSafe(byte[] archive) { + try { + Process process = new ProcessBuilder("tar", "-tf", "-") + .redirectErrorStream(true).start(); + Thread writer = new Thread(() -> { + try (java.io.OutputStream stdin = process.getOutputStream()) { + stdin.write(archive); + } catch (java.io.IOException ignored) { + // process may exit early; the exit code carries the outcome + } + }); + writer.start(); + String listing = new String(process.getInputStream().readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8); + writer.join(5000); + process.waitFor(20, TimeUnit.SECONDS); + for (String line : listing.split("\\R")) { + String entry = line.trim(); + if (entry.isEmpty()) { + continue; + } + if (entry.startsWith("/") || entry.contains("..")) { + return false; + } + } + return true; + } catch (Exception e) { + return false; + } + } + + private static void extractArchive(byte[] archive, java.nio.file.Path destination) throws Exception { + Process process = new ProcessBuilder("tar", "-xf", "-", "-C", destination.toString()) + .redirectErrorStream(true).start(); + Thread writer = new Thread(() -> { + try (java.io.OutputStream stdin = process.getOutputStream()) { + stdin.write(archive); + } catch (java.io.IOException ignored) { + // see above + } + }); + writer.start(); + String output = new String(process.getInputStream().readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8); + writer.join(10_000); + if (!process.waitFor(60, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new java.io.IOException("tar extraction timed out"); + } + if (process.exitValue() != 0) { + throw new java.io.IOException("tar extraction failed: " + output.trim()); + } + } + + private static void deleteQuietly(java.nio.file.Path root) { + try (java.util.stream.Stream walk = java.nio.file.Files.walk(root)) { + walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { + try { + java.nio.file.Files.deleteIfExists(path); + } catch (java.io.IOException ignored) { + // best effort + } + }); + } catch (java.io.IOException ignored) { + // best effort + } + } + + /** + * Run a CLI command and return its raw stdout. {@link #run(List, int)} decodes to + * String and would corrupt any binary payload, which is exactly what a tar stream is. + */ + private byte[] runBytes(List cmd, int timeoutSeconds) throws IOException, InterruptedException { + ProcessBuilder builder = new ProcessBuilder(cmd); + builder.environment().putAll(cliEnv()); + builder.redirectErrorStream(false); + builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); + Process process = builder.start(); + java.io.ByteArrayOutputStream collected = new java.io.ByteArrayOutputStream(); + Thread reader = new Thread(() -> { + try (java.io.InputStream stream = process.getInputStream()) { + stream.transferTo(collected); + } catch (java.io.IOException ignored) { + // the exit code carries the outcome + } + }, "openshell-fetchback-reader"); + reader.setDaemon(true); + reader.start(); + Thread errors = new Thread(() -> { + try (java.io.InputStream stream = process.getErrorStream()) { + stream.transferTo(java.io.OutputStream.nullOutputStream()); + } catch (java.io.IOException ignored) { + // discarded + } + }, "openshell-fetchback-errors"); + errors.setDaemon(true); + errors.start(); + if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); + } + reader.join(5000); + if (process.exitValue() != 0) { + throw new IOException("command failed (" + process.exitValue() + "): " + String.join(" ", cmd)); + } + return collected.toByteArray(); + } + /** * Stop forwarding and delete the sandbox. * diff --git a/src/prerna/util/SymlinkHelper.java b/src/prerna/util/SymlinkHelper.java index 048e55aa37..5af43ae60c 100644 --- a/src/prerna/util/SymlinkHelper.java +++ b/src/prerna/util/SymlinkHelper.java @@ -90,6 +90,15 @@ public enum SandboxMode { * re-symlink the same folders; without this the tree would be re-transferred on every request. */ private final Set activeTransports = ConcurrentHashMap.newKeySet(); + + /** + * Paths this helper has handed to the sandbox, in the order they were first + * requested. Read-only view: the fetch-back needs to know which roots exist in + * the sandbox so it can reconcile them when the process finishes a block. + */ + public java.util.List getTransportedPaths() { + return new java.util.ArrayList<>(activeTransports); + } private SandboxTransporter transporter; /** From cd87f41ec52bacf9183c02a04c5241a562616fc5 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 01:42:35 +0000 Subject: [PATCH 17/56] fix(openshell): reconcile every block, on shell writes too, and before teardown Three gaps found by testing, all in the fetch-back added in 863c3f443d: - The 1.5s throttle SKIPPED the reconcile instead of deferring it. Three Py blocks in one session left two files stranded in the sandbox until some later call. Skipping is wrong for a durable-write path; the throttle is gone. - Shell commands do not travel over the python socket, so "cat > file" or a git operation in the terminal never hit the hook. OpenShellShellRunner now reconciles after the command runs. - The last block before teardown could still be lost; shutdown() now reconciles once before deleting the sandbox. Also stopped excluding .git from the fetched archive. It costs bytes and writes nothing today -- the probe image has no git (git: not found) so the sandbox copy never changes -- but when the production image ships git, a commit made in the shell is exactly the work that must come back. Concurrent host/sandbox commits still need the revision guard. Verified: three Py writes plus one shell write in a single session all appear on the host. --- src/prerna/om/ClientProcessWrapper.java | 15 +++++++-------- src/prerna/util/OpenShellTransport.java | 12 +++++++++--- .../util/openshell/OpenShellShellRunner.java | 10 ++++++++++ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index f79fff34d9..380f7699f8 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -409,11 +409,6 @@ public int fetchBackTransportedRoots() { if (this.openshell == null || this.chrootSymlinkHelper == null) { return 0; } - long now = System.currentTimeMillis(); - if (now - this.lastFetchBackMs < FETCH_BACK_MIN_INTERVAL_MS) { - return 0; - } - this.lastFetchBackMs = now; int total = 0; for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { String normalized = Utility.normalizePath(path); @@ -434,9 +429,6 @@ public int fetchBackTransportedRoots() { return total; } - private volatile long lastFetchBackMs = 0L; - private static final long FETCH_BACK_MIN_INTERVAL_MS = 1500L; - public void shutdown(boolean cleanUpFolder) { lockDestroy.lock(); try { @@ -496,6 +488,13 @@ & deleteFolderWithRetries(this.sandboxJailDir) // teardown -- the gateway delete is authoritative and the worker cannot // outlive its sandbox. There is no graceful-shutdown verb to send first. if (this.openshell != null) { + try { + // Last reconcile before the sandbox disappears: anything written since the + // final request would otherwise die with it. + fetchBackTransportedRoots(); + } catch (Exception e) { + classLogger.warn("Fetch-back before sandbox teardown failed for port {}", this.port, e); + } try { this.openshell.close(); } catch (Exception e) { diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index d0dd93f140..ff85040d07 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -549,9 +549,15 @@ public int fetchBack(String absPath) { java.nio.file.Path temp = null; try { String cli = propOr(CLI_PROP, DEFAULT_CLI); - byte[] archive = runBytes( - Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", parent, "-cf", "-", base), - 120); + // .git is deliberately NOT excluded: a commit made in the sandbox shell is + // work the user expects to keep, and git objects are content-addressed so + // copying them back is safe. Only regenerable caches are skipped. + // Known risk, not yet guarded: a host-side commit and a sandbox-side commit + // made concurrently can leave refs diverged; the revision/conflict check that + // would catch it is still to come. + byte[] archive = runBytes(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", parent, + "--exclude=node_modules", "--exclude=__pycache__", "--exclude=.venv", + "-cf", "-", base), 120); if (archive == null || archive.length == 0) { return 0; } diff --git a/src/prerna/util/openshell/OpenShellShellRunner.java b/src/prerna/util/openshell/OpenShellShellRunner.java index e77ec8aad3..d061b26b74 100644 --- a/src/prerna/util/openshell/OpenShellShellRunner.java +++ b/src/prerna/util/openshell/OpenShellShellRunner.java @@ -129,6 +129,16 @@ public static Result execute(User user, String workingDir, String command) { return new Result(newDir, newDir); } String output = runInSandbox(sandboxName, dir, trimmed); + // Shell writes are not requests on the python socket, so they do not pass the + // fetch-back hook there. Reconcile here too -- this is the path a `git commit` + // or a `cat > file` takes, and losing that work is exactly what we are fixing. + try { + if (user.getPythonClientProcessWrapper() != null) { + user.getPythonClientProcessWrapper().fetchBackTransportedRoots(); + } + } catch (Exception ignored) { + // the command itself already ran; a failed reconcile must not fail the shell + } return new Result(output, dir); } catch (Exception e) { return new Result("Shell execution in sandbox failed: " + e.getClass().getSimpleName() + ": " From 0587f42b83a826be1f9eb5e0ca450a6b66033e78 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 03:11:30 +0000 Subject: [PATCH 18/56] feat(openshell): pull back only the files a block wrote Write-back used to tar every transported root at every execution boundary: ~190 ms per root, so `LoadApp(a); LoadApp(b); LoadApp(c)` paid ~750 ms per block whether or not anything changed. The worker now knows which paths changed and the host fetches those. Worker side (py/): - new gaas_change_watcher.py: ctypes inotify (no dependency), coalesced per-path events, 2k cap, IN_Q_OVERFLOW flag, and a baseline flag. A new directory is armed *and scanned*, because `mkdir x; write x/f` inside one block otherwise reports the directory and loses the file. - handler gains a CHANGES operation returning {changes[], overflow, baseline, watched, events, roots}. - the watcher is a module-level singleton created lazily on the first CHANGES call: the TCP handler is built per connection, and at setup() the session's project tree has not been materialised yet, so arming there yields zero roots and pins the host to the slow path forever. Host side (Java): - PayloadStruct: CHANGES operation. - NativePySocketClient.getFileChanges(); the after-execution hook now fires only for PYTHON and CMD, so PollNotifications and other internal polls no longer tar the tree. - OpenShellTransport.fetchBackPaths(): one tar of the named paths, with the same traversal and size checks as the whole-root fetch. - ClientProcessWrapper.fetchBackChangedPaths(): groups paths by transported root, deletes reported removals, and falls back to the full reconcile on baseline, overflow, or no answer. - OpenShellShellRunner takes the same path for shell commands. Verified on a 240-file project: three blocks, three unique files, all on the host; journal shows "Fetch-back wrote 1 file(s) from 1 changed path(s)"; watcher counters went watched:173 baseline:true -> changes:[x.txt] events:2 with root inodes unchanged from arm time. Cost 0.20 s/block vs ~0.30 s for the full reconcile; the first block after materialise still does one full reconcile, by design. --- py/gaas_change_watcher.py | 257 ++++++++++++++++++ py/gaas_tcp_server_handler.py | 57 ++++ src/prerna/om/ClientProcessWrapper.java | 110 ++++++++ src/prerna/tcp/PayloadStruct.java | 2 +- .../tcp/client/NativePySocketClient.java | 45 ++- src/prerna/util/OpenShellTransport.java | 74 +++++ .../util/openshell/OpenShellShellRunner.java | 2 +- 7 files changed, 543 insertions(+), 4 deletions(-) create mode 100644 py/gaas_change_watcher.py diff --git a/py/gaas_change_watcher.py b/py/gaas_change_watcher.py new file mode 100644 index 0000000000..c0bd472d90 --- /dev/null +++ b/py/gaas_change_watcher.py @@ -0,0 +1,257 @@ +"""Change tracking for the sandbox worker. + +The host needs to know *which* files a block wrote, not just *that* something +changed. A stat-walk of the tree costs O(files) per block (~6 ms for 240 files, +seconds for 100k), and spawning a process to do it costs ~80 ms before it does +any work. inotify is O(events): idle costs nothing, one write costs one event. + +Measured on the dev VM (2026-10-23), watching a project's assets tree: + + CREATE/MODIFY/CLOSE_WRITE watch_bash.txt (written by bash) + CREATE/MODIFY/CLOSE_WRITE watch_py.txt (written by a *different* python) + CREATE,DIR newdir + the first snapshot reports + ``baseline`` and is discarded, because the host already holds what it uploaded; +* the kernel queue can overflow -> ``overflow`` is reported and the host must + fall back to a full reconcile. +""" + +import ctypes +import ctypes.util +import errno +import os +import select +import struct +import threading + +IN_MODIFY = 0x00000002 +IN_CLOSE_WRITE = 0x00000008 +IN_MOVED_FROM = 0x00000040 +IN_MOVED_TO = 0x00000080 +IN_CREATE = 0x00000100 +IN_DELETE = 0x00000200 +IN_DELETE_SELF = 0x00000400 +IN_MOVE_SELF = 0x00000800 +IN_Q_OVERFLOW = 0x00004000 +IN_ISDIR = 0x40000000 + +_EVENT = struct.Struct("iIII") +_MASK = (IN_CREATE | IN_MODIFY | IN_CLOSE_WRITE | IN_MOVED_FROM | IN_MOVED_TO | IN_DELETE + | IN_DELETE_SELF | IN_MOVE_SELF) + + +class ChangeWatcher: + """Watch a set of roots and report the paths that changed between snapshots.""" + + def __init__(self, roots, max_changes=2000): + self.roots = [root for root in roots if root and os.path.isdir(root)] + self.max_changes = max_changes + self._fd = None + self._thread = None + self._stop = threading.Event() + self._lock = threading.Lock() + self._changes = {} + self._overflow = False + self._baseline_pending = True + self._armed = 0 + self._events_seen = 0 + # inotify events carry a watch descriptor, not a path; this maps it back. + self._watched = {} + self._root_ino = {} + + # -- lifecycle --------------------------------------------------------- + + def start(self): + if not self.roots: + return False + library = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6", use_errno=True) + library.inotify_init1.argtypes = [ctypes.c_int] + library.inotify_add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32] + fd = library.inotify_init1(os.O_NONBLOCK) + if fd < 0: + return False + self._fd = fd + self._libc = library + for root in self.roots: + self._arm_tree(root) + self._thread = threading.Thread(target=self._loop, name="sandbox-change-watcher", daemon=True) + self._thread.start() + return True + + def stop(self): + self._stop.set() + if self._fd is not None: + try: + os.close(self._fd) + except OSError: + pass + + # -- watch arming ------------------------------------------------------ + + def _arm(self, directory): + try: + watch = self._libc.inotify_add_watch(self._fd, directory.encode("utf-8"), _MASK) + except Exception: + return False + if watch >= 0: + self._watched[watch] = directory + self._armed += 1 + return True + return False + + def add_roots(self, roots): + """Arm roots discovered after start (a session materialises its tree late).""" + added = 0 + for root in roots: + if root in self.roots or not os.path.isdir(root): + continue + self.roots.append(root) + self._arm_tree(root) + added += 1 + return added + + def _arm_tree(self, root): + try: + self._root_ino[root] = os.stat(root).st_ino + except OSError: + pass + for current, _dirs, _files in os.walk(root): + self._arm(current) + + # -- event loop -------------------------------------------------------- + + def _loop(self): + while not self._stop.is_set(): + try: + ready, _, _ = select.select([self._fd], [], [], 0.2) + except (OSError, ValueError): + break + if not ready: + continue + try: + data = os.read(self._fd, 65536) + except OSError as exc: + if exc.errno in (errno.EAGAIN, errno.EWOULDBLOCK): + continue + break + self._consume(data) + + def _consume(self, data): + offset = 0 + additions = [] + with self._lock: + while offset + _EVENT.size <= len(data): + _watch, mask, _cookie, length = _EVENT.unpack_from(data, offset) + offset += _EVENT.size + raw = data[offset:offset + length] + offset += length + self._events_seen += 1 + if mask & IN_Q_OVERFLOW: + self._overflow = True + continue + name = raw.split(b"\0", 1)[0].decode("utf-8", "replace") + if not name: + continue + # The event does not carry the directory, only the watched descriptor's + # name. Watch descriptors map back through the path we armed, so we keep + # a lookup from descriptor to directory. + directory = self._watched.get(_watch) if hasattr(self, "_watched") else None + if directory is None: + continue + path = os.path.join(directory, name) + if mask & IN_ISDIR: + if mask & (IN_CREATE | IN_MOVED_TO): + # A new directory: arm it *and* scan what landed inside before + # the watch existed. Without the scan, `mkdir x; write x/f` + # inside one block reports the directory and misses the file. + additions.append(path) + continue + if mask & (IN_DELETE | IN_MOVED_FROM): + self._record(path, "delete") + else: + self._record(path, "modify") + for path in additions: + self._adopt_directory(path) + + def _adopt_directory(self, path): + self._arm_tree(path) + with self._lock: + for current, _dirs, files in os.walk(path): + for name in files: + self._record(os.path.join(current, name), "modify") + + def _record(self, path, op): + if len(self._changes) >= self.max_changes: + self._overflow = True + return + previous = self._changes.get(path) + if previous == "delete" and op == "modify": + # Re-created within the same block: it exists, so it is a modify. + self._changes[path] = "modify" + else: + self._changes[path] = op + + # -- snapshot ---------------------------------------------------------- + + def snapshot(self): + """Return and clear the accumulated changes. + + The first call after start returns ``baseline: True`` with an empty set: + everything already in the tree arrived before the watch existed, and the + host materialised it, so reporting it would be noise. + """ + with self._lock: + roots = [] + for root in self.roots: + try: + roots.append({ + "path": root, + "ino_now": os.stat(root).st_ino, + "ino_armed": self._root_ino.get(root), + }) + except OSError as exc: + roots.append({"path": root, "error": str(exc)}) + payload = { + "changes": [{"path": path, "op": op} for path, op in self._changes.items()], + "overflow": self._overflow, + "baseline": self._baseline_pending, + "watched": self._armed, + "events": self._events_seen, + "roots": roots, + } + self._changes = {} + self._overflow = False + self._baseline_pending = False + return payload + + +_shared = None +_shared_lock = threading.Lock() + + +def shared_watcher(roots): + """One watcher per process. + + The TCP handler is instantiated per connection, so a per-handler watcher + re-arms and re-baselines on every reconnect -- which silently pins the host to + the slow path, because every call then reports ``baseline``. Keeping the watcher + at module scope means the baseline is consumed once and subsequent calls report + real deltas. + """ + global _shared + with _shared_lock: + if _shared is not None: + _shared.add_roots(roots) + return _shared + if _shared is None: + candidate = ChangeWatcher(roots) + if candidate.start(): + _shared = candidate + return _shared diff --git a/py/gaas_tcp_server_handler.py b/py/gaas_tcp_server_handler.py index 97ffb0c716..9726909ff3 100644 --- a/py/gaas_tcp_server_handler.py +++ b/py/gaas_tcp_server_handler.py @@ -367,6 +367,14 @@ def setup(self): socket_handler=self, ) + # Track filesystem changes so the host can pull back only what a block wrote. + # Absent module or an unsupported kernel just means the host falls back to a + # full reconcile, so this must never be able to break startup. + # The watcher is created lazily on the first CHANGES call: at setup time the + # session's project tree has not been materialised into the sandbox yet, so + # arming here yields zero roots and pins the host to the slow path forever. + self.change_watcher = None + # set the thread local TCPServerHandler.thread_local = threading.local() @@ -413,6 +421,52 @@ def prod_logger(self, message: str): if self.logger: self.logger.info(message) + def _watcher_roots(self): + """Project/room/user roots that exist *now* (the worker starts without --prefix).""" + bases = [] + if self.prefix: + bases.append(self.prefix) + base_folder = os.environ.get("SEMOSS_HOME") or "/opt/semosshome" + if base_folder not in bases: + bases.append(base_folder) + for base in bases: + roots = [ + os.path.join(base, name) + for name in ("project", "room", "user") + if os.path.isdir(os.path.join(base, name)) + ] + if roots: + return roots + return [] + + def handle_changes(self): + """Return the files that changed since the last snapshot. + + The host calls this after operations that can write, then pulls exactly those + paths. The watcher is created here rather than in ``setup()`` because the + session's project tree is not materialised yet when the first connection is + accepted -- arming that early yields zero roots and pins the host to the slow + path forever. Overflow, or a set past the cap, is reported so the host falls + back to a full reconcile instead of trusting a partial list. + """ + if self.change_watcher is None: + try: + from gaas_change_watcher import shared_watcher + + self.change_watcher = shared_watcher(self._watcher_roots()) + except Exception: + self.change_watcher = None + if self.change_watcher is None: + snapshot = { + "changes": [], + "overflow": True, + "baseline": False, + "watched": 0, + } + else: + snapshot = self.change_watcher.snapshot() + self.send_output(snapshot, operation="CHANGES", response=True) + def handle(self): """ Handles incoming requests from the client. @@ -631,6 +685,9 @@ def get_final_output( elif payload["operation"] == "PYTHON": insight_id = payload.get("insightId") self.handle_python(command, insight_id) + # host asks which files changed since the last snapshot (write-back) + elif payload["operation"] == "CHANGES": + self.handle_changes() # this is when it is a response elif payload["response"]: self.handle_response() diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 380f7699f8..79fe39fd05 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -393,6 +393,116 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS * @param cleanUpFolder true to also recursively delete the server scratch * directory and any namespace-sandbox folders */ + + /** + * Reconcile only the files the sandbox worker reports as changed since the last + * snapshot. + * + *

+ * Cost is proportional to the change: a block that writes three files produces + * three paths and one small tar, instead of a tar per transported root. It falls + * back to {@link #fetchBackTransportedRoots()} whenever the worker's answer cannot + * be trusted — no watcher, the first call after materialise (everything in the tree + * arrived before the watch existed), a kernel queue overflow, or a change set past + * the worker's cap. + * + * @return number of files written back + */ + public int fetchBackChangedPaths() { + if (this.openshell == null || this.chrootSymlinkHelper == null) { + return 0; + } + Map snapshot = null; + if (this.socketClient instanceof NativePySocketClient pyClient) { + try { + snapshot = pyClient.getFileChanges(); + } catch (Exception e) { + classLogger.warn("Change query failed; falling back to a full reconcile", e); + } + } + if (snapshot == null || truthy(snapshot.get("overflow")) || truthy(snapshot.get("baseline"))) { + return fetchBackTransportedRoots(); + } + Object raw = snapshot.get("changes"); + if (!(raw instanceof List changes) || changes.isEmpty()) { + return 0; + } + + // Only roots this session transported are eligible, longest first so a nested + // root wins over its parent. + List roots = new ArrayList<>(); + for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { + String normalized = Utility.normalizePath(path); + if (normalized.contains("/" + Constants.PROJECT_FOLDER + "/") + || normalized.contains("/" + Constants.ROOM_FOLDER + "/") + || normalized.contains("/" + Constants.USER_FOLDER + "/")) { + roots.add(normalized); + } + } + roots.sort((first, second) -> second.length() - first.length()); + + Map> writes = new HashMap<>(); + List deletes = new ArrayList<>(); + for (Object entry : changes) { + if (!(entry instanceof Map change)) { + continue; + } + Object pathValue = change.get("path"); + if (pathValue == null) { + continue; + } + String path = Utility.normalizePath(pathValue.toString()); + String root = null; + for (String candidate : roots) { + if (path.startsWith(candidate + "/")) { + root = candidate; + break; + } + } + if (root == null) { + continue; + } + String relative = path.substring(root.length() + 1); + if ("delete".equals(String.valueOf(change.get("op")))) { + deletes.add(path); + } else { + List group = writes.get(root); + if (group == null) { + group = new ArrayList<>(); + writes.put(root, group); + } + group.add(relative); + } + } + + int total = 0; + for (Map.Entry> group : writes.entrySet()) { + try { + total += this.openshell.fetchBackPaths(group.getKey(), group.getValue()); + } catch (Exception e) { + classLogger.warn("Path fetch-back failed for {}", group.getKey(), e); + } + } + for (String path : deletes) { + try { + Files.deleteIfExists(Paths.get(path)); + } catch (Exception e) { + classLogger.warn("Delete after change set failed for {}", path, e); + } + } + if (total > 0 || !deletes.isEmpty()) { + classLogger.info("Fetch-back wrote {} file(s) from {} changed path(s)", total, changes.size()); + } + return total; + } + + private static boolean truthy(Object value) { + if (value instanceof Boolean flag) { + return flag; + } + return value != null && Boolean.parseBoolean(value.toString()); + } + /** * Reconcile the sandbox copies of this session's transported roots back onto the * host, so a file written by {@code Py()} or a shell command is visible to the diff --git a/src/prerna/tcp/PayloadStruct.java b/src/prerna/tcp/PayloadStruct.java index f850e4b95d..c3831e9acf 100644 --- a/src/prerna/tcp/PayloadStruct.java +++ b/src/prerna/tcp/PayloadStruct.java @@ -41,7 +41,7 @@ public class PayloadStruct implements Serializable { public enum OPERATION { R, PYTHON, NODE, CHROME, ECHO, ENGINE, REACTOR, INSIGHT, PROJECT, CMD, STDOUT, STDERR, STRUCTURED_STREAM, - CANCELLED, LOG + CANCELLED, LOG, CHANGES }; public OPERATION operation = OPERATION.R; // setting default to R diff --git a/src/prerna/tcp/client/NativePySocketClient.java b/src/prerna/tcp/client/NativePySocketClient.java index c79dcec7ac..bd69c953c7 100644 --- a/src/prerna/tcp/client/NativePySocketClient.java +++ b/src/prerna/tcp/client/NativePySocketClient.java @@ -82,6 +82,12 @@ public class NativePySocketClient extends SocketClient implements Runnable, Clos // the stdout stream private static final String INLINE_IMAGE_PREFIX = " getFileChanges() { + PayloadStruct ps = new PayloadStruct(); + ps.operation = PayloadStruct.OPERATION.CHANGES; + ps.methodName = "GET_CHANGES"; + ps.payload = new Object[] {}; + ps.hasReturn = true; + ps.longRunning = false; + IN_CHANGE_QUERY.set(Boolean.TRUE); + try { + Object reply = executeCommand(ps); + if (!(reply instanceof PayloadStruct replyStruct) || replyStruct.payload == null + || replyStruct.payload.length == 0) { + return null; + } + Object first = replyStruct.payload[0]; + return first instanceof Map ? (Map) first : null; + } finally { + IN_CHANGE_QUERY.remove(); + } + } + @Override public Object executeCommand(PayloadStruct ps) { String threadName = Thread.currentThread().getName(); @@ -659,9 +699,10 @@ public Object executeCommand(PayloadStruct ps) { // Execution boundary: a file the block wrote inside the sandbox becomes // visible to the platform here -- asset browser, git, MCP generation -- // instead of waiting for logout, where it used to be lost entirely. - if (reply != null && this.cpw != null) { + if (reply != null && this.cpw != null && !Boolean.TRUE.equals(IN_CHANGE_QUERY.get()) + && (ps.operation == PayloadStruct.OPERATION.PYTHON || ps.operation == PayloadStruct.OPERATION.CMD)) { try { - this.cpw.fetchBackTransportedRoots(); + this.cpw.fetchBackChangedPaths(); } catch (Exception fetchError) { classLogger.warn("Fetch-back after execution failed", fetchError); } diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index ff85040d07..f2a27b9da9 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -583,6 +583,80 @@ public int fetchBack(String absPath) { } } + /** + * Fetch back only the named paths under a root that is already transported. + * + *

+ * This is the narrow form of {@link #fetchBack(String)}: the sandbox worker + * reports which files a block actually wrote, so the host tars a handful of + * paths instead of the whole tree. Cost becomes proportional to the change + * rather than to the size of the project. + * + * @param absRoot absolute path of the transported root inside the sandbox + * @param relativePaths paths relative to that root, as reported by the watcher + * @return number of files written back + */ + public int fetchBackPaths(String absRoot, java.util.List relativePaths) { + if (absRoot == null || relativePaths == null || relativePaths.isEmpty()) { + return 0; + } + String baseFolder = Utility.getBaseFolder(); + String normalized = Utility.normalizePath(absRoot); + if (baseFolder == null || !normalized.startsWith(Utility.normalizePath(baseFolder))) { + classLogger.warn("Refusing path fetch-back outside the SEMOSS home: {}", absRoot); + return 0; + } + java.util.List safe = new java.util.ArrayList<>(); + for (String relative : relativePaths) { + if (relative == null || relative.isEmpty()) { + continue; + } + String cleaned = relative.replace('\\', '/'); + if (cleaned.startsWith("/") || cleaned.contains("../") || cleaned.equals("..")) { + classLogger.warn("Refusing traversal in change set: {}", relative); + continue; + } + safe.add(cleaned); + if (safe.size() >= 2000) { + break; + } + } + if (safe.isEmpty()) { + return 0; + } + + java.nio.file.Path temp = null; + try { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + java.util.List command = new java.util.ArrayList<>(Arrays.asList(cli, "sandbox", "exec", "-n", + sandboxName, "--", "tar", "-C", normalized, "--ignore-failed-read", "--exclude=node_modules", + "--exclude=__pycache__", "--exclude=.venv", "-cf", "-")); + command.addAll(safe); + byte[] archive = runBytes(command, 120); + if (archive == null || archive.length == 0) { + return 0; + } + if (archive.length > MAX_FETCH_BYTES) { + classLogger.warn("Path fetch-back from {} is {} bytes, over the cap; skipped", absRoot, archive.length); + return 0; + } + if (!archiveEntriesAreSafe(archive)) { + classLogger.warn("Path fetch-back from {} contained an absolute or traversing path; skipped", absRoot); + return 0; + } + temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-paths-"); + extractArchive(archive, temp); + return reconcile(temp, java.nio.file.Paths.get(normalized)); + } catch (Exception e) { + classLogger.warn("Path fetch-back failed for {}", absRoot, e); + return 0; + } finally { + if (temp != null) { + deleteQuietly(temp); + } + } + } + /** * Write files from {@code incoming} onto {@code target} when the sandbox copy is * newer and differs. Returns how many were written. diff --git a/src/prerna/util/openshell/OpenShellShellRunner.java b/src/prerna/util/openshell/OpenShellShellRunner.java index d061b26b74..f7242760ca 100644 --- a/src/prerna/util/openshell/OpenShellShellRunner.java +++ b/src/prerna/util/openshell/OpenShellShellRunner.java @@ -134,7 +134,7 @@ public static Result execute(User user, String workingDir, String command) { // or a `cat > file` takes, and losing that work is exactly what we are fixing. try { if (user.getPythonClientProcessWrapper() != null) { - user.getPythonClientProcessWrapper().fetchBackTransportedRoots(); + user.getPythonClientProcessWrapper().fetchBackChangedPaths(); } } catch (Exception ignored) { // the command itself already ran; a failed reconcile must not fail the shell From 784c7b4c9f532fc66cd0783df180f33c9613f2dc Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 04:07:40 +0000 Subject: [PATCH 19/56] fix(openshell): make write-back content-authoritative and never lose a host edit Four correctness defects found by external review and a seven-model council, all in the change-set fetch-back path: 1. reconcile() trusted mtime. A sandbox file whose content differed but whose mtime was older than the host copy was silently dropped (os.utime, cp -p, tar extraction). Content is now the authority; mtime only decides whether the host copy must be preserved first. 2. Reported deletes were applied unconditionally, destroying a concurrent host edit. Deletes now quarantine the host copy first, and are suppressed entirely if that copy cannot be made. 3. A full-root fallback carried survivors but no tombstones, so sandbox deletions were silently resurrected. Full captures now quarantine host files absent from the capture; directories the capture excludes (node_modules, __pycache__, .venv) are skipped, since "not in the archive" there means "never captured", not "deleted". 4. inotify cannot see a persistent mmap write. The watcher now also compares one stat tuple (mtime_ns, size, ino, ctime_ns) per file at each checkpoint, reported as "stat_scan". Measured: ~6 ms for 240 files, silent on a quiet block, catches mmap writes and deletions with no events at all. Verified live in a sandbox: backdated rewrite, mmap write, and delete all reached the host; the overwritten host version and the deleted file were both preserved under /opt/semosshome/writeback-quarantine//. --- py/gaas_change_watcher.py | 43 ++++++++ src/prerna/om/ClientProcessWrapper.java | 10 +- src/prerna/util/OpenShellTransport.java | 127 ++++++++++++++++++++++-- 3 files changed, 167 insertions(+), 13 deletions(-) diff --git a/py/gaas_change_watcher.py b/py/gaas_change_watcher.py index c0bd472d90..3f14a9412e 100644 --- a/py/gaas_change_watcher.py +++ b/py/gaas_change_watcher.py @@ -28,9 +28,13 @@ import errno import os import select +import stat import struct import threading +# Directories whose contents never need write-back; keep the tuple scan cheap. +_SKIP_DIRS = {"node_modules", "__pycache__", ".venv"} + IN_MODIFY = 0x00000002 IN_CLOSE_WRITE = 0x00000008 IN_MOVED_FROM = 0x00000040 @@ -65,6 +69,8 @@ def __init__(self, roots, max_changes=2000): # inotify events carry a watch descriptor, not a path; this maps it back. self._watched = {} self._root_ino = {} + # path -> (mtime_ns, size, ino, ctime_ns) as of the last checkpoint + self._stat_index = {} # -- lifecycle --------------------------------------------------------- @@ -200,6 +206,41 @@ def _record(self, path, op): # -- snapshot ---------------------------------------------------------- + def _scan_tuples(self): + """Report files whose metadata moved since the last checkpoint. + + inotify cannot see a persistent mmap write, and a missed event is + indistinguishable from silence. One stat tuple per file costs ~6 ms for a few + hundred files (~0.1-0.3 s at 100k) and turns those writes into reported + changes. It is the correctness floor under the event stream, not a + replacement for it: the host still decides by content. + """ + index = {} + reported = 0 + for root in list(self.roots): + for base, dirs, files in os.walk(root): + dirs[:] = [name for name in dirs if name not in _SKIP_DIRS] + for name in files: + path = os.path.join(base, name) + try: + info = os.lstat(path) + except OSError: + continue + if not stat.S_ISREG(info.st_mode): + continue + key = (info.st_mtime_ns, info.st_size, info.st_ino, info.st_ctime_ns) + index[path] = key + # The first scan only seeds the index: the tree predates the watch. + if self._stat_index and self._stat_index.get(path) != key: + self._record(path, "modify") + reported += 1 + for path in self._stat_index: + if path not in index: + self._record(path, "delete") + reported += 1 + self._stat_index = index + return reported + def snapshot(self): """Return and clear the accumulated changes. @@ -208,6 +249,7 @@ def snapshot(self): host materialised it, so reporting it would be noise. """ with self._lock: + scanned = self._scan_tuples() roots = [] for root in self.roots: try: @@ -224,6 +266,7 @@ def snapshot(self): "baseline": self._baseline_pending, "watched": self._armed, "events": self._events_seen, + "stat_scan": scanned, "roots": roots, } self._changes = {} diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 79fe39fd05..6dc1971b1c 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -442,7 +442,7 @@ public int fetchBackChangedPaths() { roots.sort((first, second) -> second.length() - first.length()); Map> writes = new HashMap<>(); - List deletes = new ArrayList<>(); + Map deletes = new HashMap<>(); for (Object entry : changes) { if (!(entry instanceof Map change)) { continue; @@ -464,7 +464,7 @@ public int fetchBackChangedPaths() { } String relative = path.substring(root.length() + 1); if ("delete".equals(String.valueOf(change.get("op")))) { - deletes.add(path); + deletes.put(path, root); } else { List group = writes.get(root); if (group == null) { @@ -483,11 +483,11 @@ public int fetchBackChangedPaths() { classLogger.warn("Path fetch-back failed for {}", group.getKey(), e); } } - for (String path : deletes) { + for (Map.Entry entry : deletes.entrySet()) { try { - Files.deleteIfExists(Paths.get(path)); + prerna.util.OpenShellTransport.quarantineAndDelete(entry.getKey(), entry.getValue()); } catch (Exception e) { - classLogger.warn("Delete after change set failed for {}", path, e); + classLogger.warn("Delete after change set failed for {}", entry.getKey(), e); } } if (total > 0 || !deletes.isEmpty()) { diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index f2a27b9da9..00487dd138 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -572,7 +572,7 @@ public int fetchBack(String absPath) { } temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-"); extractArchive(archive, temp); - return reconcile(temp.resolve(base), java.nio.file.Paths.get(normalized)); + return reconcile(temp.resolve(base), java.nio.file.Paths.get(normalized), true); } catch (Exception e) { classLogger.warn("Fetch-back failed for {}", absPath, e); return 0; @@ -646,7 +646,7 @@ public int fetchBackPaths(String absRoot, java.util.List relativePaths) } temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-paths-"); extractArchive(archive, temp); - return reconcile(temp, java.nio.file.Paths.get(normalized)); + return reconcile(temp, java.nio.file.Paths.get(normalized), false); } catch (Exception e) { classLogger.warn("Path fetch-back failed for {}", absRoot, e); return 0; @@ -658,13 +658,18 @@ public int fetchBackPaths(String absRoot, java.util.List relativePaths) } /** - * Write files from {@code incoming} onto {@code target} when the sandbox copy is - * newer and differs. Returns how many were written. + * Write files from {@code incoming} onto {@code target}. Content is the authority: + * a differing sandbox file is always written, and mtime only decides whether the + * host copy must be preserved first. When {@code fullTree} is set the target is + * also scanned for files the sandbox no longer has, so a full fallback can + * propagate deletions instead of silently resurrecting them. */ - private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target) throws java.io.IOException { + private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, boolean fullTree) + throws java.io.IOException { if (!java.nio.file.Files.exists(incoming)) { return 0; } + java.nio.file.Path store = quarantineStore(); int[] copied = { 0 }; try (java.util.stream.Stream walk = java.nio.file.Files.walk(incoming)) { walk.filter(java.nio.file.Files::isRegularFile).filter(path -> !java.nio.file.Files.isSymbolicLink(path)) @@ -673,11 +678,15 @@ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target) th java.nio.file.Path destination = target.resolve(relative.toString()); try { if (java.nio.file.Files.exists(destination)) { - if (sameContent(path, destination) - || java.nio.file.Files.getLastModifiedTime(path).compareTo( - java.nio.file.Files.getLastModifiedTime(destination)) <= 0) { + if (sameContent(path, destination)) { return; } + // Different bytes. A host copy newer than the sandbox copy may be a + // concurrent edit, so keep it rather than silently overwriting it. + if (java.nio.file.Files.getLastModifiedTime(destination) + .compareTo(java.nio.file.Files.getLastModifiedTime(path)) > 0) { + quarantineCopy(destination, target, store); + } } java.nio.file.Path folder = destination.getParent(); if (folder != null) { @@ -692,12 +701,114 @@ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target) th } }); } + if (fullTree) { + copied[0] += quarantineStale(target, incoming, store); + } if (copied[0] > 0) { classLogger.info("Fetch-back wrote {} file(s) into {}", copied[0], target); } return copied[0]; } + /** + * A full fetch carries survivors, not tombstones. Any host file under the root that + * the sandbox capture no longer contains was deleted inside the sandbox; move it to + * quarantine rather than leaving it to reappear in the project. + */ + private static int quarantineStale(java.nio.file.Path target, java.nio.file.Path incoming, + java.nio.file.Path store) { + int[] moved = { 0 }; + try (java.util.stream.Stream walk = java.nio.file.Files.walk(target)) { + walk.filter(java.nio.file.Files::isRegularFile).filter(path -> !java.nio.file.Files.isSymbolicLink(path)) + .filter(path -> !isExcludedFromCapture(target.relativize(path))) + .forEach(path -> { + java.nio.file.Path relative = target.relativize(path); + if (java.nio.file.Files.exists(incoming.resolve(relative.toString()))) { + return; + } + java.nio.file.Path copy = quarantineCopy(path, target, store); + if (copy != null) { + try { + java.nio.file.Files.deleteIfExists(path); + moved[0]++; + } catch (java.io.IOException e) { + classLogger.warn("Could not remove {} after quarantining it", path, e); + } + } + }); + } catch (java.io.IOException e) { + classLogger.warn("Could not scan {} for deletions", target, e); + } + if (moved[0] > 0) { + classLogger.info("Fetch-back moved {} deleted file(s) to {}", moved[0], store); + } + return moved[0]; + } + + /** + * Directories the root capture never contains: {@code tar} excludes them, so a file + * missing from the archive under one of these names was never captured rather than + * deleted, and must not be treated as a tombstone. + */ + private static boolean isExcludedFromCapture(java.nio.file.Path relative) { + for (java.nio.file.Path part : relative) { + String name = part.toString(); + if ("node_modules".equals(name) || "__pycache__".equals(name) || ".venv".equals(name)) { + return true; + } + } + return false; + } + + /** Timestamped quarantine root, outside every transported root so it is never pushed. */ + private static java.nio.file.Path quarantineStore() { + String base = Utility.getBaseFolder(); + java.nio.file.Path store = java.nio.file.Paths + .get(base == null ? System.getProperty("java.io.tmpdir") : base, "writeback-quarantine"); + String stamp = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss").format(new java.util.Date()); + return store.resolve(stamp); + } + + /** Copy a host file aside before it is overwritten or removed. Returns the copy, or null. */ + private static java.nio.file.Path quarantineCopy(java.nio.file.Path file, java.nio.file.Path root, + java.nio.file.Path store) { + try { + java.nio.file.Path relative = root.relativize(file); + java.nio.file.Path destination = store.resolve(root.getFileName().toString()) + .resolve(relative.toString()); + java.nio.file.Files.createDirectories(destination.getParent()); + java.nio.file.Files.copy(file, destination, java.nio.file.StandardCopyOption.REPLACE_EXISTING, + java.nio.file.StandardCopyOption.COPY_ATTRIBUTES); + classLogger.warn("Write-back preserved the host copy of {} at {}", file, destination); + return destination; + } catch (Exception e) { + classLogger.warn("Could not preserve the host copy of {}", file, e); + return null; + } + } + + /** + * Remove a host file the sandbox deleted, keeping a copy under the quarantine root. + * + * @param path absolute host path reported as deleted + * @param root transported root the path belongs to + */ + public static void quarantineAndDelete(String path, String root) { + java.nio.file.Path file = java.nio.file.Paths.get(path); + if (!java.nio.file.Files.exists(file)) { + return; + } + if (quarantineCopy(file, java.nio.file.Paths.get(Utility.normalizePath(root)), quarantineStore()) == null) { + classLogger.warn("Delete suppressed for {} because it could not be quarantined", path); + return; + } + try { + java.nio.file.Files.deleteIfExists(file); + } catch (java.io.IOException e) { + classLogger.warn("Could not delete {} after quarantining it", path, e); + } + } + private static boolean sameContent(java.nio.file.Path left, java.nio.file.Path right) throws java.io.IOException { if (java.nio.file.Files.size(left) != java.nio.file.Files.size(right)) { return false; From c5442ce3f24204c269471fe9812a83c3e098d348 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 04:16:03 +0000 Subject: [PATCH 20/56] fix(openshell): invalidate watches on directory rename, prune emptied folders Watch descriptors follow inodes, so renaming a watched directory left every descendant path stale: events arrived under the old name and were written back to the wrong place. The watcher now treats IN_MOVED_FROM, IN_MOVE_SELF, IN_IGNORED and IN_UNMOUNT as lost coverage, reports `coverage: false` with `baseline: true`, drops the tuple index and re-arms from the roots. The host falls back to a full capture for that one block and resumes the change-set path afterwards. Java also treats `coverage: false` as a fallback trigger on its own, so a future worker that reports it without `baseline` cannot leave the host trusting a partial list. Deleting a file now also prunes directories it left empty, up to but never including the transported root - an empty folder left behind after a rename reads as corruption. Verified live: rename block took the full path ("wrote 2 file(s)", "moved 1 deleted file(s)"), the next block returned to the change set ("1 changed path"), and the post-rename write reached the host. --- py/gaas_change_watcher.py | 45 +++++++++++++++++++++++-- src/prerna/om/ClientProcessWrapper.java | 3 +- src/prerna/util/OpenShellTransport.java | 25 ++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/py/gaas_change_watcher.py b/py/gaas_change_watcher.py index 3f14a9412e..561555ad4e 100644 --- a/py/gaas_change_watcher.py +++ b/py/gaas_change_watcher.py @@ -44,6 +44,8 @@ IN_DELETE_SELF = 0x00000400 IN_MOVE_SELF = 0x00000800 IN_Q_OVERFLOW = 0x00004000 +IN_UNMOUNT = 0x00002000 +IN_IGNORED = 0x00008000 IN_ISDIR = 0x40000000 _EVENT = struct.Struct("iIII") @@ -66,6 +68,9 @@ def __init__(self, roots, max_changes=2000): self._baseline_pending = True self._armed = 0 self._events_seen = 0 + # Set when the watch tree can no longer be trusted (directory rename, watch + # removed, unmount): the host must fall back to a full capture for that block. + self._coverage_lost = False # inotify events carry a watch descriptor, not a path; this maps it back. self._watched = {} self._root_ino = {} @@ -162,6 +167,11 @@ def _consume(self, data): if mask & IN_Q_OVERFLOW: self._overflow = True continue + if mask & (IN_IGNORED | IN_UNMOUNT | IN_MOVE_SELF): + # The watch itself is gone, or it moved with its inode: every path + # under it is stale. No event on this descriptor is trustworthy now. + self._coverage_lost = True + continue name = raw.split(b"\0", 1)[0].decode("utf-8", "replace") if not name: continue @@ -173,6 +183,10 @@ def _consume(self, data): continue path = os.path.join(directory, name) if mask & IN_ISDIR: + if mask & IN_MOVED_FROM: + # A watched directory left the tree; its descriptor still points + # at the inode and would report the old path forever. + self._coverage_lost = True if mask & (IN_CREATE | IN_MOVED_TO): # A new directory: arm it *and* scan what landed inside before # the watch existed. Without the scan, `mkdir x; write x/f` @@ -248,6 +262,7 @@ def snapshot(self): everything already in the tree arrived before the watch existed, and the host materialised it, so reporting it would be noise. """ + rearm = False with self._lock: scanned = self._scan_tuples() roots = [] @@ -260,10 +275,14 @@ def snapshot(self): }) except OSError as exc: roots.append({"path": root, "error": str(exc)}) + stale = self._coverage_lost payload = { "changes": [{"path": path, "op": op} for path, op in self._changes.items()], "overflow": self._overflow, - "baseline": self._baseline_pending, + # A lost watch cannot report its own gaps, so it is equivalent to a + # baseline: the host must take a full capture for this block. + "baseline": self._baseline_pending or stale, + "coverage": not stale, "watched": self._armed, "events": self._events_seen, "stat_scan": scanned, @@ -272,7 +291,29 @@ def snapshot(self): self._changes = {} self._overflow = False self._baseline_pending = False - return payload + if stale: + self._coverage_lost = False + # Paths may have moved underneath us; drop the tuple index so the next + # checkpoint reseeds instead of reporting a storm of false deletes. + self._stat_index = {} + rearm = True + if rearm: + self._rearm_all() + return payload + + def _rearm_all(self): + """Rebuild the watch tree after coverage was lost. + + Descriptors left over from the old tree stay in the kernel but are inert once + ``_watched`` no longer maps them, so re-arming from the roots restores correct + paths without having to tear the old watches down one by one. + """ + with self._lock: + self._watched = {} + self._armed = 0 + self._root_ino = {} + for root in list(self.roots): + self._arm_tree(root) _shared = None diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 6dc1971b1c..db91f71f2d 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -420,7 +420,8 @@ public int fetchBackChangedPaths() { classLogger.warn("Change query failed; falling back to a full reconcile", e); } } - if (snapshot == null || truthy(snapshot.get("overflow")) || truthy(snapshot.get("baseline"))) { + if (snapshot == null || truthy(snapshot.get("overflow")) || truthy(snapshot.get("baseline")) + || Boolean.FALSE.equals(snapshot.get("coverage"))) { return fetchBackTransportedRoots(); } Object raw = snapshot.get("changes"); diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 00487dd138..ec0687eab1 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -730,6 +730,7 @@ private static int quarantineStale(java.nio.file.Path target, java.nio.file.Path if (copy != null) { try { java.nio.file.Files.deleteIfExists(path); + pruneEmptyParents(path, target); moved[0]++; } catch (java.io.IOException e) { classLogger.warn("Could not remove {} after quarantining it", path, e); @@ -745,6 +746,30 @@ private static int quarantineStale(java.nio.file.Path target, java.nio.file.Path return moved[0]; } + /** + * Remove directories that our own deletions emptied, walking up to (but never + * including) the transported root. Only empty directories are removed, so a folder + * someone deliberately created stays where it is. + */ + private static void pruneEmptyParents(java.nio.file.Path file, java.nio.file.Path root) { + java.nio.file.Path parent = file.getParent(); + while (parent != null && parent.startsWith(root) && !parent.equals(root)) { + try (java.util.stream.Stream children = java.nio.file.Files.list(parent)) { + if (children.findAny().isPresent()) { + return; + } + } catch (java.io.IOException e) { + return; + } + try { + java.nio.file.Files.deleteIfExists(parent); + } catch (java.io.IOException e) { + return; + } + parent = parent.getParent(); + } + } + /** * Directories the root capture never contains: {@code tar} excludes them, so a file * missing from the archive under one of these names was never captured rather than From b6beeafa50e97f5f7d17557b08fc284f30eccc95 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 04:19:30 +0000 Subject: [PATCH 21/56] perf(openshell): bound the mmap backstop by wall-clock, not by block The stat-tuple scan ran on every CHANGES call, so an idle block on a 100k-file tree paid ~0.57 s for a scan that could not have found anything. Measured: 240 files 1-2 ms, 10k 54 ms, 50k 285 ms, 100k 570 ms - linear at ~5.7 us/file. The scan is now due on the first call, after a lost baseline or coverage, and otherwise at most once per interval. The interval is max(2 s, 20 x the last measured duration), so the backstop holds roughly a 5% duty cycle at any size: 2 s at 240 files, ~10 s at 100k. Skipping never drops a change - the index is untouched, so the next scan reports everything accumulated; normal writes are unaffected entirely because they arrive as events. Measured at 100k: idle block 0.01 ms (was ~0.49 s), scan 0.486 s when due, interval self-tuned to 9.7 s. Payload reports stat_scan and scan_skipped. --- py/gaas_change_watcher.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/py/gaas_change_watcher.py b/py/gaas_change_watcher.py index 561555ad4e..dab7ae8b60 100644 --- a/py/gaas_change_watcher.py +++ b/py/gaas_change_watcher.py @@ -31,10 +31,22 @@ import stat import struct import threading +import time # Directories whose contents never need write-back; keep the tuple scan cheap. _SKIP_DIRS = {"node_modules", "__pycache__", ".venv"} +# The tuple scan exists to catch changes inotify cannot see (persistent mmap writes). +# Running it on every block makes an idle session O(files) per block - measured 0.57 s +# at 100k files. Bounding it by wall-clock instead keeps staleness under this many +# seconds while an idle burst of blocks pays for at most one scan. +_SCAN_MIN_INTERVAL_S = 2.0 + +# A scan may occupy at most 1/N of wall-clock: the interval stretches to N x the +# measured duration. At 240 files a scan is ~1 ms (interval stays 2 s); at 100k it is +# ~0.57 s, so the interval becomes ~11 s and the backstop costs ~5% instead of 100%. +_SCAN_BUDGET_FACTOR = 20 + IN_MODIFY = 0x00000002 IN_CLOSE_WRITE = 0x00000008 IN_MOVED_FROM = 0x00000040 @@ -76,6 +88,11 @@ def __init__(self, roots, max_changes=2000): self._root_ino = {} # path -> (mtime_ns, size, ino, ctime_ns) as of the last checkpoint self._stat_index = {} + self._last_scan = 0.0 + self._scans_skipped = 0 + # Grows with the measured cost of a scan so the backstop never takes more than + # a few percent of wall-clock, at any tree size. + self._scan_interval = _SCAN_MIN_INTERVAL_S # -- lifecycle --------------------------------------------------------- @@ -264,7 +281,24 @@ def snapshot(self): """ rearm = False with self._lock: - scanned = self._scan_tuples() + now = time.monotonic() + # Always scan on the first call (seed), after a lost baseline/coverage, and + # otherwise at most once per interval. Skipping never drops a change: the + # index is untouched, so the next scan reports everything accumulated. + due = (self._last_scan == 0.0 or self._baseline_pending or self._coverage_lost + or (now - self._last_scan) >= self._scan_interval) + if due: + started = time.monotonic() + scanned = self._scan_tuples() + elapsed = time.monotonic() - started + self._scan_interval = max(_SCAN_MIN_INTERVAL_S, elapsed * _SCAN_BUDGET_FACTOR) + self._last_scan = now + skipped = 0 + self._scans_skipped = 0 + else: + scanned = 0 + self._scans_skipped += 1 + skipped = self._scans_skipped roots = [] for root in self.roots: try: @@ -286,6 +320,7 @@ def snapshot(self): "watched": self._armed, "events": self._events_seen, "stat_scan": scanned, + "scan_skipped": skipped, "roots": roots, } self._changes = {} From 4f1c4a26e4e89b5eb97e3f5852211bbecee5b74b Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 04:21:26 +0000 Subject: [PATCH 22/56] perf(openshell): stop watching and scanning .git and node_modules Measured on a real project: .git was 112 directories and 137 of 256 files - over half the watch set and over half of every scan - for data that cannot change, since the sandbox image has no git binary. node_modules was only excluded from the scan and the tar, so it was still watched: one npm install adds thousands of directories, exhausts fs.inotify.max_user_watches and floods the queue, turning a cheap block into a full capture. One ignore set now covers arming, adoption, scanning and pull-back: node_modules, __pycache__, .venv, .git. Directories created mid-block are adopted only if they are not in that set, so an install inside a block cannot smuggle the tree back in. Override with SEMOSS_SYNC_IGNORE. Synthetic tree shaped like the real one (20 assets, 112-dir .git, 200-dir node_modules): watches 318 -> 4, indexed files 332 -> 20, asset writes still detected, node_modules writes unreported by design. --- py/gaas_change_watcher.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/py/gaas_change_watcher.py b/py/gaas_change_watcher.py index dab7ae8b60..e726006acb 100644 --- a/py/gaas_change_watcher.py +++ b/py/gaas_change_watcher.py @@ -33,8 +33,14 @@ import threading import time -# Directories whose contents never need write-back; keep the tuple scan cheap. -_SKIP_DIRS = {"node_modules", "__pycache__", ".venv"} +# Directories that are never watched, scanned, or pulled back. `.git` is static inside +# the sandbox (the image has no git binary) yet it was 112 directories and 137 of 256 +# files in a real project - over half the watch set and over half of every scan, for +# data that cannot move. `node_modules` is worse: one install adds thousands of +# directories, exhausts fs.inotify.max_user_watches and floods the event queue. +_DEFAULT_IGNORE = ("node_modules", "__pycache__", ".venv", ".git") +_env_ignore = os.environ.get("SEMOSS_SYNC_IGNORE", "").strip() +_IGNORED_DIRS = {name.strip() for name in _env_ignore.split(",") if name.strip()} if _env_ignore else set(_DEFAULT_IGNORE) # The tuple scan exists to catch changes inotify cannot see (persistent mmap writes). # Running it on every block makes an idle session O(files) per block - measured 0.57 s @@ -150,7 +156,9 @@ def _arm_tree(self, root): self._root_ino[root] = os.stat(root).st_ino except OSError: pass - for current, _dirs, _files in os.walk(root): + for current, dirs, _files in os.walk(root): + # Prune in place so os.walk never descends into an ignored tree at all. + dirs[:] = [name for name in dirs if name not in _IGNORED_DIRS] self._arm(current) # -- event loop -------------------------------------------------------- @@ -218,6 +226,10 @@ def _consume(self, data): self._adopt_directory(path) def _adopt_directory(self, path): + # A directory created during a block is armed and scanned, unless it is one we + # ignore - an `npm install` must not pull thousands of watches into the tree. + if os.path.basename(path) in _IGNORED_DIRS: + return self._arm_tree(path) with self._lock: for current, _dirs, files in os.walk(path): @@ -250,7 +262,7 @@ def _scan_tuples(self): reported = 0 for root in list(self.roots): for base, dirs, files in os.walk(root): - dirs[:] = [name for name in dirs if name not in _SKIP_DIRS] + dirs[:] = [name for name in dirs if name not in _IGNORED_DIRS] for name in files: path = os.path.join(base, name) try: From 4b6b6b39433a5898eb2ef848f722a4842432c8dd Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 04:34:04 +0000 Subject: [PATCH 23/56] fix(openshell): refuse write-back for users who cannot edit the project The sandbox runs outside the platform's permission checks, so any member who could open a project could write a file with Py() and have fetch-back publish it into the project folder - a read-only member silently gaining write access. ClientProcessWrapper now binds the owning User (set at client creation in User.java) and checks it before any root is pulled back: the project id is taken from the root folder (`__`, or a bare uuid), and SecurityProjectUtils.getActualUserProjectPermission must answer OWNER or EDIT. Anything else is skipped with "Refusing write-back into ... has READ_ONLY access to project ...", and an unresolvable permission fails closed. The check runs in both paths - the change-set pull and the full fallback - so a baseline reconcile cannot bypass it either. Verified live: with kunal0137 downgraded to READ_ONLY on workbench_1, a Py block wrote 29 bytes inside the sandbox, the host file did not appear and the refusal was logged; restoring OWNER made the same write land on the host. Rooms and user assets are NOT gated: this build has no room permission model, and asset roots are the user's own. If rooms are meant to inherit project access, that rule needs stating before it can be enforced. --- src/prerna/auth/User.java | 2 + src/prerna/om/ClientProcessWrapper.java | 58 ++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/prerna/auth/User.java b/src/prerna/auth/User.java index 7dae918cda..9a7849647a 100644 --- a/src/prerna/auth/User.java +++ b/src/prerna/auth/User.java @@ -615,6 +615,7 @@ public void startPythonSocketServerAndClient(int port, String venvEngineId) { // we do not define the Server Directory here - because it will dynamically // generate in the chroot location try { + this.pythonCPW.setUser(this); this.pythonCPW.createProcessAndClient(nativePyServer, this.symlinkHelper, port, null, null, customClassPath, debug, "-1", loggerLevel, ThreadContext.getImmutableContext()); } catch (Exception e) { @@ -635,6 +636,7 @@ public void startPythonSocketServerAndClient(int port, String venvEngineId) { try { String venvPath = venvEngineId != null ? Utility.getVenvEngine(venvEngineId).pathToExecutable() : null; + this.pythonCPW.setUser(this); this.pythonCPW.createProcessAndClient(nativePyServer, null, port, venvPath, serverDirectoryPath.toString(), customClassPath, debug, "-1", loggerLevel, ThreadContext.getImmutableContext()); diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index db91f71f2d..18bdf9e8fb 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -127,6 +127,8 @@ public class ClientProcessWrapper { // the same JSON protocol, so it reuses NativePySocketClient. private boolean nodeServer = false; private SymlinkHelper chrootSymlinkHelper; + /** Whose permissions gate write-back; bound by {@code User} when the client is created. */ + private prerna.auth.User user; // OPENSHELL mode: owns the sandbox + port-forward for this worker's lifetime private prerna.util.OpenShellTransport openshell; private String classPath; @@ -437,6 +439,9 @@ public int fetchBackChangedPaths() { if (normalized.contains("/" + Constants.PROJECT_FOLDER + "/") || normalized.contains("/" + Constants.ROOM_FOLDER + "/") || normalized.contains("/" + Constants.USER_FOLDER + "/")) { + if (!canWriteBack(normalized)) { + continue; + } roots.add(normalized); } } @@ -504,6 +509,57 @@ private static boolean truthy(Object value) { return value != null && Boolean.parseBoolean(value.toString()); } + /** + * Bind the user whose permissions decide whether a root may be written back. + * + * @param user owner of this process + */ + public void setUser(prerna.auth.User user) { + this.user = user; + } + + /** + * A read-only member must not be able to modify a project through the sandbox: the + * worker runs outside the platform's permission checks, so without this gate a + * read-only user could write a file with {@code Py()} and have it published to the + * project folder. Project roots are gated on the platform's own permission lookup. + * Room and asset roots have no comparable model in this build and are left alone. + * + * @param normalized absolute transported root + * @return true when write-back into this root is allowed + */ + private boolean canWriteBack(String normalized) { + String marker = "/" + Constants.PROJECT_FOLDER + "/"; + int at = normalized.indexOf(marker); + if (at < 0) { + return true; + } + String rest = normalized.substring(at + marker.length()); + int slash = rest.indexOf('/'); + String folder = slash < 0 ? rest : rest.substring(0, slash); + int separator = folder.lastIndexOf("__"); + String projectId = separator < 0 ? folder : folder.substring(separator + 2); + if (this.user == null) { + classLogger.warn("Refusing write-back into {}: no user is bound to this process", normalized); + return false; + } + try { + String permission = prerna.auth.utils.SecurityProjectUtils + .getActualUserProjectPermission(this.user, projectId); + boolean allowed = prerna.auth.AccessPermissionEnum.OWNER.getPermission().equals(permission) + || prerna.auth.AccessPermissionEnum.EDIT.getPermission().equals(permission); + if (!allowed) { + classLogger.warn("Refusing write-back into {}: user {} has {} access to project {}", normalized, + prerna.auth.User.getSingleLogginName(this.user), permission, projectId); + } + return allowed; + } catch (Exception e) { + // Fail closed: an unresolvable permission is not a licence to write. + classLogger.warn("Could not resolve permission for {}; refusing write-back", normalized, e); + return false; + } + } + /** * Reconcile the sandbox copies of this session's transported roots back onto the * host, so a file written by {@code Py()} or a shell command is visible to the @@ -528,7 +584,7 @@ public int fetchBackTransportedRoots() { boolean syncable = normalized.contains("/" + Constants.PROJECT_FOLDER + "/") || normalized.contains("/" + Constants.ROOM_FOLDER + "/") || normalized.contains("/" + Constants.USER_FOLDER + "/"); - if (!syncable) { + if (!syncable || !canWriteBack(normalized)) { continue; } try { From 73b2d6fb7edbcbf1be531e5bbab16c17974595df Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 10:44:20 +0000 Subject: [PATCH 24/56] fix(openshell): gate user-asset write-back on ownership, close the fall-through Write-back now decides every root by its own space instead of special-casing rooms and letting everything else through: - user assets: the folder is Asset__ where the id must equal one of the caller's own asset project ids (User.getAssetProjectId(provider), backed by ASSETENGINE keyed on userid) - anything else under user/ is somebody else's and is refused; - rooms: ROOM.USER_ID in the model-inference-logs database via ModelInferenceLogsUtils.doCheckRoomExistsForUser; - projects: OWNER/EDIT as before; - any unrecognised space: refused, where the old code returned true. Every refusal is logged with the path and the reason, and each check fails closed on an exception. Verified: with the deployed class carrying isOwnUserAsset, a Py block writing 29 bytes into an owned project root still reached the host ("Fetch-back wrote 1 file(s)"). The asset/room denial paths are identity-keyed - this instance has one human user per asset project and only kunal0137 owns rooms, so a foreign root cannot be transported into a session to exercise the refusal. --- src/prerna/om/ClientProcessWrapper.java | 85 ++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 18bdf9e8fb..368190c7c6 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -518,21 +518,100 @@ public void setUser(prerna.auth.User user) { this.user = user; } + /** + * User assets are per-user by construction: the folder is {@code Asset__}, where the + * id is the user's own asset project id for one of their logins. Anything else under + * {@code user/} belongs to somebody else and must never be written back. + * + * @param normalized absolute user-asset root + * @return true when the asset belongs to this user + */ + private boolean isOwnUserAsset(String normalized) { + String marker = "/" + Constants.USER_FOLDER + "/"; + String rest = normalized.substring(normalized.indexOf(marker) + marker.length()); + int slash = rest.indexOf('/'); + String folder = slash < 0 ? rest : rest.substring(0, slash); + int separator = folder.lastIndexOf("__"); + String assetId = separator < 0 ? folder : folder.substring(separator + 2); + if (this.user == null) { + classLogger.warn("Refusing write-back into {}: no user is bound to this process", normalized); + return false; + } + try { + for (prerna.auth.AuthProvider provider : this.user.getLogins()) { + if (assetId.equals(this.user.getAssetProjectId(provider))) { + return true; + } + } + } catch (Exception e) { + classLogger.warn("Could not resolve asset ownership for {}; refusing write-back", normalized, e); + return false; + } + classLogger.warn("Refusing write-back into {}: asset {} does not belong to {}", normalized, assetId, + prerna.auth.User.getSingleLogginName(this.user)); + return false; + } + + /** + * A room is writable only by the member who owns it. Rooms are rows in the model + * inference logs database ({@code ROOM.USER_ID}), not insights in this build, and + * sharing grants participation rather than edit rights. + * + * @param normalized absolute room root + * @return true when the session's user owns the room + */ + private boolean canWriteRoom(String normalized) { + String marker = "/" + Constants.ROOM_FOLDER + "/"; + int at = normalized.indexOf(marker); + String rest = normalized.substring(at + marker.length()); + int slash = rest.indexOf('/'); + String roomId = slash < 0 ? rest : rest.substring(0, slash); + if (this.user == null) { + classLogger.warn("Refusing write-back into room {}: no user is bound to this process", roomId); + return false; + } + try { + // A login can be any of the providers the user authenticated with; the room was + // created under one of them. + for (prerna.auth.AuthProvider provider : this.user.getLogins()) { + prerna.auth.AccessToken token = this.user.getAccessToken(provider); + if (token != null && prerna.engine.impl.model.inferencetracking.ModelInferenceLogsUtils + .doCheckRoomExistsForUser(roomId, token.getId())) { + return true; + } + } + classLogger.warn("Refusing write-back into room {}: user {} does not own it", roomId, + prerna.auth.User.getSingleLogginName(this.user)); + return false; + } catch (Exception e) { + classLogger.warn("Could not resolve ownership of room {}; refusing write-back", roomId, e); + return false; + } + } + /** * A read-only member must not be able to modify a project through the sandbox: the * worker runs outside the platform's permission checks, so without this gate a * read-only user could write a file with {@code Py()} and have it published to the - * project folder. Project roots are gated on the platform's own permission lookup. - * Room and asset roots have no comparable model in this build and are left alone. + * project folder. Every root is decided by its own space: projects on the platform + * permission lookup, rooms on {@code ROOM.USER_ID} in the model-inference-logs + * database, user assets on the caller's own asset project id. * * @param normalized absolute transported root * @return true when write-back into this root is allowed */ private boolean canWriteBack(String normalized) { + if (normalized.contains("/" + Constants.ROOM_FOLDER + "/")) { + return canWriteRoom(normalized); + } + if (normalized.contains("/" + Constants.USER_FOLDER + "/")) { + return isOwnUserAsset(normalized); + } String marker = "/" + Constants.PROJECT_FOLDER + "/"; int at = normalized.indexOf(marker); if (at < 0) { - return true; + classLogger.warn("Refusing write-back into {}: unrecognised root space", normalized); + return false; } String rest = normalized.substring(at + marker.length()); int slash = rest.indexOf('/'); From e4004f4846fe8c760ba677528161c8a877449c92 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 10:51:39 +0000 Subject: [PATCH 25/56] build(executor): first-class Dockerfile for the sandbox image, built from py/ The executor image was assembled outside this repo: executor-probe/py was a hand copy of py/, and executor-proto just layered on top of it. That copy is how the image kept running month-old python while the repo looked fixed. docker/Dockerfile.executor replaces the two-image dance with one stage whose build context is py/, so the image carries exactly the committed worker. It keeps what matters from both predecessors: python:3.12-slim, git with a system identity (local repo work only - egress is still per-binary in the OpenShell policy), pandas + jsonpickle for the boot-time lazy imports, the insight/grants layout, uid 1000 ownership of /opt/semosshome, and the writable-workspace USER the sandbox requires. docker/build-executor-image.sh is the one command: it labels the tag with a hash of the python tree (org.semoss.py-sha) so `docker inspect` answers "which source is this", then smoke-tests the result inside the image. Built as semoss-executor:0.1 from py sha 1701eb4f4006: git 2.47.3, imports ok, worker present, watcher ignore set intact, id 1000. The worker boots as the main process and listens on 7100. RDF_Map.prop still points at semoss-executor-proto:0.2; switching is OPENSHELL_IMAGE + restart. --- docker/Dockerfile.executor | 45 ++++++++++++++++++++++++++++++++++ docker/build-executor-image.sh | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 docker/Dockerfile.executor create mode 100755 docker/build-executor-image.sh diff --git a/docker/Dockerfile.executor b/docker/Dockerfile.executor new file mode 100644 index 0000000000..acb31e0345 --- /dev/null +++ b/docker/Dockerfile.executor @@ -0,0 +1,45 @@ +# SEMOSS executor sandbox — the process that runs user code under OpenShell. +# +# docker build -f docker/Dockerfile.executor -t semoss-executor:0.3 py +# # or, with the py-source label and a smoke test: +# docker/build-executor-image.sh +# +# The build context is py/ on purpose. The image must carry the worker source that is +# in this repo; earlier builds staged py/ into a separate context by hand, and the +# image silently kept running month-old python while the repo looked fixed. +FROM python:3.12-slim + +LABEL maintainer="semoss@semoss.org" +LABEL org.semoss.component="executor-sandbox" + +# git enables local repository work inside the sandbox (status/diff/commit/branch). +# Installing it grants no network access: the OpenShell policy keys egress to a binary +# path, so every remote host needs its own rule before git can leave the sandbox. +# The identity is system-wide because commits fail without one. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && git config --system user.name "SEMOSS Sandbox" \ + && git config --system user.email "sandbox@semoss.local" \ + && git config --system safe.directory '*' + +# pandas + jsonpickle are the two modules the handler lazy-imports at boot. yfinance is +# only here for the Stock MCP example and is not part of SEMOSS's declared dependencies +# -- the real answer is a per-project venv or a manifest-driven install. +RUN pip install --no-cache-dir pandas jsonpickle yfinance + +# The worker source. Context root is py/, so this is the committed python, verbatim. +COPY . /opt/semosshome/py/ + +# The handler appends to {insight_folder}/log.txt during setup(). The grants folders are +# the read-only / read-write drop points the policy model expects to exist. +RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ + && touch /opt/semosshome/insight/log.txt \ + && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ + && chmod 755 /opt/grants-rw /opt/grants-ro + +# OpenShell refuses to start unless the workspace is writable by the sandbox identity: +# "workspace path component '' is not writable by the sandbox identity in the +# image: Permission denied" +USER 1000:1000 +WORKDIR /opt/semosshome/py diff --git a/docker/build-executor-image.sh b/docker/build-executor-image.sh new file mode 100755 index 0000000000..0a6dbd0d4d --- /dev/null +++ b/docker/build-executor-image.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Build the SEMOSS executor sandbox image from the worker source in this repo. +# +# docker/build-executor-image.sh # tag semoss-executor:dev +# TAG=semoss-executor:0.3 docker/build-executor-image.sh +# NO_CACHE=1 docker/build-executor-image.sh +# +# The build context is py/, so the image carries exactly the python committed here. +# That is the point: there is no staged copy that can go stale. The tag is labelled +# with a hash of the python tree, so `docker inspect` answers "which source is this?" +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/.." && pwd)" +TAG="${TAG:-semoss-executor:dev}" + +[ -d "$REPO/py" ] || { echo "no py/ under $REPO" >&2; exit 1; } + +PY_SHA="$(cd "$REPO/py" \ + && find . -type f -name '*.py' -print0 \ + | sort -z \ + | xargs -0 sha256sum \ + | sha256sum | cut -d' ' -f1)" + +ARGS=(build + -f "$HERE/Dockerfile.executor" + -t "$TAG" + --label "org.semoss.py-sha=$PY_SHA" + --label "org.semoss.built-at=$(date -u +%Y-%m-%dT%H:%M:%SZ)") +[ "${NO_CACHE:-0}" = "1" ] && ARGS+=(--no-cache) + +docker "${ARGS[@]}" "$REPO/py" + +echo "built $TAG from py sha ${PY_SHA:0:12}" + +# Prove what landed, rather than trusting the build log. +docker run --rm --entrypoint sh "$TAG" -c ' + git --version + python3 -c "import pandas, jsonpickle; print(\"imports ok\")" + test -f /opt/semosshome/py/gaas_tcp_socket_server.py && echo "worker present" + printf "watcher ignore refs: "; grep -c "_IGNORED_DIRS" /opt/semosshome/py/gaas_change_watcher.py + printf "id: "; id -u +' From ef1d289638f581236e3d0874c19db76f672bc849 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 10:59:27 +0000 Subject: [PATCH 26/56] docs(executor): document the OpenShell sandbox mode end to end docs/deployment/executor_sandbox.md covers the opt-in SANDBOX_MODE=OPENSHELL path: the process model and session lifecycle, the per-block write-back pipeline (CHANGES verb, targeted tar, content-authoritative reconcile, quarantine, tombstones), the two change detectors and why the stat-tuple scan exists at all, per-space write permissions, the repo-owned image build, configuration keys, a verification recipe and the known limits (2000-path cap, racing mmap writers, no host->sandbox refresh). Linked from the Deployment and operations section of docs/README.md. --- docs/README.md | 1 + docs/deployment/executor_sandbox.md | 160 ++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 docs/deployment/executor_sandbox.md diff --git a/docs/README.md b/docs/README.md index 44591b6939..6b5715243f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -82,6 +82,7 @@ The [Python GAAS tools index](python_gaas_tools/README.md) documents Python acce - [Local Docker setup](cloud_and_cluster/docker_deployment.md) - [Local Docker configuration](deployment/docker_configuration.md) +- [OpenShell executor sandbox](deployment/executor_sandbox.md): running user code in a per-session container, the write-back pipeline and change detection, per-space write permissions, and building the image from `py/` - [Supporting engine examples](../docker-compose-examples/engines/README.md) - [Cloud and cluster architecture](cloud_and_cluster/README.md) - [Central asset storage](cloud_and_cluster/central_cloud_storage.md) diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md new file mode 100644 index 0000000000..9d1ded4d23 --- /dev/null +++ b/docs/deployment/executor_sandbox.md @@ -0,0 +1,160 @@ +# OpenShell executor sandbox + +How user code runs when SEMOSS is configured with `SANDBOX_MODE=OPENSHELL`, what the sandbox +can and cannot change on the host, and how write-back decides what to copy. + +This is an opt-in execution path. With `SANDBOX_MODE` unset or set to any other value, the +platform uses the traditional local child process and none of this applies. + +## Why it exists + +`Py()` and shell commands are arbitrary code. Running them as a child of the JVM means a +process that shares the host's filesystem, environment and network namespace. The OpenShell +mode moves that process into a separate container, spawned and reaped with the user's +session, and keeps the platform's files authoritative on the host. + +## Process model + +| Piece | Runs where | Notes | +| --- | --- | --- | +| Java (Tomcat) | host | owns the socket client, decides when to sync | +| `gaas_tcp_socket_server.py` | sandbox, **main process** | the persistent REPL; one per user session | +| shell commands | sandbox | started by `OpenShellShellRunner`, same write-back rules | +| user assets / projects / rooms | host | copied in at session start, written back at block boundaries | + +The worker is stateful: variables survive between blocks, which is why it cannot be +restarted to pick up a change. It speaks the existing gaas protocol over a single TCP +connection; a second connection is accepted and then never served. + +## Session lifecycle + +- Created lazily on the first `Py()` or shell command (cold start ≈ 10–13 s). +- Deleted on logout, session timeout, and on disconnect, where `ClientProcessWrapper.shutdown` + reconciles one last time before the sandbox disappears. +- A sandbox left behind after a crash is removed by `OpenShellSandboxReaper`, which only + touches labelled `pyw-*` sandboxes past an age threshold. + +## Write-back pipeline + +Every `PYTHON` and `CMD` block ends with a sync. The steps, in order: + +1. **Ask for the change set.** `NativePySocketClient` sends a `CHANGES` request on the + existing socket. The worker answers + `{changes[], overflow, baseline, coverage, watched, events, stat_scan}`. +2. **Fail closed on doubt.** If the answer is missing, `overflow` is true, `baseline` is true, + or `coverage` is false, the host falls back to a full capture of each root instead of + trusting a partial list. +3. **Pull only what changed.** Changed paths are tarred inside the sandbox under one + `sandbox exec` and extracted to a temporary directory on the host. +4. **Reconcile.** Content is the authority: a differing sandbox file is written, and `mtime` + only decides whether the host copy is preserved first. A host file newer than its sandbox + counterpart is copied to quarantine before being replaced. +5. **Deletes.** A path reported as deleted is quarantine-then-delete, and the delete is + suppressed if the quarantine copy fails. Directories emptied by those removals are pruned + up to (never including) the transported root. + +Quarantine lives at `$BASE/writeback-quarantine///`, outside +`project/`, `room/` and `user/`, so it is never pushed to object storage. + +A whole-root fallback carries survivors, not tombstones, so it diffs the host tree against the +capture and quarantines files the capture no longer contains. Directories the capture excludes +(`node_modules`, `__pycache__`, `.venv`) are skipped: “absent from the archive” there means +“never captured”, not “deleted”. + +## Change detection + +Two mechanisms, deliberately: + +- **inotify** is the primary source. Watch descriptors are per directory, re-armed when a + directory is created, and scanned on adoption so `mkdir x; write x/f` inside one block does + not lose the file. +- **A stat-tuple scan** (`mtime_ns`, `size`, `inode`, `ctime`) is the correctness floor under + it, because a persistent `mmap` write emits no event at all. It runs at most once per + interval — `max(2 s, 20 × the last measured scan)` — so an idle block costs nothing at any + tree size. Skipping never drops a change: the index is untouched, so the next scan reports + the backlog. + +Coverage loss is a first-class outcome. `IN_MOVED_FROM`, `IN_MOVE_SELF`, `IN_IGNORED` and +`IN_UNMOUNT` all mean the path map may be stale, so the watcher reports `coverage: false`, +drops the tuple index and re-arms; the host takes one full capture for that block and returns +to the fast path afterwards. + +Excluded from watching, scanning and pulling: `node_modules`, `__pycache__`, `.venv`, `.git`. +`.git` is static inside the sandbox and dominated a real project (112 directories, 137 of 256 +files); `node_modules` is one install away from exhausting `fs.inotify.max_user_watches`. +Override with `SEMOSS_SYNC_IGNORE`. + +## Who may write back + +The worker runs outside the platform's permission checks, so the host decides per root: + +| Root | Rule | +| --- | --- | +| `project/__` | `SecurityProjectUtils.getActualUserProjectPermission` must be `OWNER` or `EDIT` | +| `room/` | `ModelInferenceLogsUtils.doCheckRoomExistsForUser` — `ROOM.USER_ID` in the model-inference-logs database | +| `user/Asset__` | the id must equal one of the caller's `User.getAssetProjectId(provider)` values | +| anything else | refused | + +Every refusal is logged with the path and the reason, and each check fails closed on error. A +read-only member can still write inside the sandbox — that is a scratch area — but the file +never reaches the host. + +## Image + +The sandbox image is built from this repository, not from a staged copy: + +```bash +docker/build-executor-image.sh # tag semoss-executor:dev +TAG=semoss-executor:0.1 docker/build-executor-image.sh +``` + +The build context is `py/`, so the image carries exactly the committed worker; +`docker/Dockerfile.executor` assembles it. The tag is labelled with `org.semoss.py-sha`, a hash +of the python tree, so the image can be traced back to its source. Contents: `python:3.12-slim`, +`git` with a system identity (local repository work only — egress is granted per binary by the +OpenShell policy), `pandas` and `jsonpickle` for the boot-time imports, the insight and grants +layout, and uid 1000 ownership of `/opt/semosshome`. + +At session start the launcher also uploads the host's `/opt/semosshome/py` over the image's +copy, but the worker's first imports happen before that upload, so the baked copy still has to +be current — a stale image shows up as code that ignores your latest change. + +## Configuration + +| Key | Meaning | +| --- | --- | +| `SANDBOX_MODE` | `OPENSHELL` enables this path | +| `OPENSHELL_IMAGE` | image a new sandbox starts from | +| `OPENSHELL_POLICY` | Landlock paths and per-binary egress rules | +| `OPENSHELL_LABEL` | label used to identify sandboxes for the reaper | +| `OPENSHELL_WORKER_PORT` | worker port inside the sandbox | +| `OPENSHELL_READY_TIMEOUT_SECONDS` | cold-start budget | + +## Verifying a deployment + +```bash +# the class actually deployed (never trust the build log) +cd /tmp && unzip -o -q /opt/tomcat/webapps/Monolith/WEB-INF/lib/semoss-*-shaded-dependencies.jar \ + 'prerna/om/ClientProcessWrapper.class' && javap -p prerna/om/ClientProcessWrapper.class | grep -E 'canWriteBack|quarantine' + +# a file written by a block reached the host +journalctl -u tomcat-semoss | grep -i "Fetch-back wrote" + +# the permission gate refuses a read-only member +sudo -u postgres psql -d semoss_security -c \ + "UPDATE projectpermission SET permission=3 WHERE projectid='' AND userid='';" +# write from Py(), confirm the host file never appears and "Refusing write-back" is logged, +# then restore permission=1 +``` + +## Known limits + +- More than ~2000 changed paths in one block sets `overflow` and forces a full capture. +- A *racing* `mmap` writer cannot be captured without cooperation (`msync`) or interception + this sandbox does not have. +- There is no host→sandbox refresh while a sandbox lives. A `git pull` on the host during a + session is not reflected inward, and the sandbox wins the next write-back for any file it + touched (recoverable from quarantine). Branch switching belongs on the host, followed by a + fresh session. +- Rooms and assets are gated as described above, but the platform has no room-sharing model + in this build; if one is added, the room branch must resolve the room to its project. From 1a596cb0647101c51684257852ca2c3c98861a6f Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 11:28:03 +0000 Subject: [PATCH 27/56] build(executor): keep the torch CPU index and CVE pins in the full image The first full build came out at 22.4 GB. Cause: flattening pyproject.toml to a requirements list dropped [tool.uv.sources], which is what maps torch/torchvision/ torchaudio to the pytorch-cpu index. PyPI then resolved CUDA builds - 3.3 GB of nvidia plus 815 MB of triton inside an image that never sees a GPU. export_requirements.py now skips the torch trio by default (--keep-torch to opt back in) so the Dockerfile can install it from the right index first, and it emits [tool.uv] override-dependencies as ordinary requirements so the sandbox keeps the platform's security pins (marshmallow, protobuf, numpy, pyasn1, lxml). Dockerfile.executor-full installs torch from download.pytorch.org/whl/cpu for the cpu extra and /cu126 for gpu, then the remaining declared dependencies from PyPI. --- docker/Dockerfile.executor-full | 79 +++++++++++++++++++++++++++++++ docker/export_requirements.py | 83 +++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 docker/Dockerfile.executor-full create mode 100644 docker/export_requirements.py diff --git a/docker/Dockerfile.executor-full b/docker/Dockerfile.executor-full new file mode 100644 index 0000000000..586599381f --- /dev/null +++ b/docker/Dockerfile.executor-full @@ -0,0 +1,79 @@ +# SEMOSS executor sandbox — full-parity build. +# +# docker build -f docker/Dockerfile.executor-full -t semoss-executor:full py +# # or: MODE=full docker/build-executor-image.sh +# +# Why this exists next to Dockerfile.executor: the minimal image carries three packages, so +# a Py() block that ran against the platform's own 2.4 GB venv (numpy, boto3, langchain, +# torch, ...) fails with ImportError the moment it is sandboxed. That is a silent capability +# regression, not a size trade-off. This build installs the dependency set that +# py/install_config/pyproject.toml declares, the same one docker/Dockerfile.python installs. +# +# The project itself is not installed (.[extra]): that pyproject has no build-system, so it +# is a dependency manifest, not a package. docker/export_requirements.py flattens it. +FROM python:3.12-slim AS deps + +ARG EXTRAS=cpu +ENV EXTRAS=${EXTRAS} + +# Compilers live in the builder stage only. Several declared dependencies ship no aarch64 +# wheel (annoy is a C++ extension and was the first to fail), so the venv has to be assembled +# from source here; the final stage still copies just the finished venv. +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential cmake \ + && rm -rf /var/lib/apt/lists/* + +# uv, the same installer SEMOSS uses for its own runtime. +COPY --from=ghcr.io/astral-sh/uv:0.9 /uv /uvx /usr/local/bin/ + +WORKDIR /build + +COPY pyproject.toml ./ +COPY export_requirements.py ./ + +RUN python3 export_requirements.py pyproject.toml "${EXTRAS}" > /tmp/requirements.txt \ + && echo "installing $(wc -l < /tmp/requirements.txt) declared dependencies (extra: ${EXTRAS:-none})" \ + && uv venv /opt/venv --python 3.12 \ + && case "${EXTRAS}" in \ + cpu) torch_index=https://download.pytorch.org/whl/cpu ;; \ + gpu) torch_index=https://download.pytorch.org/whl/cu126 ;; \ + *) torch_index= ;; \ + esac \ + && if [ -n "$torch_index" ]; then \ + echo "torch trio from $torch_index (upstream pins this in [tool.uv.sources])"; \ + uv pip install --python /opt/venv/bin/python --index-url "$torch_index" \ + torch torchvision torchaudio; \ + fi \ + && uv pip install --python /opt/venv/bin/python -r /tmp/requirements.txt \ + && uv pip install --python /opt/venv/bin/python "levenshtein>=0.27.3" + +FROM python:3.12-slim + +LABEL maintainer="semoss@semoss.org" +LABEL org.semoss.component="executor-sandbox-full" + +# git: local repository work inside the sandbox. Egress stays closed unless the OpenShell +# policy grants it per binary and per host. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && git config --system user.name "SEMOSS Sandbox" \ + && git config --system user.email "sandbox@semoss.local" \ + && git config --system safe.directory '*' + +# The venv the worker and user code import from. +COPY --from=deps /opt/venv /opt/venv +ENV VIRTUAL_ENV=/opt/venv +ENV PATH=/opt/venv/bin:$PATH + +# The worker source. The build script stages the repo's py/ as the context's py/. +COPY py/ /opt/semosshome/py/ + +RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ + && touch /opt/semosshome/insight/log.txt \ + && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro /opt/venv \ + && chmod 755 /opt/grants-rw /opt/grants-ro + +# OpenShell refuses to start unless the workspace is writable by the sandbox identity. +USER 1000:1000 +WORKDIR /opt/semosshome/py diff --git a/docker/export_requirements.py b/docker/export_requirements.py new file mode 100644 index 0000000000..891b2bf9f1 --- /dev/null +++ b/docker/export_requirements.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Flatten SEMOSS's pyproject into a requirements list for the executor image. + +SEMOSS builds its runtime python with ``uv pip install ".[cpu]"`` against +``py/install_config/pyproject.toml``. The executor sandbox must offer the same import +surface, or a ``Py()`` block that worked outside the sandbox starts failing with +ImportError inside it. + +Installing the project itself (``.[cpu]``) is not an option here: that pyproject declares +no build-system, so it is a dependency manifest rather than an installable package. This +script reads the same file and emits the dependency list instead, so nothing is duplicated +by hand. + +Two things a naive flattening gets wrong, both handled here: + +* ``[tool.uv.sources]`` maps torch/torchvision/torchaudio to the pytorch-cpu index. + Dropping it makes PyPI resolve CUDA builds -- 3.3 GB of ``nvidia`` plus triton in an + image that never sees a GPU. Those three are skipped by default; install them from the + CPU index first (see Dockerfile.executor-full). +* ``[tool.uv] override-dependencies`` carries security pins (marshmallow, protobuf, numpy, + pyasn1, lxml). They are emitted as ordinary requirements so the sandbox keeps the same + minimum versions as the platform. + +usage: export_requirements.py [extra] [--keep-torch] > requirements.txt +""" + +import sys +import tomllib + +TORCH = ("torch", "torchvision", "torchaudio") + + +def _name(spec: str) -> str: + """Package name from a requirement specifier, without version bounds or extras.""" + for separator in (">", "=", "<", "!", "~", "[", " "): + spec = spec.split(separator)[0] + return spec.strip().lower() + + +def main() -> int: + args = list(sys.argv[1:]) + keep_torch = "--keep-torch" in args + args = [a for a in args if a != "--keep-torch"] + + if not args: + print(__doc__, file=sys.stderr) + return 2 + path = args[0] + extra = args[1] if len(args) > 1 else "" + + with open(path, "rb") as handle: + data = tomllib.load(handle) + + project = data.get("project", {}) + dependencies = list(project.get("dependencies", [])) + if extra: + extras = project.get("optional-dependencies", {}) + if extra not in extras: + print( + "unknown extra %r; available: %s" % (extra, ", ".join(sorted(extras))), + file=sys.stderr, + ) + return 1 + dependencies.extend(extras[extra]) + + if not keep_torch: + dependencies = [d for d in dependencies if _name(d) not in TORCH] + + # Security pins the platform applies over transitive resolutions. + overrides = data.get("tool", {}).get("uv", {}).get("override-dependencies", []) + dependencies.extend(overrides) + + seen = set() + for dependency in dependencies: + if dependency in seen: + continue + seen.add(dependency) + print(dependency) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 876c529908a0953e6f78a4f748be95828588e78f Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 11:52:45 +0000 Subject: [PATCH 28/56] docs(executor): add topology at a glance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shows the per-session shape — Tomcat, gateway, supervisor container, workload container with network=none, PID 1 sandbox runtime, the python worker reached through openshell forward — so a reader can see why a worker connection travels four hops instead of being a local socket. --- docs/deployment/executor_sandbox.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index 9d1ded4d23..fca2228403 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -13,6 +13,28 @@ process that shares the host's filesystem, environment and network namespace. Th mode moves that process into a separate container, spawned and reaped with the user's session, and keeps the platform's files authoritative on the host. +## Topology at a glance + +``` +Tomcat (host) ──CLI──► openshell gateway (host, :17670, mTLS) + │ │ provisions + │ ▼ + │ supervisor container (uid nobody, trusted side) + │ │ control.sock (unix socket on a docker volume) + │ ▼ + │ workload container (network=none) + │ PID 1 /.openshell/runtime/openshell-sandbox + │ └── python worker (loopback :7100) + │ ▲ + └── openshell forward service --target-port 7100 --local ─┘ + (long-lived child of Tomcat; Java connects to 127.0.0.1:) +``` + +Per session there are **two containers**: the workload (your image) and its supervisor (NVIDIA's image). +The supervisor is the trusted half of the boundary and is never inside the workload. The workload has no +network interface at all — its only egress is the channel to its supervisor — so a TCP connection to the +worker has to travel `Tomcat → forward → gateway → supervisor → runtime → loopback`. + ## Process model | Piece | Runs where | Notes | From 722a7d0eecacf40749f509bdcea919dbed5e078d Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 11:53:04 +0000 Subject: [PATCH 29/56] build(executor): restore MODE=slim|full support in the image script The script that shipped with the first Dockerfile only ever built the slim image; the MODE=full path (staging pyproject.toml + export_requirements.py into the build context, passing EXTRAS) was written but never committed, and was lost to a checkout. Restored and committed so the full-image build is reproducible from the repo, not from shell history. --- docker/build-executor-image.sh | 53 +++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/docker/build-executor-image.sh b/docker/build-executor-image.sh index 0a6dbd0d4d..3f1c70097a 100755 --- a/docker/build-executor-image.sh +++ b/docker/build-executor-image.sh @@ -1,18 +1,31 @@ #!/usr/bin/env bash # Build the SEMOSS executor sandbox image from the worker source in this repo. # -# docker/build-executor-image.sh # tag semoss-executor:dev -# TAG=semoss-executor:0.3 docker/build-executor-image.sh +# docker/build-executor-image.sh # slim, tag semoss-executor:dev +# MODE=full TAG=semoss-executor:full docker/build-executor-image.sh # NO_CACHE=1 docker/build-executor-image.sh # -# The build context is py/, so the image carries exactly the python committed here. -# That is the point: there is no staged copy that can go stale. The tag is labelled -# with a hash of the python tree, so `docker inspect` answers "which source is this?" +# Two modes, one command: +# slim (default) - three packages; fast to build and start, but user code can only import +# pandas, jsonpickle and yfinance. +# full - installs the dependency set declared in py/install_config/pyproject.toml +# with uv, the same installer docker/Dockerfile.python uses, so a Py() +# block keeps the import surface it had outside the sandbox. +# +# Either way the worker python is the committed py/, never a hand-staged copy, and the tag is +# labelled with a hash of that tree so `docker inspect` answers "which source is this?". set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO="$(cd "$HERE/.." && pwd)" -TAG="${TAG:-semoss-executor:dev}" +MODE="${MODE:-slim}" +NO_CACHE="${NO_CACHE:-0}" + +case "$MODE" in + slim) TAG="${TAG:-semoss-executor:dev}" ;; + full) TAG="${TAG:-semoss-executor:full}" ;; + *) echo "MODE must be slim or full (got '$MODE')" >&2; exit 1 ;; +esac [ -d "$REPO/py" ] || { echo "no py/ under $REPO" >&2; exit 1; } @@ -22,21 +35,39 @@ PY_SHA="$(cd "$REPO/py" \ | xargs -0 sha256sum \ | sha256sum | cut -d' ' -f1)" +CTX="$(mktemp -d)" +trap 'rm -rf "$CTX"' EXIT + ARGS=(build - -f "$HERE/Dockerfile.executor" -t "$TAG" --label "org.semoss.py-sha=$PY_SHA" + --label "org.semoss.executor-mode=$MODE" --label "org.semoss.built-at=$(date -u +%Y-%m-%dT%H:%M:%SZ)") -[ "${NO_CACHE:-0}" = "1" ] && ARGS+=(--no-cache) +[ "$NO_CACHE" = "1" ] && ARGS+=(--no-cache) + +if [ "$MODE" = "slim" ]; then + # Context is py/ itself; the Dockerfile copies it verbatim. + cp "$HERE/Dockerfile.executor" "$CTX/Dockerfile" + cp -a "$REPO/py" "$CTX/py" + ARGS+=(-f "$CTX/Dockerfile") +else + # The full build also needs the manifest and the flattening helper next to py/. + cp "$HERE/Dockerfile.executor-full" "$CTX/Dockerfile" + cp "$HERE/export_requirements.py" "$CTX/export_requirements.py" + cp "$REPO/py/install_config/pyproject.toml" "$CTX/pyproject.toml" + cp -a "$REPO/py" "$CTX/py" + ARGS+=(-f "$CTX/Dockerfile" --build-arg "EXTRAS=${EXTRAS:-cpu}") +fi -docker "${ARGS[@]}" "$REPO/py" +docker "${ARGS[@]}" "$CTX" -echo "built $TAG from py sha ${PY_SHA:0:12}" +echo "built $TAG ($MODE) from py sha ${PY_SHA:0:12}" # Prove what landed, rather than trusting the build log. docker run --rm --entrypoint sh "$TAG" -c ' git --version - python3 -c "import pandas, jsonpickle; print(\"imports ok\")" + python3 -c "import pandas, jsonpickle; print(\"boot imports ok\")" + python3 -c "import numpy, boto3; print(\"parity imports ok\")" 2>/dev/null || echo "parity imports: not installed (slim)" test -f /opt/semosshome/py/gaas_tcp_socket_server.py && echo "worker present" printf "watcher ignore refs: "; grep -c "_IGNORED_DIRS" /opt/semosshome/py/gaas_change_watcher.py printf "id: "; id -u From 6551200917332bbd3f0d47e50611982053ddf99f Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 12:21:20 +0000 Subject: [PATCH 30/56] build(executor): install the executor's python into /usr/local, not a venv The sandbox's PATH belongs to OpenShell, not the image. Measured inside a live sandbox: PATH=/usr/local/bin:/usr/bin:/bin, so an image's ENV PATH=/opt/venv/bin is ignored, `python3` resolves to the base interpreter, and the worker died at boot with no pandas/jsonpickle. A symlink /usr/local/bin/python3 -> /opt/venv/bin/python does not help either: CPython looks for pyvenv.cfg relative to the invoked path and silently runs as the base interpreter. Dependencies now install with `uv pip install --system` and the final stage copies /usr/local, so whatever `python3` the sandbox resolves to already has them. No venv, no PATH dependency, and the image is smaller (3.7 GB vs 6.9 GB) because the venv duplicated the interpreter. The smoke test now runs the parity check under `env -i PATH=/usr/local/bin:/usr/bin:/bin` - the sandbox's real PATH - instead of docker run's defaults, which is why the previous build reported success while failing at worker boot. Verified in a live OpenShell sandbox: python3 -> /usr/local/bin/python3, and numpy/boto3/pandas all import. --- docker/Dockerfile.executor-full | 29 ++++++++++++++++------------- docker/build-executor-image.sh | 5 +++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docker/Dockerfile.executor-full b/docker/Dockerfile.executor-full index 586599381f..5790372616 100644 --- a/docker/Dockerfile.executor-full +++ b/docker/Dockerfile.executor-full @@ -7,18 +7,24 @@ # a Py() block that ran against the platform's own 2.4 GB venv (numpy, boto3, langchain, # torch, ...) fails with ImportError the moment it is sandboxed. That is a silent capability # regression, not a size trade-off. This build installs the dependency set that -# py/install_config/pyproject.toml declares, the same one docker/Dockerfile.python installs. +# py/install_config/pyproject.toml declares — the same one docker/Dockerfile.python installs. # # The project itself is not installed (.[extra]): that pyproject has no build-system, so it # is a dependency manifest, not a package. docker/export_requirements.py flattens it. +# +# NO VENV, deliberately. OpenShell starts the workload with its own PATH +# (/usr/local/bin:/usr/bin:/bin), so an image's ENV PATH never applies, and a venv is only +# reachable through PATH or through a symlink whose pyvenv.cfg lookup resolves to the real +# binary — which silently yields the base interpreter with none of the packages. Installing +# into /usr/local means whatever `python3` the sandbox resolves to already has them. FROM python:3.12-slim AS deps ARG EXTRAS=cpu ENV EXTRAS=${EXTRAS} # Compilers live in the builder stage only. Several declared dependencies ship no aarch64 -# wheel (annoy is a C++ extension and was the first to fail), so the venv has to be assembled -# from source here; the final stage still copies just the finished venv. +# wheel (annoy is a C++ extension and was the first to fail), so the tree has to be built +# from source here; the final stage copies only the finished /usr/local. RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential cmake \ && rm -rf /var/lib/apt/lists/* @@ -33,7 +39,6 @@ COPY export_requirements.py ./ RUN python3 export_requirements.py pyproject.toml "${EXTRAS}" > /tmp/requirements.txt \ && echo "installing $(wc -l < /tmp/requirements.txt) declared dependencies (extra: ${EXTRAS:-none})" \ - && uv venv /opt/venv --python 3.12 \ && case "${EXTRAS}" in \ cpu) torch_index=https://download.pytorch.org/whl/cpu ;; \ gpu) torch_index=https://download.pytorch.org/whl/cu126 ;; \ @@ -41,11 +46,10 @@ RUN python3 export_requirements.py pyproject.toml "${EXTRAS}" > /tmp/requirement esac \ && if [ -n "$torch_index" ]; then \ echo "torch trio from $torch_index (upstream pins this in [tool.uv.sources])"; \ - uv pip install --python /opt/venv/bin/python --index-url "$torch_index" \ - torch torchvision torchaudio; \ + uv pip install --system --index-url "$torch_index" torch torchvision torchaudio; \ fi \ - && uv pip install --python /opt/venv/bin/python -r /tmp/requirements.txt \ - && uv pip install --python /opt/venv/bin/python "levenshtein>=0.27.3" + && uv pip install --system -r /tmp/requirements.txt \ + && uv pip install --system "levenshtein>=0.27.3" FROM python:3.12-slim @@ -61,17 +65,16 @@ RUN apt-get update \ && git config --system user.email "sandbox@semoss.local" \ && git config --system safe.directory '*' -# The venv the worker and user code import from. -COPY --from=deps /opt/venv /opt/venv -ENV VIRTUAL_ENV=/opt/venv -ENV PATH=/opt/venv/bin:$PATH +# The interpreter plus every installed dependency, in the location `python3` already resolves +# to under the sandbox's PATH. No venv, no PATH dependency. +COPY --from=deps /usr/local /usr/local # The worker source. The build script stages the repo's py/ as the context's py/. COPY py/ /opt/semosshome/py/ RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ && touch /opt/semosshome/insight/log.txt \ - && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro /opt/venv \ + && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ && chmod 755 /opt/grants-rw /opt/grants-ro # OpenShell refuses to start unless the workspace is writable by the sandbox identity. diff --git a/docker/build-executor-image.sh b/docker/build-executor-image.sh index 3f1c70097a..dfbd676f51 100755 --- a/docker/build-executor-image.sh +++ b/docker/build-executor-image.sh @@ -63,11 +63,12 @@ docker "${ARGS[@]}" "$CTX" echo "built $TAG ($MODE) from py sha ${PY_SHA:0:12}" -# Prove what landed, rather than trusting the build log. +# Prove what landed, rather than trusting the build log. The parity check runs with the PATH +# OpenShell actually gives a sandbox, because the image's ENV PATH is not applied there. docker run --rm --entrypoint sh "$TAG" -c ' git --version python3 -c "import pandas, jsonpickle; print(\"boot imports ok\")" - python3 -c "import numpy, boto3; print(\"parity imports ok\")" 2>/dev/null || echo "parity imports: not installed (slim)" + env -i PATH=/usr/local/bin:/usr/bin:/bin python3 -c "import numpy, boto3; print(\"parity imports ok under sandbox PATH\")" 2>/dev/null || echo "parity imports: not installed (slim)" test -f /opt/semosshome/py/gaas_tcp_socket_server.py && echo "worker present" printf "watcher ignore refs: "; grep -c "_IGNORED_DIRS" /opt/semosshome/py/gaas_change_watcher.py printf "id: "; id -u From 837e2e1429ea10831033d931c2935bcd03a3cf1a Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 12:23:30 +0000 Subject: [PATCH 31/56] docs(executor): document the two image variants and the sandbox PATH constraint Records that OpenShell sets the workload's PATH (/usr/local/bin:/usr/bin:/bin), so an image's ENV PATH is ignored and a venv is invisible - hence the full build installs into /usr/local - and that smoke tests must run under env -i with that PATH. --- docs/deployment/executor_sandbox.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index fca2228403..b9fedc72e9 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -132,10 +132,22 @@ TAG=semoss-executor:0.1 docker/build-executor-image.sh The build context is `py/`, so the image carries exactly the committed worker; `docker/Dockerfile.executor` assembles it. The tag is labelled with `org.semoss.py-sha`, a hash -of the python tree, so the image can be traced back to its source. Contents: `python:3.12-slim`, -`git` with a system identity (local repository work only — egress is granted per binary by the -OpenShell policy), `pandas` and `jsonpickle` for the boot-time imports, the insight and grants -layout, and uid 1000 ownership of `/opt/semosshome`. +of the python tree, so the image can be traced back to its source. + +Two variants: + +| Mode | Contents | Use | +| --- | --- | --- | +| `slim` (default) | `python:3.12-slim`, git, `pandas`, `jsonpickle`, `yfinance` | quick iteration; user code can only import those | +| `full` (`MODE=full`) | the dependency set declared in `py/install_config/pyproject.toml`, installed with uv | parity with the platform runtime — a `Py()` block keeps its imports | + +The full build installs into `/usr/local` rather than a virtualenv, because **the sandbox's +`PATH` belongs to OpenShell, not the image**: a workload starts with +`/usr/local/bin:/usr/bin:/bin`, so `ENV PATH=/opt/venv/bin:$PATH` is ignored and a venv would be +invisible. (Symlinking `python3` at the venv does not help either — CPython looks for +`pyvenv.cfg` relative to the invoked path and silently falls back to the base interpreter.) Any +smoke test must therefore run under `env -i PATH=/usr/local/bin:/usr/bin:/bin`, or it passes +while the worker fails at boot. At session start the launcher also uploads the host's `/opt/semosshome/py` over the image's copy, but the worker's first imports happen before that upload, so the baked copy still has to From b52e551d3a5f2e55483cfeb9474c2cd659306954 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 12:39:43 +0000 Subject: [PATCH 32/56] build(executor): Ubuntu base, venv kept, interpreter resolved explicitly Three changes that belong together, because the sandbox runtime - not the image - decides the environment its processes get. 1. Dockerfile.executor-full now builds on ubuntu:24.04, matching docker/Dockerfile.python: same distro, glibc and native-library expectations for document/OCR stacks. python3.12 comes from apt (plus python3.12-dev, without which annoy cannot compile its C extension), and the declared dependencies install into /opt/venv with uv, as upstream does. 2. OpenShellTransport launches the worker as /opt/venv/bin/python3 when it exists and falls back to `command -v python3` otherwise. Invoking `python3` by name was the actual defect: OpenShell builds the environment for spawned processes, so an image's ENV PATH never applies and the bare name resolved to the base interpreter with none of the packages. 3. Sandbox creation passes --env PATH=/opt/venv/bin:... so user code that shells out to python3 resolves to the same venv the worker uses. Verified so far: the deployed class carries both the resolver and the --env flag, and the fallback branch works against an image without a venv (sys.executable = /usr/local/bin/python3, worker boots, Py() returns). --- docker/Dockerfile.executor-full | 69 ++++++++++++++----------- docker/build-executor-image.sh | 8 +-- src/prerna/util/OpenShellTransport.java | 13 ++++- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/docker/Dockerfile.executor-full b/docker/Dockerfile.executor-full index 5790372616..212d3ce772 100644 --- a/docker/Dockerfile.executor-full +++ b/docker/Dockerfile.executor-full @@ -1,35 +1,40 @@ # SEMOSS executor sandbox — full-parity build. # +# MODE=full TAG=semoss-executor:full docker/build-executor-image.sh +# # or directly: # docker build -f docker/Dockerfile.executor-full -t semoss-executor:full py -# # or: MODE=full docker/build-executor-image.sh # -# Why this exists next to Dockerfile.executor: the minimal image carries three packages, so -# a Py() block that ran against the platform's own 2.4 GB venv (numpy, boto3, langchain, -# torch, ...) fails with ImportError the moment it is sandboxed. That is a silent capability -# regression, not a size trade-off. This build installs the dependency set that -# py/install_config/pyproject.toml declares — the same one docker/Dockerfile.python installs. +# Why this exists next to Dockerfile.executor: the minimal image carries three packages, so a +# Py() block that ran against the platform's own runtime (numpy, boto3, langchain, torch, ...) +# fails with ImportError the moment it is sandboxed. That is a silent capability regression, not +# a size trade-off. # -# The project itself is not installed (.[extra]): that pyproject has no build-system, so it -# is a dependency manifest, not a package. docker/export_requirements.py flattens it. +# Base is Ubuntu, matching docker/Dockerfile.python, so sandbox and platform share a distro, +# glibc and the native libraries that document/OCR stacks expect. Python comes from apt (24.04 +# ships 3.12) and the declared dependencies install into a venv with uv — the same installer +# upstream uses for its own runtime. # -# NO VENV, deliberately. OpenShell starts the workload with its own PATH -# (/usr/local/bin:/usr/bin:/bin), so an image's ENV PATH never applies, and a venv is only -# reachable through PATH or through a symlink whose pyvenv.cfg lookup resolves to the real -# binary — which silently yields the base interpreter with none of the packages. Installing -# into /usr/local means whatever `python3` the sandbox resolves to already has them. -FROM python:3.12-slim AS deps +# The venv is reached by EXPLICIT PATH, never by PATH lookup: OpenShell builds the environment +# for the processes it spawns, so an image's ENV PATH is not applied inside a sandbox. The +# worker is launched as /opt/venv/bin/python3 (see OpenShellTransport) and the sandbox is +# created with `--env PATH=/opt/venv/bin:…` so user code resolves to the same interpreter. +ARG UBUNTU_VERSION=24.04 + +FROM ubuntu:${UBUNTU_VERSION} AS deps ARG EXTRAS=cpu ENV EXTRAS=${EXTRAS} +ENV DEBIAN_FRONTEND=noninteractive -# Compilers live in the builder stage only. Several declared dependencies ship no aarch64 -# wheel (annoy is a C++ extension and was the first to fail), so the tree has to be built -# from source here; the final stage copies only the finished /usr/local. +# Build toolchain lives in this stage only: several declared dependencies ship no aarch64 +# wheel (annoy is a C++ extension and was the first to fail), so they compile from source. RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential cmake \ + && apt-get install -y --no-install-recommends \ + python3 python3.12 python3.12-dev python3.12-venv python3-venv build-essential cmake pkg-config curl ca-certificates \ + libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libffi-dev \ && rm -rf /var/lib/apt/lists/* -# uv, the same installer SEMOSS uses for its own runtime. +# uv, the installer SEMOSS uses for its own runtime. COPY --from=ghcr.io/astral-sh/uv:0.9 /uv /uvx /usr/local/bin/ WORKDIR /build @@ -39,6 +44,7 @@ COPY export_requirements.py ./ RUN python3 export_requirements.py pyproject.toml "${EXTRAS}" > /tmp/requirements.txt \ && echo "installing $(wc -l < /tmp/requirements.txt) declared dependencies (extra: ${EXTRAS:-none})" \ + && uv venv /opt/venv --python /usr/bin/python3.12 --seed \ && case "${EXTRAS}" in \ cpu) torch_index=https://download.pytorch.org/whl/cpu ;; \ gpu) torch_index=https://download.pytorch.org/whl/cu126 ;; \ @@ -46,35 +52,38 @@ RUN python3 export_requirements.py pyproject.toml "${EXTRAS}" > /tmp/requirement esac \ && if [ -n "$torch_index" ]; then \ echo "torch trio from $torch_index (upstream pins this in [tool.uv.sources])"; \ - uv pip install --system --index-url "$torch_index" torch torchvision torchaudio; \ + uv pip install --python /opt/venv/bin/python --index-url "$torch_index" \ + torch torchvision torchaudio; \ fi \ - && uv pip install --system -r /tmp/requirements.txt \ - && uv pip install --system "levenshtein>=0.27.3" + && uv pip install --python /opt/venv/bin/python -r /tmp/requirements.txt \ + && uv pip install --python /opt/venv/bin/python "levenshtein>=0.27.3" -FROM python:3.12-slim +FROM ubuntu:${UBUNTU_VERSION} LABEL maintainer="semoss@semoss.org" LABEL org.semoss.component="executor-sandbox-full" -# git: local repository work inside the sandbox. Egress stays closed unless the OpenShell -# policy grants it per binary and per host. +ENV DEBIAN_FRONTEND=noninteractive + +# python3.12 must exist in this stage too: the venv's interpreter is a symlink to +# /usr/bin/python3.12. git enables local repository work; egress stays policy-gated. RUN apt-get update \ - && apt-get install -y --no-install-recommends git ca-certificates \ + && apt-get install -y --no-install-recommends python3 python3.12 git ca-certificates \ && rm -rf /var/lib/apt/lists/* \ && git config --system user.name "SEMOSS Sandbox" \ && git config --system user.email "sandbox@semoss.local" \ && git config --system safe.directory '*' -# The interpreter plus every installed dependency, in the location `python3` already resolves -# to under the sandbox's PATH. No venv, no PATH dependency. -COPY --from=deps /usr/local /usr/local +COPY --from=deps /opt/venv /opt/venv +ENV VIRTUAL_ENV=/opt/venv +ENV PATH=/opt/venv/bin:$PATH # The worker source. The build script stages the repo's py/ as the context's py/. COPY py/ /opt/semosshome/py/ RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ && touch /opt/semosshome/insight/log.txt \ - && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ + && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro /opt/venv \ && chmod 755 /opt/grants-rw /opt/grants-ro # OpenShell refuses to start unless the workspace is writable by the sandbox identity. diff --git a/docker/build-executor-image.sh b/docker/build-executor-image.sh index dfbd676f51..84739a05a1 100755 --- a/docker/build-executor-image.sh +++ b/docker/build-executor-image.sh @@ -63,12 +63,12 @@ docker "${ARGS[@]}" "$CTX" echo "built $TAG ($MODE) from py sha ${PY_SHA:0:12}" -# Prove what landed, rather than trusting the build log. The parity check runs with the PATH -# OpenShell actually gives a sandbox, because the image's ENV PATH is not applied there. +# Prove what landed, rather than trusting the build log. The interpreter is checked by absolute +# path (what the worker uses) and again through the PATH the sandbox is created with. docker run --rm --entrypoint sh "$TAG" -c ' git --version - python3 -c "import pandas, jsonpickle; print(\"boot imports ok\")" - env -i PATH=/usr/local/bin:/usr/bin:/bin python3 -c "import numpy, boto3; print(\"parity imports ok under sandbox PATH\")" 2>/dev/null || echo "parity imports: not installed (slim)" + /opt/venv/bin/python3 -c "import numpy, boto3; print(\"parity imports ok (venv)\")" 2>/dev/null || echo "no venv in this image (slim)" + env -i PATH=/opt/venv/bin:/usr/local/bin:/usr/bin:/bin python3 -c "import sys; print(\"python3 ->\", sys.executable)" 2>/dev/null || true test -f /opt/semosshome/py/gaas_tcp_socket_server.py && echo "worker present" printf "watcher ignore refs: "; grep -c "_IGNORED_DIRS" /opt/semosshome/py/gaas_change_watcher.py printf "id: "; id -u diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index ec0687eab1..7b9c8e9ca0 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -238,6 +238,11 @@ private static List createCommand(String sandboxName) throws IOException } create.add("--policy"); create.add(policy.trim()); + // Give sandboxed code the same interpreter the worker uses. The runtime does not + // apply the image's ENV PATH, so without this a `Py()` block that shells out to + // `python3` gets the bare system interpreter while the worker runs from the venv. + create.add("--env"); + create.add("PATH=/opt/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"); create.add("--"); // The worker's stdout/stderr MUST go to a file, not to the supervisor's // stdout pipe. With the pipe it came up erratically -- sometimes serving, @@ -246,9 +251,15 @@ private static List createCommand(String sandboxName) throws IOException // ready in ~0.66s. Suspected cause is output back-pressure on that pipe. // The file is also the readiness signal, read via `sandbox exec` so that no // connection is ever made to the single-connection worker. + // + // The interpreter is resolved explicitly: OpenShell builds the environment for + // the processes it spawns, so an image's ENV PATH is not applied and a bare + // `python3` means whatever the runtime's PATH points at -- which is how a venv + // image ended up running the base interpreter with none of the dependencies. create.add("sh"); create.add("-c"); - create.add("exec python3 /opt/semosshome/py/gaas_tcp_socket_server.py" // + create.add("P=/opt/venv/bin/python3; [ -x \"$P\" ] || P=$(command -v python3);" + + " exec \"$P\" /opt/semosshome/py/gaas_tcp_socket_server.py" // + " --port " + workerPort // + " --start True" // + " --py_folder /opt/semosshome/py" // From 9868fe8bdea7035dc365901707930a76a9c1ffba Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 12:53:02 +0000 Subject: [PATCH 33/56] fix(executor): put the venv where the sandbox policy allows, and launch it by path Third and final blocker in the same family: the policy allowlists /usr, /lib, /etc, /bin, /sbin, /proc (ro) plus /sandbox, /tmp, /opt/semosshome (rw). A venv at /opt/venv is outside every one of those, so the worker died with PermissionError: [Errno 13] Permission denied: '/opt/venv/pyvenv.cfg' before python could start. The venv now lives at /usr/lib/python/semossvenv - the path docker/Dockerfile.python already uses, which sits inside the allowed /usr - and OpenShellTransport resolves it explicitly (/usr/lib/python/semossvenv/bin/python3, falling back to `command -v python3` for images without a venv), with `--env PATH=/usr/lib/python/semossvenv/bin:...` for user code. Also drops the chown -R over the venv: it is read-only to the sandbox, so root-owned 755 is correct and it removes minutes from the build. Verified live through SEMOSS: sys.executable = /usr/lib/python/semossvenv/bin/python3, python 3.12.3, boto3 1.43.118, numpy 2.5.2 - imports that all failed before. That path is the proof the interpreter was resolved by name rather than by luck. --- docker/Dockerfile.executor-full | 23 +++++++++++++---------- docker/build-executor-image.sh | 4 ++-- docs/deployment/executor_sandbox.md | 24 +++++++++++++++++------- src/prerna/util/OpenShellTransport.java | 4 ++-- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/docker/Dockerfile.executor-full b/docker/Dockerfile.executor-full index 212d3ce772..b92ce5e9e9 100644 --- a/docker/Dockerfile.executor-full +++ b/docker/Dockerfile.executor-full @@ -16,8 +16,9 @@ # # The venv is reached by EXPLICIT PATH, never by PATH lookup: OpenShell builds the environment # for the processes it spawns, so an image's ENV PATH is not applied inside a sandbox. The -# worker is launched as /opt/venv/bin/python3 (see OpenShellTransport) and the sandbox is -# created with `--env PATH=/opt/venv/bin:…` so user code resolves to the same interpreter. +# worker is launched as /usr/lib/python/semossvenv/bin/python3 (see OpenShellTransport) and the +# sandbox is created with `--env PATH=/usr/lib/python/semossvenv/bin:…` so user code resolves +# to the same interpreter. ARG UBUNTU_VERSION=24.04 FROM ubuntu:${UBUNTU_VERSION} AS deps @@ -44,7 +45,7 @@ COPY export_requirements.py ./ RUN python3 export_requirements.py pyproject.toml "${EXTRAS}" > /tmp/requirements.txt \ && echo "installing $(wc -l < /tmp/requirements.txt) declared dependencies (extra: ${EXTRAS:-none})" \ - && uv venv /opt/venv --python /usr/bin/python3.12 --seed \ + && uv venv /usr/lib/python/semossvenv --python /usr/bin/python3.12 --seed \ && case "${EXTRAS}" in \ cpu) torch_index=https://download.pytorch.org/whl/cpu ;; \ gpu) torch_index=https://download.pytorch.org/whl/cu126 ;; \ @@ -52,11 +53,11 @@ RUN python3 export_requirements.py pyproject.toml "${EXTRAS}" > /tmp/requirement esac \ && if [ -n "$torch_index" ]; then \ echo "torch trio from $torch_index (upstream pins this in [tool.uv.sources])"; \ - uv pip install --python /opt/venv/bin/python --index-url "$torch_index" \ + uv pip install --python /usr/lib/python/semossvenv/bin/python --index-url "$torch_index" \ torch torchvision torchaudio; \ fi \ - && uv pip install --python /opt/venv/bin/python -r /tmp/requirements.txt \ - && uv pip install --python /opt/venv/bin/python "levenshtein>=0.27.3" + && uv pip install --python /usr/lib/python/semossvenv/bin/python -r /tmp/requirements.txt \ + && uv pip install --python /usr/lib/python/semossvenv/bin/python "levenshtein>=0.27.3" FROM ubuntu:${UBUNTU_VERSION} @@ -74,16 +75,18 @@ RUN apt-get update \ && git config --system user.email "sandbox@semoss.local" \ && git config --system safe.directory '*' -COPY --from=deps /opt/venv /opt/venv -ENV VIRTUAL_ENV=/opt/venv -ENV PATH=/opt/venv/bin:$PATH +# The venv lives under /usr/lib/python, matching docker/Dockerfile.python. That path is +# inside the sandbox policy's read-only /usr, which /opt/venv would not be. +COPY --from=deps /usr/lib/python/semossvenv /usr/lib/python/semossvenv +ENV VIRTUAL_ENV=/usr/lib/python/semossvenv +ENV PATH=/usr/lib/python/semossvenv/bin:$PATH # The worker source. The build script stages the repo's py/ as the context's py/. COPY py/ /opt/semosshome/py/ RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ && touch /opt/semosshome/insight/log.txt \ - && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro /opt/venv \ + && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ && chmod 755 /opt/grants-rw /opt/grants-ro # OpenShell refuses to start unless the workspace is writable by the sandbox identity. diff --git a/docker/build-executor-image.sh b/docker/build-executor-image.sh index 84739a05a1..cd022f8e3c 100755 --- a/docker/build-executor-image.sh +++ b/docker/build-executor-image.sh @@ -67,8 +67,8 @@ echo "built $TAG ($MODE) from py sha ${PY_SHA:0:12}" # path (what the worker uses) and again through the PATH the sandbox is created with. docker run --rm --entrypoint sh "$TAG" -c ' git --version - /opt/venv/bin/python3 -c "import numpy, boto3; print(\"parity imports ok (venv)\")" 2>/dev/null || echo "no venv in this image (slim)" - env -i PATH=/opt/venv/bin:/usr/local/bin:/usr/bin:/bin python3 -c "import sys; print(\"python3 ->\", sys.executable)" 2>/dev/null || true + /usr/lib/python/semossvenv/bin/python3 -c "import numpy, boto3; print(\"parity imports ok (venv)\")" 2>/dev/null || echo "no venv in this image (slim)" + env -i PATH=/usr/lib/python/semossvenv/bin:/usr/local/bin:/usr/bin:/bin python3 -c "import sys; print(\"python3 ->\", sys.executable)" 2>/dev/null || true test -f /opt/semosshome/py/gaas_tcp_socket_server.py && echo "worker present" printf "watcher ignore refs: "; grep -c "_IGNORED_DIRS" /opt/semosshome/py/gaas_change_watcher.py printf "id: "; id -u diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index b9fedc72e9..22ef9a79c2 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -141,13 +141,23 @@ Two variants: | `slim` (default) | `python:3.12-slim`, git, `pandas`, `jsonpickle`, `yfinance` | quick iteration; user code can only import those | | `full` (`MODE=full`) | the dependency set declared in `py/install_config/pyproject.toml`, installed with uv | parity with the platform runtime — a `Py()` block keeps its imports | -The full build installs into `/usr/local` rather than a virtualenv, because **the sandbox's -`PATH` belongs to OpenShell, not the image**: a workload starts with -`/usr/local/bin:/usr/bin:/bin`, so `ENV PATH=/opt/venv/bin:$PATH` is ignored and a venv would be -invisible. (Symlinking `python3` at the venv does not help either — CPython looks for -`pyvenv.cfg` relative to the invoked path and silently falls back to the base interpreter.) Any -smoke test must therefore run under `env -i PATH=/usr/local/bin:/usr/bin:/bin`, or it passes -while the worker fails at boot. +The full build keeps a venv at `/usr/lib/python/semossvenv` — the same path +`docker/Dockerfile.python` uses — and the worker is launched by absolute path, because +**both the environment and the filesystem inside a sandbox belong to the runtime**: + +- **`PATH`**: a workload starts with `/usr/local/bin:/usr/bin:/bin`, so an image's `ENV PATH` + is never applied and a bare `python3` resolves to the base interpreter. Symlinking + `python3` at the venv does not help either — CPython looks for `pyvenv.cfg` relative to the + invoked path and silently falls back to the base interpreter. +- **Filesystem**: the policy allowlists `/usr`, `/lib`, `/etc`, `/bin`, `/sbin`, `/proc` + read-only, plus `/sandbox`, `/tmp`, `/opt/semosshome` read-write. A venv at `/opt/venv` is + outside every one of those and dies with `PermissionError: pyvenv.cfg`; `/usr/lib/python` + is inside `/usr`, which is exactly why upstream puts it there. + +So `OpenShellTransport` resolves `/usr/lib/python/semossvenv/bin/python3` (falling back to +`command -v python3` for images without a venv) and creates the sandbox with +`--env PATH=/usr/lib/python/semossvenv/bin:…` so user code gets the same interpreter. +Smoke tests must reproduce both constraints, or they pass while the worker fails at boot. At session start the launcher also uploads the host's `/opt/semosshome/py` over the image's copy, but the worker's first imports happen before that upload, so the baked copy still has to diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 7b9c8e9ca0..3fe6e6a08a 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -242,7 +242,7 @@ private static List createCommand(String sandboxName) throws IOException // apply the image's ENV PATH, so without this a `Py()` block that shells out to // `python3` gets the bare system interpreter while the worker runs from the venv. create.add("--env"); - create.add("PATH=/opt/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"); + create.add("PATH=/usr/lib/python/semossvenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"); create.add("--"); // The worker's stdout/stderr MUST go to a file, not to the supervisor's // stdout pipe. With the pipe it came up erratically -- sometimes serving, @@ -258,7 +258,7 @@ private static List createCommand(String sandboxName) throws IOException // image ended up running the base interpreter with none of the dependencies. create.add("sh"); create.add("-c"); - create.add("P=/opt/venv/bin/python3; [ -x \"$P\" ] || P=$(command -v python3);" + create.add("P=/usr/lib/python/semossvenv/bin/python3; [ -x \"$P\" ] || P=$(command -v python3);" + " exec \"$P\" /opt/semosshome/py/gaas_tcp_socket_server.py" // + " --port " + workerPort // + " --start True" // From 491efe3389e953bd070363ceab1ba8c20cfc9bdb Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 19:41:02 +0000 Subject: [PATCH 34/56] fix(openshell): make write-back safe against loss, links and runaway captures - Infer deletions only for paths the sandbox is known to have held. A full capture used to remove every host file it did not contain, which deleted gitignored files (never uploaded) and files added on the host mid-session. - Owe a full capture when a drained change set cannot be applied, instead of losing those paths. - Serialise fetch-back per process: python blocks, shell commands and teardown no longer interleave reconciles. - Quarantine a differing host file whenever it changed since this session last left it, and skip the write if the copy fails. Sandbox mtimes no longer decide. - Refuse host writes and deletes that leave the root or cross a symlink. - Write back only git's data files (objects, refs, logs, HEAD, index): config, hooks and info are executed or obeyed by host-side git. - Stream captures to disk under the cap instead of buffering them in the heap. - Check archive paths per segment and require a clean tar listing; normalise permissions after extraction so mode-000 directories cannot leak temp dirs. - Use userCanEditProject for write-back, the check that grants the read-write upload. - Shell runner: capture output to a file (a full pipe deadlocked past 64 KB), cap it, and only change directory on a clean cd. Co-Authored-By: Claude Opus 5.5 --- src/prerna/om/ClientProcessWrapper.java | 87 ++- src/prerna/util/OpenShellTransport.java | 631 ++++++++++++------ .../util/openshell/OpenShellShellRunner.java | 88 ++- 3 files changed, 543 insertions(+), 263 deletions(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 368190c7c6..1f930b57a3 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -131,6 +131,16 @@ public class ClientProcessWrapper { private prerna.auth.User user; // OPENSHELL mode: owns the sandbox + port-forward for this worker's lifetime private prerna.util.OpenShellTransport openshell; + /** + * One write-back at a time: python blocks, shell commands and teardown all reconcile, + * and two reconciles interleaving over the same root would race their writes. + */ + private final ReentrantLock fetchBackLock = new ReentrantLock(); + /** + * Set when a change set was drained from the worker but could not be applied. The + * worker has already forgotten those paths, so only a full capture can recover them. + */ + private volatile boolean fullCaptureOwed = false; private String classPath; private boolean debug; private String timeout; @@ -414,6 +424,15 @@ public int fetchBackChangedPaths() { if (this.openshell == null || this.chrootSymlinkHelper == null) { return 0; } + fetchBackLock.lock(); + try { + return fetchBackChangedPathsLocked(); + } finally { + fetchBackLock.unlock(); + } + } + + private int fetchBackChangedPathsLocked() { Map snapshot = null; if (this.socketClient instanceof NativePySocketClient pyClient) { try { @@ -422,8 +441,10 @@ public int fetchBackChangedPaths() { classLogger.warn("Change query failed; falling back to a full reconcile", e); } } - if (snapshot == null || truthy(snapshot.get("overflow")) || truthy(snapshot.get("baseline")) - || Boolean.FALSE.equals(snapshot.get("coverage"))) { + // The snapshot is taken even when a full capture is owed: it drains the worker's + // pending set, which the full capture is about to cover anyway. + if (snapshot == null || this.fullCaptureOwed || truthy(snapshot.get("overflow")) + || truthy(snapshot.get("baseline")) || Boolean.FALSE.equals(snapshot.get("coverage"))) { return fetchBackTransportedRoots(); } Object raw = snapshot.get("changes"); @@ -448,7 +469,7 @@ public int fetchBackChangedPaths() { roots.sort((first, second) -> second.length() - first.length()); Map> writes = new HashMap<>(); - Map deletes = new HashMap<>(); + Map> deletes = new HashMap<>(); for (Object entry : changes) { if (!(entry instanceof Map change)) { continue; @@ -469,16 +490,8 @@ public int fetchBackChangedPaths() { continue; } String relative = path.substring(root.length() + 1); - if ("delete".equals(String.valueOf(change.get("op")))) { - deletes.put(path, root); - } else { - List group = writes.get(root); - if (group == null) { - group = new ArrayList<>(); - writes.put(root, group); - } - group.add(relative); - } + boolean delete = "delete".equals(String.valueOf(change.get("op"))); + (delete ? deletes : writes).computeIfAbsent(root, key -> new ArrayList<>()).add(relative); } int total = 0; @@ -486,20 +499,25 @@ public int fetchBackChangedPaths() { try { total += this.openshell.fetchBackPaths(group.getKey(), group.getValue()); } catch (Exception e) { - classLogger.warn("Path fetch-back failed for {}", group.getKey(), e); + // The worker has already dropped these paths from its pending set, so they + // would never be reported again: recover them with a full capture next block. + this.fullCaptureOwed = true; + classLogger.warn("Path fetch-back failed for {}; a full capture is owed", group.getKey(), e); } } - for (Map.Entry entry : deletes.entrySet()) { - try { - prerna.util.OpenShellTransport.quarantineAndDelete(entry.getKey(), entry.getValue()); - } catch (Exception e) { - classLogger.warn("Delete after change set failed for {}", entry.getKey(), e); + int removed = 0; + for (Map.Entry> group : deletes.entrySet()) { + for (String relative : group.getValue()) { + if (this.openshell.deleteFromHost(group.getKey(), relative)) { + removed++; + } } } - if (total > 0 || !deletes.isEmpty()) { - classLogger.info("Fetch-back wrote {} file(s) from {} changed path(s)", total, changes.size()); + if (total > 0 || removed > 0) { + classLogger.info("Fetch-back wrote {} and removed {} file(s) from {} changed path(s)", total, removed, + changes.size()); } - return total; + return total + removed; } private static boolean truthy(Object value) { @@ -623,13 +641,12 @@ private boolean canWriteBack(String normalized) { return false; } try { - String permission = prerna.auth.utils.SecurityProjectUtils - .getActualUserProjectPermission(this.user, projectId); - boolean allowed = prerna.auth.AccessPermissionEnum.OWNER.getPermission().equals(permission) - || prerna.auth.AccessPermissionEnum.EDIT.getPermission().equals(permission); + // The same check SymlinkHelper.symlinkProject uses to grant the project + // read-write: write-back must never answer that question differently. + boolean allowed = prerna.auth.utils.SecurityProjectUtils.userCanEditProject(this.user, projectId); if (!allowed) { - classLogger.warn("Refusing write-back into {}: user {} has {} access to project {}", normalized, - prerna.auth.User.getSingleLogginName(this.user), permission, projectId); + classLogger.warn("Refusing write-back into {}: user {} cannot edit project {}", normalized, + prerna.auth.User.getSingleLogginName(this.user), projectId); } return allowed; } catch (Exception e) { @@ -655,7 +672,17 @@ public int fetchBackTransportedRoots() { if (this.openshell == null || this.chrootSymlinkHelper == null) { return 0; } + fetchBackLock.lock(); + try { + return fetchBackTransportedRootsLocked(); + } finally { + fetchBackLock.unlock(); + } + } + + private int fetchBackTransportedRootsLocked() { int total = 0; + boolean failed = false; for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { String normalized = Utility.normalizePath(path); // Only the user-facing roots. Engine and runtime trees are never written by @@ -669,9 +696,11 @@ public int fetchBackTransportedRoots() { try { total += this.openshell.fetchBack(path); } catch (Exception e) { - classLogger.warn("Fetch-back failed for {}", path, e); + failed = true; + classLogger.warn("Fetch-back failed for {}; retrying with a full capture next block", path, e); } } + this.fullCaptureOwed = failed; return total; } diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 3fe6e6a08a..4f57fa5548 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -158,6 +158,23 @@ private OpenShellTransport(String sandboxName, String workerPort) { /** Cap for one fetch-back archive; larger trees are pulled path by path or not at all. */ private static final long MAX_FETCH_BYTES = 256L * 1024 * 1024; + /** Most paths one change set may name; the worker overflows to a full capture before this. */ + private static final int MAX_CHANGED_PATHS = 2000; + + /** + * Relative paths, per transported root, that the sandbox is known to have held: + * everything in its last full capture plus every path written back since. Deletions + * are inferred only for these, so a host file the sandbox never received -- + * gitignored, added on the host mid-session, missed by a partial upload -- is never + * mistaken for one the sandbox removed. + */ + private final java.util.Map> seenBySandbox = new java.util.concurrent.ConcurrentHashMap<>(); + /** + * Size and mtime of each host file as this transport last left it (uploaded or + * written back). A file that no longer matches was changed on the host since, and is + * quarantined before a sandbox copy replaces it. + */ + private final java.util.Map hostStamps = new java.util.concurrent.ConcurrentHashMap<>(); private static final String NAME_PREFIX = "pyw-"; @@ -457,6 +474,12 @@ public boolean upload(String absPath) { // `sandbox upload` prints a tick with its result; treat a non-zero exit as failure, // which run() already raises as IOException. classLogger.info("Sandbox upload result for {}: {}", absPath, out.trim()); + String normalized = Utility.normalizePath(absPath); + if (normalized.contains("/" + Constants.PROJECT_FOLDER + "/") + || normalized.contains("/" + Constants.ROOM_FOLDER + "/") + || normalized.contains("/" + Constants.USER_FOLDER + "/")) { + recordUploadedStamps(java.nio.file.Paths.get(normalized)); + } return true; } catch (Exception e) { classLogger.error("Sandbox upload failed for " + absPath + ": " + e.getClass().getSimpleName() + ": " @@ -532,15 +555,16 @@ public int getForwardedPort() { * correct for images and wheels. * *

- * Reconciliation is per file and deliberately conservative: a file is written back - * only when its content differs and the sandbox copy is newer than the host - * copy. That keeps a stale or half-written sandbox from clobbering a host edit made - * after the upload. + * A full capture also propagates deletions, but only for paths the sandbox is known + * to have held (see {@link #seenBySandbox}); absence alone proves nothing, because + * the upload skips gitignored files and the host can gain files mid-session. * * @param absPath a path previously passed to {@link #upload(String)} - * @return number of files written back + * @return number of files written back or removed + * @throws IOException when the capture could not be taken or applied; the caller must + * treat the root as unsynced */ - public int fetchBack(String absPath) { + public int fetchBack(String absPath) throws IOException { if (absPath == null || absPath.trim().isEmpty()) { return 0; } @@ -557,40 +581,28 @@ public int fetchBack(String absPath) { return 0; } - java.nio.file.Path temp = null; + java.nio.file.Path temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-"); try { String cli = propOr(CLI_PROP, DEFAULT_CLI); - // .git is deliberately NOT excluded: a commit made in the sandbox shell is - // work the user expects to keep, and git objects are content-addressed so - // copying them back is safe. Only regenerable caches are skipped. - // Known risk, not yet guarded: a host-side commit and a sandbox-side commit - // made concurrently can leave refs diverged; the revision/conflict check that - // would catch it is still to come. - byte[] archive = runBytes(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", parent, - "--exclude=node_modules", "--exclude=__pycache__", "--exclude=.venv", - "-cf", "-", base), 120); - if (archive == null || archive.length == 0) { - return 0; - } - if (archive.length > MAX_FETCH_BYTES) { - classLogger.warn("Fetch-back of {} is {} bytes, over the {} byte cap; skipped", absPath, archive.length, - MAX_FETCH_BYTES); + // .git is captured so that a commit made in the sandbox shell survives, but only + // git's data files are ever written back -- see isWritableGitPath. Regenerable + // caches are skipped. + java.nio.file.Path archive = temp.resolve("capture.tar"); + long size = runToFile(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", parent, + "--exclude=node_modules", "--exclude=__pycache__", "--exclude=.venv", "-cf", "-", base), 120, + archive); + if (size == 0) { return 0; } - if (!archiveEntriesAreSafe(archive)) { - classLogger.warn("Fetch-back of {} contained an absolute or traversing path; skipped", absPath); - return 0; - } - temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-"); - extractArchive(archive, temp); - return reconcile(temp.resolve(base), java.nio.file.Paths.get(normalized), true); - } catch (Exception e) { - classLogger.warn("Fetch-back failed for {}", absPath, e); - return 0; + requireSafeArchive(archive); + java.nio.file.Path tree = java.nio.file.Files.createDirectory(temp.resolve("tree")); + extractArchive(archive, tree); + return reconcile(tree.resolve(base), java.nio.file.Paths.get(normalized), true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("fetch-back of " + absPath + " interrupted", e); } finally { - if (temp != null) { - deleteQuietly(temp); - } + deleteQuietly(temp); } } @@ -606,8 +618,10 @@ public int fetchBack(String absPath) { * @param absRoot absolute path of the transported root inside the sandbox * @param relativePaths paths relative to that root, as reported by the watcher * @return number of files written back + * @throws IOException when the paths could not be captured or applied; the change set + * is then lost and the caller must schedule a full capture */ - public int fetchBackPaths(String absRoot, java.util.List relativePaths) { + public int fetchBackPaths(String absRoot, java.util.List relativePaths) throws IOException { if (absRoot == null || relativePaths == null || relativePaths.isEmpty()) { return 0; } @@ -623,97 +637,115 @@ public int fetchBackPaths(String absRoot, java.util.List relativePaths) continue; } String cleaned = relative.replace('\\', '/'); - if (cleaned.startsWith("/") || cleaned.contains("../") || cleaned.equals("..")) { + if (!isSafeRelative(cleaned)) { classLogger.warn("Refusing traversal in change set: {}", relative); continue; } safe.add(cleaned); - if (safe.size() >= 2000) { - break; - } } if (safe.isEmpty()) { return 0; } + if (safe.size() > MAX_CHANGED_PATHS) { + throw new IOException(safe.size() + " changed paths under " + absRoot + " exceed the " + MAX_CHANGED_PATHS + + " path limit"); + } - java.nio.file.Path temp = null; + java.nio.file.Path temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-paths-"); try { String cli = propOr(CLI_PROP, DEFAULT_CLI); java.util.List command = new java.util.ArrayList<>(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", normalized, "--ignore-failed-read", "--exclude=node_modules", - "--exclude=__pycache__", "--exclude=.venv", "-cf", "-")); + "--exclude=__pycache__", "--exclude=.venv", "-cf", "-", "--")); command.addAll(safe); - byte[] archive = runBytes(command, 120); - if (archive == null || archive.length == 0) { - return 0; - } - if (archive.length > MAX_FETCH_BYTES) { - classLogger.warn("Path fetch-back from {} is {} bytes, over the cap; skipped", absRoot, archive.length); - return 0; - } - if (!archiveEntriesAreSafe(archive)) { - classLogger.warn("Path fetch-back from {} contained an absolute or traversing path; skipped", absRoot); + java.nio.file.Path archive = temp.resolve("capture.tar"); + long size = runToFile(command, 120, archive); + if (size == 0) { return 0; } - temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-paths-"); - extractArchive(archive, temp); - return reconcile(temp, java.nio.file.Paths.get(normalized), false); - } catch (Exception e) { - classLogger.warn("Path fetch-back failed for {}", absRoot, e); - return 0; + requireSafeArchive(archive); + java.nio.file.Path tree = java.nio.file.Files.createDirectory(temp.resolve("tree")); + extractArchive(archive, tree); + return reconcile(tree, java.nio.file.Paths.get(normalized), false); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("path fetch-back from " + absRoot + " interrupted", e); } finally { - if (temp != null) { - deleteQuietly(temp); - } + deleteQuietly(temp); } } /** - * Write files from {@code incoming} onto {@code target}. Content is the authority: - * a differing sandbox file is always written, and mtime only decides whether the - * host copy must be preserved first. When {@code fullTree} is set the target is - * also scanned for files the sandbox no longer has, so a full fallback can - * propagate deletions instead of silently resurrecting them. + * Write files from {@code incoming} onto {@code target}. Content is the authority: a + * differing sandbox file is always written. The host copy is quarantined first when + * it changed since this transport last left it, and the write is skipped if that copy + * cannot be made. When {@code fullTree} is set, paths the sandbox held before but no + * longer has are propagated as deletions. */ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, boolean fullTree) - throws java.io.IOException { - if (!java.nio.file.Files.exists(incoming)) { + throws IOException { + if (!java.nio.file.Files.isDirectory(incoming)) { return 0; } + java.util.Set seen = seenBySandbox.computeIfAbsent(target.toString(), + key -> java.util.concurrent.ConcurrentHashMap.newKeySet()); + java.util.Set previous = fullTree ? new java.util.HashSet<>(seen) : null; + java.util.Set captured = new java.util.HashSet<>(); java.nio.file.Path store = quarantineStore(); int[] copied = { 0 }; try (java.util.stream.Stream walk = java.nio.file.Files.walk(incoming)) { - walk.filter(java.nio.file.Files::isRegularFile).filter(path -> !java.nio.file.Files.isSymbolicLink(path)) + walk.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)) .forEach(path -> { - java.nio.file.Path relative = incoming.relativize(path); - java.nio.file.Path destination = target.resolve(relative.toString()); + String relative = incoming.relativize(path).toString().replace('\\', '/'); + captured.add(relative); + if (!isWritableGitPath(relative)) { + classLogger.debug("Not writing back git control file {}", relative); + return; + } + java.nio.file.Path destination = containedHostPath(target, relative); + if (destination == null) { + classLogger.warn("Refusing write-back of {} under {}: path leaves the root or crosses a symlink", + relative, target); + return; + } try { - if (java.nio.file.Files.exists(destination)) { - if (sameContent(path, destination)) { + if (java.nio.file.Files.exists(destination, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + if (java.nio.file.Files.mismatch(path, destination) == -1L) { + recordStamp(destination); return; } - // Different bytes. A host copy newer than the sandbox copy may be a - // concurrent edit, so keep it rather than silently overwriting it. - if (java.nio.file.Files.getLastModifiedTime(destination) - .compareTo(java.nio.file.Files.getLastModifiedTime(path)) > 0) { - quarantineCopy(destination, target, store); + if (hostChangedSinceSync(destination) + && quarantineCopy(destination, target, store) == null) { + classLogger.warn("Write-back of {} skipped: the host copy changed and could not be preserved", + destination); + return; } } - java.nio.file.Path folder = destination.getParent(); - if (folder != null) { - java.nio.file.Files.createDirectories(folder); + java.nio.file.Files.createDirectories(destination.getParent()); + // Re-check now that the parents exist: creating them must not have + // walked through a link that appeared in between. + if (containedHostPath(target, relative) == null) { + classLogger.warn("Refusing write-back of {} under {}: path crosses a symlink", relative, + target); + return; } java.nio.file.Files.copy(path, destination, java.nio.file.StandardCopyOption.REPLACE_EXISTING, java.nio.file.StandardCopyOption.COPY_ATTRIBUTES); + recordStamp(destination); copied[0]++; - } catch (java.io.IOException e) { + } catch (IOException e) { classLogger.warn("Could not write back {}", destination, e); } }); } if (fullTree) { - copied[0] += quarantineStale(target, incoming, store); + copied[0] += removeDeletedFromHost(target, previous, captured, store); + // The capture is now the whole truth about what the sandbox holds. + seen.clear(); + seen.addAll(captured); + } else { + seen.addAll(captured); } if (copied[0] > 0) { classLogger.info("Fetch-back wrote {} file(s) into {}", copied[0], target); @@ -722,39 +754,83 @@ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, bo } /** - * A full fetch carries survivors, not tombstones. Any host file under the root that - * the sandbox capture no longer contains was deleted inside the sandbox; move it to - * quarantine rather than leaving it to reappear in the project. + * A full capture carries survivors, not tombstones. A path the sandbox held before and + * the capture no longer contains was deleted inside the sandbox: quarantine the host + * copy, then remove it. Host files the sandbox never held are left alone. */ - private static int quarantineStale(java.nio.file.Path target, java.nio.file.Path incoming, - java.nio.file.Path store) { - int[] moved = { 0 }; - try (java.util.stream.Stream walk = java.nio.file.Files.walk(target)) { - walk.filter(java.nio.file.Files::isRegularFile).filter(path -> !java.nio.file.Files.isSymbolicLink(path)) - .filter(path -> !isExcludedFromCapture(target.relativize(path))) - .forEach(path -> { - java.nio.file.Path relative = target.relativize(path); - if (java.nio.file.Files.exists(incoming.resolve(relative.toString()))) { - return; - } - java.nio.file.Path copy = quarantineCopy(path, target, store); - if (copy != null) { - try { - java.nio.file.Files.deleteIfExists(path); - pruneEmptyParents(path, target); - moved[0]++; - } catch (java.io.IOException e) { - classLogger.warn("Could not remove {} after quarantining it", path, e); - } - } - }); - } catch (java.io.IOException e) { - classLogger.warn("Could not scan {} for deletions", target, e); + private int removeDeletedFromHost(java.nio.file.Path target, java.util.Set previous, + java.util.Set captured, java.nio.file.Path store) { + int moved = 0; + for (String relative : previous) { + if (captured.contains(relative) || !isWritableGitPath(relative) + || isExcludedFromCapture(java.nio.file.Paths.get(relative))) { + continue; + } + java.nio.file.Path file = containedHostPath(target, relative); + if (file == null || !java.nio.file.Files.isRegularFile(file, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + continue; + } + if (quarantineCopy(file, target, store) == null) { + classLogger.warn("Delete suppressed for {} because it could not be quarantined", file); + continue; + } + try { + java.nio.file.Files.deleteIfExists(file); + hostStamps.remove(file.toString()); + pruneEmptyParents(file, target); + moved++; + } catch (IOException e) { + classLogger.warn("Could not remove {} after quarantining it", file, e); + } + } + if (moved > 0) { + classLogger.info("Fetch-back moved {} deleted file(s) to {}", moved, store); } - if (moved[0] > 0) { - classLogger.info("Fetch-back moved {} deleted file(s) to {}", moved[0], store); + return moved; + } + + /** + * Remove a host file the sandbox reported as deleted, keeping a copy under the + * quarantine root. Ignored unless the sandbox is known to have held the path: a file + * created and removed inside one block must not take an unrelated host file with it. + * + * @param root transported root the path belongs to + * @param relative path relative to that root + * @return true when the host file was removed + */ + public boolean deleteFromHost(String root, String relative) { + java.nio.file.Path rootPath = java.nio.file.Paths.get(Utility.normalizePath(root)); + java.util.Set seen = seenBySandbox.get(rootPath.toString()); + if (seen == null || !seen.contains(relative)) { + classLogger.debug("Ignoring delete of {} under {}: the sandbox never held it", relative, root); + return false; + } + if (!isWritableGitPath(relative)) { + return false; + } + java.nio.file.Path file = containedHostPath(rootPath, relative); + if (file == null) { + classLogger.warn("Refusing delete of {} under {}: path leaves the root or crosses a symlink", relative, root); + return false; + } + if (!java.nio.file.Files.isRegularFile(file, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + seen.remove(relative); + return false; + } + if (quarantineCopy(file, rootPath, quarantineStore()) == null) { + classLogger.warn("Delete suppressed for {} because it could not be quarantined", file); + return false; + } + try { + java.nio.file.Files.deleteIfExists(file); + hostStamps.remove(file.toString()); + seen.remove(relative); + pruneEmptyParents(file, rootPath); + return true; + } catch (IOException e) { + classLogger.warn("Could not delete {} after quarantining it", file, e); + return false; } - return moved[0]; } /** @@ -769,12 +845,12 @@ private static void pruneEmptyParents(java.nio.file.Path file, java.nio.file.Pat if (children.findAny().isPresent()) { return; } - } catch (java.io.IOException e) { + } catch (IOException e) { return; } try { java.nio.file.Files.deleteIfExists(parent); - } catch (java.io.IOException e) { + } catch (IOException e) { return; } parent = parent.getParent(); @@ -796,6 +872,144 @@ private static boolean isExcludedFromCapture(java.nio.file.Path relative) { return false; } + /** + * Inside a {@code .git} directory only git's data is written back: objects, refs, + * reflogs, HEAD and the index -- what a commit made in the sandbox produces. + * Everything else there (config, hooks, info, alternates) is code or configuration + * that host-side git runs or obeys, so a sandbox able to write it could execute + * commands on the server the next time the platform touches the repository. A + * {@code .git} file is refused too: it redirects the repository elsewhere. + */ + static boolean isWritableGitPath(String relative) { + String[] parts = relative.split("/"); + for (int i = 0; i < parts.length; i++) { + if (!".git".equals(parts[i])) { + continue; + } + if (i == parts.length - 1) { + return false; + } + String child = parts[i + 1]; + if (i + 2 == parts.length) { + return GIT_DATA_FILES.contains(child); + } + if ("objects".equals(child)) { + return !"info".equals(parts[i + 2]); + } + return "refs".equals(child) || "logs".equals(child); + } + return true; + } + + private static final java.util.Set GIT_DATA_FILES = java.util.Set.of("HEAD", "ORIG_HEAD", "FETCH_HEAD", + "MERGE_HEAD", "index", "packed-refs", "COMMIT_EDITMSG"); + + /** A relative path with no absolute prefix and no {@code ..} segment. */ + static boolean isSafeRelative(String relative) { + if (relative == null || relative.isEmpty() || relative.startsWith("/") || relative.indexOf('\0') >= 0) { + return false; + } + for (String part : relative.split("/")) { + if ("..".equals(part)) { + return false; + } + } + return true; + } + + /** + * Resolve {@code relative} under {@code root} for a host-side write or delete, or null + * when it would leave the root. No component between the root and the file may be a + * symlink: a link already in a writable project -- committed to its repository, say -- + * would otherwise redirect the write, or the delete, into another user's space. + */ + static java.nio.file.Path containedHostPath(java.nio.file.Path root, String relative) { + if (!isSafeRelative(relative)) { + return null; + } + java.nio.file.Path candidate = root.resolve(relative).normalize(); + if (!candidate.startsWith(root) || candidate.equals(root)) { + return null; + } + java.nio.file.Path current = root; + for (java.nio.file.Path part : root.relativize(candidate)) { + current = current.resolve(part); + if (java.nio.file.Files.isSymbolicLink(current)) { + return null; + } + if (!java.nio.file.Files.exists(current, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + break; + } + } + return candidate; + } + + /** Remember a host file's state as this transport left it. */ + private void recordStamp(java.nio.file.Path file) { + String stamp = stampOf(file); + if (stamp != null) { + hostStamps.put(file.toString(), stamp); + } + } + + /** True unless the host file is exactly as this transport last left it. */ + private boolean hostChangedSinceSync(java.nio.file.Path file) { + String known = hostStamps.get(file.toString()); + return known == null || !known.equals(stampOf(file)); + } + + private static String stampOf(java.nio.file.Path file) { + try { + java.nio.file.attribute.BasicFileAttributes attributes = java.nio.file.Files.readAttributes(file, + java.nio.file.attribute.BasicFileAttributes.class, java.nio.file.LinkOption.NOFOLLOW_LINKS); + return attributes.size() + ":" + attributes.lastModifiedTime().toMillis(); + } catch (IOException e) { + return null; + } + } + + /** + * Record the state of every file in a root just uploaded, so that write-back can tell + * a host file nobody touched (overwrite it) from one edited on the host since + * (quarantine it first). + */ + private void recordUploadedStamps(java.nio.file.Path root) { + if (!java.nio.file.Files.isDirectory(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + if (java.nio.file.Files.isRegularFile(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + recordStamp(root); + } + return; + } + try { + java.nio.file.Files.walkFileTree(root, new java.nio.file.SimpleFileVisitor() { + @Override + public java.nio.file.FileVisitResult preVisitDirectory(java.nio.file.Path dir, + java.nio.file.attribute.BasicFileAttributes attrs) { + String name = dir.getFileName() == null ? "" : dir.getFileName().toString(); + return isExcludedFromCapture(java.nio.file.Paths.get(name)) + ? java.nio.file.FileVisitResult.SKIP_SUBTREE + : java.nio.file.FileVisitResult.CONTINUE; + } + + @Override + public java.nio.file.FileVisitResult visitFile(java.nio.file.Path file, + java.nio.file.attribute.BasicFileAttributes attrs) { + if (attrs.isRegularFile()) { + hostStamps.put(file.toString(), attrs.size() + ":" + attrs.lastModifiedTime().toMillis()); + } + return java.nio.file.FileVisitResult.CONTINUE; + } + + @Override + public java.nio.file.FileVisitResult visitFileFailed(java.nio.file.Path file, IOException e) { + return java.nio.file.FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + classLogger.debug("Could not record upload state for {}", root, e); + } + } + /** Timestamped quarantine root, outside every transported root so it is never pushed. */ private static java.nio.file.Path quarantineStore() { String base = Utility.getBaseFolder(); @@ -814,7 +1028,7 @@ private static java.nio.file.Path quarantineCopy(java.nio.file.Path file, java.n .resolve(relative.toString()); java.nio.file.Files.createDirectories(destination.getParent()); java.nio.file.Files.copy(file, destination, java.nio.file.StandardCopyOption.REPLACE_EXISTING, - java.nio.file.StandardCopyOption.COPY_ATTRIBUTES); + java.nio.file.StandardCopyOption.COPY_ATTRIBUTES, java.nio.file.LinkOption.NOFOLLOW_LINKS); classLogger.warn("Write-back preserved the host copy of {} at {}", file, destination); return destination; } catch (Exception e) { @@ -824,140 +1038,141 @@ private static java.nio.file.Path quarantineCopy(java.nio.file.Path file, java.n } /** - * Remove a host file the sandbox deleted, keeping a copy under the quarantine root. - * - * @param path absolute host path reported as deleted - * @param root transported root the path belongs to + * Refuse an archive whose listing cannot be read, or that names an absolute path or + * a {@code ..} segment. Checked per segment, so a file called {@code v1..v2.diff} is + * fine. */ - public static void quarantineAndDelete(String path, String root) { - java.nio.file.Path file = java.nio.file.Paths.get(path); - if (!java.nio.file.Files.exists(file)) { - return; - } - if (quarantineCopy(file, java.nio.file.Paths.get(Utility.normalizePath(root)), quarantineStore()) == null) { - classLogger.warn("Delete suppressed for {} because it could not be quarantined", path); - return; - } - try { - java.nio.file.Files.deleteIfExists(file); - } catch (java.io.IOException e) { - classLogger.warn("Could not delete {} after quarantining it", path, e); + private static void requireSafeArchive(java.nio.file.Path archive) throws IOException, InterruptedException { + String listing = runLocal(Arrays.asList("tar", "-tf", archive.toString()), 60); + for (String line : listing.split("\\R")) { + String entry = line.trim(); + if (entry.isEmpty()) { + continue; + } + if (entry.endsWith("/")) { + entry = entry.substring(0, entry.length() - 1); + } + if (entry.isEmpty() || !isSafeRelative(entry)) { + throw new IOException("archive contains an absolute or traversing path: " + line); + } } } - private static boolean sameContent(java.nio.file.Path left, java.nio.file.Path right) throws java.io.IOException { - if (java.nio.file.Files.size(left) != java.nio.file.Files.size(right)) { - return false; - } - return java.util.Arrays.equals(java.nio.file.Files.readAllBytes(left), java.nio.file.Files.readAllBytes(right)); + /** + * Extract, then give the owner access to everything extracted: a directory the + * sandbox left at mode 000 would otherwise be unreadable to the reconcile walk and + * undeletable afterwards. + */ + private static void extractArchive(java.nio.file.Path archive, java.nio.file.Path destination) + throws IOException, InterruptedException { + runLocal(Arrays.asList("tar", "-xf", archive.toString(), "-C", destination.toString(), "--no-same-owner"), + 120); + makeOwnerAccessible(destination); } - private static boolean archiveEntriesAreSafe(byte[] archive) { + private static void makeOwnerAccessible(java.nio.file.Path root) { try { - Process process = new ProcessBuilder("tar", "-tf", "-") - .redirectErrorStream(true).start(); - Thread writer = new Thread(() -> { - try (java.io.OutputStream stdin = process.getOutputStream()) { - stdin.write(archive); - } catch (java.io.IOException ignored) { - // process may exit early; the exit code carries the outcome - } - }); - writer.start(); - String listing = new String(process.getInputStream().readAllBytes(), - java.nio.charset.StandardCharsets.UTF_8); - writer.join(5000); - process.waitFor(20, TimeUnit.SECONDS); - for (String line : listing.split("\\R")) { - String entry = line.trim(); - if (entry.isEmpty()) { - continue; - } - if (entry.startsWith("/") || entry.contains("..")) { - return false; - } - } - return true; + runLocal(Arrays.asList("chmod", "-R", "u+rwX", root.toString()), 60); } catch (Exception e) { - return false; - } - } - - private static void extractArchive(byte[] archive, java.nio.file.Path destination) throws Exception { - Process process = new ProcessBuilder("tar", "-xf", "-", "-C", destination.toString()) - .redirectErrorStream(true).start(); - Thread writer = new Thread(() -> { - try (java.io.OutputStream stdin = process.getOutputStream()) { - stdin.write(archive); - } catch (java.io.IOException ignored) { - // see above - } - }); - writer.start(); - String output = new String(process.getInputStream().readAllBytes(), - java.nio.charset.StandardCharsets.UTF_8); - writer.join(10_000); - if (!process.waitFor(60, TimeUnit.SECONDS)) { - process.destroyForcibly(); - throw new java.io.IOException("tar extraction timed out"); - } - if (process.exitValue() != 0) { - throw new java.io.IOException("tar extraction failed: " + output.trim()); + classLogger.debug("Could not normalise permissions under {}", root, e); } } private static void deleteQuietly(java.nio.file.Path root) { + if (root == null || !java.nio.file.Files.exists(root)) { + return; + } + makeOwnerAccessible(root); try (java.util.stream.Stream walk = java.nio.file.Files.walk(root)) { walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { try { java.nio.file.Files.deleteIfExists(path); - } catch (java.io.IOException ignored) { + } catch (IOException ignored) { // best effort } }); - } catch (java.io.IOException ignored) { + } catch (IOException ignored) { // best effort } } - /** - * Run a CLI command and return its raw stdout. {@link #run(List, int)} decodes to - * String and would corrupt any binary payload, which is exactly what a tar stream is. - */ - private byte[] runBytes(List cmd, int timeoutSeconds) throws IOException, InterruptedException { + /** Run a local helper (tar, chmod) and return its output; a non-zero exit is an error. */ + private static String runLocal(List cmd, int timeoutSeconds) throws IOException, InterruptedException { ProcessBuilder builder = new ProcessBuilder(cmd); - builder.environment().putAll(cliEnv()); - builder.redirectErrorStream(false); + builder.redirectErrorStream(true); builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); Process process = builder.start(); java.io.ByteArrayOutputStream collected = new java.io.ByteArrayOutputStream(); Thread reader = new Thread(() -> { try (java.io.InputStream stream = process.getInputStream()) { stream.transferTo(collected); - } catch (java.io.IOException ignored) { + } catch (IOException ignored) { // the exit code carries the outcome } - }, "openshell-fetchback-reader"); + }, "openshell-local-" + cmd.get(0)); reader.setDaemon(true); reader.start(); - Thread errors = new Thread(() -> { - try (java.io.InputStream stream = process.getErrorStream()) { - stream.transferTo(java.io.OutputStream.nullOutputStream()); - } catch (java.io.IOException ignored) { - // discarded - } - }, "openshell-fetchback-errors"); - errors.setDaemon(true); - errors.start(); if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { process.destroyForcibly(); throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); } reader.join(5000); + String output = collected.toString(java.nio.charset.StandardCharsets.UTF_8); + if (process.exitValue() != 0) { + throw new IOException("command failed (" + process.exitValue() + "): " + String.join(" ", cmd) + "\n" + + output.trim()); + } + return output; + } + + /** + * Run a CLI command and stream its raw stdout into {@code out}, stopping it as soon as + * the output passes {@link #MAX_FETCH_BYTES}. The stream goes to disk rather than the + * heap: one session's capture must never be able to exhaust the memory of the JVM + * every other session shares. + * + * @return bytes written + */ + private long runToFile(List cmd, int timeoutSeconds, java.nio.file.Path out) + throws IOException, InterruptedException { + ProcessBuilder builder = new ProcessBuilder(cmd); + builder.environment().putAll(cliEnv()); + builder.redirectError(ProcessBuilder.Redirect.DISCARD); + builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); + Process process = builder.start(); + java.util.concurrent.atomic.AtomicLong written = new java.util.concurrent.atomic.AtomicLong(); + java.util.concurrent.atomic.AtomicBoolean overCap = new java.util.concurrent.atomic.AtomicBoolean(); + Thread reader = new Thread(() -> { + byte[] buffer = new byte[64 * 1024]; + try (java.io.InputStream stream = process.getInputStream(); + java.io.OutputStream sink = java.nio.file.Files.newOutputStream(out)) { + int read; + while ((read = stream.read(buffer)) > 0) { + if (written.addAndGet(read) > MAX_FETCH_BYTES) { + overCap.set(true); + process.destroyForcibly(); + return; + } + sink.write(buffer, 0, read); + } + } catch (IOException ignored) { + // the exit code carries the outcome + } + }, "openshell-fetchback-reader"); + reader.setDaemon(true); + reader.start(); + if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); + } + reader.join(10_000); + if (overCap.get()) { + throw new IOException("capture passed the " + MAX_FETCH_BYTES + " byte cap: " + String.join(" ", cmd)); + } if (process.exitValue() != 0) { throw new IOException("command failed (" + process.exitValue() + "): " + String.join(" ", cmd)); } - return collected.toByteArray(); + return written.get(); } /** diff --git a/src/prerna/util/openshell/OpenShellShellRunner.java b/src/prerna/util/openshell/OpenShellShellRunner.java index f7242760ca..84ad496299 100644 --- a/src/prerna/util/openshell/OpenShellShellRunner.java +++ b/src/prerna/util/openshell/OpenShellShellRunner.java @@ -29,6 +29,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -64,6 +66,8 @@ public final class OpenShellShellRunner { private static final int DEFAULT_TIMEOUT_SECONDS = 60; + /** Most output returned to the terminal for one command. */ + private static final int MAX_OUTPUT_BYTES = 4 * 1024 * 1024; private static final String CLI = "openshell"; private OpenShellShellRunner() { @@ -120,15 +124,17 @@ public static Result execute(User user, String workingDir, String command) { if (target.equals("~")) { target = "/"; } - String resolved = runInSandbox(sandboxName, dir, "cd " + target + " && pwd"); - if (resolved == null || resolved.trim().isEmpty() || resolved.startsWith("__ERROR__")) { - return new Result(resolved == null ? "Directory doesn't exist: " + target - : resolved.replace("__ERROR__", ""), dir); + Outcome resolved = runInSandbox(sandboxName, dir, "cd " + target + " && pwd"); + String newDir = firstLine(resolved.output); + // Only a clean exit with an absolute path is a directory change; anything else + // is an error message and the working directory stays where it was. + if (resolved.exitCode != 0 || !newDir.startsWith("/")) { + return new Result(resolved.output.isEmpty() ? "Directory doesn't exist: " + target + : resolved.output, dir); } - String newDir = firstLine(resolved); return new Result(newDir, newDir); } - String output = runInSandbox(sandboxName, dir, trimmed); + String output = runInSandbox(sandboxName, dir, trimmed).output; // Shell writes are not requests on the python socket, so they do not pass the // fetch-back hook there. Reconcile here too -- this is the path a `git commit` // or a `cat > file` takes, and losing that work is exactly what we are fixing. @@ -155,6 +161,17 @@ private static String firstLine(String text) { return (newline < 0 ? text : text.substring(0, newline)).trim(); } + /** Exit code and (capped) combined output of one sandbox command. */ + private static final class Outcome { + private final int exitCode; + private final String output; + + Outcome(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + } + /** * One `openshell sandbox exec` invocation. * @@ -164,33 +181,52 @@ private static String firstLine(String text) { * image. The working directory is applied by the shell itself rather than by * {@code --workdir}, because the directory may be a path that only exists in * the sandbox after an earlier upload. + * + *

+ * Output goes to a temp file, not a pipe: a pipe nobody reads while waiting fills at + * 64 KB and blocks the command until the timeout, losing everything it printed. */ - private static String runInSandbox(String sandboxName, String workingDir, String command) throws IOException { - List argv = new ArrayList<>(Arrays.asList(CLI, "sandbox", "exec", "-n", sandboxName, "--", + private static Outcome runInSandbox(String sandboxName, String workingDir, String command) throws IOException { + List argv = new ArrayList<>(Arrays.asList(cliPath(), "sandbox", "exec", "-n", sandboxName, "--", "bash", "-lc", "cd " + shellQuote(workingDir) + " 2>/dev/null; " + command)); - ProcessBuilder builder = new ProcessBuilder(argv); - builder.redirectErrorStream(true); - builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); - // The CLI needs the same environment the transport uses (gateway config, - // TLS bundle); inherit it rather than reconstructing it here. - Process process = builder.start(); - String output; + Path capture = Files.createTempFile("openshell-shell-", ".out"); try { - if (!process.waitFor(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + ProcessBuilder builder = new ProcessBuilder(argv); + builder.redirectErrorStream(true); + builder.redirectOutput(capture.toFile()); + builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); + // The CLI needs the same environment the transport uses (gateway config, + // TLS bundle); inherit it rather than reconstructing it here. + Process process = builder.start(); + try { + if (!process.waitFor(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + process.destroyForcibly(); + return new Outcome(-1, readCapped(capture) + "\ncommand timed out after " + DEFAULT_TIMEOUT_SECONDS + + "s"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); process.destroyForcibly(); - return "__ERROR__command timed out after " + DEFAULT_TIMEOUT_SECONDS + "s"; + return new Outcome(-1, "interrupted"); } - output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - process.destroyForcibly(); - return "__ERROR__interrupted"; + return new Outcome(process.exitValue(), readCapped(capture)); + } finally { + Files.deleteIfExists(capture); + } + } + + /** The command's output, cut at {@link #MAX_OUTPUT_BYTES} with a note saying so. */ + private static String readCapped(Path capture) throws IOException { + long size = Files.size(capture); + byte[] bytes; + try (java.io.InputStream in = Files.newInputStream(capture)) { + bytes = in.readNBytes((int) Math.min(size, MAX_OUTPUT_BYTES)); } - String cleaned = output.replace("\\r", "").trim(); - if (process.exitValue() != 0 && !cleaned.isEmpty()) { - return cleaned; + String text = new String(bytes, StandardCharsets.UTF_8).replace("\r", "").trim(); + if (size > MAX_OUTPUT_BYTES) { + text += "\n[output truncated: " + size + " bytes, showing the first " + MAX_OUTPUT_BYTES + "]"; } - return cleaned; + return text; } private static String shellQuote(String value) { From c60a9435e703fb453879c74a3281eefa0af33c88 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 19:43:57 +0000 Subject: [PATCH 35/56] fix(openshell): keep the change watcher honest about what it can see - Use poll() instead of select(): select raises on any fd >= 1024 and the watcher thread exited silently while still reporting full coverage. - Report coverage loss when the loop thread is dead, when a watch cannot be added (ENOSPC, EACCES), or when a directory cannot be listed. - A scan that cannot list part of the tree no longer reports the unlisted files as deleted (a mode-000 directory read as "everything removed"). - Drain queued events inside snapshot(), so a write finished just before CHANGES belongs to that block. - Re-evaluate roots on every CHANGES and force a baseline when one is added: room/ and user/ appear after the first call and were never watched. - Prune ignored directories when adopting a new one (node_modules, .git). Co-Authored-By: Claude Opus 5.5 --- py/gaas_change_watcher.py | 98 +++++++++++++++++++++++++++-------- py/gaas_tcp_server_handler.py | 13 ++--- 2 files changed, 84 insertions(+), 27 deletions(-) diff --git a/py/gaas_change_watcher.py b/py/gaas_change_watcher.py index e726006acb..3c4c463ece 100644 --- a/py/gaas_change_watcher.py +++ b/py/gaas_change_watcher.py @@ -33,10 +33,10 @@ import threading import time -# Directories that are never watched, scanned, or pulled back. `.git` is static inside -# the sandbox (the image has no git binary) yet it was 112 directories and 137 of 256 -# files in a real project - over half the watch set and over half of every scan, for -# data that cannot move. `node_modules` is worse: one install adds thousands of +# Directories that are never watched, scanned, or pulled back. `.git` was 112 directories +# and 137 of 256 files in a real project - over half the watch set and over half of every +# scan. A commit made in the sandbox still reaches the host: the host's full capture +# carries git's data files. `node_modules` is worse: one install adds thousands of # directories, exhausts fs.inotify.max_user_watches and floods the event queue. _DEFAULT_IGNORE = ("node_modules", "__pycache__", ".venv", ".git") _env_ignore = os.environ.get("SEMOSS_SYNC_IGNORE", "").strip() @@ -81,6 +81,9 @@ def __init__(self, roots, max_changes=2000): self._thread = None self._stop = threading.Event() self._lock = threading.Lock() + # Serialises reads of the inotify fd between the loop thread and snapshot(), so + # events are consumed in the order the kernel queued them. + self._read_lock = threading.Lock() self._changes = {} self._overflow = False self._baseline_pending = True @@ -133,15 +136,25 @@ def _arm(self, directory): try: watch = self._libc.inotify_add_watch(self._fd, directory.encode("utf-8"), _MASK) except Exception: + self._coverage_lost = True return False if watch >= 0: self._watched[watch] = directory self._armed += 1 return True + # A directory removed between the walk and the arm is not a gap. Anything else + # (ENOSPC at max_user_watches, EACCES on a mode-000 directory) leaves part of the + # tree unwatched, and silence there would read as "nothing changed". + if ctypes.get_errno() != errno.ENOENT: + self._coverage_lost = True return False def add_roots(self, roots): - """Arm roots discovered after start (a session materialises its tree late).""" + """Arm roots discovered after start (a session materialises its tree late). + + Whatever was uploaded into a new root arrived before its watch, so the next + snapshot is a baseline and the host takes a full capture. + """ added = 0 for root in roots: if root in self.roots or not os.path.isdir(root): @@ -149,6 +162,9 @@ def add_roots(self, roots): self.roots.append(root) self._arm_tree(root) added += 1 + if added: + with self._lock: + self._baseline_pending = True return added def _arm_tree(self, root): @@ -156,28 +172,50 @@ def _arm_tree(self, root): self._root_ino[root] = os.stat(root).st_ino except OSError: pass - for current, dirs, _files in os.walk(root): + for current, dirs, _files in os.walk(root, onerror=self._walk_failed): # Prune in place so os.walk never descends into an ignored tree at all. dirs[:] = [name for name in dirs if name not in _IGNORED_DIRS] self._arm(current) + def _walk_failed(self, _error): + # An unreadable directory cannot be watched or listed; treat it as a gap. + self._coverage_lost = True + # -- event loop -------------------------------------------------------- def _loop(self): + # poll, not select: select() raises on any fd >= 1024, which a worker with many + # open files reaches, and the watcher would stop while still claiming coverage. + poller = select.poll() + poller.register(self._fd, select.POLLIN) while not self._stop.is_set(): try: - ready, _, _ = select.select([self._fd], [], [], 0.2) + ready = poller.poll(200) + except InterruptedError: + continue except (OSError, ValueError): break - if not ready: - continue - try: - data = os.read(self._fd, 65536) - except OSError as exc: - if exc.errno in (errno.EAGAIN, errno.EWOULDBLOCK): - continue + if ready and not self._drain(): break - self._consume(data) + + def _drain(self): + """Consume every event queued on the fd. Returns False if the fd is unusable.""" + with self._read_lock: + while True: + try: + data = os.read(self._fd, 65536) + except OSError as exc: + if exc.errno in (errno.EAGAIN, errno.EWOULDBLOCK): + return True + if exc.errno == errno.EINTR: + continue + return False + if not data: + return True + self._consume(data) + + def alive(self): + return self._thread is not None and self._thread.is_alive() def _consume(self, data): offset = 0 @@ -232,7 +270,8 @@ def _adopt_directory(self, path): return self._arm_tree(path) with self._lock: - for current, _dirs, files in os.walk(path): + for current, dirs, files in os.walk(path, onerror=self._walk_failed): + dirs[:] = [name for name in dirs if name not in _IGNORED_DIRS] for name in files: self._record(os.path.join(current, name), "modify") @@ -260,8 +299,9 @@ def _scan_tuples(self): """ index = {} reported = 0 + failures = [] for root in list(self.roots): - for base, dirs, files in os.walk(root): + for base, dirs, files in os.walk(root, onerror=failures.append): dirs[:] = [name for name in dirs if name not in _IGNORED_DIRS] for name in files: path = os.path.join(base, name) @@ -277,10 +317,18 @@ def _scan_tuples(self): if self._stat_index and self._stat_index.get(path) != key: self._record(path, "modify") reported += 1 - for path in self._stat_index: - if path not in index: - self._record(path, "delete") - reported += 1 + if failures: + # Part of the tree could not be listed, so a missing path proves nothing: a + # mode-000 directory must not read as "all of its files were deleted". Keep + # the old entries and make the host take a capture that knows it is partial. + self._coverage_lost = True + for path, key in self._stat_index.items(): + index.setdefault(path, key) + else: + for path in self._stat_index: + if path not in index: + self._record(path, "delete") + reported += 1 self._stat_index = index return reported @@ -292,7 +340,15 @@ def snapshot(self): host materialised it, so reporting it would be noise. """ rearm = False + # Pick up events the kernel has queued but the loop thread has not read yet, so + # a write that finished just before this call belongs to this block. + if self._fd is not None and not self._drain(): + with self._lock: + self._coverage_lost = True with self._lock: + if not self.alive(): + # The loop thread is gone: nothing is being watched any more. + self._coverage_lost = True now = time.monotonic() # Always scan on the first call (seed), after a lost baseline/coverage, and # otherwise at most once per interval. Skipping never drops a change: the diff --git a/py/gaas_tcp_server_handler.py b/py/gaas_tcp_server_handler.py index 9726909ff3..5ec5afbecc 100644 --- a/py/gaas_tcp_server_handler.py +++ b/py/gaas_tcp_server_handler.py @@ -449,13 +449,14 @@ def handle_changes(self): path forever. Overflow, or a set past the cap, is reported so the host falls back to a full reconcile instead of trusting a partial list. """ - if self.change_watcher is None: - try: - from gaas_change_watcher import shared_watcher + try: + from gaas_change_watcher import shared_watcher - self.change_watcher = shared_watcher(self._watcher_roots()) - except Exception: - self.change_watcher = None + # Re-evaluated on every call: room/ and user/ appear only when the session + # first touches one, which is usually after the first CHANGES. + self.change_watcher = shared_watcher(self._watcher_roots()) + except Exception: + self.change_watcher = None if self.change_watcher is None: snapshot = { "changes": [], From cb4bf047c1d8de6c5f6f58c9b69cfb8558d6e0ae Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 19:43:57 +0000 Subject: [PATCH 36/56] fix(openshell): treat a capture tar could not fully read as partial Full captures run with --ignore-failed-read so one unreadable directory no longer fails the root on every block. Anything tar reports on stderr marks the capture partial: files are written back, deletions are not inferred. Co-Authored-By: Claude Opus 5.5 --- src/prerna/util/OpenShellTransport.java | 70 ++++++++++++++++++++----- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 4f57fa5548..d00d960922 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -588,16 +588,24 @@ public int fetchBack(String absPath) throws IOException { // git's data files are ever written back -- see isWritableGitPath. Regenerable // caches are skipped. java.nio.file.Path archive = temp.resolve("capture.tar"); - long size = runToFile(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", parent, - "--exclude=node_modules", "--exclude=__pycache__", "--exclude=.venv", "-cf", "-", base), 120, - archive); - if (size == 0) { + Capture capture = runToFile(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", + parent, "--ignore-failed-read", "--exclude=node_modules", "--exclude=__pycache__", "--exclude=.venv", + "-cf", "-", base), 120, archive); + if (capture.bytes == 0) { return 0; } requireSafeArchive(archive); java.nio.file.Path tree = java.nio.file.Files.createDirectory(temp.resolve("tree")); extractArchive(archive, tree); - return reconcile(tree.resolve(base), java.nio.file.Paths.get(normalized), true); + // tar warns about anything it could not read (a mode-000 directory, say). What + // it skipped is missing from the capture without having been deleted, so a + // partial capture writes files back but infers no deletions. + boolean complete = capture.warnings.isEmpty(); + if (!complete) { + classLogger.warn("Capture of {} is partial; not propagating deletions this time: {}", absPath, + capture.warnings); + } + return reconcile(tree.resolve(base), java.nio.file.Paths.get(normalized), complete); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException("fetch-back of " + absPath + " interrupted", e); @@ -659,8 +667,7 @@ public int fetchBackPaths(String absRoot, java.util.List relativePaths) "--exclude=__pycache__", "--exclude=.venv", "-cf", "-", "--")); command.addAll(safe); java.nio.file.Path archive = temp.resolve("capture.tar"); - long size = runToFile(command, 120, archive); - if (size == 0) { + if (runToFile(command, 120, archive).bytes == 0) { return 0; } requireSafeArchive(archive); @@ -679,8 +686,9 @@ public int fetchBackPaths(String absRoot, java.util.List relativePaths) * Write files from {@code incoming} onto {@code target}. Content is the authority: a * differing sandbox file is always written. The host copy is quarantined first when * it changed since this transport last left it, and the write is skipped if that copy - * cannot be made. When {@code fullTree} is set, paths the sandbox held before but no - * longer has are propagated as deletions. + * cannot be made. {@code fullTree} means {@code incoming} is a complete capture of the + * root: paths the sandbox held before but no longer has are then propagated as + * deletions. */ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, boolean fullTree) throws IOException { @@ -1131,15 +1139,32 @@ private static String runLocal(List cmd, int timeoutSeconds) throws IOEx * heap: one session's capture must never be able to exhaust the memory of the JVM * every other session shares. * - * @return bytes written + * @return bytes written, and whatever the command wrote to stderr */ - private long runToFile(List cmd, int timeoutSeconds, java.nio.file.Path out) + private Capture runToFile(List cmd, int timeoutSeconds, java.nio.file.Path out) throws IOException, InterruptedException { ProcessBuilder builder = new ProcessBuilder(cmd); builder.environment().putAll(cliEnv()); - builder.redirectError(ProcessBuilder.Redirect.DISCARD); builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); Process process = builder.start(); + StringBuilder errors = new StringBuilder(); + Thread errorReader = new Thread(() -> { + try (java.io.BufferedReader reader = new java.io.BufferedReader( + new java.io.InputStreamReader(process.getErrorStream(), java.nio.charset.StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + synchronized (errors) { + if (errors.length() < 4096) { + errors.append(line).append('\n'); + } + } + } + } catch (IOException ignored) { + // the exit code carries the outcome + } + }, "openshell-fetchback-errors"); + errorReader.setDaemon(true); + errorReader.start(); java.util.concurrent.atomic.AtomicLong written = new java.util.concurrent.atomic.AtomicLong(); java.util.concurrent.atomic.AtomicBoolean overCap = new java.util.concurrent.atomic.AtomicBoolean(); Thread reader = new Thread(() -> { @@ -1166,13 +1191,30 @@ private long runToFile(List cmd, int timeoutSeconds, java.nio.file.Path throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); } reader.join(10_000); + errorReader.join(2000); + String warnings; + synchronized (errors) { + warnings = errors.toString().trim(); + } if (overCap.get()) { throw new IOException("capture passed the " + MAX_FETCH_BYTES + " byte cap: " + String.join(" ", cmd)); } if (process.exitValue() != 0) { - throw new IOException("command failed (" + process.exitValue() + "): " + String.join(" ", cmd)); + throw new IOException("command failed (" + process.exitValue() + "): " + String.join(" ", cmd) + + (warnings.isEmpty() ? "" : "\n" + warnings)); + } + return new Capture(written.get(), warnings); + } + + /** Size of a streamed capture and what the command reported on stderr. */ + private static final class Capture { + private final long bytes; + private final String warnings; + + Capture(long bytes, String warnings) { + this.bytes = bytes; + this.warnings = warnings; } - return written.get(); } /** From 52b5e988b09af2cf5880f3d6e865691daac97ddd Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 19:45:25 +0000 Subject: [PATCH 37/56] fix(executor): put the slim image's worker where it runs, and prove it The slim build stages py/ into the context, so `COPY .` landed the worker at /opt/semosshome/py/py/. Copy py/ like the full image does. The smoke test now fails the build unless the image's worker tree hashes to the repo's and the worker's interpreter imports the change watcher. Co-Authored-By: Claude Opus 5.5 --- docker/Dockerfile.executor | 15 +++++++-------- docker/build-executor-image.sh | 22 +++++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docker/Dockerfile.executor b/docker/Dockerfile.executor index acb31e0345..f743f68874 100644 --- a/docker/Dockerfile.executor +++ b/docker/Dockerfile.executor @@ -1,12 +1,11 @@ # SEMOSS executor sandbox — the process that runs user code under OpenShell. # -# docker build -f docker/Dockerfile.executor -t semoss-executor:0.3 py -# # or, with the py-source label and a smoke test: -# docker/build-executor-image.sh +# docker/build-executor-image.sh # stages py/ into the context, labels, smoke-tests # -# The build context is py/ on purpose. The image must carry the worker source that is -# in this repo; earlier builds staged py/ into a separate context by hand, and the -# image silently kept running month-old python while the repo looked fixed. +# The image must carry the worker source that is in this repo; earlier builds staged +# py/ into a separate context by hand, and the image silently kept running month-old +# python while the repo looked fixed. The build script copies the committed py/ into +# the context and checks the result against the repo. FROM python:3.12-slim LABEL maintainer="semoss@semoss.org" @@ -28,8 +27,8 @@ RUN apt-get update \ # -- the real answer is a per-project venv or a manifest-driven install. RUN pip install --no-cache-dir pandas jsonpickle yfinance -# The worker source. Context root is py/, so this is the committed python, verbatim. -COPY . /opt/semosshome/py/ +# The worker source: the committed py/, staged into the context by the build script. +COPY py/ /opt/semosshome/py/ # The handler appends to {insight_folder}/log.txt during setup(). The grants folders are # the read-only / read-write drop points the policy model expects to exist. diff --git a/docker/build-executor-image.sh b/docker/build-executor-image.sh index cd022f8e3c..2f54cc61a7 100755 --- a/docker/build-executor-image.sh +++ b/docker/build-executor-image.sh @@ -46,7 +46,6 @@ ARGS=(build [ "$NO_CACHE" = "1" ] && ARGS+=(--no-cache) if [ "$MODE" = "slim" ]; then - # Context is py/ itself; the Dockerfile copies it verbatim. cp "$HERE/Dockerfile.executor" "$CTX/Dockerfile" cp -a "$REPO/py" "$CTX/py" ARGS+=(-f "$CTX/Dockerfile") @@ -63,13 +62,18 @@ docker "${ARGS[@]}" "$CTX" echo "built $TAG ($MODE) from py sha ${PY_SHA:0:12}" -# Prove what landed, rather than trusting the build log. The interpreter is checked by absolute -# path (what the worker uses) and again through the PATH the sandbox is created with. -docker run --rm --entrypoint sh "$TAG" -c ' +# Prove what landed, rather than trusting the build log: the worker tree in the image must +# hash to the repo's, and the worker's interpreter must import the watcher. Any failure +# fails the build. +docker run --rm --entrypoint sh -e EXPECT="$PY_SHA" "$TAG" -ec ' + cd /opt/semosshome/py + test -f gaas_tcp_socket_server.py || { echo "worker missing at /opt/semosshome/py" >&2; exit 1; } + GOT=$(find . -type f -name "*.py" -print0 | sort -z | xargs -0 sha256sum | sha256sum | cut -d" " -f1) + [ "$GOT" = "$EXPECT" ] || { echo "py tree in image ($GOT) differs from the repo ($EXPECT)" >&2; exit 1; } + P=/usr/lib/python/semossvenv/bin/python3; [ -x "$P" ] || P=$(command -v python3) + "$P" -c "import gaas_change_watcher" || { echo "$P cannot import the change watcher" >&2; exit 1; } + echo "worker tree matches repo; $P imports the watcher" git --version - /usr/lib/python/semossvenv/bin/python3 -c "import numpy, boto3; print(\"parity imports ok (venv)\")" 2>/dev/null || echo "no venv in this image (slim)" - env -i PATH=/usr/lib/python/semossvenv/bin:/usr/local/bin:/usr/bin:/bin python3 -c "import sys; print(\"python3 ->\", sys.executable)" 2>/dev/null || true - test -f /opt/semosshome/py/gaas_tcp_socket_server.py && echo "worker present" - printf "watcher ignore refs: "; grep -c "_IGNORED_DIRS" /opt/semosshome/py/gaas_change_watcher.py - printf "id: "; id -u + [ "$(id -u)" = 1000 ] || { echo "image does not run as uid 1000" >&2; exit 1; } ' +echo "smoke test passed for $TAG" From af08e3d69e69ca0a7a7a3b0b8f17753b4d2b3dde Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 19:51:43 +0000 Subject: [PATCH 38/56] chore(openshell): keep dev harnesses out of the jar; align the sandbox doc - Remove the transport and native-client harnesses (main() classes in src/, shipped in the production jar). Move the session-lifecycle harness to test/ and require -DRDF_MAP instead of defaulting to a host path. - Doc: describe write-back as it now behaves (owed full captures, stamped quarantine, held-path deletions, partial captures, symlink containment, .git data allowlist), the permission check actually used, every OPENSHELL_* key, and how the image build proves its contents. Drop the claim that a reaper runs and the host-specific commands and timings. Co-Authored-By: Claude Opus 5.5 --- docs/deployment/executor_sandbox.md | 112 +++++++------ .../util/OpenShellNativeClientHarness.java | 107 ------------- .../util/OpenShellTransportHarness.java | 148 ------------------ .../OpenShellSessionLifecycleHarness.java | 6 +- 4 files changed, 71 insertions(+), 302 deletions(-) delete mode 100644 src/prerna/util/OpenShellNativeClientHarness.java delete mode 100644 src/prerna/util/OpenShellTransportHarness.java rename {src => test}/prerna/util/openshell/OpenShellSessionLifecycleHarness.java (97%) diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index 22ef9a79c2..43642b7d96 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -50,11 +50,12 @@ connection; a second connection is accepted and then never served. ## Session lifecycle -- Created lazily on the first `Py()` or shell command (cold start ≈ 10–13 s). +- Created lazily on the first `Py()` or shell command; the first call pays the sandbox start + and the upload of the session's roots. - Deleted on logout, session timeout, and on disconnect, where `ClientProcessWrapper.shutdown` reconciles one last time before the sandbox disappears. -- A sandbox left behind after a crash is removed by `OpenShellSandboxReaper`, which only - touches labelled `pyw-*` sandboxes past an age threshold. +- Nothing removes a sandbox orphaned by a JVM crash yet. They carry the `OPENSHELL_LABEL` + label and a `pyw-` name, so `openshell sandbox list` finds them for manual deletion. ## Write-back pipeline @@ -66,22 +67,38 @@ Every `PYTHON` and `CMD` block ends with a sync. The steps, in order: 2. **Fail closed on doubt.** If the answer is missing, `overflow` is true, `baseline` is true, or `coverage` is false, the host falls back to a full capture of each root instead of trusting a partial list. + A change set the host drained but could not apply is owed as a full capture on the next + block, because the worker has already forgotten those paths. 3. **Pull only what changed.** Changed paths are tarred inside the sandbox under one - `sandbox exec` and extracted to a temporary directory on the host. -4. **Reconcile.** Content is the authority: a differing sandbox file is written, and `mtime` - only decides whether the host copy is preserved first. A host file newer than its sandbox - counterpart is copied to quarantine before being replaced. -5. **Deletes.** A path reported as deleted is quarantine-then-delete, and the delete is - suppressed if the quarantine copy fails. Directories emptied by those removals are pruned - up to (never including) the transported root. + `sandbox exec` and streamed to a temporary file on the host, stopping at a 256 MB cap. The + archive is refused if its listing fails or names an absolute path or a `..` segment. +4. **Reconcile.** Content is the authority: a differing sandbox file is always written. The + host copy is quarantined first if it changed since this session last left it (uploaded or + written back), and the write is skipped if that copy fails. Sandbox timestamps never + decide anything. +5. **Deletes.** Only a path the sandbox is known to have held — seen in a full capture or + written back since — is ever deleted on the host, so a file created and removed within one + block cannot take an unrelated host file with it. Deletes are quarantine-then-delete, + suppressed if the copy fails, and directories they empty are pruned up to (never including) + the transported root. + +Every host-side write and delete is contained: a path that leaves the root, or crosses a +symlink on the way, is refused. Without that, a link committed into a writable project could +redirect a write or a delete into another space. + +Inside `.git`, only git's data is written back — `objects/` (not `objects/info/`), `refs/`, +`logs/`, `HEAD`, `index`, `packed-refs` and a few state files — so a commit made in the sandbox +reaches the host. `config`, `hooks/` and `info/` are never written back: host-side git runs or +obeys them, so a sandbox able to write them could execute commands on the server. Quarantine lives at `$BASE/writeback-quarantine///`, outside `project/`, `room/` and `user/`, so it is never pushed to object storage. -A whole-root fallback carries survivors, not tombstones, so it diffs the host tree against the -capture and quarantines files the capture no longer contains. Directories the capture excludes -(`node_modules`, `__pycache__`, `.venv`) are skipped: “absent from the archive” there means -“never captured”, not “deleted”. +A full capture carries survivors, not tombstones. A path the sandbox held before and the +capture no longer contains was deleted there; a host file the sandbox never held — added on the +host mid-session, or never uploaded — is left alone. If `tar` reports anything it could not +read (a directory the user made unreadable, say), the capture is partial: files are written +back and no deletion is inferred. ## Change detection @@ -97,14 +114,19 @@ Two mechanisms, deliberately: the backlog. Coverage loss is a first-class outcome. `IN_MOVED_FROM`, `IN_MOVE_SELF`, `IN_IGNORED` and -`IN_UNMOUNT` all mean the path map may be stale, so the watcher reports `coverage: false`, -drops the tuple index and re-arms; the host takes one full capture for that block and returns -to the fast path afterwards. +`IN_UNMOUNT` all mean the path map may be stale; so does a watch that could not be added +(`max_user_watches` exhausted, an unreadable directory), a directory the scan could not list, +or a watcher thread that has stopped. In each case the watcher reports `coverage: false`, drops +the tuple index and re-arms, and the host takes one full capture for that block. A scan that +could not list part of the tree never reports the unlisted files as deleted. -Excluded from watching, scanning and pulling: `node_modules`, `__pycache__`, `.venv`, `.git`. -`.git` is static inside the sandbox and dominated a real project (112 directories, 137 of 256 -files); `node_modules` is one install away from exhausting `fs.inotify.max_user_watches`. -Override with `SEMOSS_SYNC_IGNORE`. +Roots are re-evaluated on every `CHANGES` request: a root that appears later in the session (a +room created mid-session, say) is armed and forces one baseline capture. + +Excluded from watching and scanning: `node_modules`, `__pycache__`, `.venv`, `.git`. +`.git` is a large share of a typical project's directories; sandbox commits still reach the +host through the full capture. `node_modules` is one install away from exhausting +`fs.inotify.max_user_watches`. Override with `SEMOSS_SYNC_IGNORE`. ## Who may write back @@ -112,7 +134,7 @@ The worker runs outside the platform's permission checks, so the host decides pe | Root | Rule | | --- | --- | -| `project/__` | `SecurityProjectUtils.getActualUserProjectPermission` must be `OWNER` or `EDIT` | +| `project/__` | `SecurityProjectUtils.userCanEditProject` — the same check that grants the project read-write | | `room/` | `ModelInferenceLogsUtils.doCheckRoomExistsForUser` — `ROOM.USER_ID` in the model-inference-logs database | | `user/Asset__` | the id must equal one of the caller's `User.getAssetProjectId(provider)` values | | anything else | refused | @@ -126,13 +148,14 @@ never reaches the host. The sandbox image is built from this repository, not from a staged copy: ```bash -docker/build-executor-image.sh # tag semoss-executor:dev -TAG=semoss-executor:0.1 docker/build-executor-image.sh +docker/build-executor-image.sh # slim, tag semoss-executor:dev +MODE=full TAG=semoss-executor:full docker/build-executor-image.sh ``` -The build context is `py/`, so the image carries exactly the committed worker; -`docker/Dockerfile.executor` assembles it. The tag is labelled with `org.semoss.py-sha`, a hash -of the python tree, so the image can be traced back to its source. +The script stages the committed `py/` into the build context, so the image carries exactly the +worker in this repository. The tag is labelled with `org.semoss.py-sha`, a hash of the python +tree, and the build fails unless the worker tree inside the image hashes to the repository's +and the worker's interpreter imports the change watcher. Two variants: @@ -167,29 +190,26 @@ be current — a stale image shows up as code that ignores your latest change. | Key | Meaning | | --- | --- | -| `SANDBOX_MODE` | `OPENSHELL` enables this path | +| `SANDBOX_MODE` | `OPENSHELL` enables this path (requires `CHROOT_ENABLE=true`, like every jail) | | `OPENSHELL_IMAGE` | image a new sandbox starts from | -| `OPENSHELL_POLICY` | Landlock paths and per-binary egress rules | -| `OPENSHELL_LABEL` | label used to identify sandboxes for the reaper | -| `OPENSHELL_WORKER_PORT` | worker port inside the sandbox | -| `OPENSHELL_READY_TIMEOUT_SECONDS` | cold-start budget | +| `OPENSHELL_TEMPLATE` | workload template to create from instead of an image | +| `OPENSHELL_POLICY` | Landlock paths and per-binary egress rules; required | +| `OPENSHELL_CLI` | path to the `openshell` CLI (default `openshell`) | +| `OPENSHELL_CLI_HOME` | `HOME` for CLI children, where the gateway registration and mTLS bundle live | +| `OPENSHELL_LABEL` | label applied to every executor sandbox (default `app=semoss-executor`) | +| `OPENSHELL_WORKER_PORT` | worker port inside the sandbox (default `7100`) | +| `OPENSHELL_READY_TIMEOUT_SECONDS` | cold-start budget (default `180`) | +| `OPENSHELL_KEEP_SANDBOX` | `true` leaves sandboxes in place on teardown, for debugging | ## Verifying a deployment -```bash -# the class actually deployed (never trust the build log) -cd /tmp && unzip -o -q /opt/tomcat/webapps/Monolith/WEB-INF/lib/semoss-*-shaded-dependencies.jar \ - 'prerna/om/ClientProcessWrapper.class' && javap -p prerna/om/ClientProcessWrapper.class | grep -E 'canWriteBack|quarantine' - -# a file written by a block reached the host -journalctl -u tomcat-semoss | grep -i "Fetch-back wrote" - -# the permission gate refuses a read-only member -sudo -u postgres psql -d semoss_security -c \ - "UPDATE projectpermission SET permission=3 WHERE projectid='' AND userid='';" -# write from Py(), confirm the host file never appears and "Refusing write-back" is logged, -# then restore permission=1 -``` +- **The class actually deployed.** Unpack `prerna/om/ClientProcessWrapper.class` from the + deployed `semoss-*-shaded-dependencies.jar` and check it with `javap -p` for + `fetchBackChangedPaths`; a build log is not proof. +- **A write reaches the host.** Write a file from `Py()` in a loaded project and look for + `Fetch-back wrote` in the server log and the file in the project folder. +- **The permission gate holds.** Give a test user read-only access to a project, write from + `Py()`, and confirm the host file never appears and `Refusing write-back` is logged. ## Known limits diff --git a/src/prerna/util/OpenShellNativeClientHarness.java b/src/prerna/util/OpenShellNativeClientHarness.java deleted file mode 100644 index cfc1d758c8..0000000000 --- a/src/prerna/util/OpenShellNativeClientHarness.java +++ /dev/null @@ -1,107 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.util; - -import prerna.tcp.PayloadStruct; -import prerna.tcp.client.NativePySocketClient; - -/** - * Drives the REAL client ({@link NativePySocketClient}) end to end against a sandbox - * created by {@link OpenShellTransport} -- connect, send, and read the reply. - * - *

- * An earlier version only proved the handshake ({@code ready=true}). This one sends an - * actual {@link PayloadStruct} and waits for the response, which is the operation - * {@code PyTranslator} performs and the one the live pixel fails on. It carries no - * protocol code of its own: everything goes through the class that ships, because - * debugging through a hand-rolled protocol harness meant repeatedly mistaking a - * failing test client for a failing transport. - * - *

- * Usage: {@code java ... prerna.util.OpenShellNativeClientHarness [sessionKey]} - */ -public final class OpenShellNativeClientHarness { - - public static void main(String[] args) throws Exception { - String rdf = System.getProperty("RDF_MAP", "/opt/semosshome/RDF_Map.prop"); - DIHelper.getInstance().loadCoreProp(rdf); - String sessionKey = args.length > 0 ? args[0] : "native-client-1"; - - OpenShellTransport t = OpenShellTransport.forSession(sessionKey); - int port = t.start(); - System.out.println("transport up on port " + port); - - // Wait for the worker WITHOUT touching its port: a probe connect would consume - // the single connection slot and its disconnect would kill the worker. - System.out.println("settling 5s..."); - Thread.sleep(5000); - - NativePySocketClient client = new NativePySocketClient(java.util.Map.of()); - client.connect("127.0.0.1", port, false); - Thread reader = new Thread(client, "native-client-reader"); - reader.setDaemon(true); - reader.start(); - client.awaitReadyOrKill(30_000, 500); - System.out.println("connected=" + client.isConnected() + " ready=" + client.isReady()); - - System.out.println("--- sending a real PayloadStruct (what PyTranslator sends) ---"); - PayloadStruct ps = new PayloadStruct(); - ps.epoc = "00000000000000000001"; - ps.operation = PayloadStruct.OPERATION.PYTHON; - ps.methodName = "transportScriptWithExplicitPaths"; - ps.payload = new Object[] { "import os; print('NATIVE_CLIENT_OK', os.getpid())" }; - ps.payloadClasses = new Class[] { String.class }; - ps.longRunning = true; - ps.hasReturn = true; - ps.insightId = "harness-insight"; - ps.executionInsightId = "harness-insight"; - ps.sessionId = "harness-session"; - ps.jobId = "harness-job"; - - try { - Object resp = client.executeCommand(ps); - System.out.println("executeCommand returned: " + (resp == null ? "NULL" : resp.getClass().getSimpleName())); - if (resp instanceof PayloadStruct) { - PayloadStruct back = (PayloadStruct) resp; - System.out.println(" operation=" + back.operation + " response=" + back.response); - System.out.println(" payload=" + java.util.Arrays.toString(back.payload)); - } - } catch (Exception e) { - System.out.println("executeCommand FAILED: " + e.getClass().getSimpleName() + ": " + e.getMessage()); - } - - try { - client.stopServer(); - client.close(); - } catch (Exception ignored) { - // teardown is best-effort; the round trip above is the point of this harness - } - t.close(); - System.out.println("done"); - } -} diff --git a/src/prerna/util/OpenShellTransportHarness.java b/src/prerna/util/OpenShellTransportHarness.java deleted file mode 100644 index 2fe39db6d8..0000000000 --- a/src/prerna/util/OpenShellTransportHarness.java +++ /dev/null @@ -1,148 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.util; - -import java.net.InetSocketAddress; -import java.net.Socket; - -/** - * Standalone harness for {@link OpenShellTransport}, run outside Tomcat. - * - *

- * Exists because debugging this inside a servlet was guesswork: the failure - * surfaced as "Received a null PayloadStruct response" several layers away from - * the cause, and the transport's own INFO lines were not visible in the Tomcat - * log. Running the same code with a console attached removes both problems. - * - *

- * Usage: {@code java ... prerna.util.OpenShellTransportHarness } - */ -public final class OpenShellTransportHarness { - - private static String canConnect(int port) { - try (Socket s = new Socket()) { - s.connect(new InetSocketAddress("127.0.0.1", port), 2000); - return "yes"; - } catch (Exception e) { - return "no (" + e.getClass().getSimpleName() + ": " + e.getMessage() + ")"; - } - } - - private static byte[] readExactly(java.io.InputStream in, int n) throws java.io.IOException { - byte[] buf = new byte[n]; - int off = 0; - while (off < n) { - int read = in.read(buf, off, n - off); - if (read < 0) { - throw new java.io.IOException("peer closed after " + off + "/" + n + " bytes"); - } - off += read; - } - return buf; - } - - public static void main(String[] args) throws Exception { - // Bootstrap the DIHelper from the RDF map exactly as the webapp does. Without - // this, Utility.getDIHelperProperty() returns null for every property -- which - // is how an earlier round of debugging lost three iterations to a sandbox that - // was created from the stock image because OPENSHELL_IMAGE "was not set". - String rdf = System.getProperty("RDF_MAP"); - if (rdf == null || rdf.trim().isEmpty()) { - rdf = "/opt/semosshome/RDF_Map.prop"; - } - DIHelper.getInstance().loadCoreProp(rdf); - System.out.println("loaded RDF: " + rdf); - System.out.println("OPENSHELL_IMAGE=" + Utility.getDIHelperProperty(OpenShellTransport.IMAGE_PROP)); - System.out.println("OPENSHELL_POLICY=" + Utility.getDIHelperProperty(OpenShellTransport.POLICY_PROP)); - - String sessionKey = args.length > 0 ? args[0] : "harness-session-1"; - System.out.println("sandbox name would be: " + OpenShellTransport.sandboxNameFor(sessionKey)); - System.out.println("name length: " + OpenShellTransport.sandboxNameFor(sessionKey).length() + " (max " - + OpenShellTransport.MAX_SANDBOX_NAME_LENGTH + ")"); - - System.out.println("EXPECTED ARGV:"); - System.out.println(" " + OpenShellTransport.describeCreateCommand(sessionKey)); - OpenShellTransport t = OpenShellTransport.forSession(sessionKey); - System.out.println("starting transport..."); - long t0 = System.currentTimeMillis(); - int port = t.start(); - System.out.println("start() returned port " + port + " after " + (System.currentTimeMillis() - t0) + "ms"); - System.out.println("is that port actually LISTENING? " - + canConnect(port)); - - // Wait for the worker WITHOUT touching its port. "Forwarding" is printed when - // the CLI's listener binds, which precedes the worker's own readiness. A sleep - // is deliberate: probing the port would consume the worker's single connection - // slot and its disconnect would kill the worker -- see the wiki rule. - System.out.println("settling 4s before the single connection..."); - Thread.sleep(4000); - try (Socket s = new Socket()) { - s.connect(new InetSocketAddress("127.0.0.1", port), 5000); - // One framed request, exactly as the prototype does: [4-byte size][20-byte epoch][payload] - byte[] payload = ("{\"epoc\":\"00000000000000000001\",\"operation\":\"PYTHON\",\"methodName\":\"probe\"," - + "\"payload\":[\"import os; print('HARNESS_OK', os.getpid())\"],\"payloadClasses\":" - + "[\"java.lang.String\"],\"longRunning\":true,\"hasReturn\":true,\"response\":false," - + "\"interim\":false,\"insightId\":\"harness\",\"executionInsightId\":\"harness\"," - + "\"sessionId\":\"harness\",\"jobId\":\"harness\",\"mdc\":{},\"asset_paths\":[]," - + "\"runtime_vars\":{},\"append_vars\":{}}").getBytes(java.nio.charset.StandardCharsets.UTF_8); - byte[] epoc = "00000000000000000001".getBytes("UTF-8"); - java.io.OutputStream out = s.getOutputStream(); - out.write(new byte[] { (byte) (payload.length >>> 24), (byte) (payload.length >>> 16), - (byte) (payload.length >>> 8), (byte) payload.length }); - out.write(epoc); - out.write(payload); - out.flush(); - - java.io.InputStream in = s.getInputStream(); - // readNBytes() may return fewer bytes than asked on a stream (and throws on - // an empty read if the peer closed). Read exactly N, or fail with the count. - byte[] hdr = readExactly(in, 4); - int size = ((hdr[0] & 0xff) << 24) | ((hdr[1] & 0xff) << 16) | ((hdr[2] & 0xff) << 8) | (hdr[3] & 0xff); - System.out.println("reply declared size=" + size); - byte[] body = readExactly(in, size); - System.out.println("reply body=" + new String(body, java.nio.charset.StandardCharsets.UTF_8)); - // Keep reading: the worker streams LOG frames before the terminal response. - for (int i = 0; i < 5; i++) { - try { - byte[] h2 = readExactly(in, 4); - int s2 = ((h2[0] & 0xff) << 24) | ((h2[1] & 0xff) << 16) | ((h2[2] & 0xff) << 8) | (h2[3] & 0xff); - byte[] b2 = readExactly(in, s2); - System.out.println("frame " + (i + 2) + " (" + s2 + " bytes): " - + new String(b2, java.nio.charset.StandardCharsets.UTF_8)); - } catch (Exception more) { - System.out.println("(no more frames: " + more.getMessage() + ")"); - break; - } - } - } - - System.out.println("closing transport..."); - t.close(); - System.out.println("done"); - } -} diff --git a/src/prerna/util/openshell/OpenShellSessionLifecycleHarness.java b/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java similarity index 97% rename from src/prerna/util/openshell/OpenShellSessionLifecycleHarness.java rename to test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java index b7e36a5ad1..0a14d8a985 100644 --- a/src/prerna/util/openshell/OpenShellSessionLifecycleHarness.java +++ b/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java @@ -77,7 +77,11 @@ public final class OpenShellSessionLifecycleHarness { private static int failed = 0; public static void main(String[] args) throws Exception { - String rdf = System.getProperty("RDF_MAP", "/opt/semosshome/RDF_Map.prop"); + String rdf = System.getProperty("RDF_MAP"); + if (rdf == null || rdf.trim().isEmpty()) { + System.err.println("Set -DRDF_MAP="); + System.exit(2); + } DIHelper.getInstance().loadCoreProp(rdf); System.out.println("RDF: " + rdf); String mode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); From 70c61c91f807b47e14d759db0c67ecf0abba3209 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 20:03:53 +0000 Subject: [PATCH 39/56] fix(openshell): run the final reconcile while the sandbox is still alive shutdown() reconciled after stopping the socket client. The worker is the sandbox's main process, so by then the sandbox had completed and every teardown capture failed with "sandbox is not ready (phase: Completed)". Reconcile first, then stop the worker and delete the sandbox. Also log each full capture with its size and whether it was partial, so the fallback path is visible in the server log. Co-Authored-By: Claude Opus 5.5 --- src/prerna/om/ClientProcessWrapper.java | 17 ++++++++++------- src/prerna/util/OpenShellTransport.java | 5 ++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 1f930b57a3..16acb7cb77 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -707,6 +707,16 @@ private int fetchBackTransportedRootsLocked() { public void shutdown(boolean cleanUpFolder) { lockDestroy.lock(); try { + // OPENSHELL: the last reconcile must run while the sandbox is still alive. The + // worker is the sandbox's main process, so stopping the socket below ends the + // sandbox, and a capture attempted after that finds it Completed. + if (this.openshell != null) { + try { + fetchBackTransportedRoots(); + } catch (Exception e) { + classLogger.warn("Fetch-back before sandbox teardown failed for port {}", this.port, e); + } + } if (this.socketClient != null && this.socketClient.isConnected()) { try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { @@ -763,13 +773,6 @@ & deleteFolderWithRetries(this.sandboxJailDir) // teardown -- the gateway delete is authoritative and the worker cannot // outlive its sandbox. There is no graceful-shutdown verb to send first. if (this.openshell != null) { - try { - // Last reconcile before the sandbox disappears: anything written since the - // final request would otherwise die with it. - fetchBackTransportedRoots(); - } catch (Exception e) { - classLogger.warn("Fetch-back before sandbox teardown failed for port {}", this.port, e); - } try { this.openshell.close(); } catch (Exception e) { diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index d00d960922..015e9249b0 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -605,7 +605,10 @@ public int fetchBack(String absPath) throws IOException { classLogger.warn("Capture of {} is partial; not propagating deletions this time: {}", absPath, capture.warnings); } - return reconcile(tree.resolve(base), java.nio.file.Paths.get(normalized), complete); + int changed = reconcile(tree.resolve(base), java.nio.file.Paths.get(normalized), complete); + classLogger.info("Full capture of {}: {} bytes, {} file(s) changed on the host{}", absPath, capture.bytes, + changed, complete ? "" : " (partial)"); + return changed; } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException("fetch-back of " + absPath + " interrupted", e); From 6bd0071d54b447e87fc00d1bdda42c929fbc3e95 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 21:02:36 +0000 Subject: [PATCH 40/56] feat(openshell): one sandbox per user, shared across sessions and servers A user's python now runs in one persistent worker in their own sandbox. Any session, cookie-less API call or server instance attaches to it; it is deleted when the user's last live session ends, not when one session or one server goes away. - worker: --persist keeps serving after a client leaves, accepts several clients, and exits by itself after --timeout minutes with nobody connected. Callbacks to Java use the connection whose request is running. - OpenShellUserSandboxes: owner -> sandbox registry and per-session leases (Redis when configured, in-memory otherwise), a per-user create lock, one helper and wrapper per user on each instance, and a sweeper for sandboxes whose owner has no live session or that stopped running. - OpenShellTransport: attach/detach next to start/close; owner label; phase lookup and labelled listing for the sweeper. - ClientProcessWrapper: acquire through the registry, detach on shutdown, full capture on reattach. User: shared helper and wrapper in OPENSHELL mode. Co-Authored-By: Claude Opus 5.5 --- py/gaas_server_proxy.py | 2 +- py/gaas_tcp_server_handler.py | 24 +- py/gaas_tcp_socket_server.py | 53 +- src/prerna/auth/User.java | 42 +- src/prerna/om/ClientProcessWrapper.java | 45 +- src/prerna/util/OpenShellTransport.java | 236 ++++-- .../openshell/OpenShellUserSandboxes.java | 678 ++++++++++++++++++ 7 files changed, 1012 insertions(+), 68 deletions(-) create mode 100644 src/prerna/util/openshell/OpenShellUserSandboxes.java diff --git a/py/gaas_server_proxy.py b/py/gaas_server_proxy.py index 190169050b..57fbdac9af 100644 --- a/py/gaas_server_proxy.py +++ b/py/gaas_server_proxy.py @@ -13,7 +13,7 @@ def __init__(self): """ from gaas_tcp_server_handler import TCPServerHandler - self.server = TCPServerHandler.da_server + self.server = TCPServerHandler.current() def get_next_epoc(self) -> str: """This method atomically returns a random value that is unique across all thread operations and instances.""" diff --git a/py/gaas_tcp_server_handler.py b/py/gaas_tcp_server_handler.py index 5ec5afbecc..4ea3e7e9cd 100644 --- a/py/gaas_tcp_server_handler.py +++ b/py/gaas_tcp_server_handler.py @@ -151,12 +151,16 @@ def _asset_aware_import(name, _globals=None, _locals=None, fromlist=(), level=0) return mod +# Which client connection the current thread is serving; see TCPServerHandler.current(). +_current_handler = threading.local() + + def _is_engine_owned_process(): """ Whether this process runs an engine's own python rather than a user's python code. Set from the --engine_owned flag the engines launch with. """ - handler = TCPServerHandler.da_server + handler = TCPServerHandler.current() server = getattr(handler, "server", None) if handler is not None else None return bool(getattr(server, "engine_owned", False)) @@ -236,6 +240,11 @@ class TCPServerHandler(socketserver.BaseRequestHandler): # Class attribute to hold a singleton instance da_server = None + + @staticmethod + def current(): + """The handler whose request this thread is serving, else the latest client.""" + return getattr(_current_handler, "handler", None) or TCPServerHandler.da_server CANCELLED_MESSAGE = "The request was cancelled by the user" def log_level_mapper(self, log_level_name: str) -> int: @@ -305,6 +314,9 @@ def setup(self): self.residue = None TCPServerHandler.da_server = self + live = getattr(self.server, "live_handlers", None) + if live is not None: + live.add(self) # a lock to serialise all socket writes. HuggingFace (and similar libraries) # download files in parallel worker threads that all write tqdm progress to @@ -585,6 +597,9 @@ def get_final_output( epoc (Optional[str]): The epoc associated with the request. Defaults to None. """ self.log_data(data) + # Code run for this request calls back to Java over this connection, not over + # whichever client connected last: a persistent worker serves several. + _current_handler.handler = self payload = "" # payload = data @@ -978,6 +993,13 @@ def send_request(self, payload: dict): def stop_request(self): """Stops the request and closes the connection.""" if not self.stop: + live = getattr(self.server, "live_handlers", None) + if live is not None: + live.discard(self) + # A persistent worker keeps serving other clients; callbacks with no + # request of their own must not keep pointing at a closed socket. + if TCPServerHandler.da_server is self: + TCPServerHandler.da_server = next(iter(live), None) self.server.remove_handler() self.server.stop_it() self.request.close() diff --git a/py/gaas_tcp_socket_server.py b/py/gaas_tcp_socket_server.py index 619573f3ae..ae254ebed7 100644 --- a/py/gaas_tcp_socket_server.py +++ b/py/gaas_tcp_socket_server.py @@ -6,6 +6,7 @@ import threading import asyncio import os +import time # Must be imported before anything that can open a TLS connection. Importing it # leaves a single truststore owning ssl.SSLContext, without which the openai and @@ -45,6 +46,7 @@ def __init__( logger_level: str = "INFO", uds_path=None, engine_owned=False, + persist=False, ): self.logger = logging.getLogger("SocketServer") self.logger.debug("__init__") @@ -53,7 +55,14 @@ def __init__( self.uds_path = uds_path self.max_count = max_count self.cur_count = 0 - self.user_mode = self.max_count == 1 + # A persistent worker belongs to a user, not to a connection: it outlives any + # client, accepts the next one, and exits only after `timeout` minutes with no + # client at all. User mode is the opposite and closes with its single client. + self.persist = persist + self.user_mode = self.max_count == 1 and not persist + self.live_handlers = set() + self.idle_since = time.monotonic() + self.idle_limit = timeout * 60 if timeout > 0 else 0 self.insight_folder = insight_folder self.prefix = prefix # engine owned processes run platform code, not a user's python code @@ -91,7 +100,9 @@ def __init__( if timeout > 0: timeout = timeout * 60 print(f"Setting timeout to .. {timeout}") - self.timeout = timeout + # A persistent worker measures idleness from its last disconnect, so it + # wakes every minute to check rather than once per whole idle period. + self.timeout = min(timeout, 60) if persist else timeout # self.socket.settimeout(timeout*60) else: print("Setting timeout to None") @@ -108,6 +119,17 @@ def handle_timeout(self): # no clients.. kill this server, no point keeping it # give back the GPU self.timed_out = True + if self.persist: + with self.monitor: + idle = self.cur_count == 0 and time.monotonic() - self.idle_since >= self.idle_limit + if idle and self.idle_limit > 0: + self.logger.info(f"No client for {self.idle_limit / 60} minutes. Shutting down.") + self.stop = True + socketserver.TCPServer.server_close(self) + # Hard exit: a thread user code left running must not keep an abandoned + # sandbox alive, and the sandbox ends with this process. + os._exit(0) + return if self.cur_count == 0: self.logger.info( f"Server idle for {self.timeout / 60} minutes. No client connected. Shutting down." @@ -135,22 +157,30 @@ def serve_forever(self): print("Listening on port " + str(self.port)) self.handle_request() - - # also keep count updates synchronized with remove_handler - with self.monitor: - self.timed_out = False - self.cur_count += 1 except Exception as e: self.logger.error(f"Error: {e}", exc_info=True) self.stop_it() return + def process_request(self, request, client_address): + # Count the connection here, not after handle_request() returns: that also + # returns on a timeout with no client, which used to count a phantom one. + with self.monitor: + self.timed_out = False + self.cur_count += 1 + socketserver.ThreadingTCPServer.process_request(self, request, client_address) + def remove_handler(self): with self.monitor: self.cur_count = self.cur_count - 1 + self.idle_since = time.monotonic() self.monitor.notify() def stop_it(self): + if self.persist: + # One client leaving (disconnect, `stop`, logout) never ends a persistent + # worker; only idleness or deleting the sandbox does. + return print( f"Max connections = {self.max_count}, Current connections = {self.cur_count}" ) @@ -240,6 +270,14 @@ def parse_args(): "function or guardrail engine) rather than a user's python code. " "Adapters that only make sense for user code are skipped.", ) + parser.add_argument( + "--persist", + action="store_true", + help="Keep serving after a client disconnects and accept the next one, " + "up to --max_count at once. The process exits only after --timeout " + "minutes with no client connected. Used when one worker belongs to a " + "user across sessions and server instances (OpenShell).", + ) parser.add_argument( "--uds-path", type=str, @@ -306,4 +344,5 @@ def parse_args(): start=args.start, uds_path=args.uds_path, engine_owned=args.engine_owned, + persist=args.persist, ) diff --git a/src/prerna/auth/User.java b/src/prerna/auth/User.java index 9a7849647a..f62026c3f3 100644 --- a/src/prerna/auth/User.java +++ b/src/prerna/auth/User.java @@ -68,6 +68,7 @@ import prerna.util.Constants; import prerna.util.Settings; import prerna.util.SymlinkHelper; +import prerna.util.openshell.OpenShellUserSandboxes; import prerna.util.Utility; public class User implements Serializable { @@ -508,6 +509,9 @@ public SocketClient getPythonSocketClient(boolean create, int port, String venvE } return this.pythonCPW.getSocketClient(); } + if (OpenShellUserSandboxes.enabled()) { + return getOpenShellPythonSocketClient(); + } if (this.pythonCPW == null || this.pythonCPW.getSocketClient() == null) { startPythonSocketServerAndClient(-1, venvEngineId); this.pythonCPW.getSocketClient().setUser(this); @@ -528,11 +532,47 @@ public SocketClient getPythonSocketClient(boolean create, int port, String venvE } /** - * + * OPENSHELL: the user's python is one persistent worker in their own sandbox, shared + * by every session of theirs. On this instance all of those sessions use one wrapper + * (one connection, one record of what was transported); the wrapper attaches to the + * sandbox if it is running anywhere and creates it otherwise. + */ + private SocketClient getOpenShellPythonSocketClient() { + OpenShellUserSandboxes.Local local = OpenShellUserSandboxes.local(this); + synchronized (local) { + this.pythonCPW = local.getWrapper(); + // write-back permission checks run as the session that asked most recently + this.pythonCPW.setUser(this); + SocketClient client = this.pythonCPW.getSocketClient(); + if (client == null) { + startPythonSocketServerAndClient(-1, null); + } else if (!client.isConnected()) { + this.pythonCPW.shutdown(false); + try { + this.pythonCPW.reconnect(); + } catch (Exception e) { + classLogger.error("Failed to reconnect to user python process", e); + throw new IllegalArgumentException("Failed to connect to your isolated analytics engine"); + } + } + this.pythonCPW.getSocketClient().setUser(this); + } + this.insightSerializedMap.clear(); + return this.pythonCPW.getSocketClient(); + } + + /** + * * @return */ public SymlinkHelper getUserSymlinkHelper() { if (Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE))) { + if (OpenShellUserSandboxes.enabled()) { + // shared by the user's sessions on this instance; see getOpenShellPythonSocketClient + symlinkHelper = OpenShellUserSandboxes.local(this).getHelper(); + chrootPath = symlinkHelper.getUserChrootFolder(); + return symlinkHelper; + } if (symlinkHelper != null) { return symlinkHelper; } diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 16acb7cb77..c24268c7c8 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -131,6 +131,11 @@ public class ClientProcessWrapper { private prerna.auth.User user; // OPENSHELL mode: owns the sandbox + port-forward for this worker's lifetime private prerna.util.OpenShellTransport openshell; + /** + * The sandbox belongs to the user, not to this wrapper: shutdown detaches and leaves + * it running, and {@code OpenShellUserSandboxes} decides when it is deleted. + */ + private boolean openshellUserScoped = false; /** * One write-back at a time: python blocks, shell commands and teardown all reconcile, * and two reconciles interleaving over the same root would race their writes. @@ -249,10 +254,26 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS // client connects to the forwarded port. No udsPath is set, which is // what selects the TCP branch below. if (this.chrootSymlinkHelper != null && this.chrootSymlinkHelper.isExternalSandbox()) { - this.openshell = prerna.util.OpenShellTransport.forSession( - this.chrootSymlinkHelper.getUserChrootFolder()); try { - this.port = this.openshell.start(); + if (this.user != null && prerna.util.openshell.OpenShellUserSandboxes.enabled()) { + // The user's own sandbox, shared across their sessions and server + // instances: attach if it is running, create it otherwise. + prerna.util.openshell.OpenShellUserSandboxes.Attached attached = prerna.util.openshell.OpenShellUserSandboxes + .acquire(this.user); + this.openshell = attached.transport; + this.openshellUserScoped = true; + this.port = attached.port; + if (attached.reattached) { + // Another connection may have drained changes this one never + // applied; start this connection's write-back from the whole tree. + this.fullCaptureOwed = true; + } + } else { + this.openshell = prerna.util.OpenShellTransport + .forSession(this.chrootSymlinkHelper.getUserChrootFolder()); + this.openshellUserScoped = false; + this.port = this.openshell.start(); + } // Same seam as the namespace injector below: the per-mode collaborator // that receives every path symlinkFolder/symlinkProject would otherwise // have symlinked. Assets registered before the worker existed are @@ -707,9 +728,9 @@ private int fetchBackTransportedRootsLocked() { public void shutdown(boolean cleanUpFolder) { lockDestroy.lock(); try { - // OPENSHELL: the last reconcile must run while the sandbox is still alive. The - // worker is the sandbox's main process, so stopping the socket below ends the - // sandbox, and a capture attempted after that finds it Completed. + // OPENSHELL: the last reconcile must run while the sandbox is still alive. A + // session-scoped worker is the sandbox's main process, so stopping the socket + // below ends the sandbox, and a capture attempted after that finds it Completed. if (this.openshell != null) { try { fetchBackTransportedRoots(); @@ -769,12 +790,16 @@ & deleteFolderWithRetries(this.sandboxJailDir) } } // OPENSHELL: the worker lives in the sandbox, not in this JVM's process - // tree. Killing the local forward and deleting the sandbox is the whole - // teardown -- the gateway delete is authoritative and the worker cannot - // outlive its sandbox. There is no graceful-shutdown verb to send first. + // tree. A user-scoped sandbox is only detached from (the `stop` above closed + // this connection, not the persistent worker); a session-scoped one is + // deleted, and the gateway delete is authoritative. if (this.openshell != null) { try { - this.openshell.close(); + if (this.openshellUserScoped) { + this.openshell.detach(); + } else { + this.openshell.close(); + } } catch (Exception e) { classLogger.error("Failed to close OpenShell executor sandbox for port {}", this.port, e); } finally { diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 015e9249b0..73a0b42837 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -127,16 +127,36 @@ public final class OpenShellTransport { */ public static final String KEEP_SANDBOX_PROP = "OPENSHELL_KEEP_SANDBOX"; + /** Minutes a user's worker may run with no client connected before it exits itself. */ + public static final String IDLE_EXIT_PROP = "OPENSHELL_IDLE_EXIT_MINUTES"; + /** How many clients (sessions, server instances) one user's worker serves at once. */ + public static final String MAX_CLIENTS_PROP = "OPENSHELL_WORKER_MAX_CLIENTS"; + private static final int DEFAULT_IDLE_EXIT_MINUTES = 180; + private static final int DEFAULT_MAX_CLIENTS = 16; + + /** Label naming the user a sandbox belongs to (a digest, never the user id itself). */ + public static final String OWNER_LABEL = "semoss-owner"; + private final String sandboxName; private final String workerPort; private final boolean keepSandbox; + /** + * Set for a user-scoped sandbox: the worker is persistent and the sandbox carries the + * owner label. Null for the session-scoped form, whose worker ends with its client. + */ + private final String owner; private int forwardedPort = -1; private Process forwardProcess; private StringBuilder forwardLog; private OpenShellTransport(String sandboxName, String workerPort) { + this(sandboxName, workerPort, null); + } + + private OpenShellTransport(String sandboxName, String workerPort, String owner) { this.sandboxName = sandboxName; this.workerPort = workerPort; + this.owner = owner; String keep = System.getProperty(KEEP_SANDBOX_PROP); if (keep == null || keep.trim().isEmpty()) { keep = Utility.getDIHelperProperty(KEEP_SANDBOX_PROP); @@ -220,13 +240,13 @@ public static String sandboxNameFor(String sessionKey) { */ public static String describeCreateCommand(String sessionKey) { try { - return String.join(" ", createCommand(sandboxNameFor(sessionKey))); + return String.join(" ", createCommand(sandboxNameFor(sessionKey), null)); } catch (Exception e) { return "(cannot build: " + e.getMessage() + ")"; } } - private static List createCommand(String sandboxName) throws IOException { + private static List createCommand(String sandboxName, String owner) throws IOException { String cli = propOr(CLI_PROP, DEFAULT_CLI); String template = Utility.getDIHelperProperty(TEMPLATE_PROP); String label = propOr(LABEL_PROP, DEFAULT_LABEL); @@ -234,6 +254,16 @@ private static List createCommand(String sandboxName) throws IOException List create = new ArrayList<>(Arrays.asList(cli, "sandbox", "create", "--name", sandboxName, "--keep", "--detach", "--label", label)); + if (owner != null) { + create.add("--label"); + create.add(OWNER_LABEL + "=" + owner); + } + // A user's worker outlives any one client: it accepts the next session or server + // instance, and exits by itself only after a long stretch with nobody connected, + // so a sandbox every server lost track of still ends. + String workerMode = owner == null ? " --timeout 30" + : " --persist --max_count " + intPropOr(MAX_CLIENTS_PROP, DEFAULT_MAX_CLIENTS) + " --timeout " + + intPropOr(IDLE_EXIT_PROP, DEFAULT_IDLE_EXIT_MINUTES); if (template != null && !template.trim().isEmpty()) { create.add("--template"); create.add(template.trim()); @@ -281,17 +311,126 @@ private static List createCommand(String sandboxName) throws IOException + " --start True" // + " --py_folder /opt/semosshome/py" // + " --insight_folder /opt/semosshome/insight" // - + " --timeout 30" // + + workerMode // + " > " + WORKER_LOG_PATH + " 2>&1"); return create; } public static OpenShellTransport forSession(String sessionKey) { - String workerPort = Utility.getDIHelperProperty(WORKER_PORT_PROP); - if (workerPort == null || workerPort.trim().isEmpty()) { - workerPort = DEFAULT_WORKER_PORT; + return new OpenShellTransport(sandboxNameFor(sessionKey), propOr(WORKER_PORT_PROP, DEFAULT_WORKER_PORT)); + } + + /** + * A user-scoped sandbox: a persistent worker, labelled with its owner digest, that any + * session or server instance of that user can attach to. + */ + public static OpenShellTransport forUser(String sandboxName, String owner) { + if (owner == null || owner.trim().isEmpty()) { + throw new IllegalArgumentException("owner is required for a user-scoped sandbox"); } - return new OpenShellTransport(sandboxNameFor(sessionKey), workerPort.trim()); + return new OpenShellTransport(sandboxName, propOr(WORKER_PORT_PROP, DEFAULT_WORKER_PORT), owner); + } + + public String getSandboxName() { + return sandboxName; + } + + /** + * Connect to a sandbox that already runs a persistent worker: forward its port and + * nothing else. Only valid for a user-scoped sandbox; a session-scoped worker allows + * one client, and a second forward would queue behind it forever. + * + * @return the local port to connect to + */ + public int attach() throws IOException { + if (owner == null) { + throw new IllegalStateException("only a user-scoped sandbox can be attached to"); + } + int readyTimeout = intPropOr(READY_TIMEOUT_PROP, DEFAULT_READY_TIMEOUT_S); + openForward(readyTimeout); + if (!awaitWorkerReady(Math.min(30, readyTimeout))) { + detach(); + throw new IOException("worker inside sandbox " + sandboxName + " is not serving"); + } + classLogger.info("Attached to executor sandbox {}; local port {}", sandboxName, this.forwardedPort); + return this.forwardedPort; + } + + /** + * Stop this instance's forward and leave the sandbox running. The user's worker + * survives for their other sessions and server instances; deleting it is the + * registry's decision, made when the user has no live session anywhere. + */ + public void detach() { + if (forwardProcess != null) { + forwardProcess.destroy(); + try { + if (!forwardProcess.waitFor(5, TimeUnit.SECONDS)) { + forwardProcess.destroyForcibly(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + forwardProcess = null; + } + } + + /** The sandbox's phase ({@code Ready}, {@code Completed}, ...), or null when it does not exist. */ + public static String phaseOf(String sandboxName) { + try { + String out = run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "get", sandboxName, "-o", + "json"), 30); + com.google.gson.JsonObject json = com.google.gson.JsonParser.parseString(jsonBody(out)).getAsJsonObject(); + if (json.has("sandbox") && json.get("sandbox").isJsonObject()) { + json = json.getAsJsonObject("sandbox"); + } + return json.has("phase") ? json.get("phase").getAsString() : null; + } catch (Exception e) { + // "not found" exits non-zero; anything unparseable is treated the same way, + // which only ever leads to creating a fresh sandbox. + return null; + } + } + + /** Delete a sandbox by name; best effort. */ + public static void deleteSandbox(String sandboxName) { + try { + run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "delete", sandboxName), 60); + classLogger.info("Deleted executor sandbox {}", sandboxName); + } catch (Exception e) { + classLogger.warn("Failed to delete executor sandbox {}", sandboxName, e); + } + } + + /** + * Every executor sandbox on the gateway (selected by {@link #LABEL_PROP}), as name, + * phase, creation time ({@code yyyy-MM-dd HH:mm:ss}, UTC) and owner label (or null). + */ + public static List listExecutorSandboxes() throws IOException, InterruptedException { + String out = run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "list", "--selector", + propOr(LABEL_PROP, DEFAULT_LABEL), "-o", "json"), 60); + List rows = new ArrayList<>(); + com.google.gson.JsonObject json = com.google.gson.JsonParser.parseString(jsonBody(out)).getAsJsonObject(); + if (!json.has("sandboxes")) { + return rows; + } + for (com.google.gson.JsonElement element : json.getAsJsonArray("sandboxes")) { + com.google.gson.JsonObject box = element.getAsJsonObject(); + com.google.gson.JsonObject labels = box.has("labels") && box.get("labels").isJsonObject() + ? box.getAsJsonObject("labels") + : new com.google.gson.JsonObject(); + rows.add(new String[] { box.get("name").getAsString(), + box.has("phase") ? box.get("phase").getAsString() : "", + box.has("created_at") ? box.get("created_at").getAsString() : "", + labels.has(OWNER_LABEL) ? labels.get(OWNER_LABEL).getAsString() : null }); + } + return rows; + } + + /** The CLI may print notices before its JSON; keep from the first brace. */ + private static String jsonBody(String out) { + int brace = out.indexOf('{'); + return brace < 0 ? "{}" : out.substring(brace); } /** @@ -311,7 +450,7 @@ public int start() throws IOException, InterruptedException { int readyTimeout = intPropOr(READY_TIMEOUT_PROP, DEFAULT_READY_TIMEOUT_S); // Single source of truth: the argv used here is exactly the one // describeCreateCommand() prints, so a hand-run diff is meaningful. - List create = createCommand(sandboxName); + List create = createCommand(sandboxName, owner); classLogger.info("Starting executor sandbox {} (worker port {})", sandboxName, workerPort); String createOut = run(create, readyTimeout); @@ -334,6 +473,45 @@ public int start() throws IOException, InterruptedException { } } + try { + openForward(readyTimeout); + } catch (IOException e) { + close(); + throw e; + } + // Readiness must come from the worker's OWN log, and the worker must write + // that log to a FILE, not to the supervisor's stdout pipe. + // + // Two measured facts (2026-09-28): + // 1. With stdout on the supervisor's pipe the worker came up erratically -- + // sometimes serving, sometimes never opening the port at all while the + // sandbox still read `Ready`. Redirecting its stdout to a file made it + // reliable, 3 runs out of 3, and fast (~0.66s instead of ~5s). Output + // back-pressure on that pipe is the suspected cause; the file removes it. + // 2. The supervisor's `NET:OPEN 127.0.0.1:/tcp` line is a *connection* + // event, not a bind event -- it also marks NET:CLOSE, and it fired only + // because a probe connected. Waiting for it is a mistake; an earlier + // version of this method did exactly that and stalled for the full + // timeout. The worker's own "waiting for request" line is the real signal. + // + // Never probe the forwarded port to test readiness: a session-scoped worker is + // single-connection (--max_count 1), so a probe consumes its only slot and its + // disconnect ends the worker. + if (!awaitWorkerReady(readyTimeout)) { + String detail = readAvailable(forwardProcess); + close(); + throw new IOException("worker inside sandbox " + sandboxName + " did not report ready within " + + readyTimeout + "s (forward alive=" + (forwardProcess != null && forwardProcess.isAlive()) + + ")" + (detail.isEmpty() ? "" : ": " + detail)); + } + classLogger.info("Executor sandbox {} ready; worker listening, local port {}", sandboxName, + this.forwardedPort); + return this.forwardedPort; + } + + /** Start {@code openshell forward service} for the worker port and wait until it binds. */ + private void openForward(int readyTimeout) throws IOException { + String cli = propOr(CLI_PROP, DEFAULT_CLI); this.forwardedPort = freeLocalPort(); List forward = Arrays.asList(cli, "forward", "service", "--target-port", workerPort, "--local", String.valueOf(this.forwardedPort), sandboxName); @@ -371,39 +549,11 @@ public int start() throws IOException, InterruptedException { boolean alive = forwardProcess != null && forwardProcess.isAlive(); classLogger.error("Forward for sandbox {} did not become ready on 127.0.0.1:{} (alive={}): {}", sandboxName, this.forwardedPort, alive, detail); - close(); + detach(); throw new IOException("port forward for sandbox " + sandboxName + " did not become ready on 127.0.0.1:" + this.forwardedPort + " within " + readyTimeout + "s (process alive=" + alive + ")" + (detail.isEmpty() ? "" : ": " + detail)); } - // Readiness must come from the worker's OWN log, and the worker must write - // that log to a FILE, not to the supervisor's stdout pipe. - // - // Two measured facts (2026-09-28): - // 1. With stdout on the supervisor's pipe the worker came up erratically -- - // sometimes serving, sometimes never opening the port at all while the - // sandbox still read `Ready`. Redirecting its stdout to a file made it - // reliable, 3 runs out of 3, and fast (~0.66s instead of ~5s). Output - // back-pressure on that pipe is the suspected cause; the file removes it. - // 2. The supervisor's `NET:OPEN 127.0.0.1:/tcp` line is a *connection* - // event, not a bind event -- it also marks NET:CLOSE, and it fired only - // because a probe connected. Waiting for it is a mistake; an earlier - // version of this method did exactly that and stalled for the full - // timeout. The worker's own "waiting for request" line is the real signal. - // - // Never probe the forwarded port to test readiness: the worker is - // single-connection (--max_count 1), so a probe consumes its only slot and its - // disconnect ends the worker. - if (!awaitWorkerReady(readyTimeout)) { - String detail = readAvailable(forwardProcess); - close(); - throw new IOException("worker inside sandbox " + sandboxName + " did not report ready within " - + readyTimeout + "s (forward alive=" + (forwardProcess != null && forwardProcess.isAlive()) - + ")" + (detail.isEmpty() ? "" : ": " + detail)); - } - classLogger.info("Executor sandbox {} ready; worker listening, local port {}", sandboxName, - this.forwardedPort); - return this.forwardedPort; } /** @@ -1229,17 +1379,7 @@ private static final class Capture { * verb to call first, because the worker cannot outlive its sandbox. */ public void close() { - if (forwardProcess != null) { - forwardProcess.destroy(); - try { - if (!forwardProcess.waitFor(5, TimeUnit.SECONDS)) { - forwardProcess.destroyForcibly(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - forwardProcess = null; - } + detach(); if (keepSandbox) { classLogger.warn("KEEP_SANDBOX is set; leaving sandbox {} in place for inspection", sandboxName); return; diff --git a/src/prerna/util/openshell/OpenShellUserSandboxes.java b/src/prerna/util/openshell/OpenShellUserSandboxes.java new file mode 100644 index 0000000000..1de0544e04 --- /dev/null +++ b/src/prerna/util/openshell/OpenShellUserSandboxes.java @@ -0,0 +1,678 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.AccessToken; +import prerna.auth.User; +import prerna.om.ClientProcessWrapper; +import prerna.redis.RedisConnectionConfig; +import prerna.redis.RedisConnectionFactory; +import prerna.util.Constants; +import prerna.util.OpenShellTransport; +import prerna.util.SymlinkHelper; +import prerna.util.Utility; +import redis.clients.jedis.UnifiedJedis; +import redis.clients.jedis.params.SetParams; + +/** + * One OpenShell sandbox per user, shared by every session and server instance. + * + *

+ * The sandbox outlives any single session: its worker is persistent, so a server + * instance that dies, restarts or is simply a different node leaves the user's python + * process, its variables and its files in place, and the next request reattaches. The + * sandbox is deleted when the user's last live session ends, wherever that + * session lived. + * + *

+ * Three pieces of state, at two scopes: + *

    + *
  • Registry (shared): owner → sandbox name, plus the set of the user's + * live sessions, each with an expiry refreshed by every request (a lease equal to the + * session timeout). Redis when configured; an in-memory map otherwise, which is only + * correct for a single server instance.
    • + *
  • Local (this instance): one {@link SymlinkHelper} and one + * {@link ClientProcessWrapper} per user, so a user's sessions on this instance share a + * single connection and a single record of what was transported, instead of one forward + * and one worker connection per session (or per cookie-less API call).
    • + *
  • Sweeper (this instance, periodic): deletes sandboxes whose owner has no live + * session left, and labelled sandboxes that are no longer running.
    • + *
+ */ +public final class OpenShellUserSandboxes { + + private static final Logger classLogger = LogManager.getLogger(OpenShellUserSandboxes.class); + + private static final String KEY_PREFIX = "semoss:openshell:"; + /** How often a session may refresh its lease; requests in between are free. */ + private static final long TOUCH_INTERVAL_MS = TimeUnit.SECONDS.toMillis(60); + /** How long a creator may hold the per-user create lock. */ + private static final long CREATE_LOCK_MS = TimeUnit.MINUTES.toMillis(5); + /** A labelled sandbox younger than this is never swept: its registry entry may not exist yet. */ + private static final long SWEEP_GRACE_MS = TimeUnit.MINUTES.toMillis(10); + private static final DateTimeFormatter CREATED_AT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + /** Distinguishes this JVM's leases from ones a restored session carries over a restart. */ + private static final String BOOT_ID = UUID.randomUUID().toString(); + private static final String NODE = nodeName(); + + private static final Map LOCALS = new ConcurrentHashMap<>(); + private static final Map> LOCAL_SESSIONS = new ConcurrentHashMap<>(); + private static final Object LOCALS_LOCK = new Object(); + private static volatile Registry registry; + private static volatile ScheduledExecutorService sweeper; + + private OpenShellUserSandboxes() { + // static utility + } + + /** User-scoped sandboxes apply exactly when the OpenShell jail does. */ + public static boolean enabled() { + return Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE)) + && Constants.SANDBOX_MODE_OPENSHELL.equalsIgnoreCase( + String.valueOf(Utility.getDIHelperProperty(Constants.SANDBOX_MODE)).trim()); + } + + /** + * The digest a user's sandbox is filed under: provider and id of the primary login. + * Used as the registry key and the {@code semoss-owner} label, so neither carries the + * raw user id. + */ + public static String ownerOf(User user) { + AccessToken token = user == null ? null : user.getPrimaryLoginToken(); + if (token == null || token.getId() == null) { + throw new IllegalStateException("cannot scope a sandbox to a user with no primary login"); + } + return digest(token.getProvider().name() + ":" + token.getId(), 24); + } + + // ------------------------------------------------------------------ this instance + + /** What this instance holds for one user, shared by all of their sessions here. */ + public static final class Local { + private final String owner; + private final SymlinkHelper helper; + private final ClientProcessWrapper cpw = new ClientProcessWrapper(); + + private Local(String owner, SymlinkHelper helper) { + this.owner = owner; + this.helper = helper; + } + + public SymlinkHelper getHelper() { + return helper; + } + + public ClientProcessWrapper getWrapper() { + return cpw; + } + } + + /** + * The user's shared helper and wrapper on this instance, created on first use. The + * chroot folder is per instance, so its name keeps the random suffix. + */ + public static Local local(User user) { + String owner = ownerOf(user); + Local local = LOCALS.get(owner); + if (local != null) { + return local; + } + synchronized (LOCALS_LOCK) { + local = LOCALS.get(owner); + if (local == null) { + String chroot = Utility.getDIHelperProperty(Constants.CHROOT_DIR) + "/" + User.getSingleLogginName(user) + + "-" + UUID.randomUUID(); + SymlinkHelper helper = new SymlinkHelper(chroot); + local = new Local(owner, helper); + // Published before linking the asset folder: opening the asset project can + // ask the user for this helper again, and must get this one. + LOCALS.put(owner, local); + try { + helper.symlinkUserAsset(user); + } catch (Exception e) { + classLogger.warn("Unable to symlink user asset folder into chroot", e); + } + ensureSweeper(); + } + return local; + } + } + + // ------------------------------------------------------------------ attach or create + + /** A transport connected to the user's sandbox, and whether that sandbox already existed. */ + public static final class Attached { + public final OpenShellTransport transport; + public final int port; + public final boolean reattached; + + private Attached(OpenShellTransport transport, int port, boolean reattached) { + this.transport = transport; + this.port = port; + this.reattached = reattached; + } + } + + /** + * Connect to the user's sandbox, creating it if they have none that is running. + * + *

+ * Creation is serialised per user across instances with a registry lock, so two + * instances that both find nothing do not both create; the loser waits for the + * winner's entry and attaches to it. + */ + public static Attached acquire(User user) throws Exception { + String owner = ownerOf(user); + Registry store = registry(); + long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(240); + while (System.currentTimeMillis() < deadline) { + String name = store.sandboxFor(owner); + if (name != null) { + String phase = OpenShellTransport.phaseOf(name); + if ("Ready".equalsIgnoreCase(phase)) { + OpenShellTransport transport = OpenShellTransport.forUser(name, owner); + try { + int port = transport.attach(); + return new Attached(transport, port, true); + } catch (Exception e) { + classLogger.warn("Could not attach to sandbox {}; replacing it", name, e); + } + } else { + classLogger.info("Sandbox {} for this user is {}; replacing it", name, phase); + } + OpenShellTransport.deleteSandbox(name); + store.forget(owner, name); + continue; + } + if (store.lock(owner)) { + try { + if (store.sandboxFor(owner) != null) { + continue; + } + name = OpenShellTransport.sandboxNameFor(owner + ":" + UUID.randomUUID()); + OpenShellTransport transport = OpenShellTransport.forUser(name, owner); + int port = transport.start(); + store.bind(owner, name); + classLogger.info("Created sandbox {} for user {} on {}", name, owner, NODE); + return new Attached(transport, port, false); + } finally { + store.unlock(owner); + } + } + // Another instance is creating it; its entry appears when the sandbox is Ready. + Thread.sleep(1000); + } + throw new IllegalStateException("timed out waiting for the user's sandbox"); + } + + // ------------------------------------------------------------------ session leases + + /** + * Record that a session of this user is alive, for as long as the session timeout. + * Called on every authenticated request; refreshes at most once a minute per session. + * + * @param lastTouch the session's previous touch marker, or null + * @param ttlSeconds the session's max inactive interval + * @return the marker to store on the session, or null when nothing was refreshed + */ + public static String touch(User user, String sessionId, String lastTouch, int ttlSeconds) { + if (user == null || sessionId == null || !user.isLoggedIn()) { + return null; + } + long now = System.currentTimeMillis(); + if (lastTouch != null && lastTouch.startsWith(BOOT_ID + ":")) { + try { + if (now - Long.parseLong(lastTouch.substring(BOOT_ID.length() + 1)) < TOUCH_INTERVAL_MS) { + return null; + } + } catch (NumberFormatException ignored) { + // refresh below + } + } + try { + String owner = ownerOf(user); + long ttlMs = TimeUnit.SECONDS.toMillis(ttlSeconds > 0 ? ttlSeconds : TimeUnit.DAYS.toSeconds(1)); + registry().touch(owner, sessionId, now + ttlMs); + LOCAL_SESSIONS.computeIfAbsent(owner, k -> ConcurrentHashMap.newKeySet()).add(sessionId); + ensureSweeper(); + } catch (Exception e) { + classLogger.warn("Could not refresh the sandbox lease for session {}", sessionId, e); + return null; + } + return BOOT_ID + ":" + now; + } + + /** + * A session ended (logout or timeout). Releases this instance's connection when it was + * the user's last session here, and deletes the sandbox when it was their last anywhere. + */ + public static void sessionEnded(User user, String sessionId) { + if (user == null || !user.isLoggedIn()) { + return; + } + String owner; + try { + owner = ownerOf(user); + } catch (IllegalStateException e) { + return; + } + Set here = LOCAL_SESSIONS.get(owner); + if (here != null) { + here.remove(sessionId); + } + if (here == null || here.isEmpty()) { + LOCAL_SESSIONS.remove(owner); + releaseLocal(owner); + } + boolean anyLive; + try { + anyLive = registry().release(owner, sessionId, System.currentTimeMillis()); + } catch (Exception e) { + classLogger.warn("Could not release the sandbox lease for session {}; the sweeper will", sessionId, e); + return; + } + if (!anyLive) { + deleteFor(owner, "last session ended"); + } + } + + private static void releaseLocal(String owner) { + Local local = LOCALS.remove(owner); + if (local == null) { + return; + } + try { + // Write back, close this instance's connection, stop the forward. The sandbox + // itself is not touched here. + local.cpw.shutdown(true); + } catch (Exception e) { + classLogger.warn("Failed to release the sandbox connection for {}", owner, e); + } + local.helper.removeChrootFolder(); + } + + private static void deleteFor(String owner, String reason) { + Registry store = registry(); + String name = store.sandboxFor(owner); + if (name != null) { + classLogger.info("Deleting sandbox {} ({})", name, reason); + OpenShellTransport.deleteSandbox(name); + store.forget(owner, name); + } + store.drop(owner); + } + + // ------------------------------------------------------------------ sweeper + + private static void ensureSweeper() { + if (sweeper != null) { + return; + } + synchronized (OpenShellUserSandboxes.class) { + if (sweeper == null) { + ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "openshell-sandbox-sweeper"); + t.setDaemon(true); + return t; + }); + service.scheduleWithFixedDelay(OpenShellUserSandboxes::sweepQuietly, 1, 5, TimeUnit.MINUTES); + sweeper = service; + } + } + } + + private static void sweepQuietly() { + try { + sweep(); + } catch (Throwable t) { + classLogger.warn("Sandbox sweep failed", t); + } + } + + /** + * Delete what nobody can use any more. + * + *

    + *
  1. A user with no live session anywhere: their sandbox (a session lost with its + * server, so its logout never ran).
    2. + *
  2. A labelled sandbox that is not running (its worker exited after idling, or it + * was a session-scoped sandbox orphaned by a restart).
    3. + *
  3. With a shared registry only: a running user sandbox no entry points at.
    4. + *
+ */ + public static void sweep() throws Exception { + Registry store = registry(); + long now = System.currentTimeMillis(); + for (String owner : store.owners()) { + if (!store.anyLive(owner, now)) { + Set here = LOCAL_SESSIONS.remove(owner); + if (here != null) { + here.clear(); + } + releaseLocal(owner); + deleteFor(owner, "no live session for this user"); + } + } + LocalDateTime cutoff = LocalDateTime.now(ZoneOffset.UTC).minusNanos(TimeUnit.MILLISECONDS.toNanos(SWEEP_GRACE_MS)); + List boxes = OpenShellTransport.listExecutorSandboxes(); + for (String[] box : boxes) { + String name = box[0]; + String phase = box[1]; + String owner = box[3]; + LocalDateTime created; + try { + created = LocalDateTime.parse(box[2], CREATED_AT); + } catch (Exception e) { + continue; + } + if (created.isAfter(cutoff)) { + continue; + } + boolean running = "Ready".equalsIgnoreCase(phase); + if (!running) { + classLogger.info("Sweeping sandbox {} (phase {})", name, phase); + OpenShellTransport.deleteSandbox(name); + if (owner != null) { + store.forget(owner, name); + } + } else if (owner != null && store.shared() && !name.equals(store.sandboxFor(owner))) { + classLogger.info("Sweeping sandbox {} (no registry entry points at it)", name); + OpenShellTransport.deleteSandbox(name); + } + } + } + + // ------------------------------------------------------------------ registry + + private static Registry registry() { + Registry current = registry; + if (current == null) { + synchronized (OpenShellUserSandboxes.class) { + if (registry == null) { + RedisConnectionConfig config = RedisConnectionConfig.fromDIHelper(); + registry = config == null ? new LocalRegistry() : new RedisRegistry(config); + classLogger.info("OpenShell sandbox registry: {}", registry.shared() ? "redis" : "in-memory"); + } + current = registry; + } + } + return current; + } + + /** Owner → sandbox, owner → live sessions with expiry, and a create lock. */ + interface Registry { + boolean shared(); + + String sandboxFor(String owner); + + void bind(String owner, String sandboxName); + + /** Remove the entry only if it still names this sandbox. */ + void forget(String owner, String sandboxName); + + boolean lock(String owner); + + void unlock(String owner); + + void touch(String owner, String sessionId, long expiresAtMs); + + /** Remove a session; true when the user still has a live session. */ + boolean release(String owner, String sessionId, long nowMs); + + boolean anyLive(String owner, long nowMs); + + Set owners(); + + /** Forget the user entirely: sessions and index. */ + void drop(String owner); + } + + static final class RedisRegistry implements Registry { + private final RedisConnectionConfig config; + private final String lockToken = NODE + ":" + BOOT_ID; + + RedisRegistry(RedisConnectionConfig config) { + this.config = config; + } + + private UnifiedJedis redis() { + return RedisConnectionFactory.getClient(config); + } + + private static String sandboxKey(String owner) { + return KEY_PREFIX + "sandbox:" + owner; + } + + private static String sessionsKey(String owner) { + return KEY_PREFIX + "sessions:" + owner; + } + + private static String lockKey(String owner) { + return KEY_PREFIX + "lock:" + owner; + } + + private static final String OWNERS = KEY_PREFIX + "owners"; + + @Override + public boolean shared() { + return true; + } + + @Override + public String sandboxFor(String owner) { + return redis().hget(sandboxKey(owner), "name"); + } + + @Override + public void bind(String owner, String sandboxName) { + Map record = new HashMap<>(); + record.put("name", sandboxName); + record.put("node", NODE); + record.put("createdAt", String.valueOf(System.currentTimeMillis())); + record.put("workerPort", String.valueOf(Utility.getDIHelperProperty(OpenShellTransport.WORKER_PORT_PROP))); + UnifiedJedis redis = redis(); + redis.hset(sandboxKey(owner), record); + redis.sadd(OWNERS, owner); + } + + @Override + public void forget(String owner, String sandboxName) { + UnifiedJedis redis = redis(); + if (sandboxName.equals(redis.hget(sandboxKey(owner), "name"))) { + redis.del(sandboxKey(owner)); + } + } + + @Override + public boolean lock(String owner) { + return "OK".equals(redis().set(lockKey(owner), lockToken, SetParams.setParams().nx().px(CREATE_LOCK_MS))); + } + + @Override + public void unlock(String owner) { + UnifiedJedis redis = redis(); + if (lockToken.equals(redis.get(lockKey(owner)))) { + redis.del(lockKey(owner)); + } + } + + @Override + public void touch(String owner, String sessionId, long expiresAtMs) { + UnifiedJedis redis = redis(); + redis.zadd(sessionsKey(owner), expiresAtMs, sessionId); + redis.sadd(OWNERS, owner); + } + + @Override + public boolean release(String owner, String sessionId, long nowMs) { + redis().zrem(sessionsKey(owner), sessionId); + return anyLive(owner, nowMs); + } + + @Override + public boolean anyLive(String owner, long nowMs) { + UnifiedJedis redis = redis(); + redis.zremrangeByScore(sessionsKey(owner), Double.NEGATIVE_INFINITY, nowMs); + return redis.zcard(sessionsKey(owner)) > 0; + } + + @Override + public Set owners() { + return new HashSet<>(redis().smembers(OWNERS)); + } + + @Override + public void drop(String owner) { + UnifiedJedis redis = redis(); + if (redis.zcard(sessionsKey(owner)) == 0 && !redis.exists(sandboxKey(owner))) { + redis.del(sessionsKey(owner)); + redis.srem(OWNERS, owner); + } + } + } + + /** Single-instance fallback: correct only while one server owns every session. */ + static final class LocalRegistry implements Registry { + private final Map sandboxes = new ConcurrentHashMap<>(); + private final Map> sessions = new ConcurrentHashMap<>(); + private final Set locks = ConcurrentHashMap.newKeySet(); + + @Override + public boolean shared() { + return false; + } + + @Override + public String sandboxFor(String owner) { + return sandboxes.get(owner); + } + + @Override + public void bind(String owner, String sandboxName) { + sandboxes.put(owner, sandboxName); + } + + @Override + public void forget(String owner, String sandboxName) { + sandboxes.remove(owner, sandboxName); + } + + @Override + public boolean lock(String owner) { + return locks.add(owner); + } + + @Override + public void unlock(String owner) { + locks.remove(owner); + } + + @Override + public void touch(String owner, String sessionId, long expiresAtMs) { + sessions.computeIfAbsent(owner, k -> new ConcurrentHashMap<>()).put(sessionId, expiresAtMs); + } + + @Override + public boolean release(String owner, String sessionId, long nowMs) { + Map live = sessions.get(owner); + if (live != null) { + live.remove(sessionId); + } + return anyLive(owner, nowMs); + } + + @Override + public boolean anyLive(String owner, long nowMs) { + Map live = sessions.get(owner); + if (live == null) { + return false; + } + live.values().removeIf(expiry -> expiry <= nowMs); + return !live.isEmpty(); + } + + @Override + public Set owners() { + Set all = new HashSet<>(sandboxes.keySet()); + all.addAll(sessions.keySet()); + return all; + } + + @Override + public void drop(String owner) { + Map live = sessions.get(owner); + if ((live == null || live.isEmpty()) && !sandboxes.containsKey(owner)) { + sessions.remove(owner); + } + } + } + + // ------------------------------------------------------------------ helpers + + private static String digest(String value, int hexChars) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + for (int i = 0; sb.length() < hexChars; i++) { + sb.append(String.format("%02x", hash[i])); + } + return sb.substring(0, hexChars); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable", e); + } + } + + private static String nodeName() { + try { + return java.net.InetAddress.getLocalHost().getHostName(); + } catch (Exception e) { + return "unknown"; + } + } +} From 55fb52025fdd553f5a2e098c19b047202f92b37d Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 21:46:46 +0000 Subject: [PATCH 41/56] fix(openshell): a server stop does not end the user's claim on their sandbox Measured: a Tomcat restart fires sessionDestroyed for every session, which released every lease and deleted the user's sandbox. Only a logout or a real timeout now gives up a lease; a session leaving with its server keeps it, so the restarted or another instance reattaches. Co-Authored-By: Claude Opus 5.5 --- .../openshell/OpenShellUserSandboxes.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/prerna/util/openshell/OpenShellUserSandboxes.java b/src/prerna/util/openshell/OpenShellUserSandboxes.java index 1de0544e04..b1547a3da7 100644 --- a/src/prerna/util/openshell/OpenShellUserSandboxes.java +++ b/src/prerna/util/openshell/OpenShellUserSandboxes.java @@ -289,10 +289,19 @@ public static String touch(User user, String sessionId, String lastTouch, int tt } /** - * A session ended (logout or timeout). Releases this instance's connection when it was - * the user's last session here, and deletes the sandbox when it was their last anywhere. + * A session left this instance. Releases this instance's connection when it was the + * user's last session here. + * + *

+ * Only a session that is really over -- an explicit logout, or idle past its timeout -- + * gives up its lease, and the sandbox goes when that was the user's last lease anywhere. + * A server stopping also destroys every session (measured 2026-10-23: a Tomcat restart + * fired {@code sessionDestroyed} for each one and deleted the sandbox), but the user has + * not left: their lease stands, so the restarted or another instance reattaches. + * + * @param over true for a logout or a timeout; false when the server is going away */ - public static void sessionEnded(User user, String sessionId) { + public static void sessionEnded(User user, String sessionId, boolean over) { if (user == null || !user.isLoggedIn()) { return; } @@ -310,6 +319,10 @@ public static void sessionEnded(User user, String sessionId) { LOCAL_SESSIONS.remove(owner); releaseLocal(owner); } + if (!over) { + classLogger.info("Session {} left with the server; keeping its sandbox lease", sessionId); + return; + } boolean anyLive; try { anyLive = registry().release(owner, sessionId, System.currentTimeMillis()); From 3fe62770819917fb6bf3d6103d982b9c0d363979 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 21:51:35 +0000 Subject: [PATCH 42/56] fix(openshell): run shell commands in the user's actual sandbox The shell runner derived the sandbox name from this instance's chroot folder, which no longer names a user-scoped sandbox; every Command() failed with 'sandbox not found'. Ask the user's wrapper for the sandbox its worker runs in. Co-Authored-By: Claude Opus 5.5 --- src/prerna/om/ClientProcessWrapper.java | 9 +++++++++ src/prerna/util/openshell/OpenShellShellRunner.java | 9 +++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index c24268c7c8..836e9f3d3e 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -907,6 +907,15 @@ private int calculatePort(int port) { return port; } + /** + * @return the OpenShell sandbox this wrapper's worker runs in, or null outside + * OPENSHELL mode or before it started + */ + public String getOpenShellSandboxName() { + prerna.util.OpenShellTransport current = this.openshell; + return current == null ? null : current.getSandboxName(); + } + /** * @return the socket client connected to the server process, or null if not yet * created diff --git a/src/prerna/util/openshell/OpenShellShellRunner.java b/src/prerna/util/openshell/OpenShellShellRunner.java index 84ad496299..893182f492 100644 --- a/src/prerna/util/openshell/OpenShellShellRunner.java +++ b/src/prerna/util/openshell/OpenShellShellRunner.java @@ -106,8 +106,13 @@ public static Result execute(User user, String workingDir, String command) { // Same on-demand rule as Py(): the first shell command in a session is // allowed to create the sandbox. user.getPythonSocketClient(true); - String chrootFolder = user.getUserSymlinkHelper().getUserChrootFolder(); - String sandboxName = OpenShellTransport.sandboxNameFor(chrootFolder); + // The sandbox the user's worker is actually in: a user-scoped one is named by the + // registry, not derived from this instance's chroot folder. + prerna.om.ClientProcessWrapper cpw = user.getPythonClientProcessWrapper(); + String sandboxName = cpw == null ? null : cpw.getOpenShellSandboxName(); + if (sandboxName == null) { + sandboxName = OpenShellTransport.sandboxNameFor(user.getUserSymlinkHelper().getUserChrootFolder()); + } String trimmed = command == null ? "" : command.trim(); if (trimmed.isEmpty()) { From 15958b2b4b2781c0f3292ab7d64b16b97009baf5 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 21:51:54 +0000 Subject: [PATCH 43/56] docs(openshell): describe the user-scoped sandbox lifecycle and its settings Co-Authored-By: Claude Opus 5.5 --- docs/deployment/executor_sandbox.md | 44 +++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index 43642b7d96..8a64f840b7 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -40,22 +40,34 @@ worker has to travel `Tomcat → forward → gateway → supervisor → runtime | Piece | Runs where | Notes | | --- | --- | --- | | Java (Tomcat) | host | owns the socket client, decides when to sync | -| `gaas_tcp_socket_server.py` | sandbox, **main process** | the persistent REPL; one per user session | +| `gaas_tcp_socket_server.py` | sandbox, **main process** | the persistent REPL; one per **user**, shared by all their sessions | | shell commands | sandbox | started by `OpenShellShellRunner`, same write-back rules | | user assets / projects / rooms | host | copied in at session start, written back at block boundaries | The worker is stateful: variables survive between blocks, which is why it cannot be -restarted to pick up a change. It speaks the existing gaas protocol over a single TCP -connection; a second connection is accepted and then never served. - -## Session lifecycle - -- Created lazily on the first `Py()` or shell command; the first call pays the sandbox start - and the upload of the session's roots. -- Deleted on logout, session timeout, and on disconnect, where `ClientProcessWrapper.shutdown` - reconciles one last time before the sandbox disappears. -- Nothing removes a sandbox orphaned by a JVM crash yet. They carry the `OPENSHELL_LABEL` - label and a `pyw-` name, so `openshell sandbox list` finds them for manual deletion. +restarted to pick up a change. It runs with `--persist`: a client leaving (disconnect, +`stop`, logout) does not end it, it serves up to `OPENSHELL_WORKER_MAX_CLIENTS` connections +at once, and it exits by itself only after `OPENSHELL_IDLE_EXIT_MINUTES` with no client. +Each server instance holds one connection per user, shared by that user's sessions there. + +## Lifecycle (one sandbox per user) + +`OpenShellUserSandboxes` keeps a registry of owner → sandbox and owner → live sessions. It uses +Redis when one is configured (`REDIS_ENABLED`, or the cluster Redis), under `semoss:openshell:*`, +and an in-memory map otherwise. The in-memory map is only correct for a single server instance, +and it forgets the sandbox when that instance restarts. + +- **Created** lazily on the first `Py()` or shell command of a user who has no running sandbox. + Creation is serialised per user with a registry lock, so two instances do not both create one. + Any later session, cookie-less API call or server instance of that user **attaches** to it. +- **Kept alive** by a lease: every authenticated request refreshes the session's entry for the + session timeout, at most once a minute. Browsing without python counts. +- **Deleted** when the user's last live session ends, by logout or timeout. A server stopping + destroys every session, but that is not the user leaving: the leases stand, and the + restarted or another instance reattaches with the worker's variables intact. +- **Swept** every five minutes on each instance. That removes sandboxes whose owner has no live + lease (a session lost with its server), labelled sandboxes that are no longer running, and, + with Redis, running user sandboxes no registry entry names. ## Write-back pipeline @@ -200,6 +212,8 @@ be current — a stale image shows up as code that ignores your latest change. | `OPENSHELL_WORKER_PORT` | worker port inside the sandbox (default `7100`) | | `OPENSHELL_READY_TIMEOUT_SECONDS` | cold-start budget (default `180`) | | `OPENSHELL_KEEP_SANDBOX` | `true` leaves sandboxes in place on teardown, for debugging | +| `OPENSHELL_IDLE_EXIT_MINUTES` | a user's worker exits after this long with no client connected (default `180`; keep above the session timeout) | +| `OPENSHELL_WORKER_MAX_CLIENTS` | connections one user's worker serves at once (default `16`) | ## Verifying a deployment @@ -220,5 +234,11 @@ be current — a stale image shows up as code that ignores your latest change. session is not reflected inward, and the sandbox wins the next write-back for any file it touched (recoverable from quarantine). Branch switching belongs on the host, followed by a fresh session. +- Python variables are keyed by insight id, and insight ids live in one instance's memory. After + a restart or a move to another instance the worker, its modules and its files are intact, but a + new insight does not see the old insight's variables. +- One change watcher serves every connection to a worker. Two instances connected at once can + consume each other's change sets. That is harmless while they share a disk; separate disks per + instance need per-connection cursors. - Rooms and assets are gated as described above, but the platform has no room-sharing model in this build; if one is added, the room branch must resolve the room to its project. From 6b5cc08147da46b14b5003d38f7cdd224e2af9a7 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Fri, 23 Oct 2026 21:52:47 +0000 Subject: [PATCH 44/56] chore(openshell): drop the unscheduled reaper; the sweeper replaces it Co-Authored-By: Claude Opus 5.5 --- .../openshell/OpenShellSandboxReaper.java | 181 ------------------ 1 file changed, 181 deletions(-) delete mode 100644 src/prerna/util/openshell/OpenShellSandboxReaper.java diff --git a/src/prerna/util/openshell/OpenShellSandboxReaper.java b/src/prerna/util/openshell/OpenShellSandboxReaper.java deleted file mode 100644 index 36724d548f..0000000000 --- a/src/prerna/util/openshell/OpenShellSandboxReaper.java +++ /dev/null @@ -1,181 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.util.openshell; - -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; - -import prerna.util.OpenShellTransport; -import prerna.util.Utility; - -/** - * Deletes executor sandboxes that outlived their session. - * - *

- * Normal teardown is the logout path: {@code UserSessionLoader.sessionDestroyed} - * → {@code ClientProcessWrapper.shutdown(true)} → {@code OpenShellTransport.close()}. - * That covers a user who logs out or whose session times out. It does not cover - * the cases that actually leak: - * - *

    - *
  • Tomcat is killed or crashes — no session event is ever delivered;
    • - *
  • the sandbox is created and the JVM dies before it is recorded anywhere;
    • - *
  • a session is abandoned in a way the container never notices.
    • - *
- * - *

- * So this is a backstop, not the primary mechanism: it lists only sandboxes - * carrying the executor label and the {@code pyw-} name prefix, and only - * deletes those older than a threshold. Age is the only signal that is safe to - * act on without a live registry of owners: a young sandbox may belong to a - * session on another replica, and killing it would be a real user-visible bug. - * - *

- * Run it from a timer, not from the request path: - * - * 

- * java -cp ... prerna.util.openshell.OpenShellSandboxReaper --max-age-hours 12
- * java -cp ... prerna.util.openshell.OpenShellSandboxReaper --max-age-hours 12 --dry-run
- *
- */ -public final class OpenShellSandboxReaper { - - private static final DateTimeFormatter LIST_TIMESTAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - - private OpenShellSandboxReaper() { - // static utility - } - - public static void main(String[] args) { - String rdf = System.getProperty("RDF_MAP", "/opt/semosshome/RDF_Map.prop"); - prerna.util.DIHelper.getInstance().loadCoreProp(rdf); - - double maxAgeHours = 12; - boolean dryRun = false; - String label = Utility.getDIHelperProperty(OpenShellTransport.LABEL_PROP); - if (label == null || label.trim().isEmpty()) { - label = "app=semoss-executor"; - } - for (int i = 0; i < args.length; i++) { - switch (args[i]) { - case "--max-age-hours": - maxAgeHours = Double.parseDouble(args[++i]); - break; - case "--dry-run": - dryRun = true; - break; - case "--label": - label = args[++i]; - break; - default: - System.out.println("unknown argument: " + args[i]); - System.exit(2); - } - } - - System.out.println("label=" + label + " maxAgeHours=" + maxAgeHours + (dryRun ? " (dry run)" : "")); - int reaped = reap(label, maxAgeHours, dryRun); - System.out.println((dryRun ? "would reap " : "reaped ") + reaped + " sandbox(es)"); - } - - /** - * Delete labelled, prefixed sandboxes older than {@code maxAgeHours}. - * - * @return how many were (or would be) deleted - */ - public static int reap(String label, double maxAgeHours, boolean dryRun) { - List candidates = listExecutorSandboxes(label); - LocalDateTime cutoff = LocalDateTime.now().minus(Duration.ofMinutes((long) (maxAgeHours * 60))); - int count = 0; - for (String[] row : candidates) { - String name = row[0]; - String created = row[1]; - LocalDateTime timestamp; - try { - timestamp = LocalDateTime.parse(created, LIST_TIMESTAMP); - } catch (Exception e) { - // Unparseable age: skip rather than guess. Deleting a sandbox we cannot - // date is exactly the mistake this class exists to avoid. - System.out.println(" skip " + name + " (unparseable timestamp '" + created + "')"); - continue; - } - if (timestamp.isAfter(cutoff)) { - continue; - } - count++; - if (dryRun) { - System.out.println(" would delete " + name + " (created " + created + ")"); - } else { - System.out.println(" deleting " + name + " (created " + created + ")"); - run("openshell", "sandbox", "delete", name); - } - } - return count; - } - - /** - * List sandboxes carrying the executor label whose name matches the executor - * prefix. Two independent filters on purpose: the label says "this gateway - * knows about executors", the prefix says "this process created it". - */ - private static List listExecutorSandboxes(String label) { - List rows = new ArrayList<>(); - String output = run("openshell", "sandbox", "list", "--selector", label); - for (String line : output.split("\\R")) { - String trimmed = line.trim(); - if (trimmed.isEmpty() || trimmed.startsWith("NAME")) { - continue; - } - String[] columns = trimmed.split("\\s+"); - if (columns.length < 3) { - continue; - } - String name = columns[0]; - if (!name.startsWith("pyw-")) { - continue; - } - rows.add(new String[] { name, columns[1] + " " + columns[2] }); - } - return rows; - } - - private static String run(String... command) { - try { - Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); - String output = new String(process.getInputStream().readAllBytes()); - process.waitFor(); - return output; - } catch (Exception e) { - System.out.println(" command failed: " + String.join(" ", command) + " -> " + e.getMessage()); - return ""; - } - } -} From 59aa074b30dd46ea211a5fcc223cd8f6fb539421 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 01:25:29 +0000 Subject: [PATCH 45/56] feat(openshell): sync both ways between the host and a live sandbox A sandbox used to keep the copy of a project it got at first use, and its next full capture put that stale copy back over newer host files (another user's write-back, a cluster pull, a UI upload), leaving the newer version only in quarantine. Measured live before this change. - write-back: a file goes back only if the sandbox changed it since the last sync (both sides carry the same size and mtime after one; uploads and captures keep mtime). When both changed, the last write wins and the loser is kept in quarantine. A delete never removes a host file that changed since the sandbox had it. - refresh: before each python block or shell command, host files whose stamp moved are copied into the sandbox in one upload, and host deletions are applied there. - cluster: write-back into a project is pushed to cluster storage and announced, at most every ten seconds per root and on teardown, so other servers pull it and their sandboxes refresh in turn. - the user's running sandbox is recovered by its owner label when the registry has no entry (in-memory registry after a restart, Redis flushed). Co-Authored-By: Claude Opus 5.5 --- src/prerna/om/ClientProcessWrapper.java | 125 +++++++++++- .../tcp/client/NativePySocketClient.java | 11 + src/prerna/util/OpenShellTransport.java | 192 +++++++++++++++++- .../util/openshell/OpenShellShellRunner.java | 8 + .../openshell/OpenShellUserSandboxes.java | 22 ++ 5 files changed, 348 insertions(+), 10 deletions(-) diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 836e9f3d3e..2dcac408c3 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -146,6 +146,18 @@ public class ClientProcessWrapper { * worker has already forgotten those paths, so only a full capture can recover them. */ private volatile boolean fullCaptureOwed = false; + /** + * Project roots written back since they were last published to cluster storage. Pushing + * is what lets another server pull the change, and from there its sandboxes refresh. + */ + private final java.util.Set unpublishedRoots = java.util.concurrent.ConcurrentHashMap.newKeySet(); + private final Map lastPublished = new java.util.concurrent.ConcurrentHashMap<>(); + private static final long PUBLISH_INTERVAL_MS = 10_000L; + private static final ExecutorService PUBLISHER = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "openshell-writeback-publisher"); + t.setDaemon(true); + return t; + }); private String classPath; private boolean debug; private String timeout; @@ -518,7 +530,10 @@ private int fetchBackChangedPathsLocked() { int total = 0; for (Map.Entry> group : writes.entrySet()) { try { - total += this.openshell.fetchBackPaths(group.getKey(), group.getValue()); + if (this.openshell.fetchBackPaths(group.getKey(), group.getValue()) > 0) { + this.unpublishedRoots.add(group.getKey()); + total++; + } } catch (Exception e) { // The worker has already dropped these paths from its pending set, so they // would never be reported again: recover them with a full capture next block. @@ -530,17 +545,113 @@ private int fetchBackChangedPathsLocked() { for (Map.Entry> group : deletes.entrySet()) { for (String relative : group.getValue()) { if (this.openshell.deleteFromHost(group.getKey(), relative)) { + this.unpublishedRoots.add(group.getKey()); removed++; } } } if (total > 0 || removed > 0) { - classLogger.info("Fetch-back wrote {} and removed {} file(s) from {} changed path(s)", total, removed, - changes.size()); + classLogger.info("Fetch-back wrote into {} root(s) and removed {} file(s) from {} changed path(s)", total, + removed, changes.size()); } + publishWrittenRoots(false); return total + removed; } + /** + * Bring host-side changes into the sandbox before a block runs: another user's write-back, + * a cluster pull, an upload in the UI. Without this a sandbox computes on the copy it was + * given at first use for as long as it lives. + */ + public void refreshTransportedRoots() { + if (this.openshell == null || this.chrootSymlinkHelper == null) { + return; + } + fetchBackLock.lock(); + try { + if (this.fullCaptureOwed) { + // Unapplied sandbox changes first; a refresh must not overwrite them. + fetchBackTransportedRootsLocked(); + } + for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { + if (!isUserSpace(Utility.normalizePath(path))) { + continue; + } + try { + this.openshell.refreshFromHost(path); + } catch (Exception e) { + classLogger.warn("Refresh of {} into the sandbox failed; the sandbox keeps its copy", path, e); + } + } + } finally { + fetchBackLock.unlock(); + } + } + + private static boolean isUserSpace(String normalized) { + return normalized.contains("/" + Constants.PROJECT_FOLDER + "/") + || normalized.contains("/" + Constants.ROOM_FOLDER + "/") + || normalized.contains("/" + Constants.USER_FOLDER + "/"); + } + + /** + * Push written-back project roots to cluster storage and announce them, so servers that + * have the project loaded pull the change. Batched: a root is pushed at most every ten + * seconds, and whatever is left is pushed when the connection is released. + * + * @param force push everything pending now (teardown) + */ + private void publishWrittenRoots(boolean force) { + if (!prerna.cluster.util.ClusterUtil.IS_CLUSTER || this.unpublishedRoots.isEmpty()) { + return; + } + long now = System.currentTimeMillis(); + for (String root : new ArrayList<>(this.unpublishedRoots)) { + String projectId = projectIdOf(root); + if (projectId == null) { + // rooms and user assets have their own sync at logout + this.unpublishedRoots.remove(root); + continue; + } + if (!force && now - this.lastPublished.getOrDefault(root, 0L) < PUBLISH_INTERVAL_MS) { + continue; + } + this.unpublishedRoots.remove(root); + this.lastPublished.put(root, now); + Runnable push = () -> { + try { + prerna.project.api.IProject project = Utility.getProject(projectId); + if (project != null) { + prerna.cluster.util.ClusterUtil.pushProjectFolder(project, root); + classLogger.info("Published write-back of {} to cluster storage", root); + } + } catch (Exception e) { + this.unpublishedRoots.add(root); + classLogger.warn("Could not publish write-back of {}; retrying after the next block", root, e); + } + }; + if (force) { + push.run(); + } else { + PUBLISHER.submit(push); + } + } + } + + /** The project id of a root under {@code project/__/}, or null for other spaces. */ + private static String projectIdOf(String normalized) { + String marker = "/" + Constants.PROJECT_FOLDER + "/"; + int at = normalized.indexOf(marker); + if (at < 0) { + return null; + } + String rest = normalized.substring(at + marker.length()); + int slash = rest.indexOf('/'); + String folder = slash < 0 ? rest : rest.substring(0, slash); + int separator = folder.lastIndexOf("__"); + return separator < 0 ? folder : folder.substring(separator + 2); + } + private static boolean truthy(Object value) { if (value instanceof Boolean flag) { return flag; @@ -715,13 +826,18 @@ private int fetchBackTransportedRootsLocked() { continue; } try { - total += this.openshell.fetchBack(path); + int written = this.openshell.fetchBack(path); + if (written > 0) { + this.unpublishedRoots.add(normalized); + } + total += written; } catch (Exception e) { failed = true; classLogger.warn("Fetch-back failed for {}; retrying with a full capture next block", path, e); } } this.fullCaptureOwed = failed; + publishWrittenRoots(false); return total; } @@ -734,6 +850,7 @@ public void shutdown(boolean cleanUpFolder) { if (this.openshell != null) { try { fetchBackTransportedRoots(); + publishWrittenRoots(true); } catch (Exception e) { classLogger.warn("Fetch-back before sandbox teardown failed for port {}", this.port, e); } diff --git a/src/prerna/tcp/client/NativePySocketClient.java b/src/prerna/tcp/client/NativePySocketClient.java index bd69c953c7..026d087122 100644 --- a/src/prerna/tcp/client/NativePySocketClient.java +++ b/src/prerna/tcp/client/NativePySocketClient.java @@ -605,6 +605,17 @@ public Object executeCommand(PayloadStruct ps) { addEpocForJob(ps.jobId, ps.epoc); } ps.longRunning = true; + // The other half of the execution boundary: before a block runs, host-side changes + // (another user's write-back, a cluster pull, the UI) are copied into the sandbox, so + // the block sees what the platform holds now rather than its first-use copy. + if (!ps.response && this.cpw != null && !Boolean.TRUE.equals(IN_CHANGE_QUERY.get()) + && (ps.operation == PayloadStruct.OPERATION.PYTHON || ps.operation == PayloadStruct.OPERATION.CMD)) { + try { + this.cpw.refreshTransportedRoots(); + } catch (Exception refreshError) { + classLogger.warn("Refresh before execution failed", refreshError); + } + } try { // ReentrantLock + Condition (not synchronized/wait) so a virtual thread diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 73a0b42837..0263dd50db 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -782,6 +782,127 @@ public int fetchBack(String absPath) throws IOException { * @throws IOException when the paths could not be captured or applied; the change set * is then lost and the caller must schedule a full capture */ + /** + * Bring host-side changes to one transported root into the sandbox: files another user, + * a cluster pull or the UI wrote since this sandbox last had them, and files removed on + * the host. Run before a block, after the previous block's write-back, so the sandbox + * computes on what the platform holds now. + * + *

+ * Only files whose host stamp moved since the last sync are sent, in one upload. A file + * the sandbox also changed was reconciled by the previous write-back (last write won + * and the loser was kept), so sending the host's copy here cannot drop work. + * + * @param absRoot a path previously passed to {@link #upload(String)} + * @return number of files copied in or removed + */ + public int refreshFromHost(String absRoot) throws IOException { + java.nio.file.Path root = java.nio.file.Paths.get(Utility.normalizePath(absRoot)); + if (!java.nio.file.Files.isDirectory(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + return 0; + } + java.util.Set seen = seenBySandbox.computeIfAbsent(root.toString(), + key -> java.util.concurrent.ConcurrentHashMap.newKeySet()); + java.util.List changed = new ArrayList<>(); + java.util.Set present = new java.util.HashSet<>(); + java.nio.file.Files.walkFileTree(root, new java.nio.file.SimpleFileVisitor() { + @Override + public java.nio.file.FileVisitResult preVisitDirectory(java.nio.file.Path dir, + java.nio.file.attribute.BasicFileAttributes attrs) { + String name = dir.getFileName() == null ? "" : dir.getFileName().toString(); + return !dir.equals(root) && (attrs.isSymbolicLink() || isExcludedFromCapture(java.nio.file.Paths.get(name))) + ? java.nio.file.FileVisitResult.SKIP_SUBTREE + : java.nio.file.FileVisitResult.CONTINUE; + } + + @Override + public java.nio.file.FileVisitResult visitFile(java.nio.file.Path file, + java.nio.file.attribute.BasicFileAttributes attrs) { + if (attrs.isRegularFile()) { + present.add(file.toString()); + String stamp = attrs.size() + ":" + attrs.lastModifiedTime().to(TimeUnit.SECONDS); + if (!stamp.equals(hostStamps.get(file.toString()))) { + changed.add(file); + } + } + return java.nio.file.FileVisitResult.CONTINUE; + } + + @Override + public java.nio.file.FileVisitResult visitFileFailed(java.nio.file.Path file, IOException e) { + return java.nio.file.FileVisitResult.CONTINUE; + } + }); + // Removed on the host since the sandbox had it: only paths this transport synced. + String prefix = root.toString() + java.io.File.separator; + java.util.List removed = new ArrayList<>(); + for (String known : new ArrayList<>(hostStamps.keySet())) { + if (known.startsWith(prefix) && !present.contains(known) + && !java.nio.file.Files.exists(java.nio.file.Paths.get(known), java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + removed.add(known); + } + } + if (changed.isEmpty() && removed.isEmpty()) { + return 0; + } + if (changed.size() + removed.size() > MAX_CHANGED_PATHS) { + // A whole-tree change (a branch switch, a re-import): send the root again. + classLogger.info("Refreshing all of {} in sandbox {}: {} host change(s)", root, sandboxName, + changed.size() + removed.size()); + if (!upload(root.toString())) { + throw new IOException("re-upload of " + root + " failed"); + } + return changed.size(); + } + String cli = propOr(CLI_PROP, DEFAULT_CLI); + int count = 0; + if (!changed.isEmpty()) { + java.nio.file.Path stage = java.nio.file.Files.createTempDirectory("openshell-refresh-"); + try { + java.nio.file.Path staged = stage.resolve(root.getFileName().toString()); + for (java.nio.file.Path file : changed) { + java.nio.file.Path copy = staged.resolve(root.relativize(file).toString()); + java.nio.file.Files.createDirectories(copy.getParent()); + java.nio.file.Files.copy(file, copy, java.nio.file.StandardCopyOption.COPY_ATTRIBUTES, + java.nio.file.LinkOption.NOFOLLOW_LINKS); + } + // `sandbox upload` merges into the existing tree and keeps mtimes, so after it + // both sides carry the stamp recorded below. + run(Arrays.asList(cli, "sandbox", "upload", sandboxName, staged.toString(), + root.getParent().toString()), 300); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted refreshing " + root, e); + } finally { + deleteQuietly(stage); + } + for (java.nio.file.Path file : changed) { + recordStamp(file); + seen.add(root.relativize(file).toString().replace('\\', '/')); + } + count += changed.size(); + } + if (!removed.isEmpty()) { + List rm = new ArrayList<>(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "rm", + "-f", "--")); + rm.addAll(removed); + try { + run(rm, 60); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted refreshing " + root, e); + } + for (String known : removed) { + hostStamps.remove(known); + seen.remove(root.relativize(java.nio.file.Paths.get(known)).toString().replace('\\', '/')); + } + count += removed.size(); + } + classLogger.info("Refreshed sandbox {} from {}: {} file(s) in, {} removed", sandboxName, root, changed.size(), + removed.size()); + return count; + } + public int fetchBackPaths(String absRoot, java.util.List relativePaths) throws IOException { if (absRoot == null || relativePaths == null || relativePaths.isEmpty()) { return 0; @@ -875,12 +996,33 @@ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, bo recordStamp(destination); return; } - if (hostChangedSinceSync(destination) - && quarantineCopy(destination, target, store) == null) { - classLogger.warn("Write-back of {} skipped: the host copy changed and could not be preserved", - destination); + // Which side changed since the last sync? Both copies carried the same + // size and mtime then (uploads and captures preserve mtime), so a side + // that no longer matches the recorded stamp is a side that changed. A + // path the watcher reported was changed by the sandbox by definition. + String synced = hostStamps.get(destination.toString()); + boolean sandboxChanged = !fullTree || synced == null || !synced.equals(stampOf(path)); + boolean hostChanged = synced == null || !synced.equals(stampOf(destination)); + if (!sandboxChanged) { + // Only the host moved on (another user, a cluster pull, the UI): a + // full capture must not put the sandbox's stale copy back. The + // refresh before the next block brings the host's into the sandbox. return; } + if (hostChanged) { + // Both changed: last write wins, and the loser is kept. + if (mtimeOf(destination) > mtimeOf(path)) { + quarantineIncoming(path, target, relative, store); + classLogger.info("Write-back of {} skipped: the host copy is newer", destination); + return; + } + if (quarantineCopy(destination, target, store) == null) { + classLogger.warn( + "Write-back of {} skipped: the host copy changed and could not be preserved", + destination); + return; + } + } } java.nio.file.Files.createDirectories(destination.getParent()); // Re-check now that the parents exist: creating them must not have @@ -931,6 +1073,12 @@ private int removeDeletedFromHost(java.nio.file.Path target, java.util.Set Date: Sat, 24 Oct 2026 01:25:49 +0000 Subject: [PATCH 46/56] docs(openshell): two-way sync, last write wins, cluster publish Co-Authored-By: Claude Opus 5.5 --- docs/deployment/executor_sandbox.md | 43 +++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index 8a64f840b7..c6196164f8 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -71,7 +71,8 @@ and it forgets the sandbox when that instance restarts. ## Write-back pipeline -Every `PYTHON` and `CMD` block ends with a sync. The steps, in order: +Every `PYTHON` and `CMD` block starts with a refresh (host to sandbox) and ends with a write-back +(sandbox to host). The write-back steps, in order: 1. **Ask for the change set.** `NativePySocketClient` sends a `CHANGES` request on the existing socket. The worker answers @@ -84,15 +85,26 @@ Every `PYTHON` and `CMD` block ends with a sync. The steps, in order: 3. **Pull only what changed.** Changed paths are tarred inside the sandbox under one `sandbox exec` and streamed to a temporary file on the host, stopping at a 256 MB cap. The archive is refused if its listing fails or names an absolute path or a `..` segment. -4. **Reconcile.** Content is the authority: a differing sandbox file is always written. The - host copy is quarantined first if it changed since this session last left it (uploaded or - written back), and the write is skipped if that copy fails. Sandbox timestamps never - decide anything. +4. **Reconcile.** Identical content is skipped. Otherwise each side is checked against the + stamp (size and mtime, in seconds) recorded at the last sync, when both copies matched: + uploads and captures preserve mtime, so a side that no longer matches is the side that + changed. + - Only the sandbox changed: the file is written back. + - Only the host changed (another user, a cluster pull, the UI): nothing is written. The + next refresh brings the host copy in. This is what keeps a full capture from putting a + stale copy back. + - Both changed: the **last write wins**. The losing copy is kept in quarantine (the + sandbox's under `.sandbox/`), and the write is skipped if the host copy cannot be + preserved. 5. **Deletes.** Only a path the sandbox is known to have held — seen in a full capture or written back since — is ever deleted on the host, so a file created and removed within one - block cannot take an unrelated host file with it. Deletes are quarantine-then-delete, - suppressed if the copy fails, and directories they empty are pruned up to (never including) - the transported root. + block cannot take an unrelated host file with it. A host file that changed since the + sandbox had it is never deleted. Deletes are quarantine-then-delete, suppressed if the copy + fails, and directories they empty are pruned up to (never including) the transported root. +6. **Publish.** In cluster mode, a project root that was written back is pushed to cluster + storage and announced on the cluster synchronizer, at most every ten seconds per root and + once more on teardown. Servers that have the project loaded pull it, and their sandboxes + refresh from that before their next block. Every host-side write and delete is contained: a path that leaves the root, or crosses a symlink on the way, is refused. Without that, a link committed into a writable project could @@ -103,6 +115,12 @@ Inside `.git`, only git's data is written back — `objects/` (not `objects/info reaches the host. `config`, `hooks/` and `info/` are never written back: host-side git runs or obeys them, so a sandbox able to write them could execute commands on the server. +The **refresh** before each block walks every transported root on the host. Files whose stamp +moved since the last sync are copied in with a single `sandbox upload`, which merges into the +existing tree and keeps mtimes. Files removed on the host are removed in the sandbox. More +than ~2000 changes re-upload the whole root. Any sandbox change still owed from a failed +write-back is applied first, so a refresh never overwrites unsynced work. + Quarantine lives at `$BASE/writeback-quarantine///`, outside `project/`, `room/` and `user/`, so it is never pushed to object storage. @@ -230,10 +248,11 @@ be current — a stale image shows up as code that ignores your latest change. - More than ~2000 changed paths in one block sets `overflow` and forces a full capture. - A *racing* `mmap` writer cannot be captured without cooperation (`msync`) or interception this sandbox does not have. -- There is no host→sandbox refresh while a sandbox lives. A `git pull` on the host during a - session is not reflected inward, and the sandbox wins the next write-back for any file it - touched (recoverable from quarantine). Branch switching belongs on the host, followed by a - fresh session. +- Conflicts are decided by modification time, so a change made within the same second as + the last sync, and leaving the size unchanged, is not detected. +- The refresh lists each transported root on every block; for very large roots that cost + shows up as block latency. +- Background code that writes between blocks is not synced until the next block ends. - Python variables are keyed by insight id, and insight ids live in one instance's memory. After a restart or a move to another instance the worker, its modules and its files are intact, but a new insight does not see the old insight's variables. From 4a01efc9ce09487ea5338d00a9fb9853eab147bb Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 01:55:08 +0000 Subject: [PATCH 47/56] refactor(openshell): review cleanup of the user-scoped sandbox Correctness: - phaseOf throws when the gateway cannot be asked instead of answering "gone", and acquire retries rather than deleting a live user's sandbox over a hiccup; one failed attach is retried before the sandbox is replaced - a per-user lock serialises session bookkeeping with the release of the shared wrapper, and User retries if a release won the race to the wrapper - every request records its session here, not only lease refreshes - the worker's live-handler set is guarded by the server monitor - a whole-tree refresh still applies host deletions, in batches - fetch-back counts files again, not roots Redis registry follows the room-coordination layout: openshell:{owner}:* hash-tag keys, SET NX PX lock with a random token, compare-and-delete scripts for forget/unlock/drop. Cleanup: TransportedRoots replaces four copies of the user-space check and three of the id parsing; SandboxInfo record; one stamp helper; imports instead of fully qualified names; constants for the timing values; appliesTo() keeps users without a primary login on the per-session path; fetchBackPaths gets its javadoc back; stale single-connection docs updated. Co-Authored-By: Claude Opus 5.5 --- docs/deployment/executor_sandbox.md | 7 +- py/gaas_tcp_server_handler.py | 18 +- py/gaas_tcp_socket_server.py | 1 - src/prerna/auth/User.java | 47 +- src/prerna/om/ClientProcessWrapper.java | 104 ++-- src/prerna/util/OpenShellTransport.java | 508 ++++++++++-------- .../util/openshell/OpenShellShellRunner.java | 33 +- .../openshell/OpenShellUserSandboxes.java | 339 +++++++----- .../util/openshell/TransportedRoots.java | 100 ++++ 9 files changed, 704 insertions(+), 453 deletions(-) create mode 100644 src/prerna/util/openshell/TransportedRoots.java diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index c6196164f8..f363fa4d57 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -53,8 +53,11 @@ Each server instance holds one connection per user, shared by that user's sessio ## Lifecycle (one sandbox per user) `OpenShellUserSandboxes` keeps a registry of owner → sandbox and owner → live sessions. It uses -Redis when one is configured (`REDIS_ENABLED`, or the cluster Redis), under `semoss:openshell:*`, -and an in-memory map otherwise. The in-memory map is only correct for a single server instance, +Redis when one is configured (`REDIS_ENABLED`, or the cluster Redis), and an in-memory map +otherwise. The Redis keys follow the room-coordination layout: `openshell:{}:sandbox`, +`:sessions` and `:lock`, all with the `{owner}` hash tag so one user's keys share a Redis Cluster +slot, plus an `openshell:owners` index. The create lock is `SET NX PX` with a random token and is +released by a compare-and-delete script, as room locks are. The in-memory map is only correct for a single server instance, and it forgets the sandbox when that instance restarts. - **Created** lazily on the first `Py()` or shell command of a user who has no running sandbox. diff --git a/py/gaas_tcp_server_handler.py b/py/gaas_tcp_server_handler.py index 4ea3e7e9cd..352c1a7560 100644 --- a/py/gaas_tcp_server_handler.py +++ b/py/gaas_tcp_server_handler.py @@ -240,12 +240,12 @@ class TCPServerHandler(socketserver.BaseRequestHandler): # Class attribute to hold a singleton instance da_server = None + CANCELLED_MESSAGE = "The request was cancelled by the user" @staticmethod def current(): """The handler whose request this thread is serving, else the latest client.""" return getattr(_current_handler, "handler", None) or TCPServerHandler.da_server - CANCELLED_MESSAGE = "The request was cancelled by the user" def log_level_mapper(self, log_level_name: str) -> int: """ @@ -316,7 +316,8 @@ def setup(self): TCPServerHandler.da_server = self live = getattr(self.server, "live_handlers", None) if live is not None: - live.add(self) + with self.server.monitor: + live.add(self) # a lock to serialise all socket writes. HuggingFace (and similar libraries) # download files in parallel worker threads that all write tqdm progress to @@ -995,11 +996,14 @@ def stop_request(self): if not self.stop: live = getattr(self.server, "live_handlers", None) if live is not None: - live.discard(self) - # A persistent worker keeps serving other clients; callbacks with no - # request of their own must not keep pointing at a closed socket. - if TCPServerHandler.da_server is self: - TCPServerHandler.da_server = next(iter(live), None) + # Other connection threads add and remove concurrently; iterating the + # set unguarded can raise "set changed size during iteration". + with self.server.monitor: + live.discard(self) + # A persistent worker keeps serving other clients; callbacks with no + # request of their own must not keep pointing at a closed socket. + if TCPServerHandler.da_server is self: + TCPServerHandler.da_server = next(iter(live), None) self.server.remove_handler() self.server.stop_it() self.request.close() diff --git a/py/gaas_tcp_socket_server.py b/py/gaas_tcp_socket_server.py index ae254ebed7..759ce2866a 100644 --- a/py/gaas_tcp_socket_server.py +++ b/py/gaas_tcp_socket_server.py @@ -125,7 +125,6 @@ def handle_timeout(self): if idle and self.idle_limit > 0: self.logger.info(f"No client for {self.idle_limit / 60} minutes. Shutting down.") self.stop = True - socketserver.TCPServer.server_close(self) # Hard exit: a thread user code left running must not keep an abandoned # sandbox alive, and the sandbox ends with this process. os._exit(0) diff --git a/src/prerna/auth/User.java b/src/prerna/auth/User.java index f62026c3f3..43a8e6dca4 100644 --- a/src/prerna/auth/User.java +++ b/src/prerna/auth/User.java @@ -509,8 +509,8 @@ public SocketClient getPythonSocketClient(boolean create, int port, String venvE } return this.pythonCPW.getSocketClient(); } - if (OpenShellUserSandboxes.enabled()) { - return getOpenShellPythonSocketClient(); + if (OpenShellUserSandboxes.appliesTo(this)) { + return getOpenShellPythonSocketClient(venvEngineId); } if (this.pythonCPW == null || this.pythonCPW.getSocketClient() == null) { startPythonSocketServerAndClient(-1, venvEngineId); @@ -537,25 +537,32 @@ public SocketClient getPythonSocketClient(boolean create, int port, String venvE * (one connection, one record of what was transported); the wrapper attaches to the * sandbox if it is running anywhere and creates it otherwise. */ - private SocketClient getOpenShellPythonSocketClient() { - OpenShellUserSandboxes.Local local = OpenShellUserSandboxes.local(this); - synchronized (local) { - this.pythonCPW = local.getWrapper(); - // write-back permission checks run as the session that asked most recently - this.pythonCPW.setUser(this); - SocketClient client = this.pythonCPW.getSocketClient(); - if (client == null) { - startPythonSocketServerAndClient(-1, null); - } else if (!client.isConnected()) { - this.pythonCPW.shutdown(false); - try { - this.pythonCPW.reconnect(); - } catch (Exception e) { - classLogger.error("Failed to reconnect to user python process", e); - throw new IllegalArgumentException("Failed to connect to your isolated analytics engine"); + private SocketClient getOpenShellPythonSocketClient(String venvEngineId) { + while (true) { + OpenShellUserSandboxes.Local local = OpenShellUserSandboxes.local(this); + synchronized (local) { + if (local.isReleased()) { + // the user's last other session here ended between the lookup and the lock + continue; } + this.pythonCPW = local.getWrapper(); + // write-back permission checks run as the session that asked most recently + this.pythonCPW.setUser(this); + SocketClient client = this.pythonCPW.getSocketClient(); + if (client == null) { + startPythonSocketServerAndClient(-1, venvEngineId); + } else if (!client.isConnected()) { + this.pythonCPW.shutdown(false); + try { + this.pythonCPW.reconnect(); + } catch (Exception e) { + classLogger.error("Failed to reconnect to user python process", e); + throw new IllegalArgumentException("Failed to connect to your isolated analytics engine"); + } + } + this.pythonCPW.getSocketClient().setUser(this); + break; } - this.pythonCPW.getSocketClient().setUser(this); } this.insightSerializedMap.clear(); return this.pythonCPW.getSocketClient(); @@ -567,7 +574,7 @@ private SocketClient getOpenShellPythonSocketClient() { */ public SymlinkHelper getUserSymlinkHelper() { if (Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE))) { - if (OpenShellUserSandboxes.enabled()) { + if (OpenShellUserSandboxes.appliesTo(this)) { // shared by the user's sessions on this instance; see getOpenShellPythonSocketClient symlinkHelper = OpenShellUserSandboxes.local(this).getHelper(); chrootPath = symlinkHelper.getUserChrootFolder(); diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index 2dcac408c3..260f47d4c2 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -44,7 +44,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -62,15 +64,21 @@ import com.google.common.base.Strings; import com.google.gson.GsonBuilder; +import prerna.cluster.util.ClusterUtil; +import prerna.project.api.IProject; import prerna.tcp.client.NativePySocketClient; import prerna.tcp.client.SocketClient; import prerna.util.Constants; import prerna.util.DIHelper; +import prerna.util.OpenShellTransport; +import prerna.util.OpenShellTransporter; import prerna.util.PortAllocator; import prerna.util.SandboxInjector; import prerna.util.Settings; import prerna.util.SymlinkHelper; import prerna.util.Utility; +import prerna.util.openshell.OpenShellUserSandboxes; +import prerna.util.openshell.TransportedRoots; public class ClientProcessWrapper { @@ -130,7 +138,7 @@ public class ClientProcessWrapper { /** Whose permissions gate write-back; bound by {@code User} when the client is created. */ private prerna.auth.User user; // OPENSHELL mode: owns the sandbox + port-forward for this worker's lifetime - private prerna.util.OpenShellTransport openshell; + private OpenShellTransport openshell; /** * The sandbox belongs to the user, not to this wrapper: shutdown detaches and leaves * it running, and {@code OpenShellUserSandboxes} decides when it is deleted. @@ -150,8 +158,8 @@ public class ClientProcessWrapper { * Project roots written back since they were last published to cluster storage. Pushing * is what lets another server pull the change, and from there its sandboxes refresh. */ - private final java.util.Set unpublishedRoots = java.util.concurrent.ConcurrentHashMap.newKeySet(); - private final Map lastPublished = new java.util.concurrent.ConcurrentHashMap<>(); + private final Set unpublishedRoots = ConcurrentHashMap.newKeySet(); + private final Map lastPublished = new ConcurrentHashMap<>(); private static final long PUBLISH_INTERVAL_MS = 10_000L; private static final ExecutorService PUBLISHER = Executors.newSingleThreadExecutor(r -> { Thread t = new Thread(r, "openshell-writeback-publisher"); @@ -267,11 +275,10 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS // what selects the TCP branch below. if (this.chrootSymlinkHelper != null && this.chrootSymlinkHelper.isExternalSandbox()) { try { - if (this.user != null && prerna.util.openshell.OpenShellUserSandboxes.enabled()) { + if (OpenShellUserSandboxes.appliesTo(this.user)) { // The user's own sandbox, shared across their sessions and server // instances: attach if it is running, create it otherwise. - prerna.util.openshell.OpenShellUserSandboxes.Attached attached = prerna.util.openshell.OpenShellUserSandboxes - .acquire(this.user); + OpenShellUserSandboxes.Attached attached = OpenShellUserSandboxes.acquire(this.user); this.openshell = attached.transport; this.openshellUserScoped = true; this.port = attached.port; @@ -281,8 +288,7 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS this.fullCaptureOwed = true; } } else { - this.openshell = prerna.util.OpenShellTransport - .forSession(this.chrootSymlinkHelper.getUserChrootFolder()); + this.openshell = OpenShellTransport.forSession(this.chrootSymlinkHelper.getUserChrootFolder()); this.openshellUserScoped = false; this.port = this.openshell.start(); } @@ -292,7 +298,7 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS // replayed by setTransport, which is why this happens here rather than // before start(). this.chrootSymlinkHelper.setTransport( - new prerna.util.OpenShellTransporter(this.openshell)); + new OpenShellTransporter(this.openshell)); } catch (Exception e) { // Fail closed: never fall back to an in-Tomcat worker when the // deployment asked for an external boundary. @@ -490,12 +496,7 @@ private int fetchBackChangedPathsLocked() { List roots = new ArrayList<>(); for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { String normalized = Utility.normalizePath(path); - if (normalized.contains("/" + Constants.PROJECT_FOLDER + "/") - || normalized.contains("/" + Constants.ROOM_FOLDER + "/") - || normalized.contains("/" + Constants.USER_FOLDER + "/")) { - if (!canWriteBack(normalized)) { - continue; - } + if (TransportedRoots.isUserSpace(normalized) && canWriteBack(normalized)) { roots.add(normalized); } } @@ -530,9 +531,10 @@ private int fetchBackChangedPathsLocked() { int total = 0; for (Map.Entry> group : writes.entrySet()) { try { - if (this.openshell.fetchBackPaths(group.getKey(), group.getValue()) > 0) { + int written = this.openshell.fetchBackPaths(group.getKey(), group.getValue()); + if (written > 0) { this.unpublishedRoots.add(group.getKey()); - total++; + total += written; } } catch (Exception e) { // The worker has already dropped these paths from its pending set, so they @@ -551,8 +553,8 @@ private int fetchBackChangedPathsLocked() { } } if (total > 0 || removed > 0) { - classLogger.info("Fetch-back wrote into {} root(s) and removed {} file(s) from {} changed path(s)", total, - removed, changes.size()); + classLogger.info("Fetch-back wrote {} and removed {} file(s) from {} changed path(s)", total, removed, + changes.size()); } publishWrittenRoots(false); return total + removed; @@ -574,7 +576,7 @@ public void refreshTransportedRoots() { fetchBackTransportedRootsLocked(); } for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { - if (!isUserSpace(Utility.normalizePath(path))) { + if (!TransportedRoots.isUserSpace(Utility.normalizePath(path))) { continue; } try { @@ -588,12 +590,6 @@ public void refreshTransportedRoots() { } } - private static boolean isUserSpace(String normalized) { - return normalized.contains("/" + Constants.PROJECT_FOLDER + "/") - || normalized.contains("/" + Constants.ROOM_FOLDER + "/") - || normalized.contains("/" + Constants.USER_FOLDER + "/"); - } - /** * Push written-back project roots to cluster storage and announce them, so servers that * have the project loaded pull the change. Batched: a root is pushed at most every ten @@ -602,12 +598,12 @@ private static boolean isUserSpace(String normalized) { * @param force push everything pending now (teardown) */ private void publishWrittenRoots(boolean force) { - if (!prerna.cluster.util.ClusterUtil.IS_CLUSTER || this.unpublishedRoots.isEmpty()) { + if (!ClusterUtil.IS_CLUSTER || this.unpublishedRoots.isEmpty()) { return; } long now = System.currentTimeMillis(); for (String root : new ArrayList<>(this.unpublishedRoots)) { - String projectId = projectIdOf(root); + String projectId = TransportedRoots.projectIdOf(root); if (projectId == null) { // rooms and user assets have their own sync at logout this.unpublishedRoots.remove(root); @@ -620,9 +616,9 @@ private void publishWrittenRoots(boolean force) { this.lastPublished.put(root, now); Runnable push = () -> { try { - prerna.project.api.IProject project = Utility.getProject(projectId); + IProject project = Utility.getProject(projectId); if (project != null) { - prerna.cluster.util.ClusterUtil.pushProjectFolder(project, root); + ClusterUtil.pushProjectFolder(project, root); classLogger.info("Published write-back of {} to cluster storage", root); } } catch (Exception e) { @@ -638,20 +634,6 @@ private void publishWrittenRoots(boolean force) { } } - /** The project id of a root under {@code project/__/}, or null for other spaces. */ - private static String projectIdOf(String normalized) { - String marker = "/" + Constants.PROJECT_FOLDER + "/"; - int at = normalized.indexOf(marker); - if (at < 0) { - return null; - } - String rest = normalized.substring(at + marker.length()); - int slash = rest.indexOf('/'); - String folder = slash < 0 ? rest : rest.substring(0, slash); - int separator = folder.lastIndexOf("__"); - return separator < 0 ? folder : folder.substring(separator + 2); - } - private static boolean truthy(Object value) { if (value instanceof Boolean flag) { return flag; @@ -677,12 +659,7 @@ public void setUser(prerna.auth.User user) { * @return true when the asset belongs to this user */ private boolean isOwnUserAsset(String normalized) { - String marker = "/" + Constants.USER_FOLDER + "/"; - String rest = normalized.substring(normalized.indexOf(marker) + marker.length()); - int slash = rest.indexOf('/'); - String folder = slash < 0 ? rest : rest.substring(0, slash); - int separator = folder.lastIndexOf("__"); - String assetId = separator < 0 ? folder : folder.substring(separator + 2); + String assetId = TransportedRoots.userAssetIdOf(normalized); if (this.user == null) { classLogger.warn("Refusing write-back into {}: no user is bound to this process", normalized); return false; @@ -711,11 +688,7 @@ private boolean isOwnUserAsset(String normalized) { * @return true when the session's user owns the room */ private boolean canWriteRoom(String normalized) { - String marker = "/" + Constants.ROOM_FOLDER + "/"; - int at = normalized.indexOf(marker); - String rest = normalized.substring(at + marker.length()); - int slash = rest.indexOf('/'); - String roomId = slash < 0 ? rest : rest.substring(0, slash); + String roomId = TransportedRoots.roomIdOf(normalized); if (this.user == null) { classLogger.warn("Refusing write-back into room {}: no user is bound to this process", roomId); return false; @@ -751,23 +724,17 @@ private boolean canWriteRoom(String normalized) { * @return true when write-back into this root is allowed */ private boolean canWriteBack(String normalized) { - if (normalized.contains("/" + Constants.ROOM_FOLDER + "/")) { + if (TransportedRoots.isRoom(normalized)) { return canWriteRoom(normalized); } - if (normalized.contains("/" + Constants.USER_FOLDER + "/")) { + if (TransportedRoots.isUserAsset(normalized)) { return isOwnUserAsset(normalized); } - String marker = "/" + Constants.PROJECT_FOLDER + "/"; - int at = normalized.indexOf(marker); - if (at < 0) { + String projectId = TransportedRoots.projectIdOf(normalized); + if (projectId == null) { classLogger.warn("Refusing write-back into {}: unrecognised root space", normalized); return false; } - String rest = normalized.substring(at + marker.length()); - int slash = rest.indexOf('/'); - String folder = slash < 0 ? rest : rest.substring(0, slash); - int separator = folder.lastIndexOf("__"); - String projectId = separator < 0 ? folder : folder.substring(separator + 2); if (this.user == null) { classLogger.warn("Refusing write-back into {}: no user is bound to this process", normalized); return false; @@ -819,10 +786,7 @@ private int fetchBackTransportedRootsLocked() { String normalized = Utility.normalizePath(path); // Only the user-facing roots. Engine and runtime trees are never written by // user code, and pulling them back would waste a tar per block. - boolean syncable = normalized.contains("/" + Constants.PROJECT_FOLDER + "/") - || normalized.contains("/" + Constants.ROOM_FOLDER + "/") - || normalized.contains("/" + Constants.USER_FOLDER + "/"); - if (!syncable || !canWriteBack(normalized)) { + if (!TransportedRoots.isUserSpace(normalized) || !canWriteBack(normalized)) { continue; } try { @@ -1029,7 +993,7 @@ private int calculatePort(int port) { * OPENSHELL mode or before it started */ public String getOpenShellSandboxName() { - prerna.util.OpenShellTransport current = this.openshell; + OpenShellTransport current = this.openshell; return current == null ? null : current.getSandboxName(); } diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 0263dd50db..4bf5f62acb 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -27,15 +27,50 @@ *******************************************************************************/ package prerna.util; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import prerna.util.openshell.TransportedRoots; + /** * Transport to a SEMOSS python worker running inside an OpenShell sandbox. * @@ -51,11 +86,12 @@ * loopback over the gateway ({@code openshell forward service}). The existing * {@code NativePySocketClient} is unchanged: it connects to * {@code 127.0.0.1:} exactly as it does for the non-sandbox path. - *

  • What "the connection" means. The worker is single-connection - * ({@code --max_count 1}) and it is the sandbox's main process, so a client - * disconnect ends the worker and the sandbox. The connection must therefore - * be held for the session's life, and re-established by re-creating the sandbox, - * not merely by reconnecting a socket.
    • + *
  • What "the connection" means. In the session-scoped form + * ({@link #forSession}) the worker is single-connection ({@code --max_count 1}) and + * the sandbox's main process, so a client disconnect ends the worker and the + * sandbox; the connection is held for the session's life. In the user-scoped form + * ({@link #forUser}) the worker is persistent: it serves several clients, survives + * their disconnects, and any server instance can {@link #attach()} to it.
    • * * *

    @@ -180,6 +216,8 @@ private OpenShellTransport(String sandboxName, String workerPort, String owner) private static final long MAX_FETCH_BYTES = 256L * 1024 * 1024; /** Most paths one change set may name; the worker overflows to a full capture before this. */ private static final int MAX_CHANGED_PATHS = 2000; + /** Paths per {@code rm} when a refresh removes host-deleted files from the sandbox. */ + private static final int REFRESH_DELETE_BATCH = 500; /** * Relative paths, per transported root, that the sandbox is known to have held: @@ -188,13 +226,13 @@ private OpenShellTransport(String sandboxName, String workerPort, String owner) * gitignored, added on the host mid-session, missed by a partial upload -- is never * mistaken for one the sandbox removed. */ - private final java.util.Map> seenBySandbox = new java.util.concurrent.ConcurrentHashMap<>(); + private final Map> seenBySandbox = new ConcurrentHashMap<>(); /** * Size and mtime of each host file as this transport last left it (uploaded or * written back). A file that no longer matches was changed on the host since, and is * quarantined before a sandbox copy replaces it. */ - private final java.util.Map hostStamps = new java.util.concurrent.ConcurrentHashMap<>(); + private final Map hostStamps = new ConcurrentHashMap<>(); private static final String NAME_PREFIX = "pyw-"; @@ -215,14 +253,14 @@ public static String sandboxNameFor(String sessionKey) { int digestChars = MAX_SANDBOX_NAME_LENGTH - NAME_PREFIX.length(); String digest; try { - java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256"); - byte[] hash = md.digest(sessionKey.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] hash = md.digest(sessionKey.getBytes(StandardCharsets.UTF_8)); StringBuilder sb = new StringBuilder(); for (int i = 0; sb.length() < digestChars; i++) { sb.append(String.format("%02x", hash[i])); } digest = sb.substring(0, digestChars); - } catch (java.security.NoSuchAlgorithmException e) { + } catch (NoSuchAlgorithmException e) { throw new IllegalStateException("SHA-256 unavailable", e); } String name = NAME_PREFIX + digest; @@ -375,20 +413,39 @@ public void detach() { } } - /** The sandbox's phase ({@code Ready}, {@code Completed}, ...), or null when it does not exist. */ - public static String phaseOf(String sandboxName) { + /** + * The sandbox's phase ({@code Ready}, {@code Completed}, ...), or null when the gateway + * says it does not exist. + * + * @throws IOException when the gateway could not be asked or its answer not read. That + * is not "gone": a caller that deleted on it would destroy a live + * user's python over a network hiccup. + */ + public static String phaseOf(String sandboxName) throws IOException { + String out; try { - String out = run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "get", sandboxName, "-o", - "json"), 30); - com.google.gson.JsonObject json = com.google.gson.JsonParser.parseString(jsonBody(out)).getAsJsonObject(); + out = run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "get", sandboxName, "-o", "json"), + 30); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted asking for sandbox " + sandboxName, e); + } catch (IOException e) { + if (String.valueOf(e.getMessage()).toLowerCase().contains("not found")) { + return null; + } + throw e; + } + try { + JsonObject json = JsonParser.parseString(jsonBody(out)).getAsJsonObject(); if (json.has("sandbox") && json.get("sandbox").isJsonObject()) { json = json.getAsJsonObject("sandbox"); } - return json.has("phase") ? json.get("phase").getAsString() : null; - } catch (Exception e) { - // "not found" exits non-zero; anything unparseable is treated the same way, - // which only ever leads to creating a fresh sandbox. - return null; + if (!json.has("phase")) { + throw new IOException("no phase in the gateway's answer for sandbox " + sandboxName); + } + return json.get("phase").getAsString(); + } catch (RuntimeException e) { + throw new IOException("unreadable answer for sandbox " + sandboxName + ": " + out, e); } } @@ -403,26 +460,35 @@ public static void deleteSandbox(String sandboxName) { } /** - * Every executor sandbox on the gateway (selected by {@link #LABEL_PROP}), as name, - * phase, creation time ({@code yyyy-MM-dd HH:mm:ss}, UTC) and owner label (or null). + * One executor sandbox as the gateway lists it. + * + * @param createdAt {@code yyyy-MM-dd HH:mm:ss}, UTC, or empty + * @param owner the {@link #OWNER_LABEL} digest, or null for a session-scoped sandbox */ - public static List listExecutorSandboxes() throws IOException, InterruptedException { + public record SandboxInfo(String name, String phase, String createdAt, String owner) { + public boolean isReady() { + return "Ready".equalsIgnoreCase(phase); + } + } + + /** Every executor sandbox on the gateway, selected by {@link #LABEL_PROP}. */ + public static List listExecutorSandboxes() throws IOException, InterruptedException { String out = run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "list", "--selector", propOr(LABEL_PROP, DEFAULT_LABEL), "-o", "json"), 60); - List rows = new ArrayList<>(); - com.google.gson.JsonObject json = com.google.gson.JsonParser.parseString(jsonBody(out)).getAsJsonObject(); + List rows = new ArrayList<>(); + JsonObject json = JsonParser.parseString(jsonBody(out)).getAsJsonObject(); if (!json.has("sandboxes")) { return rows; } - for (com.google.gson.JsonElement element : json.getAsJsonArray("sandboxes")) { - com.google.gson.JsonObject box = element.getAsJsonObject(); - com.google.gson.JsonObject labels = box.has("labels") && box.get("labels").isJsonObject() + for (JsonElement element : json.getAsJsonArray("sandboxes")) { + JsonObject box = element.getAsJsonObject(); + JsonObject labels = box.has("labels") && box.get("labels").isJsonObject() ? box.getAsJsonObject("labels") - : new com.google.gson.JsonObject(); - rows.add(new String[] { box.get("name").getAsString(), + : new JsonObject(); + rows.add(new SandboxInfo(box.get("name").getAsString(), box.has("phase") ? box.get("phase").getAsString() : "", box.has("created_at") ? box.get("created_at").getAsString() : "", - labels.has(OWNER_LABEL) ? labels.get(OWNER_LABEL).getAsString() : null }); + labels.has(OWNER_LABEL) ? labels.get(OWNER_LABEL).getAsString() : null)); } return rows; } @@ -524,8 +590,8 @@ private void openForward(int readyTimeout) throws IOException { // A child whose pipe fills up blocks forever. The forward emits periodic // output, so it must be drained continuously or the port never binds. Thread drain = new Thread(() -> { - try (java.io.BufferedReader r = new java.io.BufferedReader(new java.io.InputStreamReader( - forwardProcess.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { + try (BufferedReader r = new BufferedReader(new InputStreamReader( + forwardProcess.getInputStream(), StandardCharsets.UTF_8))) { String line; while ((line = r.readLine()) != null) { synchronized (forwardLog) { @@ -616,8 +682,8 @@ public boolean upload(String absPath) { // produced /a/b/app_root/app_root/version/... (observed 2026-09-28, and the same // shape as the earlier X.py -> /path/X.py/X.py trap). The destination must be the // PARENT so the tree lands at the absolute path the worker expects. - String destination = java.nio.file.Paths.get(absPath).getParent() == null ? "/" - : java.nio.file.Paths.get(absPath).getParent().toString(); + String destination = Paths.get(absPath).getParent() == null ? "/" + : Paths.get(absPath).getParent().toString(); classLogger.info("Sandbox upload {} -> {}:{}", absPath, sandboxName, destination); try { String out = run(Arrays.asList(cli, "sandbox", "upload", sandboxName, absPath, destination), 300); @@ -625,10 +691,8 @@ public boolean upload(String absPath) { // which run() already raises as IOException. classLogger.info("Sandbox upload result for {}: {}", absPath, out.trim()); String normalized = Utility.normalizePath(absPath); - if (normalized.contains("/" + Constants.PROJECT_FOLDER + "/") - || normalized.contains("/" + Constants.ROOM_FOLDER + "/") - || normalized.contains("/" + Constants.USER_FOLDER + "/")) { - recordUploadedStamps(java.nio.file.Paths.get(normalized)); + if (TransportedRoots.isUserSpace(normalized)) { + recordUploadedStamps(Paths.get(normalized)); } return true; } catch (Exception e) { @@ -724,20 +788,20 @@ public int fetchBack(String absPath) throws IOException { classLogger.warn("Refusing fetch-back outside the SEMOSS home: {}", absPath); return 0; } - java.io.File source = new java.io.File(normalized); + File source = new File(normalized); String parent = source.getParent(); String base = source.getName(); if (parent == null) { return 0; } - java.nio.file.Path temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-"); + Path temp = Files.createTempDirectory("openshell-fetchback-"); try { String cli = propOr(CLI_PROP, DEFAULT_CLI); // .git is captured so that a commit made in the sandbox shell survives, but only // git's data files are ever written back -- see isWritableGitPath. Regenerable // caches are skipped. - java.nio.file.Path archive = temp.resolve("capture.tar"); + Path archive = temp.resolve("capture.tar"); Capture capture = runToFile(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", parent, "--ignore-failed-read", "--exclude=node_modules", "--exclude=__pycache__", "--exclude=.venv", "-cf", "-", base), 120, archive); @@ -745,7 +809,7 @@ public int fetchBack(String absPath) throws IOException { return 0; } requireSafeArchive(archive); - java.nio.file.Path tree = java.nio.file.Files.createDirectory(temp.resolve("tree")); + Path tree = Files.createDirectory(temp.resolve("tree")); extractArchive(archive, tree); // tar warns about anything it could not read (a mode-000 directory, say). What // it skipped is missing from the capture without having been deleted, so a @@ -755,7 +819,7 @@ public int fetchBack(String absPath) throws IOException { classLogger.warn("Capture of {} is partial; not propagating deletions this time: {}", absPath, capture.warnings); } - int changed = reconcile(tree.resolve(base), java.nio.file.Paths.get(normalized), complete); + int changed = reconcile(tree.resolve(base), Paths.get(normalized), complete); classLogger.info("Full capture of {}: {} bytes, {} file(s) changed on the host{}", absPath, capture.bytes, changed, complete ? "" : " (partial)"); return changed; @@ -767,21 +831,6 @@ public int fetchBack(String absPath) throws IOException { } } - /** - * Fetch back only the named paths under a root that is already transported. - * - *

    - * This is the narrow form of {@link #fetchBack(String)}: the sandbox worker - * reports which files a block actually wrote, so the host tars a handful of - * paths instead of the whole tree. Cost becomes proportional to the change - * rather than to the size of the project. - * - * @param absRoot absolute path of the transported root inside the sandbox - * @param relativePaths paths relative to that root, as reported by the watcher - * @return number of files written back - * @throws IOException when the paths could not be captured or applied; the change set - * is then lost and the caller must schedule a full capture - */ /** * Bring host-side changes to one transported root into the sandbox: files another user, * a cluster pull or the UI wrote since this sandbox last had them, and files removed on @@ -797,74 +846,74 @@ public int fetchBack(String absPath) throws IOException { * @return number of files copied in or removed */ public int refreshFromHost(String absRoot) throws IOException { - java.nio.file.Path root = java.nio.file.Paths.get(Utility.normalizePath(absRoot)); - if (!java.nio.file.Files.isDirectory(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + Path root = Paths.get(Utility.normalizePath(absRoot)); + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) { return 0; } - java.util.Set seen = seenBySandbox.computeIfAbsent(root.toString(), - key -> java.util.concurrent.ConcurrentHashMap.newKeySet()); - java.util.List changed = new ArrayList<>(); - java.util.Set present = new java.util.HashSet<>(); - java.nio.file.Files.walkFileTree(root, new java.nio.file.SimpleFileVisitor() { + Set seen = seenBySandbox.computeIfAbsent(root.toString(), + key -> ConcurrentHashMap.newKeySet()); + List changed = new ArrayList<>(); + Set present = new HashSet<>(); + Files.walkFileTree(root, new SimpleFileVisitor() { @Override - public java.nio.file.FileVisitResult preVisitDirectory(java.nio.file.Path dir, - java.nio.file.attribute.BasicFileAttributes attrs) { + public FileVisitResult preVisitDirectory(Path dir, + BasicFileAttributes attrs) { String name = dir.getFileName() == null ? "" : dir.getFileName().toString(); - return !dir.equals(root) && (attrs.isSymbolicLink() || isExcludedFromCapture(java.nio.file.Paths.get(name))) - ? java.nio.file.FileVisitResult.SKIP_SUBTREE - : java.nio.file.FileVisitResult.CONTINUE; + return !dir.equals(root) && (attrs.isSymbolicLink() || isExcludedFromCapture(Paths.get(name))) + ? FileVisitResult.SKIP_SUBTREE + : FileVisitResult.CONTINUE; } @Override - public java.nio.file.FileVisitResult visitFile(java.nio.file.Path file, - java.nio.file.attribute.BasicFileAttributes attrs) { + public FileVisitResult visitFile(Path file, + BasicFileAttributes attrs) { if (attrs.isRegularFile()) { present.add(file.toString()); - String stamp = attrs.size() + ":" + attrs.lastModifiedTime().to(TimeUnit.SECONDS); + String stamp = stampOf(attrs); if (!stamp.equals(hostStamps.get(file.toString()))) { changed.add(file); } } - return java.nio.file.FileVisitResult.CONTINUE; + return FileVisitResult.CONTINUE; } @Override - public java.nio.file.FileVisitResult visitFileFailed(java.nio.file.Path file, IOException e) { - return java.nio.file.FileVisitResult.CONTINUE; + public FileVisitResult visitFileFailed(Path file, IOException e) { + return FileVisitResult.CONTINUE; } }); // Removed on the host since the sandbox had it: only paths this transport synced. - String prefix = root.toString() + java.io.File.separator; - java.util.List removed = new ArrayList<>(); + String prefix = root.toString() + File.separator; + List removed = new ArrayList<>(); for (String known : new ArrayList<>(hostStamps.keySet())) { if (known.startsWith(prefix) && !present.contains(known) - && !java.nio.file.Files.exists(java.nio.file.Paths.get(known), java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + && !Files.exists(Paths.get(known), LinkOption.NOFOLLOW_LINKS)) { removed.add(known); } } if (changed.isEmpty() && removed.isEmpty()) { return 0; } + String cli = propOr(CLI_PROP, DEFAULT_CLI); if (changed.size() + removed.size() > MAX_CHANGED_PATHS) { - // A whole-tree change (a branch switch, a re-import): send the root again. + // A whole-tree change (a branch switch, a re-import): send the root again. The + // upload records every stamp and only adds, so host deletions still follow. classLogger.info("Refreshing all of {} in sandbox {}: {} host change(s)", root, sandboxName, changed.size() + removed.size()); if (!upload(root.toString())) { throw new IOException("re-upload of " + root + " failed"); } - return changed.size(); - } - String cli = propOr(CLI_PROP, DEFAULT_CLI); - int count = 0; - if (!changed.isEmpty()) { - java.nio.file.Path stage = java.nio.file.Files.createTempDirectory("openshell-refresh-"); + for (Path file : changed) { + seen.add(relativeKey(root, file)); + } + } else if (!changed.isEmpty()) { + Path stage = Files.createTempDirectory("openshell-refresh-"); try { - java.nio.file.Path staged = stage.resolve(root.getFileName().toString()); - for (java.nio.file.Path file : changed) { - java.nio.file.Path copy = staged.resolve(root.relativize(file).toString()); - java.nio.file.Files.createDirectories(copy.getParent()); - java.nio.file.Files.copy(file, copy, java.nio.file.StandardCopyOption.COPY_ATTRIBUTES, - java.nio.file.LinkOption.NOFOLLOW_LINKS); + Path staged = stage.resolve(root.getFileName().toString()); + for (Path file : changed) { + Path copy = staged.resolve(root.relativize(file).toString()); + Files.createDirectories(copy.getParent()); + Files.copy(file, copy, StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS); } // `sandbox upload` merges into the existing tree and keeps mtimes, so after it // both sides carry the stamp recorded below. @@ -876,34 +925,54 @@ public java.nio.file.FileVisitResult visitFileFailed(java.nio.file.Path file, IO } finally { deleteQuietly(stage); } - for (java.nio.file.Path file : changed) { + for (Path file : changed) { recordStamp(file); - seen.add(root.relativize(file).toString().replace('\\', '/')); + seen.add(relativeKey(root, file)); } - count += changed.size(); } - if (!removed.isEmpty()) { - List rm = new ArrayList<>(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "rm", - "-f", "--")); - rm.addAll(removed); + // In batches, so a mass delete cannot exceed the argument-length limit. + for (int from = 0; from < removed.size(); from += REFRESH_DELETE_BATCH) { + List batch = removed.subList(from, Math.min(removed.size(), from + REFRESH_DELETE_BATCH)); + List rm = new ArrayList<>( + Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "rm", "-f", "--")); + rm.addAll(batch); try { run(rm, 60); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException("interrupted refreshing " + root, e); } - for (String known : removed) { + for (String known : batch) { hostStamps.remove(known); - seen.remove(root.relativize(java.nio.file.Paths.get(known)).toString().replace('\\', '/')); + seen.remove(relativeKey(root, Paths.get(known))); } - count += removed.size(); } classLogger.info("Refreshed sandbox {} from {}: {} file(s) in, {} removed", sandboxName, root, changed.size(), removed.size()); - return count; + return changed.size() + removed.size(); + } + + /** A path's key in {@link #seenBySandbox}: relative to its root, forward slashes. */ + private static String relativeKey(Path root, Path file) { + return root.relativize(file).toString().replace('\\', '/'); } - public int fetchBackPaths(String absRoot, java.util.List relativePaths) throws IOException { + /** + * Fetch back only the named paths under a root that is already transported. + * + *

    + * This is the narrow form of {@link #fetchBack(String)}: the sandbox worker + * reports which files a block actually wrote, so the host tars a handful of + * paths instead of the whole tree. Cost becomes proportional to the change + * rather than to the size of the project. + * + * @param absRoot absolute path of the transported root inside the sandbox + * @param relativePaths paths relative to that root, as reported by the watcher + * @return number of files written back + * @throws IOException when the paths could not be captured or applied; the change set + * is then lost and the caller must schedule a full capture + */ + public int fetchBackPaths(String absRoot, List relativePaths) throws IOException { if (absRoot == null || relativePaths == null || relativePaths.isEmpty()) { return 0; } @@ -913,7 +982,7 @@ public int fetchBackPaths(String absRoot, java.util.List relativePaths) classLogger.warn("Refusing path fetch-back outside the SEMOSS home: {}", absRoot); return 0; } - java.util.List safe = new java.util.ArrayList<>(); + List safe = new ArrayList<>(); for (String relative : relativePaths) { if (relative == null || relative.isEmpty()) { continue; @@ -933,21 +1002,21 @@ public int fetchBackPaths(String absRoot, java.util.List relativePaths) + " path limit"); } - java.nio.file.Path temp = java.nio.file.Files.createTempDirectory("openshell-fetchback-paths-"); + Path temp = Files.createTempDirectory("openshell-fetchback-paths-"); try { String cli = propOr(CLI_PROP, DEFAULT_CLI); - java.util.List command = new java.util.ArrayList<>(Arrays.asList(cli, "sandbox", "exec", "-n", + List command = new ArrayList<>(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", normalized, "--ignore-failed-read", "--exclude=node_modules", "--exclude=__pycache__", "--exclude=.venv", "-cf", "-", "--")); command.addAll(safe); - java.nio.file.Path archive = temp.resolve("capture.tar"); + Path archive = temp.resolve("capture.tar"); if (runToFile(command, 120, archive).bytes == 0) { return 0; } requireSafeArchive(archive); - java.nio.file.Path tree = java.nio.file.Files.createDirectory(temp.resolve("tree")); + Path tree = Files.createDirectory(temp.resolve("tree")); extractArchive(archive, tree); - return reconcile(tree, java.nio.file.Paths.get(normalized), false); + return reconcile(tree, Paths.get(normalized), false); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException("path fetch-back from " + absRoot + " interrupted", e); @@ -964,19 +1033,19 @@ public int fetchBackPaths(String absRoot, java.util.List relativePaths) * root: paths the sandbox held before but no longer has are then propagated as * deletions. */ - private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, boolean fullTree) + private int reconcile(Path incoming, Path target, boolean fullTree) throws IOException { - if (!java.nio.file.Files.isDirectory(incoming)) { + if (!Files.isDirectory(incoming)) { return 0; } - java.util.Set seen = seenBySandbox.computeIfAbsent(target.toString(), - key -> java.util.concurrent.ConcurrentHashMap.newKeySet()); - java.util.Set previous = fullTree ? new java.util.HashSet<>(seen) : null; - java.util.Set captured = new java.util.HashSet<>(); - java.nio.file.Path store = quarantineStore(); + Set seen = seenBySandbox.computeIfAbsent(target.toString(), + key -> ConcurrentHashMap.newKeySet()); + Set previous = fullTree ? new HashSet<>(seen) : null; + Set captured = new HashSet<>(); + Path store = quarantineStore(); int[] copied = { 0 }; - try (java.util.stream.Stream walk = java.nio.file.Files.walk(incoming)) { - walk.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)) + try (Stream walk = Files.walk(incoming)) { + walk.filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) .forEach(path -> { String relative = incoming.relativize(path).toString().replace('\\', '/'); captured.add(relative); @@ -984,15 +1053,15 @@ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, bo classLogger.debug("Not writing back git control file {}", relative); return; } - java.nio.file.Path destination = containedHostPath(target, relative); + Path destination = containedHostPath(target, relative); if (destination == null) { classLogger.warn("Refusing write-back of {} under {}: path leaves the root or crosses a symlink", relative, target); return; } try { - if (java.nio.file.Files.exists(destination, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { - if (java.nio.file.Files.mismatch(path, destination) == -1L) { + if (Files.exists(destination, LinkOption.NOFOLLOW_LINKS)) { + if (Files.mismatch(path, destination) == -1L) { recordStamp(destination); return; } @@ -1024,7 +1093,7 @@ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, bo } } } - java.nio.file.Files.createDirectories(destination.getParent()); + Files.createDirectories(destination.getParent()); // Re-check now that the parents exist: creating them must not have // walked through a link that appeared in between. if (containedHostPath(target, relative) == null) { @@ -1032,9 +1101,9 @@ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, bo target); return; } - java.nio.file.Files.copy(path, destination, - java.nio.file.StandardCopyOption.REPLACE_EXISTING, - java.nio.file.StandardCopyOption.COPY_ATTRIBUTES); + Files.copy(path, destination, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES); recordStamp(destination); copied[0]++; } catch (IOException e) { @@ -1061,16 +1130,16 @@ private int reconcile(java.nio.file.Path incoming, java.nio.file.Path target, bo * the capture no longer contains was deleted inside the sandbox: quarantine the host * copy, then remove it. Host files the sandbox never held are left alone. */ - private int removeDeletedFromHost(java.nio.file.Path target, java.util.Set previous, - java.util.Set captured, java.nio.file.Path store) { + private int removeDeletedFromHost(Path target, Set previous, + Set captured, Path store) { int moved = 0; for (String relative : previous) { if (captured.contains(relative) || !isWritableGitPath(relative) - || isExcludedFromCapture(java.nio.file.Paths.get(relative))) { + || isExcludedFromCapture(Paths.get(relative))) { continue; } - java.nio.file.Path file = containedHostPath(target, relative); - if (file == null || !java.nio.file.Files.isRegularFile(file, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + Path file = containedHostPath(target, relative); + if (file == null || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { continue; } if (hostChangedSinceSync(file)) { @@ -1084,7 +1153,7 @@ private int removeDeletedFromHost(java.nio.file.Path target, java.util.Set seen = seenBySandbox.get(rootPath.toString()); + Path rootPath = Paths.get(Utility.normalizePath(root)); + Set seen = seenBySandbox.get(rootPath.toString()); if (seen == null || !seen.contains(relative)) { classLogger.debug("Ignoring delete of {} under {}: the sandbox never held it", relative, root); return false; @@ -1117,12 +1186,12 @@ public boolean deleteFromHost(String root, String relative) { if (!isWritableGitPath(relative)) { return false; } - java.nio.file.Path file = containedHostPath(rootPath, relative); + Path file = containedHostPath(rootPath, relative); if (file == null) { classLogger.warn("Refusing delete of {} under {}: path leaves the root or crosses a symlink", relative, root); return false; } - if (!java.nio.file.Files.isRegularFile(file, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { seen.remove(relative); return false; } @@ -1135,7 +1204,7 @@ public boolean deleteFromHost(String root, String relative) { return false; } try { - java.nio.file.Files.deleteIfExists(file); + Files.deleteIfExists(file); hostStamps.remove(file.toString()); seen.remove(relative); pruneEmptyParents(file, rootPath); @@ -1151,10 +1220,10 @@ public boolean deleteFromHost(String root, String relative) { * including) the transported root. Only empty directories are removed, so a folder * someone deliberately created stays where it is. */ - private static void pruneEmptyParents(java.nio.file.Path file, java.nio.file.Path root) { - java.nio.file.Path parent = file.getParent(); + private static void pruneEmptyParents(Path file, Path root) { + Path parent = file.getParent(); while (parent != null && parent.startsWith(root) && !parent.equals(root)) { - try (java.util.stream.Stream children = java.nio.file.Files.list(parent)) { + try (Stream children = Files.list(parent)) { if (children.findAny().isPresent()) { return; } @@ -1162,7 +1231,7 @@ private static void pruneEmptyParents(java.nio.file.Path file, java.nio.file.Pat return; } try { - java.nio.file.Files.deleteIfExists(parent); + Files.deleteIfExists(parent); } catch (IOException e) { return; } @@ -1175,8 +1244,8 @@ private static void pruneEmptyParents(java.nio.file.Path file, java.nio.file.Pat * missing from the archive under one of these names was never captured rather than * deleted, and must not be treated as a tombstone. */ - private static boolean isExcludedFromCapture(java.nio.file.Path relative) { - for (java.nio.file.Path part : relative) { + private static boolean isExcludedFromCapture(Path relative) { + for (Path part : relative) { String name = part.toString(); if ("node_modules".equals(name) || "__pycache__".equals(name) || ".venv".equals(name)) { return true; @@ -1214,7 +1283,7 @@ static boolean isWritableGitPath(String relative) { return true; } - private static final java.util.Set GIT_DATA_FILES = java.util.Set.of("HEAD", "ORIG_HEAD", "FETCH_HEAD", + private static final Set GIT_DATA_FILES = Set.of("HEAD", "ORIG_HEAD", "FETCH_HEAD", "MERGE_HEAD", "index", "packed-refs", "COMMIT_EDITMSG"); /** A relative path with no absolute prefix and no {@code ..} segment. */ @@ -1236,21 +1305,21 @@ static boolean isSafeRelative(String relative) { * symlink: a link already in a writable project -- committed to its repository, say -- * would otherwise redirect the write, or the delete, into another user's space. */ - static java.nio.file.Path containedHostPath(java.nio.file.Path root, String relative) { + static Path containedHostPath(Path root, String relative) { if (!isSafeRelative(relative)) { return null; } - java.nio.file.Path candidate = root.resolve(relative).normalize(); + Path candidate = root.resolve(relative).normalize(); if (!candidate.startsWith(root) || candidate.equals(root)) { return null; } - java.nio.file.Path current = root; - for (java.nio.file.Path part : root.relativize(candidate)) { + Path current = root; + for (Path part : root.relativize(candidate)) { current = current.resolve(part); - if (java.nio.file.Files.isSymbolicLink(current)) { + if (Files.isSymbolicLink(current)) { return null; } - if (!java.nio.file.Files.exists(current, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) { break; } } @@ -1258,7 +1327,7 @@ static java.nio.file.Path containedHostPath(java.nio.file.Path root, String rela } /** Remember a host file's state as this transport left it. */ - private void recordStamp(java.nio.file.Path file) { + private void recordStamp(Path file) { String stamp = stampOf(file); if (stamp != null) { hostStamps.put(file.toString(), stamp); @@ -1266,28 +1335,31 @@ private void recordStamp(java.nio.file.Path file) { } /** True unless the host file is exactly as this transport last left it. */ - private boolean hostChangedSinceSync(java.nio.file.Path file) { + private boolean hostChangedSinceSync(Path file) { String known = hostStamps.get(file.toString()); return known == null || !known.equals(stampOf(file)); } - private static String stampOf(java.nio.file.Path file) { + private static String stampOf(Path file) { try { - java.nio.file.attribute.BasicFileAttributes attributes = java.nio.file.Files.readAttributes(file, - java.nio.file.attribute.BasicFileAttributes.class, java.nio.file.LinkOption.NOFOLLOW_LINKS); - return attributes.size() + ":" + attributes.lastModifiedTime().to(TimeUnit.SECONDS); + return stampOf(Files.readAttributes(file, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS)); } catch (IOException e) { return null; } } /** - * Seconds, not milliseconds: tar and {@code sandbox upload} keep whole seconds, so a finer - * stamp would read as changed after every round trip. + * {@code size:mtime} in whole seconds: tar and {@code sandbox upload} keep only seconds, + * so a finer stamp would read as changed after every round trip. */ - private static long mtimeOf(java.nio.file.Path file) { + private static String stampOf(BasicFileAttributes attributes) { + return attributes.size() + ":" + attributes.lastModifiedTime().to(TimeUnit.SECONDS); + } + + /** Modification time in whole seconds, the resolution both sides of a sync share. */ + private static long mtimeOf(Path file) { try { - return java.nio.file.Files.getLastModifiedTime(file, java.nio.file.LinkOption.NOFOLLOW_LINKS) + return Files.getLastModifiedTime(file, LinkOption.NOFOLLOW_LINKS) .to(TimeUnit.SECONDS); } catch (IOException e) { return 0L; @@ -1295,14 +1367,14 @@ private static long mtimeOf(java.nio.file.Path file) { } /** Keep the sandbox's copy of a file whose host copy won a conflict. */ - private static void quarantineIncoming(java.nio.file.Path file, java.nio.file.Path target, String relative, - java.nio.file.Path store) { + private static void quarantineIncoming(Path file, Path target, String relative, + Path store) { try { - java.nio.file.Path destination = store.resolve(target.getFileName().toString() + ".sandbox") + Path destination = store.resolve(target.getFileName().toString() + ".sandbox") .resolve(relative); - java.nio.file.Files.createDirectories(destination.getParent()); - java.nio.file.Files.copy(file, destination, java.nio.file.StandardCopyOption.REPLACE_EXISTING, - java.nio.file.StandardCopyOption.COPY_ATTRIBUTES, java.nio.file.LinkOption.NOFOLLOW_LINKS); + Files.createDirectories(destination.getParent()); + Files.copy(file, destination, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS); classLogger.warn("Write-back kept the sandbox copy of {} at {}", relative, destination); } catch (Exception e) { classLogger.warn("Could not keep the sandbox copy of {}", relative, e); @@ -1314,36 +1386,36 @@ private static void quarantineIncoming(java.nio.file.Path file, java.nio.file.Pa * a host file nobody touched (overwrite it) from one edited on the host since * (quarantine it first). */ - private void recordUploadedStamps(java.nio.file.Path root) { - if (!java.nio.file.Files.isDirectory(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { - if (java.nio.file.Files.isRegularFile(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + private void recordUploadedStamps(Path root) { + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) { + if (Files.isRegularFile(root, LinkOption.NOFOLLOW_LINKS)) { recordStamp(root); } return; } try { - java.nio.file.Files.walkFileTree(root, new java.nio.file.SimpleFileVisitor() { + Files.walkFileTree(root, new SimpleFileVisitor() { @Override - public java.nio.file.FileVisitResult preVisitDirectory(java.nio.file.Path dir, - java.nio.file.attribute.BasicFileAttributes attrs) { + public FileVisitResult preVisitDirectory(Path dir, + BasicFileAttributes attrs) { String name = dir.getFileName() == null ? "" : dir.getFileName().toString(); - return isExcludedFromCapture(java.nio.file.Paths.get(name)) - ? java.nio.file.FileVisitResult.SKIP_SUBTREE - : java.nio.file.FileVisitResult.CONTINUE; + return isExcludedFromCapture(Paths.get(name)) + ? FileVisitResult.SKIP_SUBTREE + : FileVisitResult.CONTINUE; } @Override - public java.nio.file.FileVisitResult visitFile(java.nio.file.Path file, - java.nio.file.attribute.BasicFileAttributes attrs) { + public FileVisitResult visitFile(Path file, + BasicFileAttributes attrs) { if (attrs.isRegularFile()) { - hostStamps.put(file.toString(), attrs.size() + ":" + attrs.lastModifiedTime().to(TimeUnit.SECONDS)); + hostStamps.put(file.toString(), stampOf(attrs)); } - return java.nio.file.FileVisitResult.CONTINUE; + return FileVisitResult.CONTINUE; } @Override - public java.nio.file.FileVisitResult visitFileFailed(java.nio.file.Path file, IOException e) { - return java.nio.file.FileVisitResult.CONTINUE; + public FileVisitResult visitFileFailed(Path file, IOException e) { + return FileVisitResult.CONTINUE; } }); } catch (IOException e) { @@ -1352,24 +1424,24 @@ public java.nio.file.FileVisitResult visitFileFailed(java.nio.file.Path file, IO } /** Timestamped quarantine root, outside every transported root so it is never pushed. */ - private static java.nio.file.Path quarantineStore() { + private static Path quarantineStore() { String base = Utility.getBaseFolder(); - java.nio.file.Path store = java.nio.file.Paths + Path store = Paths .get(base == null ? System.getProperty("java.io.tmpdir") : base, "writeback-quarantine"); - String stamp = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss").format(new java.util.Date()); + String stamp = new SimpleDateFormat("yyyyMMdd-HHmmss").format(new Date()); return store.resolve(stamp); } /** Copy a host file aside before it is overwritten or removed. Returns the copy, or null. */ - private static java.nio.file.Path quarantineCopy(java.nio.file.Path file, java.nio.file.Path root, - java.nio.file.Path store) { + private static Path quarantineCopy(Path file, Path root, + Path store) { try { - java.nio.file.Path relative = root.relativize(file); - java.nio.file.Path destination = store.resolve(root.getFileName().toString()) + Path relative = root.relativize(file); + Path destination = store.resolve(root.getFileName().toString()) .resolve(relative.toString()); - java.nio.file.Files.createDirectories(destination.getParent()); - java.nio.file.Files.copy(file, destination, java.nio.file.StandardCopyOption.REPLACE_EXISTING, - java.nio.file.StandardCopyOption.COPY_ATTRIBUTES, java.nio.file.LinkOption.NOFOLLOW_LINKS); + Files.createDirectories(destination.getParent()); + Files.copy(file, destination, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS); classLogger.warn("Write-back preserved the host copy of {} at {}", file, destination); return destination; } catch (Exception e) { @@ -1383,7 +1455,7 @@ private static java.nio.file.Path quarantineCopy(java.nio.file.Path file, java.n * a {@code ..} segment. Checked per segment, so a file called {@code v1..v2.diff} is * fine. */ - private static void requireSafeArchive(java.nio.file.Path archive) throws IOException, InterruptedException { + private static void requireSafeArchive(Path archive) throws IOException, InterruptedException { String listing = runLocal(Arrays.asList("tar", "-tf", archive.toString()), 60); for (String line : listing.split("\\R")) { String entry = line.trim(); @@ -1404,14 +1476,14 @@ private static void requireSafeArchive(java.nio.file.Path archive) throws IOExce * sandbox left at mode 000 would otherwise be unreadable to the reconcile walk and * undeletable afterwards. */ - private static void extractArchive(java.nio.file.Path archive, java.nio.file.Path destination) + private static void extractArchive(Path archive, Path destination) throws IOException, InterruptedException { runLocal(Arrays.asList("tar", "-xf", archive.toString(), "-C", destination.toString(), "--no-same-owner"), 120); makeOwnerAccessible(destination); } - private static void makeOwnerAccessible(java.nio.file.Path root) { + private static void makeOwnerAccessible(Path root) { try { runLocal(Arrays.asList("chmod", "-R", "u+rwX", root.toString()), 60); } catch (Exception e) { @@ -1419,15 +1491,15 @@ private static void makeOwnerAccessible(java.nio.file.Path root) { } } - private static void deleteQuietly(java.nio.file.Path root) { - if (root == null || !java.nio.file.Files.exists(root)) { + private static void deleteQuietly(Path root) { + if (root == null || !Files.exists(root)) { return; } makeOwnerAccessible(root); - try (java.util.stream.Stream walk = java.nio.file.Files.walk(root)) { - walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { + try (Stream walk = Files.walk(root)) { + walk.sorted(Comparator.reverseOrder()).forEach(path -> { try { - java.nio.file.Files.deleteIfExists(path); + Files.deleteIfExists(path); } catch (IOException ignored) { // best effort } @@ -1441,11 +1513,11 @@ private static void deleteQuietly(java.nio.file.Path root) { private static String runLocal(List cmd, int timeoutSeconds) throws IOException, InterruptedException { ProcessBuilder builder = new ProcessBuilder(cmd); builder.redirectErrorStream(true); - builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); + builder.redirectInput(ProcessBuilder.Redirect.from(new File("/dev/null"))); Process process = builder.start(); - java.io.ByteArrayOutputStream collected = new java.io.ByteArrayOutputStream(); + ByteArrayOutputStream collected = new ByteArrayOutputStream(); Thread reader = new Thread(() -> { - try (java.io.InputStream stream = process.getInputStream()) { + try (InputStream stream = process.getInputStream()) { stream.transferTo(collected); } catch (IOException ignored) { // the exit code carries the outcome @@ -1458,7 +1530,7 @@ private static String runLocal(List cmd, int timeoutSeconds) throws IOEx throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); } reader.join(5000); - String output = collected.toString(java.nio.charset.StandardCharsets.UTF_8); + String output = collected.toString(StandardCharsets.UTF_8); if (process.exitValue() != 0) { throw new IOException("command failed (" + process.exitValue() + "): " + String.join(" ", cmd) + "\n" + output.trim()); @@ -1474,16 +1546,16 @@ private static String runLocal(List cmd, int timeoutSeconds) throws IOEx * * @return bytes written, and whatever the command wrote to stderr */ - private Capture runToFile(List cmd, int timeoutSeconds, java.nio.file.Path out) + private Capture runToFile(List cmd, int timeoutSeconds, Path out) throws IOException, InterruptedException { ProcessBuilder builder = new ProcessBuilder(cmd); builder.environment().putAll(cliEnv()); - builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); + builder.redirectInput(ProcessBuilder.Redirect.from(new File("/dev/null"))); Process process = builder.start(); StringBuilder errors = new StringBuilder(); Thread errorReader = new Thread(() -> { - try (java.io.BufferedReader reader = new java.io.BufferedReader( - new java.io.InputStreamReader(process.getErrorStream(), java.nio.charset.StandardCharsets.UTF_8))) { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getErrorStream(), StandardCharsets.UTF_8))) { String line; while ((line = reader.readLine()) != null) { synchronized (errors) { @@ -1498,12 +1570,12 @@ private Capture runToFile(List cmd, int timeoutSeconds, java.nio.file.Pa }, "openshell-fetchback-errors"); errorReader.setDaemon(true); errorReader.start(); - java.util.concurrent.atomic.AtomicLong written = new java.util.concurrent.atomic.AtomicLong(); - java.util.concurrent.atomic.AtomicBoolean overCap = new java.util.concurrent.atomic.AtomicBoolean(); + AtomicLong written = new AtomicLong(); + AtomicBoolean overCap = new AtomicBoolean(); Thread reader = new Thread(() -> { byte[] buffer = new byte[64 * 1024]; - try (java.io.InputStream stream = process.getInputStream(); - java.io.OutputStream sink = java.nio.file.Files.newOutputStream(out)) { + try (InputStream stream = process.getInputStream(); + OutputStream sink = Files.newOutputStream(out)) { int read; while ((read = stream.read(buffer)) > 0) { if (written.addAndGet(read) > MAX_FETCH_BYTES) { @@ -1577,8 +1649,8 @@ public void close() { * Environment for CLI children: inherit the caller's, but pin {@code HOME} so the * CLI always finds the same gateway registration and client certificate bundle. */ - private static java.util.Map cliEnv() { - java.util.Map env = new java.util.HashMap<>(System.getenv()); + private static Map cliEnv() { + Map env = new HashMap<>(System.getenv()); String home = System.getProperty(CLI_HOME_PROP); if (home == null || home.trim().isEmpty()) { home = Utility.getDIHelperProperty(CLI_HOME_PROP); @@ -1611,7 +1683,7 @@ private static int intPropOr(String key, int fallback) { * host-local, and two sessions must not collide. */ private static int freeLocalPort() throws IOException { - try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { + try (ServerSocket s = new ServerSocket(0)) { return s.getLocalPort(); } } @@ -1627,7 +1699,7 @@ private static String run(List cmd, int timeoutSeconds) throws IOExcepti // instantly from a shell (stdin at EOF) and timed out after 30s under // Tomcat, which is what made the readiness poll fail and the pixel stall for // the full 180s. A shell would not have shown this. - pb.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); + pb.redirectInput(ProcessBuilder.Redirect.from(new File("/dev/null"))); Process p = pb.start(); // Read on a separate thread. A blocking readLine() here hangs the CALLER -- @@ -1636,8 +1708,8 @@ private static String run(List cmd, int timeoutSeconds) throws IOExcepti // no timeout, and the sandbox was never cleaned up. Measured 2026-09-28. StringBuilder out = new StringBuilder(); Thread reader = new Thread(() -> { - try (java.io.BufferedReader r = new java.io.BufferedReader(new java.io.InputStreamReader( - p.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { + try (BufferedReader r = new BufferedReader(new InputStreamReader( + p.getInputStream(), StandardCharsets.UTF_8))) { String line; while ((line = r.readLine()) != null) { synchronized (out) { diff --git a/src/prerna/util/openshell/OpenShellShellRunner.java b/src/prerna/util/openshell/OpenShellShellRunner.java index 75db50b420..daac612686 100644 --- a/src/prerna/util/openshell/OpenShellShellRunner.java +++ b/src/prerna/util/openshell/OpenShellShellRunner.java @@ -27,7 +27,9 @@ *******************************************************************************/ package prerna.util.openshell; +import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -36,7 +38,11 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.auth.User; +import prerna.om.ClientProcessWrapper; import prerna.util.OpenShellTransport; import prerna.util.Utility; @@ -53,10 +59,9 @@ * *

    * The command is run with {@code openshell sandbox exec}, which starts a process - * inside the sandbox. That is deliberately not the worker socket: the - * worker serves one connection and a shell command must not compete with the - * REPL for it. It also means shell output does not require the python worker to - * be healthy. + * inside the sandbox. That is deliberately not the worker socket: a shell + * command must not queue behind a running python block on the same connection, + * and shell output does not require the python worker to be healthy. * *

    * The sandbox is started on demand if it is not running yet, so a terminal @@ -65,6 +70,8 @@ */ public final class OpenShellShellRunner { + private static final Logger classLogger = LogManager.getLogger(OpenShellShellRunner.class); + private static final int DEFAULT_TIMEOUT_SECONDS = 60; /** Most output returned to the terminal for one command. */ private static final int MAX_OUTPUT_BYTES = 4 * 1024 * 1024; @@ -108,7 +115,7 @@ public static Result execute(User user, String workingDir, String command) { user.getPythonSocketClient(true); // The sandbox the user's worker is actually in: a user-scoped one is named by the // registry, not derived from this instance's chroot folder. - prerna.om.ClientProcessWrapper cpw = user.getPythonClientProcessWrapper(); + ClientProcessWrapper cpw = user.getPythonClientProcessWrapper(); String sandboxName = cpw == null ? null : cpw.getOpenShellSandboxName(); if (sandboxName == null) { sandboxName = OpenShellTransport.sandboxNameFor(user.getUserSymlinkHelper().getUserChrootFolder()); @@ -145,18 +152,20 @@ public static Result execute(User user, String workingDir, String command) { cpw.refreshTransportedRoots(); } catch (Exception refreshError) { // a stale copy is better than a refused command + classLogger.warn("Refresh before shell command failed", refreshError); } } String output = runInSandbox(sandboxName, dir, trimmed).output; // Shell writes are not requests on the python socket, so they do not pass the // fetch-back hook there. Reconcile here too -- this is the path a `git commit` // or a `cat > file` takes, and losing that work is exactly what we are fixing. - try { - if (user.getPythonClientProcessWrapper() != null) { - user.getPythonClientProcessWrapper().fetchBackChangedPaths(); + if (cpw != null) { + try { + cpw.fetchBackChangedPaths(); + } catch (Exception reconcileError) { + // the command itself already ran; a failed reconcile must not fail the shell + classLogger.warn("Write-back after shell command failed", reconcileError); } - } catch (Exception ignored) { - // the command itself already ran; a failed reconcile must not fail the shell } return new Result(output, dir); } catch (Exception e) { @@ -207,7 +216,7 @@ private static Outcome runInSandbox(String sandboxName, String workingDir, Strin ProcessBuilder builder = new ProcessBuilder(argv); builder.redirectErrorStream(true); builder.redirectOutput(capture.toFile()); - builder.redirectInput(ProcessBuilder.Redirect.from(new java.io.File("/dev/null"))); + builder.redirectInput(ProcessBuilder.Redirect.from(new File("/dev/null"))); // The CLI needs the same environment the transport uses (gateway config, // TLS bundle); inherit it rather than reconstructing it here. Process process = builder.start(); @@ -232,7 +241,7 @@ private static Outcome runInSandbox(String sandboxName, String workingDir, Strin private static String readCapped(Path capture) throws IOException { long size = Files.size(capture); byte[] bytes; - try (java.io.InputStream in = Files.newInputStream(capture)) { + try (InputStream in = Files.newInputStream(capture)) { bytes = in.readNBytes((int) Math.min(size, MAX_OUTPUT_BYTES)); } String text = new String(bytes, StandardCharsets.UTF_8).replace("\r", "").trim(); diff --git a/src/prerna/util/openshell/OpenShellUserSandboxes.java b/src/prerna/util/openshell/OpenShellUserSandboxes.java index 0679ac3e2e..12ec969ec9 100644 --- a/src/prerna/util/openshell/OpenShellUserSandboxes.java +++ b/src/prerna/util/openshell/OpenShellUserSandboxes.java @@ -27,14 +27,20 @@ *******************************************************************************/ package prerna.util.openshell; +import java.io.IOException; +import java.net.InetAddress; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.Duration; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.HexFormat; import java.util.List; import java.util.Map; import java.util.Set; @@ -54,6 +60,7 @@ import prerna.redis.RedisConnectionFactory; import prerna.util.Constants; import prerna.util.OpenShellTransport; +import prerna.util.OpenShellTransport.SandboxInfo; import prerna.util.SymlinkHelper; import prerna.util.Utility; import redis.clients.jedis.UnifiedJedis; @@ -88,13 +95,21 @@ public final class OpenShellUserSandboxes { private static final Logger classLogger = LogManager.getLogger(OpenShellUserSandboxes.class); - private static final String KEY_PREFIX = "semoss:openshell:"; /** How often a session may refresh its lease; requests in between are free. */ private static final long TOUCH_INTERVAL_MS = TimeUnit.SECONDS.toMillis(60); /** How long a creator may hold the per-user create lock. */ private static final long CREATE_LOCK_MS = TimeUnit.MINUTES.toMillis(5); /** A labelled sandbox younger than this is never swept: its registry entry may not exist yet. */ - private static final long SWEEP_GRACE_MS = TimeUnit.MINUTES.toMillis(10); + private static final Duration SWEEP_GRACE = Duration.ofMinutes(10); + private static final long SWEEP_INTERVAL_MINUTES = 5; + /** How long {@link #acquire} keeps trying: a create is bounded by the CLI's ready timeout. */ + private static final long ACQUIRE_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(240); + /** Pause between attempts while another instance creates, or the gateway does not answer. */ + private static final long ACQUIRE_POLL_MS = 1000; + /** Failed attaches to a Ready sandbox before it is replaced; one failure may be a hiccup. */ + private static final int MAX_ATTACH_ATTEMPTS = 2; + /** Session lease when the container gives no timeout (a negative interval never expires). */ + private static final long DEFAULT_LEASE_MS = TimeUnit.DAYS.toMillis(1); private static final DateTimeFormatter CREATED_AT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); /** Distinguishes this JVM's leases from ones a restored session carries over a restart. */ @@ -104,6 +119,12 @@ public final class OpenShellUserSandboxes { private static final Map LOCALS = new ConcurrentHashMap<>(); private static final Map> LOCAL_SESSIONS = new ConcurrentHashMap<>(); private static final Object LOCALS_LOCK = new Object(); + /** + * Serialises, per user, the changes to which of their sessions live here and the release + * of their connection, so a session ending cannot tear down a wrapper another session + * of theirs just started using. Always taken before a {@link Local}'s monitor. + */ + private static final Map OWNER_LOCKS = new ConcurrentHashMap<>(); private static volatile Registry registry; private static volatile ScheduledExecutorService sweeper; @@ -118,6 +139,18 @@ public static boolean enabled() { String.valueOf(Utility.getDIHelperProperty(Constants.SANDBOX_MODE)).trim()); } + /** + * Whether this user's python lives in a user-scoped sandbox: the jail is OpenShell and + * the user has a primary login to scope it by. Anyone else keeps the per-session form. + */ + public static boolean appliesTo(User user) { + if (user == null || !enabled()) { + return false; + } + AccessToken token = user.getPrimaryLoginToken(); + return token != null && token.getId() != null; + } + /** * The digest a user's sandbox is filed under: provider and id of the primary login. * Used as the registry key and the {@code semoss-owner} label, so neither carries the @@ -135,12 +168,12 @@ public static String ownerOf(User user) { /** What this instance holds for one user, shared by all of their sessions here. */ public static final class Local { - private final String owner; private final SymlinkHelper helper; private final ClientProcessWrapper cpw = new ClientProcessWrapper(); + /** Set under this object's monitor when the instance gives the user up. */ + private boolean released; - private Local(String owner, SymlinkHelper helper) { - this.owner = owner; + private Local(SymlinkHelper helper) { this.helper = helper; } @@ -151,6 +184,11 @@ public SymlinkHelper getHelper() { public ClientProcessWrapper getWrapper() { return cpw; } + + /** Whether this was released; read under this object's monitor. */ + public boolean isReleased() { + return released; + } } /** @@ -169,7 +207,7 @@ public static Local local(User user) { String chroot = Utility.getDIHelperProperty(Constants.CHROOT_DIR) + "/" + User.getSingleLogginName(user) + "-" + UUID.randomUUID(); SymlinkHelper helper = new SymlinkHelper(chroot); - local = new Local(owner, helper); + local = new Local(helper); // Published before linking the asset folder: opening the asset project can // ask the user for this helper again, and must get this one. LOCALS.put(owner, local); @@ -210,27 +248,44 @@ private Attached(OpenShellTransport transport, int port, boolean reattached) { public static Attached acquire(User user) throws Exception { String owner = ownerOf(user); Registry store = registry(); - long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(240); + long deadline = System.currentTimeMillis() + ACQUIRE_TIMEOUT_MS; + int attachFailures = 0; while (System.currentTimeMillis() < deadline) { String name = store.sandboxFor(owner); if (name != null) { - String phase = OpenShellTransport.phaseOf(name); + String phase; + try { + phase = OpenShellTransport.phaseOf(name); + } catch (IOException e) { + // Unknown is not gone: deleting here would destroy the user's python over a + // gateway hiccup. + classLogger.warn("Could not read the state of sandbox {}; retrying", name, e); + Thread.sleep(ACQUIRE_POLL_MS); + continue; + } if ("Ready".equalsIgnoreCase(phase)) { OpenShellTransport transport = OpenShellTransport.forUser(name, owner); try { - int port = transport.attach(); - return new Attached(transport, port, true); + return new Attached(transport, transport.attach(), true); } catch (Exception e) { + if (++attachFailures < MAX_ATTACH_ATTEMPTS) { + classLogger.warn("Could not attach to sandbox {}; retrying", name, e); + Thread.sleep(ACQUIRE_POLL_MS); + continue; + } classLogger.warn("Could not attach to sandbox {}; replacing it", name, e); } } else { - classLogger.info("Sandbox {} for this user is {}; replacing it", name, phase); + classLogger.info("Sandbox {} for this user is {}; replacing it", name, + phase == null ? "gone" : phase); } OpenShellTransport.deleteSandbox(name); store.forget(owner, name); + attachFailures = 0; continue; } - if (store.lock(owner)) { + String lockToken = store.tryLock(owner); + if (lockToken != null) { try { if (store.sandboxFor(owner) != null) { continue; @@ -250,11 +305,11 @@ public static Attached acquire(User user) throws Exception { classLogger.info("Created sandbox {} for user {} on {}", name, owner, NODE); return new Attached(transport, port, false); } finally { - store.unlock(owner); + store.unlock(owner, lockToken); } } // Another instance is creating it; its entry appears when the sandbox is Ready. - Thread.sleep(1000); + Thread.sleep(ACQUIRE_POLL_MS); } throw new IllegalStateException("timed out waiting for the user's sandbox"); } @@ -262,9 +317,9 @@ public static Attached acquire(User user) throws Exception { /** A {@code Ready} sandbox carrying this owner's label, or null. */ private static String runningSandboxLabelled(String owner) { try { - for (String[] box : OpenShellTransport.listExecutorSandboxes()) { - if (owner.equals(box[3]) && "Ready".equalsIgnoreCase(box[1])) { - return box[0]; + for (SandboxInfo box : OpenShellTransport.listExecutorSandboxes()) { + if (owner.equals(box.owner()) && box.isReady()) { + return box.name(); } } } catch (Exception e) { @@ -284,9 +339,15 @@ private static String runningSandboxLabelled(String owner) { * @return the marker to store on the session, or null when nothing was refreshed */ public static String touch(User user, String sessionId, String lastTouch, int ttlSeconds) { - if (user == null || sessionId == null || !user.isLoggedIn()) { + if (sessionId == null || !appliesTo(user) || !user.isLoggedIn()) { return null; } + String owner = ownerOf(user); + // Every request, not only refreshing ones: this set is what keeps the user's + // connection here from being released while a session of theirs is still active. + synchronized (ownerLock(owner)) { + LOCAL_SESSIONS.computeIfAbsent(owner, k -> ConcurrentHashMap.newKeySet()).add(sessionId); + } long now = System.currentTimeMillis(); if (lastTouch != null && lastTouch.startsWith(BOOT_ID + ":")) { try { @@ -298,10 +359,8 @@ public static String touch(User user, String sessionId, String lastTouch, int tt } } try { - String owner = ownerOf(user); - long ttlMs = TimeUnit.SECONDS.toMillis(ttlSeconds > 0 ? ttlSeconds : TimeUnit.DAYS.toSeconds(1)); + long ttlMs = ttlSeconds > 0 ? TimeUnit.SECONDS.toMillis(ttlSeconds) : DEFAULT_LEASE_MS; registry().touch(owner, sessionId, now + ttlMs); - LOCAL_SESSIONS.computeIfAbsent(owner, k -> ConcurrentHashMap.newKeySet()).add(sessionId); ensureSweeper(); } catch (Exception e) { classLogger.warn("Could not refresh the sandbox lease for session {}", sessionId, e); @@ -324,22 +383,19 @@ public static String touch(User user, String sessionId, String lastTouch, int tt * @param over true for a logout or a timeout; false when the server is going away */ public static void sessionEnded(User user, String sessionId, boolean over) { - if (user == null || !user.isLoggedIn()) { + if (!appliesTo(user) || !user.isLoggedIn()) { return; } - String owner; - try { - owner = ownerOf(user); - } catch (IllegalStateException e) { - return; - } - Set here = LOCAL_SESSIONS.get(owner); - if (here != null) { - here.remove(sessionId); - } - if (here == null || here.isEmpty()) { - LOCAL_SESSIONS.remove(owner); - releaseLocal(owner); + String owner = ownerOf(user); + synchronized (ownerLock(owner)) { + Set here = LOCAL_SESSIONS.get(owner); + if (here != null) { + here.remove(sessionId); + } + if (here == null || here.isEmpty()) { + LOCAL_SESSIONS.remove(owner); + releaseLocal(owner); + } } if (!over) { classLogger.info("Session {} left with the server; keeping its sandbox lease", sessionId); @@ -357,19 +413,29 @@ public static void sessionEnded(User user, String sessionId, boolean over) { } } + /** Caller holds {@link #ownerLock}. */ private static void releaseLocal(String owner) { Local local = LOCALS.remove(owner); if (local == null) { return; } - try { - // Write back, close this instance's connection, stop the forward. The sandbox - // itself is not touched here. - local.cpw.shutdown(true); - } catch (Exception e) { - classLogger.warn("Failed to release the sandbox connection for {}", owner, e); + // The same monitor User takes while it binds and starts this wrapper, so a release + // never runs underneath a request that is connecting. + synchronized (local) { + local.released = true; + try { + // Write back, close this instance's connection, stop the forward. The sandbox + // itself is not touched here. + local.cpw.shutdown(true); + } catch (Exception e) { + classLogger.warn("Failed to release the sandbox connection for {}", owner, e); + } + local.helper.removeChrootFolder(); } - local.helper.removeChrootFolder(); + } + + private static Object ownerLock(String owner) { + return OWNER_LOCKS.computeIfAbsent(owner, k -> new Object()); } private static void deleteFor(String owner, String reason) { @@ -396,7 +462,8 @@ private static void ensureSweeper() { t.setDaemon(true); return t; }); - service.scheduleWithFixedDelay(OpenShellUserSandboxes::sweepQuietly, 1, 5, TimeUnit.MINUTES); + service.scheduleWithFixedDelay(OpenShellUserSandboxes::sweepQuietly, 1, SWEEP_INTERVAL_MINUTES, + TimeUnit.MINUTES); sweeper = service; } } @@ -426,41 +493,41 @@ public static void sweep() throws Exception { long now = System.currentTimeMillis(); for (String owner : store.owners()) { if (!store.anyLive(owner, now)) { - Set here = LOCAL_SESSIONS.remove(owner); - if (here != null) { - here.clear(); + synchronized (ownerLock(owner)) { + LOCAL_SESSIONS.remove(owner); + releaseLocal(owner); } - releaseLocal(owner); deleteFor(owner, "no live session for this user"); } } - LocalDateTime cutoff = LocalDateTime.now(ZoneOffset.UTC).minusNanos(TimeUnit.MILLISECONDS.toNanos(SWEEP_GRACE_MS)); - List boxes = OpenShellTransport.listExecutorSandboxes(); - for (String[] box : boxes) { - String name = box[0]; - String phase = box[1]; - String owner = box[3]; + LocalDateTime cutoff = LocalDateTime.now(ZoneOffset.UTC).minus(SWEEP_GRACE); + int unreadable = 0; + for (SandboxInfo box : OpenShellTransport.listExecutorSandboxes()) { LocalDateTime created; try { - created = LocalDateTime.parse(box[2], CREATED_AT); - } catch (Exception e) { + created = LocalDateTime.parse(box.createdAt(), CREATED_AT); + } catch (DateTimeParseException e) { + unreadable++; continue; } if (created.isAfter(cutoff)) { continue; } - boolean running = "Ready".equalsIgnoreCase(phase); - if (!running) { - classLogger.info("Sweeping sandbox {} (phase {})", name, phase); - OpenShellTransport.deleteSandbox(name); - if (owner != null) { - store.forget(owner, name); + if (!box.isReady()) { + classLogger.info("Sweeping sandbox {} (phase {})", box.name(), box.phase()); + OpenShellTransport.deleteSandbox(box.name()); + if (box.owner() != null) { + store.forget(box.owner(), box.name()); } - } else if (owner != null && store.shared() && !name.equals(store.sandboxFor(owner))) { - classLogger.info("Sweeping sandbox {} (no registry entry points at it)", name); - OpenShellTransport.deleteSandbox(name); + } else if (box.owner() != null && store.shared() && !box.name().equals(store.sandboxFor(box.owner()))) { + classLogger.info("Sweeping sandbox {} (no registry entry points at it)", box.name()); + OpenShellTransport.deleteSandbox(box.name()); } } + if (unreadable > 0) { + // Said out loud: a changed CLI date format would otherwise silently stop all sweeping. + classLogger.warn("Skipped {} sandbox(es) whose creation time could not be read", unreadable); + } } // ------------------------------------------------------------------ registry @@ -491,9 +558,11 @@ interface Registry { /** Remove the entry only if it still names this sandbox. */ void forget(String owner, String sandboxName); - boolean lock(String owner); + /** Take the per-user create lock; the token that holds it, or null when another does. */ + String tryLock(String owner); - void unlock(String owner); + /** Release the create lock, only if this token still holds it. */ + void unlock(String owner, String token); void touch(String owner, String sessionId, long expiresAtMs); @@ -508,32 +577,27 @@ interface Registry { void drop(String owner); } + /** + * The shared registry. Laid out like the room coordination keys + * ({@code RoomMessageRedisClient}): every key of one user carries the + * {@code {owner}} hash tag, so they share a Redis Cluster slot and a script can + * touch them together; the lock is {@code SET NX PX} with a random token, released + * only by its holder. + */ static final class RedisRegistry implements Registry { + private static final String UNLOCK_SCRIPT = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end"; + private static final String FORGET_SCRIPT = "if redis.call('hget', KEYS[1], 'name') == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end"; + /** Forget a user with no session and no sandbox left; 1 when it did. */ + private static final String DROP_SCRIPT = "if redis.call('zcard', KEYS[1]) == 0 and redis.call('exists', KEYS[2]) == 0 then return redis.call('del', KEYS[1]) + 1 else return 0 end"; + /** Every user with an entry or a lease; a separate slot, so kept apart from the scripts. */ + private static final String OWNERS_KEY = "openshell:owners"; + private final RedisConnectionConfig config; - private final String lockToken = NODE + ":" + BOOT_ID; RedisRegistry(RedisConnectionConfig config) { this.config = config; } - private UnifiedJedis redis() { - return RedisConnectionFactory.getClient(config); - } - - private static String sandboxKey(String owner) { - return KEY_PREFIX + "sandbox:" + owner; - } - - private static String sessionsKey(String owner) { - return KEY_PREFIX + "sessions:" + owner; - } - - private static String lockKey(String owner) { - return KEY_PREFIX + "lock:" + owner; - } - - private static final String OWNERS = KEY_PREFIX + "owners"; - @Override public boolean shared() { return true; @@ -541,7 +605,7 @@ public boolean shared() { @Override public String sandboxFor(String owner) { - return redis().hget(sandboxKey(owner), "name"); + return withJedis(jedis -> jedis.hget(sandboxKey(owner), "name")); } @Override @@ -551,72 +615,104 @@ public void bind(String owner, String sandboxName) { record.put("node", NODE); record.put("createdAt", String.valueOf(System.currentTimeMillis())); record.put("workerPort", String.valueOf(Utility.getDIHelperProperty(OpenShellTransport.WORKER_PORT_PROP))); - UnifiedJedis redis = redis(); - redis.hset(sandboxKey(owner), record); - redis.sadd(OWNERS, owner); + withJedis(jedis -> { + jedis.hset(sandboxKey(owner), record); + return jedis.sadd(OWNERS_KEY, owner); + }); } @Override public void forget(String owner, String sandboxName) { - UnifiedJedis redis = redis(); - if (sandboxName.equals(redis.hget(sandboxKey(owner), "name"))) { - redis.del(sandboxKey(owner)); - } + withJedis(jedis -> jedis.eval(FORGET_SCRIPT, Collections.singletonList(sandboxKey(owner)), + Collections.singletonList(sandboxName))); } @Override - public boolean lock(String owner) { - return "OK".equals(redis().set(lockKey(owner), lockToken, SetParams.setParams().nx().px(CREATE_LOCK_MS))); + public String tryLock(String owner) { + String token = UUID.randomUUID().toString(); + boolean held = withJedis(jedis -> "OK" + .equals(jedis.set(lockKey(owner), token, SetParams.setParams().nx().px(CREATE_LOCK_MS)))); + return held ? token : null; } @Override - public void unlock(String owner) { - UnifiedJedis redis = redis(); - if (lockToken.equals(redis.get(lockKey(owner)))) { - redis.del(lockKey(owner)); - } + public void unlock(String owner, String token) { + withJedis(jedis -> jedis.eval(UNLOCK_SCRIPT, Collections.singletonList(lockKey(owner)), + Collections.singletonList(token))); } @Override public void touch(String owner, String sessionId, long expiresAtMs) { - UnifiedJedis redis = redis(); - redis.zadd(sessionsKey(owner), expiresAtMs, sessionId); - redis.sadd(OWNERS, owner); + withJedis(jedis -> { + jedis.zadd(sessionsKey(owner), expiresAtMs, sessionId); + return jedis.sadd(OWNERS_KEY, owner); + }); } @Override public boolean release(String owner, String sessionId, long nowMs) { - redis().zrem(sessionsKey(owner), sessionId); + withJedis(jedis -> jedis.zrem(sessionsKey(owner), sessionId)); return anyLive(owner, nowMs); } @Override public boolean anyLive(String owner, long nowMs) { - UnifiedJedis redis = redis(); - redis.zremrangeByScore(sessionsKey(owner), Double.NEGATIVE_INFINITY, nowMs); - return redis.zcard(sessionsKey(owner)) > 0; + return withJedis(jedis -> { + jedis.zremrangeByScore(sessionsKey(owner), Double.NEGATIVE_INFINITY, nowMs); + return jedis.zcard(sessionsKey(owner)) > 0; + }); } @Override public Set owners() { - return new HashSet<>(redis().smembers(OWNERS)); + return new HashSet<>(withJedis(jedis -> jedis.smembers(OWNERS_KEY))); } @Override public void drop(String owner) { - UnifiedJedis redis = redis(); - if (redis.zcard(sessionsKey(owner)) == 0 && !redis.exists(sandboxKey(owner))) { - redis.del(sessionsKey(owner)); - redis.srem(OWNERS, owner); + Object dropped = withJedis(jedis -> jedis.eval(DROP_SCRIPT, List.of(sessionsKey(owner), sandboxKey(owner)), + Collections.emptyList())); + if (dropped instanceof Long count && count > 0) { + // A touch racing this re-adds the owner, so the index cannot lose a live user. + withJedis(jedis -> jedis.srem(OWNERS_KEY, owner)); } } + + private T withJedis(JedisCallback callback) { + // UnifiedJedis is thread-safe and internally pooled; it is shared and must + // NOT be closed per call (that would tear down the whole client). + try { + return callback.apply(RedisConnectionFactory.getClient(config)); + } catch (Exception e) { + throw new IllegalStateException("Redis sandbox-registry command failed: " + e.getMessage(), e); + } + } + + // The {owner} hash tag co-locates all of a user's keys on a single Redis Cluster + // hash slot, so multi-key EVAL operations never trigger CROSSSLOT. It is inert on + // standalone/Sentinel deployments. + private static String sandboxKey(String owner) { + return "openshell:{" + owner + "}:sandbox"; + } + + private static String sessionsKey(String owner) { + return "openshell:{" + owner + "}:sessions"; + } + + private static String lockKey(String owner) { + return "openshell:{" + owner + "}:lock"; + } + + private interface JedisCallback { + T apply(UnifiedJedis jedis); + } } /** Single-instance fallback: correct only while one server owns every session. */ static final class LocalRegistry implements Registry { private final Map sandboxes = new ConcurrentHashMap<>(); private final Map> sessions = new ConcurrentHashMap<>(); - private final Set locks = ConcurrentHashMap.newKeySet(); + private final Map locks = new ConcurrentHashMap<>(); @Override public boolean shared() { @@ -639,13 +735,14 @@ public void forget(String owner, String sandboxName) { } @Override - public boolean lock(String owner) { - return locks.add(owner); + public String tryLock(String owner) { + String token = UUID.randomUUID().toString(); + return locks.putIfAbsent(owner, token) == null ? token : null; } @Override - public void unlock(String owner) { - locks.remove(owner); + public void unlock(String owner, String token) { + locks.remove(owner, token); } @Override @@ -693,11 +790,7 @@ public void drop(String owner) { private static String digest(String value, int hexChars) { try { byte[] hash = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(); - for (int i = 0; sb.length() < hexChars; i++) { - sb.append(String.format("%02x", hash[i])); - } - return sb.substring(0, hexChars); + return HexFormat.of().formatHex(hash).substring(0, hexChars); } catch (NoSuchAlgorithmException e) { throw new IllegalStateException("SHA-256 unavailable", e); } @@ -705,7 +798,7 @@ private static String digest(String value, int hexChars) { private static String nodeName() { try { - return java.net.InetAddress.getLocalHost().getHostName(); + return InetAddress.getLocalHost().getHostName(); } catch (Exception e) { return "unknown"; } diff --git a/src/prerna/util/openshell/TransportedRoots.java b/src/prerna/util/openshell/TransportedRoots.java new file mode 100644 index 0000000000..86b3d2b48c --- /dev/null +++ b/src/prerna/util/openshell/TransportedRoots.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import prerna.util.Constants; + +/** + * Which space a transported root belongs to, read from its normalized absolute path. + * + *

    + * Only user spaces -- projects, rooms and user assets -- are written by user code, so + * only they are synced between the host and a sandbox. Engine and runtime trees are + * uploaded once and never pulled back. + */ +public final class TransportedRoots { + + private static final String PROJECT_MARKER = "/" + Constants.PROJECT_FOLDER + "/"; + private static final String ROOM_MARKER = "/" + Constants.ROOM_FOLDER + "/"; + private static final String USER_MARKER = "/" + Constants.USER_FOLDER + "/"; + + private TransportedRoots() { + // static utility + } + + /** A project, room or user-asset root: the spaces user code writes and sync covers. */ + public static boolean isUserSpace(String normalized) { + return isProject(normalized) || isRoom(normalized) || isUserAsset(normalized); + } + + public static boolean isProject(String normalized) { + return normalized.contains(PROJECT_MARKER); + } + + public static boolean isRoom(String normalized) { + return normalized.contains(ROOM_MARKER); + } + + public static boolean isUserAsset(String normalized) { + return normalized.contains(USER_MARKER); + } + + /** The project id of a root under {@code project/__/}, or null outside a project. */ + public static String projectIdOf(String normalized) { + return idOf(folderAfter(normalized, PROJECT_MARKER)); + } + + /** The asset project id of a root under {@code user/Asset__/}, or null outside one. */ + public static String userAssetIdOf(String normalized) { + return idOf(folderAfter(normalized, USER_MARKER)); + } + + /** {@code __} folders carry the id after the last separator. */ + private static String idOf(String folder) { + if (folder == null) { + return null; + } + int separator = folder.lastIndexOf("__"); + return separator < 0 ? folder : folder.substring(separator + 2); + } + + /** The room id of a root under {@code room//}, or null outside a room. */ + public static String roomIdOf(String normalized) { + return folderAfter(normalized, ROOM_MARKER); + } + + private static String folderAfter(String normalized, String marker) { + int at = normalized.indexOf(marker); + if (at < 0) { + return null; + } + String rest = normalized.substring(at + marker.length()); + int slash = rest.indexOf('/'); + return slash < 0 ? rest : rest.substring(0, slash); + } +} From 1407808e71731ca71a452fb91e6ec95854891435 Mon Sep 17 00:00:00 2001 From: Kunal Patel Date: Fri, 23 Oct 2026 22:51:06 -0400 Subject: [PATCH 48/56] build(docker): single sandbox image on python-builder, with CI Replace the slim/full executor Dockerfiles and export_requirements.py with docker/Dockerfile.sandbox: ubuntu 24.04 plus /usr/lib/python copied from quay.io/semoss/python-builder:3.14-cpu (same as the Tomcat images), git, a uid 1000 user and py/ as the worker. Add sandbox-builder.yml to build amd64/arm64 on py/ changes and after Python Builder runs, smoke-test each digest, then tag quay.io/semoss/semoss-sandbox:-3.14-cpu. The local build script and CI share docker/sandbox-smoke-test.sh. --- .github/workflows/sandbox-builder.yml | 144 ++++++++++++++++++ docker/Dockerfile.executor | 44 ------ docker/Dockerfile.executor-full | 94 ------------ docker/Dockerfile.sandbox | 50 ++++++ docker/Dockerfile.sandbox.dockerignore | 7 + docker/build-executor-image.sh | 79 ---------- docker/build-sandbox-image.sh | 26 ++++ docker/export_requirements.py | 83 ---------- docker/sandbox-smoke-test.sh | 33 ++++ .../github_actions_workflows.md | 1 + docs/deployment/executor_sandbox.md | 37 +++-- 11 files changed, 282 insertions(+), 316 deletions(-) create mode 100644 .github/workflows/sandbox-builder.yml delete mode 100644 docker/Dockerfile.executor delete mode 100644 docker/Dockerfile.executor-full create mode 100644 docker/Dockerfile.sandbox create mode 100644 docker/Dockerfile.sandbox.dockerignore delete mode 100755 docker/build-executor-image.sh create mode 100755 docker/build-sandbox-image.sh delete mode 100644 docker/export_requirements.py create mode 100755 docker/sandbox-smoke-test.sh diff --git a/.github/workflows/sandbox-builder.yml b/.github/workflows/sandbox-builder.yml new file mode 100644 index 0000000000..7247945c9a --- /dev/null +++ b/.github/workflows/sandbox-builder.yml @@ -0,0 +1,144 @@ +name: Sandbox Builder +# Builds docker/Dockerfile.sandbox: the OpenShell sandbox image that runs a user's python +# worker. It is the python-builder image plus this repo's py/, so it rebuilds when py/ changes +# and after every successful Python Builder run (new interpreter or dependencies). +on: + workflow_dispatch: + push: + branches: ["dev", "main"] + paths: + - "py/**" + - "docker/Dockerfile.sandbox" + - "docker/Dockerfile.sandbox.dockerignore" + - "docker/sandbox-smoke-test.sh" + - ".github/workflows/sandbox-builder.yml" + workflow_run: + workflows: ["Python Builder"] + types: [completed] + branches: ["dev", "main"] + +concurrency: + group: sandbox-builder-${{ github.event.workflow_run.head_branch || github.ref_name }} + cancel-in-progress: true + +env: + IMAGE: quay.io/semoss/semoss-sandbox + # must match the COPY --from tag in docker/Dockerfile.sandbox + PYTHON_BUILDER_TAG: "3.14-cpu" + BRANCH: ${{ github.event.workflow_run.head_branch || github.ref_name }} + SOURCE_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} + +jobs: + # --------------------------------------------------------------------------- + # Build each architecture natively, push BY DIGEST (no tag), then smoke-test + # that digest. A failed smoke test never reaches the tagging job. + # --------------------------------------------------------------------------- + build: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + arch: amd64 + runner_prefix: codebuild-semoss-github-runner + container: quay.io/semoss/test-quay:ubuntu-dind + - platform: linux/arm64 + arch: arm64 + runner_prefix: codebuild-semoss-github-runner-arm64 + container: quay.io/semoss/test-quay:ubuntu-dind-arm64 + runs-on: ${{ matrix.runner_prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + container: + image: ${{ matrix.container }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ env.SOURCE_SHA }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Quay + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_ROBOT_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_PASSWORD }} + + - name: Build and push by digest (${{ matrix.platform }}) + id: build + uses: docker/build-push-action@v6 + with: + # context is py/; the Dockerfile and its .dockerignore live in docker/ + context: ./py + file: ./docker/Dockerfile.sandbox + platforms: ${{ matrix.platform }} + pull: true + labels: | + org.opencontainers.image.revision=${{ env.SOURCE_SHA }} + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.semoss.python-builder=${{ env.PYTHON_BUILDER_TAG }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + + - name: Smoke test (${{ matrix.platform }}) + run: bash docker/sandbox-smoke-test.sh "${IMAGE}@${{ steps.build.outputs.digest }}" py + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # --------------------------------------------------------------------------- + # Stitch the per-arch digests into one tagged manifest list. + # --------------------------------------------------------------------------- + merge: + needs: build + runs-on: ubuntu-latest + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Quay + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_ROBOT_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_PASSWORD }} + + - name: Compute tags + id: meta + shell: bash + run: | + BASE="${BRANCH//\//-}-${PYTHON_BUILDER_TAG}" + TAGS=( "-t" "${IMAGE}:${BASE}" "-t" "${IMAGE}:${BASE}-${SOURCE_SHA:0:7}" ) + if [[ "${{ vars.RELEASE }}" == "TRUE" ]]; then + TAGS+=( "-t" "${IMAGE}:${PYTHON_BUILDER_TAG}-${{ vars.VERSION }}" ) + fi + echo "args=${TAGS[*]}" >> "$GITHUB_OUTPUT" + echo "primary=${IMAGE}:${BASE}" >> "$GITHUB_OUTPUT" + + - name: Create and push manifest list + working-directory: /tmp/digests + run: | + docker buildx imagetools create ${{ steps.meta.outputs.args }} \ + $(printf "${IMAGE}@sha256:%s " *) + + - name: Inspect manifest list + run: docker buildx imagetools inspect "${{ steps.meta.outputs.primary }}" diff --git a/docker/Dockerfile.executor b/docker/Dockerfile.executor deleted file mode 100644 index f743f68874..0000000000 --- a/docker/Dockerfile.executor +++ /dev/null @@ -1,44 +0,0 @@ -# SEMOSS executor sandbox — the process that runs user code under OpenShell. -# -# docker/build-executor-image.sh # stages py/ into the context, labels, smoke-tests -# -# The image must carry the worker source that is in this repo; earlier builds staged -# py/ into a separate context by hand, and the image silently kept running month-old -# python while the repo looked fixed. The build script copies the committed py/ into -# the context and checks the result against the repo. -FROM python:3.12-slim - -LABEL maintainer="semoss@semoss.org" -LABEL org.semoss.component="executor-sandbox" - -# git enables local repository work inside the sandbox (status/diff/commit/branch). -# Installing it grants no network access: the OpenShell policy keys egress to a binary -# path, so every remote host needs its own rule before git can leave the sandbox. -# The identity is system-wide because commits fail without one. -RUN apt-get update \ - && apt-get install -y --no-install-recommends git ca-certificates \ - && rm -rf /var/lib/apt/lists/* \ - && git config --system user.name "SEMOSS Sandbox" \ - && git config --system user.email "sandbox@semoss.local" \ - && git config --system safe.directory '*' - -# pandas + jsonpickle are the two modules the handler lazy-imports at boot. yfinance is -# only here for the Stock MCP example and is not part of SEMOSS's declared dependencies -# -- the real answer is a per-project venv or a manifest-driven install. -RUN pip install --no-cache-dir pandas jsonpickle yfinance - -# The worker source: the committed py/, staged into the context by the build script. -COPY py/ /opt/semosshome/py/ - -# The handler appends to {insight_folder}/log.txt during setup(). The grants folders are -# the read-only / read-write drop points the policy model expects to exist. -RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ - && touch /opt/semosshome/insight/log.txt \ - && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ - && chmod 755 /opt/grants-rw /opt/grants-ro - -# OpenShell refuses to start unless the workspace is writable by the sandbox identity: -# "workspace path component '' is not writable by the sandbox identity in the -# image: Permission denied" -USER 1000:1000 -WORKDIR /opt/semosshome/py diff --git a/docker/Dockerfile.executor-full b/docker/Dockerfile.executor-full deleted file mode 100644 index b92ce5e9e9..0000000000 --- a/docker/Dockerfile.executor-full +++ /dev/null @@ -1,94 +0,0 @@ -# SEMOSS executor sandbox — full-parity build. -# -# MODE=full TAG=semoss-executor:full docker/build-executor-image.sh -# # or directly: -# docker build -f docker/Dockerfile.executor-full -t semoss-executor:full py -# -# Why this exists next to Dockerfile.executor: the minimal image carries three packages, so a -# Py() block that ran against the platform's own runtime (numpy, boto3, langchain, torch, ...) -# fails with ImportError the moment it is sandboxed. That is a silent capability regression, not -# a size trade-off. -# -# Base is Ubuntu, matching docker/Dockerfile.python, so sandbox and platform share a distro, -# glibc and the native libraries that document/OCR stacks expect. Python comes from apt (24.04 -# ships 3.12) and the declared dependencies install into a venv with uv — the same installer -# upstream uses for its own runtime. -# -# The venv is reached by EXPLICIT PATH, never by PATH lookup: OpenShell builds the environment -# for the processes it spawns, so an image's ENV PATH is not applied inside a sandbox. The -# worker is launched as /usr/lib/python/semossvenv/bin/python3 (see OpenShellTransport) and the -# sandbox is created with `--env PATH=/usr/lib/python/semossvenv/bin:…` so user code resolves -# to the same interpreter. -ARG UBUNTU_VERSION=24.04 - -FROM ubuntu:${UBUNTU_VERSION} AS deps - -ARG EXTRAS=cpu -ENV EXTRAS=${EXTRAS} -ENV DEBIAN_FRONTEND=noninteractive - -# Build toolchain lives in this stage only: several declared dependencies ship no aarch64 -# wheel (annoy is a C++ extension and was the first to fail), so they compile from source. -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 python3.12 python3.12-dev python3.12-venv python3-venv build-essential cmake pkg-config curl ca-certificates \ - libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libffi-dev \ - && rm -rf /var/lib/apt/lists/* - -# uv, the installer SEMOSS uses for its own runtime. -COPY --from=ghcr.io/astral-sh/uv:0.9 /uv /uvx /usr/local/bin/ - -WORKDIR /build - -COPY pyproject.toml ./ -COPY export_requirements.py ./ - -RUN python3 export_requirements.py pyproject.toml "${EXTRAS}" > /tmp/requirements.txt \ - && echo "installing $(wc -l < /tmp/requirements.txt) declared dependencies (extra: ${EXTRAS:-none})" \ - && uv venv /usr/lib/python/semossvenv --python /usr/bin/python3.12 --seed \ - && case "${EXTRAS}" in \ - cpu) torch_index=https://download.pytorch.org/whl/cpu ;; \ - gpu) torch_index=https://download.pytorch.org/whl/cu126 ;; \ - *) torch_index= ;; \ - esac \ - && if [ -n "$torch_index" ]; then \ - echo "torch trio from $torch_index (upstream pins this in [tool.uv.sources])"; \ - uv pip install --python /usr/lib/python/semossvenv/bin/python --index-url "$torch_index" \ - torch torchvision torchaudio; \ - fi \ - && uv pip install --python /usr/lib/python/semossvenv/bin/python -r /tmp/requirements.txt \ - && uv pip install --python /usr/lib/python/semossvenv/bin/python "levenshtein>=0.27.3" - -FROM ubuntu:${UBUNTU_VERSION} - -LABEL maintainer="semoss@semoss.org" -LABEL org.semoss.component="executor-sandbox-full" - -ENV DEBIAN_FRONTEND=noninteractive - -# python3.12 must exist in this stage too: the venv's interpreter is a symlink to -# /usr/bin/python3.12. git enables local repository work; egress stays policy-gated. -RUN apt-get update \ - && apt-get install -y --no-install-recommends python3 python3.12 git ca-certificates \ - && rm -rf /var/lib/apt/lists/* \ - && git config --system user.name "SEMOSS Sandbox" \ - && git config --system user.email "sandbox@semoss.local" \ - && git config --system safe.directory '*' - -# The venv lives under /usr/lib/python, matching docker/Dockerfile.python. That path is -# inside the sandbox policy's read-only /usr, which /opt/venv would not be. -COPY --from=deps /usr/lib/python/semossvenv /usr/lib/python/semossvenv -ENV VIRTUAL_ENV=/usr/lib/python/semossvenv -ENV PATH=/usr/lib/python/semossvenv/bin:$PATH - -# The worker source. The build script stages the repo's py/ as the context's py/. -COPY py/ /opt/semosshome/py/ - -RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ - && touch /opt/semosshome/insight/log.txt \ - && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ - && chmod 755 /opt/grants-rw /opt/grants-ro - -# OpenShell refuses to start unless the workspace is writable by the sandbox identity. -USER 1000:1000 -WORKDIR /opt/semosshome/py diff --git a/docker/Dockerfile.sandbox b/docker/Dockerfile.sandbox new file mode 100644 index 0000000000..fad492565e --- /dev/null +++ b/docker/Dockerfile.sandbox @@ -0,0 +1,50 @@ +# SEMOSS sandbox image: the OpenShell sandbox that runs a user's persistent python worker. +# +# docker/build-sandbox-image.sh # local build + smoke test +# docker build -f docker/Dockerfile.sandbox -t semoss-sandbox py # context is py/ +# +# Python and every dependency come from the python-builder image (docker/Dockerfile.python, +# uv + py/install_config/pyproject.toml). Like the Tomcat images, we copy its /usr/lib/python +# onto a plain Ubuntu base, so a Py() block has the same interpreter and imports inside the +# sandbox as outside it. +# +# The venv is reached by explicit path, never by PATH lookup: OpenShell builds the environment +# for the processes it spawns, so ENV PATH is not applied inside a sandbox. See +# docs/deployment/executor_sandbox.md. +ARG BASE_REGISTRY=public.ecr.aws/docker/library +ARG BASE_IMAGE=ubuntu +ARG BASE_TAG=24.04 + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +LABEL maintainer="semoss@semoss.org" +LABEL org.semoss.component="sandbox" + +ENV DEBIAN_FRONTEND=noninteractive + +# git for local repository work; egress stays policy-gated per binary +RUN for i in 1 2 3; do apt-get update && break || (sleep 10 && rm -rf /var/lib/apt/lists/*); done \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# uv-managed CPython + semossvenv; /usr/lib/python sits inside the policy's read-only /usr +COPY --from=quay.io/semoss/python-builder:3.14-cpu /usr/lib/python /usr/lib/python +ENV VIRTUAL_ENV=/usr/lib/python/semossvenv +ENV PATH=/usr/lib/python/semossvenv/bin:$PATH + +# the sandbox identity is uid/gid 1000 (ubuntu:24.04 already has it as "ubuntu") +RUN if ! getent group 1000 >/dev/null; then groupadd -g 1000 sandbox; fi \ + && if ! getent passwd 1000 >/dev/null; then useradd -u 1000 -g 1000 -m -s /bin/bash sandbox; fi + +# the worker source (build context is the repo's py/) +COPY . /opt/semosshome/py/ + +# setup() appends to insight/log.txt; grants-* are the policy's ro/rw drop points +RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ + && touch /opt/semosshome/insight/log.txt \ + && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ + && chmod 755 /opt/grants-rw /opt/grants-ro + +# OpenShell refuses to start unless the workspace is writable by the sandbox identity +USER 1000:1000 +WORKDIR /opt/semosshome/py diff --git a/docker/Dockerfile.sandbox.dockerignore b/docker/Dockerfile.sandbox.dockerignore new file mode 100644 index 0000000000..64900ac9d8 --- /dev/null +++ b/docker/Dockerfile.sandbox.dockerignore @@ -0,0 +1,7 @@ +# context is py/; keep caches and local tooling out of the sandbox image +**/__pycache__ +**/*.pyc +**/.DS_Store +**/*.egg-info +.pytest_cache +.continue diff --git a/docker/build-executor-image.sh b/docker/build-executor-image.sh deleted file mode 100755 index 2f54cc61a7..0000000000 --- a/docker/build-executor-image.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# Build the SEMOSS executor sandbox image from the worker source in this repo. -# -# docker/build-executor-image.sh # slim, tag semoss-executor:dev -# MODE=full TAG=semoss-executor:full docker/build-executor-image.sh -# NO_CACHE=1 docker/build-executor-image.sh -# -# Two modes, one command: -# slim (default) - three packages; fast to build and start, but user code can only import -# pandas, jsonpickle and yfinance. -# full - installs the dependency set declared in py/install_config/pyproject.toml -# with uv, the same installer docker/Dockerfile.python uses, so a Py() -# block keeps the import surface it had outside the sandbox. -# -# Either way the worker python is the committed py/, never a hand-staged copy, and the tag is -# labelled with a hash of that tree so `docker inspect` answers "which source is this?". -set -euo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO="$(cd "$HERE/.." && pwd)" -MODE="${MODE:-slim}" -NO_CACHE="${NO_CACHE:-0}" - -case "$MODE" in - slim) TAG="${TAG:-semoss-executor:dev}" ;; - full) TAG="${TAG:-semoss-executor:full}" ;; - *) echo "MODE must be slim or full (got '$MODE')" >&2; exit 1 ;; -esac - -[ -d "$REPO/py" ] || { echo "no py/ under $REPO" >&2; exit 1; } - -PY_SHA="$(cd "$REPO/py" \ - && find . -type f -name '*.py' -print0 \ - | sort -z \ - | xargs -0 sha256sum \ - | sha256sum | cut -d' ' -f1)" - -CTX="$(mktemp -d)" -trap 'rm -rf "$CTX"' EXIT - -ARGS=(build - -t "$TAG" - --label "org.semoss.py-sha=$PY_SHA" - --label "org.semoss.executor-mode=$MODE" - --label "org.semoss.built-at=$(date -u +%Y-%m-%dT%H:%M:%SZ)") -[ "$NO_CACHE" = "1" ] && ARGS+=(--no-cache) - -if [ "$MODE" = "slim" ]; then - cp "$HERE/Dockerfile.executor" "$CTX/Dockerfile" - cp -a "$REPO/py" "$CTX/py" - ARGS+=(-f "$CTX/Dockerfile") -else - # The full build also needs the manifest and the flattening helper next to py/. - cp "$HERE/Dockerfile.executor-full" "$CTX/Dockerfile" - cp "$HERE/export_requirements.py" "$CTX/export_requirements.py" - cp "$REPO/py/install_config/pyproject.toml" "$CTX/pyproject.toml" - cp -a "$REPO/py" "$CTX/py" - ARGS+=(-f "$CTX/Dockerfile" --build-arg "EXTRAS=${EXTRAS:-cpu}") -fi - -docker "${ARGS[@]}" "$CTX" - -echo "built $TAG ($MODE) from py sha ${PY_SHA:0:12}" - -# Prove what landed, rather than trusting the build log: the worker tree in the image must -# hash to the repo's, and the worker's interpreter must import the watcher. Any failure -# fails the build. -docker run --rm --entrypoint sh -e EXPECT="$PY_SHA" "$TAG" -ec ' - cd /opt/semosshome/py - test -f gaas_tcp_socket_server.py || { echo "worker missing at /opt/semosshome/py" >&2; exit 1; } - GOT=$(find . -type f -name "*.py" -print0 | sort -z | xargs -0 sha256sum | sha256sum | cut -d" " -f1) - [ "$GOT" = "$EXPECT" ] || { echo "py tree in image ($GOT) differs from the repo ($EXPECT)" >&2; exit 1; } - P=/usr/lib/python/semossvenv/bin/python3; [ -x "$P" ] || P=$(command -v python3) - "$P" -c "import gaas_change_watcher" || { echo "$P cannot import the change watcher" >&2; exit 1; } - echo "worker tree matches repo; $P imports the watcher" - git --version - [ "$(id -u)" = 1000 ] || { echo "image does not run as uid 1000" >&2; exit 1; } -' -echo "smoke test passed for $TAG" diff --git a/docker/build-sandbox-image.sh b/docker/build-sandbox-image.sh new file mode 100755 index 0000000000..cb02a986fa --- /dev/null +++ b/docker/build-sandbox-image.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Build the SEMOSS sandbox image locally from this repo's py/ and smoke-test it. +# CI builds and publishes the same image: .github/workflows/sandbox-builder.yml +# +# docker/build-sandbox-image.sh # tag semoss-sandbox:dev +# TAG=semoss-sandbox:test NO_CACHE=1 docker/build-sandbox-image.sh +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/.." && pwd)" +TAG="${TAG:-semoss-sandbox:dev}" +NO_CACHE="${NO_CACHE:-0}" + +[ -d "$REPO/py" ] || { echo "no py/ under $REPO" >&2; exit 1; } + +ARGS=(build + -f "$HERE/Dockerfile.sandbox" + -t "$TAG" + --label "org.opencontainers.image.revision=$(git -C "$REPO" rev-parse HEAD 2>/dev/null || echo unknown)" + --label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)") +[ "$NO_CACHE" = "1" ] && ARGS+=(--no-cache --pull) + +# context is py/ itself, so the image carries this checkout's worker (uncommitted edits included) +DOCKER_BUILDKIT=1 docker "${ARGS[@]}" "$REPO/py" + +"$HERE/sandbox-smoke-test.sh" "$TAG" "$REPO/py" diff --git a/docker/export_requirements.py b/docker/export_requirements.py deleted file mode 100644 index 891b2bf9f1..0000000000 --- a/docker/export_requirements.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -"""Flatten SEMOSS's pyproject into a requirements list for the executor image. - -SEMOSS builds its runtime python with ``uv pip install ".[cpu]"`` against -``py/install_config/pyproject.toml``. The executor sandbox must offer the same import -surface, or a ``Py()`` block that worked outside the sandbox starts failing with -ImportError inside it. - -Installing the project itself (``.[cpu]``) is not an option here: that pyproject declares -no build-system, so it is a dependency manifest rather than an installable package. This -script reads the same file and emits the dependency list instead, so nothing is duplicated -by hand. - -Two things a naive flattening gets wrong, both handled here: - -* ``[tool.uv.sources]`` maps torch/torchvision/torchaudio to the pytorch-cpu index. - Dropping it makes PyPI resolve CUDA builds -- 3.3 GB of ``nvidia`` plus triton in an - image that never sees a GPU. Those three are skipped by default; install them from the - CPU index first (see Dockerfile.executor-full). -* ``[tool.uv] override-dependencies`` carries security pins (marshmallow, protobuf, numpy, - pyasn1, lxml). They are emitted as ordinary requirements so the sandbox keeps the same - minimum versions as the platform. - -usage: export_requirements.py [extra] [--keep-torch] > requirements.txt -""" - -import sys -import tomllib - -TORCH = ("torch", "torchvision", "torchaudio") - - -def _name(spec: str) -> str: - """Package name from a requirement specifier, without version bounds or extras.""" - for separator in (">", "=", "<", "!", "~", "[", " "): - spec = spec.split(separator)[0] - return spec.strip().lower() - - -def main() -> int: - args = list(sys.argv[1:]) - keep_torch = "--keep-torch" in args - args = [a for a in args if a != "--keep-torch"] - - if not args: - print(__doc__, file=sys.stderr) - return 2 - path = args[0] - extra = args[1] if len(args) > 1 else "" - - with open(path, "rb") as handle: - data = tomllib.load(handle) - - project = data.get("project", {}) - dependencies = list(project.get("dependencies", [])) - if extra: - extras = project.get("optional-dependencies", {}) - if extra not in extras: - print( - "unknown extra %r; available: %s" % (extra, ", ".join(sorted(extras))), - file=sys.stderr, - ) - return 1 - dependencies.extend(extras[extra]) - - if not keep_torch: - dependencies = [d for d in dependencies if _name(d) not in TORCH] - - # Security pins the platform applies over transitive resolutions. - overrides = data.get("tool", {}).get("uv", {}).get("override-dependencies", []) - dependencies.extend(overrides) - - seen = set() - for dependency in dependencies: - if dependency in seen: - continue - seen.add(dependency) - print(dependency) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/docker/sandbox-smoke-test.sh b/docker/sandbox-smoke-test.sh new file mode 100755 index 0000000000..ba77503d96 --- /dev/null +++ b/docker/sandbox-smoke-test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Prove a built sandbox image is usable: the worker tree matches the given py/, the venv +# python imports the worker, git is present and the image runs as uid 1000. +# +# docker/sandbox-smoke-test.sh [py-dir] # py-dir defaults to the repo's py/ +set -euo pipefail + +IMAGE="${1:?usage: sandbox-smoke-test.sh [py-dir]}" +PY_DIR="${2:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/py}" + +# same exclusions as Dockerfile.sandbox.dockerignore +py_sha() { + find . -type f -name '*.py' -not -path '*/__pycache__/*' -not -path './.pytest_cache/*' \ + -not -path './.continue/*' -not -path '*.egg-info/*' -print0 \ + | sort -z | xargs -0 sha256sum | sha256sum | cut -d' ' -f1 +} + +EXPECT="$(cd "$PY_DIR" && py_sha)" + +docker run --rm --entrypoint bash -e EXPECT="$EXPECT" "$IMAGE" -ec " + $(declare -f py_sha) + cd /opt/semosshome/py + test -f gaas_tcp_socket_server.py || { echo 'worker missing at /opt/semosshome/py' >&2; exit 1; } + GOT=\$(py_sha) + [ \"\$GOT\" = \"\$EXPECT\" ] || { echo \"py tree in image (\$GOT) differs from $PY_DIR (\$EXPECT)\" >&2; exit 1; } + P=/usr/lib/python/semossvenv/bin/python3 + [ -x \"\$P\" ] || { echo \"no venv python at \$P\" >&2; exit 1; } + # same launch shape as OpenShellTransport: explicit interpreter, bare PATH + env -i PATH=/usr/local/bin:/usr/bin:/bin \"\$P\" -c 'import sys, pandas, numpy, jsonpickle, gaas_tcp_server_handler, gaas_change_watcher; print(sys.version)' + git --version + [ \"\$(id -u)\" = 1000 ] || { echo 'image does not run as uid 1000' >&2; exit 1; } +" +echo "smoke test passed for $IMAGE" diff --git a/docs/cloud_and_cluster/github_actions_workflows.md b/docs/cloud_and_cluster/github_actions_workflows.md index f7639942d2..8e6b28ddf2 100644 --- a/docs/cloud_and_cluster/github_actions_workflows.md +++ b/docs/cloud_and_cluster/github_actions_workflows.md @@ -8,6 +8,7 @@ The workflows in [.github/workflows](../../.github/workflows/) build Java artifa | --- | --- | | [dev_workflow.yml](../../.github/workflows/dev_workflow.yml) | Java build, test selection, artifact signing, and publication paths | | [python-builder.yml](../../.github/workflows/python-builder.yml) | Python runtime/dependency image | +| [sandbox-builder.yml](../../.github/workflows/sandbox-builder.yml) | OpenShell sandbox image: python-builder plus the `py/` worker ([details](../deployment/executor_sandbox.md#image)) | | [tomcat-builder.yml](../../.github/workflows/tomcat-builder.yml) | Java/Tomcat builder image | | [all-docker-builds.yml](../../.github/workflows/all-docker-builds.yml) | Application image build orchestration | | [ubuntu2204.yml](../../.github/workflows/ubuntu2204.yml), [ubuntu2404.yml](../../.github/workflows/ubuntu2404.yml) | Ubuntu application images | diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index f363fa4d57..16f8f24a03 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -178,27 +178,32 @@ never reaches the host. ## Image -The sandbox image is built from this repository, not from a staged copy: +One image, `docker/Dockerfile.sandbox`, serves every sandbox. Like the Tomcat images, it is a +plain `ubuntu:24.04` with `/usr/lib/python` copied in from the python-builder image +(`quay.io/semoss/python-builder:3.14-cpu`, built by `docker/Dockerfile.python` from +`py/install_config/pyproject.toml` with uv). A `Py()` block therefore gets the same Python and +the same packages inside the sandbox as outside it. On top of that the image adds git, a uid +1000 user and this repository's `py/` as the worker. -```bash -docker/build-executor-image.sh # slim, tag semoss-executor:dev -MODE=full TAG=semoss-executor:full docker/build-executor-image.sh -``` +CI publishes it as `quay.io/semoss/semoss-sandbox` from +`.github/workflows/sandbox-builder.yml`. It rebuilds on any `py/` change on `dev`/`main` and +after every successful Python Builder run. Tags are `-` (for example +`dev-3.14-cpu`) plus a `-` pin. Each architecture is smoke-tested before anything is +tagged. Point `OPENSHELL_IMAGE` at one of these tags. -The script stages the committed `py/` into the build context, so the image carries exactly the -worker in this repository. The tag is labelled with `org.semoss.py-sha`, a hash of the python -tree, and the build fails unless the worker tree inside the image hashes to the repository's -and the worker's interpreter imports the change watcher. +For a local build: -Two variants: +```bash +docker/build-sandbox-image.sh # tag semoss-sandbox:dev +TAG=semoss-sandbox:test NO_CACHE=1 docker/build-sandbox-image.sh +``` -| Mode | Contents | Use | -| --- | --- | --- | -| `slim` (default) | `python:3.12-slim`, git, `pandas`, `jsonpickle`, `yfinance` | quick iteration; user code can only import those | -| `full` (`MODE=full`) | the dependency set declared in `py/install_config/pyproject.toml`, installed with uv | parity with the platform runtime — a `Py()` block keeps its imports | +The build context is `py/` itself, so a local build carries the checkout's worker, uncommitted +edits included. Both the script and CI run `docker/sandbox-smoke-test.sh`, which fails unless +the worker tree in the image hashes to the source `py/`, the venv python imports the worker +under a bare `PATH`, git is present and the image runs as uid 1000. -The full build keeps a venv at `/usr/lib/python/semossvenv` — the same path -`docker/Dockerfile.python` uses — and the worker is launched by absolute path, because +The venv lives at `/usr/lib/python/semossvenv` and the worker is launched by absolute path, because **both the environment and the filesystem inside a sandbox belong to the runtime**: - **`PATH`**: a workload starts with `/usr/local/bin:/usr/bin:/bin`, so an image's `ENV PATH` From 7fb40da66202896bc614c11b97a85edd22b111d8 Mon Sep 17 00:00:00 2001 From: Kunal Patel Date: Fri, 23 Oct 2026 22:51:07 -0400 Subject: [PATCH 49/56] fix(py): per-connection thread_local in the TCP handler setup() replaced a class-level thread_local on every new connection, so on a persistent worker a client connecting mid-request wiped the payload of a Py() still running on another connection; its reply went out with a random epoc and no insightId and the Java caller hung. Each handler now owns its own thread_local. Also log why the change watcher failed instead of silently forcing a full reconcile. --- py/gaas_tcp_server_handler.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/py/gaas_tcp_server_handler.py b/py/gaas_tcp_server_handler.py index 352c1a7560..3f0a5992be 100644 --- a/py/gaas_tcp_server_handler.py +++ b/py/gaas_tcp_server_handler.py @@ -388,8 +388,9 @@ def setup(self): # arming here yields zero roots and pins the host to the slow path forever. self.change_watcher = None - # set the thread local - TCPServerHandler.thread_local = threading.local() + # per connection: a class-level one is replaced by every new client, wiping the + # payload of requests still running on other connections to a persistent worker + self.thread_local = threading.local() # Sometimes the debugger is not effective or cannot handle certain troubleshooting scenarios. # This is where you can use the custom_log() method. It writes to the log txt file, ensuring the file exists, creates a new line, adds the message, and flushes the log. @@ -468,7 +469,13 @@ def handle_changes(self): # Re-evaluated on every call: room/ and user/ appear only when the session # first touches one, which is usually after the first CHANGES. self.change_watcher = shared_watcher(self._watcher_roots()) - except Exception: + except Exception as e: + # an unsupported kernel returns None without raising, so this is a real fault + if self.logger: + self.logger.warning( + f"Change watcher failed, host falls back to a full reconcile: {e}", + exc_info=True, + ) self.change_watcher = None if self.change_watcher is None: snapshot = { From 4cceeae9920fdb88fb8ad3083fbb07b9d26197ca Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 11:33:40 +0000 Subject: [PATCH 50/56] fix(openshell): release the sandbox lease on a provider logout Logging out of the last provider drops its token before the session is destroyed. sessionDestroyed then threw in getSingleLogginName, which skipped every cleanup step, and even without the throw the owner could no longer be derived from the user, so the lease was never released. - getSingleLogginName returns "" when the primary login's token is gone - sessionEnded takes the owner; ownerForSession reads it from the session (recorded by the filter) and falls back to the user's login --- src/prerna/auth/User.java | 4 ++- .../openshell/OpenShellUserSandboxes.java | 30 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/prerna/auth/User.java b/src/prerna/auth/User.java index 43a8e6dca4..40cdb97888 100644 --- a/src/prerna/auth/User.java +++ b/src/prerna/auth/User.java @@ -1086,7 +1086,9 @@ public static String getSingleLogginName(User semossUser) { } AccessToken token = semossUser.accessTokens.get(semossUser.getPrimaryLogin()); - return token.getId(); + // null after logging out of the primary provider: the token is dropped before the + // session is destroyed, and the session cleanup logs this name + return token == null ? "" : token.getId(); } public static List> getUserIdAndType(User user) { diff --git a/src/prerna/util/openshell/OpenShellUserSandboxes.java b/src/prerna/util/openshell/OpenShellUserSandboxes.java index 12ec969ec9..0cd9861063 100644 --- a/src/prerna/util/openshell/OpenShellUserSandboxes.java +++ b/src/prerna/util/openshell/OpenShellUserSandboxes.java @@ -95,6 +95,8 @@ public final class OpenShellUserSandboxes { private static final Logger classLogger = LogManager.getLogger(OpenShellUserSandboxes.class); + /** Session attribute holding the owner a session's lease was taken under. */ + public static final String SESSION_OWNER_ATTR = "OPENSHELL_SANDBOX_OWNER"; /** How often a session may refresh its lease; requests in between are free. */ private static final long TOUCH_INTERVAL_MS = TimeUnit.SECONDS.toMillis(60); /** How long a creator may hold the per-user create lock. */ @@ -151,6 +153,25 @@ public static boolean appliesTo(User user) { return token != null && token.getId() != null; } + /** + * The owner a session's sandbox lease was taken under, for cleanup when the session is + * destroyed. Read from the session first: logging out of a provider drops its token + * before the session is invalidated, so by then the user may have no login to derive + * the owner from, and the lease would never be released. + * + * @param recorded the session's {@link #SESSION_OWNER_ATTR} value, or null + * @return the owner, or null when this session has no user-scoped sandbox + */ + public static String ownerForSession(User user, Object recorded) { + if (!enabled()) { + return null; + } + if (recorded instanceof String owner && !owner.isEmpty()) { + return owner; + } + return appliesTo(user) ? ownerOf(user) : null; + } + /** * The digest a user's sandbox is filed under: provider and id of the primary login. * Used as the registry key and the {@code semoss-owner} label, so neither carries the @@ -380,13 +401,14 @@ public static String touch(User user, String sessionId, String lastTouch, int tt * fired {@code sessionDestroyed} for each one and deleted the sandbox), but the user has * not left: their lease stands, so the restarted or another instance reattaches. * - * @param over true for a logout or a timeout; false when the server is going away + * @param owner the owner recorded on the session ({@link #ownerForSession}); a logout + * may already have dropped the login it was derived from + * @param over true for a logout or a timeout; false when the server is going away */ - public static void sessionEnded(User user, String sessionId, boolean over) { - if (!appliesTo(user) || !user.isLoggedIn()) { + public static void sessionEnded(String owner, String sessionId, boolean over) { + if (owner == null) { return; } - String owner = ownerOf(user); synchronized (ownerLock(owner)) { Set here = LOCAL_SESSIONS.get(owner); if (here != null) { From 01e0bcba7a95bfe1383092dadcf8957ecdbd3a95 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 11:54:08 +0000 Subject: [PATCH 51/56] feat(openshell): run on a workstation whose SEMOSS home is not /opt/semosshome Files cross into a sandbox at their host path, which is what keeps the python SEMOSS generates (MCP loaders, PySource, the pandas JSON hand-off) working, since it embeds absolute host paths. A Mac's BaseFolder is somewhere like /Users/me/Documents/semoss, which no image has. The sandbox can only create it under a root that exists at start and that the policy grants: uploads run as the sandbox user inside Landlock. - Dockerfile.sandbox makes /Users and /home, owned by uid 1000; at create time the policy grants the top folder of BaseFolder's path, and the transport makes the home inside the sandbox. One generic image serves every developer; nothing is built per machine. - The policy is rendered from a template bundled in the jar (or OPENSHELL_POLICY), with ${SEMOSS_HOME_ROOT} and ${PYTHON} filled in. OPENSHELL_POLICY is no longer required, so a checkout needs no policy file from outside the repo. Egress is pypi only. - Dockerfile.sandbox moves uv's CPython to /usr/lib/python/cpython, so the egress rule names /usr/lib/python/cpython/bin/python3.14 on every patch version and architecture; the smoke test checks it. - Sandbox creation fails with the cause in the message when the home cannot be made, instead of the bare upload error. - OPENSHELL_IMAGE defaults to the published quay.io/semoss/semoss-sandbox:dev-3.14-cpu, so a deployment that tracks dev sets neither the image nor the policy; build-sandbox-image.sh is gone, and the doc keeps the one docker build line for trying a py/ change before it is published --- docker/Dockerfile.sandbox | 23 ++- docker/build-sandbox-image.sh | 26 --- docker/sandbox-smoke-test.sh | 3 + docs/deployment/executor_sandbox.md | 74 +++++++-- src/prerna/util/OpenShellTransport.java | 156 ++++++++++++++++-- src/prerna/util/openshell/sandbox-policy.yaml | 43 +++++ .../OpenShellSessionLifecycleHarness.java | 7 +- 7 files changed, 274 insertions(+), 58 deletions(-) delete mode 100755 docker/build-sandbox-image.sh create mode 100644 src/prerna/util/openshell/sandbox-policy.yaml diff --git a/docker/Dockerfile.sandbox b/docker/Dockerfile.sandbox index fad492565e..f3631fc2bd 100644 --- a/docker/Dockerfile.sandbox +++ b/docker/Dockerfile.sandbox @@ -1,7 +1,10 @@ # SEMOSS sandbox image: the OpenShell sandbox that runs a user's persistent python worker. # -# docker/build-sandbox-image.sh # local build + smoke test -# docker build -f docker/Dockerfile.sandbox -t semoss-sandbox py # context is py/ +# Published by .github/workflows/sandbox-builder.yml as quay.io/semoss/semoss-sandbox, the default +# OPENSHELL_IMAGE. To try a py/ change before it is published: +# +# docker build -f docker/Dockerfile.sandbox -t semoss-sandbox:dev py # context is py/ +# docker/sandbox-smoke-test.sh semoss-sandbox:dev # # Python and every dependency come from the python-builder image (docker/Dockerfile.python, # uv + py/install_config/pyproject.toml). Like the Tomcat images, we copy its /usr/lib/python @@ -29,6 +32,16 @@ RUN for i in 1 2 3; do apt-get update && break || (sleep 10 && rm -rf /var/lib/a # uv-managed CPython + semossvenv; /usr/lib/python sits inside the policy's read-only /usr COPY --from=quay.io/semoss/python-builder:3.14-cpu /usr/lib/python /usr/lib/python + +# One interpreter path for every build: the sandbox policy's egress rules name the kernel-reported +# executable, and uv's directory carries the patch version and the CPU architecture +# (cpython-3.14.8-linux-aarch64-gnu). Move the tree to a fixed name and leave the original as a +# symlink, so the venv still resolves and the policy can say /usr/lib/python/cpython/bin/python3.14. +RUN real="$(readlink -f /usr/lib/python/semossvenv/bin/python3)" \ + && dir="$(dirname "$(dirname "$real")")" \ + && mv "$dir" /usr/lib/python/cpython \ + && ln -s cpython "$dir" \ + && test "$(readlink -f /usr/lib/python/semossvenv/bin/python3)" = "/usr/lib/python/cpython/bin/$(basename "$real")" ENV VIRTUAL_ENV=/usr/lib/python/semossvenv ENV PATH=/usr/lib/python/semossvenv/bin:$PATH @@ -45,6 +58,12 @@ RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ && chmod 755 /opt/grants-rw /opt/grants-ro +# Workstation homes. Files cross into a sandbox at their host path, so a Mac's BaseFolder +# (/Users/me/...) or a Linux one (/home/me/...) must exist inside it. A sandbox can only create +# directories under a path its policy grants and that exists at start, so these roots are made +# here, owned by the sandbox user; OpenShellTransport grants the root and creates the home. +RUN mkdir -p /Users && chown 1000:1000 /Users /home + # OpenShell refuses to start unless the workspace is writable by the sandbox identity USER 1000:1000 WORKDIR /opt/semosshome/py diff --git a/docker/build-sandbox-image.sh b/docker/build-sandbox-image.sh deleted file mode 100755 index cb02a986fa..0000000000 --- a/docker/build-sandbox-image.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# Build the SEMOSS sandbox image locally from this repo's py/ and smoke-test it. -# CI builds and publishes the same image: .github/workflows/sandbox-builder.yml -# -# docker/build-sandbox-image.sh # tag semoss-sandbox:dev -# TAG=semoss-sandbox:test NO_CACHE=1 docker/build-sandbox-image.sh -set -euo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO="$(cd "$HERE/.." && pwd)" -TAG="${TAG:-semoss-sandbox:dev}" -NO_CACHE="${NO_CACHE:-0}" - -[ -d "$REPO/py" ] || { echo "no py/ under $REPO" >&2; exit 1; } - -ARGS=(build - -f "$HERE/Dockerfile.sandbox" - -t "$TAG" - --label "org.opencontainers.image.revision=$(git -C "$REPO" rev-parse HEAD 2>/dev/null || echo unknown)" - --label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)") -[ "$NO_CACHE" = "1" ] && ARGS+=(--no-cache --pull) - -# context is py/ itself, so the image carries this checkout's worker (uncommitted edits included) -DOCKER_BUILDKIT=1 docker "${ARGS[@]}" "$REPO/py" - -"$HERE/sandbox-smoke-test.sh" "$TAG" "$REPO/py" diff --git a/docker/sandbox-smoke-test.sh b/docker/sandbox-smoke-test.sh index ba77503d96..c909a008a9 100755 --- a/docker/sandbox-smoke-test.sh +++ b/docker/sandbox-smoke-test.sh @@ -25,6 +25,9 @@ docker run --rm --entrypoint bash -e EXPECT="$EXPECT" "$IMAGE" -ec " [ \"\$GOT\" = \"\$EXPECT\" ] || { echo \"py tree in image (\$GOT) differs from $PY_DIR (\$EXPECT)\" >&2; exit 1; } P=/usr/lib/python/semossvenv/bin/python3 [ -x \"\$P\" ] || { echo \"no venv python at \$P\" >&2; exit 1; } + # the sandbox policy's egress rules name this path (OPENSHELL_PYTHON_BINARY's default) + [ \"\$(readlink -f \$P)\" = /usr/lib/python/cpython/bin/python3.14 ] \\ + || { echo \"interpreter is \$(readlink -f \$P), not /usr/lib/python/cpython/bin/python3.14\" >&2; exit 1; } # same launch shape as OpenShellTransport: explicit interpreter, bare PATH env -i PATH=/usr/local/bin:/usr/bin:/bin \"\$P\" -c 'import sys, pandas, numpy, jsonpickle, gaas_tcp_server_handler, gaas_change_watcher; print(sys.version)' git --version diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index 16f8f24a03..e97f08da62 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -191,17 +191,19 @@ after every successful Python Builder run. Tags are `-` pin. Each architecture is smoke-tested before anything is tagged. Point `OPENSHELL_IMAGE` at one of these tags. -For a local build: +`OPENSHELL_IMAGE` defaults to `quay.io/semoss/semoss-sandbox:dev-3.14-cpu`, so a deployment or +a workstation that tracks `dev` sets nothing. To try a `py/` change before CI publishes it: ```bash -docker/build-sandbox-image.sh # tag semoss-sandbox:dev -TAG=semoss-sandbox:test NO_CACHE=1 docker/build-sandbox-image.sh +docker build -f docker/Dockerfile.sandbox -t semoss-sandbox:dev py # context is py/ +docker/sandbox-smoke-test.sh semoss-sandbox:dev ``` -The build context is `py/` itself, so a local build carries the checkout's worker, uncommitted -edits included. Both the script and CI run `docker/sandbox-smoke-test.sh`, which fails unless -the worker tree in the image hashes to the source `py/`, the venv python imports the worker -under a bare `PATH`, git is present and the image runs as uid 1000. +and set `OPENSHELL_IMAGE semoss-sandbox:dev`. The build context is `py/` itself, so a local +build carries the checkout's worker, uncommitted edits included. CI runs the same +`docker/sandbox-smoke-test.sh`, which fails unless the worker tree in the image hashes to the +source `py/`, the venv python imports the worker under a bare `PATH`, the interpreter is at the +path the policy names, git is present and the image runs as uid 1000. The venv lives at `/usr/lib/python/semossvenv` and the worker is launched by absolute path, because **both the environment and the filesystem inside a sandbox belong to the runtime**: @@ -211,9 +213,16 @@ The venv lives at `/usr/lib/python/semossvenv` and the worker is launched by abs `python3` at the venv does not help either — CPython looks for `pyvenv.cfg` relative to the invoked path and silently falls back to the base interpreter. - **Filesystem**: the policy allowlists `/usr`, `/lib`, `/etc`, `/bin`, `/sbin`, `/proc` - read-only, plus `/sandbox`, `/tmp`, `/opt/semosshome` read-write. A venv at `/opt/venv` is - outside every one of those and dies with `PermissionError: pyvenv.cfg`; `/usr/lib/python` - is inside `/usr`, which is exactly why upstream puts it there. + read-only, plus `/sandbox`, `/tmp`, `/opt/semosshome` and the platform's SEMOSS home + read-write. A venv at `/opt/venv` is outside every one of those and dies with + `PermissionError: pyvenv.cfg`; `/usr/lib/python` is inside `/usr`, which is exactly why + upstream puts it there. +- **Egress**: the bundled policy allows only pypi (so `pip install` works), and a network + rule names the kernel-reported executable, so it is granted to the interpreter's *real* path. + Other egress is not configurable yet beyond supplying a whole `OPENSHELL_POLICY`. uv's directory carries the patch version and + the CPU (`cpython-3.14.8-linux-aarch64-gnu`), so the image moves it to + `/usr/lib/python/cpython` and leaves the old name as a symlink: the rule is + `/usr/lib/python/cpython/bin/python3.14` on every build and architecture. So `OpenShellTransport` resolves `/usr/lib/python/semossvenv/bin/python3` (falling back to `command -v python3` for images without a venv) and creates the sandbox with @@ -224,14 +233,55 @@ At session start the launcher also uploads the host's `/opt/semosshome/py` over copy, but the worker's first imports happen before that upload, so the baked copy still has to be current — a stale image shows up as code that ignores your latest change. +## Policy + +The policy is generated, not hand-kept. `OpenShellTransport` reads `OPENSHELL_POLICY` if it is +set, otherwise the template bundled in the jar +(`src/prerna/util/openshell/sandbox-policy.yaml`), replaces `${SEMOSS_HOME_ROOT}` with the top folder of `BaseFolder` (see below) +and `${PYTHON}` with `OPENSHELL_PYTHON_BINARY`, and creates the sandbox with the result (a file +in the temp directory named by its content). On a server the home is the image's own +`/opt/semosshome`, and that line is dropped. A custom policy may use the same +placeholders. + +## Workstations (BaseFolder is not /opt/semosshome) + +Files cross into a sandbox **at the path they have on the host**: a project uploaded from +`/Users/me/Documents/semoss/project/x__id` is at that same path inside the sandbox. That is what +keeps the python SEMOSS generates working unchanged: MCP tool loaders, `PySource` and the pandas +JSON hand-off all embed absolute host paths. Translating paths instead would mean finding and +rewriting every such string. + +Nothing has to be built per developer. The home is taken from `BaseFolder` at runtime: + +- the policy grants the top folder of its path (`/Users` for `/Users/me/Documents/semoss`, + `/home` for `/home/me/semoss`), and +- when the sandbox starts, `OpenShellTransport` creates the home under it as the sandbox user. + +Both steps need that root to exist in the image when the sandbox starts, writable by uid 1000: +uploads run as the sandbox user inside the filesystem policy (measured: `ssh tar extract exited +with status 1` for a path the image did not have), and the policy can only grant a path that +exists. `docker/Dockerfile.sandbox` makes `/Users` and `/home` for this. A home under any other +root needs that root added the same way, for example: + +```dockerfile +FROM semoss-sandbox:dev +USER root +RUN mkdir -p /data && chown 1000:1000 /data +USER 1000:1000 +``` + +If the root is missing, sandbox creation fails with a message saying so, rather than with the +upload error. + ## Configuration | Key | Meaning | | --- | --- | | `SANDBOX_MODE` | `OPENSHELL` enables this path (requires `CHROOT_ENABLE=true`, like every jail) | -| `OPENSHELL_IMAGE` | image a new sandbox starts from | +| `OPENSHELL_IMAGE` | image a new sandbox starts from (default `quay.io/semoss/semoss-sandbox:dev-3.14-cpu`) | | `OPENSHELL_TEMPLATE` | workload template to create from instead of an image | -| `OPENSHELL_POLICY` | Landlock paths and per-binary egress rules; required | +| `OPENSHELL_POLICY` | policy template to render instead of the bundled one (see Policy); optional | +| `OPENSHELL_PYTHON_BINARY` | interpreter path the policy's egress rules name (default `/usr/lib/python/cpython/bin/python3.14`) | | `OPENSHELL_CLI` | path to the `openshell` CLI (default `openshell`) | | `OPENSHELL_CLI_HOME` | `HOME` for CLI children, where the gateway registration and mTLS bundle live | | `OPENSHELL_LABEL` | label applied to every executor sandbox (default `app=semoss-executor`) | diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 4bf5f62acb..472a05e040 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -53,6 +53,7 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.HexFormat; import java.util.List; import java.util.Map; import java.util.Set; @@ -131,10 +132,30 @@ public final class OpenShellTransport { public static final String CLI_HOME_PROP = "OPENSHELL_CLI_HOME"; /** Property naming the sandbox workload template to create from. */ public static final String TEMPLATE_PROP = "OPENSHELL_TEMPLATE"; - /** Property naming the sandbox image (used when no template is set). */ + /** + * Property naming the sandbox image (used when no template is set). Defaults to the image + * CI publishes from {@code docker/Dockerfile.sandbox} for {@code dev}. + */ public static final String IMAGE_PROP = "OPENSHELL_IMAGE"; - /** Property naming a filesystem-policy YAML applied at creation. */ + private static final String DEFAULT_IMAGE = "quay.io/semoss/semoss-sandbox:dev-3.14-cpu"; + /** + * Property naming a sandbox-policy YAML applied at creation. Optional: unset, the policy + * bundled with SEMOSS ({@value #POLICY_RESOURCE}) is used. Either way + * {@code ${SEMOSS_HOME_ROOT}} and {@code ${PYTHON}} are filled in first; see + * {@link #renderPolicy}. + */ public static final String POLICY_PROP = "OPENSHELL_POLICY"; + /** + * Property naming the sandbox interpreter's real path, which the policy's egress rules + * name. The default matches {@code docker/Dockerfile.sandbox}, which moves CPython to a + * fixed directory so the path does not change with the patch version or the CPU. + */ + public static final String PYTHON_BINARY_PROP = "OPENSHELL_PYTHON_BINARY"; + private static final String DEFAULT_PYTHON_BINARY = "/usr/lib/python/cpython/bin/python3.14"; + /** The bundled policy template, on the classpath. */ + private static final String POLICY_RESOURCE = "/prerna/util/openshell/sandbox-policy.yaml"; + /** The SEMOSS home inside the sandbox image: the worker's py/ and insight folder. */ + private static final String IMAGE_HOME = "/opt/semosshome"; /** Property naming the port the worker listens on *inside* the sandbox. */ public static final String WORKER_PORT_PROP = "OPENSHELL_WORKER_PORT"; /** Property naming the label value used to group executor sandboxes. */ @@ -306,23 +327,13 @@ private static List createCommand(String sandboxName, String owner) thro create.add("--template"); create.add(template.trim()); } else { - String image = Utility.getDIHelperProperty(IMAGE_PROP); - if (image == null || image.trim().isEmpty()) { - throw new IOException("no sandbox image: set " + IMAGE_PROP + " (or " + TEMPLATE_PROP - + "). The stock OpenShell image has no python3, so omitting this produces a sandbox" - + " that starts and then cannot run the worker."); - } + // Never the stock OpenShell image: it has no python3, so the sandbox would start and + // then fail to run the worker. create.add("--from"); - create.add(image.trim()); - } - String policy = Utility.getDIHelperProperty(POLICY_PROP); - if (policy == null || policy.trim().isEmpty()) { - throw new IOException("no sandbox policy: set " + POLICY_PROP - + ". Creating a sandbox without a filesystem policy leaves the workload unfenced, which" - + " is never what a deployment intends."); + create.add(propOr(IMAGE_PROP, DEFAULT_IMAGE)); } create.add("--policy"); - create.add(policy.trim()); + create.add(renderPolicy().toString()); // Give sandboxed code the same interpreter the worker uses. The runtime does not // apply the image's ENV PATH, so without this a `Py()` block that shells out to // `python3` gets the bare system interpreter while the worker runs from the venv. @@ -493,6 +504,118 @@ public static List listExecutorSandboxes() throws IOException, Inte return rows; } + /** + * The policy file to create sandboxes with: {@link #POLICY_PROP} or the bundled template, + * with {@code ${SEMOSS_HOME_ROOT}} replaced by {@link #homeRoot()} and {@code ${PYTHON}} by + * {@link #PYTHON_BINARY_PROP}. + * + *

    + * Generated rather than shipped as a fixed file because the home differs per deployment: + * files cross into the sandbox at their host path, and the filesystem policy can only grant + * a path by name. On a server the home is the image's own /opt/semosshome, and the line is + * dropped. + * + * @return a file under the temp directory, named by its content, so concurrent creates and + * restarts reuse one rendering + */ + static Path renderPolicy() throws IOException { + String configured = Utility.getDIHelperProperty(POLICY_PROP); + String template; + if (configured != null && !configured.trim().isEmpty()) { + template = Files.readString(Paths.get(configured.trim()), StandardCharsets.UTF_8); + } else { + try (InputStream in = OpenShellTransport.class.getResourceAsStream(POLICY_RESOURCE)) { + if (in == null) { + throw new IOException("bundled sandbox policy " + POLICY_RESOURCE + " is missing from the build"); + } + template = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + } + String root = homeRoot(); + StringBuilder rendered = new StringBuilder(); + for (String line : template.split("\n", -1)) { + if (line.trim().startsWith("#")) { + // the template documents its placeholders; leave the comments as written + rendered.append(line).append('\n'); + continue; + } + if (line.trim().equals("- ${SEMOSS_HOME_ROOT}") && root == null) { + continue; + } + rendered.append(line.replace("${SEMOSS_HOME_ROOT}", root == null ? IMAGE_HOME : root) + .replace("${PYTHON}", propOr(PYTHON_BINARY_PROP, DEFAULT_PYTHON_BINARY))).append('\n'); + } + String text = rendered.toString(); + String hash; + try { + hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))) + .substring(0, 16); + } catch (NoSuchAlgorithmException e) { + throw new IOException("SHA-256 unavailable", e); + } + Path file = Paths.get(System.getProperty("java.io.tmpdir"), "openshell-policy-" + hash + ".yaml"); + if (!Files.exists(file)) { + Path partial = Files.createTempFile(file.getParent(), "openshell-policy-", ".partial"); + Files.writeString(partial, text, StandardCharsets.UTF_8); + Files.move(partial, file, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } + return file; + } + + /** The platform's SEMOSS home (BaseFolder), normalized and without a trailing slash. */ + private static String semossHome() { + String home = Utility.normalizePath(Utility.getBaseFolder()); + return home.length() > 1 && home.endsWith("/") ? home.substring(0, home.length() - 1) : home; + } + + /** Whether the home is the image's own, which the image and the policy already provide. */ + private static boolean homeIsImageHome() { + String home = semossHome(); + return home.equals(IMAGE_HOME) || home.startsWith(IMAGE_HOME + "/"); + } + + /** + * The folder the policy grants so the home can be made inside the sandbox: the top folder + * of the home's path, {@code /Users} for {@code /Users/me/Documents/semoss}. Landlock only + * grants a path that exists when the sandbox starts, and the home itself does not exist in + * a generic image; {@code docker/Dockerfile.sandbox} makes {@code /Users} and {@code /home} + * for exactly this. Null when the home is the image's own. + */ + static String homeRoot() { + if (homeIsImageHome()) { + return null; + } + String home = semossHome(); + int slash = home.indexOf('/', 1); + return slash < 0 ? home : home.substring(0, slash); + } + + /** + * Make the SEMOSS home inside the sandbox, at its host path, and fail with the fix in the + * message when that is not possible. Otherwise the first upload fails with only "ssh tar + * extract exited with status 1": uploads run as the sandbox user, who can only create + * directories under a root the image has and the policy grants. + */ + private void checkSandboxHome() throws IOException { + if (homeIsImageHome()) { + return; + } + String home = semossHome(); + try { + run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "exec", "-n", sandboxName, "--", "sh", + "-c", "mkdir -p \"$1\" && test -w \"$1\"", "sh", home), 60); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted checking the SEMOSS home in sandbox " + sandboxName, e); + } catch (IOException e) { + throw new IOException("Could not make " + home + ", the SEMOSS home (BaseFolder), inside the sandbox." + + " Files cross into the sandbox at their host path, so the image must have a writable " + + homeRoot() + ": docker/Dockerfile.sandbox provides /Users and /home. For a home under" + + " another root, add that root to the image owned by uid 1000 (see" + + " docs/deployment/executor_sandbox.md) and set " + IMAGE_PROP + " to it.", e); + } + } + /** The CLI may print notices before its JSON; keep from the first brace. */ private static String jsonBody(String out) { int brace = out.indexOf('{'); @@ -540,6 +663,7 @@ public int start() throws IOException, InterruptedException { } try { + checkSandboxHome(); openForward(readyTimeout); } catch (IOException e) { close(); diff --git a/src/prerna/util/openshell/sandbox-policy.yaml b/src/prerna/util/openshell/sandbox-policy.yaml new file mode 100644 index 0000000000..2ac2048c46 --- /dev/null +++ b/src/prerna/util/openshell/sandbox-policy.yaml @@ -0,0 +1,43 @@ +version: 1 +# SEMOSS sandbox policy, the default when OPENSHELL_POLICY is unset. OpenShellTransport fills in +# ${SEMOSS_HOME_ROOT} the top folder of BaseFolder's path (/Users for /Users/me/...); files +# cross into the sandbox at their host path, and the home is created +# under this root when the sandbox starts +# ${PYTHON} the interpreter's real path (OPENSHELL_PYTHON_BINARY), which egress rules name +# and passes the result to `openshell sandbox create --policy`. A custom OPENSHELL_POLICY file may +# use the same placeholders. +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /etc + - /bin + - /sbin + # process inspection (ps, /proc scans, pid-based kill) fails without it + - /proc + - /dev/urandom + read_write: + - /sandbox + - /tmp + # the image's own home: the worker (py/) and its insight folder + - /opt/semosshome + # where the platform's home is made, for projects, rooms and user assets; on a server + # the home is /opt/semosshome itself and the line is dropped + - ${SEMOSS_HOME_ROOT} + # redirects fail unless it is listed + - /dev/null +landlock: + compatibility: hard_requirement + +# The only egress the workload gets. pip is a script, so a rule names the INTERPRETER: the +# kernel-reported executable, not a symlink to it. +network_policies: + pypi: + endpoints: + - host: pypi.org + port: 443 + - host: files.pythonhosted.org + port: 443 + binaries: + - path: ${PYTHON} diff --git a/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java b/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java index 0a14d8a985..a8f8cdd48b 100644 --- a/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java +++ b/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java @@ -89,8 +89,11 @@ public static void main(String[] args) throws Exception { String policy = Utility.getDIHelperProperty(OpenShellTransport.POLICY_PROP); System.out.println("SANDBOX_MODE=" + mode + " image=" + image + " policy=" + policy); check("P-0 SANDBOX_MODE is OPENSHELL", "OPENSHELL".equalsIgnoreCase(mode), String.valueOf(mode)); - check("P-1 image is configured", image != null && !image.trim().isEmpty(), String.valueOf(image)); - check("P-2 policy is configured", policy != null && !policy.trim().isEmpty(), String.valueOf(policy)); + check("P-1 image is configured or defaulted", true, image == null || image.trim().isEmpty() ? "default" : image); + // unset means the policy bundled with SEMOSS, rendered for this home at create time + boolean bundled = OpenShellTransport.class.getResource("/prerna/util/openshell/sandbox-policy.yaml") != null; + check("P-2 policy is configured or bundled", (policy != null && !policy.trim().isEmpty()) || bundled, + policy == null || policy.trim().isEmpty() ? "bundled=" + bundled : policy); check("P-3 CHROOT_ENABLE is on", Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE)), String.valueOf(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE))); if (failed > 0) { From ec1517fe4a6795e2c3baa82e7f438ffd063c46e1 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 12:51:01 +0000 Subject: [PATCH 52/56] feat(openshell): RDF settings for sandbox egress and CPU/memory caps - OPENSHELL_EGRESS: host[:port] list rendered into the bundled policy's network_policies for the sandbox interpreter; default pypi, "none" for no egress; entries are validated as plain hostnames - OPENSHELL_CPU / OPENSHELL_MEMORY: per-sandbox caps passed to sandbox create, default 2 CPUs and 4Gi, "none" for no cap; caps not reservations, so idle sandboxes (~130 MB worker) cost little - the policy grants /dev/shm: multiprocessing, joblib and torch data loaders failed with PermissionError on SemLock without it Measured through the docker driver: caps land on the container (NanoCpus/Memory), an allocation past the cap is killed, custom egress allows only what is listed. --- docs/deployment/executor_sandbox.md | 35 ++++++- src/prerna/util/OpenShellTransport.java | 91 +++++++++++++++++++ src/prerna/util/openshell/sandbox-policy.yaml | 19 ++-- 3 files changed, 131 insertions(+), 14 deletions(-) diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index e97f08da62..95bad20ddd 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -217,9 +217,9 @@ The venv lives at `/usr/lib/python/semossvenv` and the worker is launched by abs read-write. A venv at `/opt/venv` is outside every one of those and dies with `PermissionError: pyvenv.cfg`; `/usr/lib/python` is inside `/usr`, which is exactly why upstream puts it there. -- **Egress**: the bundled policy allows only pypi (so `pip install` works), and a network - rule names the kernel-reported executable, so it is granted to the interpreter's *real* path. - Other egress is not configurable yet beyond supplying a whole `OPENSHELL_POLICY`. uv's directory carries the patch version and +- **Egress**: `OPENSHELL_EGRESS` lists the hosts sandboxed code may reach (default pypi, so + `pip install` works). A network rule names the kernel-reported executable, so the list is + granted to the interpreter's *real* path. uv's directory carries the patch version and the CPU (`cpython-3.14.8-linux-aarch64-gnu`), so the image moves it to `/usr/lib/python/cpython` and leaves the old name as a symlink: the rule is `/usr/lib/python/cpython/bin/python3.14` on every build and architecture. @@ -243,6 +243,32 @@ in the temp directory named by its content). On a server the home is the image's `/opt/semosshome`, and that line is dropped. A custom policy may use the same placeholders. +## Egress and sizing + +Set in `RDF_Map.prop`, applied to every sandbox when it is created (an existing user sandbox keeps +what it was created with until it is replaced): + +``` +OPENSHELL_EGRESS pypi.org, files.pythonhosted.org, query1.finance.yahoo.com:443 +OPENSHELL_CPU 2 +OPENSHELL_MEMORY 4Gi +``` + +- `OPENSHELL_EGRESS` is `host[:port]` separated by commas or spaces, port 443 when omitted. + Unset means pypi only; `none` means no egress. Every host is granted to the sandbox + interpreter; other programs (`git`, `curl`) get no network. Entries are validated as plain + hostnames, so the setting cannot inject policy. It is rendered through the + `${NETWORK_POLICIES}` placeholder, so a custom `OPENSHELL_POLICY` without it keeps its own rules. +- `OPENSHELL_CPU` and `OPENSHELL_MEMORY` are caps, not reservations: an idle sandbox costs + little (the worker is about 130 MB with pandas loaded), so the sum of the caps may exceed + the host. `none` removes a cap. Defaults are 2 CPUs and 4 GiB. +- Past the memory cap the kernel kills the largest process in the sandbox. A `Py()` block runs + in the worker, so an over-allocating block is expected to take the worker down and the + user's next block to start a fresh python: variables lost, files kept. (Measured so far only + for a separate process, which was killed while the sandbox stayed up.) +- Measured through the docker driver: the caps land on the container + (`NanoCpus`, `Memory`), and an allocation is killed at the memory cap. + ## Workstations (BaseFolder is not /opt/semosshome) Files cross into a sandbox **at the path they have on the host**: a project uploaded from @@ -281,6 +307,9 @@ upload error. | `OPENSHELL_IMAGE` | image a new sandbox starts from (default `quay.io/semoss/semoss-sandbox:dev-3.14-cpu`) | | `OPENSHELL_TEMPLATE` | workload template to create from instead of an image | | `OPENSHELL_POLICY` | policy template to render instead of the bundled one (see Policy); optional | +| `OPENSHELL_EGRESS` | hosts sandboxed code may reach, `host[:port]` list (default pypi; `none` for no egress) | +| `OPENSHELL_CPU` | CPU cap per sandbox, e.g. `2`, `1.5`, `500m` (default `2`; `none` for no cap) | +| `OPENSHELL_MEMORY` | memory cap per sandbox, e.g. `4Gi`, `512Mi` (default `4Gi`; `none` for no cap) | | `OPENSHELL_PYTHON_BINARY` | interpreter path the policy's egress rules name (default `/usr/lib/python/cpython/bin/python3.14`) | | `OPENSHELL_CLI` | path to the `openshell` CLI (default `openshell`) | | `OPENSHELL_CLI_HOME` | `HOME` for CLI children, where the gateway registration and mTLS bundle live | diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index 472a05e040..d694181988 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -61,6 +61,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; @@ -152,6 +153,34 @@ public final class OpenShellTransport { */ public static final String PYTHON_BINARY_PROP = "OPENSHELL_PYTHON_BINARY"; private static final String DEFAULT_PYTHON_BINARY = "/usr/lib/python/cpython/bin/python3.14"; + /** + * Property listing the hosts sandboxed code may reach, as {@code host[:port]} separated by + * commas or spaces (port 443 when omitted). Unset means pypi, so {@code pip install} works; + * {@code none} means no egress at all. Granted to the sandbox interpreter only, and only + * applied through the {@code ${NETWORK_POLICIES}} placeholder of the policy template. + */ + public static final String EGRESS_PROP = "OPENSHELL_EGRESS"; + private static final String DEFAULT_EGRESS = "pypi.org:443, files.pythonhosted.org:443"; + /** A hostname, nothing a YAML scalar could be broken out of with. */ + private static final Pattern EGRESS_HOST = Pattern + .compile("[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?(\\.[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?)*"); + /** + * Property capping each sandbox's CPU, in Kubernetes quantity form ({@code 2}, {@code 1.5}, + * {@code 500m}); {@code none} for no cap. A cap, not a reservation: idle sandboxes cost nothing. + */ + public static final String CPU_PROP = "OPENSHELL_CPU"; + private static final String DEFAULT_CPU = "2"; + /** + * Property capping each sandbox's memory ({@code 4Gi}, {@code 512Mi}); {@code none} for no + * cap. Past it the kernel kills the largest process in the sandbox, which for a {@code Py()} + * block is the worker: the user gets a fresh python on the next block. + */ + public static final String MEMORY_PROP = "OPENSHELL_MEMORY"; + private static final String DEFAULT_MEMORY = "4Gi"; + private static final Pattern CPU_QUANTITY = Pattern + .compile("\\d+(\\.\\d+)?m?"); + private static final Pattern MEMORY_QUANTITY = Pattern + .compile("\\d+(\\.\\d+)?(Ki|Mi|Gi|Ti|K|M|G|T)?"); /** The bundled policy template, on the classpath. */ private static final String POLICY_RESOURCE = "/prerna/util/openshell/sandbox-policy.yaml"; /** The SEMOSS home inside the sandbox image: the worker's py/ and insight folder. */ @@ -334,6 +363,16 @@ private static List createCommand(String sandboxName, String owner) thro } create.add("--policy"); create.add(renderPolicy().toString()); + String cpu = resourceLimit(CPU_PROP, DEFAULT_CPU, CPU_QUANTITY); + if (cpu != null) { + create.add("--cpu"); + create.add(cpu); + } + String memory = resourceLimit(MEMORY_PROP, DEFAULT_MEMORY, MEMORY_QUANTITY); + if (memory != null) { + create.add("--memory"); + create.add(memory); + } // Give sandboxed code the same interpreter the worker uses. The runtime does not // apply the image's ENV PATH, so without this a `Py()` block that shells out to // `python3` gets the bare system interpreter while the worker runs from the venv. @@ -542,6 +581,10 @@ static Path renderPolicy() throws IOException { if (line.trim().equals("- ${SEMOSS_HOME_ROOT}") && root == null) { continue; } + if (line.trim().equals("${NETWORK_POLICIES}")) { + rendered.append(networkPolicies(propOr(PYTHON_BINARY_PROP, DEFAULT_PYTHON_BINARY))); + continue; + } rendered.append(line.replace("${SEMOSS_HOME_ROOT}", root == null ? IMAGE_HOME : root) .replace("${PYTHON}", propOr(PYTHON_BINARY_PROP, DEFAULT_PYTHON_BINARY))).append('\n'); } @@ -562,6 +605,54 @@ static Path renderPolicy() throws IOException { return file; } + /** + * The {@code network_policies} section for {@link #EGRESS_PROP}: one rule granting every + * listed endpoint to the interpreter, or an empty section for {@code none}. + */ + static String networkPolicies(String python) throws IOException { + String configured = Utility.getDIHelperProperty(EGRESS_PROP); + String list = configured == null || configured.trim().isEmpty() ? DEFAULT_EGRESS : configured.trim(); + if (list.equalsIgnoreCase("none")) { + return "network_policies: {}\n"; + } + StringBuilder yaml = new StringBuilder("network_policies:\n semoss_egress:\n endpoints:\n"); + for (String entry : list.split("[,\\s]+")) { + if (entry.isEmpty()) { + continue; + } + int colon = entry.lastIndexOf(':'); + String host = colon < 0 ? entry : entry.substring(0, colon); + int port; + try { + port = colon < 0 ? 443 : Integer.parseInt(entry.substring(colon + 1)); + } catch (NumberFormatException e) { + port = -1; + } + if (!EGRESS_HOST.matcher(host).matches() || port < 1 || port > 65535) { + throw new IOException(EGRESS_PROP + " entry '" + entry + "' is not host[:port]"); + } + yaml.append(" - host: ").append(host).append("\n port: ").append(port).append('\n'); + } + yaml.append(" binaries:\n - path: ").append(python).append('\n'); + return yaml.toString(); + } + + /** + * A sandbox resource cap from its property: the default when unset, null for {@code none}, + * and an error for anything that is not a quantity the CLI accepts. + */ + private static String resourceLimit(String prop, String fallback, Pattern format) + throws IOException { + String value = propOr(prop, fallback); + if (value.equalsIgnoreCase("none")) { + return null; + } + if (!format.matcher(value).matches()) { + throw new IOException(prop + " '" + value + "' is not a valid quantity"); + } + return value; + } + /** The platform's SEMOSS home (BaseFolder), normalized and without a trailing slash. */ private static String semossHome() { String home = Utility.normalizePath(Utility.getBaseFolder()); diff --git a/src/prerna/util/openshell/sandbox-policy.yaml b/src/prerna/util/openshell/sandbox-policy.yaml index 2ac2048c46..445d4d0f6a 100644 --- a/src/prerna/util/openshell/sandbox-policy.yaml +++ b/src/prerna/util/openshell/sandbox-policy.yaml @@ -4,6 +4,7 @@ version: 1 # cross into the sandbox at their host path, and the home is created # under this root when the sandbox starts # ${PYTHON} the interpreter's real path (OPENSHELL_PYTHON_BINARY), which egress rules name +# ${NETWORK_POLICIES} the network_policies section, generated from OPENSHELL_EGRESS # and passes the result to `openshell sandbox create --policy`. A custom OPENSHELL_POLICY file may # use the same placeholders. filesystem_policy: @@ -27,17 +28,13 @@ filesystem_policy: - ${SEMOSS_HOME_ROOT} # redirects fail unless it is listed - /dev/null + # POSIX semaphores and shared memory: multiprocessing, joblib (scikit-learn n_jobs) and + # torch data loaders fail with PermissionError on SemLock without it + - /dev/shm landlock: compatibility: hard_requirement -# The only egress the workload gets. pip is a script, so a rule names the INTERPRETER: the -# kernel-reported executable, not a symlink to it. -network_policies: - pypi: - endpoints: - - host: pypi.org - port: 443 - - host: files.pythonhosted.org - port: 443 - binaries: - - path: ${PYTHON} +# The only egress the workload gets: OPENSHELL_EGRESS (default pypi, so pip works), granted to +# the interpreter. pip is a script, so a rule names the INTERPRETER: the kernel-reported +# executable, not a symlink to it. +${NETWORK_POLICIES} From ba2a73a4b3b091a680e725abd48c4306011fc726 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 13:25:15 +0000 Subject: [PATCH 53/56] docs(openshell): multiprocessing in Py() is still blocked by the import guard /dev/shm makes it work for processes in the sandbox; inside Py() the worker's InsightGlobalStore guard removes os.fork/os.kill and bans subprocess, so Pool fails there (measured on the live instance). --- docs/deployment/executor_sandbox.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index 95bad20ddd..155b4f4177 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -268,6 +268,11 @@ OPENSHELL_MEMORY 4Gi for a separate process, which was killed while the sandbox stayed up.) - Measured through the docker driver: the caps land on the container (`NanoCpus`, `Memory`), and an allocation is killed at the memory cap. +- `/dev/shm` is granted so POSIX semaphores work for processes in the sandbox (a shell + command running a script that uses `multiprocessing`). Inside `Py()` itself, multiprocessing + still fails: the worker's import guard (`InsightGlobalStore`) bans `subprocess` and `socket` + and deletes `os.fork`, `os.kill`, `os.system` and the `exec*`/`spawn*` family. That guard + predates OpenShell; whether to relax it now that the sandbox is the boundary is open. ## Workstations (BaseFolder is not /opt/semosshome) From c6ce071ed0b0be6c04d5bfe94f3be1b22e2baead Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 17:46:11 +0000 Subject: [PATCH 54/56] feat(openshell): broker app secrets into the sandbox with OpenShell providers A project declares SANDBOX_SECRETS (name + hosts) in its .smss and sets SANDBOX_SECRET_. When a user's sandbox loads the project, each secret becomes an OpenShell profile + provider attached to that sandbox, so code gets a placeholder (smss_secrets.secret(name)) that OpenShell's supervisor swaps for the value only in requests to the declared hosts. The .smss never enters the sandbox, so the value stays on the host. - credential names carry the project key, so two apps may share a name - the value reaches the CLI through the environment, never argv - an attach is waited out, and the placeholder file is rewritten after every attach or update once the new generation appears; old generations stop resolving (attach) or keep the old value (update) - an unchanged value is not re-applied, so later loads cost ~0.5 s Measured against a real sandbox with a live OpenWeather key: 200 through secret(), refused at another app's host, rotation reaches running code. --- docs/deployment/executor_sandbox.md | 33 ++ py/smss_secrets.py | 49 +++ src/prerna/util/OpenShellTransport.java | 32 ++ src/prerna/util/OpenShellTransporter.java | 15 +- .../util/openshell/OpenShellSecrets.java | 393 ++++++++++++++++++ 5 files changed, 521 insertions(+), 1 deletion(-) create mode 100644 py/smss_secrets.py create mode 100644 src/prerna/util/openshell/OpenShellSecrets.java diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md index 155b4f4177..07a8c7e2f5 100644 --- a/docs/deployment/executor_sandbox.md +++ b/docs/deployment/executor_sandbox.md @@ -274,6 +274,39 @@ OPENSHELL_MEMORY 4Gi and deletes `os.fork`, `os.kill`, `os.system` and the `exec*`/`spawn*` family. That guard predates OpenShell; whether to relax it now that the sandbox is the boundary is open. +## App secrets + +A project can give its sandboxed code a secret the code can use but never read. In the project +`.smss` (project owners edit it): + +``` +SANDBOX_SECRETS [{"name":"OPENWEATHER_API_KEY","hosts":["api.openweathermap.org"]}] +SANDBOX_SECRET_OPENWEATHER_API_KEY +``` + +and in python: + +```python +from smss_secrets import secret +requests.get(url, params={"appid": secret("OPENWEATHER_API_KEY")}) +``` + +- The `.smss` never enters a sandbox, so the value stays on the host. Each secret becomes one + OpenShell provider; code gets a placeholder, and OpenShell's supervisor swaps it for the value in + requests to the declared hosts (headers and query string verified) and refuses it anywhere else + (`403 credential_endpoint_mismatch`, logged as a HIGH finding). Any request method is allowed. + HTTP(S) only: a database password cannot be brokered. +- Brokered when a user's sandbox first loads the project (`OpenShellSecrets`, called from the + transporter for a project root). The first load waits for the attach to apply (~10 s). +- Names are per project: two apps may both declare `OPENAI_API_KEY`; with both loaded, + `secret(name, project=)` picks one. +- A changed value is applied on the next load of the project (an edited `.smss` needs a project + reload) and reaches code that is already running: `secret()` re-reads the placeholders, which + OpenShell replaces on every update and attach. +- The value property can later come from the secret store, which overlays the same properties. +- All code in one user's sandbox runs as that user, so any app the user has loaded can use the + placeholders of another app they have loaded, at that app's hosts. Isolation is per user. + ## Workstations (BaseFolder is not /opt/semosshome) Files cross into a sandbox **at the path they have on the host**: a project uploaded from diff --git a/py/smss_secrets.py b/py/smss_secrets.py new file mode 100644 index 0000000000..8a5b0b4cd3 --- /dev/null +++ b/py/smss_secrets.py @@ -0,0 +1,49 @@ +"""Secrets an app declares in its project .smss, as OpenShell placeholders. + +The project declares SANDBOX_SECRETS (name + hosts) and sets SANDBOX_SECRET_; SEMOSS +brokers each into the user's sandbox when the project is loaded. Code here only ever holds a +placeholder: OpenShell's supervisor swaps it for the value in requests to the secret's declared +hosts (headers, query string) and refuses it anywhere else. + + from smss_secrets import secret + key = secret("OPENWEATHER_API_KEY") + requests.get(url, params={"appid": key}) +""" + +import json +import re + +# written by prerna.util.openshell.OpenShellSecrets after every attach +_PLACEHOLDERS = "/opt/semosshome/insight/sandbox-secrets/placeholders.json" + + +def _project_key(project_id): + return re.sub(r"[^a-z0-9]", "", project_id.lower())[:8] + + +def secret(name, project=None): + """The placeholder for a declared secret. + + Args: + name: the name in the project's SANDBOX_SECRETS, e.g. "OPENWEATHER_API_KEY". + project: the project id, needed only when several loaded projects declare the name. + """ + try: + with open(_PLACEHOLDERS) as f: + loaded = json.load(f) + except FileNotFoundError: + raise KeyError( + f"no sandbox secrets are available: declare {name} in the project's " + "SANDBOX_SECRETS, set its value, and load the project" + ) from None + wanted = _project_key(project) if project else None + found = { + key: values[name] + for key, values in loaded.items() + if name in values and (wanted is None or key == wanted) + } + if not found: + raise KeyError(f"{name} is not a secret of any loaded project") + if len(found) > 1: + raise KeyError(f"{name} is declared by several loaded projects; pass project=") + return next(iter(found.values())) diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java index d694181988..da4a85e80c 100644 --- a/src/prerna/util/OpenShellTransport.java +++ b/src/prerna/util/OpenShellTransport.java @@ -637,6 +637,16 @@ static String networkPolicies(String python) throws IOException { return yaml.toString(); } + /** The interpreter path network rules name ({@link #PYTHON_BINARY_PROP}). */ + public static String pythonBinary() { + return propOr(PYTHON_BINARY_PROP, DEFAULT_PYTHON_BINARY); + } + + /** Whether a value is a plain hostname, safe to place in a policy or profile. */ + public static boolean isValidHost(String host) { + return host != null && EGRESS_HOST.matcher(host).matches(); + } + /** * A sandbox resource cap from its property: the default when unset, null for {@code none}, * and an error for anything that is not a quantity the CLI accepts. @@ -1903,9 +1913,31 @@ private static int freeLocalPort() throws IOException { } } + /** + * Run an {@code openshell} subcommand, for collaborators such as the secret broker. + * + * @param args everything after the CLI name + * @param extraEnv added to the child's environment; how a credential value reaches the CLI + * ({@code --credential KEY} reads it from there), so it is never on a command + * line or in a process listing + */ + public static String cli(List args, int timeoutSeconds, Map extraEnv) + throws IOException, InterruptedException { + List cmd = new ArrayList<>(); + cmd.add(propOr(CLI_PROP, DEFAULT_CLI)); + cmd.addAll(args); + return run(cmd, timeoutSeconds, extraEnv); + } + private static String run(List cmd, int timeoutSeconds) throws IOException, InterruptedException { + return run(cmd, timeoutSeconds, Map.of()); + } + + private static String run(List cmd, int timeoutSeconds, Map extraEnv) + throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(cmd); pb.environment().putAll(cliEnv()); + pb.environment().putAll(extraEnv); pb.redirectErrorStream(true); // Close the child's stdin immediately. Without this it inherits a pipe that // stays open for the JVM's lifetime and any subcommand that reads stdin -- diff --git a/src/prerna/util/OpenShellTransporter.java b/src/prerna/util/OpenShellTransporter.java index 25f19e6bc4..4b0833c376 100644 --- a/src/prerna/util/OpenShellTransporter.java +++ b/src/prerna/util/OpenShellTransporter.java @@ -27,6 +27,9 @@ *******************************************************************************/ package prerna.util; +import prerna.util.openshell.OpenShellSecrets; +import prerna.util.openshell.TransportedRoots; + /** * {@link SandboxTransporter} for {@link SymlinkHelper.SandboxMode#OPENSHELL}: copies a host path into * the sandbox with {@code openshell sandbox upload}. @@ -57,7 +60,17 @@ public boolean transport(String absPath) { if (absPath == null || absPath.trim().isEmpty()) { return false; } - return this.transport.upload(absPath); + boolean uploaded = this.transport.upload(absPath); + if (uploaded) { + // A project root arriving in the sandbox is when its declared secrets are attached: + // only users allowed to load the project get here. + String normalized = Utility.normalizePath(absPath); + String projectId = TransportedRoots.projectIdOf(normalized); + if (projectId != null && normalized.matches(".*/" + Constants.PROJECT_FOLDER + "/[^/]+/app_root/?$")) { + OpenShellSecrets.ensureForProject(this.transport, projectId); + } + } + return uploaded; } @Override diff --git a/src/prerna/util/openshell/OpenShellSecrets.java b/src/prerna/util/openshell/OpenShellSecrets.java new file mode 100644 index 0000000000..d3274e4ab8 --- /dev/null +++ b/src/prerna/util/openshell/OpenShellSecrets.java @@ -0,0 +1,393 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import prerna.project.api.IProject; +import prerna.util.OpenShellTransport; +import prerna.util.Utility; + +/** + * Secrets an app declares in its project {@code .smss}, brokered into a user's sandbox by + * OpenShell providers, so sandboxed code can use a secret without ever holding it. + * + * 

    + * SANDBOX_SECRETS                     [{"name":"OPENWEATHER_API_KEY","hosts":["api.openweathermap.org"]}]
+ * SANDBOX_SECRET_OPENWEATHER_API_KEY  the value
+ *
    + * + *

    + * The value stays on the host: the {@code .smss} is never uploaded into a sandbox. Each secret + * becomes one OpenShell profile and provider, so it can only reach its own hosts. Code in the + * sandbox gets a placeholder ({@code secret("OPENWEATHER_API_KEY")} in {@code smss_secrets}), and + * OpenShell's supervisor swaps it for the value in requests to those hosts and refuses it + * anywhere else. Any request method is allowed. + * + *

    + * Measured behaviour this follows (2026-10-24, OpenShell 0.1.2): + *

      + *
    • attaching a provider to a running sandbox takes tens of seconds to apply, and requests + * in that window hang, so an attach is waited out;
      • + *
    • an attach changes the placeholder generation and the old placeholders stop resolving, so + * the placeholder file is rewritten from the sandbox's current environment after every attach + * (the worker's own environment is fixed at start);
      • + *
    • {@code provider update} also starts a new generation, within seconds, and placeholders + * of the old one keep resolving to the old value; so after an update the file is + * rewritten once the new generation appears, and an unchanged value is not updated at all.
      • + *
    + * The helper reads the file on every call, which is what lets the long-lived worker follow both. + * The value in the property can later come from the secret store instead, which overlays the + * same properties. + */ +public final class OpenShellSecrets { + + private static final Logger classLogger = LogManager.getLogger(OpenShellSecrets.class); + + /** The project property declaring the secrets, as a JSON list of {name, hosts}. */ + public static final String SECRETS_PROP = "SANDBOX_SECRETS"; + /** The project property holding one secret's value, followed by its name. */ + public static final String VALUE_PROP_PREFIX = "SANDBOX_SECRET_"; + /** Where the placeholders are written in the sandbox; read by {@code py/smss_secrets.py}. */ + private static final String PLACEHOLDER_PARENT = "/opt/semosshome/insight"; + private static final String PLACEHOLDER_DIR = "sandbox-secrets"; + private static final String PLACEHOLDER_FILE = "placeholders.json"; + + private static final Pattern NAME = Pattern.compile("[A-Z][A-Z0-9_]{0,63}"); + /** A credential in the sandbox environment: {@code __}. */ + private static final Pattern CREDENTIAL_ENV = Pattern.compile("([A-Z][A-Z0-9_]*)__([a-z0-9]{8})=(openshell:resolve:\\S+)"); + private static final long READY_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(90); + private static final long READY_POLL_MS = 2000; + private static final int CLI_TIMEOUT_S = 60; + + /** How long a new placeholder generation may take to appear after an update (measured 5–15 s). */ + private static final long GENERATION_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(60); + private static final Pattern GENERATION = Pattern.compile("=openshell:resolve:env:v(\\d+)_"); + + private static final Map SANDBOX_LOCKS = new ConcurrentHashMap<>(); + /** Digest of the value last given to each provider by this JVM; unknown after a restart. */ + private static final Map APPLIED = new ConcurrentHashMap<>(); + + private OpenShellSecrets() { + // static utility + } + + /** One declared secret. */ + record Declared(String name, List hosts) { + } + + /** + * Make a project's declared secrets usable in this sandbox: register each with OpenShell, + * update its value, attach it if it is not yet, and refresh the placeholder file. A secret + * that is declared wrongly or has no value is skipped with a warning; nothing here fails + * the project load. + */ + public static void ensureForProject(OpenShellTransport transport, String projectId) { + IProject project = Utility.getProject(projectId); + if (project != null && project.getSmssProp() != null) { + ensureForProject(transport, projectId, project.getSmssProp()); + } + } + + /** {@link #ensureForProject(OpenShellTransport, String)} for a project's properties. */ + public static void ensureForProject(OpenShellTransport transport, String projectId, Properties props) { + List declared = parse(projectId, props.getProperty(SECRETS_PROP)); + if (declared.isEmpty()) { + return; + } + String sandbox = transport.getSandboxName(); + synchronized (SANDBOX_LOCKS.computeIfAbsent(sandbox, k -> new Object())) { + try { + String attached = OpenShellTransport.cli(List.of("sandbox", "provider", "list", sandbox), + CLI_TIMEOUT_S, Map.of()); + String generation = generation(sandbox); + boolean changed = false; + for (Declared secret : declared) { + String value = props.getProperty(VALUE_PROP_PREFIX + secret.name()); + if (value == null || value.trim().isEmpty()) { + classLogger.warn("Project {} declares sandbox secret {} but sets no {}{}", projectId, + secret.name(), VALUE_PROP_PREFIX, secret.name()); + continue; + } + String id = providerId(projectId, secret); + String credential = credentialName(projectId, secret.name()); + ensureProfile(id, projectId, secret, credential); + changed |= ensureProvider(id, credential, value.trim()); + if (!containsWord(attached, id)) { + changed = true; + OpenShellTransport.cli(List.of("sandbox", "provider", "attach", sandbox, id), CLI_TIMEOUT_S, + Map.of()); + awaitApplied(sandbox, id); + classLogger.info("Attached sandbox secret {} of project {} to sandbox {}", secret.name(), + projectId, sandbox); + } + } + if (changed && generation != null) { + awaitNewGeneration(sandbox, generation); + } + writePlaceholders(sandbox); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + classLogger.warn("Interrupted brokering secrets of project {} into sandbox {}", projectId, sandbox); + } catch (Exception e) { + classLogger.warn("Could not broker secrets of project {} into sandbox {}", projectId, sandbox, e); + } + } + } + + /** The declared secrets, skipping (with a warning) any entry that is not well formed. */ + static List parse(String projectId, String json) { + List declared = new ArrayList<>(); + if (json == null || json.trim().isEmpty()) { + return declared; + } + JsonArray list; + try { + list = JsonParser.parseString(json).getAsJsonArray(); + } catch (RuntimeException e) { + classLogger.warn("Project {} has an unreadable {}: expected a JSON list", projectId, SECRETS_PROP); + return declared; + } + for (JsonElement element : list) { + try { + JsonObject entry = element.getAsJsonObject(); + String name = entry.get("name").getAsString().trim(); + List hosts = new ArrayList<>(); + for (JsonElement host : entry.getAsJsonArray("hosts")) { + hosts.add(host.getAsString().trim().toLowerCase()); + } + if (!NAME.matcher(name).matches() || hosts.isEmpty() + || !hosts.stream().allMatch(OpenShellTransport::isValidHost)) { + classLogger.warn("Project {} {} entry {} needs an upper-case name and plain host names", projectId, + SECRETS_PROP, element); + continue; + } + declared.add(new Declared(name, List.copyOf(new TreeSet<>(hosts)))); + } catch (RuntimeException e) { + classLogger.warn("Project {} {} entry {} is not {\"name\", \"hosts\"}", projectId, SECRETS_PROP, element); + } + } + return declared; + } + + /** The project's key in credential names and the placeholder file: its first 8 id characters. */ + static String projectKey(String projectId) { + String compact = projectId.toLowerCase().replaceAll("[^a-z0-9]", ""); + return compact.length() >= 8 ? compact.substring(0, 8) : String.format("%-8s", compact).replace(' ', '0'); + } + + /** The credential's name in the sandbox, unique per project so two apps may share a name. */ + static String credentialName(String projectId, String name) { + return name + "__" + projectKey(projectId); + } + + /** + * The profile and provider id. It includes a digest of the hosts and the interpreter, so a + * changed declaration is a new profile rather than an edit of one that sandboxes use. + */ + static String providerId(String projectId, Declared secret) { + String hosts = digest(String.join(",", secret.hosts()) + "|" + OpenShellTransport.pythonBinary()).substring(0, 6); + return "semoss-" + projectKey(projectId) + "-" + secret.name().toLowerCase().replace('_', '-') + "-" + hosts; + } + + private static void ensureProfile(String id, String projectId, Declared secret, String credential) + throws IOException, InterruptedException { + try { + OpenShellTransport.cli(List.of("profile", "describe", id), CLI_TIMEOUT_S, Map.of()); + return; + } catch (IOException missing) { + // not registered yet + } + StringBuilder yaml = new StringBuilder(); + yaml.append("id: ").append(id).append('\n'); + yaml.append("display_name: ").append(secret.name()).append(" (SEMOSS project ").append(projectId).append(")\n"); + yaml.append("description: sandbox secret declared by SEMOSS project ").append(projectId).append('\n'); + yaml.append("category: other\n"); + yaml.append("credentials:\n - name: ").append(credential).append("\n required: true\n"); + yaml.append("endpoints:\n"); + for (String host : secret.hosts()) { + // credentialed endpoints must be inspected (L7) for the supervisor to swap the value in + yaml.append(" - host: ").append(host).append("\n port: 443\n protocol: rest\n access: full\n"); + } + // without a binary the generated rule lets no program connect + yaml.append("binaries:\n - ").append(OpenShellTransport.pythonBinary()).append('\n'); + Path dir = Files.createTempDirectory("openshell-profile-"); + try { + Path file = dir.resolve(id + ".yaml"); + Files.writeString(file, yaml.toString(), StandardCharsets.UTF_8); + OpenShellTransport.cli(List.of("profile", "import", "--file", file.toString()), CLI_TIMEOUT_S, Map.of()); + } finally { + deleteQuietly(dir); + } + } + + /** + * Create the provider, or update its value when it changed. + * + * @return whether the value was set, which starts a new placeholder generation + */ + private static boolean ensureProvider(String id, String credential, String value) + throws IOException, InterruptedException { + String digest = digest(value); + Map env = Map.of(credential, value); + boolean exists; + try { + OpenShellTransport.cli(List.of("provider", "get", id), CLI_TIMEOUT_S, Map.of()); + exists = true; + } catch (IOException missing) { + exists = false; + } + if (exists && digest.equals(APPLIED.get(id))) { + return false; + } + if (exists) { + OpenShellTransport.cli(List.of("provider", "update", id, "--credential", credential), CLI_TIMEOUT_S, env); + } else { + OpenShellTransport.cli(List.of("provider", "create", "--name", id, "--type", id, "--credential", credential), + CLI_TIMEOUT_S, env); + } + APPLIED.put(id, digest); + return true; + } + + /** The placeholder generation new processes in the sandbox get, or null with no secret yet. */ + private static String generation(String sandbox) throws IOException, InterruptedException { + Matcher m = GENERATION.matcher(OpenShellTransport.cli(List.of("sandbox", "exec", "-n", sandbox, "--", "env"), + CLI_TIMEOUT_S, Map.of())); + return m.find() ? m.group(1) : null; + } + + /** Wait for the generation an update or attach starts; the old one keeps the old value. */ + private static void awaitNewGeneration(String sandbox, String before) throws IOException, InterruptedException { + long deadline = System.currentTimeMillis() + GENERATION_TIMEOUT_MS; + while (System.currentTimeMillis() < deadline) { + String now = generation(sandbox); + if (now != null && !now.equals(before)) { + return; + } + Thread.sleep(READY_POLL_MS); + } + classLogger.warn("Sandbox {} kept placeholder generation {} for {} s after a secret change", sandbox, before, + GENERATION_TIMEOUT_MS / 1000); + } + + private static String digest(String value) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable", e); + } + } + + /** Wait until the sandbox reports the provider change applied; requests hang until then. */ + private static void awaitApplied(String sandbox, String id) throws IOException, InterruptedException { + long deadline = System.currentTimeMillis() + READY_TIMEOUT_MS; + String status = ""; + while (System.currentTimeMillis() < deadline) { + status = OpenShellTransport.cli(List.of("sandbox", "provider", "status", sandbox, id), CLI_TIMEOUT_S, Map.of()); + if (status.contains(": ready")) { + return; + } + Thread.sleep(READY_POLL_MS); + } + classLogger.warn("Sandbox {} did not report secret {} applied within {} s: {}", sandbox, id, + READY_TIMEOUT_MS / 1000, status.lines().findFirst().orElse("")); + } + + /** + * Write every brokered secret's current placeholder into the sandbox, by project key. The + * placeholders are not secret; they only resolve at the secret's own hosts. + */ + private static void writePlaceholders(String sandbox) throws IOException, InterruptedException { + String env = OpenShellTransport.cli(List.of("sandbox", "exec", "-n", sandbox, "--", "env"), CLI_TIMEOUT_S, + Map.of()); + Map> byProject = new TreeMap<>(); + for (String line : env.split("\\R")) { + Matcher m = CREDENTIAL_ENV.matcher(line.trim()); + if (m.matches()) { + byProject.computeIfAbsent(m.group(2), k -> new LinkedHashMap<>()).put(m.group(1), m.group(3)); + } + } + // A directory, not a lone file: for a file the CLI takes the destination as the file's + // own name ("tar: insight: Cannot open: File exists"), while a directory lands inside it. + Path dir = Files.createTempDirectory("openshell-secrets-"); + try { + Path stage = Files.createDirectory(dir.resolve(PLACEHOLDER_DIR)); + Files.writeString(stage.resolve(PLACEHOLDER_FILE), new Gson().toJson(byProject), StandardCharsets.UTF_8); + OpenShellTransport.cli(List.of("sandbox", "upload", sandbox, stage.toString(), PLACEHOLDER_PARENT), + CLI_TIMEOUT_S, Map.of()); + } finally { + deleteQuietly(dir); + } + } + + private static boolean containsWord(String text, String word) { + return Pattern.compile("(^|\\s)" + Pattern.quote(word) + "(\\s|$)", Pattern.MULTILINE).matcher(text).find(); + } + + private static void deleteQuietly(Path dir) { + try (var files = Files.walk(dir)) { + files.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // temp file + } + }); + } catch (IOException ignored) { + // temp dir + } + } +} From 30763638095c5be6336abdeae3194117351ea451 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 19:13:05 +0000 Subject: [PATCH 55/56] build(docker): restore yfinance in the sandbox image for the Stock MCP example The old slim image pip-installed it as a stopgap for the example; the python-builder venv does not declare it, so Stock MCP failed with ModuleNotFoundError after the 3.14 switch. APP_PACKAGES keeps the list explicit until apps can declare their own dependencies. --- docker/Dockerfile.sandbox | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/Dockerfile.sandbox b/docker/Dockerfile.sandbox index f3631fc2bd..3dd5c1bc4b 100644 --- a/docker/Dockerfile.sandbox +++ b/docker/Dockerfile.sandbox @@ -42,6 +42,12 @@ RUN real="$(readlink -f /usr/lib/python/semossvenv/bin/python3)" \ && mv "$dir" /usr/lib/python/cpython \ && ln -s cpython "$dir" \ && test "$(readlink -f /usr/lib/python/semossvenv/bin/python3)" = "/usr/lib/python/cpython/bin/$(basename "$real")" +# Packages example apps need that SEMOSS does not declare (Stock MCP: yfinance). A stopgap until an +# app can declare its own dependencies; keep the list short, and pass APP_PACKAGES="" to omit it. +ARG APP_PACKAGES="yfinance" +RUN if [ -n "${APP_PACKAGES}" ]; then \ + /usr/lib/python/semossvenv/bin/python3 -m pip install --no-cache-dir ${APP_PACKAGES}; \ + fi ENV VIRTUAL_ENV=/usr/lib/python/semossvenv ENV PATH=/usr/lib/python/semossvenv/bin:$PATH From ace8346ba2b10f0fca611f7a7ff6e6b15753faf4 Mon Sep 17 00:00:00 2001 From: kunal0137 Date: Sat, 24 Oct 2026 21:46:53 +0000 Subject: [PATCH 56/56] Revert "build(docker): restore yfinance in the sandbox image for the Stock MCP example" Skip yfinance as an image package for now; the Stock MCP example is parked. This reverts commit 3076363809. --- docker/Dockerfile.sandbox | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker/Dockerfile.sandbox b/docker/Dockerfile.sandbox index 3dd5c1bc4b..f3631fc2bd 100644 --- a/docker/Dockerfile.sandbox +++ b/docker/Dockerfile.sandbox @@ -42,12 +42,6 @@ RUN real="$(readlink -f /usr/lib/python/semossvenv/bin/python3)" \ && mv "$dir" /usr/lib/python/cpython \ && ln -s cpython "$dir" \ && test "$(readlink -f /usr/lib/python/semossvenv/bin/python3)" = "/usr/lib/python/cpython/bin/$(basename "$real")" -# Packages example apps need that SEMOSS does not declare (Stock MCP: yfinance). A stopgap until an -# app can declare its own dependencies; keep the list short, and pass APP_PACKAGES="" to omit it. -ARG APP_PACKAGES="yfinance" -RUN if [ -n "${APP_PACKAGES}" ]; then \ - /usr/lib/python/semossvenv/bin/python3 -m pip install --no-cache-dir ${APP_PACKAGES}; \ - fi ENV VIRTUAL_ENV=/usr/lib/python/semossvenv ENV PATH=/usr/lib/python/semossvenv/bin:$PATH