diff --git a/.github/workflows/sandbox-builder.yml b/.github/workflows/sandbox-builder.yml new file mode 100644 index 0000000000..7247945c9a --- /dev/null +++ b/.github/workflows/sandbox-builder.yml @@ -0,0 +1,144 @@ +name: Sandbox Builder +# Builds docker/Dockerfile.sandbox: the OpenShell sandbox image that runs a user's python +# worker. It is the python-builder image plus this repo's py/, so it rebuilds when py/ changes +# and after every successful Python Builder run (new interpreter or dependencies). +on: + workflow_dispatch: + push: + branches: ["dev", "main"] + paths: + - "py/**" + - "docker/Dockerfile.sandbox" + - "docker/Dockerfile.sandbox.dockerignore" + - "docker/sandbox-smoke-test.sh" + - ".github/workflows/sandbox-builder.yml" + workflow_run: + workflows: ["Python Builder"] + types: [completed] + branches: ["dev", "main"] + +concurrency: + group: sandbox-builder-${{ github.event.workflow_run.head_branch || github.ref_name }} + cancel-in-progress: true + +env: + IMAGE: quay.io/semoss/semoss-sandbox + # must match the COPY --from tag in docker/Dockerfile.sandbox + PYTHON_BUILDER_TAG: "3.14-cpu" + BRANCH: ${{ github.event.workflow_run.head_branch || github.ref_name }} + SOURCE_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} + +jobs: + # --------------------------------------------------------------------------- + # Build each architecture natively, push BY DIGEST (no tag), then smoke-test + # that digest. A failed smoke test never reaches the tagging job. + # --------------------------------------------------------------------------- + build: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + arch: amd64 + runner_prefix: codebuild-semoss-github-runner + container: quay.io/semoss/test-quay:ubuntu-dind + - platform: linux/arm64 + arch: arm64 + runner_prefix: codebuild-semoss-github-runner-arm64 + container: quay.io/semoss/test-quay:ubuntu-dind-arm64 + runs-on: ${{ matrix.runner_prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + container: + image: ${{ matrix.container }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ env.SOURCE_SHA }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Quay + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_ROBOT_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_PASSWORD }} + + - name: Build and push by digest (${{ matrix.platform }}) + id: build + uses: docker/build-push-action@v6 + with: + # context is py/; the Dockerfile and its .dockerignore live in docker/ + context: ./py + file: ./docker/Dockerfile.sandbox + platforms: ${{ matrix.platform }} + pull: true + labels: | + org.opencontainers.image.revision=${{ env.SOURCE_SHA }} + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.semoss.python-builder=${{ env.PYTHON_BUILDER_TAG }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + + - name: Smoke test (${{ matrix.platform }}) + run: bash docker/sandbox-smoke-test.sh "${IMAGE}@${{ steps.build.outputs.digest }}" py + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # --------------------------------------------------------------------------- + # Stitch the per-arch digests into one tagged manifest list. + # --------------------------------------------------------------------------- + merge: + needs: build + runs-on: ubuntu-latest + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Quay + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_ROBOT_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_PASSWORD }} + + - name: Compute tags + id: meta + shell: bash + run: | + BASE="${BRANCH//\//-}-${PYTHON_BUILDER_TAG}" + TAGS=( "-t" "${IMAGE}:${BASE}" "-t" "${IMAGE}:${BASE}-${SOURCE_SHA:0:7}" ) + if [[ "${{ vars.RELEASE }}" == "TRUE" ]]; then + TAGS+=( "-t" "${IMAGE}:${PYTHON_BUILDER_TAG}-${{ vars.VERSION }}" ) + fi + echo "args=${TAGS[*]}" >> "$GITHUB_OUTPUT" + echo "primary=${IMAGE}:${BASE}" >> "$GITHUB_OUTPUT" + + - name: Create and push manifest list + working-directory: /tmp/digests + run: | + docker buildx imagetools create ${{ steps.meta.outputs.args }} \ + $(printf "${IMAGE}@sha256:%s " *) + + - name: Inspect manifest list + run: docker buildx imagetools inspect "${{ steps.meta.outputs.primary }}" diff --git a/docker/Dockerfile.sandbox b/docker/Dockerfile.sandbox new file mode 100644 index 0000000000..f3631fc2bd --- /dev/null +++ b/docker/Dockerfile.sandbox @@ -0,0 +1,69 @@ +# SEMOSS sandbox image: the OpenShell sandbox that runs a user's persistent python worker. +# +# Published by .github/workflows/sandbox-builder.yml as quay.io/semoss/semoss-sandbox, the default +# OPENSHELL_IMAGE. To try a py/ change before it is published: +# +# docker build -f docker/Dockerfile.sandbox -t semoss-sandbox:dev py # context is py/ +# docker/sandbox-smoke-test.sh semoss-sandbox:dev +# +# Python and every dependency come from the python-builder image (docker/Dockerfile.python, +# uv + py/install_config/pyproject.toml). Like the Tomcat images, we copy its /usr/lib/python +# onto a plain Ubuntu base, so a Py() block has the same interpreter and imports inside the +# sandbox as outside it. +# +# The venv is reached by explicit path, never by PATH lookup: OpenShell builds the environment +# for the processes it spawns, so ENV PATH is not applied inside a sandbox. See +# docs/deployment/executor_sandbox.md. +ARG BASE_REGISTRY=public.ecr.aws/docker/library +ARG BASE_IMAGE=ubuntu +ARG BASE_TAG=24.04 + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +LABEL maintainer="semoss@semoss.org" +LABEL org.semoss.component="sandbox" + +ENV DEBIAN_FRONTEND=noninteractive + +# git for local repository work; egress stays policy-gated per binary +RUN for i in 1 2 3; do apt-get update && break || (sleep 10 && rm -rf /var/lib/apt/lists/*); done \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# uv-managed CPython + semossvenv; /usr/lib/python sits inside the policy's read-only /usr +COPY --from=quay.io/semoss/python-builder:3.14-cpu /usr/lib/python /usr/lib/python + +# One interpreter path for every build: the sandbox policy's egress rules name the kernel-reported +# executable, and uv's directory carries the patch version and the CPU architecture +# (cpython-3.14.8-linux-aarch64-gnu). Move the tree to a fixed name and leave the original as a +# symlink, so the venv still resolves and the policy can say /usr/lib/python/cpython/bin/python3.14. +RUN real="$(readlink -f /usr/lib/python/semossvenv/bin/python3)" \ + && dir="$(dirname "$(dirname "$real")")" \ + && mv "$dir" /usr/lib/python/cpython \ + && ln -s cpython "$dir" \ + && test "$(readlink -f /usr/lib/python/semossvenv/bin/python3)" = "/usr/lib/python/cpython/bin/$(basename "$real")" +ENV VIRTUAL_ENV=/usr/lib/python/semossvenv +ENV PATH=/usr/lib/python/semossvenv/bin:$PATH + +# the sandbox identity is uid/gid 1000 (ubuntu:24.04 already has it as "ubuntu") +RUN if ! getent group 1000 >/dev/null; then groupadd -g 1000 sandbox; fi \ + && if ! getent passwd 1000 >/dev/null; then useradd -u 1000 -g 1000 -m -s /bin/bash sandbox; fi + +# the worker source (build context is the repo's py/) +COPY . /opt/semosshome/py/ + +# setup() appends to insight/log.txt; grants-* are the policy's ro/rw drop points +RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \ + && touch /opt/semosshome/insight/log.txt \ + && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \ + && chmod 755 /opt/grants-rw /opt/grants-ro + +# Workstation homes. Files cross into a sandbox at their host path, so a Mac's BaseFolder +# (/Users/me/...) or a Linux one (/home/me/...) must exist inside it. A sandbox can only create +# directories under a path its policy grants and that exists at start, so these roots are made +# here, owned by the sandbox user; OpenShellTransport grants the root and creates the home. +RUN mkdir -p /Users && chown 1000:1000 /Users /home + +# OpenShell refuses to start unless the workspace is writable by the sandbox identity +USER 1000:1000 +WORKDIR /opt/semosshome/py diff --git a/docker/Dockerfile.sandbox.dockerignore b/docker/Dockerfile.sandbox.dockerignore new file mode 100644 index 0000000000..64900ac9d8 --- /dev/null +++ b/docker/Dockerfile.sandbox.dockerignore @@ -0,0 +1,7 @@ +# context is py/; keep caches and local tooling out of the sandbox image +**/__pycache__ +**/*.pyc +**/.DS_Store +**/*.egg-info +.pytest_cache +.continue diff --git a/docker/sandbox-smoke-test.sh b/docker/sandbox-smoke-test.sh new file mode 100755 index 0000000000..c909a008a9 --- /dev/null +++ b/docker/sandbox-smoke-test.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Prove a built sandbox image is usable: the worker tree matches the given py/, the venv +# python imports the worker, git is present and the image runs as uid 1000. +# +# docker/sandbox-smoke-test.sh [py-dir] # py-dir defaults to the repo's py/ +set -euo pipefail + +IMAGE="${1:?usage: sandbox-smoke-test.sh [py-dir]}" +PY_DIR="${2:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/py}" + +# same exclusions as Dockerfile.sandbox.dockerignore +py_sha() { + find . -type f -name '*.py' -not -path '*/__pycache__/*' -not -path './.pytest_cache/*' \ + -not -path './.continue/*' -not -path '*.egg-info/*' -print0 \ + | sort -z | xargs -0 sha256sum | sha256sum | cut -d' ' -f1 +} + +EXPECT="$(cd "$PY_DIR" && py_sha)" + +docker run --rm --entrypoint bash -e EXPECT="$EXPECT" "$IMAGE" -ec " + $(declare -f py_sha) + cd /opt/semosshome/py + test -f gaas_tcp_socket_server.py || { echo 'worker missing at /opt/semosshome/py' >&2; exit 1; } + GOT=\$(py_sha) + [ \"\$GOT\" = \"\$EXPECT\" ] || { echo \"py tree in image (\$GOT) differs from $PY_DIR (\$EXPECT)\" >&2; exit 1; } + P=/usr/lib/python/semossvenv/bin/python3 + [ -x \"\$P\" ] || { echo \"no venv python at \$P\" >&2; exit 1; } + # the sandbox policy's egress rules name this path (OPENSHELL_PYTHON_BINARY's default) + [ \"\$(readlink -f \$P)\" = /usr/lib/python/cpython/bin/python3.14 ] \\ + || { echo \"interpreter is \$(readlink -f \$P), not /usr/lib/python/cpython/bin/python3.14\" >&2; exit 1; } + # same launch shape as OpenShellTransport: explicit interpreter, bare PATH + env -i PATH=/usr/local/bin:/usr/bin:/bin \"\$P\" -c 'import sys, pandas, numpy, jsonpickle, gaas_tcp_server_handler, gaas_change_watcher; print(sys.version)' + git --version + [ \"\$(id -u)\" = 1000 ] || { echo 'image does not run as uid 1000' >&2; exit 1; } +" +echo "smoke test passed for $IMAGE" diff --git a/docs/README.md b/docs/README.md index 44591b6939..6b5715243f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -82,6 +82,7 @@ The [Python GAAS tools index](python_gaas_tools/README.md) documents Python acce - [Local Docker setup](cloud_and_cluster/docker_deployment.md) - [Local Docker configuration](deployment/docker_configuration.md) +- [OpenShell executor sandbox](deployment/executor_sandbox.md): running user code in a per-session container, the write-back pipeline and change detection, per-space write permissions, and building the image from `py/` - [Supporting engine examples](../docker-compose-examples/engines/README.md) - [Cloud and cluster architecture](cloud_and_cluster/README.md) - [Central asset storage](cloud_and_cluster/central_cloud_storage.md) diff --git a/docs/cloud_and_cluster/github_actions_workflows.md b/docs/cloud_and_cluster/github_actions_workflows.md index f7639942d2..8e6b28ddf2 100644 --- a/docs/cloud_and_cluster/github_actions_workflows.md +++ b/docs/cloud_and_cluster/github_actions_workflows.md @@ -8,6 +8,7 @@ The workflows in [.github/workflows](../../.github/workflows/) build Java artifa | --- | --- | | [dev_workflow.yml](../../.github/workflows/dev_workflow.yml) | Java build, test selection, artifact signing, and publication paths | | [python-builder.yml](../../.github/workflows/python-builder.yml) | Python runtime/dependency image | +| [sandbox-builder.yml](../../.github/workflows/sandbox-builder.yml) | OpenShell sandbox image: python-builder plus the `py/` worker ([details](../deployment/executor_sandbox.md#image)) | | [tomcat-builder.yml](../../.github/workflows/tomcat-builder.yml) | Java/Tomcat builder image | | [all-docker-builds.yml](../../.github/workflows/all-docker-builds.yml) | Application image build orchestration | | [ubuntu2204.yml](../../.github/workflows/ubuntu2204.yml), [ubuntu2404.yml](../../.github/workflows/ubuntu2404.yml) | Ubuntu application images | diff --git a/docs/deployment/executor_sandbox.md b/docs/deployment/executor_sandbox.md new file mode 100644 index 0000000000..07a8c7e2f5 --- /dev/null +++ b/docs/deployment/executor_sandbox.md @@ -0,0 +1,388 @@ +# OpenShell executor sandbox + +How user code runs when SEMOSS is configured with `SANDBOX_MODE=OPENSHELL`, what the sandbox +can and cannot change on the host, and how write-back decides what to copy. + +This is an opt-in execution path. With `SANDBOX_MODE` unset or set to any other value, the +platform uses the traditional local child process and none of this applies. + +## Why it exists + +`Py()` and shell commands are arbitrary code. Running them as a child of the JVM means a +process that shares the host's filesystem, environment and network namespace. The OpenShell +mode moves that process into a separate container, spawned and reaped with the user's +session, and keeps the platform's files authoritative on the host. + +## Topology at a glance + +``` +Tomcat (host) ──CLI──► openshell gateway (host, :17670, mTLS) + │ │ provisions + │ ▼ + │ supervisor container (uid nobody, trusted side) + │ │ control.sock (unix socket on a docker volume) + │ ▼ + │ workload container (network=none) + │ PID 1 /.openshell/runtime/openshell-sandbox + │ └── python worker (loopback :7100) + │ ▲ + └── openshell forward service --target-port 7100 --local ─┘ + (long-lived child of Tomcat; Java connects to 127.0.0.1:) +``` + +Per session there are **two containers**: the workload (your image) and its supervisor (NVIDIA's image). +The supervisor is the trusted half of the boundary and is never inside the workload. The workload has no +network interface at all — its only egress is the channel to its supervisor — so a TCP connection to the +worker has to travel `Tomcat → forward → gateway → supervisor → runtime → loopback`. + +## Process model + +| Piece | Runs where | Notes | +| --- | --- | --- | +| Java (Tomcat) | host | owns the socket client, decides when to sync | +| `gaas_tcp_socket_server.py` | sandbox, **main process** | the persistent REPL; one per **user**, shared by all their sessions | +| shell commands | sandbox | started by `OpenShellShellRunner`, same write-back rules | +| user assets / projects / rooms | host | copied in at session start, written back at block boundaries | + +The worker is stateful: variables survive between blocks, which is why it cannot be +restarted to pick up a change. It runs with `--persist`: a client leaving (disconnect, +`stop`, logout) does not end it, it serves up to `OPENSHELL_WORKER_MAX_CLIENTS` connections +at once, and it exits by itself only after `OPENSHELL_IDLE_EXIT_MINUTES` with no client. +Each server instance holds one connection per user, shared by that user's sessions there. + +## Lifecycle (one sandbox per user) + +`OpenShellUserSandboxes` keeps a registry of owner → sandbox and owner → live sessions. It uses +Redis when one is configured (`REDIS_ENABLED`, or the cluster Redis), and an in-memory map +otherwise. The Redis keys follow the room-coordination layout: `openshell:{}:sandbox`, +`:sessions` and `:lock`, all with the `{owner}` hash tag so one user's keys share a Redis Cluster +slot, plus an `openshell:owners` index. The create lock is `SET NX PX` with a random token and is +released by a compare-and-delete script, as room locks are. The in-memory map is only correct for a single server instance, +and it forgets the sandbox when that instance restarts. + +- **Created** lazily on the first `Py()` or shell command of a user who has no running sandbox. + Creation is serialised per user with a registry lock, so two instances do not both create one. + Any later session, cookie-less API call or server instance of that user **attaches** to it. +- **Kept alive** by a lease: every authenticated request refreshes the session's entry for the + session timeout, at most once a minute. Browsing without python counts. +- **Deleted** when the user's last live session ends, by logout or timeout. A server stopping + destroys every session, but that is not the user leaving: the leases stand, and the + restarted or another instance reattaches with the worker's variables intact. +- **Swept** every five minutes on each instance. That removes sandboxes whose owner has no live + lease (a session lost with its server), labelled sandboxes that are no longer running, and, + with Redis, running user sandboxes no registry entry names. + +## Write-back pipeline + +Every `PYTHON` and `CMD` block starts with a refresh (host to sandbox) and ends with a write-back +(sandbox to host). The write-back steps, in order: + +1. **Ask for the change set.** `NativePySocketClient` sends a `CHANGES` request on the + existing socket. The worker answers + `{changes[], overflow, baseline, coverage, watched, events, stat_scan}`. +2. **Fail closed on doubt.** If the answer is missing, `overflow` is true, `baseline` is true, + or `coverage` is false, the host falls back to a full capture of each root instead of + trusting a partial list. + A change set the host drained but could not apply is owed as a full capture on the next + block, because the worker has already forgotten those paths. +3. **Pull only what changed.** Changed paths are tarred inside the sandbox under one + `sandbox exec` and streamed to a temporary file on the host, stopping at a 256 MB cap. The + archive is refused if its listing fails or names an absolute path or a `..` segment. +4. **Reconcile.** Identical content is skipped. Otherwise each side is checked against the + stamp (size and mtime, in seconds) recorded at the last sync, when both copies matched: + uploads and captures preserve mtime, so a side that no longer matches is the side that + changed. + - Only the sandbox changed: the file is written back. + - Only the host changed (another user, a cluster pull, the UI): nothing is written. The + next refresh brings the host copy in. This is what keeps a full capture from putting a + stale copy back. + - Both changed: the **last write wins**. The losing copy is kept in quarantine (the + sandbox's under `.sandbox/`), and the write is skipped if the host copy cannot be + preserved. +5. **Deletes.** Only a path the sandbox is known to have held — seen in a full capture or + written back since — is ever deleted on the host, so a file created and removed within one + block cannot take an unrelated host file with it. A host file that changed since the + sandbox had it is never deleted. Deletes are quarantine-then-delete, suppressed if the copy + fails, and directories they empty are pruned up to (never including) the transported root. +6. **Publish.** In cluster mode, a project root that was written back is pushed to cluster + storage and announced on the cluster synchronizer, at most every ten seconds per root and + once more on teardown. Servers that have the project loaded pull it, and their sandboxes + refresh from that before their next block. + +Every host-side write and delete is contained: a path that leaves the root, or crosses a +symlink on the way, is refused. Without that, a link committed into a writable project could +redirect a write or a delete into another space. + +Inside `.git`, only git's data is written back — `objects/` (not `objects/info/`), `refs/`, +`logs/`, `HEAD`, `index`, `packed-refs` and a few state files — so a commit made in the sandbox +reaches the host. `config`, `hooks/` and `info/` are never written back: host-side git runs or +obeys them, so a sandbox able to write them could execute commands on the server. + +The **refresh** before each block walks every transported root on the host. Files whose stamp +moved since the last sync are copied in with a single `sandbox upload`, which merges into the +existing tree and keeps mtimes. Files removed on the host are removed in the sandbox. More +than ~2000 changes re-upload the whole root. Any sandbox change still owed from a failed +write-back is applied first, so a refresh never overwrites unsynced work. + +Quarantine lives at `$BASE/writeback-quarantine///`, outside +`project/`, `room/` and `user/`, so it is never pushed to object storage. + +A full capture carries survivors, not tombstones. A path the sandbox held before and the +capture no longer contains was deleted there; a host file the sandbox never held — added on the +host mid-session, or never uploaded — is left alone. If `tar` reports anything it could not +read (a directory the user made unreadable, say), the capture is partial: files are written +back and no deletion is inferred. + +## Change detection + +Two mechanisms, deliberately: + +- **inotify** is the primary source. Watch descriptors are per directory, re-armed when a + directory is created, and scanned on adoption so `mkdir x; write x/f` inside one block does + not lose the file. +- **A stat-tuple scan** (`mtime_ns`, `size`, `inode`, `ctime`) is the correctness floor under + it, because a persistent `mmap` write emits no event at all. It runs at most once per + interval — `max(2 s, 20 × the last measured scan)` — so an idle block costs nothing at any + tree size. Skipping never drops a change: the index is untouched, so the next scan reports + the backlog. + +Coverage loss is a first-class outcome. `IN_MOVED_FROM`, `IN_MOVE_SELF`, `IN_IGNORED` and +`IN_UNMOUNT` all mean the path map may be stale; so does a watch that could not be added +(`max_user_watches` exhausted, an unreadable directory), a directory the scan could not list, +or a watcher thread that has stopped. In each case the watcher reports `coverage: false`, drops +the tuple index and re-arms, and the host takes one full capture for that block. A scan that +could not list part of the tree never reports the unlisted files as deleted. + +Roots are re-evaluated on every `CHANGES` request: a root that appears later in the session (a +room created mid-session, say) is armed and forces one baseline capture. + +Excluded from watching and scanning: `node_modules`, `__pycache__`, `.venv`, `.git`. +`.git` is a large share of a typical project's directories; sandbox commits still reach the +host through the full capture. `node_modules` is one install away from exhausting +`fs.inotify.max_user_watches`. Override with `SEMOSS_SYNC_IGNORE`. + +## Who may write back + +The worker runs outside the platform's permission checks, so the host decides per root: + +| Root | Rule | +| --- | --- | +| `project/__` | `SecurityProjectUtils.userCanEditProject` — the same check that grants the project read-write | +| `room/` | `ModelInferenceLogsUtils.doCheckRoomExistsForUser` — `ROOM.USER_ID` in the model-inference-logs database | +| `user/Asset__` | the id must equal one of the caller's `User.getAssetProjectId(provider)` values | +| anything else | refused | + +Every refusal is logged with the path and the reason, and each check fails closed on error. A +read-only member can still write inside the sandbox — that is a scratch area — but the file +never reaches the host. + +## Image + +One image, `docker/Dockerfile.sandbox`, serves every sandbox. Like the Tomcat images, it is a +plain `ubuntu:24.04` with `/usr/lib/python` copied in from the python-builder image +(`quay.io/semoss/python-builder:3.14-cpu`, built by `docker/Dockerfile.python` from +`py/install_config/pyproject.toml` with uv). A `Py()` block therefore gets the same Python and +the same packages inside the sandbox as outside it. On top of that the image adds git, a uid +1000 user and this repository's `py/` as the worker. + +CI publishes it as `quay.io/semoss/semoss-sandbox` from +`.github/workflows/sandbox-builder.yml`. It rebuilds on any `py/` change on `dev`/`main` and +after every successful Python Builder run. Tags are `-` (for example +`dev-3.14-cpu`) plus a `-` pin. Each architecture is smoke-tested before anything is +tagged. Point `OPENSHELL_IMAGE` at one of these tags. + +`OPENSHELL_IMAGE` defaults to `quay.io/semoss/semoss-sandbox:dev-3.14-cpu`, so a deployment or +a workstation that tracks `dev` sets nothing. To try a `py/` change before CI publishes it: + +```bash +docker build -f docker/Dockerfile.sandbox -t semoss-sandbox:dev py # context is py/ +docker/sandbox-smoke-test.sh semoss-sandbox:dev +``` + +and set `OPENSHELL_IMAGE semoss-sandbox:dev`. The build context is `py/` itself, so a local +build carries the checkout's worker, uncommitted edits included. CI runs the same +`docker/sandbox-smoke-test.sh`, which fails unless the worker tree in the image hashes to the +source `py/`, the venv python imports the worker under a bare `PATH`, the interpreter is at the +path the policy names, git is present and the image runs as uid 1000. + +The venv lives at `/usr/lib/python/semossvenv` and the worker is launched by absolute path, because +**both the environment and the filesystem inside a sandbox belong to the runtime**: + +- **`PATH`**: a workload starts with `/usr/local/bin:/usr/bin:/bin`, so an image's `ENV PATH` + is never applied and a bare `python3` resolves to the base interpreter. Symlinking + `python3` at the venv does not help either — CPython looks for `pyvenv.cfg` relative to the + invoked path and silently falls back to the base interpreter. +- **Filesystem**: the policy allowlists `/usr`, `/lib`, `/etc`, `/bin`, `/sbin`, `/proc` + read-only, plus `/sandbox`, `/tmp`, `/opt/semosshome` and the platform's SEMOSS home + read-write. A venv at `/opt/venv` is outside every one of those and dies with + `PermissionError: pyvenv.cfg`; `/usr/lib/python` is inside `/usr`, which is exactly why + upstream puts it there. +- **Egress**: `OPENSHELL_EGRESS` lists the hosts sandboxed code may reach (default pypi, so + `pip install` works). A network rule names the kernel-reported executable, so the list is + granted to the interpreter's *real* path. uv's directory carries the patch version and + the CPU (`cpython-3.14.8-linux-aarch64-gnu`), so the image moves it to + `/usr/lib/python/cpython` and leaves the old name as a symlink: the rule is + `/usr/lib/python/cpython/bin/python3.14` on every build and architecture. + +So `OpenShellTransport` resolves `/usr/lib/python/semossvenv/bin/python3` (falling back to +`command -v python3` for images without a venv) and creates the sandbox with +`--env PATH=/usr/lib/python/semossvenv/bin:…` so user code gets the same interpreter. +Smoke tests must reproduce both constraints, or they pass while the worker fails at boot. + +At session start the launcher also uploads the host's `/opt/semosshome/py` over the image's +copy, but the worker's first imports happen before that upload, so the baked copy still has to +be current — a stale image shows up as code that ignores your latest change. + +## Policy + +The policy is generated, not hand-kept. `OpenShellTransport` reads `OPENSHELL_POLICY` if it is +set, otherwise the template bundled in the jar +(`src/prerna/util/openshell/sandbox-policy.yaml`), replaces `${SEMOSS_HOME_ROOT}` with the top folder of `BaseFolder` (see below) +and `${PYTHON}` with `OPENSHELL_PYTHON_BINARY`, and creates the sandbox with the result (a file +in the temp directory named by its content). On a server the home is the image's own +`/opt/semosshome`, and that line is dropped. A custom policy may use the same +placeholders. + +## Egress and sizing + +Set in `RDF_Map.prop`, applied to every sandbox when it is created (an existing user sandbox keeps +what it was created with until it is replaced): + +``` +OPENSHELL_EGRESS pypi.org, files.pythonhosted.org, query1.finance.yahoo.com:443 +OPENSHELL_CPU 2 +OPENSHELL_MEMORY 4Gi +``` + +- `OPENSHELL_EGRESS` is `host[:port]` separated by commas or spaces, port 443 when omitted. + Unset means pypi only; `none` means no egress. Every host is granted to the sandbox + interpreter; other programs (`git`, `curl`) get no network. Entries are validated as plain + hostnames, so the setting cannot inject policy. It is rendered through the + `${NETWORK_POLICIES}` placeholder, so a custom `OPENSHELL_POLICY` without it keeps its own rules. +- `OPENSHELL_CPU` and `OPENSHELL_MEMORY` are caps, not reservations: an idle sandbox costs + little (the worker is about 130 MB with pandas loaded), so the sum of the caps may exceed + the host. `none` removes a cap. Defaults are 2 CPUs and 4 GiB. +- Past the memory cap the kernel kills the largest process in the sandbox. A `Py()` block runs + in the worker, so an over-allocating block is expected to take the worker down and the + user's next block to start a fresh python: variables lost, files kept. (Measured so far only + for a separate process, which was killed while the sandbox stayed up.) +- Measured through the docker driver: the caps land on the container + (`NanoCpus`, `Memory`), and an allocation is killed at the memory cap. +- `/dev/shm` is granted so POSIX semaphores work for processes in the sandbox (a shell + command running a script that uses `multiprocessing`). Inside `Py()` itself, multiprocessing + still fails: the worker's import guard (`InsightGlobalStore`) bans `subprocess` and `socket` + and deletes `os.fork`, `os.kill`, `os.system` and the `exec*`/`spawn*` family. That guard + predates OpenShell; whether to relax it now that the sandbox is the boundary is open. + +## App secrets + +A project can give its sandboxed code a secret the code can use but never read. In the project +`.smss` (project owners edit it): + +``` +SANDBOX_SECRETS [{"name":"OPENWEATHER_API_KEY","hosts":["api.openweathermap.org"]}] +SANDBOX_SECRET_OPENWEATHER_API_KEY +``` + +and in python: + +```python +from smss_secrets import secret +requests.get(url, params={"appid": secret("OPENWEATHER_API_KEY")}) +``` + +- The `.smss` never enters a sandbox, so the value stays on the host. Each secret becomes one + OpenShell provider; code gets a placeholder, and OpenShell's supervisor swaps it for the value in + requests to the declared hosts (headers and query string verified) and refuses it anywhere else + (`403 credential_endpoint_mismatch`, logged as a HIGH finding). Any request method is allowed. + HTTP(S) only: a database password cannot be brokered. +- Brokered when a user's sandbox first loads the project (`OpenShellSecrets`, called from the + transporter for a project root). The first load waits for the attach to apply (~10 s). +- Names are per project: two apps may both declare `OPENAI_API_KEY`; with both loaded, + `secret(name, project=)` picks one. +- A changed value is applied on the next load of the project (an edited `.smss` needs a project + reload) and reaches code that is already running: `secret()` re-reads the placeholders, which + OpenShell replaces on every update and attach. +- The value property can later come from the secret store, which overlays the same properties. +- All code in one user's sandbox runs as that user, so any app the user has loaded can use the + placeholders of another app they have loaded, at that app's hosts. Isolation is per user. + +## Workstations (BaseFolder is not /opt/semosshome) + +Files cross into a sandbox **at the path they have on the host**: a project uploaded from +`/Users/me/Documents/semoss/project/x__id` is at that same path inside the sandbox. That is what +keeps the python SEMOSS generates working unchanged: MCP tool loaders, `PySource` and the pandas +JSON hand-off all embed absolute host paths. Translating paths instead would mean finding and +rewriting every such string. + +Nothing has to be built per developer. The home is taken from `BaseFolder` at runtime: + +- the policy grants the top folder of its path (`/Users` for `/Users/me/Documents/semoss`, + `/home` for `/home/me/semoss`), and +- when the sandbox starts, `OpenShellTransport` creates the home under it as the sandbox user. + +Both steps need that root to exist in the image when the sandbox starts, writable by uid 1000: +uploads run as the sandbox user inside the filesystem policy (measured: `ssh tar extract exited +with status 1` for a path the image did not have), and the policy can only grant a path that +exists. `docker/Dockerfile.sandbox` makes `/Users` and `/home` for this. A home under any other +root needs that root added the same way, for example: + +```dockerfile +FROM semoss-sandbox:dev +USER root +RUN mkdir -p /data && chown 1000:1000 /data +USER 1000:1000 +``` + +If the root is missing, sandbox creation fails with a message saying so, rather than with the +upload error. + +## Configuration + +| Key | Meaning | +| --- | --- | +| `SANDBOX_MODE` | `OPENSHELL` enables this path (requires `CHROOT_ENABLE=true`, like every jail) | +| `OPENSHELL_IMAGE` | image a new sandbox starts from (default `quay.io/semoss/semoss-sandbox:dev-3.14-cpu`) | +| `OPENSHELL_TEMPLATE` | workload template to create from instead of an image | +| `OPENSHELL_POLICY` | policy template to render instead of the bundled one (see Policy); optional | +| `OPENSHELL_EGRESS` | hosts sandboxed code may reach, `host[:port]` list (default pypi; `none` for no egress) | +| `OPENSHELL_CPU` | CPU cap per sandbox, e.g. `2`, `1.5`, `500m` (default `2`; `none` for no cap) | +| `OPENSHELL_MEMORY` | memory cap per sandbox, e.g. `4Gi`, `512Mi` (default `4Gi`; `none` for no cap) | +| `OPENSHELL_PYTHON_BINARY` | interpreter path the policy's egress rules name (default `/usr/lib/python/cpython/bin/python3.14`) | +| `OPENSHELL_CLI` | path to the `openshell` CLI (default `openshell`) | +| `OPENSHELL_CLI_HOME` | `HOME` for CLI children, where the gateway registration and mTLS bundle live | +| `OPENSHELL_LABEL` | label applied to every executor sandbox (default `app=semoss-executor`) | +| `OPENSHELL_WORKER_PORT` | worker port inside the sandbox (default `7100`) | +| `OPENSHELL_READY_TIMEOUT_SECONDS` | cold-start budget (default `180`) | +| `OPENSHELL_KEEP_SANDBOX` | `true` leaves sandboxes in place on teardown, for debugging | +| `OPENSHELL_IDLE_EXIT_MINUTES` | a user's worker exits after this long with no client connected (default `180`; keep above the session timeout) | +| `OPENSHELL_WORKER_MAX_CLIENTS` | connections one user's worker serves at once (default `16`) | + +## Verifying a deployment + +- **The class actually deployed.** Unpack `prerna/om/ClientProcessWrapper.class` from the + deployed `semoss-*-shaded-dependencies.jar` and check it with `javap -p` for + `fetchBackChangedPaths`; a build log is not proof. +- **A write reaches the host.** Write a file from `Py()` in a loaded project and look for + `Fetch-back wrote` in the server log and the file in the project folder. +- **The permission gate holds.** Give a test user read-only access to a project, write from + `Py()`, and confirm the host file never appears and `Refusing write-back` is logged. + +## Known limits + +- More than ~2000 changed paths in one block sets `overflow` and forces a full capture. +- A *racing* `mmap` writer cannot be captured without cooperation (`msync`) or interception + this sandbox does not have. +- Conflicts are decided by modification time, so a change made within the same second as + the last sync, and leaving the size unchanged, is not detected. +- The refresh lists each transported root on every block; for very large roots that cost + shows up as block latency. +- Background code that writes between blocks is not synced until the next block ends. +- Python variables are keyed by insight id, and insight ids live in one instance's memory. After + a restart or a move to another instance the worker, its modules and its files are intact, but a + new insight does not see the old insight's variables. +- One change watcher serves every connection to a worker. Two instances connected at once can + consume each other's change sets. That is harmless while they share a disk; separate disks per + instance need per-connection cursors. +- Rooms and assets are gated as described above, but the platform has no room-sharing model + in this build; if one is added, the room branch must resolve the room to its project. diff --git a/py/gaas_change_watcher.py b/py/gaas_change_watcher.py new file mode 100644 index 0000000000..3c4c463ece --- /dev/null +++ b/py/gaas_change_watcher.py @@ -0,0 +1,444 @@ +"""Change tracking for the sandbox worker. + +The host needs to know *which* files a block wrote, not just *that* something +changed. A stat-walk of the tree costs O(files) per block (~6 ms for 240 files, +seconds for 100k), and spawning a process to do it costs ~80 ms before it does +any work. inotify is O(events): idle costs nothing, one write costs one event. + +Measured on the dev VM (2026-10-23), watching a project's assets tree: + + CREATE/MODIFY/CLOSE_WRITE watch_bash.txt (written by bash) + CREATE/MODIFY/CLOSE_WRITE watch_py.txt (written by a *different* python) + CREATE,DIR newdir + the first snapshot reports + ``baseline`` and is discarded, because the host already holds what it uploaded; +* the kernel queue can overflow -> ``overflow`` is reported and the host must + fall back to a full reconcile. +""" + +import ctypes +import ctypes.util +import errno +import os +import select +import stat +import struct +import threading +import time + +# Directories that are never watched, scanned, or pulled back. `.git` was 112 directories +# and 137 of 256 files in a real project - over half the watch set and over half of every +# scan. A commit made in the sandbox still reaches the host: the host's full capture +# carries git's data files. `node_modules` is worse: one install adds thousands of +# directories, exhausts fs.inotify.max_user_watches and floods the event queue. +_DEFAULT_IGNORE = ("node_modules", "__pycache__", ".venv", ".git") +_env_ignore = os.environ.get("SEMOSS_SYNC_IGNORE", "").strip() +_IGNORED_DIRS = {name.strip() for name in _env_ignore.split(",") if name.strip()} if _env_ignore else set(_DEFAULT_IGNORE) + +# The tuple scan exists to catch changes inotify cannot see (persistent mmap writes). +# Running it on every block makes an idle session O(files) per block - measured 0.57 s +# at 100k files. Bounding it by wall-clock instead keeps staleness under this many +# seconds while an idle burst of blocks pays for at most one scan. +_SCAN_MIN_INTERVAL_S = 2.0 + +# A scan may occupy at most 1/N of wall-clock: the interval stretches to N x the +# measured duration. At 240 files a scan is ~1 ms (interval stays 2 s); at 100k it is +# ~0.57 s, so the interval becomes ~11 s and the backstop costs ~5% instead of 100%. +_SCAN_BUDGET_FACTOR = 20 + +IN_MODIFY = 0x00000002 +IN_CLOSE_WRITE = 0x00000008 +IN_MOVED_FROM = 0x00000040 +IN_MOVED_TO = 0x00000080 +IN_CREATE = 0x00000100 +IN_DELETE = 0x00000200 +IN_DELETE_SELF = 0x00000400 +IN_MOVE_SELF = 0x00000800 +IN_Q_OVERFLOW = 0x00004000 +IN_UNMOUNT = 0x00002000 +IN_IGNORED = 0x00008000 +IN_ISDIR = 0x40000000 + +_EVENT = struct.Struct("iIII") +_MASK = (IN_CREATE | IN_MODIFY | IN_CLOSE_WRITE | IN_MOVED_FROM | IN_MOVED_TO | IN_DELETE + | IN_DELETE_SELF | IN_MOVE_SELF) + + +class ChangeWatcher: + """Watch a set of roots and report the paths that changed between snapshots.""" + + def __init__(self, roots, max_changes=2000): + self.roots = [root for root in roots if root and os.path.isdir(root)] + self.max_changes = max_changes + self._fd = None + self._thread = None + self._stop = threading.Event() + self._lock = threading.Lock() + # Serialises reads of the inotify fd between the loop thread and snapshot(), so + # events are consumed in the order the kernel queued them. + self._read_lock = threading.Lock() + self._changes = {} + self._overflow = False + self._baseline_pending = True + self._armed = 0 + self._events_seen = 0 + # Set when the watch tree can no longer be trusted (directory rename, watch + # removed, unmount): the host must fall back to a full capture for that block. + self._coverage_lost = False + # inotify events carry a watch descriptor, not a path; this maps it back. + self._watched = {} + self._root_ino = {} + # path -> (mtime_ns, size, ino, ctime_ns) as of the last checkpoint + self._stat_index = {} + self._last_scan = 0.0 + self._scans_skipped = 0 + # Grows with the measured cost of a scan so the backstop never takes more than + # a few percent of wall-clock, at any tree size. + self._scan_interval = _SCAN_MIN_INTERVAL_S + + # -- lifecycle --------------------------------------------------------- + + def start(self): + if not self.roots: + return False + library = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6", use_errno=True) + library.inotify_init1.argtypes = [ctypes.c_int] + library.inotify_add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32] + fd = library.inotify_init1(os.O_NONBLOCK) + if fd < 0: + return False + self._fd = fd + self._libc = library + for root in self.roots: + self._arm_tree(root) + self._thread = threading.Thread(target=self._loop, name="sandbox-change-watcher", daemon=True) + self._thread.start() + return True + + def stop(self): + self._stop.set() + if self._fd is not None: + try: + os.close(self._fd) + except OSError: + pass + + # -- watch arming ------------------------------------------------------ + + def _arm(self, directory): + try: + watch = self._libc.inotify_add_watch(self._fd, directory.encode("utf-8"), _MASK) + except Exception: + self._coverage_lost = True + return False + if watch >= 0: + self._watched[watch] = directory + self._armed += 1 + return True + # A directory removed between the walk and the arm is not a gap. Anything else + # (ENOSPC at max_user_watches, EACCES on a mode-000 directory) leaves part of the + # tree unwatched, and silence there would read as "nothing changed". + if ctypes.get_errno() != errno.ENOENT: + self._coverage_lost = True + return False + + def add_roots(self, roots): + """Arm roots discovered after start (a session materialises its tree late). + + Whatever was uploaded into a new root arrived before its watch, so the next + snapshot is a baseline and the host takes a full capture. + """ + added = 0 + for root in roots: + if root in self.roots or not os.path.isdir(root): + continue + self.roots.append(root) + self._arm_tree(root) + added += 1 + if added: + with self._lock: + self._baseline_pending = True + return added + + def _arm_tree(self, root): + try: + self._root_ino[root] = os.stat(root).st_ino + except OSError: + pass + for current, dirs, _files in os.walk(root, onerror=self._walk_failed): + # Prune in place so os.walk never descends into an ignored tree at all. + dirs[:] = [name for name in dirs if name not in _IGNORED_DIRS] + self._arm(current) + + def _walk_failed(self, _error): + # An unreadable directory cannot be watched or listed; treat it as a gap. + self._coverage_lost = True + + # -- event loop -------------------------------------------------------- + + def _loop(self): + # poll, not select: select() raises on any fd >= 1024, which a worker with many + # open files reaches, and the watcher would stop while still claiming coverage. + poller = select.poll() + poller.register(self._fd, select.POLLIN) + while not self._stop.is_set(): + try: + ready = poller.poll(200) + except InterruptedError: + continue + except (OSError, ValueError): + break + if ready and not self._drain(): + break + + def _drain(self): + """Consume every event queued on the fd. Returns False if the fd is unusable.""" + with self._read_lock: + while True: + try: + data = os.read(self._fd, 65536) + except OSError as exc: + if exc.errno in (errno.EAGAIN, errno.EWOULDBLOCK): + return True + if exc.errno == errno.EINTR: + continue + return False + if not data: + return True + self._consume(data) + + def alive(self): + return self._thread is not None and self._thread.is_alive() + + def _consume(self, data): + offset = 0 + additions = [] + with self._lock: + while offset + _EVENT.size <= len(data): + _watch, mask, _cookie, length = _EVENT.unpack_from(data, offset) + offset += _EVENT.size + raw = data[offset:offset + length] + offset += length + self._events_seen += 1 + if mask & IN_Q_OVERFLOW: + self._overflow = True + continue + if mask & (IN_IGNORED | IN_UNMOUNT | IN_MOVE_SELF): + # The watch itself is gone, or it moved with its inode: every path + # under it is stale. No event on this descriptor is trustworthy now. + self._coverage_lost = True + continue + name = raw.split(b"\0", 1)[0].decode("utf-8", "replace") + if not name: + continue + # The event does not carry the directory, only the watched descriptor's + # name. Watch descriptors map back through the path we armed, so we keep + # a lookup from descriptor to directory. + directory = self._watched.get(_watch) if hasattr(self, "_watched") else None + if directory is None: + continue + path = os.path.join(directory, name) + if mask & IN_ISDIR: + if mask & IN_MOVED_FROM: + # A watched directory left the tree; its descriptor still points + # at the inode and would report the old path forever. + self._coverage_lost = True + if mask & (IN_CREATE | IN_MOVED_TO): + # A new directory: arm it *and* scan what landed inside before + # the watch existed. Without the scan, `mkdir x; write x/f` + # inside one block reports the directory and misses the file. + additions.append(path) + continue + if mask & (IN_DELETE | IN_MOVED_FROM): + self._record(path, "delete") + else: + self._record(path, "modify") + for path in additions: + self._adopt_directory(path) + + def _adopt_directory(self, path): + # A directory created during a block is armed and scanned, unless it is one we + # ignore - an `npm install` must not pull thousands of watches into the tree. + if os.path.basename(path) in _IGNORED_DIRS: + return + self._arm_tree(path) + with self._lock: + for current, dirs, files in os.walk(path, onerror=self._walk_failed): + dirs[:] = [name for name in dirs if name not in _IGNORED_DIRS] + for name in files: + self._record(os.path.join(current, name), "modify") + + def _record(self, path, op): + if len(self._changes) >= self.max_changes: + self._overflow = True + return + previous = self._changes.get(path) + if previous == "delete" and op == "modify": + # Re-created within the same block: it exists, so it is a modify. + self._changes[path] = "modify" + else: + self._changes[path] = op + + # -- snapshot ---------------------------------------------------------- + + def _scan_tuples(self): + """Report files whose metadata moved since the last checkpoint. + + inotify cannot see a persistent mmap write, and a missed event is + indistinguishable from silence. One stat tuple per file costs ~6 ms for a few + hundred files (~0.1-0.3 s at 100k) and turns those writes into reported + changes. It is the correctness floor under the event stream, not a + replacement for it: the host still decides by content. + """ + index = {} + reported = 0 + failures = [] + for root in list(self.roots): + for base, dirs, files in os.walk(root, onerror=failures.append): + dirs[:] = [name for name in dirs if name not in _IGNORED_DIRS] + for name in files: + path = os.path.join(base, name) + try: + info = os.lstat(path) + except OSError: + continue + if not stat.S_ISREG(info.st_mode): + continue + key = (info.st_mtime_ns, info.st_size, info.st_ino, info.st_ctime_ns) + index[path] = key + # The first scan only seeds the index: the tree predates the watch. + if self._stat_index and self._stat_index.get(path) != key: + self._record(path, "modify") + reported += 1 + if failures: + # Part of the tree could not be listed, so a missing path proves nothing: a + # mode-000 directory must not read as "all of its files were deleted". Keep + # the old entries and make the host take a capture that knows it is partial. + self._coverage_lost = True + for path, key in self._stat_index.items(): + index.setdefault(path, key) + else: + for path in self._stat_index: + if path not in index: + self._record(path, "delete") + reported += 1 + self._stat_index = index + return reported + + def snapshot(self): + """Return and clear the accumulated changes. + + The first call after start returns ``baseline: True`` with an empty set: + everything already in the tree arrived before the watch existed, and the + host materialised it, so reporting it would be noise. + """ + rearm = False + # Pick up events the kernel has queued but the loop thread has not read yet, so + # a write that finished just before this call belongs to this block. + if self._fd is not None and not self._drain(): + with self._lock: + self._coverage_lost = True + with self._lock: + if not self.alive(): + # The loop thread is gone: nothing is being watched any more. + self._coverage_lost = True + now = time.monotonic() + # Always scan on the first call (seed), after a lost baseline/coverage, and + # otherwise at most once per interval. Skipping never drops a change: the + # index is untouched, so the next scan reports everything accumulated. + due = (self._last_scan == 0.0 or self._baseline_pending or self._coverage_lost + or (now - self._last_scan) >= self._scan_interval) + if due: + started = time.monotonic() + scanned = self._scan_tuples() + elapsed = time.monotonic() - started + self._scan_interval = max(_SCAN_MIN_INTERVAL_S, elapsed * _SCAN_BUDGET_FACTOR) + self._last_scan = now + skipped = 0 + self._scans_skipped = 0 + else: + scanned = 0 + self._scans_skipped += 1 + skipped = self._scans_skipped + roots = [] + for root in self.roots: + try: + roots.append({ + "path": root, + "ino_now": os.stat(root).st_ino, + "ino_armed": self._root_ino.get(root), + }) + except OSError as exc: + roots.append({"path": root, "error": str(exc)}) + stale = self._coverage_lost + payload = { + "changes": [{"path": path, "op": op} for path, op in self._changes.items()], + "overflow": self._overflow, + # A lost watch cannot report its own gaps, so it is equivalent to a + # baseline: the host must take a full capture for this block. + "baseline": self._baseline_pending or stale, + "coverage": not stale, + "watched": self._armed, + "events": self._events_seen, + "stat_scan": scanned, + "scan_skipped": skipped, + "roots": roots, + } + self._changes = {} + self._overflow = False + self._baseline_pending = False + if stale: + self._coverage_lost = False + # Paths may have moved underneath us; drop the tuple index so the next + # checkpoint reseeds instead of reporting a storm of false deletes. + self._stat_index = {} + rearm = True + if rearm: + self._rearm_all() + return payload + + def _rearm_all(self): + """Rebuild the watch tree after coverage was lost. + + Descriptors left over from the old tree stay in the kernel but are inert once + ``_watched`` no longer maps them, so re-arming from the roots restores correct + paths without having to tear the old watches down one by one. + """ + with self._lock: + self._watched = {} + self._armed = 0 + self._root_ino = {} + for root in list(self.roots): + self._arm_tree(root) + + +_shared = None +_shared_lock = threading.Lock() + + +def shared_watcher(roots): + """One watcher per process. + + The TCP handler is instantiated per connection, so a per-handler watcher + re-arms and re-baselines on every reconnect -- which silently pins the host to + the slow path, because every call then reports ``baseline``. Keeping the watcher + at module scope means the baseline is consumed once and subsequent calls report + real deltas. + """ + global _shared + with _shared_lock: + if _shared is not None: + _shared.add_roots(roots) + return _shared + if _shared is None: + candidate = ChangeWatcher(roots) + if candidate.start(): + _shared = candidate + return _shared diff --git a/py/gaas_server_proxy.py b/py/gaas_server_proxy.py index 190169050b..57fbdac9af 100644 --- a/py/gaas_server_proxy.py +++ b/py/gaas_server_proxy.py @@ -13,7 +13,7 @@ def __init__(self): """ from gaas_tcp_server_handler import TCPServerHandler - self.server = TCPServerHandler.da_server + self.server = TCPServerHandler.current() def get_next_epoc(self) -> str: """This method atomically returns a random value that is unique across all thread operations and instances.""" diff --git a/py/gaas_tcp_server_handler.py b/py/gaas_tcp_server_handler.py index 97ffb0c716..3f0a5992be 100644 --- a/py/gaas_tcp_server_handler.py +++ b/py/gaas_tcp_server_handler.py @@ -151,12 +151,16 @@ def _asset_aware_import(name, _globals=None, _locals=None, fromlist=(), level=0) return mod +# Which client connection the current thread is serving; see TCPServerHandler.current(). +_current_handler = threading.local() + + def _is_engine_owned_process(): """ Whether this process runs an engine's own python rather than a user's python code. Set from the --engine_owned flag the engines launch with. """ - handler = TCPServerHandler.da_server + handler = TCPServerHandler.current() server = getattr(handler, "server", None) if handler is not None else None return bool(getattr(server, "engine_owned", False)) @@ -238,6 +242,11 @@ class TCPServerHandler(socketserver.BaseRequestHandler): da_server = None CANCELLED_MESSAGE = "The request was cancelled by the user" + @staticmethod + def current(): + """The handler whose request this thread is serving, else the latest client.""" + return getattr(_current_handler, "handler", None) or TCPServerHandler.da_server + def log_level_mapper(self, log_level_name: str) -> int: """ Maps a log level name to its corresponding logging constant. @@ -305,6 +314,10 @@ def setup(self): self.residue = None TCPServerHandler.da_server = self + live = getattr(self.server, "live_handlers", None) + if live is not None: + with self.server.monitor: + live.add(self) # a lock to serialise all socket writes. HuggingFace (and similar libraries) # download files in parallel worker threads that all write tqdm progress to @@ -367,8 +380,17 @@ def setup(self): socket_handler=self, ) - # set the thread local - TCPServerHandler.thread_local = threading.local() + # Track filesystem changes so the host can pull back only what a block wrote. + # Absent module or an unsupported kernel just means the host falls back to a + # full reconcile, so this must never be able to break startup. + # The watcher is created lazily on the first CHANGES call: at setup time the + # session's project tree has not been materialised into the sandbox yet, so + # arming here yields zero roots and pins the host to the slow path forever. + self.change_watcher = None + + # per connection: a class-level one is replaced by every new client, wiping the + # payload of requests still running on other connections to a persistent worker + self.thread_local = threading.local() # Sometimes the debugger is not effective or cannot handle certain troubleshooting scenarios. # This is where you can use the custom_log() method. It writes to the log txt file, ensuring the file exists, creates a new line, adds the message, and flushes the log. @@ -413,6 +435,59 @@ def prod_logger(self, message: str): if self.logger: self.logger.info(message) + def _watcher_roots(self): + """Project/room/user roots that exist *now* (the worker starts without --prefix).""" + bases = [] + if self.prefix: + bases.append(self.prefix) + base_folder = os.environ.get("SEMOSS_HOME") or "/opt/semosshome" + if base_folder not in bases: + bases.append(base_folder) + for base in bases: + roots = [ + os.path.join(base, name) + for name in ("project", "room", "user") + if os.path.isdir(os.path.join(base, name)) + ] + if roots: + return roots + return [] + + def handle_changes(self): + """Return the files that changed since the last snapshot. + + The host calls this after operations that can write, then pulls exactly those + paths. The watcher is created here rather than in ``setup()`` because the + session's project tree is not materialised yet when the first connection is + accepted -- arming that early yields zero roots and pins the host to the slow + path forever. Overflow, or a set past the cap, is reported so the host falls + back to a full reconcile instead of trusting a partial list. + """ + try: + from gaas_change_watcher import shared_watcher + + # Re-evaluated on every call: room/ and user/ appear only when the session + # first touches one, which is usually after the first CHANGES. + self.change_watcher = shared_watcher(self._watcher_roots()) + except Exception as e: + # an unsupported kernel returns None without raising, so this is a real fault + if self.logger: + self.logger.warning( + f"Change watcher failed, host falls back to a full reconcile: {e}", + exc_info=True, + ) + self.change_watcher = None + if self.change_watcher is None: + snapshot = { + "changes": [], + "overflow": True, + "baseline": False, + "watched": 0, + } + else: + snapshot = self.change_watcher.snapshot() + self.send_output(snapshot, operation="CHANGES", response=True) + def handle(self): """ Handles incoming requests from the client. @@ -530,6 +605,9 @@ def get_final_output( epoc (Optional[str]): The epoc associated with the request. Defaults to None. """ self.log_data(data) + # Code run for this request calls back to Java over this connection, not over + # whichever client connected last: a persistent worker serves several. + _current_handler.handler = self payload = "" # payload = data @@ -631,6 +709,9 @@ def get_final_output( elif payload["operation"] == "PYTHON": insight_id = payload.get("insightId") self.handle_python(command, insight_id) + # host asks which files changed since the last snapshot (write-back) + elif payload["operation"] == "CHANGES": + self.handle_changes() # this is when it is a response elif payload["response"]: self.handle_response() @@ -920,6 +1001,16 @@ def send_request(self, payload: dict): def stop_request(self): """Stops the request and closes the connection.""" if not self.stop: + live = getattr(self.server, "live_handlers", None) + if live is not None: + # Other connection threads add and remove concurrently; iterating the + # set unguarded can raise "set changed size during iteration". + with self.server.monitor: + live.discard(self) + # A persistent worker keeps serving other clients; callbacks with no + # request of their own must not keep pointing at a closed socket. + if TCPServerHandler.da_server is self: + TCPServerHandler.da_server = next(iter(live), None) self.server.remove_handler() self.server.stop_it() self.request.close() diff --git a/py/gaas_tcp_socket_server.py b/py/gaas_tcp_socket_server.py index 619573f3ae..759ce2866a 100644 --- a/py/gaas_tcp_socket_server.py +++ b/py/gaas_tcp_socket_server.py @@ -6,6 +6,7 @@ import threading import asyncio import os +import time # Must be imported before anything that can open a TLS connection. Importing it # leaves a single truststore owning ssl.SSLContext, without which the openai and @@ -45,6 +46,7 @@ def __init__( logger_level: str = "INFO", uds_path=None, engine_owned=False, + persist=False, ): self.logger = logging.getLogger("SocketServer") self.logger.debug("__init__") @@ -53,7 +55,14 @@ def __init__( self.uds_path = uds_path self.max_count = max_count self.cur_count = 0 - self.user_mode = self.max_count == 1 + # A persistent worker belongs to a user, not to a connection: it outlives any + # client, accepts the next one, and exits only after `timeout` minutes with no + # client at all. User mode is the opposite and closes with its single client. + self.persist = persist + self.user_mode = self.max_count == 1 and not persist + self.live_handlers = set() + self.idle_since = time.monotonic() + self.idle_limit = timeout * 60 if timeout > 0 else 0 self.insight_folder = insight_folder self.prefix = prefix # engine owned processes run platform code, not a user's python code @@ -91,7 +100,9 @@ def __init__( if timeout > 0: timeout = timeout * 60 print(f"Setting timeout to .. {timeout}") - self.timeout = timeout + # A persistent worker measures idleness from its last disconnect, so it + # wakes every minute to check rather than once per whole idle period. + self.timeout = min(timeout, 60) if persist else timeout # self.socket.settimeout(timeout*60) else: print("Setting timeout to None") @@ -108,6 +119,16 @@ def handle_timeout(self): # no clients.. kill this server, no point keeping it # give back the GPU self.timed_out = True + if self.persist: + with self.monitor: + idle = self.cur_count == 0 and time.monotonic() - self.idle_since >= self.idle_limit + if idle and self.idle_limit > 0: + self.logger.info(f"No client for {self.idle_limit / 60} minutes. Shutting down.") + self.stop = True + # Hard exit: a thread user code left running must not keep an abandoned + # sandbox alive, and the sandbox ends with this process. + os._exit(0) + return if self.cur_count == 0: self.logger.info( f"Server idle for {self.timeout / 60} minutes. No client connected. Shutting down." @@ -135,22 +156,30 @@ def serve_forever(self): print("Listening on port " + str(self.port)) self.handle_request() - - # also keep count updates synchronized with remove_handler - with self.monitor: - self.timed_out = False - self.cur_count += 1 except Exception as e: self.logger.error(f"Error: {e}", exc_info=True) self.stop_it() return + def process_request(self, request, client_address): + # Count the connection here, not after handle_request() returns: that also + # returns on a timeout with no client, which used to count a phantom one. + with self.monitor: + self.timed_out = False + self.cur_count += 1 + socketserver.ThreadingTCPServer.process_request(self, request, client_address) + def remove_handler(self): with self.monitor: self.cur_count = self.cur_count - 1 + self.idle_since = time.monotonic() self.monitor.notify() def stop_it(self): + if self.persist: + # One client leaving (disconnect, `stop`, logout) never ends a persistent + # worker; only idleness or deleting the sandbox does. + return print( f"Max connections = {self.max_count}, Current connections = {self.cur_count}" ) @@ -240,6 +269,14 @@ def parse_args(): "function or guardrail engine) rather than a user's python code. " "Adapters that only make sense for user code are skipped.", ) + parser.add_argument( + "--persist", + action="store_true", + help="Keep serving after a client disconnects and accept the next one, " + "up to --max_count at once. The process exits only after --timeout " + "minutes with no client connected. Used when one worker belongs to a " + "user across sessions and server instances (OpenShell).", + ) parser.add_argument( "--uds-path", type=str, @@ -306,4 +343,5 @@ def parse_args(): start=args.start, uds_path=args.uds_path, engine_owned=args.engine_owned, + persist=args.persist, ) diff --git a/py/smss_secrets.py b/py/smss_secrets.py new file mode 100644 index 0000000000..8a5b0b4cd3 --- /dev/null +++ b/py/smss_secrets.py @@ -0,0 +1,49 @@ +"""Secrets an app declares in its project .smss, as OpenShell placeholders. + +The project declares SANDBOX_SECRETS (name + hosts) and sets SANDBOX_SECRET_; SEMOSS +brokers each into the user's sandbox when the project is loaded. Code here only ever holds a +placeholder: OpenShell's supervisor swaps it for the value in requests to the secret's declared +hosts (headers, query string) and refuses it anywhere else. + + from smss_secrets import secret + key = secret("OPENWEATHER_API_KEY") + requests.get(url, params={"appid": key}) +""" + +import json +import re + +# written by prerna.util.openshell.OpenShellSecrets after every attach +_PLACEHOLDERS = "/opt/semosshome/insight/sandbox-secrets/placeholders.json" + + +def _project_key(project_id): + return re.sub(r"[^a-z0-9]", "", project_id.lower())[:8] + + +def secret(name, project=None): + """The placeholder for a declared secret. + + Args: + name: the name in the project's SANDBOX_SECRETS, e.g. "OPENWEATHER_API_KEY". + project: the project id, needed only when several loaded projects declare the name. + """ + try: + with open(_PLACEHOLDERS) as f: + loaded = json.load(f) + except FileNotFoundError: + raise KeyError( + f"no sandbox secrets are available: declare {name} in the project's " + "SANDBOX_SECRETS, set its value, and load the project" + ) from None + wanted = _project_key(project) if project else None + found = { + key: values[name] + for key, values in loaded.items() + if name in values and (wanted is None or key == wanted) + } + if not found: + raise KeyError(f"{name} is not a secret of any loaded project") + if len(found) > 1: + raise KeyError(f"{name} is declared by several loaded projects; pass project=") + return next(iter(found.values())) diff --git a/src/prerna/auth/User.java b/src/prerna/auth/User.java index 7dae918cda..40cdb97888 100644 --- a/src/prerna/auth/User.java +++ b/src/prerna/auth/User.java @@ -68,6 +68,7 @@ import prerna.util.Constants; import prerna.util.Settings; import prerna.util.SymlinkHelper; +import prerna.util.openshell.OpenShellUserSandboxes; import prerna.util.Utility; public class User implements Serializable { @@ -508,6 +509,9 @@ public SocketClient getPythonSocketClient(boolean create, int port, String venvE } return this.pythonCPW.getSocketClient(); } + if (OpenShellUserSandboxes.appliesTo(this)) { + return getOpenShellPythonSocketClient(venvEngineId); + } if (this.pythonCPW == null || this.pythonCPW.getSocketClient() == null) { startPythonSocketServerAndClient(-1, venvEngineId); this.pythonCPW.getSocketClient().setUser(this); @@ -528,11 +532,54 @@ public SocketClient getPythonSocketClient(boolean create, int port, String venvE } /** - * + * OPENSHELL: the user's python is one persistent worker in their own sandbox, shared + * by every session of theirs. On this instance all of those sessions use one wrapper + * (one connection, one record of what was transported); the wrapper attaches to the + * sandbox if it is running anywhere and creates it otherwise. + */ + private SocketClient getOpenShellPythonSocketClient(String venvEngineId) { + while (true) { + OpenShellUserSandboxes.Local local = OpenShellUserSandboxes.local(this); + synchronized (local) { + if (local.isReleased()) { + // the user's last other session here ended between the lookup and the lock + continue; + } + this.pythonCPW = local.getWrapper(); + // write-back permission checks run as the session that asked most recently + this.pythonCPW.setUser(this); + SocketClient client = this.pythonCPW.getSocketClient(); + if (client == null) { + startPythonSocketServerAndClient(-1, venvEngineId); + } else if (!client.isConnected()) { + this.pythonCPW.shutdown(false); + try { + this.pythonCPW.reconnect(); + } catch (Exception e) { + classLogger.error("Failed to reconnect to user python process", e); + throw new IllegalArgumentException("Failed to connect to your isolated analytics engine"); + } + } + this.pythonCPW.getSocketClient().setUser(this); + break; + } + } + this.insightSerializedMap.clear(); + return this.pythonCPW.getSocketClient(); + } + + /** + * * @return */ public SymlinkHelper getUserSymlinkHelper() { if (Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE))) { + if (OpenShellUserSandboxes.appliesTo(this)) { + // shared by the user's sessions on this instance; see getOpenShellPythonSocketClient + symlinkHelper = OpenShellUserSandboxes.local(this).getHelper(); + chrootPath = symlinkHelper.getUserChrootFolder(); + return symlinkHelper; + } if (symlinkHelper != null) { return symlinkHelper; } @@ -615,6 +662,7 @@ public void startPythonSocketServerAndClient(int port, String venvEngineId) { // we do not define the Server Directory here - because it will dynamically // generate in the chroot location try { + this.pythonCPW.setUser(this); this.pythonCPW.createProcessAndClient(nativePyServer, this.symlinkHelper, port, null, null, customClassPath, debug, "-1", loggerLevel, ThreadContext.getImmutableContext()); } catch (Exception e) { @@ -635,6 +683,7 @@ public void startPythonSocketServerAndClient(int port, String venvEngineId) { try { String venvPath = venvEngineId != null ? Utility.getVenvEngine(venvEngineId).pathToExecutable() : null; + this.pythonCPW.setUser(this); this.pythonCPW.createProcessAndClient(nativePyServer, null, port, venvPath, serverDirectoryPath.toString(), customClassPath, debug, "-1", loggerLevel, ThreadContext.getImmutableContext()); @@ -1037,7 +1086,9 @@ public static String getSingleLogginName(User semossUser) { } AccessToken token = semossUser.accessTokens.get(semossUser.getPrimaryLogin()); - return token.getId(); + // null after logging out of the primary provider: the token is dropped before the + // session is destroyed, and the session cleanup logs this name + return token == null ? "" : token.getId(); } public static List> getUserIdAndType(User user) { diff --git a/src/prerna/om/ClientProcessWrapper.java b/src/prerna/om/ClientProcessWrapper.java index bde8082c19..260f47d4c2 100644 --- a/src/prerna/om/ClientProcessWrapper.java +++ b/src/prerna/om/ClientProcessWrapper.java @@ -44,7 +44,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -62,15 +64,21 @@ import com.google.common.base.Strings; import com.google.gson.GsonBuilder; +import prerna.cluster.util.ClusterUtil; +import prerna.project.api.IProject; import prerna.tcp.client.NativePySocketClient; import prerna.tcp.client.SocketClient; import prerna.util.Constants; import prerna.util.DIHelper; +import prerna.util.OpenShellTransport; +import prerna.util.OpenShellTransporter; import prerna.util.PortAllocator; import prerna.util.SandboxInjector; import prerna.util.Settings; import prerna.util.SymlinkHelper; import prerna.util.Utility; +import prerna.util.openshell.OpenShellUserSandboxes; +import prerna.util.openshell.TransportedRoots; public class ClientProcessWrapper { @@ -127,6 +135,37 @@ public class ClientProcessWrapper { // the same JSON protocol, so it reuses NativePySocketClient. private boolean nodeServer = false; private SymlinkHelper chrootSymlinkHelper; + /** Whose permissions gate write-back; bound by {@code User} when the client is created. */ + private prerna.auth.User user; + // OPENSHELL mode: owns the sandbox + port-forward for this worker's lifetime + private OpenShellTransport openshell; + /** + * The sandbox belongs to the user, not to this wrapper: shutdown detaches and leaves + * it running, and {@code OpenShellUserSandboxes} decides when it is deleted. + */ + private boolean openshellUserScoped = false; + /** + * One write-back at a time: python blocks, shell commands and teardown all reconcile, + * and two reconciles interleaving over the same root would race their writes. + */ + private final ReentrantLock fetchBackLock = new ReentrantLock(); + /** + * Set when a change set was drained from the worker but could not be applied. The + * worker has already forgotten those paths, so only a full capture can recover them. + */ + private volatile boolean fullCaptureOwed = false; + /** + * Project roots written back since they were last published to cluster storage. Pushing + * is what lets another server pull the change, and from there its sandboxes refresh. + */ + private final Set unpublishedRoots = ConcurrentHashMap.newKeySet(); + private final Map lastPublished = new ConcurrentHashMap<>(); + private static final long PUBLISH_INTERVAL_MS = 10_000L; + private static final ExecutorService PUBLISHER = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "openshell-writeback-publisher"); + t.setDaemon(true); + return t; + }); private String classPath; private boolean debug; private String timeout; @@ -223,7 +262,52 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS boolean serverRunning = debug && port > 0; if (!serverRunning) { if (nativePyServer) { - if (this.chrootSymlinkHelper != null && this.chrootSymlinkHelper.isInjectMode()) { + // The jail is selected the same way the fakechroot and namespace + // jails have always been selected: by the chain below, on the + // SymlinkHelper the caller supplied. User-authored python gets one + // (User.startPythonSocketServerAndClient); engine-owned workers and + // Project python pass null and so are never jailed at all. + // + // OPENSHELL is simply the third branch of that chain. The sandbox is + // created ahead of the worker, so the worker becomes the sandbox's + // canonical process instead of a child of Tomcat, and the socket + // client connects to the forwarded port. No udsPath is set, which is + // what selects the TCP branch below. + if (this.chrootSymlinkHelper != null && this.chrootSymlinkHelper.isExternalSandbox()) { + try { + if (OpenShellUserSandboxes.appliesTo(this.user)) { + // The user's own sandbox, shared across their sessions and server + // instances: attach if it is running, create it otherwise. + OpenShellUserSandboxes.Attached attached = OpenShellUserSandboxes.acquire(this.user); + this.openshell = attached.transport; + this.openshellUserScoped = true; + this.port = attached.port; + if (attached.reattached) { + // Another connection may have drained changes this one never + // applied; start this connection's write-back from the whole tree. + this.fullCaptureOwed = true; + } + } else { + this.openshell = OpenShellTransport.forSession(this.chrootSymlinkHelper.getUserChrootFolder()); + this.openshellUserScoped = false; + this.port = this.openshell.start(); + } + // Same seam as the namespace injector below: the per-mode collaborator + // that receives every path symlinkFolder/symlinkProject would otherwise + // have symlinked. Assets registered before the worker existed are + // replayed by setTransport, which is why this happens here rather than + // before start(). + this.chrootSymlinkHelper.setTransport( + new OpenShellTransporter(this.openshell)); + } catch (Exception e) { + // Fail closed: never fall back to an in-Tomcat worker when the + // deployment asked for an external boundary. + throw new IllegalStateException( + "OpenShell executor sandbox unavailable and no fallback is permitted in " + + "SANDBOX_MODE=OPENSHELL", + e); + } + } else if (this.chrootSymlinkHelper != null && this.chrootSymlinkHelper.isInjectMode()) { String insightCache = Utility.getDIHelperProperty(Constants.INSIGHT_CACHE_DIR); Path serverDirectoryPath = Files.createTempDirectory(Paths.get(insightCache), "a"); this.serverDirectory = serverDirectoryPath.toString(); @@ -324,6 +408,9 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS this.socketClient = new SocketClient(threadLoggerCtx); } this.socketClient.setCpw(this); + classLogger.info( + "Connecting socket client: udsPath={} port={} openshell={} nativePyServer={}", + this.udsPath, this.port, this.openshell != null, this.nativePyServer); if (this.udsPath != null) { this.socketClient.connectUds(this.udsPath); } else { @@ -357,9 +444,381 @@ public void createProcessAndClient(boolean nativePyServer, SymlinkHelper chrootS * @param cleanUpFolder true to also recursively delete the server scratch * directory and any namespace-sandbox folders */ + + /** + * Reconcile only the files the sandbox worker reports as changed since the last + * snapshot. + * + *

+ * Cost is proportional to the change: a block that writes three files produces + * three paths and one small tar, instead of a tar per transported root. It falls + * back to {@link #fetchBackTransportedRoots()} whenever the worker's answer cannot + * be trusted — no watcher, the first call after materialise (everything in the tree + * arrived before the watch existed), a kernel queue overflow, or a change set past + * the worker's cap. + * + * @return number of files written back + */ + public int fetchBackChangedPaths() { + if (this.openshell == null || this.chrootSymlinkHelper == null) { + return 0; + } + fetchBackLock.lock(); + try { + return fetchBackChangedPathsLocked(); + } finally { + fetchBackLock.unlock(); + } + } + + private int fetchBackChangedPathsLocked() { + Map snapshot = null; + if (this.socketClient instanceof NativePySocketClient pyClient) { + try { + snapshot = pyClient.getFileChanges(); + } catch (Exception e) { + classLogger.warn("Change query failed; falling back to a full reconcile", e); + } + } + // The snapshot is taken even when a full capture is owed: it drains the worker's + // pending set, which the full capture is about to cover anyway. + if (snapshot == null || this.fullCaptureOwed || truthy(snapshot.get("overflow")) + || truthy(snapshot.get("baseline")) || Boolean.FALSE.equals(snapshot.get("coverage"))) { + return fetchBackTransportedRoots(); + } + Object raw = snapshot.get("changes"); + if (!(raw instanceof List changes) || changes.isEmpty()) { + return 0; + } + + // Only roots this session transported are eligible, longest first so a nested + // root wins over its parent. + List roots = new ArrayList<>(); + for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { + String normalized = Utility.normalizePath(path); + if (TransportedRoots.isUserSpace(normalized) && canWriteBack(normalized)) { + roots.add(normalized); + } + } + roots.sort((first, second) -> second.length() - first.length()); + + Map> writes = new HashMap<>(); + Map> deletes = new HashMap<>(); + for (Object entry : changes) { + if (!(entry instanceof Map change)) { + continue; + } + Object pathValue = change.get("path"); + if (pathValue == null) { + continue; + } + String path = Utility.normalizePath(pathValue.toString()); + String root = null; + for (String candidate : roots) { + if (path.startsWith(candidate + "/")) { + root = candidate; + break; + } + } + if (root == null) { + continue; + } + String relative = path.substring(root.length() + 1); + boolean delete = "delete".equals(String.valueOf(change.get("op"))); + (delete ? deletes : writes).computeIfAbsent(root, key -> new ArrayList<>()).add(relative); + } + + int total = 0; + for (Map.Entry> group : writes.entrySet()) { + try { + int written = this.openshell.fetchBackPaths(group.getKey(), group.getValue()); + if (written > 0) { + this.unpublishedRoots.add(group.getKey()); + total += written; + } + } catch (Exception e) { + // The worker has already dropped these paths from its pending set, so they + // would never be reported again: recover them with a full capture next block. + this.fullCaptureOwed = true; + classLogger.warn("Path fetch-back failed for {}; a full capture is owed", group.getKey(), e); + } + } + int removed = 0; + for (Map.Entry> group : deletes.entrySet()) { + for (String relative : group.getValue()) { + if (this.openshell.deleteFromHost(group.getKey(), relative)) { + this.unpublishedRoots.add(group.getKey()); + removed++; + } + } + } + if (total > 0 || removed > 0) { + classLogger.info("Fetch-back wrote {} and removed {} file(s) from {} changed path(s)", total, removed, + changes.size()); + } + publishWrittenRoots(false); + return total + removed; + } + + /** + * Bring host-side changes into the sandbox before a block runs: another user's write-back, + * a cluster pull, an upload in the UI. Without this a sandbox computes on the copy it was + * given at first use for as long as it lives. + */ + public void refreshTransportedRoots() { + if (this.openshell == null || this.chrootSymlinkHelper == null) { + return; + } + fetchBackLock.lock(); + try { + if (this.fullCaptureOwed) { + // Unapplied sandbox changes first; a refresh must not overwrite them. + fetchBackTransportedRootsLocked(); + } + for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { + if (!TransportedRoots.isUserSpace(Utility.normalizePath(path))) { + continue; + } + try { + this.openshell.refreshFromHost(path); + } catch (Exception e) { + classLogger.warn("Refresh of {} into the sandbox failed; the sandbox keeps its copy", path, e); + } + } + } finally { + fetchBackLock.unlock(); + } + } + + /** + * Push written-back project roots to cluster storage and announce them, so servers that + * have the project loaded pull the change. Batched: a root is pushed at most every ten + * seconds, and whatever is left is pushed when the connection is released. + * + * @param force push everything pending now (teardown) + */ + private void publishWrittenRoots(boolean force) { + if (!ClusterUtil.IS_CLUSTER || this.unpublishedRoots.isEmpty()) { + return; + } + long now = System.currentTimeMillis(); + for (String root : new ArrayList<>(this.unpublishedRoots)) { + String projectId = TransportedRoots.projectIdOf(root); + if (projectId == null) { + // rooms and user assets have their own sync at logout + this.unpublishedRoots.remove(root); + continue; + } + if (!force && now - this.lastPublished.getOrDefault(root, 0L) < PUBLISH_INTERVAL_MS) { + continue; + } + this.unpublishedRoots.remove(root); + this.lastPublished.put(root, now); + Runnable push = () -> { + try { + IProject project = Utility.getProject(projectId); + if (project != null) { + ClusterUtil.pushProjectFolder(project, root); + classLogger.info("Published write-back of {} to cluster storage", root); + } + } catch (Exception e) { + this.unpublishedRoots.add(root); + classLogger.warn("Could not publish write-back of {}; retrying after the next block", root, e); + } + }; + if (force) { + push.run(); + } else { + PUBLISHER.submit(push); + } + } + } + + private static boolean truthy(Object value) { + if (value instanceof Boolean flag) { + return flag; + } + return value != null && Boolean.parseBoolean(value.toString()); + } + + /** + * Bind the user whose permissions decide whether a root may be written back. + * + * @param user owner of this process + */ + public void setUser(prerna.auth.User user) { + this.user = user; + } + + /** + * User assets are per-user by construction: the folder is {@code Asset__}, where the + * id is the user's own asset project id for one of their logins. Anything else under + * {@code user/} belongs to somebody else and must never be written back. + * + * @param normalized absolute user-asset root + * @return true when the asset belongs to this user + */ + private boolean isOwnUserAsset(String normalized) { + String assetId = TransportedRoots.userAssetIdOf(normalized); + if (this.user == null) { + classLogger.warn("Refusing write-back into {}: no user is bound to this process", normalized); + return false; + } + try { + for (prerna.auth.AuthProvider provider : this.user.getLogins()) { + if (assetId.equals(this.user.getAssetProjectId(provider))) { + return true; + } + } + } catch (Exception e) { + classLogger.warn("Could not resolve asset ownership for {}; refusing write-back", normalized, e); + return false; + } + classLogger.warn("Refusing write-back into {}: asset {} does not belong to {}", normalized, assetId, + prerna.auth.User.getSingleLogginName(this.user)); + return false; + } + + /** + * A room is writable only by the member who owns it. Rooms are rows in the model + * inference logs database ({@code ROOM.USER_ID}), not insights in this build, and + * sharing grants participation rather than edit rights. + * + * @param normalized absolute room root + * @return true when the session's user owns the room + */ + private boolean canWriteRoom(String normalized) { + String roomId = TransportedRoots.roomIdOf(normalized); + if (this.user == null) { + classLogger.warn("Refusing write-back into room {}: no user is bound to this process", roomId); + return false; + } + try { + // A login can be any of the providers the user authenticated with; the room was + // created under one of them. + for (prerna.auth.AuthProvider provider : this.user.getLogins()) { + prerna.auth.AccessToken token = this.user.getAccessToken(provider); + if (token != null && prerna.engine.impl.model.inferencetracking.ModelInferenceLogsUtils + .doCheckRoomExistsForUser(roomId, token.getId())) { + return true; + } + } + classLogger.warn("Refusing write-back into room {}: user {} does not own it", roomId, + prerna.auth.User.getSingleLogginName(this.user)); + return false; + } catch (Exception e) { + classLogger.warn("Could not resolve ownership of room {}; refusing write-back", roomId, e); + return false; + } + } + + /** + * A read-only member must not be able to modify a project through the sandbox: the + * worker runs outside the platform's permission checks, so without this gate a + * read-only user could write a file with {@code Py()} and have it published to the + * project folder. Every root is decided by its own space: projects on the platform + * permission lookup, rooms on {@code ROOM.USER_ID} in the model-inference-logs + * database, user assets on the caller's own asset project id. + * + * @param normalized absolute transported root + * @return true when write-back into this root is allowed + */ + private boolean canWriteBack(String normalized) { + if (TransportedRoots.isRoom(normalized)) { + return canWriteRoom(normalized); + } + if (TransportedRoots.isUserAsset(normalized)) { + return isOwnUserAsset(normalized); + } + String projectId = TransportedRoots.projectIdOf(normalized); + if (projectId == null) { + classLogger.warn("Refusing write-back into {}: unrecognised root space", normalized); + return false; + } + if (this.user == null) { + classLogger.warn("Refusing write-back into {}: no user is bound to this process", normalized); + return false; + } + try { + // The same check SymlinkHelper.symlinkProject uses to grant the project + // read-write: write-back must never answer that question differently. + boolean allowed = prerna.auth.utils.SecurityProjectUtils.userCanEditProject(this.user, projectId); + if (!allowed) { + classLogger.warn("Refusing write-back into {}: user {} cannot edit project {}", normalized, + prerna.auth.User.getSingleLogginName(this.user), projectId); + } + return allowed; + } catch (Exception e) { + // Fail closed: an unresolvable permission is not a licence to write. + classLogger.warn("Could not resolve permission for {}; refusing write-back", normalized, e); + return false; + } + } + + /** + * Reconcile the sandbox copies of this session's transported roots back onto the + * host, so a file written by {@code Py()} or a shell command is visible to the + * platform (asset browser, MCP generators, git) without waiting for logout. + * + *

+ * Called at the execution boundary. Throttled: a REPL that runs many small blocks + * must not tar its roots per call, and the reconcile itself skips files that are + * identical or where the sandbox copy is older. + * + * @return number of files written back + */ + public int fetchBackTransportedRoots() { + if (this.openshell == null || this.chrootSymlinkHelper == null) { + return 0; + } + fetchBackLock.lock(); + try { + return fetchBackTransportedRootsLocked(); + } finally { + fetchBackLock.unlock(); + } + } + + private int fetchBackTransportedRootsLocked() { + int total = 0; + boolean failed = false; + for (String path : this.chrootSymlinkHelper.getTransportedPaths()) { + String normalized = Utility.normalizePath(path); + // Only the user-facing roots. Engine and runtime trees are never written by + // user code, and pulling them back would waste a tar per block. + if (!TransportedRoots.isUserSpace(normalized) || !canWriteBack(normalized)) { + continue; + } + try { + int written = this.openshell.fetchBack(path); + if (written > 0) { + this.unpublishedRoots.add(normalized); + } + total += written; + } catch (Exception e) { + failed = true; + classLogger.warn("Fetch-back failed for {}; retrying with a full capture next block", path, e); + } + } + this.fullCaptureOwed = failed; + publishWrittenRoots(false); + return total; + } + public void shutdown(boolean cleanUpFolder) { lockDestroy.lock(); try { + // OPENSHELL: the last reconcile must run while the sandbox is still alive. A + // session-scoped worker is the sandbox's main process, so stopping the socket + // below ends the sandbox, and a capture attempted after that finds it Completed. + if (this.openshell != null) { + try { + fetchBackTransportedRoots(); + publishWrittenRoots(true); + } catch (Exception e) { + classLogger.warn("Fetch-back before sandbox teardown failed for port {}", this.port, e); + } + } if (this.socketClient != null && this.socketClient.isConnected()) { try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { @@ -411,6 +870,23 @@ & deleteFolderWithRetries(this.sandboxJailDir) classLogger.error("Failed to destroy isolated analytics process for port {}", this.port, e); } } + // OPENSHELL: the worker lives in the sandbox, not in this JVM's process + // tree. A user-scoped sandbox is only detached from (the `stop` above closed + // this connection, not the persistent worker); a session-scoped one is + // deleted, and the gateway delete is authoritative. + if (this.openshell != null) { + try { + if (this.openshellUserScoped) { + this.openshell.detach(); + } else { + this.openshell.close(); + } + } catch (Exception e) { + classLogger.error("Failed to close OpenShell executor sandbox for port {}", this.port, e); + } finally { + this.openshell = null; + } + } } finally { lockDestroy.unlock(); } @@ -512,6 +988,15 @@ private int calculatePort(int port) { return port; } + /** + * @return the OpenShell sandbox this wrapper's worker runs in, or null outside + * OPENSHELL mode or before it started + */ + public String getOpenShellSandboxName() { + OpenShellTransport current = this.openshell; + return current == null ? null : current.getSandboxName(); + } + /** * @return the socket client connected to the server process, or null if not yet * created diff --git a/src/prerna/tcp/PayloadStruct.java b/src/prerna/tcp/PayloadStruct.java index f850e4b95d..c3831e9acf 100644 --- a/src/prerna/tcp/PayloadStruct.java +++ b/src/prerna/tcp/PayloadStruct.java @@ -41,7 +41,7 @@ public class PayloadStruct implements Serializable { public enum OPERATION { R, PYTHON, NODE, CHROME, ECHO, ENGINE, REACTOR, INSIGHT, PROJECT, CMD, STDOUT, STDERR, STRUCTURED_STREAM, - CANCELLED, LOG + CANCELLED, LOG, CHANGES }; public OPERATION operation = OPERATION.R; // setting default to R diff --git a/src/prerna/tcp/client/NativePySocketClient.java b/src/prerna/tcp/client/NativePySocketClient.java index e0219c9ecb..026d087122 100644 --- a/src/prerna/tcp/client/NativePySocketClient.java +++ b/src/prerna/tcp/client/NativePySocketClient.java @@ -82,6 +82,12 @@ public class NativePySocketClient extends SocketClient implements Runnable, Clos // the stdout stream private static final String INLINE_IMAGE_PREFIX = " getFileChanges() { + PayloadStruct ps = new PayloadStruct(); + ps.operation = PayloadStruct.OPERATION.CHANGES; + ps.methodName = "GET_CHANGES"; + ps.payload = new Object[] {}; + ps.hasReturn = true; + ps.longRunning = false; + IN_CHANGE_QUERY.set(Boolean.TRUE); + try { + Object reply = executeCommand(ps); + if (!(reply instanceof PayloadStruct replyStruct) || replyStruct.payload == null + || replyStruct.payload.length == 0) { + return null; + } + Object first = replyStruct.payload[0]; + return first instanceof Map ? (Map) first : null; + } finally { + IN_CHANGE_QUERY.remove(); + } + } + @Override public Object executeCommand(PayloadStruct ps) { String threadName = Thread.currentThread().getName(); @@ -565,6 +605,17 @@ public Object executeCommand(PayloadStruct ps) { addEpocForJob(ps.jobId, ps.epoc); } ps.longRunning = true; + // The other half of the execution boundary: before a block runs, host-side changes + // (another user's write-back, a cluster pull, the UI) are copied into the sandbox, so + // the block sees what the platform holds now rather than its first-use copy. + if (!ps.response && this.cpw != null && !Boolean.TRUE.equals(IN_CHANGE_QUERY.get()) + && (ps.operation == PayloadStruct.OPERATION.PYTHON || ps.operation == PayloadStruct.OPERATION.CMD)) { + try { + this.cpw.refreshTransportedRoots(); + } catch (Exception refreshError) { + classLogger.warn("Refresh before execution failed", refreshError); + } + } try { // ReentrantLock + Condition (not synchronized/wait) so a virtual thread @@ -655,7 +706,19 @@ public Object executeCommand(PayloadStruct ps) { } } - return responseMap.remove(ps.epoc); + PayloadStruct reply = responseMap.remove(ps.epoc); + // Execution boundary: a file the block wrote inside the sandbox becomes + // visible to the platform here -- asset browser, git, MCP generation -- + // instead of waiting for logout, where it used to be lost entirely. + if (reply != null && this.cpw != null && !Boolean.TRUE.equals(IN_CHANGE_QUERY.get()) + && (ps.operation == PayloadStruct.OPERATION.PYTHON || ps.operation == PayloadStruct.OPERATION.CMD)) { + try { + this.cpw.fetchBackChangedPaths(); + } catch (Exception fetchError) { + classLogger.warn("Fetch-back after execution failed", fetchError); + } + } + return reply; } finally { ps.unlockResponse(); } diff --git a/src/prerna/util/CmdExecUtil.java b/src/prerna/util/CmdExecUtil.java index a1b8a77b1f..23ff9790a8 100644 --- a/src/prerna/util/CmdExecUtil.java +++ b/src/prerna/util/CmdExecUtil.java @@ -45,6 +45,7 @@ import org.apache.logging.log4j.Logger; import prerna.auth.User; +import prerna.util.openshell.OpenShellShellRunner; public class CmdExecUtil { @@ -57,14 +58,21 @@ public class CmdExecUtil { private String workingDir = null; private String contextDir = null; private String commandAppender = "cmd"; + /** Non-null only when this helper is driving an OpenShell sandbox. */ + private User user = null; + private boolean openshellSandbox = false; /** * Constructor */ public CmdExecUtil(User user, String insightId, String contextDir) { + this.user = user; if (user != null && Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE))) { SymlinkHelper chrootHelper = user.getUserSymlinkHelper(); this.chrootFolderPath = chrootHelper.getUserChrootFolder(); + // The sandbox is the boundary in this mode, so commands must run inside it + // rather than in a chroot on Tomcat's filesystem. + this.openshellSandbox = chrootHelper.isExternalSandbox(); } getCommandAppender(); this.insightId = insightId; @@ -138,6 +146,13 @@ private boolean directoryExists(String chrootRelativePath) { * Execute command with proper path handling */ public String executeCommand(String command) { + if (this.openshellSandbox) { + // Delegate the whole line, including cd/pwd/reset, to the sandbox. Doing it + // here rather than in runCommand keeps the host-path logic below untouched. + OpenShellShellRunner.Result result = OpenShellShellRunner.execute(this.user, this.workingDir, command); + this.workingDir = result.getWorkingDir(); + return result.getOutput(); + } String output = null; try { if (command.equalsIgnoreCase("reset")) { diff --git a/src/prerna/util/Constants.java b/src/prerna/util/Constants.java index 1faba0f61d..75b7c451f2 100644 --- a/src/prerna/util/Constants.java +++ b/src/prerna/util/Constants.java @@ -767,7 +767,11 @@ public class Constants { public static final String CHROOT_DIR = "CHROOT_DIR"; // OFF / FAKECHROOT / NAMESPACE / SECCOMP_ONLY (legacy NSJAIL is accepted) + // OPENSHELL is an additional mode, not a replacement for the others: the worker + // runs as the canonical main process of an OpenShell sandbox instead of a child + // process of Tomcat. See prerna.util.OpenShellTransport. public static final String SANDBOX_MODE = "SANDBOX_MODE"; + public static final String SANDBOX_MODE_OPENSHELL = "OPENSHELL"; public static final String SANDBOX_IO_DIR = "SANDBOX_IO_DIR"; // Colon/semicolon-separated executable directories exposed inside sandboxes public static final String SANDBOX_PATH = "SANDBOX_PATH"; diff --git a/src/prerna/util/OpenShellTransport.java b/src/prerna/util/OpenShellTransport.java new file mode 100644 index 0000000000..da4a85e80c --- /dev/null +++ b/src/prerna/util/OpenShellTransport.java @@ -0,0 +1,1996 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import prerna.util.openshell.TransportedRoots; + +/** + * Transport to a SEMOSS python worker running inside an OpenShell sandbox. + * + *

+ * OpenShell ({@code SANDBOX_MODE=OPENSHELL}) makes the worker the canonical main + * process of its own sandbox rather than a child process of Tomcat. That changes + * three things and nothing else: + * + *

    + *
  1. Where the worker runs. A sandbox, created with + * {@code openshell sandbox create} driven from an admitted workload template.
    2. + *
  2. How bytes reach it. A local port is forwarded into the sandbox's + * loopback over the gateway ({@code openshell forward service}). The existing + * {@code NativePySocketClient} is unchanged: it connects to + * {@code 127.0.0.1:} exactly as it does for the non-sandbox path.
    3. + *
  3. What "the connection" means. In the session-scoped form + * ({@link #forSession}) the worker is single-connection ({@code --max_count 1}) and + * the sandbox's main process, so a client disconnect ends the worker and the + * sandbox; the connection is held for the session's life. In the user-scoped form + * ({@link #forUser}) the worker is persistent: it serves several clients, survives + * their disconnects, and any server instance can {@link #attach()} to it.
    4. + *
+ * + *

+ * What this class deliberately does not do: it does not choose the image, + * the policy, the runtime class, or the egress profile. Those are deployment + * properties admitted by the workload template, so that neither the user nor + * model-generated code can widen its own boundary. + */ +public final class OpenShellTransport { + + private static final Logger classLogger = LogManager.getLogger(OpenShellTransport.class); + + /** Property naming the OpenShell CLI. */ + public static final String CLI_PROP = "OPENSHELL_CLI"; + /** + * Explicit {@code HOME} for the CLI child process. + * + *

+ * This is required, and it is the subtlest thing in this class. The + * {@code openshell} CLI resolves both its gateway registration and its mTLS client + * bundle from $HOME/.local/state/openshell/. A caller with a different + * (or unset) {@code HOME} therefore talks to a different, or non-existent, gateway -- + * and the failure is quiet: the sandbox is still created, but it is not the one the + * caller can reach, so the worker appears to die and the caller sees an unrelated + * null response several layers away. + * + *

+ * Measured 2026-09-28: the same argv stays {@code Ready} when launched from a shell + * with a full environment, and dies when launched from a process with Tomcat's much + * thinner one (13 variables versus 124). With {@code HOME} unset the CLI reports + * "Or register one with: openshell gateway add". + * + *

+ * Pinned explicitly rather than inherited, so behaviour does not depend on how the + * JVM happened to be started. + */ + public static final String CLI_HOME_PROP = "OPENSHELL_CLI_HOME"; + /** Property naming the sandbox workload template to create from. */ + public static final String TEMPLATE_PROP = "OPENSHELL_TEMPLATE"; + /** + * Property naming the sandbox image (used when no template is set). Defaults to the image + * CI publishes from {@code docker/Dockerfile.sandbox} for {@code dev}. + */ + public static final String IMAGE_PROP = "OPENSHELL_IMAGE"; + private static final String DEFAULT_IMAGE = "quay.io/semoss/semoss-sandbox:dev-3.14-cpu"; + /** + * Property naming a sandbox-policy YAML applied at creation. Optional: unset, the policy + * bundled with SEMOSS ({@value #POLICY_RESOURCE}) is used. Either way + * {@code ${SEMOSS_HOME_ROOT}} and {@code ${PYTHON}} are filled in first; see + * {@link #renderPolicy}. + */ + public static final String POLICY_PROP = "OPENSHELL_POLICY"; + /** + * Property naming the sandbox interpreter's real path, which the policy's egress rules + * name. The default matches {@code docker/Dockerfile.sandbox}, which moves CPython to a + * fixed directory so the path does not change with the patch version or the CPU. + */ + public static final String PYTHON_BINARY_PROP = "OPENSHELL_PYTHON_BINARY"; + private static final String DEFAULT_PYTHON_BINARY = "/usr/lib/python/cpython/bin/python3.14"; + /** + * Property listing the hosts sandboxed code may reach, as {@code host[:port]} separated by + * commas or spaces (port 443 when omitted). Unset means pypi, so {@code pip install} works; + * {@code none} means no egress at all. Granted to the sandbox interpreter only, and only + * applied through the {@code ${NETWORK_POLICIES}} placeholder of the policy template. + */ + public static final String EGRESS_PROP = "OPENSHELL_EGRESS"; + private static final String DEFAULT_EGRESS = "pypi.org:443, files.pythonhosted.org:443"; + /** A hostname, nothing a YAML scalar could be broken out of with. */ + private static final Pattern EGRESS_HOST = Pattern + .compile("[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?(\\.[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?)*"); + /** + * Property capping each sandbox's CPU, in Kubernetes quantity form ({@code 2}, {@code 1.5}, + * {@code 500m}); {@code none} for no cap. A cap, not a reservation: idle sandboxes cost nothing. + */ + public static final String CPU_PROP = "OPENSHELL_CPU"; + private static final String DEFAULT_CPU = "2"; + /** + * Property capping each sandbox's memory ({@code 4Gi}, {@code 512Mi}); {@code none} for no + * cap. Past it the kernel kills the largest process in the sandbox, which for a {@code Py()} + * block is the worker: the user gets a fresh python on the next block. + */ + public static final String MEMORY_PROP = "OPENSHELL_MEMORY"; + private static final String DEFAULT_MEMORY = "4Gi"; + private static final Pattern CPU_QUANTITY = Pattern + .compile("\\d+(\\.\\d+)?m?"); + private static final Pattern MEMORY_QUANTITY = Pattern + .compile("\\d+(\\.\\d+)?(Ki|Mi|Gi|Ti|K|M|G|T)?"); + /** The bundled policy template, on the classpath. */ + private static final String POLICY_RESOURCE = "/prerna/util/openshell/sandbox-policy.yaml"; + /** The SEMOSS home inside the sandbox image: the worker's py/ and insight folder. */ + private static final String IMAGE_HOME = "/opt/semosshome"; + /** Property naming the port the worker listens on *inside* the sandbox. */ + public static final String WORKER_PORT_PROP = "OPENSHELL_WORKER_PORT"; + /** Property naming the label value used to group executor sandboxes. */ + public static final String LABEL_PROP = "OPENSHELL_LABEL"; + /** How long to wait for the sandbox to reach Ready. */ + public static final String READY_TIMEOUT_PROP = "OPENSHELL_READY_TIMEOUT_SECONDS"; + + private static final String DEFAULT_CLI = "openshell"; + private static final String DEFAULT_WORKER_PORT = "7100"; + private static final String DEFAULT_LABEL = "app=semoss-executor"; + private static final int DEFAULT_READY_TIMEOUT_S = 180; + + /** Where the worker writes its own log inside the sandbox. */ + public static final String WORKER_LOG_PATH = "/opt/semosshome/insight/worker.log"; + /** The worker's own line announcing that it is serving; the readiness signal. */ + public static final String WORKER_READY_MARKER = "waiting for request"; + + /** + * When true, {@link #close()} stops the forward but leaves the sandbox in place. + * + *

+ * Exists because the default teardown deletes the sandbox, which destroys the + * evidence needed to diagnose a startup failure: the worker's own output and the + * sandbox's final phase both vanish. Set with {@code -DOPENSHELL_KEEP_SANDBOX=true} + * or the {@link #KEEP_SANDBOX_PROP} property while investigating. + */ + public static final String KEEP_SANDBOX_PROP = "OPENSHELL_KEEP_SANDBOX"; + + /** Minutes a user's worker may run with no client connected before it exits itself. */ + public static final String IDLE_EXIT_PROP = "OPENSHELL_IDLE_EXIT_MINUTES"; + /** How many clients (sessions, server instances) one user's worker serves at once. */ + public static final String MAX_CLIENTS_PROP = "OPENSHELL_WORKER_MAX_CLIENTS"; + private static final int DEFAULT_IDLE_EXIT_MINUTES = 180; + private static final int DEFAULT_MAX_CLIENTS = 16; + + /** Label naming the user a sandbox belongs to (a digest, never the user id itself). */ + public static final String OWNER_LABEL = "semoss-owner"; + + private final String sandboxName; + private final String workerPort; + private final boolean keepSandbox; + /** + * Set for a user-scoped sandbox: the worker is persistent and the sandbox carries the + * owner label. Null for the session-scoped form, whose worker ends with its client. + */ + private final String owner; + private int forwardedPort = -1; + private Process forwardProcess; + private StringBuilder forwardLog; + + private OpenShellTransport(String sandboxName, String workerPort) { + this(sandboxName, workerPort, null); + } + + private OpenShellTransport(String sandboxName, String workerPort, String owner) { + this.sandboxName = sandboxName; + this.workerPort = workerPort; + this.owner = owner; + String keep = System.getProperty(KEEP_SANDBOX_PROP); + if (keep == null || keep.trim().isEmpty()) { + keep = Utility.getDIHelperProperty(KEEP_SANDBOX_PROP); + } + this.keepSandbox = "true".equalsIgnoreCase(keep == null ? "" : keep.trim()); + } + + /** + * Maximum sandbox name length the gateway accepts. + * + *

+ * Measured 2026-09-28: {@code openshell sandbox create} rejects longer names + * with Client specified an invalid argument: name exceeds maximum length + * (25 > 19). This is stricter than the DNS-1123 limit the name is + * otherwise modelled on, and it is not documented at the call site, so it is + * pinned here with its own regression check. + */ + public static final int MAX_SANDBOX_NAME_LENGTH = 19; + + /** Cap for one fetch-back archive; larger trees are pulled path by path or not at all. */ + private static final long MAX_FETCH_BYTES = 256L * 1024 * 1024; + /** Most paths one change set may name; the worker overflows to a full capture before this. */ + private static final int MAX_CHANGED_PATHS = 2000; + /** Paths per {@code rm} when a refresh removes host-deleted files from the sandbox. */ + private static final int REFRESH_DELETE_BATCH = 500; + + /** + * Relative paths, per transported root, that the sandbox is known to have held: + * everything in its last full capture plus every path written back since. Deletions + * are inferred only for these, so a host file the sandbox never received -- + * gitignored, added on the host mid-session, missed by a partial upload -- is never + * mistaken for one the sandbox removed. + */ + private final Map> seenBySandbox = new ConcurrentHashMap<>(); + /** + * Size and mtime of each host file as this transport last left it (uploaded or + * written back). A file that no longer matches was changed on the host since, and is + * quarantined before a sandbox copy replaces it. + */ + private final Map hostStamps = new ConcurrentHashMap<>(); + + private static final String NAME_PREFIX = "pyw-"; + + /** + * Deterministic sandbox name for a session key. + * + *

+ * Kept in Java so the name is a pure function of the session key and needs no + * lookup: the gateway's own object store is the registry. The digest is + * truncated so the whole name fits {@link #MAX_SANDBOX_NAME_LENGTH}: with a + * 4-character prefix that leaves 15 hex characters (60 bits), which is still + * far beyond any realistic collision concern for concurrently active sessions. + */ + public static String sandboxNameFor(String sessionKey) { + if (sessionKey == null || sessionKey.trim().isEmpty()) { + throw new IllegalArgumentException("sessionKey is required to address an executor sandbox"); + } + int digestChars = MAX_SANDBOX_NAME_LENGTH - NAME_PREFIX.length(); + String digest; + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] hash = md.digest(sessionKey.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + for (int i = 0; sb.length() < digestChars; i++) { + sb.append(String.format("%02x", hash[i])); + } + digest = sb.substring(0, digestChars); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable", e); + } + String name = NAME_PREFIX + digest; + if (name.length() > MAX_SANDBOX_NAME_LENGTH) { + throw new IllegalStateException("sandbox name exceeds the gateway limit: " + name); + } + return name; + } + + /** + * The exact argv {@link #start()} will run. Exposed so a hand-run command can be + * diffed against it byte for byte: every debugging round that failed here failed + * because the Java-side argv differed from the hand-run one in a way that was not + * visible by eye. + */ + public static String describeCreateCommand(String sessionKey) { + try { + return String.join(" ", createCommand(sandboxNameFor(sessionKey), null)); + } catch (Exception e) { + return "(cannot build: " + e.getMessage() + ")"; + } + } + + private static List createCommand(String sandboxName, String owner) throws IOException { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + String template = Utility.getDIHelperProperty(TEMPLATE_PROP); + String label = propOr(LABEL_PROP, DEFAULT_LABEL); + String workerPort = propOr(WORKER_PORT_PROP, DEFAULT_WORKER_PORT); + + List create = new ArrayList<>(Arrays.asList(cli, "sandbox", "create", "--name", sandboxName, + "--keep", "--detach", "--label", label)); + if (owner != null) { + create.add("--label"); + create.add(OWNER_LABEL + "=" + owner); + } + // A user's worker outlives any one client: it accepts the next session or server + // instance, and exits by itself only after a long stretch with nobody connected, + // so a sandbox every server lost track of still ends. + String workerMode = owner == null ? " --timeout 30" + : " --persist --max_count " + intPropOr(MAX_CLIENTS_PROP, DEFAULT_MAX_CLIENTS) + " --timeout " + + intPropOr(IDLE_EXIT_PROP, DEFAULT_IDLE_EXIT_MINUTES); + if (template != null && !template.trim().isEmpty()) { + create.add("--template"); + create.add(template.trim()); + } else { + // Never the stock OpenShell image: it has no python3, so the sandbox would start and + // then fail to run the worker. + create.add("--from"); + create.add(propOr(IMAGE_PROP, DEFAULT_IMAGE)); + } + create.add("--policy"); + create.add(renderPolicy().toString()); + String cpu = resourceLimit(CPU_PROP, DEFAULT_CPU, CPU_QUANTITY); + if (cpu != null) { + create.add("--cpu"); + create.add(cpu); + } + String memory = resourceLimit(MEMORY_PROP, DEFAULT_MEMORY, MEMORY_QUANTITY); + if (memory != null) { + create.add("--memory"); + create.add(memory); + } + // Give sandboxed code the same interpreter the worker uses. The runtime does not + // apply the image's ENV PATH, so without this a `Py()` block that shells out to + // `python3` gets the bare system interpreter while the worker runs from the venv. + create.add("--env"); + create.add("PATH=/usr/lib/python/semossvenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"); + create.add("--"); + // The worker's stdout/stderr MUST go to a file, not to the supervisor's + // stdout pipe. With the pipe it came up erratically -- sometimes serving, + // sometimes never binding at all while the sandbox still read `Ready` -- and + // slow (~5s). Redirecting to a file made it reliable, 3 runs out of 3, and + // ready in ~0.66s. Suspected cause is output back-pressure on that pipe. + // The file is also the readiness signal, read via `sandbox exec` so that no + // connection is ever made to the single-connection worker. + // + // The interpreter is resolved explicitly: OpenShell builds the environment for + // the processes it spawns, so an image's ENV PATH is not applied and a bare + // `python3` means whatever the runtime's PATH points at -- which is how a venv + // image ended up running the base interpreter with none of the dependencies. + create.add("sh"); + create.add("-c"); + create.add("P=/usr/lib/python/semossvenv/bin/python3; [ -x \"$P\" ] || P=$(command -v python3);" + + " exec \"$P\" /opt/semosshome/py/gaas_tcp_socket_server.py" // + + " --port " + workerPort // + + " --start True" // + + " --py_folder /opt/semosshome/py" // + + " --insight_folder /opt/semosshome/insight" // + + workerMode // + + " > " + WORKER_LOG_PATH + " 2>&1"); + return create; + } + + public static OpenShellTransport forSession(String sessionKey) { + return new OpenShellTransport(sandboxNameFor(sessionKey), propOr(WORKER_PORT_PROP, DEFAULT_WORKER_PORT)); + } + + /** + * A user-scoped sandbox: a persistent worker, labelled with its owner digest, that any + * session or server instance of that user can attach to. + */ + public static OpenShellTransport forUser(String sandboxName, String owner) { + if (owner == null || owner.trim().isEmpty()) { + throw new IllegalArgumentException("owner is required for a user-scoped sandbox"); + } + return new OpenShellTransport(sandboxName, propOr(WORKER_PORT_PROP, DEFAULT_WORKER_PORT), owner); + } + + public String getSandboxName() { + return sandboxName; + } + + /** + * Connect to a sandbox that already runs a persistent worker: forward its port and + * nothing else. Only valid for a user-scoped sandbox; a session-scoped worker allows + * one client, and a second forward would queue behind it forever. + * + * @return the local port to connect to + */ + public int attach() throws IOException { + if (owner == null) { + throw new IllegalStateException("only a user-scoped sandbox can be attached to"); + } + int readyTimeout = intPropOr(READY_TIMEOUT_PROP, DEFAULT_READY_TIMEOUT_S); + openForward(readyTimeout); + if (!awaitWorkerReady(Math.min(30, readyTimeout))) { + detach(); + throw new IOException("worker inside sandbox " + sandboxName + " is not serving"); + } + classLogger.info("Attached to executor sandbox {}; local port {}", sandboxName, this.forwardedPort); + return this.forwardedPort; + } + + /** + * Stop this instance's forward and leave the sandbox running. The user's worker + * survives for their other sessions and server instances; deleting it is the + * registry's decision, made when the user has no live session anywhere. + */ + public void detach() { + if (forwardProcess != null) { + forwardProcess.destroy(); + try { + if (!forwardProcess.waitFor(5, TimeUnit.SECONDS)) { + forwardProcess.destroyForcibly(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + forwardProcess = null; + } + } + + /** + * The sandbox's phase ({@code Ready}, {@code Completed}, ...), or null when the gateway + * says it does not exist. + * + * @throws IOException when the gateway could not be asked or its answer not read. That + * is not "gone": a caller that deleted on it would destroy a live + * user's python over a network hiccup. + */ + public static String phaseOf(String sandboxName) throws IOException { + String out; + try { + out = run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "get", sandboxName, "-o", "json"), + 30); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted asking for sandbox " + sandboxName, e); + } catch (IOException e) { + if (String.valueOf(e.getMessage()).toLowerCase().contains("not found")) { + return null; + } + throw e; + } + try { + JsonObject json = JsonParser.parseString(jsonBody(out)).getAsJsonObject(); + if (json.has("sandbox") && json.get("sandbox").isJsonObject()) { + json = json.getAsJsonObject("sandbox"); + } + if (!json.has("phase")) { + throw new IOException("no phase in the gateway's answer for sandbox " + sandboxName); + } + return json.get("phase").getAsString(); + } catch (RuntimeException e) { + throw new IOException("unreadable answer for sandbox " + sandboxName + ": " + out, e); + } + } + + /** Delete a sandbox by name; best effort. */ + public static void deleteSandbox(String sandboxName) { + try { + run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "delete", sandboxName), 60); + classLogger.info("Deleted executor sandbox {}", sandboxName); + } catch (Exception e) { + classLogger.warn("Failed to delete executor sandbox {}", sandboxName, e); + } + } + + /** + * One executor sandbox as the gateway lists it. + * + * @param createdAt {@code yyyy-MM-dd HH:mm:ss}, UTC, or empty + * @param owner the {@link #OWNER_LABEL} digest, or null for a session-scoped sandbox + */ + public record SandboxInfo(String name, String phase, String createdAt, String owner) { + public boolean isReady() { + return "Ready".equalsIgnoreCase(phase); + } + } + + /** Every executor sandbox on the gateway, selected by {@link #LABEL_PROP}. */ + public static List listExecutorSandboxes() throws IOException, InterruptedException { + String out = run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "list", "--selector", + propOr(LABEL_PROP, DEFAULT_LABEL), "-o", "json"), 60); + List rows = new ArrayList<>(); + JsonObject json = JsonParser.parseString(jsonBody(out)).getAsJsonObject(); + if (!json.has("sandboxes")) { + return rows; + } + for (JsonElement element : json.getAsJsonArray("sandboxes")) { + JsonObject box = element.getAsJsonObject(); + JsonObject labels = box.has("labels") && box.get("labels").isJsonObject() + ? box.getAsJsonObject("labels") + : new JsonObject(); + rows.add(new SandboxInfo(box.get("name").getAsString(), + box.has("phase") ? box.get("phase").getAsString() : "", + box.has("created_at") ? box.get("created_at").getAsString() : "", + labels.has(OWNER_LABEL) ? labels.get(OWNER_LABEL).getAsString() : null)); + } + return rows; + } + + /** + * The policy file to create sandboxes with: {@link #POLICY_PROP} or the bundled template, + * with {@code ${SEMOSS_HOME_ROOT}} replaced by {@link #homeRoot()} and {@code ${PYTHON}} by + * {@link #PYTHON_BINARY_PROP}. + * + *

+ * Generated rather than shipped as a fixed file because the home differs per deployment: + * files cross into the sandbox at their host path, and the filesystem policy can only grant + * a path by name. On a server the home is the image's own /opt/semosshome, and the line is + * dropped. + * + * @return a file under the temp directory, named by its content, so concurrent creates and + * restarts reuse one rendering + */ + static Path renderPolicy() throws IOException { + String configured = Utility.getDIHelperProperty(POLICY_PROP); + String template; + if (configured != null && !configured.trim().isEmpty()) { + template = Files.readString(Paths.get(configured.trim()), StandardCharsets.UTF_8); + } else { + try (InputStream in = OpenShellTransport.class.getResourceAsStream(POLICY_RESOURCE)) { + if (in == null) { + throw new IOException("bundled sandbox policy " + POLICY_RESOURCE + " is missing from the build"); + } + template = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + } + String root = homeRoot(); + StringBuilder rendered = new StringBuilder(); + for (String line : template.split("\n", -1)) { + if (line.trim().startsWith("#")) { + // the template documents its placeholders; leave the comments as written + rendered.append(line).append('\n'); + continue; + } + if (line.trim().equals("- ${SEMOSS_HOME_ROOT}") && root == null) { + continue; + } + if (line.trim().equals("${NETWORK_POLICIES}")) { + rendered.append(networkPolicies(propOr(PYTHON_BINARY_PROP, DEFAULT_PYTHON_BINARY))); + continue; + } + rendered.append(line.replace("${SEMOSS_HOME_ROOT}", root == null ? IMAGE_HOME : root) + .replace("${PYTHON}", propOr(PYTHON_BINARY_PROP, DEFAULT_PYTHON_BINARY))).append('\n'); + } + String text = rendered.toString(); + String hash; + try { + hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))) + .substring(0, 16); + } catch (NoSuchAlgorithmException e) { + throw new IOException("SHA-256 unavailable", e); + } + Path file = Paths.get(System.getProperty("java.io.tmpdir"), "openshell-policy-" + hash + ".yaml"); + if (!Files.exists(file)) { + Path partial = Files.createTempFile(file.getParent(), "openshell-policy-", ".partial"); + Files.writeString(partial, text, StandardCharsets.UTF_8); + Files.move(partial, file, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } + return file; + } + + /** + * The {@code network_policies} section for {@link #EGRESS_PROP}: one rule granting every + * listed endpoint to the interpreter, or an empty section for {@code none}. + */ + static String networkPolicies(String python) throws IOException { + String configured = Utility.getDIHelperProperty(EGRESS_PROP); + String list = configured == null || configured.trim().isEmpty() ? DEFAULT_EGRESS : configured.trim(); + if (list.equalsIgnoreCase("none")) { + return "network_policies: {}\n"; + } + StringBuilder yaml = new StringBuilder("network_policies:\n semoss_egress:\n endpoints:\n"); + for (String entry : list.split("[,\\s]+")) { + if (entry.isEmpty()) { + continue; + } + int colon = entry.lastIndexOf(':'); + String host = colon < 0 ? entry : entry.substring(0, colon); + int port; + try { + port = colon < 0 ? 443 : Integer.parseInt(entry.substring(colon + 1)); + } catch (NumberFormatException e) { + port = -1; + } + if (!EGRESS_HOST.matcher(host).matches() || port < 1 || port > 65535) { + throw new IOException(EGRESS_PROP + " entry '" + entry + "' is not host[:port]"); + } + yaml.append(" - host: ").append(host).append("\n port: ").append(port).append('\n'); + } + yaml.append(" binaries:\n - path: ").append(python).append('\n'); + return yaml.toString(); + } + + /** The interpreter path network rules name ({@link #PYTHON_BINARY_PROP}). */ + public static String pythonBinary() { + return propOr(PYTHON_BINARY_PROP, DEFAULT_PYTHON_BINARY); + } + + /** Whether a value is a plain hostname, safe to place in a policy or profile. */ + public static boolean isValidHost(String host) { + return host != null && EGRESS_HOST.matcher(host).matches(); + } + + /** + * A sandbox resource cap from its property: the default when unset, null for {@code none}, + * and an error for anything that is not a quantity the CLI accepts. + */ + private static String resourceLimit(String prop, String fallback, Pattern format) + throws IOException { + String value = propOr(prop, fallback); + if (value.equalsIgnoreCase("none")) { + return null; + } + if (!format.matcher(value).matches()) { + throw new IOException(prop + " '" + value + "' is not a valid quantity"); + } + return value; + } + + /** The platform's SEMOSS home (BaseFolder), normalized and without a trailing slash. */ + private static String semossHome() { + String home = Utility.normalizePath(Utility.getBaseFolder()); + return home.length() > 1 && home.endsWith("/") ? home.substring(0, home.length() - 1) : home; + } + + /** Whether the home is the image's own, which the image and the policy already provide. */ + private static boolean homeIsImageHome() { + String home = semossHome(); + return home.equals(IMAGE_HOME) || home.startsWith(IMAGE_HOME + "/"); + } + + /** + * The folder the policy grants so the home can be made inside the sandbox: the top folder + * of the home's path, {@code /Users} for {@code /Users/me/Documents/semoss}. Landlock only + * grants a path that exists when the sandbox starts, and the home itself does not exist in + * a generic image; {@code docker/Dockerfile.sandbox} makes {@code /Users} and {@code /home} + * for exactly this. Null when the home is the image's own. + */ + static String homeRoot() { + if (homeIsImageHome()) { + return null; + } + String home = semossHome(); + int slash = home.indexOf('/', 1); + return slash < 0 ? home : home.substring(0, slash); + } + + /** + * Make the SEMOSS home inside the sandbox, at its host path, and fail with the fix in the + * message when that is not possible. Otherwise the first upload fails with only "ssh tar + * extract exited with status 1": uploads run as the sandbox user, who can only create + * directories under a root the image has and the policy grants. + */ + private void checkSandboxHome() throws IOException { + if (homeIsImageHome()) { + return; + } + String home = semossHome(); + try { + run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "exec", "-n", sandboxName, "--", "sh", + "-c", "mkdir -p \"$1\" && test -w \"$1\"", "sh", home), 60); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted checking the SEMOSS home in sandbox " + sandboxName, e); + } catch (IOException e) { + throw new IOException("Could not make " + home + ", the SEMOSS home (BaseFolder), inside the sandbox." + + " Files cross into the sandbox at their host path, so the image must have a writable " + + homeRoot() + ": docker/Dockerfile.sandbox provides /Users and /home. For a home under" + + " another root, add that root to the image owned by uid 1000 (see" + + " docs/deployment/executor_sandbox.md) and set " + IMAGE_PROP + " to it.", e); + } + } + + /** The CLI may print notices before its JSON; keep from the first brace. */ + private static String jsonBody(String out) { + int brace = out.indexOf('{'); + return brace < 0 ? "{}" : out.substring(brace); + } + + /** + * Create the sandbox if it does not exist, then forward its worker port. + * + *

+ * Create is idempotent from the caller's perspective: an "already exists" result + * is success, not an error. A sandbox that exists but has finished (phase + * {@code Completed}) is not reusable — the worker has exited and the + * next request must get a fresh sandbox, so callers should treat + * {@link #awaitReady()} failure as "delete and re-create". + * + * @return the local port to connect to + */ + public int start() throws IOException, InterruptedException { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + int readyTimeout = intPropOr(READY_TIMEOUT_PROP, DEFAULT_READY_TIMEOUT_S); + // Single source of truth: the argv used here is exactly the one + // describeCreateCommand() prints, so a hand-run diff is meaningful. + List create = createCommand(sandboxName, owner); + + classLogger.info("Starting executor sandbox {} (worker port {})", sandboxName, workerPort); + String createOut = run(create, readyTimeout); + classLogger.info("Sandbox create output for {}: {}", sandboxName, createOut); + if (createOut.toLowerCase().contains("already exists")) { + // A sandbox with this name exists, but this transport did not create it in + // this call. That is not a reusable state: the worker inside allows one + // connection, and whatever forward reached it first still holds that slot, + // so a second forward leaves the real client queued and every request + // blocks forever. Measured 2026-10-22: a Py() that never returned and a + // logout thread stuck in clearInsightGlobals until the sandbox was deleted + // by hand. Deleting and recreating is the only safe interpretation -- + // a sandbox this instance is not driving belongs to a dead session. + classLogger.warn("Sandbox {} already exists; deleting and recreating it", sandboxName); + run(Arrays.asList(cli, "sandbox", "delete", sandboxName), 60); + createOut = run(create, readyTimeout); + classLogger.info("Sandbox re-create output for {}: {}", sandboxName, createOut); + if (createOut.toLowerCase().contains("already exists")) { + throw new IOException("sandbox " + sandboxName + " still exists after delete: " + createOut); + } + } + + try { + checkSandboxHome(); + openForward(readyTimeout); + } catch (IOException e) { + close(); + throw e; + } + // Readiness must come from the worker's OWN log, and the worker must write + // that log to a FILE, not to the supervisor's stdout pipe. + // + // Two measured facts (2026-09-28): + // 1. With stdout on the supervisor's pipe the worker came up erratically -- + // sometimes serving, sometimes never opening the port at all while the + // sandbox still read `Ready`. Redirecting its stdout to a file made it + // reliable, 3 runs out of 3, and fast (~0.66s instead of ~5s). Output + // back-pressure on that pipe is the suspected cause; the file removes it. + // 2. The supervisor's `NET:OPEN 127.0.0.1:/tcp` line is a *connection* + // event, not a bind event -- it also marks NET:CLOSE, and it fired only + // because a probe connected. Waiting for it is a mistake; an earlier + // version of this method did exactly that and stalled for the full + // timeout. The worker's own "waiting for request" line is the real signal. + // + // Never probe the forwarded port to test readiness: a session-scoped worker is + // single-connection (--max_count 1), so a probe consumes its only slot and its + // disconnect ends the worker. + if (!awaitWorkerReady(readyTimeout)) { + String detail = readAvailable(forwardProcess); + close(); + throw new IOException("worker inside sandbox " + sandboxName + " did not report ready within " + + readyTimeout + "s (forward alive=" + (forwardProcess != null && forwardProcess.isAlive()) + + ")" + (detail.isEmpty() ? "" : ": " + detail)); + } + classLogger.info("Executor sandbox {} ready; worker listening, local port {}", sandboxName, + this.forwardedPort); + return this.forwardedPort; + } + + /** Start {@code openshell forward service} for the worker port and wait until it binds. */ + private void openForward(int readyTimeout) throws IOException { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + this.forwardedPort = freeLocalPort(); + List forward = Arrays.asList(cli, "forward", "service", "--target-port", workerPort, "--local", + String.valueOf(this.forwardedPort), sandboxName); + classLogger.info("Forwarding localhost:{} -> sandbox {}:{}", this.forwardedPort, sandboxName, workerPort); + ProcessBuilder pb = new ProcessBuilder(forward); + pb.environment().putAll(cliEnv()); + pb.redirectErrorStream(true); + this.forwardProcess = pb.start(); + this.forwardLog = new StringBuilder(); + // A child whose pipe fills up blocks forever. The forward emits periodic + // output, so it must be drained continuously or the port never binds. + Thread drain = new Thread(() -> { + try (BufferedReader r = new BufferedReader(new InputStreamReader( + forwardProcess.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + synchronized (forwardLog) { + forwardLog.append(line).append('\n'); + } + } + } catch (IOException e) { + // process exited; the readiness check will report it + } + }, "openshell-forward-" + sandboxName); + drain.setDaemon(true); + drain.start(); + + // The forward binds lazily: the CLI starts, then opens the local listener. + // Returning before that produces a connection refused at the socket client + // and surfaces as "Received a null PayloadStruct response" -- measured + // 2026-09-28. Poll until the port actually accepts, or fail here with a + // clear message rather than downstream with a confusing one. + if (!awaitForwardReady(readyTimeout)) { + String detail = readAvailable(forwardProcess); + boolean alive = forwardProcess != null && forwardProcess.isAlive(); + classLogger.error("Forward for sandbox {} did not become ready on 127.0.0.1:{} (alive={}): {}", + sandboxName, this.forwardedPort, alive, detail); + detach(); + throw new IOException("port forward for sandbox " + sandboxName + " did not become ready on 127.0.0.1:" + + this.forwardedPort + " within " + readyTimeout + "s (process alive=" + alive + ")" + + (detail.isEmpty() ? "" : ": " + detail)); + } + } + + /** + * Poll until the forward is up, without ever connecting to the forwarded port. + * + *

+ * Do not replace this with a TCP connect. An earlier version did exactly + * that and it killed every sandbox it was meant to check: the forward proxies + * straight into the worker, the worker is single-connection ({@code --max_count 1}), + * so the probe consumed the worker's only slot and its disconnect triggered + * {@code stop_request()} -- ending the worker and taking the sandbox to + * {@code Completed}. The symptom was indistinguishable from "the worker never + * started", which is why it took so long to find. See the wiki's + * "never open a second connection to the worker's port" rule; this method exists to + * honour it. + * + *

+ * Readiness is therefore taken from the CLI's own output: {@code forward service} + * prints a line containing {@code "Forwarding"} once the local listener is bound. + */ + private boolean awaitForwardReady(int timeoutSeconds) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + while (System.nanoTime() < deadline) { + if (forwardProcess == null || !forwardProcess.isAlive()) { + // Died: record why before returning, or the reason is lost with the process. + classLogger.error("Forward process exited early (exit={}): {}", + forwardProcess == null ? "null" : forwardProcess.exitValue(), readAvailable(forwardProcess)); + return false; + } + synchronized (forwardLog) { + if (forwardLog.indexOf("Forwarding") >= 0) { + return true; + } + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return false; + } + + /** + * Copy a host file or directory into this sandbox, preserving its absolute path. + * + *

+ * Used by {@link OpenShellTransporter} so that asset and project folders the worker refers to by + * absolute path exist inside the sandbox. Idempotency is the caller's job + * ({@code SymlinkHelper.activeTransports}); this method always transfers. + * + * @param absPath host path to copy + * @return true when the CLI reported success + */ + public boolean upload(String absPath) { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + // `sandbox upload ` copies src INTO dst, i.e. it appends basename(src). + // Passing dst == src therefore nests: uploading /a/b/app_root to /a/b/app_root + // produced /a/b/app_root/app_root/version/... (observed 2026-09-28, and the same + // shape as the earlier X.py -> /path/X.py/X.py trap). The destination must be the + // PARENT so the tree lands at the absolute path the worker expects. + String destination = Paths.get(absPath).getParent() == null ? "/" + : Paths.get(absPath).getParent().toString(); + classLogger.info("Sandbox upload {} -> {}:{}", absPath, sandboxName, destination); + try { + String out = run(Arrays.asList(cli, "sandbox", "upload", sandboxName, absPath, destination), 300); + // `sandbox upload` prints a tick with its result; treat a non-zero exit as failure, + // which run() already raises as IOException. + classLogger.info("Sandbox upload result for {}: {}", absPath, out.trim()); + String normalized = Utility.normalizePath(absPath); + if (TransportedRoots.isUserSpace(normalized)) { + recordUploadedStamps(Paths.get(normalized)); + } + return true; + } catch (Exception e) { + classLogger.error("Sandbox upload failed for " + absPath + ": " + e.getClass().getSimpleName() + ": " + + e.getMessage(), e); + return false; + } + } + + /** + * Poll until the worker reports that it is serving, without ever connecting to + * the forwarded port. + * + *

+ * The signal is the worker's own log line {@code "waiting for request"}. It is read + * back with {@code sandbox exec}, so no socket is opened to the worker. Measured + * ready at +0.66s, against ~5s when the worker's stdout went to the supervisor's + * pipe instead of a file. + */ + private boolean awaitWorkerReady(int timeoutSeconds) { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + while (System.nanoTime() < deadline) { + try { + List cmd = Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "cat", + WORKER_LOG_PATH); + String out = run(cmd, Math.min(30, timeoutSeconds)); + if (out.contains(WORKER_READY_MARKER)) { + return true; + } + } catch (Exception e) { + // Log the message AND the exception: with just "{}" and no argument the + // cause vanished and the failure read as a bare timeout. + classLogger.error("Could not read the worker log while waiting for it to become ready: " + + e.getClass().getSimpleName() + ": " + e.getMessage(), e); + } + try { + Thread.sleep(300); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return false; + } + + private String readAvailable(Process p) { + if (forwardLog == null) { + return ""; + } + synchronized (forwardLog) { + String s = forwardLog.toString().trim(); + return s.length() > 600 ? s.substring(s.length() - 600) : s; + } + } + + public int getForwardedPort() { + return forwardedPort; + } + + /** + * Copy one uploaded root back out of the sandbox onto the host. + * + *

+ * This is the missing half of the data plane: {@link #upload(String)} moves a tree + * in, and without this nothing ever moves back, so a file an agent or a + * {@code Py()} call writes inside the sandbox is destroyed with it. + * + *

+ * {@code openshell sandbox download} cannot be used directly: it is scoped to the + * sandbox workspace ({@code /opt/semosshome/py}) and refuses project paths with + * "outside the sandbox workspace". So the tree is streamed out through + * {@code sandbox exec -- tar -cf -} instead, which is byte-safe and therefore also + * correct for images and wheels. + * + *

+ * A full capture also propagates deletions, but only for paths the sandbox is known + * to have held (see {@link #seenBySandbox}); absence alone proves nothing, because + * the upload skips gitignored files and the host can gain files mid-session. + * + * @param absPath a path previously passed to {@link #upload(String)} + * @return number of files written back or removed + * @throws IOException when the capture could not be taken or applied; the caller must + * treat the root as unsynced + */ + public int fetchBack(String absPath) throws IOException { + if (absPath == null || absPath.trim().isEmpty()) { + return 0; + } + String baseFolder = Utility.getBaseFolder(); + String normalized = Utility.normalizePath(absPath); + if (baseFolder == null || !normalized.startsWith(Utility.normalizePath(baseFolder))) { + classLogger.warn("Refusing fetch-back outside the SEMOSS home: {}", absPath); + return 0; + } + File source = new File(normalized); + String parent = source.getParent(); + String base = source.getName(); + if (parent == null) { + return 0; + } + + Path temp = Files.createTempDirectory("openshell-fetchback-"); + try { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + // .git is captured so that a commit made in the sandbox shell survives, but only + // git's data files are ever written back -- see isWritableGitPath. Regenerable + // caches are skipped. + Path archive = temp.resolve("capture.tar"); + Capture capture = runToFile(Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "tar", "-C", + parent, "--ignore-failed-read", "--exclude=node_modules", "--exclude=__pycache__", "--exclude=.venv", + "-cf", "-", base), 120, archive); + if (capture.bytes == 0) { + return 0; + } + requireSafeArchive(archive); + Path tree = Files.createDirectory(temp.resolve("tree")); + extractArchive(archive, tree); + // tar warns about anything it could not read (a mode-000 directory, say). What + // it skipped is missing from the capture without having been deleted, so a + // partial capture writes files back but infers no deletions. + boolean complete = capture.warnings.isEmpty(); + if (!complete) { + classLogger.warn("Capture of {} is partial; not propagating deletions this time: {}", absPath, + capture.warnings); + } + int changed = reconcile(tree.resolve(base), Paths.get(normalized), complete); + classLogger.info("Full capture of {}: {} bytes, {} file(s) changed on the host{}", absPath, capture.bytes, + changed, complete ? "" : " (partial)"); + return changed; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("fetch-back of " + absPath + " interrupted", e); + } finally { + deleteQuietly(temp); + } + } + + /** + * Bring host-side changes to one transported root into the sandbox: files another user, + * a cluster pull or the UI wrote since this sandbox last had them, and files removed on + * the host. Run before a block, after the previous block's write-back, so the sandbox + * computes on what the platform holds now. + * + *

+ * Only files whose host stamp moved since the last sync are sent, in one upload. A file + * the sandbox also changed was reconciled by the previous write-back (last write won + * and the loser was kept), so sending the host's copy here cannot drop work. + * + * @param absRoot a path previously passed to {@link #upload(String)} + * @return number of files copied in or removed + */ + public int refreshFromHost(String absRoot) throws IOException { + Path root = Paths.get(Utility.normalizePath(absRoot)); + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) { + return 0; + } + Set seen = seenBySandbox.computeIfAbsent(root.toString(), + key -> ConcurrentHashMap.newKeySet()); + List changed = new ArrayList<>(); + Set present = new HashSet<>(); + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, + BasicFileAttributes attrs) { + String name = dir.getFileName() == null ? "" : dir.getFileName().toString(); + return !dir.equals(root) && (attrs.isSymbolicLink() || isExcludedFromCapture(Paths.get(name))) + ? FileVisitResult.SKIP_SUBTREE + : FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, + BasicFileAttributes attrs) { + if (attrs.isRegularFile()) { + present.add(file.toString()); + String stamp = stampOf(attrs); + if (!stamp.equals(hostStamps.get(file.toString()))) { + changed.add(file); + } + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) { + return FileVisitResult.CONTINUE; + } + }); + // Removed on the host since the sandbox had it: only paths this transport synced. + String prefix = root.toString() + File.separator; + List removed = new ArrayList<>(); + for (String known : new ArrayList<>(hostStamps.keySet())) { + if (known.startsWith(prefix) && !present.contains(known) + && !Files.exists(Paths.get(known), LinkOption.NOFOLLOW_LINKS)) { + removed.add(known); + } + } + if (changed.isEmpty() && removed.isEmpty()) { + return 0; + } + String cli = propOr(CLI_PROP, DEFAULT_CLI); + if (changed.size() + removed.size() > MAX_CHANGED_PATHS) { + // A whole-tree change (a branch switch, a re-import): send the root again. The + // upload records every stamp and only adds, so host deletions still follow. + classLogger.info("Refreshing all of {} in sandbox {}: {} host change(s)", root, sandboxName, + changed.size() + removed.size()); + if (!upload(root.toString())) { + throw new IOException("re-upload of " + root + " failed"); + } + for (Path file : changed) { + seen.add(relativeKey(root, file)); + } + } else if (!changed.isEmpty()) { + Path stage = Files.createTempDirectory("openshell-refresh-"); + try { + Path staged = stage.resolve(root.getFileName().toString()); + for (Path file : changed) { + Path copy = staged.resolve(root.relativize(file).toString()); + Files.createDirectories(copy.getParent()); + Files.copy(file, copy, StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS); + } + // `sandbox upload` merges into the existing tree and keeps mtimes, so after it + // both sides carry the stamp recorded below. + run(Arrays.asList(cli, "sandbox", "upload", sandboxName, staged.toString(), + root.getParent().toString()), 300); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted refreshing " + root, e); + } finally { + deleteQuietly(stage); + } + for (Path file : changed) { + recordStamp(file); + seen.add(relativeKey(root, file)); + } + } + // In batches, so a mass delete cannot exceed the argument-length limit. + for (int from = 0; from < removed.size(); from += REFRESH_DELETE_BATCH) { + List batch = removed.subList(from, Math.min(removed.size(), from + REFRESH_DELETE_BATCH)); + List rm = new ArrayList<>( + Arrays.asList(cli, "sandbox", "exec", "-n", sandboxName, "--", "rm", "-f", "--")); + rm.addAll(batch); + try { + run(rm, 60); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted refreshing " + root, e); + } + for (String known : batch) { + hostStamps.remove(known); + seen.remove(relativeKey(root, Paths.get(known))); + } + } + classLogger.info("Refreshed sandbox {} from {}: {} file(s) in, {} removed", sandboxName, root, changed.size(), + removed.size()); + return changed.size() + removed.size(); + } + + /** A path's key in {@link #seenBySandbox}: relative to its root, forward slashes. */ + private static String relativeKey(Path root, Path file) { + return root.relativize(file).toString().replace('\\', '/'); + } + + /** + * Fetch back only the named paths under a root that is already transported. + * + *

+ * This is the narrow form of {@link #fetchBack(String)}: the sandbox worker + * reports which files a block actually wrote, so the host tars a handful of + * paths instead of the whole tree. Cost becomes proportional to the change + * rather than to the size of the project. + * + * @param absRoot absolute path of the transported root inside the sandbox + * @param relativePaths paths relative to that root, as reported by the watcher + * @return number of files written back + * @throws IOException when the paths could not be captured or applied; the change set + * is then lost and the caller must schedule a full capture + */ + public int fetchBackPaths(String absRoot, List relativePaths) throws IOException { + if (absRoot == null || relativePaths == null || relativePaths.isEmpty()) { + return 0; + } + String baseFolder = Utility.getBaseFolder(); + String normalized = Utility.normalizePath(absRoot); + if (baseFolder == null || !normalized.startsWith(Utility.normalizePath(baseFolder))) { + classLogger.warn("Refusing path fetch-back outside the SEMOSS home: {}", absRoot); + return 0; + } + List safe = new ArrayList<>(); + for (String relative : relativePaths) { + if (relative == null || relative.isEmpty()) { + continue; + } + String cleaned = relative.replace('\\', '/'); + if (!isSafeRelative(cleaned)) { + classLogger.warn("Refusing traversal in change set: {}", relative); + continue; + } + safe.add(cleaned); + } + if (safe.isEmpty()) { + return 0; + } + if (safe.size() > MAX_CHANGED_PATHS) { + throw new IOException(safe.size() + " changed paths under " + absRoot + " exceed the " + MAX_CHANGED_PATHS + + " path limit"); + } + + Path temp = Files.createTempDirectory("openshell-fetchback-paths-"); + try { + String cli = propOr(CLI_PROP, DEFAULT_CLI); + List command = new ArrayList<>(Arrays.asList(cli, "sandbox", "exec", "-n", + sandboxName, "--", "tar", "-C", normalized, "--ignore-failed-read", "--exclude=node_modules", + "--exclude=__pycache__", "--exclude=.venv", "-cf", "-", "--")); + command.addAll(safe); + Path archive = temp.resolve("capture.tar"); + if (runToFile(command, 120, archive).bytes == 0) { + return 0; + } + requireSafeArchive(archive); + Path tree = Files.createDirectory(temp.resolve("tree")); + extractArchive(archive, tree); + return reconcile(tree, Paths.get(normalized), false); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("path fetch-back from " + absRoot + " interrupted", e); + } finally { + deleteQuietly(temp); + } + } + + /** + * Write files from {@code incoming} onto {@code target}. Content is the authority: a + * differing sandbox file is always written. The host copy is quarantined first when + * it changed since this transport last left it, and the write is skipped if that copy + * cannot be made. {@code fullTree} means {@code incoming} is a complete capture of the + * root: paths the sandbox held before but no longer has are then propagated as + * deletions. + */ + private int reconcile(Path incoming, Path target, boolean fullTree) + throws IOException { + if (!Files.isDirectory(incoming)) { + return 0; + } + Set seen = seenBySandbox.computeIfAbsent(target.toString(), + key -> ConcurrentHashMap.newKeySet()); + Set previous = fullTree ? new HashSet<>(seen) : null; + Set captured = new HashSet<>(); + Path store = quarantineStore(); + int[] copied = { 0 }; + try (Stream walk = Files.walk(incoming)) { + walk.filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) + .forEach(path -> { + String relative = incoming.relativize(path).toString().replace('\\', '/'); + captured.add(relative); + if (!isWritableGitPath(relative)) { + classLogger.debug("Not writing back git control file {}", relative); + return; + } + Path destination = containedHostPath(target, relative); + if (destination == null) { + classLogger.warn("Refusing write-back of {} under {}: path leaves the root or crosses a symlink", + relative, target); + return; + } + try { + if (Files.exists(destination, LinkOption.NOFOLLOW_LINKS)) { + if (Files.mismatch(path, destination) == -1L) { + recordStamp(destination); + return; + } + // Which side changed since the last sync? Both copies carried the same + // size and mtime then (uploads and captures preserve mtime), so a side + // that no longer matches the recorded stamp is a side that changed. A + // path the watcher reported was changed by the sandbox by definition. + String synced = hostStamps.get(destination.toString()); + boolean sandboxChanged = !fullTree || synced == null || !synced.equals(stampOf(path)); + boolean hostChanged = synced == null || !synced.equals(stampOf(destination)); + if (!sandboxChanged) { + // Only the host moved on (another user, a cluster pull, the UI): a + // full capture must not put the sandbox's stale copy back. The + // refresh before the next block brings the host's into the sandbox. + return; + } + if (hostChanged) { + // Both changed: last write wins, and the loser is kept. + if (mtimeOf(destination) > mtimeOf(path)) { + quarantineIncoming(path, target, relative, store); + classLogger.info("Write-back of {} skipped: the host copy is newer", destination); + return; + } + if (quarantineCopy(destination, target, store) == null) { + classLogger.warn( + "Write-back of {} skipped: the host copy changed and could not be preserved", + destination); + return; + } + } + } + Files.createDirectories(destination.getParent()); + // Re-check now that the parents exist: creating them must not have + // walked through a link that appeared in between. + if (containedHostPath(target, relative) == null) { + classLogger.warn("Refusing write-back of {} under {}: path crosses a symlink", relative, + target); + return; + } + Files.copy(path, destination, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES); + recordStamp(destination); + copied[0]++; + } catch (IOException e) { + classLogger.warn("Could not write back {}", destination, e); + } + }); + } + if (fullTree) { + copied[0] += removeDeletedFromHost(target, previous, captured, store); + // The capture is now the whole truth about what the sandbox holds. + seen.clear(); + seen.addAll(captured); + } else { + seen.addAll(captured); + } + if (copied[0] > 0) { + classLogger.info("Fetch-back wrote {} file(s) into {}", copied[0], target); + } + return copied[0]; + } + + /** + * A full capture carries survivors, not tombstones. A path the sandbox held before and + * the capture no longer contains was deleted inside the sandbox: quarantine the host + * copy, then remove it. Host files the sandbox never held are left alone. + */ + private int removeDeletedFromHost(Path target, Set previous, + Set captured, Path store) { + int moved = 0; + for (String relative : previous) { + if (captured.contains(relative) || !isWritableGitPath(relative) + || isExcludedFromCapture(Paths.get(relative))) { + continue; + } + Path file = containedHostPath(target, relative); + if (file == null || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { + continue; + } + if (hostChangedSinceSync(file)) { + // Someone updated it on the host after the sandbox last had it: keep the newer + // data. The next refresh puts it back into the sandbox. + classLogger.info("Delete of {} skipped: the host copy changed since the sandbox had it", file); + continue; + } + if (quarantineCopy(file, target, store) == null) { + classLogger.warn("Delete suppressed for {} because it could not be quarantined", file); + continue; + } + try { + Files.deleteIfExists(file); + hostStamps.remove(file.toString()); + pruneEmptyParents(file, target); + moved++; + } catch (IOException e) { + classLogger.warn("Could not remove {} after quarantining it", file, e); + } + } + if (moved > 0) { + classLogger.info("Fetch-back moved {} deleted file(s) to {}", moved, store); + } + return moved; + } + + /** + * Remove a host file the sandbox reported as deleted, keeping a copy under the + * quarantine root. Ignored unless the sandbox is known to have held the path: a file + * created and removed inside one block must not take an unrelated host file with it. + * + * @param root transported root the path belongs to + * @param relative path relative to that root + * @return true when the host file was removed + */ + public boolean deleteFromHost(String root, String relative) { + Path rootPath = Paths.get(Utility.normalizePath(root)); + Set seen = seenBySandbox.get(rootPath.toString()); + if (seen == null || !seen.contains(relative)) { + classLogger.debug("Ignoring delete of {} under {}: the sandbox never held it", relative, root); + return false; + } + if (!isWritableGitPath(relative)) { + return false; + } + Path file = containedHostPath(rootPath, relative); + if (file == null) { + classLogger.warn("Refusing delete of {} under {}: path leaves the root or crosses a symlink", relative, root); + return false; + } + if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { + seen.remove(relative); + return false; + } + if (hostChangedSinceSync(file)) { + classLogger.info("Delete of {} skipped: the host copy changed since the sandbox had it", file); + return false; + } + if (quarantineCopy(file, rootPath, quarantineStore()) == null) { + classLogger.warn("Delete suppressed for {} because it could not be quarantined", file); + return false; + } + try { + Files.deleteIfExists(file); + hostStamps.remove(file.toString()); + seen.remove(relative); + pruneEmptyParents(file, rootPath); + return true; + } catch (IOException e) { + classLogger.warn("Could not delete {} after quarantining it", file, e); + return false; + } + } + + /** + * Remove directories that our own deletions emptied, walking up to (but never + * including) the transported root. Only empty directories are removed, so a folder + * someone deliberately created stays where it is. + */ + private static void pruneEmptyParents(Path file, Path root) { + Path parent = file.getParent(); + while (parent != null && parent.startsWith(root) && !parent.equals(root)) { + try (Stream children = Files.list(parent)) { + if (children.findAny().isPresent()) { + return; + } + } catch (IOException e) { + return; + } + try { + Files.deleteIfExists(parent); + } catch (IOException e) { + return; + } + parent = parent.getParent(); + } + } + + /** + * Directories the root capture never contains: {@code tar} excludes them, so a file + * missing from the archive under one of these names was never captured rather than + * deleted, and must not be treated as a tombstone. + */ + private static boolean isExcludedFromCapture(Path relative) { + for (Path part : relative) { + String name = part.toString(); + if ("node_modules".equals(name) || "__pycache__".equals(name) || ".venv".equals(name)) { + return true; + } + } + return false; + } + + /** + * Inside a {@code .git} directory only git's data is written back: objects, refs, + * reflogs, HEAD and the index -- what a commit made in the sandbox produces. + * Everything else there (config, hooks, info, alternates) is code or configuration + * that host-side git runs or obeys, so a sandbox able to write it could execute + * commands on the server the next time the platform touches the repository. A + * {@code .git} file is refused too: it redirects the repository elsewhere. + */ + static boolean isWritableGitPath(String relative) { + String[] parts = relative.split("/"); + for (int i = 0; i < parts.length; i++) { + if (!".git".equals(parts[i])) { + continue; + } + if (i == parts.length - 1) { + return false; + } + String child = parts[i + 1]; + if (i + 2 == parts.length) { + return GIT_DATA_FILES.contains(child); + } + if ("objects".equals(child)) { + return !"info".equals(parts[i + 2]); + } + return "refs".equals(child) || "logs".equals(child); + } + return true; + } + + private static final Set GIT_DATA_FILES = Set.of("HEAD", "ORIG_HEAD", "FETCH_HEAD", + "MERGE_HEAD", "index", "packed-refs", "COMMIT_EDITMSG"); + + /** A relative path with no absolute prefix and no {@code ..} segment. */ + static boolean isSafeRelative(String relative) { + if (relative == null || relative.isEmpty() || relative.startsWith("/") || relative.indexOf('\0') >= 0) { + return false; + } + for (String part : relative.split("/")) { + if ("..".equals(part)) { + return false; + } + } + return true; + } + + /** + * Resolve {@code relative} under {@code root} for a host-side write or delete, or null + * when it would leave the root. No component between the root and the file may be a + * symlink: a link already in a writable project -- committed to its repository, say -- + * would otherwise redirect the write, or the delete, into another user's space. + */ + static Path containedHostPath(Path root, String relative) { + if (!isSafeRelative(relative)) { + return null; + } + Path candidate = root.resolve(relative).normalize(); + if (!candidate.startsWith(root) || candidate.equals(root)) { + return null; + } + Path current = root; + for (Path part : root.relativize(candidate)) { + current = current.resolve(part); + if (Files.isSymbolicLink(current)) { + return null; + } + if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) { + break; + } + } + return candidate; + } + + /** Remember a host file's state as this transport left it. */ + private void recordStamp(Path file) { + String stamp = stampOf(file); + if (stamp != null) { + hostStamps.put(file.toString(), stamp); + } + } + + /** True unless the host file is exactly as this transport last left it. */ + private boolean hostChangedSinceSync(Path file) { + String known = hostStamps.get(file.toString()); + return known == null || !known.equals(stampOf(file)); + } + + private static String stampOf(Path file) { + try { + return stampOf(Files.readAttributes(file, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS)); + } catch (IOException e) { + return null; + } + } + + /** + * {@code size:mtime} in whole seconds: tar and {@code sandbox upload} keep only seconds, + * so a finer stamp would read as changed after every round trip. + */ + private static String stampOf(BasicFileAttributes attributes) { + return attributes.size() + ":" + attributes.lastModifiedTime().to(TimeUnit.SECONDS); + } + + /** Modification time in whole seconds, the resolution both sides of a sync share. */ + private static long mtimeOf(Path file) { + try { + return Files.getLastModifiedTime(file, LinkOption.NOFOLLOW_LINKS) + .to(TimeUnit.SECONDS); + } catch (IOException e) { + return 0L; + } + } + + /** Keep the sandbox's copy of a file whose host copy won a conflict. */ + private static void quarantineIncoming(Path file, Path target, String relative, + Path store) { + try { + Path destination = store.resolve(target.getFileName().toString() + ".sandbox") + .resolve(relative); + Files.createDirectories(destination.getParent()); + Files.copy(file, destination, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS); + classLogger.warn("Write-back kept the sandbox copy of {} at {}", relative, destination); + } catch (Exception e) { + classLogger.warn("Could not keep the sandbox copy of {}", relative, e); + } + } + + /** + * Record the state of every file in a root just uploaded, so that write-back can tell + * a host file nobody touched (overwrite it) from one edited on the host since + * (quarantine it first). + */ + private void recordUploadedStamps(Path root) { + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) { + if (Files.isRegularFile(root, LinkOption.NOFOLLOW_LINKS)) { + recordStamp(root); + } + return; + } + try { + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, + BasicFileAttributes attrs) { + String name = dir.getFileName() == null ? "" : dir.getFileName().toString(); + return isExcludedFromCapture(Paths.get(name)) + ? FileVisitResult.SKIP_SUBTREE + : FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, + BasicFileAttributes attrs) { + if (attrs.isRegularFile()) { + hostStamps.put(file.toString(), stampOf(attrs)); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) { + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + classLogger.debug("Could not record upload state for {}", root, e); + } + } + + /** Timestamped quarantine root, outside every transported root so it is never pushed. */ + private static Path quarantineStore() { + String base = Utility.getBaseFolder(); + Path store = Paths + .get(base == null ? System.getProperty("java.io.tmpdir") : base, "writeback-quarantine"); + String stamp = new SimpleDateFormat("yyyyMMdd-HHmmss").format(new Date()); + return store.resolve(stamp); + } + + /** Copy a host file aside before it is overwritten or removed. Returns the copy, or null. */ + private static Path quarantineCopy(Path file, Path root, + Path store) { + try { + Path relative = root.relativize(file); + Path destination = store.resolve(root.getFileName().toString()) + .resolve(relative.toString()); + Files.createDirectories(destination.getParent()); + Files.copy(file, destination, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS); + classLogger.warn("Write-back preserved the host copy of {} at {}", file, destination); + return destination; + } catch (Exception e) { + classLogger.warn("Could not preserve the host copy of {}", file, e); + return null; + } + } + + /** + * Refuse an archive whose listing cannot be read, or that names an absolute path or + * a {@code ..} segment. Checked per segment, so a file called {@code v1..v2.diff} is + * fine. + */ + private static void requireSafeArchive(Path archive) throws IOException, InterruptedException { + String listing = runLocal(Arrays.asList("tar", "-tf", archive.toString()), 60); + for (String line : listing.split("\\R")) { + String entry = line.trim(); + if (entry.isEmpty()) { + continue; + } + if (entry.endsWith("/")) { + entry = entry.substring(0, entry.length() - 1); + } + if (entry.isEmpty() || !isSafeRelative(entry)) { + throw new IOException("archive contains an absolute or traversing path: " + line); + } + } + } + + /** + * Extract, then give the owner access to everything extracted: a directory the + * sandbox left at mode 000 would otherwise be unreadable to the reconcile walk and + * undeletable afterwards. + */ + private static void extractArchive(Path archive, Path destination) + throws IOException, InterruptedException { + runLocal(Arrays.asList("tar", "-xf", archive.toString(), "-C", destination.toString(), "--no-same-owner"), + 120); + makeOwnerAccessible(destination); + } + + private static void makeOwnerAccessible(Path root) { + try { + runLocal(Arrays.asList("chmod", "-R", "u+rwX", root.toString()), 60); + } catch (Exception e) { + classLogger.debug("Could not normalise permissions under {}", root, e); + } + } + + private static void deleteQuietly(Path root) { + if (root == null || !Files.exists(root)) { + return; + } + makeOwnerAccessible(root); + try (Stream walk = Files.walk(root)) { + walk.sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + // best effort + } + }); + } catch (IOException ignored) { + // best effort + } + } + + /** Run a local helper (tar, chmod) and return its output; a non-zero exit is an error. */ + private static String runLocal(List cmd, int timeoutSeconds) throws IOException, InterruptedException { + ProcessBuilder builder = new ProcessBuilder(cmd); + builder.redirectErrorStream(true); + builder.redirectInput(ProcessBuilder.Redirect.from(new File("/dev/null"))); + Process process = builder.start(); + ByteArrayOutputStream collected = new ByteArrayOutputStream(); + Thread reader = new Thread(() -> { + try (InputStream stream = process.getInputStream()) { + stream.transferTo(collected); + } catch (IOException ignored) { + // the exit code carries the outcome + } + }, "openshell-local-" + cmd.get(0)); + reader.setDaemon(true); + reader.start(); + if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); + } + reader.join(5000); + String output = collected.toString(StandardCharsets.UTF_8); + if (process.exitValue() != 0) { + throw new IOException("command failed (" + process.exitValue() + "): " + String.join(" ", cmd) + "\n" + + output.trim()); + } + return output; + } + + /** + * Run a CLI command and stream its raw stdout into {@code out}, stopping it as soon as + * the output passes {@link #MAX_FETCH_BYTES}. The stream goes to disk rather than the + * heap: one session's capture must never be able to exhaust the memory of the JVM + * every other session shares. + * + * @return bytes written, and whatever the command wrote to stderr + */ + private Capture runToFile(List cmd, int timeoutSeconds, Path out) + throws IOException, InterruptedException { + ProcessBuilder builder = new ProcessBuilder(cmd); + builder.environment().putAll(cliEnv()); + builder.redirectInput(ProcessBuilder.Redirect.from(new File("/dev/null"))); + Process process = builder.start(); + StringBuilder errors = new StringBuilder(); + Thread errorReader = new Thread(() -> { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getErrorStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + synchronized (errors) { + if (errors.length() < 4096) { + errors.append(line).append('\n'); + } + } + } + } catch (IOException ignored) { + // the exit code carries the outcome + } + }, "openshell-fetchback-errors"); + errorReader.setDaemon(true); + errorReader.start(); + AtomicLong written = new AtomicLong(); + AtomicBoolean overCap = new AtomicBoolean(); + Thread reader = new Thread(() -> { + byte[] buffer = new byte[64 * 1024]; + try (InputStream stream = process.getInputStream(); + OutputStream sink = Files.newOutputStream(out)) { + int read; + while ((read = stream.read(buffer)) > 0) { + if (written.addAndGet(read) > MAX_FETCH_BYTES) { + overCap.set(true); + process.destroyForcibly(); + return; + } + sink.write(buffer, 0, read); + } + } catch (IOException ignored) { + // the exit code carries the outcome + } + }, "openshell-fetchback-reader"); + reader.setDaemon(true); + reader.start(); + if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); + } + reader.join(10_000); + errorReader.join(2000); + String warnings; + synchronized (errors) { + warnings = errors.toString().trim(); + } + if (overCap.get()) { + throw new IOException("capture passed the " + MAX_FETCH_BYTES + " byte cap: " + String.join(" ", cmd)); + } + if (process.exitValue() != 0) { + throw new IOException("command failed (" + process.exitValue() + "): " + String.join(" ", cmd) + + (warnings.isEmpty() ? "" : "\n" + warnings)); + } + return new Capture(written.get(), warnings); + } + + /** Size of a streamed capture and what the command reported on stderr. */ + private static final class Capture { + private final long bytes; + private final String warnings; + + Capture(long bytes, String warnings) { + this.bytes = bytes; + this.warnings = warnings; + } + } + + /** + * Stop forwarding and delete the sandbox. + * + *

+ * This is the whole teardown: the gateway delete is authoritative and handles + * process, cgroup and network cleanup. There is no "graceful worker shutdown" + * verb to call first, because the worker cannot outlive its sandbox. + */ + public void close() { + detach(); + if (keepSandbox) { + classLogger.warn("KEEP_SANDBOX is set; leaving sandbox {} in place for inspection", sandboxName); + return; + } + try { + run(Arrays.asList(propOr(CLI_PROP, DEFAULT_CLI), "sandbox", "delete", sandboxName), 60); + } catch (Exception e) { + classLogger.warn("Failed to delete executor sandbox {}", sandboxName, e); + } + } + + // ------------------------------------------------------------------ helpers + + /** + * Environment for CLI children: inherit the caller's, but pin {@code HOME} so the + * CLI always finds the same gateway registration and client certificate bundle. + */ + private static Map cliEnv() { + Map env = new HashMap<>(System.getenv()); + String home = System.getProperty(CLI_HOME_PROP); + if (home == null || home.trim().isEmpty()) { + home = Utility.getDIHelperProperty(CLI_HOME_PROP); + } + if (home != null && !home.trim().isEmpty()) { + env.put("HOME", home.trim()); + } + return env; + } + + private static String propOr(String key, String fallback) { + String v = Utility.getDIHelperProperty(key); + return (v == null || v.trim().isEmpty()) ? fallback : v.trim(); + } + + private static int intPropOr(String key, int fallback) { + String v = Utility.getDIHelperProperty(key); + if (v == null || v.trim().isEmpty()) { + return fallback; + } + try { + return Integer.parseInt(v.trim()); + } catch (NumberFormatException e) { + return fallback; + } + } + + /** + * Ask the OS for a free port rather than picking one: the forward binds + * host-local, and two sessions must not collide. + */ + private static int freeLocalPort() throws IOException { + try (ServerSocket s = new ServerSocket(0)) { + return s.getLocalPort(); + } + } + + /** + * Run an {@code openshell} subcommand, for collaborators such as the secret broker. + * + * @param args everything after the CLI name + * @param extraEnv added to the child's environment; how a credential value reaches the CLI + * ({@code --credential KEY} reads it from there), so it is never on a command + * line or in a process listing + */ + public static String cli(List args, int timeoutSeconds, Map extraEnv) + throws IOException, InterruptedException { + List cmd = new ArrayList<>(); + cmd.add(propOr(CLI_PROP, DEFAULT_CLI)); + cmd.addAll(args); + return run(cmd, timeoutSeconds, extraEnv); + } + + private static String run(List cmd, int timeoutSeconds) throws IOException, InterruptedException { + return run(cmd, timeoutSeconds, Map.of()); + } + + private static String run(List cmd, int timeoutSeconds, Map extraEnv) + throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.environment().putAll(cliEnv()); + pb.environment().putAll(extraEnv); + pb.redirectErrorStream(true); + // Close the child's stdin immediately. Without this it inherits a pipe that + // stays open for the JVM's lifetime and any subcommand that reads stdin -- + // `openshell sandbox exec` does -- blocks until the timeout instead of + // running. Measured 2026-09-28: the same `sandbox exec ... cat` returned + // instantly from a shell (stdin at EOF) and timed out after 30s under + // Tomcat, which is what made the readiness poll fail and the pixel stall for + // the full 180s. A shell would not have shown this. + pb.redirectInput(ProcessBuilder.Redirect.from(new File("/dev/null"))); + Process p = pb.start(); + + // Read on a separate thread. A blocking readLine() here hangs the CALLER -- + // and because close() also calls run(), a wedged CLI child used to deadlock the + // failure path itself: the request thread sat in readLine() inside close() with + // no timeout, and the sandbox was never cleaned up. Measured 2026-09-28. + StringBuilder out = new StringBuilder(); + Thread reader = new Thread(() -> { + try (BufferedReader r = new BufferedReader(new InputStreamReader( + p.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + synchronized (out) { + out.append(line).append('\n'); + } + } + } catch (IOException ignored) { + // stream closed with the process; the exit code carries the outcome + } + }, "openshell-cli-" + cmd.get(0)); + reader.setDaemon(true); + reader.start(); + + if (!p.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + p.destroyForcibly(); + reader.interrupt(); + throw new IOException("timed out after " + timeoutSeconds + "s: " + String.join(" ", cmd)); + } + // Give the reader a moment to drain what the process wrote before it exited. + reader.join(2000); + if (p.exitValue() != 0) { + // A sandbox that already exists is the normal re-entry case, not a failure. + String collected; + synchronized (out) { + collected = out.toString(); + } + String lower = collected.toLowerCase(); + if (!lower.contains("already exists")) { + throw new IOException("command failed (" + p.exitValue() + "): " + String.join(" ", cmd) + "\n" + + collected.trim()); + } + } + synchronized (out) { + return out.toString().trim(); + } + } +} diff --git a/src/prerna/util/OpenShellTransporter.java b/src/prerna/util/OpenShellTransporter.java new file mode 100644 index 0000000000..4b0833c376 --- /dev/null +++ b/src/prerna/util/OpenShellTransporter.java @@ -0,0 +1,80 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +import prerna.util.openshell.OpenShellSecrets; +import prerna.util.openshell.TransportedRoots; + +/** + * {@link SandboxTransporter} for {@link SymlinkHelper.SandboxMode#OPENSHELL}: copies a host path into + * the sandbox with {@code openshell sandbox upload}. + * + *

+ * The sandbox has its own filesystem, so unlike the namespace jail -- which runs on this host and is + * satisfied by binding the real path -- the bytes must actually be transferred. Uploads preserve the + * absolute path, so a worker that opens an asset by absolute path or inserts it into {@code sys.path} + * finds it where it expects. + */ +public class OpenShellTransporter implements SandboxTransporter { + + private static final org.apache.logging.log4j.Logger classLogger = org.apache.logging.log4j.LogManager + .getLogger(OpenShellTransporter.class); + + private final OpenShellTransport transport; + + public OpenShellTransporter(OpenShellTransport transport) { + this.transport = transport; + } + + @Override + public boolean transport(String absPath) { + if (this.transport == null) { + classLogger.warn("No OpenShell transport; cannot send {}", absPath); + return false; + } + if (absPath == null || absPath.trim().isEmpty()) { + return false; + } + boolean uploaded = this.transport.upload(absPath); + if (uploaded) { + // A project root arriving in the sandbox is when its declared secrets are attached: + // only users allowed to load the project get here. + String normalized = Utility.normalizePath(absPath); + String projectId = TransportedRoots.projectIdOf(normalized); + if (projectId != null && normalized.matches(".*/" + Constants.PROJECT_FOLDER + "/[^/]+/app_root/?$")) { + OpenShellSecrets.ensureForProject(this.transport, projectId); + } + } + return uploaded; + } + + @Override + public boolean close() { + return true; + } +} diff --git a/src/prerna/util/SandboxTransporter.java b/src/prerna/util/SandboxTransporter.java new file mode 100644 index 0000000000..6c1481ea00 --- /dev/null +++ b/src/prerna/util/SandboxTransporter.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +/** + * Moves host paths into an external sandbox. + * + *

+ * The OPENSHELL counterpart of {@link SandboxInjector}. Both are the per-mode collaborator that + * {@link SymlinkHelper} hands a path to when it would otherwise have symlinked it: + * + * + * + * + * + * + *
mode collaborators
ModeCollaboratorWhat it does with the path
NAMESPACE{@link SandboxInjector}adds it to the namespace jail's + * allowed set over the control socket
OPENSHELL{@link SandboxTransporter}copies the tree into the + * sandbox
+ * + *

+ * The distinction is worth stating because it is not an implementation detail: a namespace jail runs + * on this host, so binding the real path is enough. A sandbox has its own filesystem, so the bytes + * have to be transferred. Making the path visible is not making it present. A worker that calls + * {@code sys.path.insert} on an asset path, or opens an asset by absolute path, fails with + * {@code FileNotFoundError} when only the path exists -- observed with a project MCP driver, where + * {@code PyTranslator} passed the asset paths correctly and the import still failed because the file + * was never in the sandbox. + */ +public interface SandboxTransporter { + + /** + * Make {@code absPath} (a file or directory tree on this host) available inside the sandbox at the + * same absolute path. + * + *

+ * Implementations must be idempotent: the same path is requested repeatedly (every insight, + * project open and MCP call re-symlinks), and re-transferring a tree on each call would be a + * per-request cost with no benefit. + * + * @param absPath host path, already normalised + * @return true if the path is present in the sandbox after the call + */ + boolean transport(String absPath); + + /** + * Best-effort teardown. + * + * @return true if the collaborator released cleanly + */ + boolean close(); +} diff --git a/src/prerna/util/SymlinkHelper.java b/src/prerna/util/SymlinkHelper.java index 8fa49cfab0..5af43ae60c 100644 --- a/src/prerna/util/SymlinkHelper.java +++ b/src/prerna/util/SymlinkHelper.java @@ -41,6 +41,8 @@ import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; @@ -55,6 +57,27 @@ public class SymlinkHelper { + /** + * Sandbox modes this user-level jail can be in. + * + *

+ * The distinction is who supplies the {@link SymlinkHelper}, not a flag checked at the call + * site: {@code User.startPythonSocketServerAndClient} passes one for user-authored python, while + * engine-owned workers (model engines, vector databases, guardrails, local function engines) and + * {@code Project} pass {@code null}. That is existing semantics and the reason a user-authored + * worker is the only one that ever reaches this class. + */ + public enum SandboxMode { + /** No kernel jail: fake chroot with symlinks. */ + FAKECHROOT, + /** Namespace jail (nsjail/landlock style) with a control-socket injector. */ + NAMESPACE, + /** External sandbox (OpenShell) created ahead of the worker; no in-process jail. */ + OPENSHELL + } + + private final SandboxMode sandboxMode; + private static final Logger classLogger = LogManager.getLogger(SymlinkHelper.class); private String userChrootFolder = null; @@ -62,6 +85,21 @@ public class SymlinkHelper { private final boolean injectMode; private SandboxInjector injector; private final Map activeInjects = new LinkedHashMap<>(); + /** + * Paths already handed to the external sandbox. A project open, an insight and an MCP call all + * re-symlink the same folders; without this the tree would be re-transferred on every request. + */ + private final Set activeTransports = ConcurrentHashMap.newKeySet(); + + /** + * Paths this helper has handed to the sandbox, in the order they were first + * requested. Read-only view: the fetch-back needs to know which roots exist in + * the sandbox so it can reconcile them when the process finishes a block. + */ + public java.util.List getTransportedPaths() { + return new java.util.ArrayList<>(activeTransports); + } + private SandboxTransporter transporter; /** * Construct a SymlinkHelper bound to a user-specific chroot folder and run the @@ -75,8 +113,11 @@ public class SymlinkHelper { */ public SymlinkHelper(String targetDirName) { this.userChrootFolder = targetDirName; - String sandboxMode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); - this.injectMode = "NAMESPACE".equalsIgnoreCase(sandboxMode) || "NSJAIL".equalsIgnoreCase(sandboxMode); + // The mode is resolved once, here, because this object exists only for a + // user-authored worker. Callers that must NOT be jailed (engine-owned + // workers, Project python) pass a null helper and never reach this. + this.sandboxMode = resolveSandboxMode(); + this.injectMode = this.sandboxMode == SandboxMode.NAMESPACE; File targetDir = new File(Utility.normalizePath(userChrootFolder)); if (!targetDir.exists()) { @@ -100,6 +141,29 @@ public SymlinkHelper(String targetDirName) { classLogger.info("Chroot init completed in {} ms for: {}", elapsedMs, userChrootFolder); } + /** + * Wire the per-mode collaborator that moves paths into an external sandbox. + * + *

+ * The exact counterpart of {@link #setInjector(SandboxInjector)}: both are set once by + * {@code ClientProcessWrapper} when the worker starts, and both are replayed with everything + * requested so far, because asset paths are registered before the worker exists. + * + * @param transporter the transporter for {@link SandboxMode#OPENSHELL} + */ + public synchronized void setTransport(SandboxTransporter transporter) { + this.transporter = transporter; + if (transporter != null && !activeTransports.isEmpty()) { + for (String absPath : activeTransports) { + try { + transporter.transport(absPath); + } catch (RuntimeException e) { + classLogger.warn("Skipping sandbox transport for {}", absPath, e); + } + } + } + } + public synchronized void setInjector(SandboxInjector injector) { this.injector = injector; if (injector != null && !activeInjects.isEmpty()) { @@ -117,6 +181,78 @@ public boolean isInjectMode() { return injectMode; } + /** The user-level jail this helper's worker runs under. */ + public SandboxMode getSandboxMode() { + return this.sandboxMode; + } + + /** + * True when the worker runs inside an external sandbox created ahead of it + * (OpenShell), so no in-process jail is applied. + */ + public boolean isExternalSandbox() { + return this.sandboxMode == SandboxMode.OPENSHELL; + } + + private static SandboxMode resolveSandboxMode() { + String mode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); + if (mode == null || (mode = mode.trim()).isEmpty()) { + // Historically an unset mode means the plain fakechroot jail. + return SandboxMode.FAKECHROOT; + } + if (Constants.SANDBOX_MODE_OPENSHELL.equalsIgnoreCase(mode)) { + return SandboxMode.OPENSHELL; + } + if ("NAMESPACE".equalsIgnoreCase(mode) || "NSJAIL".equalsIgnoreCase(mode)) { + return SandboxMode.NAMESPACE; + } + if ("FAKECHROOT".equalsIgnoreCase(mode)) { + return SandboxMode.FAKECHROOT; + } + // SECCOMP_ONLY and anything unrecognised keep the pre-existing fakechroot + // behaviour rather than silently gaining a jail. + return SandboxMode.FAKECHROOT; + } + + /** + * The single place a path is handed to the active jail. + * + *

+ * Every {@code symlinkFolder}/{@code symlinkProject} caller routes through here, so this is the + * seam that gives the sandbox the same files the other two jails get. A path requested before the + * worker exists is remembered and replayed by {@link #setTransport(SandboxTransporter)}, exactly as + * the namespace injector replays {@code activeInjects}. + */ + private synchronized void transportOrEnqueue(String absPath, boolean readWrite) { + if (sandboxMode == SandboxMode.OPENSHELL) { + // Same scope rule as the namespace injector. Without it the transporter is handed + // system paths -- /usr/bin, /etc/ssl, /usr/lib/git-core -- because + // MCPUtility.runPythonTool also symlinks git's own directories. Uploading those is + // wrong and enormous: observed as several 30s CLI timeouts, a 77s pixel, and the + // project never arriving. The sandbox image already provides them. + if (!isNamespaceInjectablePath(absPath)) { + classLogger.debug("Skipping sandbox transport for path outside SEMOSS home: {}", absPath); + return; + } + boolean existing = !activeTransports.add(absPath); + if (existing) { + // Already sent once; transport() is idempotent but there is no reason to call it. + return; + } + if (transporter != null) { + try { + if (!transporter.transport(absPath)) { + classLogger.warn("Sandbox transport reported failure for {}", absPath); + } + } catch (RuntimeException e) { + classLogger.warn("Sandbox transport failed for {}", absPath, e); + } + } + return; + } + enqueueOrInject(absPath, readWrite); + } + private synchronized void enqueueOrInject(String absPath, boolean readWrite) { if (!isNamespaceInjectablePath(absPath)) { classLogger.warn("Skipping namespace injection for path outside SEMOSS home: {}", absPath); @@ -310,8 +446,13 @@ private void copyDirectoryRecursively(Path source, Path target) throws IOExcepti */ public void symlinkFolder(String sourceDirName) { sourceDirName = Utility.normalizePath(sourceDirName); - if (injectMode) { - enqueueOrInject(sourceDirName, true); + if (sandboxMode != SandboxMode.FAKECHROOT) { + transportOrEnqueue(sourceDirName, true); + } + if (sandboxMode == SandboxMode.OPENSHELL) { + // The sandbox has its own filesystem root; a chroot symlink under the host user folder + // would point at a path the sandbox cannot see, so it is meaningless here. + return; } createChrootSymlink(sourceDirName); } @@ -436,8 +577,8 @@ public void symlinkProject(User user, String projectId) { boolean canEdit = SecurityProjectUtils.userCanEditProject(user, projectId); - if (injectMode) { - enqueueOrInject(Utility.normalizePath(projectAppRootFolder), canEdit); + if (sandboxMode != SandboxMode.FAKECHROOT) { + transportOrEnqueue(Utility.normalizePath(projectAppRootFolder), canEdit); } if (canEdit) { diff --git a/src/prerna/util/openshell/OpenShellSecrets.java b/src/prerna/util/openshell/OpenShellSecrets.java new file mode 100644 index 0000000000..d3274e4ab8 --- /dev/null +++ b/src/prerna/util/openshell/OpenShellSecrets.java @@ -0,0 +1,393 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import prerna.project.api.IProject; +import prerna.util.OpenShellTransport; +import prerna.util.Utility; + +/** + * Secrets an app declares in its project {@code .smss}, brokered into a user's sandbox by + * OpenShell providers, so sandboxed code can use a secret without ever holding it. + * + * 

+ * SANDBOX_SECRETS                     [{"name":"OPENWEATHER_API_KEY","hosts":["api.openweathermap.org"]}]
+ * SANDBOX_SECRET_OPENWEATHER_API_KEY  the value
+ *
+ * + *

+ * The value stays on the host: the {@code .smss} is never uploaded into a sandbox. Each secret + * becomes one OpenShell profile and provider, so it can only reach its own hosts. Code in the + * sandbox gets a placeholder ({@code secret("OPENWEATHER_API_KEY")} in {@code smss_secrets}), and + * OpenShell's supervisor swaps it for the value in requests to those hosts and refuses it + * anywhere else. Any request method is allowed. + * + *

+ * Measured behaviour this follows (2026-10-24, OpenShell 0.1.2): + *

    + *
  • attaching a provider to a running sandbox takes tens of seconds to apply, and requests + * in that window hang, so an attach is waited out;
    • + *
  • an attach changes the placeholder generation and the old placeholders stop resolving, so + * the placeholder file is rewritten from the sandbox's current environment after every attach + * (the worker's own environment is fixed at start);
    • + *
  • {@code provider update} also starts a new generation, within seconds, and placeholders + * of the old one keep resolving to the old value; so after an update the file is + * rewritten once the new generation appears, and an unchanged value is not updated at all.
    • + *
+ * The helper reads the file on every call, which is what lets the long-lived worker follow both. + * The value in the property can later come from the secret store instead, which overlays the + * same properties. + */ +public final class OpenShellSecrets { + + private static final Logger classLogger = LogManager.getLogger(OpenShellSecrets.class); + + /** The project property declaring the secrets, as a JSON list of {name, hosts}. */ + public static final String SECRETS_PROP = "SANDBOX_SECRETS"; + /** The project property holding one secret's value, followed by its name. */ + public static final String VALUE_PROP_PREFIX = "SANDBOX_SECRET_"; + /** Where the placeholders are written in the sandbox; read by {@code py/smss_secrets.py}. */ + private static final String PLACEHOLDER_PARENT = "/opt/semosshome/insight"; + private static final String PLACEHOLDER_DIR = "sandbox-secrets"; + private static final String PLACEHOLDER_FILE = "placeholders.json"; + + private static final Pattern NAME = Pattern.compile("[A-Z][A-Z0-9_]{0,63}"); + /** A credential in the sandbox environment: {@code __}. */ + private static final Pattern CREDENTIAL_ENV = Pattern.compile("([A-Z][A-Z0-9_]*)__([a-z0-9]{8})=(openshell:resolve:\\S+)"); + private static final long READY_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(90); + private static final long READY_POLL_MS = 2000; + private static final int CLI_TIMEOUT_S = 60; + + /** How long a new placeholder generation may take to appear after an update (measured 5–15 s). */ + private static final long GENERATION_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(60); + private static final Pattern GENERATION = Pattern.compile("=openshell:resolve:env:v(\\d+)_"); + + private static final Map SANDBOX_LOCKS = new ConcurrentHashMap<>(); + /** Digest of the value last given to each provider by this JVM; unknown after a restart. */ + private static final Map APPLIED = new ConcurrentHashMap<>(); + + private OpenShellSecrets() { + // static utility + } + + /** One declared secret. */ + record Declared(String name, List hosts) { + } + + /** + * Make a project's declared secrets usable in this sandbox: register each with OpenShell, + * update its value, attach it if it is not yet, and refresh the placeholder file. A secret + * that is declared wrongly or has no value is skipped with a warning; nothing here fails + * the project load. + */ + public static void ensureForProject(OpenShellTransport transport, String projectId) { + IProject project = Utility.getProject(projectId); + if (project != null && project.getSmssProp() != null) { + ensureForProject(transport, projectId, project.getSmssProp()); + } + } + + /** {@link #ensureForProject(OpenShellTransport, String)} for a project's properties. */ + public static void ensureForProject(OpenShellTransport transport, String projectId, Properties props) { + List declared = parse(projectId, props.getProperty(SECRETS_PROP)); + if (declared.isEmpty()) { + return; + } + String sandbox = transport.getSandboxName(); + synchronized (SANDBOX_LOCKS.computeIfAbsent(sandbox, k -> new Object())) { + try { + String attached = OpenShellTransport.cli(List.of("sandbox", "provider", "list", sandbox), + CLI_TIMEOUT_S, Map.of()); + String generation = generation(sandbox); + boolean changed = false; + for (Declared secret : declared) { + String value = props.getProperty(VALUE_PROP_PREFIX + secret.name()); + if (value == null || value.trim().isEmpty()) { + classLogger.warn("Project {} declares sandbox secret {} but sets no {}{}", projectId, + secret.name(), VALUE_PROP_PREFIX, secret.name()); + continue; + } + String id = providerId(projectId, secret); + String credential = credentialName(projectId, secret.name()); + ensureProfile(id, projectId, secret, credential); + changed |= ensureProvider(id, credential, value.trim()); + if (!containsWord(attached, id)) { + changed = true; + OpenShellTransport.cli(List.of("sandbox", "provider", "attach", sandbox, id), CLI_TIMEOUT_S, + Map.of()); + awaitApplied(sandbox, id); + classLogger.info("Attached sandbox secret {} of project {} to sandbox {}", secret.name(), + projectId, sandbox); + } + } + if (changed && generation != null) { + awaitNewGeneration(sandbox, generation); + } + writePlaceholders(sandbox); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + classLogger.warn("Interrupted brokering secrets of project {} into sandbox {}", projectId, sandbox); + } catch (Exception e) { + classLogger.warn("Could not broker secrets of project {} into sandbox {}", projectId, sandbox, e); + } + } + } + + /** The declared secrets, skipping (with a warning) any entry that is not well formed. */ + static List parse(String projectId, String json) { + List declared = new ArrayList<>(); + if (json == null || json.trim().isEmpty()) { + return declared; + } + JsonArray list; + try { + list = JsonParser.parseString(json).getAsJsonArray(); + } catch (RuntimeException e) { + classLogger.warn("Project {} has an unreadable {}: expected a JSON list", projectId, SECRETS_PROP); + return declared; + } + for (JsonElement element : list) { + try { + JsonObject entry = element.getAsJsonObject(); + String name = entry.get("name").getAsString().trim(); + List hosts = new ArrayList<>(); + for (JsonElement host : entry.getAsJsonArray("hosts")) { + hosts.add(host.getAsString().trim().toLowerCase()); + } + if (!NAME.matcher(name).matches() || hosts.isEmpty() + || !hosts.stream().allMatch(OpenShellTransport::isValidHost)) { + classLogger.warn("Project {} {} entry {} needs an upper-case name and plain host names", projectId, + SECRETS_PROP, element); + continue; + } + declared.add(new Declared(name, List.copyOf(new TreeSet<>(hosts)))); + } catch (RuntimeException e) { + classLogger.warn("Project {} {} entry {} is not {\"name\", \"hosts\"}", projectId, SECRETS_PROP, element); + } + } + return declared; + } + + /** The project's key in credential names and the placeholder file: its first 8 id characters. */ + static String projectKey(String projectId) { + String compact = projectId.toLowerCase().replaceAll("[^a-z0-9]", ""); + return compact.length() >= 8 ? compact.substring(0, 8) : String.format("%-8s", compact).replace(' ', '0'); + } + + /** The credential's name in the sandbox, unique per project so two apps may share a name. */ + static String credentialName(String projectId, String name) { + return name + "__" + projectKey(projectId); + } + + /** + * The profile and provider id. It includes a digest of the hosts and the interpreter, so a + * changed declaration is a new profile rather than an edit of one that sandboxes use. + */ + static String providerId(String projectId, Declared secret) { + String hosts = digest(String.join(",", secret.hosts()) + "|" + OpenShellTransport.pythonBinary()).substring(0, 6); + return "semoss-" + projectKey(projectId) + "-" + secret.name().toLowerCase().replace('_', '-') + "-" + hosts; + } + + private static void ensureProfile(String id, String projectId, Declared secret, String credential) + throws IOException, InterruptedException { + try { + OpenShellTransport.cli(List.of("profile", "describe", id), CLI_TIMEOUT_S, Map.of()); + return; + } catch (IOException missing) { + // not registered yet + } + StringBuilder yaml = new StringBuilder(); + yaml.append("id: ").append(id).append('\n'); + yaml.append("display_name: ").append(secret.name()).append(" (SEMOSS project ").append(projectId).append(")\n"); + yaml.append("description: sandbox secret declared by SEMOSS project ").append(projectId).append('\n'); + yaml.append("category: other\n"); + yaml.append("credentials:\n - name: ").append(credential).append("\n required: true\n"); + yaml.append("endpoints:\n"); + for (String host : secret.hosts()) { + // credentialed endpoints must be inspected (L7) for the supervisor to swap the value in + yaml.append(" - host: ").append(host).append("\n port: 443\n protocol: rest\n access: full\n"); + } + // without a binary the generated rule lets no program connect + yaml.append("binaries:\n - ").append(OpenShellTransport.pythonBinary()).append('\n'); + Path dir = Files.createTempDirectory("openshell-profile-"); + try { + Path file = dir.resolve(id + ".yaml"); + Files.writeString(file, yaml.toString(), StandardCharsets.UTF_8); + OpenShellTransport.cli(List.of("profile", "import", "--file", file.toString()), CLI_TIMEOUT_S, Map.of()); + } finally { + deleteQuietly(dir); + } + } + + /** + * Create the provider, or update its value when it changed. + * + * @return whether the value was set, which starts a new placeholder generation + */ + private static boolean ensureProvider(String id, String credential, String value) + throws IOException, InterruptedException { + String digest = digest(value); + Map env = Map.of(credential, value); + boolean exists; + try { + OpenShellTransport.cli(List.of("provider", "get", id), CLI_TIMEOUT_S, Map.of()); + exists = true; + } catch (IOException missing) { + exists = false; + } + if (exists && digest.equals(APPLIED.get(id))) { + return false; + } + if (exists) { + OpenShellTransport.cli(List.of("provider", "update", id, "--credential", credential), CLI_TIMEOUT_S, env); + } else { + OpenShellTransport.cli(List.of("provider", "create", "--name", id, "--type", id, "--credential", credential), + CLI_TIMEOUT_S, env); + } + APPLIED.put(id, digest); + return true; + } + + /** The placeholder generation new processes in the sandbox get, or null with no secret yet. */ + private static String generation(String sandbox) throws IOException, InterruptedException { + Matcher m = GENERATION.matcher(OpenShellTransport.cli(List.of("sandbox", "exec", "-n", sandbox, "--", "env"), + CLI_TIMEOUT_S, Map.of())); + return m.find() ? m.group(1) : null; + } + + /** Wait for the generation an update or attach starts; the old one keeps the old value. */ + private static void awaitNewGeneration(String sandbox, String before) throws IOException, InterruptedException { + long deadline = System.currentTimeMillis() + GENERATION_TIMEOUT_MS; + while (System.currentTimeMillis() < deadline) { + String now = generation(sandbox); + if (now != null && !now.equals(before)) { + return; + } + Thread.sleep(READY_POLL_MS); + } + classLogger.warn("Sandbox {} kept placeholder generation {} for {} s after a secret change", sandbox, before, + GENERATION_TIMEOUT_MS / 1000); + } + + private static String digest(String value) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable", e); + } + } + + /** Wait until the sandbox reports the provider change applied; requests hang until then. */ + private static void awaitApplied(String sandbox, String id) throws IOException, InterruptedException { + long deadline = System.currentTimeMillis() + READY_TIMEOUT_MS; + String status = ""; + while (System.currentTimeMillis() < deadline) { + status = OpenShellTransport.cli(List.of("sandbox", "provider", "status", sandbox, id), CLI_TIMEOUT_S, Map.of()); + if (status.contains(": ready")) { + return; + } + Thread.sleep(READY_POLL_MS); + } + classLogger.warn("Sandbox {} did not report secret {} applied within {} s: {}", sandbox, id, + READY_TIMEOUT_MS / 1000, status.lines().findFirst().orElse("")); + } + + /** + * Write every brokered secret's current placeholder into the sandbox, by project key. The + * placeholders are not secret; they only resolve at the secret's own hosts. + */ + private static void writePlaceholders(String sandbox) throws IOException, InterruptedException { + String env = OpenShellTransport.cli(List.of("sandbox", "exec", "-n", sandbox, "--", "env"), CLI_TIMEOUT_S, + Map.of()); + Map> byProject = new TreeMap<>(); + for (String line : env.split("\\R")) { + Matcher m = CREDENTIAL_ENV.matcher(line.trim()); + if (m.matches()) { + byProject.computeIfAbsent(m.group(2), k -> new LinkedHashMap<>()).put(m.group(1), m.group(3)); + } + } + // A directory, not a lone file: for a file the CLI takes the destination as the file's + // own name ("tar: insight: Cannot open: File exists"), while a directory lands inside it. + Path dir = Files.createTempDirectory("openshell-secrets-"); + try { + Path stage = Files.createDirectory(dir.resolve(PLACEHOLDER_DIR)); + Files.writeString(stage.resolve(PLACEHOLDER_FILE), new Gson().toJson(byProject), StandardCharsets.UTF_8); + OpenShellTransport.cli(List.of("sandbox", "upload", sandbox, stage.toString(), PLACEHOLDER_PARENT), + CLI_TIMEOUT_S, Map.of()); + } finally { + deleteQuietly(dir); + } + } + + private static boolean containsWord(String text, String word) { + return Pattern.compile("(^|\\s)" + Pattern.quote(word) + "(\\s|$)", Pattern.MULTILINE).matcher(text).find(); + } + + private static void deleteQuietly(Path dir) { + try (var files = Files.walk(dir)) { + files.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // temp file + } + }); + } catch (IOException ignored) { + // temp dir + } + } +} diff --git a/src/prerna/util/openshell/OpenShellShellRunner.java b/src/prerna/util/openshell/OpenShellShellRunner.java new file mode 100644 index 0000000000..daac612686 --- /dev/null +++ b/src/prerna/util/openshell/OpenShellShellRunner.java @@ -0,0 +1,269 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.om.ClientProcessWrapper; +import prerna.util.OpenShellTransport; +import prerna.util.Utility; + +/** + * Runs shell commands inside the user's OpenShell sandbox. + * + *

+ * Why this exists: {@code CmdExecUtil} executes commands with Apache + * {@code commons-exec} on Tomcat's own machine, inside a chroot. That is correct + * for the legacy jail and wrong for a sandbox — measured 2026-10-22, a + * {@code Command("hostname")} pixel returned the host's identity while + * {@code Py()} returned the sandbox's. The terminal was simply not part of the + * migration. + * + *

+ * The command is run with {@code openshell sandbox exec}, which starts a process + * inside the sandbox. That is deliberately not the worker socket: a shell + * command must not queue behind a running python block on the same connection, + * and shell output does not require the python worker to be healthy. + * + *

+ * The sandbox is started on demand if it is not running yet, so a terminal + * command works as the first action in a session rather than requiring a + * {@code Py()} first. + */ +public final class OpenShellShellRunner { + + private static final Logger classLogger = LogManager.getLogger(OpenShellShellRunner.class); + + private static final int DEFAULT_TIMEOUT_SECONDS = 60; + /** Most output returned to the terminal for one command. */ + private static final int MAX_OUTPUT_BYTES = 4 * 1024 * 1024; + private static final String CLI = "openshell"; + + private OpenShellShellRunner() { + // static utility + } + + /** Outcome of one shell command. */ + public static final class Result { + private final String output; + private final String workingDir; + + Result(String output, String workingDir) { + this.output = output; + this.workingDir = workingDir; + } + + public String getOutput() { + return output; + } + + public String getWorkingDir() { + return workingDir; + } + } + + /** + * Execute one command in the sandbox belonging to {@code user}. + * + * @param user the session user; used to make sure the sandbox exists + * @param workingDir directory to run in (an absolute path inside the sandbox) + * @param command the command line, as typed at the terminal + */ + public static Result execute(User user, String workingDir, String command) { + String dir = workingDir == null || workingDir.trim().isEmpty() ? "/" : workingDir.trim(); + try { + // Same on-demand rule as Py(): the first shell command in a session is + // allowed to create the sandbox. + user.getPythonSocketClient(true); + // The sandbox the user's worker is actually in: a user-scoped one is named by the + // registry, not derived from this instance's chroot folder. + ClientProcessWrapper cpw = user.getPythonClientProcessWrapper(); + String sandboxName = cpw == null ? null : cpw.getOpenShellSandboxName(); + if (sandboxName == null) { + sandboxName = OpenShellTransport.sandboxNameFor(user.getUserSymlinkHelper().getUserChrootFolder()); + } + + String trimmed = command == null ? "" : command.trim(); + if (trimmed.isEmpty()) { + return new Result("", dir); + } + if (equalsIgnoreCase(trimmed, "pwd")) { + return new Result(dir, dir); + } + if (equalsIgnoreCase(trimmed, "reset")) { + return new Result(dir, dir); + } + if (trimmed.startsWith("cd ") || trimmed.equals("cd")) { + String target = trimmed.length() > 2 ? trimmed.substring(3).trim() : "/"; + if (target.equals("~")) { + target = "/"; + } + Outcome resolved = runInSandbox(sandboxName, dir, "cd " + target + " && pwd"); + String newDir = firstLine(resolved.output); + // Only a clean exit with an absolute path is a directory change; anything else + // is an error message and the working directory stays where it was. + if (resolved.exitCode != 0 || !newDir.startsWith("/")) { + return new Result(resolved.output.isEmpty() ? "Directory doesn't exist: " + target + : resolved.output, dir); + } + return new Result(newDir, newDir); + } + // Same boundary as a python block: host-side changes go in first. + if (cpw != null) { + try { + cpw.refreshTransportedRoots(); + } catch (Exception refreshError) { + // a stale copy is better than a refused command + classLogger.warn("Refresh before shell command failed", refreshError); + } + } + String output = runInSandbox(sandboxName, dir, trimmed).output; + // Shell writes are not requests on the python socket, so they do not pass the + // fetch-back hook there. Reconcile here too -- this is the path a `git commit` + // or a `cat > file` takes, and losing that work is exactly what we are fixing. + if (cpw != null) { + try { + cpw.fetchBackChangedPaths(); + } catch (Exception reconcileError) { + // the command itself already ran; a failed reconcile must not fail the shell + classLogger.warn("Write-back after shell command failed", reconcileError); + } + } + return new Result(output, dir); + } catch (Exception e) { + return new Result("Shell execution in sandbox failed: " + e.getClass().getSimpleName() + ": " + + e.getMessage(), dir); + } + } + + private static boolean equalsIgnoreCase(String left, String right) { + return left != null && left.equalsIgnoreCase(right); + } + + private static String firstLine(String text) { + int newline = text.indexOf('\n'); + return (newline < 0 ? text : text.substring(0, newline)).trim(); + } + + /** Exit code and (capped) combined output of one sandbox command. */ + private static final class Outcome { + private final int exitCode; + private final String output; + + Outcome(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + } + + /** + * One `openshell sandbox exec` invocation. + * + *

+ * {@code bash -lc} on purpose: the command string users type is a shell line + * (pipes, redirects, &&), and a login shell gives a predictable PATH inside the + * image. The working directory is applied by the shell itself rather than by + * {@code --workdir}, because the directory may be a path that only exists in + * the sandbox after an earlier upload. + * + *

+ * Output goes to a temp file, not a pipe: a pipe nobody reads while waiting fills at + * 64 KB and blocks the command until the timeout, losing everything it printed. + */ + private static Outcome runInSandbox(String sandboxName, String workingDir, String command) throws IOException { + List argv = new ArrayList<>(Arrays.asList(cliPath(), "sandbox", "exec", "-n", sandboxName, "--", + "bash", "-lc", "cd " + shellQuote(workingDir) + " 2>/dev/null; " + command)); + Path capture = Files.createTempFile("openshell-shell-", ".out"); + try { + ProcessBuilder builder = new ProcessBuilder(argv); + builder.redirectErrorStream(true); + builder.redirectOutput(capture.toFile()); + builder.redirectInput(ProcessBuilder.Redirect.from(new File("/dev/null"))); + // The CLI needs the same environment the transport uses (gateway config, + // TLS bundle); inherit it rather than reconstructing it here. + Process process = builder.start(); + try { + if (!process.waitFor(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + process.destroyForcibly(); + return new Outcome(-1, readCapped(capture) + "\ncommand timed out after " + DEFAULT_TIMEOUT_SECONDS + + "s"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + process.destroyForcibly(); + return new Outcome(-1, "interrupted"); + } + return new Outcome(process.exitValue(), readCapped(capture)); + } finally { + Files.deleteIfExists(capture); + } + } + + /** The command's output, cut at {@link #MAX_OUTPUT_BYTES} with a note saying so. */ + private static String readCapped(Path capture) throws IOException { + long size = Files.size(capture); + byte[] bytes; + try (InputStream in = Files.newInputStream(capture)) { + bytes = in.readNBytes((int) Math.min(size, MAX_OUTPUT_BYTES)); + } + String text = new String(bytes, StandardCharsets.UTF_8).replace("\r", "").trim(); + if (size > MAX_OUTPUT_BYTES) { + text += "\n[output truncated: " + size + " bytes, showing the first " + MAX_OUTPUT_BYTES + "]"; + } + return text; + } + + private static String shellQuote(String value) { + return "'" + value.replace("'", "'\\''") + "'"; + } + + /** Exposed for tests: the exact argv shape used for one command. */ + public static List describeCommand(String sandboxName, String workingDir, String command) { + return Arrays.asList(CLI, "sandbox", "exec", "-n", sandboxName, "--", "bash", "-lc", + "cd " + shellQuote(workingDir) + " 2>/dev/null; " + command); + } + + /** Kept for symmetry with the transport's property handling. */ + static String cliPath() { + String configured = Utility.getDIHelperProperty(OpenShellTransport.CLI_PROP); + return configured == null || configured.trim().isEmpty() ? CLI : configured.trim(); + } +} diff --git a/src/prerna/util/openshell/OpenShellUserSandboxes.java b/src/prerna/util/openshell/OpenShellUserSandboxes.java new file mode 100644 index 0000000000..0cd9861063 --- /dev/null +++ b/src/prerna/util/openshell/OpenShellUserSandboxes.java @@ -0,0 +1,828 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.io.IOException; +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.AccessToken; +import prerna.auth.User; +import prerna.om.ClientProcessWrapper; +import prerna.redis.RedisConnectionConfig; +import prerna.redis.RedisConnectionFactory; +import prerna.util.Constants; +import prerna.util.OpenShellTransport; +import prerna.util.OpenShellTransport.SandboxInfo; +import prerna.util.SymlinkHelper; +import prerna.util.Utility; +import redis.clients.jedis.UnifiedJedis; +import redis.clients.jedis.params.SetParams; + +/** + * One OpenShell sandbox per user, shared by every session and server instance. + * + *

+ * The sandbox outlives any single session: its worker is persistent, so a server + * instance that dies, restarts or is simply a different node leaves the user's python + * process, its variables and its files in place, and the next request reattaches. The + * sandbox is deleted when the user's last live session ends, wherever that + * session lived. + * + *

+ * Three pieces of state, at two scopes: + *

    + *
  • Registry (shared): owner → sandbox name, plus the set of the user's + * live sessions, each with an expiry refreshed by every request (a lease equal to the + * session timeout). Redis when configured; an in-memory map otherwise, which is only + * correct for a single server instance.
    • + *
  • Local (this instance): one {@link SymlinkHelper} and one + * {@link ClientProcessWrapper} per user, so a user's sessions on this instance share a + * single connection and a single record of what was transported, instead of one forward + * and one worker connection per session (or per cookie-less API call).
    • + *
  • Sweeper (this instance, periodic): deletes sandboxes whose owner has no live + * session left, and labelled sandboxes that are no longer running.
    • + *
+ */ +public final class OpenShellUserSandboxes { + + private static final Logger classLogger = LogManager.getLogger(OpenShellUserSandboxes.class); + + /** Session attribute holding the owner a session's lease was taken under. */ + public static final String SESSION_OWNER_ATTR = "OPENSHELL_SANDBOX_OWNER"; + /** How often a session may refresh its lease; requests in between are free. */ + private static final long TOUCH_INTERVAL_MS = TimeUnit.SECONDS.toMillis(60); + /** How long a creator may hold the per-user create lock. */ + private static final long CREATE_LOCK_MS = TimeUnit.MINUTES.toMillis(5); + /** A labelled sandbox younger than this is never swept: its registry entry may not exist yet. */ + private static final Duration SWEEP_GRACE = Duration.ofMinutes(10); + private static final long SWEEP_INTERVAL_MINUTES = 5; + /** How long {@link #acquire} keeps trying: a create is bounded by the CLI's ready timeout. */ + private static final long ACQUIRE_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(240); + /** Pause between attempts while another instance creates, or the gateway does not answer. */ + private static final long ACQUIRE_POLL_MS = 1000; + /** Failed attaches to a Ready sandbox before it is replaced; one failure may be a hiccup. */ + private static final int MAX_ATTACH_ATTEMPTS = 2; + /** Session lease when the container gives no timeout (a negative interval never expires). */ + private static final long DEFAULT_LEASE_MS = TimeUnit.DAYS.toMillis(1); + private static final DateTimeFormatter CREATED_AT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + /** Distinguishes this JVM's leases from ones a restored session carries over a restart. */ + private static final String BOOT_ID = UUID.randomUUID().toString(); + private static final String NODE = nodeName(); + + private static final Map LOCALS = new ConcurrentHashMap<>(); + private static final Map> LOCAL_SESSIONS = new ConcurrentHashMap<>(); + private static final Object LOCALS_LOCK = new Object(); + /** + * Serialises, per user, the changes to which of their sessions live here and the release + * of their connection, so a session ending cannot tear down a wrapper another session + * of theirs just started using. Always taken before a {@link Local}'s monitor. + */ + private static final Map OWNER_LOCKS = new ConcurrentHashMap<>(); + private static volatile Registry registry; + private static volatile ScheduledExecutorService sweeper; + + private OpenShellUserSandboxes() { + // static utility + } + + /** User-scoped sandboxes apply exactly when the OpenShell jail does. */ + public static boolean enabled() { + return Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE)) + && Constants.SANDBOX_MODE_OPENSHELL.equalsIgnoreCase( + String.valueOf(Utility.getDIHelperProperty(Constants.SANDBOX_MODE)).trim()); + } + + /** + * Whether this user's python lives in a user-scoped sandbox: the jail is OpenShell and + * the user has a primary login to scope it by. Anyone else keeps the per-session form. + */ + public static boolean appliesTo(User user) { + if (user == null || !enabled()) { + return false; + } + AccessToken token = user.getPrimaryLoginToken(); + return token != null && token.getId() != null; + } + + /** + * The owner a session's sandbox lease was taken under, for cleanup when the session is + * destroyed. Read from the session first: logging out of a provider drops its token + * before the session is invalidated, so by then the user may have no login to derive + * the owner from, and the lease would never be released. + * + * @param recorded the session's {@link #SESSION_OWNER_ATTR} value, or null + * @return the owner, or null when this session has no user-scoped sandbox + */ + public static String ownerForSession(User user, Object recorded) { + if (!enabled()) { + return null; + } + if (recorded instanceof String owner && !owner.isEmpty()) { + return owner; + } + return appliesTo(user) ? ownerOf(user) : null; + } + + /** + * The digest a user's sandbox is filed under: provider and id of the primary login. + * Used as the registry key and the {@code semoss-owner} label, so neither carries the + * raw user id. + */ + public static String ownerOf(User user) { + AccessToken token = user == null ? null : user.getPrimaryLoginToken(); + if (token == null || token.getId() == null) { + throw new IllegalStateException("cannot scope a sandbox to a user with no primary login"); + } + return digest(token.getProvider().name() + ":" + token.getId(), 24); + } + + // ------------------------------------------------------------------ this instance + + /** What this instance holds for one user, shared by all of their sessions here. */ + public static final class Local { + private final SymlinkHelper helper; + private final ClientProcessWrapper cpw = new ClientProcessWrapper(); + /** Set under this object's monitor when the instance gives the user up. */ + private boolean released; + + private Local(SymlinkHelper helper) { + this.helper = helper; + } + + public SymlinkHelper getHelper() { + return helper; + } + + public ClientProcessWrapper getWrapper() { + return cpw; + } + + /** Whether this was released; read under this object's monitor. */ + public boolean isReleased() { + return released; + } + } + + /** + * The user's shared helper and wrapper on this instance, created on first use. The + * chroot folder is per instance, so its name keeps the random suffix. + */ + public static Local local(User user) { + String owner = ownerOf(user); + Local local = LOCALS.get(owner); + if (local != null) { + return local; + } + synchronized (LOCALS_LOCK) { + local = LOCALS.get(owner); + if (local == null) { + String chroot = Utility.getDIHelperProperty(Constants.CHROOT_DIR) + "/" + User.getSingleLogginName(user) + + "-" + UUID.randomUUID(); + SymlinkHelper helper = new SymlinkHelper(chroot); + local = new Local(helper); + // Published before linking the asset folder: opening the asset project can + // ask the user for this helper again, and must get this one. + LOCALS.put(owner, local); + try { + helper.symlinkUserAsset(user); + } catch (Exception e) { + classLogger.warn("Unable to symlink user asset folder into chroot", e); + } + ensureSweeper(); + } + return local; + } + } + + // ------------------------------------------------------------------ attach or create + + /** A transport connected to the user's sandbox, and whether that sandbox already existed. */ + public static final class Attached { + public final OpenShellTransport transport; + public final int port; + public final boolean reattached; + + private Attached(OpenShellTransport transport, int port, boolean reattached) { + this.transport = transport; + this.port = port; + this.reattached = reattached; + } + } + + /** + * Connect to the user's sandbox, creating it if they have none that is running. + * + *

+ * Creation is serialised per user across instances with a registry lock, so two + * instances that both find nothing do not both create; the loser waits for the + * winner's entry and attaches to it. + */ + public static Attached acquire(User user) throws Exception { + String owner = ownerOf(user); + Registry store = registry(); + long deadline = System.currentTimeMillis() + ACQUIRE_TIMEOUT_MS; + int attachFailures = 0; + while (System.currentTimeMillis() < deadline) { + String name = store.sandboxFor(owner); + if (name != null) { + String phase; + try { + phase = OpenShellTransport.phaseOf(name); + } catch (IOException e) { + // Unknown is not gone: deleting here would destroy the user's python over a + // gateway hiccup. + classLogger.warn("Could not read the state of sandbox {}; retrying", name, e); + Thread.sleep(ACQUIRE_POLL_MS); + continue; + } + if ("Ready".equalsIgnoreCase(phase)) { + OpenShellTransport transport = OpenShellTransport.forUser(name, owner); + try { + return new Attached(transport, transport.attach(), true); + } catch (Exception e) { + if (++attachFailures < MAX_ATTACH_ATTEMPTS) { + classLogger.warn("Could not attach to sandbox {}; retrying", name, e); + Thread.sleep(ACQUIRE_POLL_MS); + continue; + } + classLogger.warn("Could not attach to sandbox {}; replacing it", name, e); + } + } else { + classLogger.info("Sandbox {} for this user is {}; replacing it", name, + phase == null ? "gone" : phase); + } + OpenShellTransport.deleteSandbox(name); + store.forget(owner, name); + attachFailures = 0; + continue; + } + String lockToken = store.tryLock(owner); + if (lockToken != null) { + try { + if (store.sandboxFor(owner) != null) { + continue; + } + // No entry, but the gateway may still run the user's sandbox: the in-memory + // registry after a restart, or Redis flushed. The owner label finds it. + String labelled = runningSandboxLabelled(owner); + if (labelled != null) { + classLogger.info("Recovered sandbox {} for user {} from its owner label", labelled, owner); + store.bind(owner, labelled); + continue; + } + name = OpenShellTransport.sandboxNameFor(owner + ":" + UUID.randomUUID()); + OpenShellTransport transport = OpenShellTransport.forUser(name, owner); + int port = transport.start(); + store.bind(owner, name); + classLogger.info("Created sandbox {} for user {} on {}", name, owner, NODE); + return new Attached(transport, port, false); + } finally { + store.unlock(owner, lockToken); + } + } + // Another instance is creating it; its entry appears when the sandbox is Ready. + Thread.sleep(ACQUIRE_POLL_MS); + } + throw new IllegalStateException("timed out waiting for the user's sandbox"); + } + + /** A {@code Ready} sandbox carrying this owner's label, or null. */ + private static String runningSandboxLabelled(String owner) { + try { + for (SandboxInfo box : OpenShellTransport.listExecutorSandboxes()) { + if (owner.equals(box.owner()) && box.isReady()) { + return box.name(); + } + } + } catch (Exception e) { + classLogger.warn("Could not list sandboxes to recover one by label", e); + } + return null; + } + + // ------------------------------------------------------------------ session leases + + /** + * Record that a session of this user is alive, for as long as the session timeout. + * Called on every authenticated request; refreshes at most once a minute per session. + * + * @param lastTouch the session's previous touch marker, or null + * @param ttlSeconds the session's max inactive interval + * @return the marker to store on the session, or null when nothing was refreshed + */ + public static String touch(User user, String sessionId, String lastTouch, int ttlSeconds) { + if (sessionId == null || !appliesTo(user) || !user.isLoggedIn()) { + return null; + } + String owner = ownerOf(user); + // Every request, not only refreshing ones: this set is what keeps the user's + // connection here from being released while a session of theirs is still active. + synchronized (ownerLock(owner)) { + LOCAL_SESSIONS.computeIfAbsent(owner, k -> ConcurrentHashMap.newKeySet()).add(sessionId); + } + long now = System.currentTimeMillis(); + if (lastTouch != null && lastTouch.startsWith(BOOT_ID + ":")) { + try { + if (now - Long.parseLong(lastTouch.substring(BOOT_ID.length() + 1)) < TOUCH_INTERVAL_MS) { + return null; + } + } catch (NumberFormatException ignored) { + // refresh below + } + } + try { + long ttlMs = ttlSeconds > 0 ? TimeUnit.SECONDS.toMillis(ttlSeconds) : DEFAULT_LEASE_MS; + registry().touch(owner, sessionId, now + ttlMs); + ensureSweeper(); + } catch (Exception e) { + classLogger.warn("Could not refresh the sandbox lease for session {}", sessionId, e); + return null; + } + return BOOT_ID + ":" + now; + } + + /** + * A session left this instance. Releases this instance's connection when it was the + * user's last session here. + * + *

+ * Only a session that is really over -- an explicit logout, or idle past its timeout -- + * gives up its lease, and the sandbox goes when that was the user's last lease anywhere. + * A server stopping also destroys every session (measured 2026-10-23: a Tomcat restart + * fired {@code sessionDestroyed} for each one and deleted the sandbox), but the user has + * not left: their lease stands, so the restarted or another instance reattaches. + * + * @param owner the owner recorded on the session ({@link #ownerForSession}); a logout + * may already have dropped the login it was derived from + * @param over true for a logout or a timeout; false when the server is going away + */ + public static void sessionEnded(String owner, String sessionId, boolean over) { + if (owner == null) { + return; + } + synchronized (ownerLock(owner)) { + Set here = LOCAL_SESSIONS.get(owner); + if (here != null) { + here.remove(sessionId); + } + if (here == null || here.isEmpty()) { + LOCAL_SESSIONS.remove(owner); + releaseLocal(owner); + } + } + if (!over) { + classLogger.info("Session {} left with the server; keeping its sandbox lease", sessionId); + return; + } + boolean anyLive; + try { + anyLive = registry().release(owner, sessionId, System.currentTimeMillis()); + } catch (Exception e) { + classLogger.warn("Could not release the sandbox lease for session {}; the sweeper will", sessionId, e); + return; + } + if (!anyLive) { + deleteFor(owner, "last session ended"); + } + } + + /** Caller holds {@link #ownerLock}. */ + private static void releaseLocal(String owner) { + Local local = LOCALS.remove(owner); + if (local == null) { + return; + } + // The same monitor User takes while it binds and starts this wrapper, so a release + // never runs underneath a request that is connecting. + synchronized (local) { + local.released = true; + try { + // Write back, close this instance's connection, stop the forward. The sandbox + // itself is not touched here. + local.cpw.shutdown(true); + } catch (Exception e) { + classLogger.warn("Failed to release the sandbox connection for {}", owner, e); + } + local.helper.removeChrootFolder(); + } + } + + private static Object ownerLock(String owner) { + return OWNER_LOCKS.computeIfAbsent(owner, k -> new Object()); + } + + private static void deleteFor(String owner, String reason) { + Registry store = registry(); + String name = store.sandboxFor(owner); + if (name != null) { + classLogger.info("Deleting sandbox {} ({})", name, reason); + OpenShellTransport.deleteSandbox(name); + store.forget(owner, name); + } + store.drop(owner); + } + + // ------------------------------------------------------------------ sweeper + + private static void ensureSweeper() { + if (sweeper != null) { + return; + } + synchronized (OpenShellUserSandboxes.class) { + if (sweeper == null) { + ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "openshell-sandbox-sweeper"); + t.setDaemon(true); + return t; + }); + service.scheduleWithFixedDelay(OpenShellUserSandboxes::sweepQuietly, 1, SWEEP_INTERVAL_MINUTES, + TimeUnit.MINUTES); + sweeper = service; + } + } + } + + private static void sweepQuietly() { + try { + sweep(); + } catch (Throwable t) { + classLogger.warn("Sandbox sweep failed", t); + } + } + + /** + * Delete what nobody can use any more. + * + *

    + *
  1. A user with no live session anywhere: their sandbox (a session lost with its + * server, so its logout never ran).
    2. + *
  2. A labelled sandbox that is not running (its worker exited after idling, or it + * was a session-scoped sandbox orphaned by a restart).
    3. + *
  3. With a shared registry only: a running user sandbox no entry points at.
    4. + *
+ */ + public static void sweep() throws Exception { + Registry store = registry(); + long now = System.currentTimeMillis(); + for (String owner : store.owners()) { + if (!store.anyLive(owner, now)) { + synchronized (ownerLock(owner)) { + LOCAL_SESSIONS.remove(owner); + releaseLocal(owner); + } + deleteFor(owner, "no live session for this user"); + } + } + LocalDateTime cutoff = LocalDateTime.now(ZoneOffset.UTC).minus(SWEEP_GRACE); + int unreadable = 0; + for (SandboxInfo box : OpenShellTransport.listExecutorSandboxes()) { + LocalDateTime created; + try { + created = LocalDateTime.parse(box.createdAt(), CREATED_AT); + } catch (DateTimeParseException e) { + unreadable++; + continue; + } + if (created.isAfter(cutoff)) { + continue; + } + if (!box.isReady()) { + classLogger.info("Sweeping sandbox {} (phase {})", box.name(), box.phase()); + OpenShellTransport.deleteSandbox(box.name()); + if (box.owner() != null) { + store.forget(box.owner(), box.name()); + } + } else if (box.owner() != null && store.shared() && !box.name().equals(store.sandboxFor(box.owner()))) { + classLogger.info("Sweeping sandbox {} (no registry entry points at it)", box.name()); + OpenShellTransport.deleteSandbox(box.name()); + } + } + if (unreadable > 0) { + // Said out loud: a changed CLI date format would otherwise silently stop all sweeping. + classLogger.warn("Skipped {} sandbox(es) whose creation time could not be read", unreadable); + } + } + + // ------------------------------------------------------------------ registry + + private static Registry registry() { + Registry current = registry; + if (current == null) { + synchronized (OpenShellUserSandboxes.class) { + if (registry == null) { + RedisConnectionConfig config = RedisConnectionConfig.fromDIHelper(); + registry = config == null ? new LocalRegistry() : new RedisRegistry(config); + classLogger.info("OpenShell sandbox registry: {}", registry.shared() ? "redis" : "in-memory"); + } + current = registry; + } + } + return current; + } + + /** Owner → sandbox, owner → live sessions with expiry, and a create lock. */ + interface Registry { + boolean shared(); + + String sandboxFor(String owner); + + void bind(String owner, String sandboxName); + + /** Remove the entry only if it still names this sandbox. */ + void forget(String owner, String sandboxName); + + /** Take the per-user create lock; the token that holds it, or null when another does. */ + String tryLock(String owner); + + /** Release the create lock, only if this token still holds it. */ + void unlock(String owner, String token); + + void touch(String owner, String sessionId, long expiresAtMs); + + /** Remove a session; true when the user still has a live session. */ + boolean release(String owner, String sessionId, long nowMs); + + boolean anyLive(String owner, long nowMs); + + Set owners(); + + /** Forget the user entirely: sessions and index. */ + void drop(String owner); + } + + /** + * The shared registry. Laid out like the room coordination keys + * ({@code RoomMessageRedisClient}): every key of one user carries the + * {@code {owner}} hash tag, so they share a Redis Cluster slot and a script can + * touch them together; the lock is {@code SET NX PX} with a random token, released + * only by its holder. + */ + static final class RedisRegistry implements Registry { + private static final String UNLOCK_SCRIPT = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end"; + private static final String FORGET_SCRIPT = "if redis.call('hget', KEYS[1], 'name') == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end"; + /** Forget a user with no session and no sandbox left; 1 when it did. */ + private static final String DROP_SCRIPT = "if redis.call('zcard', KEYS[1]) == 0 and redis.call('exists', KEYS[2]) == 0 then return redis.call('del', KEYS[1]) + 1 else return 0 end"; + /** Every user with an entry or a lease; a separate slot, so kept apart from the scripts. */ + private static final String OWNERS_KEY = "openshell:owners"; + + private final RedisConnectionConfig config; + + RedisRegistry(RedisConnectionConfig config) { + this.config = config; + } + + @Override + public boolean shared() { + return true; + } + + @Override + public String sandboxFor(String owner) { + return withJedis(jedis -> jedis.hget(sandboxKey(owner), "name")); + } + + @Override + public void bind(String owner, String sandboxName) { + Map record = new HashMap<>(); + record.put("name", sandboxName); + record.put("node", NODE); + record.put("createdAt", String.valueOf(System.currentTimeMillis())); + record.put("workerPort", String.valueOf(Utility.getDIHelperProperty(OpenShellTransport.WORKER_PORT_PROP))); + withJedis(jedis -> { + jedis.hset(sandboxKey(owner), record); + return jedis.sadd(OWNERS_KEY, owner); + }); + } + + @Override + public void forget(String owner, String sandboxName) { + withJedis(jedis -> jedis.eval(FORGET_SCRIPT, Collections.singletonList(sandboxKey(owner)), + Collections.singletonList(sandboxName))); + } + + @Override + public String tryLock(String owner) { + String token = UUID.randomUUID().toString(); + boolean held = withJedis(jedis -> "OK" + .equals(jedis.set(lockKey(owner), token, SetParams.setParams().nx().px(CREATE_LOCK_MS)))); + return held ? token : null; + } + + @Override + public void unlock(String owner, String token) { + withJedis(jedis -> jedis.eval(UNLOCK_SCRIPT, Collections.singletonList(lockKey(owner)), + Collections.singletonList(token))); + } + + @Override + public void touch(String owner, String sessionId, long expiresAtMs) { + withJedis(jedis -> { + jedis.zadd(sessionsKey(owner), expiresAtMs, sessionId); + return jedis.sadd(OWNERS_KEY, owner); + }); + } + + @Override + public boolean release(String owner, String sessionId, long nowMs) { + withJedis(jedis -> jedis.zrem(sessionsKey(owner), sessionId)); + return anyLive(owner, nowMs); + } + + @Override + public boolean anyLive(String owner, long nowMs) { + return withJedis(jedis -> { + jedis.zremrangeByScore(sessionsKey(owner), Double.NEGATIVE_INFINITY, nowMs); + return jedis.zcard(sessionsKey(owner)) > 0; + }); + } + + @Override + public Set owners() { + return new HashSet<>(withJedis(jedis -> jedis.smembers(OWNERS_KEY))); + } + + @Override + public void drop(String owner) { + Object dropped = withJedis(jedis -> jedis.eval(DROP_SCRIPT, List.of(sessionsKey(owner), sandboxKey(owner)), + Collections.emptyList())); + if (dropped instanceof Long count && count > 0) { + // A touch racing this re-adds the owner, so the index cannot lose a live user. + withJedis(jedis -> jedis.srem(OWNERS_KEY, owner)); + } + } + + private T withJedis(JedisCallback callback) { + // UnifiedJedis is thread-safe and internally pooled; it is shared and must + // NOT be closed per call (that would tear down the whole client). + try { + return callback.apply(RedisConnectionFactory.getClient(config)); + } catch (Exception e) { + throw new IllegalStateException("Redis sandbox-registry command failed: " + e.getMessage(), e); + } + } + + // The {owner} hash tag co-locates all of a user's keys on a single Redis Cluster + // hash slot, so multi-key EVAL operations never trigger CROSSSLOT. It is inert on + // standalone/Sentinel deployments. + private static String sandboxKey(String owner) { + return "openshell:{" + owner + "}:sandbox"; + } + + private static String sessionsKey(String owner) { + return "openshell:{" + owner + "}:sessions"; + } + + private static String lockKey(String owner) { + return "openshell:{" + owner + "}:lock"; + } + + private interface JedisCallback { + T apply(UnifiedJedis jedis); + } + } + + /** Single-instance fallback: correct only while one server owns every session. */ + static final class LocalRegistry implements Registry { + private final Map sandboxes = new ConcurrentHashMap<>(); + private final Map> sessions = new ConcurrentHashMap<>(); + private final Map locks = new ConcurrentHashMap<>(); + + @Override + public boolean shared() { + return false; + } + + @Override + public String sandboxFor(String owner) { + return sandboxes.get(owner); + } + + @Override + public void bind(String owner, String sandboxName) { + sandboxes.put(owner, sandboxName); + } + + @Override + public void forget(String owner, String sandboxName) { + sandboxes.remove(owner, sandboxName); + } + + @Override + public String tryLock(String owner) { + String token = UUID.randomUUID().toString(); + return locks.putIfAbsent(owner, token) == null ? token : null; + } + + @Override + public void unlock(String owner, String token) { + locks.remove(owner, token); + } + + @Override + public void touch(String owner, String sessionId, long expiresAtMs) { + sessions.computeIfAbsent(owner, k -> new ConcurrentHashMap<>()).put(sessionId, expiresAtMs); + } + + @Override + public boolean release(String owner, String sessionId, long nowMs) { + Map live = sessions.get(owner); + if (live != null) { + live.remove(sessionId); + } + return anyLive(owner, nowMs); + } + + @Override + public boolean anyLive(String owner, long nowMs) { + Map live = sessions.get(owner); + if (live == null) { + return false; + } + live.values().removeIf(expiry -> expiry <= nowMs); + return !live.isEmpty(); + } + + @Override + public Set owners() { + Set all = new HashSet<>(sandboxes.keySet()); + all.addAll(sessions.keySet()); + return all; + } + + @Override + public void drop(String owner) { + Map live = sessions.get(owner); + if ((live == null || live.isEmpty()) && !sandboxes.containsKey(owner)) { + sessions.remove(owner); + } + } + } + + // ------------------------------------------------------------------ helpers + + private static String digest(String value, int hexChars) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(hash).substring(0, hexChars); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable", e); + } + } + + private static String nodeName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (Exception e) { + return "unknown"; + } + } +} diff --git a/src/prerna/util/openshell/TransportedRoots.java b/src/prerna/util/openshell/TransportedRoots.java new file mode 100644 index 0000000000..86b3d2b48c --- /dev/null +++ b/src/prerna/util/openshell/TransportedRoots.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import prerna.util.Constants; + +/** + * Which space a transported root belongs to, read from its normalized absolute path. + * + *

+ * Only user spaces -- projects, rooms and user assets -- are written by user code, so + * only they are synced between the host and a sandbox. Engine and runtime trees are + * uploaded once and never pulled back. + */ +public final class TransportedRoots { + + private static final String PROJECT_MARKER = "/" + Constants.PROJECT_FOLDER + "/"; + private static final String ROOM_MARKER = "/" + Constants.ROOM_FOLDER + "/"; + private static final String USER_MARKER = "/" + Constants.USER_FOLDER + "/"; + + private TransportedRoots() { + // static utility + } + + /** A project, room or user-asset root: the spaces user code writes and sync covers. */ + public static boolean isUserSpace(String normalized) { + return isProject(normalized) || isRoom(normalized) || isUserAsset(normalized); + } + + public static boolean isProject(String normalized) { + return normalized.contains(PROJECT_MARKER); + } + + public static boolean isRoom(String normalized) { + return normalized.contains(ROOM_MARKER); + } + + public static boolean isUserAsset(String normalized) { + return normalized.contains(USER_MARKER); + } + + /** The project id of a root under {@code project/__/}, or null outside a project. */ + public static String projectIdOf(String normalized) { + return idOf(folderAfter(normalized, PROJECT_MARKER)); + } + + /** The asset project id of a root under {@code user/Asset__/}, or null outside one. */ + public static String userAssetIdOf(String normalized) { + return idOf(folderAfter(normalized, USER_MARKER)); + } + + /** {@code __} folders carry the id after the last separator. */ + private static String idOf(String folder) { + if (folder == null) { + return null; + } + int separator = folder.lastIndexOf("__"); + return separator < 0 ? folder : folder.substring(separator + 2); + } + + /** The room id of a root under {@code room//}, or null outside a room. */ + public static String roomIdOf(String normalized) { + return folderAfter(normalized, ROOM_MARKER); + } + + private static String folderAfter(String normalized, String marker) { + int at = normalized.indexOf(marker); + if (at < 0) { + return null; + } + String rest = normalized.substring(at + marker.length()); + int slash = rest.indexOf('/'); + return slash < 0 ? rest : rest.substring(0, slash); + } +} diff --git a/src/prerna/util/openshell/sandbox-policy.yaml b/src/prerna/util/openshell/sandbox-policy.yaml new file mode 100644 index 0000000000..445d4d0f6a --- /dev/null +++ b/src/prerna/util/openshell/sandbox-policy.yaml @@ -0,0 +1,40 @@ +version: 1 +# SEMOSS sandbox policy, the default when OPENSHELL_POLICY is unset. OpenShellTransport fills in +# ${SEMOSS_HOME_ROOT} the top folder of BaseFolder's path (/Users for /Users/me/...); files +# cross into the sandbox at their host path, and the home is created +# under this root when the sandbox starts +# ${PYTHON} the interpreter's real path (OPENSHELL_PYTHON_BINARY), which egress rules name +# ${NETWORK_POLICIES} the network_policies section, generated from OPENSHELL_EGRESS +# and passes the result to `openshell sandbox create --policy`. A custom OPENSHELL_POLICY file may +# use the same placeholders. +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /etc + - /bin + - /sbin + # process inspection (ps, /proc scans, pid-based kill) fails without it + - /proc + - /dev/urandom + read_write: + - /sandbox + - /tmp + # the image's own home: the worker (py/) and its insight folder + - /opt/semosshome + # where the platform's home is made, for projects, rooms and user assets; on a server + # the home is /opt/semosshome itself and the line is dropped + - ${SEMOSS_HOME_ROOT} + # redirects fail unless it is listed + - /dev/null + # POSIX semaphores and shared memory: multiprocessing, joblib (scikit-learn n_jobs) and + # torch data loaders fail with PermissionError on SemLock without it + - /dev/shm +landlock: + compatibility: hard_requirement + +# The only egress the workload gets: OPENSHELL_EGRESS (default pypi, so pip works), granted to +# the interpreter. pip is a script, so a rule names the INTERPRETER: the kernel-reported +# executable, not a symlink to it. +${NETWORK_POLICIES} diff --git a/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java b/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java new file mode 100644 index 0000000000..a8f8cdd48b --- /dev/null +++ b/test/prerna/util/openshell/OpenShellSessionLifecycleHarness.java @@ -0,0 +1,256 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.openshell; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +import prerna.om.ClientProcessWrapper; +import prerna.tcp.PayloadStruct; +import prerna.tcp.client.SocketClient; +import prerna.util.Constants; +import prerna.util.DIHelper; +import prerna.util.OpenShellTransport; +import prerna.util.SymlinkHelper; +import prerna.util.Utility; + +/** + * Proves the sandbox lifecycle that login/logout is supposed to give us, + * without booting Tomcat. + * + *

+ * The claim being tested is not "a sandbox can be created" — the transport + * harness already shows that. It is the session-shaped claim: + * + *

    + *
  1. nothing exists while the user has not run any code;
    2. + *
  2. the first user-authored python request creates the sandbox on demand;
    3. + *
  3. {@code Py()} actually executes inside it (real worker, not a stock image);
    4. + *
  4. the exact call made on logout — {@code ClientProcessWrapper.shutdown(true)} + * from {@code UserSessionLoader.sessionDestroyed} — deletes it.
    5. + *
+ * + *

+ * Step 4 is the one worth automating: it is a single call buried in a servlet + * listener, and if the wrapper forgets its transport, logout silently leaks a + * sandbox. + * + *

+ * Run with an RDF map whose {@code SANDBOX_MODE} is {@code OPENSHELL}: + * + * 

+ * java -DRDF_MAP=/tmp/RDF_Map.openshell-harness.prop -cp ... \
+ *      prerna.util.openshell.OpenShellSessionLifecycleHarness
+ *
+ */ +public final class OpenShellSessionLifecycleHarness { + + private static int passed = 0; + private static int failed = 0; + + public static void main(String[] args) throws Exception { + String rdf = System.getProperty("RDF_MAP"); + if (rdf == null || rdf.trim().isEmpty()) { + System.err.println("Set -DRDF_MAP="); + System.exit(2); + } + DIHelper.getInstance().loadCoreProp(rdf); + System.out.println("RDF: " + rdf); + String mode = Utility.getDIHelperProperty(Constants.SANDBOX_MODE); + String image = Utility.getDIHelperProperty(OpenShellTransport.IMAGE_PROP); + String policy = Utility.getDIHelperProperty(OpenShellTransport.POLICY_PROP); + System.out.println("SANDBOX_MODE=" + mode + " image=" + image + " policy=" + policy); + check("P-0 SANDBOX_MODE is OPENSHELL", "OPENSHELL".equalsIgnoreCase(mode), String.valueOf(mode)); + check("P-1 image is configured or defaulted", true, image == null || image.trim().isEmpty() ? "default" : image); + // unset means the policy bundled with SEMOSS, rendered for this home at create time + boolean bundled = OpenShellTransport.class.getResource("/prerna/util/openshell/sandbox-policy.yaml") != null; + check("P-2 policy is configured or bundled", (policy != null && !policy.trim().isEmpty()) || bundled, + policy == null || policy.trim().isEmpty() ? "bundled=" + bundled : policy); + check("P-3 CHROOT_ENABLE is on", Boolean.parseBoolean(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE)), + String.valueOf(Utility.getDIHelperProperty(Constants.CHROOT_ENABLE))); + if (failed > 0) { + System.out.println("aborting: the properties above decide whether this test means anything"); + System.exit(1); + } + + String chrootRoot = Utility.getDIHelperProperty(Constants.CHROOT_DIR); + String chrootPath = chrootRoot + File.separator + "lifecycle-" + UUID.randomUUID().toString().substring(0, 8); + Files.createDirectories(Paths.get(chrootPath)); + String sandboxName = OpenShellTransport.sandboxNameFor(chrootPath); + System.out.println("chroot: " + chrootPath); + System.out.println("sandbox: " + sandboxName); + + ClientProcessWrapper cpw = new ClientProcessWrapper(); + try { + check("L-1 no sandbox before first use", !sandboxExists(sandboxName), sandboxName); + + // This is the same call the first Py() makes: User.getPythonSocketClient(true) + // -> ClientProcessWrapper.createProcessAndClient(...) -> OpenShellTransport.start(). + System.out.println("\nstarting on demand (this creates the sandbox)..."); + long started = System.currentTimeMillis(); + SymlinkHelper helper = new SymlinkHelper(chrootPath); + check("L-2 helper selected the external sandbox", helper.isExternalSandbox(), helper.getSandboxMode().name()); + cpw.createProcessAndClient(true, helper, -1, null, null, null, false, "30", "INFO", null); + System.out.println(" up in " + (System.currentTimeMillis() - started) + " ms"); + check("L-3 sandbox exists after start", sandboxExists(sandboxName), sandboxName); + + // The real point: user-authored python runs in the sandbox, through the + // shipping client, not a probe socket. + SocketClient client = cpw.getSocketClient(); + check("L-4 socket client connected", client != null && client.isConnected(), "client=" + client); + if (client != null && client.isConnected()) { + PayloadStruct ps = new PayloadStruct(); + ps.operation = PayloadStruct.OPERATION.PYTHON; + ps.methodName = "lifecycleHarness"; + ps.payload = new Object[] { "import os,sys; print('LIFECYCLE_OK', os.getpid(), sys.version.split()[0])" }; + ps.payloadClasses = new Class[] { String.class }; + ps.longRunning = true; + ps.response = false; + ps.interim = false; + ps.insightId = "lifecycle-harness"; + ps.executionInsightId = "lifecycle-harness"; + ps.sessionId = "lifecycle-harness"; + ps.jobId = "lifecycle-harness"; + Object reply = client.executeCommand(ps); + String text; + if (reply instanceof PayloadStruct) { + PayloadStruct replyStruct = (PayloadStruct) reply; + text = java.util.Arrays.deepToString(replyStruct.payload); + System.out.println(" response=" + replyStruct.response + " payload=" + text); + } else { + text = String.valueOf(reply); + System.out.println(" reply=" + text); + } + check("L-5 Py() executed inside the sandbox", text.contains("LIFECYCLE_OK"), abbreviate(text)); + } + + // Exactly what UserSessionLoader.sessionDestroyed does for the session user. + System.out.println("\nshutting down as logout does (cpw.shutdown(true))..."); + long stopped = System.currentTimeMillis(); + cpw.shutdown(true); + System.out.println(" down in " + (System.currentTimeMillis() - stopped) + " ms"); + Thread.sleep(2000); + check("L-6 sandbox deleted on logout", !sandboxExists(sandboxName), sandboxName); + + // The second half of the on-demand claim: after teardown the same wrapper + // can start a fresh sandbox, so a new session does not inherit a dead one. + System.out.println("\nstarting again (second session)..."); + cpw.createProcessAndClient(true, new SymlinkHelper(chrootPath), -1, null, null, null, false, "30", "INFO", + null); + check("L-7 a later session starts a fresh sandbox", sandboxExists(sandboxName), sandboxName); + } catch (Exception e) { + check("L-0 lifecycle completed without exception", false, e.getClass().getSimpleName() + ": " + e.getMessage()); + e.printStackTrace(); + } finally { + try { + cpw.shutdown(true); + } catch (Exception ignored) { + // best effort + } + Thread.sleep(1000); + if (sandboxExists(sandboxName)) { + System.out.println("cleaning up leftover sandbox " + sandboxName); + deleteSandbox(sandboxName); + } + deleteRecursively(Paths.get(chrootPath)); + } + + int total = passed + failed; + System.out.println("\n=== " + (failed == 0 ? "ALL PASS" : "FAILURES") + " (" + passed + "/" + total + + " checks passed) ==="); + if (failed > 0) { + System.exit(1); + } + } + + // ------------------------------------------------------------------ + // helpers + // ------------------------------------------------------------------ + + private static boolean sandboxExists(String name) { + String output = run("openshell", "sandbox", "list"); + for (String line : output.split("\\R")) { + String trimmed = line.trim(); + if (trimmed.startsWith(name + " ") || trimmed.equals(name)) { + return true; + } + } + return false; + } + + private static void deleteSandbox(String name) { + run("openshell", "sandbox", "delete", name); + } + + private static String run(String... command) { + try { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + return output; + } catch (Exception e) { + return ""; + } + } + + private static void deleteRecursively(Path root) { + if (!Files.exists(root)) { + return; + } + try (var stream = Files.walk(root)) { + stream.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (Exception ignored) { + // best effort + } + }); + } catch (Exception ignored) { + // best effort + } + } + + private static String abbreviate(String text) { + if (text == null) { + return ""; + } + return text.length() <= 120 ? text : text.substring(0, 117) + "..."; + } + + private static void check(String name, boolean ok, String detail) { + if (ok) { + passed++; + } else { + failed++; + } + System.out.printf(" %s %-48s %s%n", ok ? "[PASS]" : "[FAIL]", name, detail == null ? "" : detail); + } +}