feat: openshell user sandbox

.github/workflows/sandbox-builder.yml

@@ -0,0 +1,144 @@
+name: Sandbox Builder
+# Builds docker/Dockerfile.sandbox: the OpenShell sandbox image that runs a user's python
+# worker. It is the python-builder image plus this repo's py/, so it rebuilds when py/ changes
+# and after every successful Python Builder run (new interpreter or dependencies).
+on:
+  workflow_dispatch:
+  push:
+    branches: ["dev", "main"]
+    paths:
+      - "py/**"
+      - "docker/Dockerfile.sandbox"
+      - "docker/Dockerfile.sandbox.dockerignore"
+      - "docker/sandbox-smoke-test.sh"
+      - ".github/workflows/sandbox-builder.yml"
+  workflow_run:
+    workflows: ["Python Builder"]
+    types: [completed]
+    branches: ["dev", "main"]
+
+concurrency:
+  group: sandbox-builder-${{ github.event.workflow_run.head_branch || github.ref_name }}
+  cancel-in-progress: true
+
+env:
+  IMAGE: quay.io/semoss/semoss-sandbox
+  # must match the COPY --from tag in docker/Dockerfile.sandbox
+  PYTHON_BUILDER_TAG: "3.14-cpu"
+  BRANCH: ${{ github.event.workflow_run.head_branch || github.ref_name }}
+  SOURCE_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Build each architecture natively, push BY DIGEST (no tag), then smoke-test
+  # that digest. A failed smoke test never reaches the tagging job.
+  # ---------------------------------------------------------------------------
+  build:
+    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            arch: amd64
+            runner_prefix: codebuild-semoss-github-runner
+            container: quay.io/semoss/test-quay:ubuntu-dind
+          - platform: linux/arm64
+            arch: arm64
+            runner_prefix: codebuild-semoss-github-runner-arm64
+            container: quay.io/semoss/test-quay:ubuntu-dind-arm64
+    runs-on: ${{ matrix.runner_prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
+    container:
+      image: ${{ matrix.container }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.SOURCE_SHA }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Quay
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_ROBOT_USERNAME }}
+          password: ${{ secrets.QUAY_ROBOT_PASSWORD }}
+
+      - name: Build and push by digest (${{ matrix.platform }})
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          # context is py/; the Dockerfile and its .dockerignore live in docker/
+          context: ./py
+          file: ./docker/Dockerfile.sandbox
+          platforms: ${{ matrix.platform }}
+          pull: true
+          labels: |
+            org.opencontainers.image.revision=${{ env.SOURCE_SHA }}
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.semoss.python-builder=${{ env.PYTHON_BUILDER_TAG }}
+          outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Smoke test (${{ matrix.platform }})
+        run: bash docker/sandbox-smoke-test.sh "${IMAGE}@${{ steps.build.outputs.digest }}" py
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.arch }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # ---------------------------------------------------------------------------
+  # Stitch the per-arch digests into one tagged manifest list.
+  # ---------------------------------------------------------------------------
+  merge:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Quay
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_ROBOT_USERNAME }}
+          password: ${{ secrets.QUAY_ROBOT_PASSWORD }}
+
+      - name: Compute tags
+        id: meta
+        shell: bash
+        run: |
+          BASE="${BRANCH//\//-}-${PYTHON_BUILDER_TAG}"
+          TAGS=( "-t" "${IMAGE}:${BASE}" "-t" "${IMAGE}:${BASE}-${SOURCE_SHA:0:7}" )
+          if [[ "${{ vars.RELEASE }}" == "TRUE" ]]; then
+            TAGS+=( "-t" "${IMAGE}:${PYTHON_BUILDER_TAG}-${{ vars.VERSION }}" )
+          fi
+          echo "args=${TAGS[*]}" >> "$GITHUB_OUTPUT"
+          echo "primary=${IMAGE}:${BASE}" >> "$GITHUB_OUTPUT"
+
+      - name: Create and push manifest list
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create ${{ steps.meta.outputs.args }} \
+            $(printf "${IMAGE}@sha256:%s " *)
+
+      - name: Inspect manifest list
+        run: docker buildx imagetools inspect "${{ steps.meta.outputs.primary }}"

docker/Dockerfile.sandbox

@@ -0,0 +1,69 @@
+# SEMOSS sandbox image: the OpenShell sandbox that runs a user's persistent python worker.
+#
+# Published by .github/workflows/sandbox-builder.yml as quay.io/semoss/semoss-sandbox, the default
+# OPENSHELL_IMAGE. To try a py/ change before it is published:
+#
+#   docker build -f docker/Dockerfile.sandbox -t semoss-sandbox:dev py    # context is py/
+#   docker/sandbox-smoke-test.sh semoss-sandbox:dev
+#
+# Python and every dependency come from the python-builder image (docker/Dockerfile.python,
+# uv + py/install_config/pyproject.toml). Like the Tomcat images, we copy its /usr/lib/python
+# onto a plain Ubuntu base, so a Py() block has the same interpreter and imports inside the
+# sandbox as outside it.
+#
+# The venv is reached by explicit path, never by PATH lookup: OpenShell builds the environment
+# for the processes it spawns, so ENV PATH is not applied inside a sandbox. See
+# docs/deployment/executor_sandbox.md.
+ARG BASE_REGISTRY=public.ecr.aws/docker/library
+ARG BASE_IMAGE=ubuntu
+ARG BASE_TAG=24.04
+
+FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
+
+LABEL maintainer="semoss@semoss.org"
+LABEL org.semoss.component="sandbox"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# git for local repository work; egress stays policy-gated per binary
+RUN for i in 1 2 3; do apt-get update && break || (sleep 10 && rm -rf /var/lib/apt/lists/*); done \
+ && apt-get install -y --no-install-recommends git ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+# uv-managed CPython + semossvenv; /usr/lib/python sits inside the policy's read-only /usr
+COPY --from=quay.io/semoss/python-builder:3.14-cpu  /usr/lib/python /usr/lib/python
+
+# One interpreter path for every build: the sandbox policy's egress rules name the kernel-reported
+# executable, and uv's directory carries the patch version and the CPU architecture
+# (cpython-3.14.8-linux-aarch64-gnu). Move the tree to a fixed name and leave the original as a
+# symlink, so the venv still resolves and the policy can say /usr/lib/python/cpython/bin/python3.14.
+RUN real="$(readlink -f /usr/lib/python/semossvenv/bin/python3)" \
+ && dir="$(dirname "$(dirname "$real")")" \
+ && mv "$dir" /usr/lib/python/cpython \
+ && ln -s cpython "$dir" \
+ && test "$(readlink -f /usr/lib/python/semossvenv/bin/python3)" = "/usr/lib/python/cpython/bin/$(basename "$real")"
+ENV VIRTUAL_ENV=/usr/lib/python/semossvenv
+ENV PATH=/usr/lib/python/semossvenv/bin:$PATH
+
+# the sandbox identity is uid/gid 1000 (ubuntu:24.04 already has it as "ubuntu")
+RUN if ! getent group 1000 >/dev/null; then groupadd -g 1000 sandbox; fi \
+ && if ! getent passwd 1000 >/dev/null; then useradd -u 1000 -g 1000 -m -s /bin/bash sandbox; fi
+
+# the worker source (build context is the repo's py/)
+COPY . /opt/semosshome/py/
+
+# setup() appends to insight/log.txt; grants-* are the policy's ro/rw drop points
+RUN mkdir -p /opt/semosshome/insight /opt/grants-rw /opt/grants-ro \
+ && touch /opt/semosshome/insight/log.txt \
+ && chown -R 1000:1000 /opt/semosshome /opt/grants-rw /opt/grants-ro \
+ && chmod 755 /opt/grants-rw /opt/grants-ro
+
+# Workstation homes. Files cross into a sandbox at their host path, so a Mac's BaseFolder
+# (/Users/me/...) or a Linux one (/home/me/...) must exist inside it. A sandbox can only create
+# directories under a path its policy grants and that exists at start, so these roots are made
+# here, owned by the sandbox user; OpenShellTransport grants the root and creates the home.
+RUN mkdir -p /Users && chown 1000:1000 /Users /home
+
+# OpenShell refuses to start unless the workspace is writable by the sandbox identity
+USER 1000:1000
+WORKDIR /opt/semosshome/py

docker/Dockerfile.sandbox.dockerignore

@@ -0,0 +1,7 @@
+# context is py/; keep caches and local tooling out of the sandbox image
+**/__pycache__
+**/*.pyc
+**/.DS_Store
+**/*.egg-info
+.pytest_cache
+.continue

docker/sandbox-smoke-test.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Prove a built sandbox image is usable: the worker tree matches the given py/, the venv
+# python imports the worker, git is present and the image runs as uid 1000.
+#
+#   docker/sandbox-smoke-test.sh <image> [py-dir]      # py-dir defaults to the repo's py/
+set -euo pipefail
+
+IMAGE="${1:?usage: sandbox-smoke-test.sh <image> [py-dir]}"
+PY_DIR="${2:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/py}"
+
+# same exclusions as Dockerfile.sandbox.dockerignore
+py_sha() {
+  find . -type f -name '*.py' -not -path '*/__pycache__/*' -not -path './.pytest_cache/*' \
+    -not -path './.continue/*' -not -path '*.egg-info/*' -print0 \
+    | sort -z | xargs -0 sha256sum | sha256sum | cut -d' ' -f1
+}
+
+EXPECT="$(cd "$PY_DIR" && py_sha)"
+
+docker run --rm --entrypoint bash -e EXPECT="$EXPECT" "$IMAGE" -ec "
+  $(declare -f py_sha)
+  cd /opt/semosshome/py
+  test -f gaas_tcp_socket_server.py || { echo 'worker missing at /opt/semosshome/py' >&2; exit 1; }
+  GOT=\$(py_sha)
+  [ \"\$GOT\" = \"\$EXPECT\" ] || { echo \"py tree in image (\$GOT) differs from $PY_DIR (\$EXPECT)\" >&2; exit 1; }
+  P=/usr/lib/python/semossvenv/bin/python3
+  [ -x \"\$P\" ] || { echo \"no venv python at \$P\" >&2; exit 1; }
+  # the sandbox policy's egress rules name this path (OPENSHELL_PYTHON_BINARY's default)
+  [ \"\$(readlink -f \$P)\" = /usr/lib/python/cpython/bin/python3.14 ] \\
+    || { echo \"interpreter is \$(readlink -f \$P), not /usr/lib/python/cpython/bin/python3.14\" >&2; exit 1; }
+  # same launch shape as OpenShellTransport: explicit interpreter, bare PATH
+  env -i PATH=/usr/local/bin:/usr/bin:/bin \"\$P\" -c 'import sys, pandas, numpy, jsonpickle, gaas_tcp_server_handler, gaas_change_watcher; print(sys.version)'
+  git --version
+  [ \"\$(id -u)\" = 1000 ] || { echo 'image does not run as uid 1000' >&2; exit 1; }
+"
+echo "smoke test passed for $IMAGE"

docs/README.md

@@ -82,6 +82,7 @@ The [Python GAAS tools index](python_gaas_tools/README.md) documents Python acce
 
 - [Local Docker setup](cloud_and_cluster/docker_deployment.md)
 - [Local Docker configuration](deployment/docker_configuration.md)
+- [OpenShell executor sandbox](deployment/executor_sandbox.md): running user code in a per-session container, the write-back pipeline and change detection, per-space write permissions, and building the image from `py/`
 - [Supporting engine examples](../docker-compose-examples/engines/README.md)
 - [Cloud and cluster architecture](cloud_and_cluster/README.md)
 - [Central asset storage](cloud_and_cluster/central_cloud_storage.md)

docs/cloud_and_cluster/github_actions_workflows.md

@@ -8,6 +8,7 @@ The workflows in [.github/workflows](../../.github/workflows/) build Java artifa
 | --- | --- |
 | [dev_workflow.yml](../../.github/workflows/dev_workflow.yml) | Java build, test selection, artifact signing, and publication paths |
 | [python-builder.yml](../../.github/workflows/python-builder.yml) | Python runtime/dependency image |
+| [sandbox-builder.yml](../../.github/workflows/sandbox-builder.yml) | OpenShell sandbox image: python-builder plus the `py/` worker ([details](../deployment/executor_sandbox.md#image)) |
 | [tomcat-builder.yml](../../.github/workflows/tomcat-builder.yml) | Java/Tomcat builder image |
 | [all-docker-builds.yml](../../.github/workflows/all-docker-builds.yml) | Application image build orchestration |
 | [ubuntu2204.yml](../../.github/workflows/ubuntu2204.yml), [ubuntu2404.yml](../../.github/workflows/ubuntu2404.yml) | Ubuntu application images |