diff --git a/.devcontainer/.env.example b/.devcontainer/.env.example new file mode 100644 index 000000000..f28d52879 --- /dev/null +++ b/.devcontainer/.env.example @@ -0,0 +1,12 @@ +# Optional non-production AAF configuration for the development container. +# Copy to .devcontainer/.env and obtain a dedicated registration from the +# identity owner. Never reuse or commit a production/institution shared secret. +# Keep database unless every AAF value below is complete; then set this to aaf. +DF_AUTH_METHOD=database +DF_AAF_ISSUER_URL= +DF_AAF_AUDIENCE_URL=http://localhost:3000 +DF_AAF_CALLBACK_URL=http://localhost:3000/api/auth/jwt +DF_AAF_IDENTITY_PROVIDER_URL= +DF_AAF_UNIQUE_URL= +DF_AAF_AUTH_SIGNOUT_URL= +DF_SECRET_KEY_AAF= diff --git a/.devcontainer/devcontainer.env b/.devcontainer/devcontainer.env index ac60b770a..8e5bbeb3b 100644 --- a/.devcontainer/devcontainer.env +++ b/.devcontainer/devcontainer.env @@ -12,6 +12,9 @@ DF_JPLAG_REPORT_DIR=/jplag/results DF_JPLAG_SKIP_CLUSTER_CHECK=true DF_JPLAG_MAX_SHOWN_COMPARISONS=-1 +# Peer Progress Indicator local development defaults +DF_PPI_MINIMUM_COHORT_SIZE=21 +DF_PPI_STALE_AFTER_HOURS=48 # Overseer - enabled! OVERSEER_ENABLED=1 @@ -43,14 +46,15 @@ DF_ACCESS_TOKEN_EXPIRY_SECONDS=3600 # 7 Days DF_REFRESH_TOKEN_EXPIRY_SECONDS=604800 -# AAF settings - for testing on local host -DF_AAF_ISSUER_URL=https://rapid.test.aaf.edu.au -DF_AAF_AUDIENCE_URL=http://localhost:3000 -DF_AAF_CALLBACK_URL=http://localhost:3000/api/auth/jwt -DF_AAF_IDENTITY_PROVIDER_URL=https://signon-uat.deakin.edu.au/idp/shibboleth -DF_AAF_UNIQUE_URL=https://rapid.test.aaf.edu.au/jwt/authnrequest/research/Ag4EJJhjf0zXHqlKvKZEbg -DF_AAF_AUTH_SIGNOUT_URL=https://sync-uat.deakin.edu.au/auth/logout -DF_SECRET_KEY_AAF=v4~LMFLzzwRGZdju\5QBa@FiHIN9 +# Optional AAF development settings are deliberately blank. Supply a dedicated +# non-production registration through .devcontainer/.env; never commit it. +DF_AAF_ISSUER_URL= +DF_AAF_AUDIENCE_URL= +DF_AAF_CALLBACK_URL= +DF_AAF_IDENTITY_PROVIDER_URL= +DF_AAF_UNIQUE_URL= +DF_AAF_AUTH_SIGNOUT_URL= +DF_SECRET_KEY_AAF= # Database settings DF_DEV_DB_ADAPTER=mysql2 diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index c7f42dd79..622eca979 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -48,6 +48,17 @@ services: environment: LATEX_CONTAINER_NAME: 1-formatif-texlive-container LATEX_BUILD_PATH: /texlive/shell/latex_build.sh + # Optional AAF development uses a dedicated registration supplied from + # an ignored .devcontainer/.env file. Database auth remains the default. + DF_AUTH_METHOD: ${DF_AUTH_METHOD:-database} + TUTORIAL_ENABLED: ${TUTORIAL_ENABLED:-0} + DF_AAF_ISSUER_URL: ${DF_AAF_ISSUER_URL:-} + DF_AAF_AUDIENCE_URL: ${DF_AAF_AUDIENCE_URL:-} + DF_AAF_CALLBACK_URL: ${DF_AAF_CALLBACK_URL:-} + DF_AAF_IDENTITY_PROVIDER_URL: ${DF_AAF_IDENTITY_PROVIDER_URL:-} + DF_AAF_UNIQUE_URL: ${DF_AAF_UNIQUE_URL:-} + DF_AAF_AUTH_SIGNOUT_URL: ${DF_AAF_AUTH_SIGNOUT_URL:-} + DF_SECRET_KEY_AAF: ${DF_SECRET_KEY_AAF:-} depends_on: - dev-db diff --git a/.github/workflows/build-dev-container.yml b/.github/workflows/build-dev-container.yml index 76eda1af2..81912e912 100644 --- a/.github/workflows/build-dev-container.yml +++ b/.github/workflows/build-dev-container.yml @@ -25,29 +25,29 @@ jobs: context: ./doubtfire-api/ steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: submodules: recursive - name: Setup meta for development image id: docker_meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 with: images: ${{ matrix.image }} tags: | type=semver,pattern={{major}}.{{minor}} - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 if: github.event_name != 'pull_request' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 - name: Set up Docker id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Cache Docker layers - uses: actions/cache@v3 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: /tmp/.buildx-cache-app key: ${{ runner.os }}-buildx-app-${{ github.sha }} @@ -55,14 +55,14 @@ jobs: ${{ runner.os }}-buildx-app - name: Build and push dev container image id: docker_build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: file: ${{ matrix.dockerfile }} context: ${{ matrix.context }} builder: ${{ steps.buildx.outputs.name }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + labels: ${{ steps.docker_meta.outputs.labels }} platforms: linux/amd64,linux/arm64 cache-from: type=local,src=/tmp/.buildx-cache-app # Reuse cached layers from previous builds. cache-to: type=local,dest=/tmp/.buildx-cache-app-new,mode=max # Save new cache layers after the build. diff --git a/.github/workflows/validate-production-compose.yml b/.github/workflows/validate-production-compose.yml new file mode 100644 index 000000000..4ae59e914 --- /dev/null +++ b/.github/workflows/validate-production-compose.yml @@ -0,0 +1,57 @@ +name: validate-production-compose + +on: + pull_request: + paths: + - ".github/workflows/validate-production-compose.yml" + - ".gitmodules" + - ".devcontainer/**" + - "development/**" + - "HANDOVER.md" + - "MIGRATING.md" + - "RELEASING.md" + - "DEPLOYING.md" + - "DESKTOP-PWA.md" + - "doubtfire-api" + - "doubtfire-web" + - "production/**" + push: + branches: + - 11.0.x + paths: + - ".github/workflows/validate-production-compose.yml" + - ".gitmodules" + - ".devcontainer/**" + - "development/**" + - "HANDOVER.md" + - "MIGRATING.md" + - "RELEASING.md" + - "DEPLOYING.md" + - "DESKTOP-PWA.md" + - "doubtfire-api" + - "doubtfire-web" + - "production/**" + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Check out repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Test desktop PWA public-asset verification + run: python3 -B production/tests/verify_pwa_test.py + + - name: Test release publication and recovery cache controls + run: python3 -B production/tests/publish_release_test.py + + - name: Validate production configuration + run: production/tests/validate_test.sh + + - name: Verify upload boundaries through the production Nginx template + run: python3 -B production/tests/nginx_upload_test.py diff --git a/.github/workflows/validate-secret-audit.yml b/.github/workflows/validate-secret-audit.yml new file mode 100644 index 000000000..dd04609d4 --- /dev/null +++ b/.github/workflows/validate-secret-audit.yml @@ -0,0 +1,22 @@ +name: Validate secret audit tool +on: + pull_request: + paths: + - tools/audit-vapid-secret.py + - tools/tests/test_audit_vapid_secret.py + - .github/workflows/validate-secret-audit.yml + push: + branches: ["11.0.x"] + paths: + - tools/audit-vapid-secret.py + - tools/tests/test_audit_vapid_secret.py + - .github/workflows/validate-secret-audit.yml +permissions: + contents: read +jobs: + synthetic-fixtures: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: python3 -B -m unittest discover -s tools/tests -p 'test_audit_vapid_secret.py' diff --git a/.gitignore b/.gitignore index 5c46ce646..1512a188e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,16 @@ !/data/**/ !/data/**/.gitkeep /node_modules/ +/.devcontainer/.env +/development/.env + +# Production runtime configuration, credentials, and local TLS material. +/production/.env.production +/production/secrets/* +!/production/secrets/.gitkeep +/production/shared-files/aliases +/production/shared-files/msmtprc +/production/*.sql # Created by https://www.toptal.com/developers/gitignore/api/macos # Edit at https://www.toptal.com/developers/gitignore?templates=macos diff --git a/DEPLOYING.md b/DEPLOYING.md index 930547a2c..a10c4af05 100644 --- a/DEPLOYING.md +++ b/DEPLOYING.md @@ -1,102 +1,679 @@ -![Doubtfire Logo](http://puu.sh/lyClF/fde5bfbbe7.png) - # Deploying Doubtfire -Doubtfire is deployed using Docker containers described in a docker compose. This document outlines the main steps in the process for deploying this as an application, which require additional considerations to ensure that the setup is appropriate for your situation. Settings provided, and changes recommended, need to be reviewed by an appropriate security and deployment professional. - -When setup, launching the application should only require the docker compose to be started: e.g. `docker compose up`. When setup, the application involves the following components: - -- a proxy, based on nginx, that handles HTTPS and routes traffic to the webserver or apiserver containers. -- a webserver, based on nginx, that serves the static html/css/javascript/etc files. -- an apiserver, based on rails, that serves the restful API used by the application. -- an application server (pdfgen), based on rails, that uses cron jobs to periodically generate PDFs from student submissions, and send status emails. -- a database server, based on Maria DB or MySql used by the api and application servers to persist data -- file storage connected to the apiserver and application server for storing student work -- an external mail server to send emails -- an external authentication server to authenticate users. - -The setups to configure these components include: - -1. You will need: - - a host on to which to deploy the application - - certificates for https access to the application, and network setup to route that traffic to the host - - a mail server to enable notification emails, and a support email address that the application can use - - an external authentication sources (AAF, SAML, or LDAP) - - file storage (and backups) for student work - - optional: separate database server -2. Copy the following files from the **production** folder of this project to the host machine. - - docker-compose.yml - this file contains the configuration of all containers needed. - - .env.production - this contains all of the environment variables needed to configure the containers. - - shared-files folder - this contains configuration files that need to be mapped into containers for email configurations and proxy nginx settings. -3. Adjust settings in **docker-compose.yml** - - Proxy - - Replace certificate and key files and volume mappings with certificate created for the institution URL. - - Apiserver - - The **student_work** volume should be mapped to the host, and this location should be backed up. - - Consider where to store doubtfire-logs and update volume if needed - - Doubtfire-db - - Remove container if using external database, otherwise configure the MariaDB for your production needs. - - Pdfgen - - Adjust **student-work** to match app server. - - Add monitoring as needed to ensure ongoing operation -4. Adjust **.env.production**: - - **DF_PRODUCTION_DB_*** settings - adjust database settings for adapter type, host name, database name, and password. The provided setting work with the database setup in the compose file. The password should be updates as a minimum. - - **DF_SECRET_KEY_DEVISE** - contains the key used to encrypt the [Devise](https://github.com/heartcombo/devise) user data in the database. Keys can be generated with `bundle exec rake secret` run in the *apiserver* container. - - **DF_SECRET_KEY_BASE** and **DF_SECRET_KEY_ATTR** - these are historic keys used to encrypt data in the database. Generate as with the Devise key. - - **DF_SECRET_KEY_MOSS** - the key used to connect with the [MOSS](http://moss.stanford.edu) system for checking code similarity. - - **DF_INSTITUTION_HOST** - change to the URL used for the application - - **DF_INSTITUTION_PRODUCT_NAME** - change to the name of the deployed product - - **DF_INSTITUTION_SETTINGS_RB** - can be used when institution specific integrations for synchronising student enrolments. - - **DF_INSTITUTION_EMAIL_DOMAIN** - the domain name associated with student emails. - - **DF_INSTITUTION_NAME** - change to the the name of the institution - - **DF_AUTH_METHOD** - change to the intended authentication method. Do not use database. The authentication source needs to exist externally, or be setup within the system. - - AAF authentication uses [rapid connect](https://rapid.aaf.edu.au). This is configured with the following settings: - - DF_AAF_ISSUER_URL - - DF_AAF_AUDIENCE_URL - - DF_AAF_CALLBACK_URL - - DF_AAF_IDENTITY_PROVIDER_URL - - DF_AAF_UNIQUE_URL - - DF_AAF_AUTH_SIGNOUT_URL - - DF_SECRET_KEY_AAF - - SAML authentication uses: - - DF_SAML_METADATA_URL - - DF_SAML_CONSUMER_SERVICE_URL - - DF_SAML_SP_ENTITY_ID - - DF_SAML_IDP_TARGET_URL - - DF_SAML_IDP_SAML_NAME_IDENTIFIER_FORMAT - - LDAP uses: - - DF_LDAP_HOST - - DF_LDAP_PORT - - DF_LDAP_ATTRIBUTE - - DF_LDAP_BASE - - DF_LDAP_SSL - - DF_LDAP_USE_ADMIN_TO_BIND - - DF_LDAP_ADMIN_USER - - DF_LDAP_ADMIN_PWD - - **DF_ENCRYPTION_*** - is used in encrypting data in the database. Generate these from within the *apiserver* container using `rails db:encryption:init`. See [Active Record Encryption](https://guides.rubyonrails.org/active_record_encryption.html). - - Setup email settings, see [Action Mailer](https://guides.rubyonrails.org/action_mailer_basics.html). - - Set **DF_MAIL_PERFORM_DELIVERIES** to `yes` (lowercase) to enable mail sending - - Set **DF_MAIL_DELIVERY_METHOD** to *smtp* or disable by setting to *none*. Set the following variables to configure mail sending. - - DF_SMTP_ADDRESS - - DF_SMTP_PORT - - DF_SMTP_DOMAIN - - DF_SMTP_USERNAME - - DF_SMTP_PASSWORD - - DF_SMTP_AUTHENTICATION -5. Adjust **shared-files/proxy-nginx.conf**. This file configures the proxy nginx container that routes traffic to the webserver and apiserver containers. It needs to be update with the host URLs. The provided file uses localhost as the server URL. -6. Configure email settings: - - Edit **shared-files/aliases** to map the root user email to the support email address that the application will use. - - Edit **shared-files/msmtprc** to include the necessary details used to connect to your mail server. -7. Initialise the database by running: - - `DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bundle exec rake db:setup db:init`. When prompted about running in production, respond with `Yes` (case sensitive) to allow the database initialisation to proceed. - - Verify the database setup in the rails console `bundle exec rails c`, and update username and email to match expected admin user: - - ```ruby - u = User.first - u.email = '...' - u.username = '...' - u.save! - ``` - - When successful you should be able to login as the admin user. +The `production` directory is a fail-closed, single-host Docker Compose +deployment for the core Doubtfire application. It runs the web and API services, +MariaDB, Redis/Sidekiq, scheduled work, PDF generation, and JPlag behind HTTPS. + +It does not contain credentials, institution settings, TLS private keys, or a +default image release. Those values are deployment-specific and must be supplied +before validation will pass. LTI and Gotenberg are optional integrations with +separate component contracts and are not enabled by this stack. + +## Requirements and trust boundary + +Use a dedicated Linux host with: + +- Docker Engine and Docker Compose v2.33.1 or newer; +- `curl`, OpenSSL, and standard GNU utilities including `timeout`; +- DNS for the production hostname and a trusted TLS certificate; +- an external AAF, SAML, or LDAP identity provider; +- an SMTP service and an operational sender address; +- outbound HTTPS access to the push services used by supported browsers; +- persistent storage for MariaDB, Redis, logs, and student work; and +- monitored, regularly restored backups of the database and student work. + +The PDF and JPlag implementation invokes existing helper containers through the +Docker API. The included HAProxy policy allows only ping/version plus +inspect/exec requests for `jplag` and this project's two TexLive containers; all +other Docker API paths and methods are denied. The proxy is reachable only from +the worker network and is never published. Possession of a Docker socket is +still a high-trust boundary, so use a host dedicated to Doubtfire, pin the tested +HAProxy Alpine image, review policy changes, and monitor denials. The API expects +the JPlag container name `jplag`, so run only one production project per daemon. + +### SCORM credential-path mitigation + +The inherited SCORM content route is +`/api/scorm/:task_def_id/:username/:auth_token/*file_path`. Its `auth_token` +segment is a reusable credential until the corresponding SCORM token expires. +Putting a credential in a URL path can expose it through access and error logs, +browser history, copied links, monitoring and analytics systems, upstream edge +services, and referrer headers. + +For this release, the included Nginx proxy suppresses access-log entries for +`/api/scorm/` paths and returns `Referrer-Policy: no-referrer` on those responses. +The rule is applied without a separate location block, so SCORM requests retain +the common API routing, forwarding headers, and timeout. These controls are +mitigation, not closure: they do not remove the credential from browser history, +the API request target, error reporting, an external load balancer, CDN, WAF, or +APM instrumentation. Configure equivalent redaction or suppression on every +component in front of or observing Doubtfire, restrict access to historic logs, +and treat any previously logged SCORM token as exposed until it expires or is +revoked. + +The SCORM launch flow must be redesigned so reusable credentials are carried in +a Secure, HttpOnly, appropriately scoped SameSite cookie or an authorisation +header. Where SCORM clients cannot supply either, exchange a single-use, +short-lived launch code for scoped session state and keep usernames and tokens +out of subsequent asset URLs. That application redesign and token migration is +required before this risk can be considered closed. + +During production acceptance, use a disposable SCORM test token and confirm its +asset responses include `Referrer-Policy: no-referrer`, no `/api/scorm/` request +appears in the proxy access log, and an ordinary API request is still logged. +Also inspect API, platform-edge, observability, and error-reporting destinations +for the disposable token, then revoke it. Never paste a real SCORM URL into a +ticket, chat, command history, or acceptance record. + +## 1. Prepare runtime configuration + +From the repository root: + +```bash +cd production +cp .env.production.example .env.production +chmod 600 .env.production +``` + +Replace every `REPLACE_ME` value. The validator rejects placeholder values, +development database authentication, weak/short critical secrets, mutable image +tags, unsafe peer-progress settings, malformed or development VAPID keys, +missing processor limits, unsupported Overseer activation, and runtime files +with unsafe permissions. + +Keep each setting in canonical `KEY=value` form. Full-line comments are allowed, +but quoted or escaped values, inline comments, dollar-sign interpolation, and +leading or trailing whitespace are rejected so validation and Compose cannot +interpret the same file differently. + +Generate independent random values for database and Rails secrets. For example: + +```bash +openssl rand -hex 32 +``` + +Generate a new value for each setting. Existing installations must retain their +Rails signing and Active Record encryption values; changing them without a +planned rotation can invalidate sessions or make encrypted data unreadable. + +Set `DF_AUTH_METHOD` to `aaf`, `saml`, or `ldap` and complete that provider's +settings. `database` authentication is deliberately rejected for production. +Set institution, SMTP, and sender values to real operational endpoints. Overseer, +LTI, and Gotenberg are not enabled by this stack. Peer Progress and Web Push are +enabled only through the explicit, fail-closed settings below. + +### Sidekiq concurrency contract + +Start with `DF_SIDEKIQ_CONCURRENCY=5`. Production validation permits only an +integer from 2 through 5, and Compose injects it only into Sidekiq. Five is the +upper bound of the default Active Record connection pool, which ensures each +worker thread can obtain a database connection without silently overcommitting +the pool. The lower bound keeps one long-running document or similarity job from +being the worker's only available execution path while notification and Peer +Progress jobs are queued. + +This range is a release safety bound, not a capacity guarantee. Monitor queue +latency, retry/dead-job counts, database connection saturation, CPU, memory, and +the external PDF/JPlag helpers before lowering concurrency or planning a larger +worker topology. Raising it above 5 requires a coordinated application change, +an explicitly larger Active Record pool, and database-capacity testing; changing +only the environment file is deliberately rejected. Use API and app-worker +images from the same tested release so their concurrency validation, database +pool assumptions, job classes, and cron schedule remain compatible. + +### Peer Progress privacy contract + +Keep `DF_PPI_MINIMUM_COHORT_SIZE=21`, or raise it if the institution approves a +stricter privacy threshold. Values below 21 are rejected. Keep +`DF_PPI_STALE_AFTER_HOURS=48`, or lower it for a stricter freshness window; +zero, negative, non-integer, and values above 48 are rejected. The API also +fails closed rather than returning peer percentages if either setting is +missing or invalid. + +The threshold is the number of remaining peers after the authenticated +student's known contribution has been removed from the exact internal +aggregate. A configured floor of 21 therefore requires at least 22 eligible +students in the target-grade band, including the reader. The same peer-only +threshold protects both the compact completion view and the advanced +task-status distribution. The submitted percentage remains an additive wire +compatibility metric for rolling API/web upgrades; it is not the primary +compact meaning. + +The API must fail closed unless the snapshot contains the exact internal +submitted and status counts needed to subtract the reader. It also fails closed +if the reader's project membership or task changed after aggregation, rather +than subtracting a stale known contribution. Advanced percentages are released only when the API's +whole-vector ambiguity check confirms that every displayed peer status still +has multiple feasible raw counts, even if an observer already knows the peer +cohort size. Otherwise the entire distribution is unavailable; never weaken +the cohort setting or bypass that guard to fill the advanced display. Neither +view may expose peer identities, project records, marks, raw cohort size, or raw +submitted/status counts. + +Each released status is rounded into its own privacy bucket. The displayed +percentages may therefore total slightly above or below 100%; neither the API +nor the client may renormalise them, because doing so would change the reviewed +privacy mapping and can reveal extra information. + +Configuration does not opt units in. `units.peer_progress_enabled` defaults to +false and an authorised convenor must explicitly enable Peer Progress for each +approved unit. Start with a populated test unit, confirm the cohort policy with +the responsible privacy owner, and complete the activation and verification +steps in section 5 before enabling more units. Do not run demo seed tasks against +production. + +### Web Push VAPID contract + +Generate a new P-256 VAPID pair for this production installation from the exact +accepted API image on a trusted administration host. The production host does +not need Ruby or Bundler. Set `DF_API_IMAGE` to the accepted immutable API digest +and `DF_VAPID_DIR` to a new protected directory; the container has no network, +does not log the keys, and writes them directly to that directory: + +```bash +: "${DF_API_IMAGE:?Set the accepted apiserver image@sha256 digest}" +: "${DF_VAPID_DIR:?Set a new absolute protected key directory}" +[[ "${DF_API_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]] +[[ "${DF_VAPID_DIR}" == /* && ! -e "${DF_VAPID_DIR}" ]] +sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "${DF_VAPID_DIR}" +docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges --log-driver none \ + --volume "${DF_VAPID_DIR}:/keys" \ + "${DF_API_IMAGE}" \ + bundle exec ruby -e \ + 'require "web_push"; key = WebPush.generate_key; {"/keys/public" => key.public_key, "/keys/private" => key.private_key}.each { |path, value| File.open(path, File::WRONLY | File::CREAT | File::EXCL, 0o600) { |file| file.write(value) } }' +sudo chown "$(id -u):$(id -g)" "${DF_VAPID_DIR}/public" "${DF_VAPID_DIR}/private" +chmod 0600 "${DF_VAPID_DIR}/public" "${DF_VAPID_DIR}/private" +``` + +Set `DOUBTFIRE_VAPID_PUBLIC_KEY`, `DOUBTFIRE_VAPID_PRIVATE_KEY`, and an explicit +`DOUBTFIRE_VAPID_SUBJECT`. The subject must be an operational `mailto:` address +or HTTPS contact URL. The example values are deliberately non-secret +placeholders and validation will reject them. The checked-in development pair +is also rejected. The private key is supplied only to the API and Sidekiq +containers; transfer it into the approved secret manager using a secure editor +or secret-ingestion mechanism, never by printing it to a terminal. Never put it +in Git, tickets, command arguments, build logs, or browser configuration. +Securely delete the temporary key directory after escrow and deployment are +verified. + +Use the [private-key audit runbook](docs/vapid-secret-audit.md) to check the +actual secret against local Git history, original image layers and protected +log exports without printing its value. A source review or a clean current +container filesystem cannot establish that earlier layers and logs are clean. + +Store the private key in the institution's secret manager and encrypted recovery +escrow alongside the Rails encryption keys. Preserve it across deployments and +restores. Rotating the pair invalidates every existing browser subscription: +schedule the rotation, deploy both API and Sidekiq with the new pair, notify +users, and require them to enable push again. A database restore without the +matching VAPID private key cannot deliver to its saved subscriptions. + +Allow outbound TCP 443 from API and Sidekiq to the endpoint hosts issued by the +supported browsers. The current application allowlist includes +`fcm.googleapis.com`, `android.googleapis.com`, +`updates.push.services.mozilla.com`, subdomains of `push.apple.com`, subdomains +of `push.services.microsoft.com`, and subdomains of `notify.windows.com`. Review +captured subscription endpoint hosts whenever the supported browser matrix +changes; do not assume browser vendors will keep endpoints static. + +## 2. Pin tested images + +Every image setting must use a fully qualified image name followed by +`@sha256:` and exactly 64 hexadecimal digest characters. + +Build the mutually compatible API, app-worker, web, TexLive, and JPlag images +through the revision-locked publisher in `RELEASING.md`; use only the complete +digest manifest it emits after every registry verification succeeds. Resolve +the separately selected proxy, MariaDB, Redis, and HAProxy images to digests +through the institution's approved registry process. Never place the candidate +tag itself in `.env.production`. + +Record the tested digest in `.env.production`, including for Nginx, MariaDB, +Redis, and `haproxy:3.2-alpine`. The app-worker image must include the secure +production runtime scripts from the matching Doubtfire API release; older +`pdfgen_entry_point.sh` versions print their environment and must not be used. +The API image must come from the same release and provide the dependency-aware +`/readiness` endpoint used by the Compose and public health checks. + +## 3. Prepare host storage and service files + +Create the absolute directories configured by `STUDENT_WORK_PATH`, +`DOUBTFIRE_LOG_PATH`, `MARIADB_DATA_PATH`, and `REDIS_DATA_PATH`. Also create +`JPLAG_REPORT_PATH` and `JPLAG_ARCHIVE_REPORT_PATH` beneath the student-work +directory. The JPlag container receives only those two report paths, and the +TexLive containers receive no student-work mount. Keep runtime data outside the +Git checkout and restrict ownership to the deployment operators and container +users that require access. + +Set explicit CPU, memory, and PID ceilings for TexLive and JPlag based on host +capacity and measured workload. The supplied example is a conservative starting +point, not a sizing guarantee. + +Provide the certificate and unencrypted private key configured by +`TLS_CERTIFICATE_PATH` and `TLS_PRIVATE_KEY_PATH`. The validator confirms that +the certificate matches the key and hostname and will remain valid for at least +30 days. Restrict the private key to its owner: + +```bash +chmod 600 /absolute/path/to/private.key +``` + +Copy the mail helper examples outside the repository and replace every +placeholder: + +```bash +sudo install -d -m 0755 /etc/doubtfire +sudo install -m 0644 shared-files/aliases.example /etc/doubtfire/aliases +sudo install -m 0600 shared-files/msmtprc.example /etc/doubtfire/msmtprc +``` + +Update the two absolute paths in `.env.production`. The Rails SMTP settings and +`msmtprc` should describe the same approved mail service. + +Automate certificate renewal. Because the certificate and key are file bind +mounts, recreate only the proxy after a successful renewal so Docker remounts +the replacement files, then verify both public health endpoints: + +```bash +./compose.sh up --detach --no-deps --force-recreate --wait --wait-timeout 60 proxy +``` + +## 4. Validate and deploy + +Run the preflight before every deployment: + +```bash +./validate.sh +``` + +Deploy only after it passes: + +```bash +./deploy.sh +./verify.sh +``` + +The default health wait is five minutes. Supply a different bounded value from +1 through 86400 seconds as the second argument when an approved migration needs +longer: + +```bash +./deploy.sh .env.production 900 +``` + +The deployment helper validates configuration, pulls the exact image digests, +stages and health-checks the hardened web image, then enters a bounded +maintenance window by stopping public ingress and all application writers before +applying the API migration and worker stack. Before invoking it, the release +owner must confirm the consistent recovery set and migration rehearsal are +current and use database monitoring to clear or explicitly accept long-running +transactions. This order is required because the new web accepts both legacy +query and fragment authentication callbacks, while the hardened API emits +fragments that an older web build cannot consume. The helper waits for declared +health checks and running worker processes, keeps ingress and writers stopped on +a failed rollout, and prints diagnostics. `verify.sh` then runs +the fail-closed post-deploy gates against the same environment file and local +Compose wrapper. A one-shot `migrate` service runs after MariaDB is healthy; API +and worker startup is gated on successful migrations. API readiness checks both +MariaDB and Redis. Only ports 80 and 443 are published. MariaDB, Redis, the +Docker API proxy, and application services remain on private Compose networks. + +Both helpers accept an explicit environment file. `verify.sh` also accepts a +bounded per-check timeout: + +```bash +./verify.sh .env.production 60 +``` + +It verifies public TLS/proxy readiness, direct API database/Redis readiness, +the web index and PWA control files, pending migrations, a live Sidekiq process +at the configured concurrency, the required peer-progress and notification cron +jobs, and live PPI/VAPID configuration without printing credentials or student +data. A successful exit means only that the automated gates passed; its clearly +labelled manual gates remain mandatory before traffic or a rollback is accepted. + +For a release supporting desktop installation, also run the public-asset gate +and installed-browser acceptance matrix in [DESKTOP-PWA.md](DESKTOP-PWA.md). +The standalone gate uses Python 3.9+ and supplements `verify.sh`: + +```bash +python3 -B ./verify-pwa.py https://ontrack.example.edu --timeout 30 +``` + +Replace the example with the canonical public origin. This additional gate +checks the web manifest, PNG icons, worker caching and released asset hashes; +a passing result does not install or certify an operating-system app. + +`compose.sh` deliberately clears the ambient shell before invoking Compose and +pins the Docker CLI to the local `/var/run/docker.sock`. This prevents exported +variables or a remote Docker context from changing the configuration that was +validated. The production host must expose its local Docker daemon at that +standard socket path. + +On the first deployment, keep external ingress blocked with the host firewall or +institutional edge until initialisation and manual identity acceptance are +complete. Initialise Doubtfire's roles, statuses, and initial administrator once: + +```bash +./compose.sh exec apiserver bundle exec rake db:init +``` + +Then use the Rails console to replace the initial administrator's placeholder +profile with the authorised operational identity. Do not enable database +authentication or retain a default password to make first login easier. Complete +the administrator IdP login/logout and the applicable verification gates before +allowing public ingress. + +## 5. Verify the deployment + +Run the automated gate at minimum: + +```bash +./compose.sh ps +./verify.sh +``` + +Complete every `MANUAL-*` gate printed by `verify.sh`. In particular, exercise a +real login through the configured identity provider, send a notification email, +generate a PDF through Sidekiq/TexLive, and run a permitted JPlag check before +accepting traffic. Confirm that database and Redis ports are not published and +that container logs contain no credentials or private keys. + +For Peer Progress, have an authorised convenor enable it in Unit Settings for a +single approved test unit. Enqueue the first aggregation rather than waiting for +the nightly schedule (replace `UNIT_CODE` with the exact production unit code): + +```bash +./compose.sh exec apiserver bundle exec rails runner \ + 'unit = Unit.find_by!(code: "UNIT_CODE"); abort "Peer Progress is not enabled" unless unit.peer_progress_enabled?; AggregatePeerProgressJob.perform_async(unit.id); puts "Peer Progress aggregation queued"' +``` + +Wait for Sidekiq to log completion, then verify that the schedule is loaded and +that a fresh snapshot exists without printing student-level data: + +```bash +./compose.sh exec sidekiq bundle exec rails runner \ + 'abort "Peer Progress schedule missing" unless Sidekiq::Cron::Job.find("aggregate_peer_progress"); puts "Peer Progress schedule loaded"' +./compose.sh exec apiserver bundle exec rails runner \ + 'unit = Unit.find_by!(code: "UNIT_CODE"); snapshot = unit.peer_progress_snapshots.order(calculated_at: :desc).first; abort "No Peer Progress snapshot" unless snapshot; abort "Peer Progress exact aggregates are incomplete" if snapshot.submitted_count.nil? || snapshot.status_counts.nil?; abort "Peer Progress snapshot is stale" if snapshot.calculated_at < ENV.fetch("DF_PPI_STALE_AFTER_HOURS").to_i.hours.ago; puts "Fresh Peer Progress snapshot with exact peer-only inputs present"' +``` + +Sign in as a student in that test unit and inspect the Peer Progress response and +display. Confirm that the indicator appears below the task submission area, the +compact view is the initial task view, and its small advanced control reveals +the quantised canonical status distribution. Exercise representative statuses, +including redo and fix-and-resubmit. Confirm the profile preference is on for a +new student and that switching it off removes peer progress without affecting +submission controls. + +For an eligible remaining-peer cohort, the response may contain only approved state fields, +the quantised completed percentage, the additive submitted compatibility +percentage, and (when its independent ambiguity guard passes) the quantised +`status_distribution`. Confirm that a cohort below the configured threshold +(which can never be lower than 21 peers after excluding the reader) suppresses +all peer percentages. Confirm that a legacy snapshot without exact submitted +and status counts, and a snapshot older than the reader's current project/task +context, also fail closed until aggregation refreshes them. Separately +exercise a cohort whose detailed vector is withheld by the vector privacy guard: +the compact view may remain available, but advanced data must fail closed with a +non-sensitive explanation and useful navigation or retry choices. No response +or display may expose names, identifiers, peer project records, marks, raw +cohort size, or raw status counts. + +Web Push acceptance requires a real supported browser and cannot be replaced by +Compose health checks. Over HTTPS, sign in and confirm `/ngsw.json` loads and the +authenticated `/api/settings` response reports `pushEnabled: true` with the +configured public key. Use the application's user-gesture control to grant +notification permission and create a subscription. Trigger a known notification +that is processed by Sidekiq, verify it appears with the application in the +background, and verify clicking it opens the intended OnTrack route. Record the +browser, operating system, endpoint host, and result. Repeat on at least one +desktop browser and every mobile/browser combination the institution promises +to support; iOS/iPadOS testing must use an installed Home Screen web app. + +Monitor at least: + +- external HTTPS and `/healthz` availability; +- container health/restarts and migration failures; +- Sidekiq queue latency, retries, and dead jobs; +- Peer Progress aggregation age and suppressed/error responses; +- scheduled PDF/JPlag job failures; +- Web Push delivery failures, expired subscriptions, and rejected endpoint hosts; +- SMTP delivery failures; +- database/storage capacity and backup age; and +- TLS certificate expiry. + +## Backups and restore tests + +The receiving institution must set and approve a recovery point objective +(RPO), recovery time objective (RTO), backup frequency, retention, encryption +owner, off-host copy location, and restore-test frequency. Record those values +and the last successful isolated restore in the release change. A backup is not +accepted merely because a file exists. + +Back up the MariaDB database and the complete `STUDENT_WORK_PATH` as one labelled +recovery set. For the cleanest consistency boundary, enter maintenance, stop all +application writers, let Sidekiq stop gracefully, capture both components, and +then restart. Run the following as one reviewed Bash block from the `production` +directory. Replace the student-work source with the exact path from the validated +environment file: + +```bash +( + set -Eeuo pipefail + umask 077 + + DF_BACKUP_ROOT=/srv/doubtfire/backups + DF_STUDENT_WORK_SOURCE=/srv/doubtfire/student-work + DF_BACKUP_ID="doubtfire-$(date -u +%Y%m%dT%H%M%SZ)" + DF_BACKUP_PATH="${DF_BACKUP_ROOT}/${DF_BACKUP_ID}" + DF_BACKUP_OPERATOR_UID="$(id -u)" + DF_BACKUP_OPERATOR_GID="$(id -g)" + DF_MAINTENANCE_STARTED=0 + + backup_failure() { + local status="$1" + trap - ERR INT TERM + if [[ "${DF_MAINTENANCE_STARTED}" == 1 ]]; then + timeout --kill-after=10s 120s \ + ./compose.sh stop --timeout 60 proxy apiserver sidekiq pdfgen || true + fi + printf 'Backup failed; if maintenance started, application writers remain stopped.\n' >&2 + exit "${status}" + } + trap 'backup_failure $?' ERR + trap 'backup_failure 130' INT + trap 'backup_failure 143' TERM + + DF_RUNNING_SERVICES="$(./compose.sh ps --status running --services)" + if grep -Fxq migrate <<< "${DF_RUNNING_SERVICES}"; then + printf 'Refusing backup while the migration service is running.\n' >&2 + exit 1 + fi + [[ -d "${DF_STUDENT_WORK_SOURCE}" ]] || { + printf 'Student-work source is not a directory.\n' >&2 + exit 1 + } + + sudo install -d -m 0700 \ + -o "${DF_BACKUP_OPERATOR_UID}" \ + -g "${DF_BACKUP_OPERATOR_GID}" \ + "${DF_BACKUP_PATH}" + + DF_MAINTENANCE_STARTED=1 + timeout --kill-after=10s 120s \ + ./compose.sh stop --timeout 60 proxy apiserver sidekiq pdfgen + ./compose.sh exec -T doubtfire-db sh -c \ + 'exec mariadb-dump --single-transaction --routines --events \ + --user="$MARIADB_USER" --password="$MARIADB_PASSWORD" "$MARIADB_DATABASE"' \ + > "${DF_BACKUP_PATH}/database.sql" + sudo tar --acls --xattrs --numeric-owner \ + -C "${DF_STUDENT_WORK_SOURCE}" \ + -cpf "${DF_BACKUP_PATH}/student-work.tar" \ + . + sudo chown \ + "${DF_BACKUP_OPERATOR_UID}:${DF_BACKUP_OPERATOR_GID}" \ + "${DF_BACKUP_PATH}/student-work.tar" + chmod 0600 \ + "${DF_BACKUP_PATH}/database.sql" \ + "${DF_BACKUP_PATH}/student-work.tar" + + ( + cd "${DF_BACKUP_PATH}" + sha256sum database.sql student-work.tar > SHA256SUMS + ) + ./compose.sh up --detach --wait --wait-timeout 300 + ./verify.sh .env.production 60 + + DF_MAINTENANCE_STARTED=0 + trap - ERR INT TERM +) +``` + +Stopping the writers creates a short outage but avoids a database record being +captured without its corresponding student file. A storage platform may instead +use an application-consistent snapshot mechanism, but document and test its +equivalent quiesce boundary. If any backup command fails, keep traffic stopped, +preserve diagnostics, and do not label the partial set usable. + +Encrypt the set, restrict access, validate checksums after transfer, and keep an +off-host copy. Record the matching secret-manager/escrow versions for all Rails +signing and Active Record encryption keys, database credentials, VAPID private +key, TLS material, and external-service configuration. Do not place those +secrets inside the Git repository or an unencrypted backup manifest. + +Redis/Sidekiq is not part of the authoritative recovery set. Restoring an old +queue can replay non-idempotent exports or notifications, while discarding it +can lose jobs accepted after the database recovery point. The maintenance +boundary minimises that window. The incident/recovery owner must record the +queue-loss decision, inspect database state, let cron recreate scheduled work, +queue a fresh PPI aggregation for approved units, and have users request lost +one-off exports/PDFs again. Never bulk replay notification jobs without a +product/privacy review because recipients may receive duplicates. + +### Isolated restore rehearsal + +Restore only onto an isolated host with empty data paths, blocked production +email/push egress, a test hostname/certificate, the exact release manifest, and +the matching escrowed Rails/encryption/VAPID material. Never test by overwriting +the live paths. + +1. Verify the encrypted set and its checksums before extraction. Keep the + restore path explicit rather than relying on a later working directory: + + ```bash + : "${DF_RESTORE_DIR:?Set the absolute restored backup directory}" + [[ "${DF_RESTORE_DIR}" == /* ]] + cd "${DF_RESTORE_DIR}" + sha256sum --check SHA256SUMS + ``` + +2. Restore `student-work.tar` to the empty `STUDENT_WORK_PATH`, preserving + numeric ownership, ACLs and extended attributes. Set the exact validated + target path; the block refuses a non-empty target: + + ```bash + ( + set -Eeuo pipefail + : "${DF_RESTORE_DIR:?Set the absolute restored backup directory}" + [[ "${DF_RESTORE_DIR}" == /* ]] + DF_STUDENT_WORK_TARGET=/srv/doubtfire/student-work + sudo install -d -m 0750 "${DF_STUDENT_WORK_TARGET}" + [[ -z "$(sudo find "${DF_STUDENT_WORK_TARGET}" -mindepth 1 -print -quit)" ]] || { + printf 'Student-work restore target is not empty.\n' >&2 + exit 1 + } + sudo tar --acls --xattrs --numeric-owner \ + -C "${DF_STUDENT_WORK_TARGET}" \ + -xpf "${DF_RESTORE_DIR}/student-work.tar" + ) + ``` + + Confirm the configured JPlag report/archive subdirectories are present and + container-readable. + +3. From the release's `production` directory, validate the isolated + `.env.production`, start only an empty MariaDB and Redis, wait for both to be + healthy, prove the target schema has no tables, then import the dump. Set + `DF_RESTORE_DIR` to the same absolute path used in step 1: + + ```bash + DF_PRODUCTION_DIR="$(pwd)" + : "${DF_RESTORE_DIR:?Set the absolute restored backup directory}" + [[ "${DF_RESTORE_DIR}" == /* ]] + test -x "${DF_PRODUCTION_DIR}/validate.sh" + ./validate.sh + ./compose.sh up --detach --wait --wait-timeout 120 \ + doubtfire-db redis-sidekiq + ./compose.sh exec -T doubtfire-db sh -ec \ + 'count="$(mariadb --batch --skip-column-names \ + --user="$MARIADB_USER" --password="$MARIADB_PASSWORD" \ + --execute="SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=\"$MARIADB_DATABASE\"")"; \ + test "$count" = 0' + ./compose.sh exec -T doubtfire-db sh -c \ + 'exec mariadb --user="$MARIADB_USER" \ + --password="$MARIADB_PASSWORD" "$MARIADB_DATABASE"' \ + < "${DF_RESTORE_DIR}/database.sql" + ``` + +4. Run `./deploy.sh` and `./verify.sh`. Confirm the restored migration version, + representative users/units/submissions, recent notification state, fresh PPI + regeneration, and several sampled student files by application download and + checksum. Exercise identity using an approved test integration and keep real + SMTP/push delivery blocked. + +5. Record start/finish time, achieved RPO/RTO, manifest, backup ID, checksums, + secret escrow versions, row/file sampling, queue-loss decision, verifier + output, discrepancies and named acceptance. Destroy the isolated copy under + the institution's data-handling policy. + +Rehearse at the receiving team's approved cadence and before a migration whose +rollback depends on restore. A failed or stale rehearsal blocks release. + +## Upgrades and rollback + +Before an upgrade: + +1. confirm the backup and restore test are current; +2. record the running image digests and Git revision; +3. review API migrations for backward compatibility; +4. update only the tested digests in `.env.production`; and +5. run `./validate.sh`, `./deploy.sh`, and `./verify.sh`, then complete the + printed manual gates. + +Afterward, repeat the verification checks and inspect worker queues and logs. +For an application rollback, enter a maintenance/drain window and restore the +previous image digests. Roll back the API before the web so the hardened web can +continue to accept either authentication callback form; do not expose an old +web image to a still-hardened API. Use the verifier and acceptance procedure +retained with the previous release manifest and repeat every applicable +release-independent manual gate before accepting the rollback. The current +`verify.sh` intentionally requires this release's exact schema, cron jobs and +PPI/VAPID contract; do not weaken it or claim its expected failure against older +images is a valid acceptance result. If the previous release has no compatible +automated verifier, record and complete a named `MANUAL-ROLLBACK-CONTRACT` gate +covering public and dependency readiness, migrations, queues, identity, +messaging, data integrity, ports and credential-safe logs. If a migration is not +backward-compatible, image rollback alone is unsafe; follow the migration's +documented rollback or restore the matching database and student-work recovery +set in a controlled outage, then run the same applicable automated and manual +acceptance gates. + +`deploy.sh` is a forward-only helper and must not be used for rollback because it +stages web first. Before release, the previous manifest must contain the exact +previous deploy checkout, environment/digest manifest, bounded API/app-worker- +before-web commands, migration decision, failure diagnostics and version-matched +acceptance procedure, all proven in rehearsal. Absence of that executable +rollback record is a release blocker rather than permission to improvise during +an incident. + +Never run `./compose.sh down --volumes` against production. The temporary +TexLive and JPlag volumes are disposable, but the host paths configured for the +database, Redis, logs, and student work are production data. + +## CI-only configuration rendering +`DOUBTFIRE_CONFIG_ONLY=1 ./validate.sh /path/to/fixture.env` skips only the live +Docker socket accessibility check. It exists for isolated CI validation and +must not be used by `deploy.sh` or as a production preflight substitute. diff --git a/DESKTOP-PWA.md b/DESKTOP-PWA.md new file mode 100644 index 000000000..6ea93df34 --- /dev/null +++ b/DESKTOP-PWA.md @@ -0,0 +1,236 @@ +# OnTrack on desktop + +OnTrack's desktop app is the existing website installed through a desktop +browser. It opens in its own window, uses the same account and API, and receives +web releases through the existing Angular service worker. Desktop and mobile +share the same web build; no additional server, API endpoint, database migration, +app store account, or native installer is required for desktop installation. + +Installation is not an offline submission feature. Previously cached interface +assets may load without a connection, but sign-in, current unit/task data, +submissions, comments, file uploads and downloads, and server notifications need +the network. Save work before reloading or applying an update. + +## Install and remove + +Use the institution's normal HTTPS OnTrack address in a regular browser window. +Open **Install OnTrack** below sign-in or in the account menu for guidance, or use the browser's +own install control. Browser policies, versions, and existing installations can +change which controls appear. A dismissed suggestion does not require clearing +site data: the account-menu entry remains available. + +| Desktop | Browser | Installation | +| --- | --- | --- | +| Windows | Chrome | Select the install icon in the address bar, or **More → Cast, save, and share → Install page as app**; confirm installation. | +| Windows | Edge | Select the app-available icon in the address bar, or **Settings and more → More tools → Apps → Install this site as an app**. | +| macOS | Chrome or Edge | Use the browser's install icon/menu; launch OnTrack from the installed app shortcut and optionally keep it in the Dock. | +| macOS Sonoma 14 or later | Safari | Open the OnTrack home page, choose **File → Add to Dock** (also available from Share), then **Add**. Open the Dock app and sign in if requested. | +| Linux | Chrome | Use the install icon or **More → Cast, save, and share → Install page as app**; launch from the desktop environment's application menu. | + +Chrome documents desktop installation, shortcuts and uninstall for Windows, +macOS and Linux in [Use web apps](https://support.google.com/chrome/answer/9658361?co=GENIE.Platform%3DDesktop&hl=en). +Edge documents installation and app management in +[Install, manage, or uninstall apps](https://support.microsoft.com/en-us/edge/install-manage-or-uninstall-apps-in-microsoft-edge). +Safari web apps require macOS Sonoma 14 or later and maintain their own cookies +and website data, so a new app may need another sign-in. See +[Use Safari web apps on Mac](https://support.apple.com/en-us/104996). + +Firefox on Windows also offers manual web apps, starting with version 143 +(version 150 for Microsoft Store installations). Its web-app behavior differs +from Chromium's manifest-driven install prompt; validate it separately if the +institution includes it in the support matrix. Do not present Firefox as +universally unable to install desktop web apps. See +[Mozilla's Windows web app guide](https://support.mozilla.org/en-US/kb/web-apps-firefox-windows). +Other browser/platform combinations can continue using the website. + +Launch the installed OnTrack shortcut to get the app window. Existing browser +tabs remain tabs. Desktop taskbar/Dock integration and shortcut menus vary by +browser and OS; the manifest's Home, Notifications and Unit Hub shortcuts are +optional browser features. An unauthenticated shortcut must send the user through +sign-in before displaying account data. + +For removal, use the installed app's browser menu: Chrome offers +**Uninstall OnTrack**, and Edge manages installed apps at `edge://apps`. +Safari creates the app in the user's Applications folder; move that app to Trash +to remove it. Removing a local app does not delete the OnTrack account or server +submissions. Deleting website data may remove local preferences and sessions. + +## Deployment contract + +Build the production `doubtfire-web` release with its existing Docker build and +Angular service worker enabled. The development server is not the installation +acceptance target. Keep the web app at the root of its canonical HTTPS origin, +with `/api` forwarded to the existing API. Retain production certificate checks; +HTTP is only suitable for localhost development. + +The web manifest is `/manifest.webmanifest`, linked from `/index.html` with a +root ``. The explicit app `id` and `start_url` both resolve to +`/index.html`, matching the previous implicit identity; `scope` stays `/` and +`display` is `standalone`. Keep that origin and identity stable across upgrades +to avoid a second installed app. Changing the installation origin is a migration +that requires separate browser acceptance. + +Serve these public assets directly, before any identity-provider redirect: + +| Resource | Response contract | +| --- | --- | +| `/index.html` and application navigation routes | HTML app shell, with root base and manifest link | +| `/manifest.webmanifest` | JSON with `application/manifest+json` or `application/json` | +| `/ngsw.json` | Angular-generated control manifest with `application/json` | +| `/ngsw-worker.js` | Angular worker JavaScript with a JavaScript content type | +| Manifest icons | Real PNG files with declared dimensions, including general-purpose 192×192 and 512×512 icons | +| Bundled scripts and styles | Correct JavaScript/CSS content types and bytes matching the generated `ngsw.json` hashes | + +The index, web manifest and worker control files must prevent stale caching; +query parameters must not bypass the policy. The existing web `nginx.conf` +provides these headers and returns real 404s for missing control files and +`/assets/` paths. The production proxy forwards those responses. Preserve these +rules in any institutional CDN, ingress, or reverse proxy. Fingerprinted bundles +may be cached normally. Do not upload a new `ngsw.json` ahead of its corresponding +assets or substitute the SPA HTML for a missing worker, manifest or icon. + +The API uses its existing authentication callbacks and push configuration. +Desktop installation needs no CORS exception, new auth callback scheme, or +additional VAPID key. A browser may return an external identity-provider flow to +a normal tab; acceptance must confirm the installed app can recover the session +or complete sign-in itself, with no login loop or lost destination. Safari's +separate app storage makes this particularly important. + +## Verify a candidate release + +From the deploy repository root, run the public-asset gate with Python 3.9 or +later (standard library only): + +```bash +python3 -B production/verify-pwa.py https://ontrack.example.edu --timeout 30 +``` + +Replace the example with the actual staging or production origin. For a +production build served through local Nginx, loopback HTTP is also allowed: + +```bash +python3 -B production/verify-pwa.py http://localhost:8080 +``` + +The command issues bounded GET requests to public static assets only, does not +use a login or call `/api`, verifies TLS, and rejects redirects. It bypasses +ambient proxy environment variables so it checks the named origin directly. +Each request has a 1–300 second timeout and 32 MiB response limit. It verifies +manifest identity, start/scope, icon PNG data, MIME types, fresh control-file +policy including query variants, entry bundles, and app-shell hash consistency. +Third-party font stylesheets are left to browser acceptance. It does not fetch +every lazy chunk, prove authentication, register a worker, install an OS app, +or verify notification delivery. A pass is evidence about the publicly served +assets, not full browser or operating-system certification. + +Run verifier regression tests without a running OnTrack stack or Docker: + +```bash +python3 -B production/tests/verify_pwa_test.py +``` + +CI runs these tests with the required deployment checks. They use a disposable +loopback HTTP server and synthetic assets; they do not contact a live account. + +## Browser acceptance and release + +1. Build the web image from the exact reviewed desktop-PWA web revision and run + its focused tests and production build. Use the release process in + [RELEASING.md](RELEASING.md) to publish the tested image and record its immutable + digest. Keep deploy/web revision references aligned with that release. +2. Deploy the candidate to a staging HTTPS origin using the existing + [deployment runbook](DEPLOYING.md), then run `production/verify-pwa.py` against + that public origin in addition to `production/verify.sh`. +3. On the actual operating systems and browsers the institution will support, + use a normal fresh browser profile and install OnTrack. Record OS and browser + versions, web image digest, origin, date, and outcome for each check below. +4. Also test an app installed from the previous release, then deliver the + candidate at the same origin. Confirm the existing app updates without a + duplicate installation, the update notice waits for the user's action, and + canceling/deferring the update keeps the current work available. +5. After acceptance, follow the normal production release procedure. Run both + production verification and the separate PWA public-asset command against + the final public origin, then repeat the installed-app smoke checks. Record + a real browser result before describing a platform as supported. + +| Smoke check | Expected result | +| --- | --- | +| Install from the browser and account menu | Guidance matches the platform; a supported browser prompts only after user action; canceling permits a later attempt. | +| Launch, close, relaunch; check taskbar/Dock icon | OnTrack opens in its app window with the expected icon and name; it does not install a duplicate at the same origin. | +| Identity-provider login and logout | Full external login round trip succeeds; callback credentials disappear from the URL; logout and a subsequent launch obey the institution's session policy. | +| Remember-me and non-remembered sessions | Relaunch behaves according to the selected option and existing session expiry rules, including Safari's separate app storage. | +| Home, Notifications, Unit Hub and a permitted task link | In-scope navigation and reload work; signed-out routes require sign-in; browser-provided shortcuts work where offered. | +| Resize, maximize and keyboard navigation | Existing desktop layout remains usable; install guidance traps/returns focus correctly; buttons and close controls are accessible. | +| Submit a permitted test file, download feedback, open an external link | Existing workflows still work; external content follows browser policy without losing OnTrack state. | +| Opt in to notifications using a real test account | OS/browser permission is requested on user action; a background notification arrives and its click opens the intended route. Never promise delivery after the browser is fully quit. | +| Go offline after an online visit, then reconnect | Cached shell may load, while API-backed operations cannot be accepted offline; reconnect restores current data without claiming an unsent submission succeeded. | +| Apply a second staged release | Existing update flow completes after work is saved; the running app and cached resources settle on the new version. | +| Uninstall and reinstall | Browser removal works; reinstall opens the same institution app and server data remains available after sign-in. | + +Repeat the existing mobile PWA acceptance on a representative Android and iOS +installation because these platforms share the manifest and worker. Browser +installation, SSO and push need interactive checks; a headless unit test cannot +replace this matrix. + +## Rollback and recovery + +Retain the prior source revision, web image digest, deploy checkout, and browser +acceptance record. Restoring a byte-identical previous image is **insufficient +for service-worker clients that already know that version**. In Angular 22.0.3, +an already-known `ngsw.json` hash produces `NO_NEW_VERSION_DETECTED` without +changing the worker's latest version. Both an existing window and a newly opened +window can therefore continue using the newer cached release after an old image +is restored. + +For a desktop-only web change with no backend or schema change: + +1. Rebuild the prior source revision as a **new recovery release**. Run the + production Angular build again so it generates a fresh `ngsw.json` timestamp + and version hash. Use the release publisher's `--no-cache-web` option to + prevent reuse of a Docker layer containing the old output. Give the image a + distinct immutable release tag and record its digest. Changing only a tag or + image label does not force Angular to rebuild. +2. Check that the recovery manifest differs from the failed and previous + manifests, and that all asset hashes match the recovery output. Preserve the + canonical origin, `/index.html` app identity and `/` scope. Publish the complete + coherent image, not individually edited control files or bundles. +3. Deploy through the existing change-control and Compose procedure. Run the + verifier belonging to that source revision against the public origin. +4. In an app still running the failed version, check for an update. Confirm the + recovery release produces the existing **Reload** notice, leaves unsaved work + intact until the user acts, and loads the intended recovery content after + saving work and choosing Reload. Verify a new app window as well, then check + `/ngsw/state` reports `NORMAL` and repeat the offline/reconnect smoke check. + +After pinning the intended recovery source in a clean, reviewed deploy checkout, +use the protected evidence directory and release-version setup from +[RELEASING.md](RELEASING.md#2-publish-attested-application-images), adding the flag +to its publication command: + +```bash +PUBLISH_RELEASE_CONFIRM=1 production/publish-release.sh --no-cache-web \ + registry.example.edu/ontrack "$DF_RELEASE_VERSION" linux/amd64,linux/arm64 \ + > "$DF_RELEASE_EVIDENCE_DIR/release-application-digests.txt" +``` + +The publisher still publishes the full five-image release; only the web build +bypasses cache. Review the output under the normal release policy, and deploy +only the intended component digests. The flag does not deploy or clear browser +data. Use a new evidence file and immutable release version for each attempt. + +For a combined application release, use the ordered API/web rollback and +migration decision in [DEPLOYING.md](DEPLOYING.md#upgrades-and-rollback), applying +the same fresh-recovery-build requirement to the web image. The forward-only +deploy helper is not a rollback procedure. + +An open window can remain on its prior version until the user reloads; publishing +a recovery image is not an instant client rollback. Check control-file caching +at the public edge if the recovery update is not found. Avoid clearing site data +or changing app identity as the default recovery step: that can remove local +work/preferences or create a second installation. + +An older mobile-PWA release +may omit the explicit identity or description required by this desktop verifier; +that expected mismatch must not be reported as a passing desktop release. +Repeat the applicable login, navigation, upload/download, notification and +update smoke checks before accepting the rollback. diff --git a/MIGRATING.md b/MIGRATING.md index 5d1cb33f9..187f0c0cc 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -1,24 +1,104 @@ -![Doubtfire Logo](http://puu.sh/lyClF/fde5bfbbe7.png) # Migrating between Doubtfire Versions -## Version 5 to Version 6 +## OnTrack 11.0.x combined all-features release (24 August 2026) + +This dossier applies only to the API revision pinned in `HANDOVER.md`. The +expected final Rails schema version is `20260824000003`; `production/verify.sh` +fails if the running database does not match it. Re-audit this matrix if the API +revision or migration set changes. + +The supported paths are a fresh empty database or an exact prior release/schema +that the receiving institution records in the change manifest and successfully +rehearses from a production-sized isolated restore. This dossier does not claim +that an arbitrary older OnTrack database can upgrade directly. Any unrecorded +source version, skipped historical migration, vendor fork, or schema drift is a +separate blocked migration project until its full path and rollback have been +reviewed and rehearsed. + +The production stack runs migrations as a one-shot service before starting the +new API and workers. The hardened web is staged first for authentication +callback compatibility. Do not run multiple migration jobs, use `db:populate`, +or run any demo seed task against production. + +### Migration matrix + +| Version | Change and expected data work | Availability and rollback classification | +| --- | --- | --- | +| `20260722000001` | Creates `notifications` and its user/read index. | Additive. An application rollback can leave the table in place; reversing it deletes notification history. | +| `20260802000001` | Adds required notification `event`, backfills any existing rows to `legacy`, adds an index, then removes the write default. | May alter/lock `notifications`. Do not roll back to an intermediate app that writes notifications without `event`. Reversal deletes event provenance. | +| `20260802000002` | Widens notification `message` from `VARCHAR(255)` to `TEXT`. | The alter may rebuild/lock the table. Forward schema is compatible with older readers. Reversal is lossy or can fail once a message exceeds 255 characters. | +| `20260802000003` | Creates `push_subscriptions` and a unique endpoint index. | Additive. Reversal deletes every browser subscription and forces opt-in again. | +| `20260809153000` | Creates privacy-internal `peer_progress_snapshots` and its unique unit/task/grade index. | Additive. Reversal deletes aggregates; the student endpoint must remain disabled/fail closed until regenerated. | +| `20260810033824` | Adds `units.peer_progress_enabled`, default `false`, not null. | Safe default keeps all existing units opted out. The alter may lock `units`. Reversal loses unit opt-in decisions. | +| `20260818160804` | Adds `projects.target_grade_changed_at`, backfills every existing project to migration time, retains a database current-time default, then enforces not null. | The bulk update may lock or generate substantial redo on a large `projects` table. The retained default protects older writers. Reversal loses freshness history and requires new PPI aggregation after re-upgrade. | +| `20260824000001` | Adds task-notification tracking/default/index plus notification dedupe key, delivery timestamp and unique user/dedupe index. | Alters `task_definitions` and `notifications`. Additive to old readers. Reversal loses dedupe/delivery state and can cause duplicate mail after re-upgrade. | +| `20260824000002` | Repairs/retains the database default for `projects.target_grade_changed_at` on installations that previously recorded an older form of the preceding migration. | Idempotent metadata repair with no row backfill. Its down path intentionally preserves the compatibility default. | +| `20260824000003` | Adds nullable, internal `peer_progress_snapshots.submitted_count` and `peer_progress_snapshots.status_counts` JSON, plus `users.display_peer_progress` with database default `true` and `NOT NULL`. Exact counts let the student API subtract the authenticated reader before privacy thresholding and quantisation. Existing snapshots keep their legacy stored percentage for older readers, but the new API deliberately withholds compact and detailed output until reaggregation fills both exact fields. Existing and future users start with the requested display preference enabled and may persist `false` through Profile. | Additive for old readers/writers, but the `users` alter/default backfill may lock a large table and must be measured. Reaggregate every approved PPI unit before acceptance. Prefer keeping the forward schema during application rollback. Reversal removes peer-only aggregate inputs and erases every explicit user opt-out, so re-upgrade would default those users on again unless the preference is separately restored. | -Version 6 includes changes to database encryption that require custom migration processes in order to retain current authentication tokens in use. Where this is skipped, the standard migration will lose these tokens and users will need to login again. +No migration intentionally deletes an existing pre-release application table or +column. Database reversal is nevertheless not the preferred application +rollback: several `down` paths destroy new feature state, and the message-width +reversal is explicitly lossy. For a compatible previous release, normally keep +the forward schema and roll back application images during a maintenance window. +Use database restore only when the reviewed previous application is not forward- +schema compatible or data integrity is in doubt. -1. While running Version 5: +### Required rehearsal and evidence + +Before the change window, restore a recent production-sized recovery set to an +isolated host using `DEPLOYING.md`, then record: + +- exact source/target schema versions and row counts for `projects`, + `task_definitions`, `notifications`, `units` and `users`; +- table sizes, free disk space, migration start/finish time, longest metadata or + row lock, database CPU/IO/redo growth, and any blocked query; +- counts before/after the project timestamp backfill and confirmation that no + resulting value is null; +- confirmation that existing units remain PPI-disabled, notification dedupe + indexes are valid, no user has a null PPI display preference, the new API + suppresses legacy snapshots whose exact count fields are still null, and no + synthetic/demo records were introduced; +- `rails db:abort_if_pending_migrations`, `production/verify.sh`, and application + smoke-test results; and +- whether the measured outage fits the institution's approved maintenance + window and RTO. + +Fresh-install and simulated previously-recorded migration repair tests passed +against MariaDB 12.3 during branch validation. That proves migration semantics, +not production-scale duration; the receiving team's rehearsal remains a release +gate. + +Immediately before migration, create the consistent database/student-work +recovery set described in `DEPLOYING.md`, verify its checksums and restore-test +record, drain traffic, and check for pending/long-running database work. After +migration, require schema `20260824000003`, inspect Sidekiq queues/retries, queue +the first PPI aggregation only for an approved unit, verify its exact aggregate +fields are populated without printing them, and complete every manual gate in +`HANDOVER.md`. + +## Version 5 to Version 6 - The authentication tokens are encrupted in the database in a way that will not be accessible in version 6. In order to migrate the tokens, you will need to login to the server and save the current authentication tokens using the following script in the rails console. This will get the auth tokens for all logged in users, and export them to a tokens.rb file in the log folder. Alternate locations can be used as required. +Version 6 changed database encryption, so a direct upgrade cannot preserve old +sessions without a dedicated token migration. The former procedure in this file +wrote every live bearer token as executable plaintext under the application log +directory. That procedure is not approved for use: it did not guarantee mode +0600, encrypted transport or storage, safe escaping, deletion, expiry or +rotation. - ```ruby - lines = User.where("auth_token_expiry > CURRENT_DATE()").map { |user| "AuthToken.create(user_id: #{user.id}, authentication_token: '#{user.auth_token}', auth_token_expiry: DateTime.parse('#{user.auth_token_expiry}'))" }.join("\n") - File.write("#{Rails.root}/log/tokens.rb", lines) - ``` +The supported safe default is to skip token preservation and require every user +to sign in again after migration. If preserving sessions is a business +requirement, stop the release until the security and identity owners approve and +test a one-time migration tool that writes outside logs and routine backups, +uses mode 0600 plus encrypted transfer/storage, records no token values in +evidence, verifies import counts without printing credentials, and securely +deletes or cryptographically destroys the export immediately after acceptance. +Rotate or expire the migrated tokens on an approved schedule. -2. Ensure you have sufficient space to recreate the database - we suggest ensuring you can double the size of the current database during the migration process. This extra size will reduce after migration, once the tables have been re-written. -3. Backup the database prior to upgrading, as data will be lost in the process and it will be easier to restore from a backup if the migration fails. -4. Update to Version 6 and migrate the database. -5. Execute the exported script to create the required authentication tokens. Failure to do this will only result in users needing to login again to continue use of the application. +Ensure there is enough space to recreate the database (up to twice its current +size during the migration), take and restore-test a complete backup, migrate in +a maintenance window, and verify that all prior sessions are invalid when using +the safe default. ## Version 4 to Version 5 diff --git a/README.md b/README.md index ae43f755e..5acaa349b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -![Doubtfire Logo](http://puu.sh/lyClF/fde5bfbbe7.png) # Doubtfire Deploy @@ -6,17 +5,44 @@ Doubtfire is a feedback-driven learning support system. The Doubtfire Deploy repository is used to manage releases of Doubtfire using containers. +## Documentation + +- [Demo guide](DEMO.md) +- [Release guide](RELEASING.md) +- [Desktop PWA installation and release checks](DESKTOP-PWA.md) +- [Migration guide](MIGRATING.md) +- [Notifications integration](NOTIFICATIONS-INTEGRATION.md) +- [All-features integration](ALL-FEATURES-INTEGRATION.md) +- [Unit Hub, Teams announcements and class calendars](UNIT-HUB.md) +- [Development Docker files](development/development-dockerfiles.md) +- [OnTrack Podman setup](docs/ONTRACK_PODMAN_SETUP.md) + ## Table of Contents - [Doubtfire Deploy](#doubtfire-deploy) + - [Documentation](#documentation) - [Table of Contents](#table-of-contents) - [How to use this project for development](#how-to-use-this-project-for-development) - [How to use this project for deployment](#how-to-use-this-project-for-deployment) + ## How to use this project for development -Refer to [CONTRIBUTING.md](CONTRIBUTING.md) for details on getting started contributing to Doubtfire. +Use [RUNNING-LOCALLY.md](RUNNING-LOCALLY.md) for the current combined checkout +and demo workflow. [CONTRIBUTING.md](CONTRIBUTING.md) retains general project and +commit guidance, but its older runtime and branch examples are not the 11.0.x +release contract. ## How to use this project for deployment - +Start with the release-owner checklist in [HANDOVER.md](HANDOVER.md), then use +the fail-closed production Compose stack and operator runbook in +[DEPLOYING.md](DEPLOYING.md). The deployment requires institution-owned secrets, +TLS, identity-provider and SMTP settings, persistent storage, monitoring, +restored backups, manual acceptance, and immutable container image digests; +none are defaulted in the repository. + +The isolated all-features demonstration remains available under +[`development/all-features-demo`](development/all-features-demo/README.md). It +uses synthetic data and development-only accounts and is never production +configuration. diff --git a/RELEASING.md b/RELEASING.md index 6b51a3072..fb8b2730e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,14 +1,154 @@ -![Doubtfire Logo](http://puu.sh/lyClF/fde5bfbbe7.png) +# Publishing an OnTrack production release -# Releasing a new version of Doubtfire +This procedure turns the exact API and web gitlinks in this repository into +immutable production images. It complements `HANDOVER.md` (release ownership +and acceptance), `MIGRATING.md` (schema dossier), and `DEPLOYING.md` (host +configuration, rollout, verification and recovery). -## How to use this project for releasing new versions +The historical root `release.sh` only creates source tags. It does not build, +scan, attest, sign or deploy the production API, app-worker, web, TexLive and +JPlag artifacts. Do not use it as evidence that a deployable release exists. -Once you have a working product in the development environment described above, the next step is to prepare a release. +## 1. Lock and approve source -1. Commit and push everything. Make sure that all sub modules, and the deploy project are up to date and have all of the changes you want included. -2. Make sure that each sub module has incorporated any other changes from development into its branch. -3. Run `build.sh` this will prepare local containers that mirror those that will be made available to deployments. -4. Test the local containers - run `docker compose up` from the doubtfire deploy project root. You can test the solution using http://localhost:3000/ -5. Make any necessary changes to correct issues identified. Repeat 3, 4, and 5 until you are ready to release. -6. Run `release.sh` +1. Complete human review and required checks on the API, web and deploy pull + requests. Keep the umbrella branches draft until their focused review gates + are satisfied; an integration branch does not bypass repository rules. +2. Pin `doubtfire-api` and `doubtfire-web` to the accepted commits and update + their revision rows in `HANDOVER.md` in the same deploy commit. The deploy + commit cannot embed its own hash; its protected signed tag is resolved and + recorded in the external release manifest after that commit exists. +3. From a fresh recursive clone, prove both gitlinks and the nested web JPlag + submodule initialise, then run the automated API/web/deploy checks and the + isolated all-features demo. +4. Create protected signed API and web release refs at the pinned component + commits, then tag the accepted deploy commit under the same retention policy. + Record all three refs and resolved Git revisions in the change record. Prove + a fresh recursive clone can fetch them through this repository's configured + submodule URLs before publishing. + +The API and app-worker images must always be built from the same API commit. +The web image must be built from the pinned web commit. Never rebuild a release +tag from a different checkout and never promote a mutable tag by name alone. + +## 2. Publish attested application images + +Use an institution-controlled OCI registry with retention/deletion protection. +Authenticate Docker, create a clean recursive clone at the accepted deploy +commit, and run: + +```bash +( + set -Eeuo pipefail + set -o noclobber + umask 077 + : "${DF_RELEASE_VERSION:?Set the release-owner-approved version before publishing}" + : "${DF_RELEASE_EVIDENCE_DIR:?Set an absolute protected directory outside the Git checkout}" + [[ "${DF_RELEASE_EVIDENCE_DIR}" == /* ]] + test -d "${DF_RELEASE_EVIDENCE_DIR}" + PUBLISH_RELEASE_CONFIRM=1 \ + production/publish-release.sh \ + registry.example.edu/ontrack \ + "${DF_RELEASE_VERSION}" \ + linux/amd64,linux/arm64 \ + > "${DF_RELEASE_EVIDENCE_DIR}/release-application-digests.txt" +) +``` + +Choose the version under the release owner's normal versioning policy and record +it in the manifest. Do not copy an example version: `CHANGELOG.md` already +contains 11.0.1, and publishing an older-looking version would make promotion +and rollback evidence ambiguous. The publisher rejects mutable names, `.x` +series labels and common placeholder text, but the registry must still enforce +tag immutability and retention. Keep the evidence output outside the clean +checkout: shell redirection creates its target before the publisher checks Git, +so writing it into the repository would correctly make publication fail. A +nonzero publisher exit leaves incomplete evidence that must be quarantined and +must never be accepted as a release manifest. + +For a PWA recovery release rebuilt from prior web source, add `--no-cache-web` to +the publisher invocation. This bypasses Docker's build cache for the web image +so Angular generates a fresh service-worker manifest; the other four images +retain their normal cache behavior. A new tag or version label alone does not +invalidate the cached Angular build. Use a distinct immutable recovery version +and follow [desktop client recovery](DESKTOP-PWA.md#rollback-and-recovery), +including acceptance in a client still running the failed release. + +The publisher refuses dirty or mismatched component worktrees, then creates +temporary build contexts only from files tracked by the pinned Git commits. It +populates nested web assets from the exact nested gitlink, so ignored or +untracked host files cannot enter an image through `COPY .`. It builds: + +| Manifest key | Source | +| --- | --- | +| `DOUBTFIRE_API_IMAGE` | `doubtfire-api/deployApi.Dockerfile` | +| `DOUBTFIRE_APP_IMAGE` | `doubtfire-api/deployAppSvr.Dockerfile` | +| `DOUBTFIRE_WEB_IMAGE` | `doubtfire-web/deploy.Dockerfile` | +| `TEXLIVE_IMAGE` | `doubtfire-api/texlive.Dockerfile` | +| `JPLAG_IMAGE` | `doubtfire-api/jplag.Dockerfile` | + +Every build is multi-platform, labels each image with its verified component Git +revision and release version, pushes a manifest, and requests BuildKit SBOM and +maximum provenance attestations. The stdout file contains Git revisions and +`image@sha256:...` references; protect it as change evidence. The script never +reads or writes `.env.production` and does not print registry credentials. Tags +are only publication labels and may be mutable unless the registry enforces +immutability; the digest is the release identity. + +## 3. Supply-chain acceptance + +Before an image digest can enter production configuration: + +- verify its registry provenance/SBOM attestations identify the expected Git + revision, Dockerfile, build platform and builder identity; +- scan the immutable digest with the institution-approved container and licence + scanners, triage every critical/high result, and record any time-bounded risk + acceptance with an owner and expiry; +- sign the digest using the institution-approved keyless or protected-key + identity, verify the signature under policy, and retain transparency/audit + evidence; +- confirm API/app-worker labels and bundled source identify the same API commit, + the web JPlag assets are populated, and no secrets appear in layers, history, + SBOM or build logs; and +- retain the exact digest, attestations, signature and scan report for rollback, + rather than relying on a tag that can move. + +The publisher does not choose the Nginx proxy, MariaDB, Redis or HAProxy images. +Select separately reviewed versions, resolve their multi-platform digests, scan +and sign/verify them under the same policy, and record them in the release +manifest. Refresh a pinned base or runtime image only as a reviewed dependency +change followed by a complete rebuild and acceptance cycle. + +## 4. Complete the manifest and stage + +Copy accepted immutable references into a protected `.env.production` created +from `production/.env.production.example`. Complete the release-manifest fields +in `HANDOVER.md`, including the previous release, migration target, recovery +set, secret-escrow versions, scans, attestations, signatures, manual owners and +rollback decision. + +Run in a production-equivalent staging environment: + +```bash +cd production +./validate.sh +./deploy.sh +./verify.sh +``` + +Complete the real IdP, SMTP, PPI, push-device, PDF/JPlag, SCORM and recovery +gates without production student data. The forward helper stages compatible web +before API. If LTI is separately enabled, its redirector must be verified not to +place `ltik`, auth tokens or usernames in a query URL. + +## 5. Production promotion + +Promote by digest, not by rebuilding. Re-run `validate.sh` against the exact +production environment file, confirm the consistent backup/restore evidence and +migration rehearsal are current, then follow `DEPLOYING.md`. Save verifier and +manual-gate evidence and obtain the named release owner's go/no-go. + +A release is complete only when monitoring and on-call routing are active, the +final manifest is immutable, and both forward and rollback references are +retained. A failed gate stops promotion; do not relabel or retag an unaccepted +digest to work around it. diff --git a/RUNNING-LOCALLY.md b/RUNNING-LOCALLY.md new file mode 100644 index 000000000..145a61750 --- /dev/null +++ b/RUNNING-LOCALLY.md @@ -0,0 +1,602 @@ +# Running OnTrack locally (web and api) + +How to run OnTrack on your computer with Docker. It also lists the problems we hit and how +to fix them. + +## What runs + +- doubtfire-api: the backend (Rails). Host port 3200, container port 3000. +- doubtfire-sidekiq: the background worker for queued notification email. See below. +- doubtfire-web: the frontend (Angular). Host port 4400, container port 4200. +- Mailpit: catches every email the app sends. Web inbox on host port 8225. +- A database (MariaDB) and Redis. Docker starts these for you. + +## Before you start + +- Install Docker Desktop and start it. +- Set your git remotes. `origin` is the team org, `upstream` is thoth-tech. +- Do not use the `development` branch. It is old and frozen. +- Do not install Ruby or Node. They run inside the Docker images. The api needs Ruby 3.4. + The web needs Node 22.22.3 or newer. +- Do not run `rails`, `rubocop`, or `bundle` on your own computer. Your Mac has old Ruby + (2.6). Run those inside the container instead. + +## Clone all three repos side by side + +This is the most common reason the build fails. + +Docker builds the api and web containers from folders it expects to find next to the deploy +folder. The compose file hardcodes `../../doubtfire-api` and `../../doubtfire-web`. If your +folders have different names, or are nested, or you only cloned the deploy repo, the build +fails and the error will not tell you why. + +Make one parent folder and clone all three into it: + + mkdir ontrack && cd ontrack + git clone https://github.com/ontrack-features-t2-2026/doubtfire-deploy.git + git clone https://github.com/ontrack-features-t2-2026/doubtfire-api.git + git clone https://github.com/ontrack-features-t2-2026/doubtfire-web.git + +You should end up with exactly this: + + ontrack/ + doubtfire-deploy/ + doubtfire-api/ + doubtfire-web/ + +Do not rename the folders. Do not put doubtfire-api inside doubtfire-deploy. The deploy repo +already has empty folders with those names. They are uninitialised submodules. Your code +does not go there. + +## Combined all-features 11.0.x + +For a deterministic, disposable demonstration with a UI data switch, use the +isolated [all-features demo runtime](development/all-features-demo/README.md). +It owns a separate Compose project and database and is the preferred option for +showing CPD, PPI, and notification examples without retaining ad-hoc seed data. + +The final source revisions are recorded in [HANDOVER.md](HANDOVER.md) and pinned +by this repository's API and web gitlinks. A fresh clone can initialise them +recursively. The demo launcher resolves sources in this order: explicit +`DF_DEMO_API_PATH`/`DF_DEMO_WEB_PATH`, complete sibling checkouts, then the +gitlinks. It fails before Compose if the chosen source is incomplete. + +For editable sibling checkouts, use the published integration branch in each +repository and verify that its exact revision matches `HANDOVER.md`: + +- **doubtfire-deploy**: `integration/deploy-all-features-foundation-20260824` +- **doubtfire-api**: `integration/11.0.x-all-features-20260824` +- **doubtfire-web**: `integration/11.0.x-all-features-20260824` + +This is the singular local CPD, PPI, Email Notifications, and Mobile Notifications +integration. Do not mix it with the older individual feature branches. The api and web +containers run exactly what is checked out in the sibling folders, including uncommitted +changes. + +The normal combined stack uses the base and local-path files under the dedicated +`notifications-demo` Compose project. That project name keeps its database and dependency +volumes separate from any older base-only stack and matches the notification verification +helpers. It exposes web, API, and Mailpit on , +, and respectively. Use the +ordered startup under **Steps to run** so a destructive demo seed cannot race +an eagerly starting API or worker. + +To apply the retained isolated PPI setup as well, put its overlay last on every Compose +command: + +```bash +docker compose -p ppi-live \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.ppi-live.yml \ + up -d --build +``` + +With the PPI overlay last, the effective web, API, and Mailpit ports are 4300, 3100, and +8125. The worker and API use the same PPI API image and generate links to the web app at +. + +If either overlay is missing, one of the repositories is not on the combined integration +branch. + +## Steps to run + +All commands run from the deploy folder: + + cd doubtfire-deploy/development + +**Keep the same `-p` project name and use both `-f` flags on every command.** The second file +points the build at your sibling folders and fixes the api proxy. If you selected the PPI +setup above, use `-p ppi-live` and include its third +`-f docker-compose.ppi-live.yml` flag last on every command too. + +**Do not run `run-api-web.sh`.** It sits in this folder and looks like the way to start +things. It leaves out the second `-f` flag and fails on an empty build context. + +1. Build the application images. Do this the first time and after switching the + integration revision. + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml build \ + doubtfire-api doubtfire-sidekiq doubtfire-web + + The first build is slow. It installs gems and node packages. + +2. Start only the infrastructure and wait for it to be ready. + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml up -d --wait \ + dev-db redis-sidekiq mailpit + +3. Set up the database. Do this the first time, or any time the database is broken. + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml run --rm --no-deps doubtfire-api \ + bash -c "bundle exec rake db:populate" + + `db:populate` already drops, creates, migrates and seeds the database on its own. The + longer command you may see elsewhere does the slowest part of setup twice. + + It takes a while and prints a lot. That is normal. + + Keep the API and worker stopped until this finishes. `db:populate` rebuilds + the schema; letting an application entrypoint load the same schema at the + same time can produce a duplicate-constraint failure. + +4. Start the application processes and wait for them. + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml up -d --wait --wait-timeout 300 \ + doubtfire-api doubtfire-sidekiq doubtfire-web + +5. Open the app. + + - Web: http://localhost:4400 + - API docs: http://localhost:3200/api/docs + - Mail inbox: http://localhost:8225 + +6. Log in. Every test user has the password "password". + + - Student: student_1 + - Admin: aadmin + - Convenor: aconvenor + - Tutor: atutor + + You will usually want two of these signed in at once, because most notification work is + one person doing something and another person being told about it. Use a private browser + window for the second account instead of logging in and out. + + A student's dashboard is at `/projects//dashboard`. There is no top-level + `/dashboard` page. Typing that address sends you back to the home page. + +## Where the emails go + +**Open http://localhost:8225** + +That is Mailpit, a mail catcher. Every email the app sends arrives there and you can read +it in your browser, subject, recipient and all. New mail appears without reloading the page. + +The app never sends real email in development. Mailpit accepts everything and forwards +nothing, so you can safely put your own address on a test account. + +- Web inbox: http://localhost:8225 +- The api sends to it over SMTP on port 1025 inside Docker. + +The `doubtfire-sidekiq` service takes provider delivery off the api request path. It +reads email from `mailers` and Web Push from `notifications`; the normal `up` command +starts it and sends caught email to Mailpit. + +Both provider channels are asynchronous. If the worker is stopped, the in-app bell record +still appears immediately, while email and push wait safely in Redis until it starts again. + +The worker listens only on `mailers` and `notifications`, so it does not run the rest of +the background jobs on `default`. That is on purpose. Several of them need a LaTeX +container this stack does not have, and a worker that picked those up would fail every +task submission and push the task back to "fix". + +Check the worker and its recent job output: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml ps doubtfire-sidekiq + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml logs --tail 100 doubtfire-sidekiq + +Stopping the worker does not lose queued notification email or push delivery. Starting it +again processes the pending channel work: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml stop doubtfire-sidekiq + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml start doubtfire-sidekiq + +If the inbox stays empty: + +1. Check the service is running: + `docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml ps mailpit` +2. Check the api knows about it: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml exec doubtfire-api printenv DF_SMTP_ADDRESS + + You want `mailpit`. If it is blank, your api container was started before the + mail catcher was added. Recreate it: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml up -d doubtfire-api + + A plain `restart` is not enough. Environment variables only change on recreate. + +**Without Docker**, or if `DF_SMTP_ADDRESS` is unset, the api falls back to writing each +email to a file under `doubtfire-deploy/data/tmp/mails/`. One file per recipient address, +with new mail appended to the end. That is the old behaviour and it still works. + +The comment in `doubtfire-api/config/environments/development.rb` used to say mail landed in +`doubtfire-api/tmp/mails`, which was wrong under Docker and sent people looking in an empty +folder in the wrong repository. That comment is now fixed. + +## How to check it is working + +- See the containers: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml ps -a + +- Check the api answers, from inside the container: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml exec doubtfire-api curl -s localhost:3000/api/settings/public + +- Check the web can reach the api through its proxy. You want 200: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml exec doubtfire-web curl -s -o /dev/null -w "%{http_code}\n" localhost:4200/api/settings/public + +- Check the mail catcher answers. You want 200: + + curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8225/ + +- List what is in the mail inbox without opening a browser: + + curl -s http://localhost:8225/api/v1/messages | head -c 400 + +- Read the logs: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml logs doubtfire-api + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml logs doubtfire-sidekiq + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml logs doubtfire-web + +## Asking for help + +Grab these before you ask. The second one answers most questions on its own. + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml ps -a + +On macOS or Linux: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml logs --no-color --tail 200 doubtfire-api > api-log.txt 2>&1 + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml logs --no-color --tail 200 doubtfire-web > web-log.txt 2>&1 + +**On Windows, wrap it in `cmd /c` or the file comes out unreadable.** + + cmd /c "docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml logs --no-color --tail 200 doubtfire-api > api-log.txt 2>&1" + cmd /c "docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml logs --no-color --tail 200 doubtfire-web > web-log.txt 2>&1" + +PowerShell does two things to a plain `>` redirect that ruin the file. It writes UTF-16, so +every character comes out with a null byte next to it and most tools see binary rather than +text. And it treats anything the command sends to stderr as a PowerShell error object, so the +real message gets buried under `At line:1 char:1`, `CategoryInfo` and `FullyQualifiedErrorId` +noise, with the actual error split away from its own stack trace. `cmd /c` does neither. + +Use Compose `ps -a` and not plain `docker ps`. Plain `docker ps` hides containers that have +already exited, and a container that exited is usually the whole problem. If the +`doubtfire-api` service says Exited, its log says why. + +**Send logs as text, not as a screenshot.** Attach the two files, or paste the output inside +a fenced code block with three backticks. A screenshot of a terminal crops the part that +matters, cannot be searched, and in a Ruby crash the line you need is usually well below the +line you can see. Text can be matched against the errors in the next section in seconds. A +screenshot cannot. + +Screenshots are still the right thing for anything visual. "The page says Temporarily +Unavailable" is a screenshot, because the rendering is the evidence. Anything with a stack +trace in it is text. + +Say which branch each repo is on as well, and which Compose overlays you used. A lot of the +answers below turn on those two things. From `doubtfire-deploy/development`: + + git branch --show-current + git -C ../../doubtfire-api branch --show-current + git -C ../../doubtfire-web branch --show-current + +## Problems and fixes + +1. The api will not start. Error: "Your Ruby version is 3.1.7, but your Gemfile specified + ~> 3.4.0". + Cause: the image was built with old Ruby. `11.0.x` needs Ruby 3.4. + Fix: rebuild the images. Add `--build` to the up command. + +2. The web will not start. Error: "The Angular CLI requires a minimum Node.js version of + v22". + Cause: the image was built with old Node. `11.0.x` needs Node 22. + Fix: rebuild the images. Add `--build`. + +3. `up` stops straight away. Error: "service overseer-worker-1 has neither an image nor a + build context". + Cause: an old local-paths file had overseer services with no image. + Fix: already fixed. The combined local-paths overlay contains complete overrides only + for services used by this stack. + +4. The web crashes. Error: "Missing script: start-compose". + Cause: `11.0.x` renamed that script to "start". + Fix: already fixed. The local-paths file runs "npm start". + +5. The api crashes while migrating. Error: "Table 'doubtfire-dev.task_prerequisites' doesn't + exist". + Cause: the database has old, half-set-up data. The api container runs `db:migrate` every + time it starts, so a half-populated database makes it crash on boot over and over, before + it ever listens on port 3000. + Fix: reset the database. Run step 2 above. + + **If step 2 fails too, throw the database away and start it again.** Step 2 asks MariaDB + to drop the database, and a server that cannot read its own files cannot drop them + either. `-v` deletes the volume the database lives in, which is the only thing that + clears it. + + ```bash + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml down -v + ``` + + Then repeat **Steps to run** 1–4. Those steps wait for MariaDB, keep API and + worker schema readers stopped during `db:populate`, and start them only after + the seed is complete. + + This deletes your local data. That is fine, everything in it came from `db:populate` and + the command above puts it all back. `-v` also clears the web `node_modules` volume, so the + next start is slower while npm reinstalls. Your code is untouched either way: the repos + are bind mounted, not copied. + +6. The app loads but shows "Temporarily Unavailable" and the title stays "Loading...". + **Check the api is running before you read any further.** Run the Compose `ps -a` command + under **How to check it is working**. If the `doubtfire-api` service is missing or says + Exited, this is not a proxy problem; it is problems 5, 10 or 14, and the Compose service + log says which. + Cause: the web app cannot reach the api. The proxy points at localhost:3000, which is + wrong inside the container. The api is a different Compose service named doubtfire-api. + Fix: already fixed. The local-paths file mounts `proxy.conf.docker.json`, which points at + doubtfire-api:3000. If you still see the error, rebuild the web container and reload: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml up -d --build doubtfire-web + +7. The build fails straight away, or complains about an empty or missing build context. + Cause: your folders are not laid out the way the compose file expects, or you only cloned + the deploy repo. + Fix: see "Clone all three repos side by side" above. All three must sit next to each + other, with their original names. + +8. You switched branch, and now the web container fails on a package it should have. + Cause: node_modules lives in a Docker volume that survives `docker compose down`, so a + branch with different dependencies installs on top of stale packages. + Fix: clear the volume and rebuild. + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml down -v + + **`-v` does delete your database**, because that is a volume too. Repeat + **Steps to run** 1–4 to rebuild, repopulate, and start safely. Your code is + not touched; the repos are bind mounted rather than copied. + +9. You trigger an email and nothing appears at http://localhost:8225. + Cause: nearly always an api container started before the mail catcher existed, so it + still has no `DF_SMTP_ADDRESS` and is writing files instead. + Fix: recreate it. `restart` does not pick up new environment variables. + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml up -d doubtfire-api + + Confirm with the Compose `exec doubtfire-api printenv DF_SMTP_ADDRESS` command under + **Where the emails go**, which should print `mailpit`. + +10. The api container will not start. Error: "Could not find in locally installed + gems (Bundler::GemNotFound)". + Cause: somebody added a gem to the api `Gemfile`. Gems are installed into the image when + it is built, not into a volume, so a container started from the old image does not have + it. The api then crash-loops before it ever listens on port 3000. + Fix: rebuild the image. + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml up -d --build doubtfire-api + + Running `bundle install` with Compose `exec` looks like it works and does not survive. + The gems land in the running container's writable layer and are thrown away the next + time the container is recreated. + +11. The app starts throwing 500s while you are using it, and the log says + "ActiveRecord::LockWaitTimeout: Lock wait timeout exceeded". + Cause: the base Compose file uses one database for development and tests. The combined + local-paths overlay fixes this by creating and selecting + `doubtfire-notifications-test`. Seeing this error usually means the api was started + without the required local-paths overlay. + Fix: stop the base-only stack and restart with both required `-f` flags. Do not run a + database reset while anyone is using the app or during a demo. + +12. You post a comment, get a 403 back, and no email arrives. The error says "Comment + duplicates last comment, so ignored". + Cause: OnTrack drops a comment whose text is identical to the previous comment on that + task. No comment is created, so no notification and no email. This is existing behaviour + in `app/api/task_comments_api.rb`, not something notifications introduced. + Fix: type something different. When rehearsing a demo, vary the text each time. + +13. `git status` in doubtfire-deploy shows an untracked `doubtfire-overseer/` folder. + Cause: it is a leftover checkout from another branch. `11.0.x` does not use it. Most + people will never see it. + Fix: none needed. Leave it alone. Do not `git add` it and do not delete it. + +14. `rake db:populate` fails part way through. Error: "Error on rename of + './doubtfire@002ddev/' to './doubtfire@002ddev/#sql-backup-1-7' (errno: 194 + "Tablespace is missing for a table")". + **This is a Windows problem and it is not your data.** It happens on a completely fresh + database, so deleting things and starting again does not help. Three people tried that and + got the identical error on the identical table. + Cause: the database used to live in a bind mount, `../data/database`, a folder on your own + machine shared into the container. InnoDB cannot reliably rename a table across that share + on Windows, and it reports errno 194. Loading the schema renames tables while it adds + foreign keys, so `db:populate` trips over it on the first table in that pass every time. + It is not a Rails problem and it is not corruption. See docker-library/mariadb#331, which + reproduces it in three SQL statements. + Fix: already fixed. The database is a named Docker volume now, which lives inside Docker's + own filesystem and never touches the Windows one. Pull the latest `11.0.x`, then: + + ```bash + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml down -v + ``` + + Then repeat **Steps to run** 1–4 so schema writers cannot race. + + The old `doubtfire-deploy/data/database` folder is dead after that and you can delete it. + Nothing reads it any more. + You will usually see the api container die too, because it migrates on boot. Same problem, + and the same reset fixes both. + +15. `bundle exec rake db:populate` fails instantly. On Windows the error is "WSL ... ERROR: + CreateProcessCommon:800: execvpe(/bin/bash) failed: No such file or directory". On macOS + it is "bundle: command not found". + Cause: the command ran on your own machine instead of inside the api container. Ruby and + the gems are only in the container. Nothing needs to be installed on your machine. + Fix: use the whole command from step 2. The part before `bash -c` is not optional. + + ```bash + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml run --rm --no-deps doubtfire-api bash -c "bundle exec rake db:populate" + ``` + + This single-line form avoids the PowerShell line-continuation issue. + +## Notes + +- Docker mounts your local folders. The web and api run your branch code, including changes + you have not committed yet. +- The first time you move to `11.0.x` you must rebuild the images with `--build`. Old images + will not work. +- The normal combined stack publishes API 3200, web 4400, Mailpit 8225, and Mailpit SMTP + 1225. The database and Redis are internal to Docker, so a database client on your Mac + cannot connect to them. To look at the database, go through the api service: + + docker compose -p notifications-demo -f docker-compose.yml -f docker-compose.local-paths.yml exec doubtfire-api bash -c "bundle exec rails console" + +## Peer Progress Indicator configuration + +The local API container receives these non-secret development defaults: + +- `DF_PPI_MINIMUM_COHORT_SIZE=21` +- `DF_PPI_STALE_AFTER_HOURS=48` + +`DF_PPI_MINIMUM_COHORT_SIZE=21` matches the API's own floor. +`PeerProgressApi::MINIMUM_SAFE_COHORT_SIZE` is 21 and `minimum_cohort_size!` +returns 503 for anything below it, so a lower value disables the endpoint for +enabled units rather than publishing a smaller cohort. You can raise this +value, you cannot lower it. `positive_integer_env!` separately rejects a +missing, zero, negative or non-integer value. + +The floor is 21 remaining peers because the API quantises percentages into +10-point buckets. Before thresholding or quantising, it uses the exact internal +snapshot counts to remove the authenticated student's known task status and +upload contribution. A target-grade band therefore needs at least 22 eligible +students, including the reader, to leave 21 peers. At 20 peers each person +accounts for exactly half a bucket, leaving some rounded outputs that map to +only one possible count. At 21 peers each person's share is smaller than half a +bucket, so every published scalar output maps to at least two possible counts, +including the edge buckets. Changing either number without the other breaks +that guarantee, so `MINIMUM_SAFE_COHORT_SIZE` and +`PERCENTAGE_BUCKET_SIZE` are asserted together in the API test suite. + +That scalar guarantee protects the compact completed percentage and the +additive submitted wire-compatibility percentage. The advanced 15-status +distribution also considers all peer-only rounded buckets together: the API +releases the entire vector only when every status has at least two feasible raw +peer counts after applying the sum constraint. If that stronger check fails, +the compact value can remain available while the detailed vector is withheld. +Do not lower the peer floor to force either view to appear. + +Reader subtraction requires exact internal `submitted_count` and +`status_counts` fields. Snapshots created before migration `20260824000003` +have neither and deliberately return no percentages through the new API until +the aggregation job refreshes them. The endpoint also fails closed if the +reader's project membership or task changed after the snapshot, because +subtracting newer viewer context from an older aggregate would be unsafe. + +Use the combined API integration branch named above; the older +`ppi/student-progress-endpoint` verification pin is superseded. That API +revision must include the privacy fix that raises +`PeerProgressApi::MINIMUM_SAFE_COHORT_SIZE` to 21. The deploy and API changes +must not be merged independently. + +`DF_PPI_STALE_AFTER_HOURS=48` is the local maximum snapshot age. A snapshot +older than this is returned as stale, and the response withholds the +percentage entirely rather than returning an old one. + +The combined local stack starts Redis and the Sidekiq worker. That worker only +listens on the `mailers` and `notifications` queues, so it does not pick up +`AggregatePeerProgressJob` from `default`; the sample task below does the work +directly and does not depend on it. +To create the privacy-safe PPI demo data deterministically, run the dedicated +sample task in the same `ppi-live` project: + +```bash +docker compose -p ppi-live \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.ppi-live.yml \ + run --rm doubtfire-api bash -c "bundle exec rake db:ppi_sample_data" +``` + +This task creates or repairs the synthetic `PPI1001` and `PPI1002` units. It +derives the students per class from `DF_PPI_MINIMUM_COHORT_SIZE`, rounding up so +the two-class exact-grade cohorts leave at least the configured number of peers +after excluding any one reader. With the local setting, each class has 11 +students per grade and each target-grade cohort on a fresh or legacy sample +database has 22 students: exactly 21 remaining peers for any reader. The task +changes the PPI setting only for those synthetic +units, repairs the current seed-owned user roles, unit and task definitions, +tutorial capacity and enrolments, and writes and validates fresh snapshots +before it returns. It is safe to run again against a database previously +populated by this task; surplus students from an earlier higher threshold are +retained, so later cohorts can be above the configured minimum. + +Verify the feature flag, exact-grade cohort sizes, and the 28 seed-owned fresh +task/grade snapshots per unit: + +```bash +docker compose -p ppi-live \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.ppi-live.yml \ + exec doubtfire-api bundle exec rails runner \ + 'minimum = Integer(ENV.fetch("DF_PPI_MINIMUM_COHORT_SIZE"), 10); \ + stale_after_hours = Integer(ENV.fetch("DF_PPI_STALE_AFTER_HOURS"), 10); \ + raise "Invalid PPI configuration" unless minimum >= PeerProgressApi::MINIMUM_SAFE_COHORT_SIZE && stale_after_hours.positive?; \ + fresh_after = stale_after_hours.hours.ago; \ + grades = [0, 1, 2, 3]; codes = %w[PPI1001 PPI1002]; \ + abbreviations = (1..7).map { |number| "T#{number}" }; \ + units = codes.map do |code| \ + matches = Unit.where(code: code).to_a; \ + raise "Expected exactly one #{code} unit" unless matches.one?; \ + matches.first; \ + end; \ + units.each do |unit| \ + cohorts = unit.active_projects.group(:target_grade).count; \ + task_defs = unit.task_definitions.where(abbreviation: abbreviations); \ + unit_number = unit.code.delete_prefix("PPI100"); \ + projects = unit.active_projects.joins(:user).where("users.username LIKE ?", "ppi_u#{unit_number}c%"); \ + tasks = Task.where(project_id: projects.select(:id), task_definition_id: task_defs.select(:id)).includes(:task_definition, :project); \ + demo = unit.peer_progress_snapshots.where(task_definition_id: task_defs.select(:id), target_grade: grades); \ + latest_changes = grades.index_with { |grade| unit.active_projects.where(target_grade: grade).maximum(:target_grade_changed_at) }; \ + valid = unit.active? && unit.peer_progress_enabled? && !unit.allow_flexible_dates? && \ + task_defs.count == 7 && task_defs.all? { |definition| definition.target_grade.zero? } && \ + projects.count >= 2 * grades.length * minimum.fdiv(2).ceil && \ + projects.all? { |project| project.enrolled? && project.user.role_id == Role.student_id && grades.include?(project.target_grade) } && \ + tasks.count == projects.count * task_defs.count && \ + tasks.all? { |task| task.local_start_date.present? && task.local_start_date <= Time.zone.now && task.task_definition.target_grade <= task.project.target_grade } && \ + demo.count == 28 && \ + grades.all? { |grade| cohorts.fetch(grade, 0) - 1 >= minimum } && \ + demo.all? { |snapshot| !snapshot.submitted_count.nil? && !snapshot.status_counts.nil? && snapshot.cohort_size == cohorts.fetch(snapshot.target_grade) && \ + snapshot.calculated_at >= fresh_after && snapshot.calculated_at >= latest_changes.fetch(snapshot.target_grade) }; \ + raise "#{unit.code} is not PPI demo-ready" unless valid; \ + puts "#{unit.code}: enabled=true cohorts=#{cohorts.slice(*grades).inspect} fresh_demo_snapshots=#{demo.count}"; \ + end' +``` + +At the checked-in local setting on a fresh or legacy sample database, both lines +report `enabled=true`, cohorts `{0=>22, 1=>22, 2=>22, 3=>22}`, and +`fresh_demo_snapshots=28`. Cohorts can be higher after a run with a higher valid +threshold because the seed never removes students. In every case it adds enough +students per class to leave at least the configured peer threshold after the +reader is excluded. Never lower the configured floor to make a demo visible. + +The production Compose template supplies the same approved values through +`production/.env.production.example`, which you copy to `production/.env.production` +on the host. That file is deliberately not committed. These values are not +secrets, but both must be present because an enabled unit with an eligible +snapshot returns 503 when either setting is missing or invalid. diff --git a/development/.env.example b/development/.env.example new file mode 100644 index 000000000..91c1afd38 --- /dev/null +++ b/development/.env.example @@ -0,0 +1,14 @@ +# Optional non-production AAF configuration for development Compose. +# Copy to development/.env and obtain a dedicated registration from the +# identity owner. Never reuse or commit a production/institution shared secret. +DF_AUTH_METHOD=database +DF_AAF_ISSUER_URL= +DF_AAF_AUDIENCE_URL=http://localhost:3000 +DF_AAF_CALLBACK_URL=http://localhost:3000/api/auth/jwt +DF_AAF_IDENTITY_PROVIDER_URL= +DF_AAF_UNIQUE_URL= +DF_AAF_AUTH_SIGNOUT_URL= +DF_SECRET_KEY_AAF= + +# First-time tutorial rollout; recreate doubtfire-api after changing. +TUTORIAL_ENABLED=0 diff --git a/development/api.env b/development/api.env index 72c33f9c5..986f36352 100644 --- a/development/api.env +++ b/development/api.env @@ -5,6 +5,10 @@ RAILS_ENV=development TZ=Australia/Melbourne +# Peer Progress Indicator local development defaults +DF_PPI_MINIMUM_COHORT_SIZE=21 +DF_PPI_STALE_AFTER_HOURS=48 + # Student work location (in container) DF_STUDENT_WORK_DIR=/student-work @@ -29,14 +33,15 @@ DF_SECRET_KEY_DEVISE=test-secret-key-test-secret-key! # Authentication settings DF_AUTH_METHOD=database -# AAF settings - for testing on local host -DF_AAF_ISSUER_URL=https://rapid.test.aaf.edu.au -DF_AAF_AUDIENCE_URL=http://localhost:3000 -DF_AAF_CALLBACK_URL=http://localhost:3000/api/auth/jwt -DF_AAF_IDENTITY_PROVIDER_URL=https://signon-uat.deakin.edu.au/idp/shibboleth -DF_AAF_UNIQUE_URL=https://rapid.test.aaf.edu.au/jwt/authnrequest/research/Ag4EJJhjf0zXHqlKvKZEbg -DF_AAF_AUTH_SIGNOUT_URL=https://sync-uat.deakin.edu.au/auth/logout -DF_SECRET_KEY_AAF=v4~LMFLzzwRGZdju\5QBa@FiHIN9 +# Optional AAF development settings are deliberately blank. Compose users can +# supply a dedicated non-production registration from development/.env. +DF_AAF_ISSUER_URL= +DF_AAF_AUDIENCE_URL= +DF_AAF_CALLBACK_URL= +DF_AAF_IDENTITY_PROVIDER_URL= +DF_AAF_UNIQUE_URL= +DF_AAF_AUTH_SIGNOUT_URL= +DF_SECRET_KEY_AAF= # Database settings DF_DEV_DB_ADAPTER=mysql2 @@ -61,4 +66,3 @@ DF_PRODUCTION_DB_PASSWORD=pwd # Mail settings DF_MAIL_DELIVERY_METHOD=test - diff --git a/development/development-dockerfiles.md b/development/development-dockerfiles.md index 48f4990da..21bb787fe 100644 --- a/development/development-dockerfiles.md +++ b/development/development-dockerfiles.md @@ -1,5 +1,10 @@ # Development docker images +> **Legacy development-only notes.** The `build.sh`, `publish.sh`, version tags, +> and Docker Hub flow below are not production release evidence and must not be +> used for the 11.0.x handover. Use [RELEASING.md](../RELEASING.md) for the +> revision-locked, digest-based production publication contract. + This folder contains docker compose files, and scripts to build the associated container images from the api and web projects. ## Running in development diff --git a/development/docker-compose.full.yml b/development/docker-compose.full.yml index 69d54c401..b1ee69d4c 100644 --- a/development/docker-compose.full.yml +++ b/development/docker-compose.full.yml @@ -1,19 +1,22 @@ -version: '3' services: dev-db: container_name: df-compose-dev-db - image: mariadb + # Kept identical to docker-compose.yml. Both share the db_data volume, so + # they must agree on the server version and on the mount. + image: mariadb:12.3 environment: MYSQL_ROOT_PASSWORD: db-root-password MYSQL_DATABASE: doubtfire-dev MYSQL_USER: dfire MYSQL_PASSWORD: pwd volumes: - - ../data/database:/var/lib/mysql + # Named volume, not ../data/database. See the comment in + # docker-compose.yml for why a bind mount breaks InnoDB on Windows. + - db_data:/var/lib/mysql doubtfire-api: container_name: doubtfire-api - image: lmsdoubtfire/doubtfire-api:10.0.x-dev + image: lmsdoubtfire/doubtfire-api:11.0.x-dev build: ../doubtfire-api ports: - "3000:3000" @@ -34,15 +37,20 @@ services: DF_SECRET_KEY_ATTR: test-secret-key-test-secret-key! DF_SECRET_KEY_DEVISE: test-secret-key-test-secret-key! - # Authentication method - can set to AAF or ldap - DF_AUTH_METHOD: database - DF_AAF_ISSUER_URL: https://rapid.test.aaf.edu.au - DF_AAF_AUDIENCE_URL: http://localhost:3000 - DF_AAF_CALLBACK_URL: http://localhost:3000/api/auth/jwt - DF_AAF_IDENTITY_PROVIDER_URL: https://signon-uat.deakin.edu.au/idp/shibboleth - DF_AAF_UNIQUE_URL: https://rapid.test.aaf.edu.au/jwt/authnrequest/research/Ag4EJJhjf0zXHqlKvKZEbg - DF_AAF_AUTH_SIGNOUT_URL: https://sync-uat.deakin.edu.au/auth/logout - DF_SECRET_KEY_AAF: v4~LMFLzzwRGZdju\5QBa@FiHIN9 + # Database authentication is the safe local default. Optional AAF values + # must come from an ignored development/.env file. + DF_AUTH_METHOD: ${DF_AUTH_METHOD:-database} + DF_AAF_ISSUER_URL: ${DF_AAF_ISSUER_URL:-} + DF_AAF_AUDIENCE_URL: ${DF_AAF_AUDIENCE_URL:-http://localhost:3000} + DF_AAF_CALLBACK_URL: ${DF_AAF_CALLBACK_URL:-http://localhost:3000/api/auth/jwt} + DF_AAF_IDENTITY_PROVIDER_URL: ${DF_AAF_IDENTITY_PROVIDER_URL:-} + DF_AAF_UNIQUE_URL: ${DF_AAF_UNIQUE_URL:-} + DF_AAF_AUTH_SIGNOUT_URL: ${DF_AAF_AUTH_SIGNOUT_URL:-} + DF_SECRET_KEY_AAF: ${DF_SECRET_KEY_AAF:-} + + # Peer Progress Indicator local defaults, aligned with production policy. + DF_PPI_MINIMUM_COHORT_SIZE: '21' + DF_PPI_STALE_AFTER_HOURS: '48' # Database settings - for development env DF_DEV_DB_ADAPTER: mysql2 @@ -67,6 +75,7 @@ services: # Overseer - disabled! OVERSEER_ENABLED: 1 + TUTORIAL_ENABLED: ${TUTORIAL_ENABLED:-0} RABBITMQ_HOSTNAME: doubtfire-mq RABBITMQ_USERNAME: secure_credentials RABBITMQ_PASSWORD: secure_credentials @@ -80,18 +89,19 @@ services: doubtfire-web: container_name: doubtfire-web - image: lmsdoubtfire/doubtfire-web:10.0.x-dev + image: lmsdoubtfire/doubtfire-web:11.0.x-dev build: ../doubtfire-web - command: /bin/bash -c 'npm install; npm run start-compose' + command: /bin/bash -c 'npm install && npm start' ports: - "4200:4200" depends_on: - doubtfire-api volumes: - ../doubtfire-web:/doubtfire-web + - ./proxy.conf.docker.json:/doubtfire-web/proxy.conf.json:ro overseer-receive: - image: lmsdoubtfire/doubtfire-api:development-dev + image: lmsdoubtfire/doubtfire-api:11.0.x-dev build: ../doubtfire-api env_file: - api.env @@ -136,3 +146,6 @@ services: environment: RABBITMQ_DEFAULT_USER: secure_credentials RABBITMQ_DEFAULT_PASS: secure_credentials + +volumes: + db_data: diff --git a/development/docker-compose.local-paths.yml b/development/docker-compose.local-paths.yml new file mode 100644 index 000000000..db428d339 --- /dev/null +++ b/development/docker-compose.local-paths.yml @@ -0,0 +1,91 @@ +# Overlay for `docker-compose.yml` (api + web + db, overseer disabled). +# Repoints the api/web build contexts from the doubtfire-deploy submodule +# placeholders to the real sibling checkouts under ontrack/ (the working repos, +# incl. uncommitted changes). Overseer services are intentionally omitted — the +# base compose has none (OVERSEER_ENABLED: 0); use docker-compose.full.yml if you +# need overseer. +x-notifications-environment: ¬ifications-environment + DF_SMTP_ADDRESS: mailpit + DF_DEV_DB_HOST: dev-db + DF_TEST_DB_HOST: dev-db + DF_PRODUCTION_DB_HOST: dev-db + DF_REDIS_SIDEKIQ_URL: redis://redis-sidekiq:6379/0 + DF_INSTITUTION_HOST: http://localhost:4400 + +services: + dev-db: + container_name: notifications-demo-db + volumes: + - ./notifications-test-db.sql:/docker-entrypoint-initdb.d/10-notifications-test-db.sql:ro + + redis-sidekiq: + container_name: notifications-demo-redis + + mailpit: + container_name: notifications-demo-mailpit + ports: !override + - "8225:8025" + - "1225:1025" + + doubtfire-api: + container_name: notifications-demo-api + # Use environment-specific image tags so another OnTrack worktree cannot + # silently replace the image while this demo is running. + image: ontrack-notifications-api:20260823 + build: ../../doubtfire-api + ports: !override + - "3200:3000" + volumes: + - ../../doubtfire-api/:/doubtfire + - ../data/tmp:/doubtfire/tmp + - ../data/student-work:/student-work + environment: + <<: *notifications-environment + # api.env doesn't define these, but ActiveRecord's db tasks always + # process the "test" config alongside "development", crashing with a + # NoMethodError on the blank adapter if they're unset. + DF_TEST_DB_ADAPTER: mysql2 + DF_TEST_DB_DATABASE: doubtfire-notifications-test + DF_TEST_DB_USERNAME: dfire + DF_TEST_DB_PASSWORD: pwd + + doubtfire-sidekiq: + container_name: notifications-demo-sidekiq + image: ontrack-notifications-api:20260823 + build: ../../doubtfire-api + volumes: + - ../../doubtfire-api/:/doubtfire + - ../data/tmp:/doubtfire/tmp + - ../data/student-work:/student-work + environment: + <<: *notifications-environment + + doubtfire-web: + container_name: notifications-demo-web + image: ontrack-notifications-web:20260823 + build: ../../doubtfire-web + ports: !override + - "4400:4200" + # The base compose now runs `npm start` and mounts proxy.conf.docker.json too, so the + # command and proxy volume below match the base. They are kept spelled out here only so + # this overlay reads as self-contained; the repo's own proxy.conf.json (which targets + # localhost:3000, correct only for host-based ng serve) is left untouched on the host. + command: /bin/bash -c 'npm install && npm start' + volumes: + - ../../doubtfire-web:/doubtfire-web + - ./proxy.conf.docker.json:/doubtfire-web/proxy.conf.json:ro + + # Static, production/service-worker build used only for the update-prompt + # recording. `prepare-pwa-update-demo.sh prepare` creates its A/B bundles. + pwa-update-demo: + profiles: [pwa-update-demo] + container_name: notifications-demo-pwa-update + image: ontrack-notifications-web:20260823 + command: ["node", "/demo-server/pwa-update-demo-server.mjs"] + ports: !override + - "4500:8080" + depends_on: + - doubtfire-api + volumes: + - ../data/pwa-update-demo:/demo:ro + - ./pwa-update-demo-server.mjs:/demo-server/pwa-update-demo-server.mjs:ro diff --git a/development/docker-compose.podman.yml b/development/docker-compose.podman.yml new file mode 100644 index 000000000..aca530e06 --- /dev/null +++ b/development/docker-compose.podman.yml @@ -0,0 +1,31 @@ +services: + dev-db: + volumes: + - podman_db_data:/var/lib/mysql + + doubtfire-api: + image: localhost/ontrack-doubtfire-api:11.0-local + volumes: + - ../../doubtfire-api/:/doubtfire:z + - ../data/tmp:/doubtfire/tmp:z + - ../data/student-work:/student-work:z + + doubtfire-sidekiq: + image: localhost/ontrack-doubtfire-api:11.0-local + volumes: + - ../../doubtfire-api/:/doubtfire:z + - ../data/tmp:/doubtfire/tmp:z + - ../data/student-work:/student-work:z + + doubtfire-web: + image: localhost/ontrack-doubtfire-web:11.0-local + userns_mode: "keep-id:uid=1000,gid=1000" + security_opt: + - label=disable + command: /bin/bash -c 'npm install && npm start' + volumes: + - ../../doubtfire-web:/doubtfire-web + - ./proxy.conf.docker.json:/doubtfire-web/proxy.conf.json:ro + +volumes: + podman_db_data: diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 1e5f98f04..fb9491e4c 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -1,25 +1,58 @@ -version: '3' services: dev-db: container_name: df-compose-dev-db - image: mariadb + # Pinned so everyone runs the same server. An unpinned `mariadb` tag means + # two people who set up a month apart get two different majors, and a data + # directory written by one is not readable by the other. + image: mariadb:12.3 environment: MYSQL_ROOT_PASSWORD: db-root-password MYSQL_DATABASE: doubtfire-dev MYSQL_USER: dfire MYSQL_PASSWORD: pwd volumes: - - ../data/database:/var/lib/mysql + # A named volume, NOT a bind mount to ../data/database. InnoDB cannot + # reliably rename a table on a host directory shared into the container + # on Windows, and it fails with errno 194, "Tablespace is missing for a + # table". Rails hits that on a clean `rake db:populate`, because loading + # the schema renames tables while adding foreign keys. It reads as a + # Rails or a data corruption problem and is neither: the same database + # on a named volume is fine. docker-library/mariadb#331. + # docs/ONTRACK_PODMAN_SETUP.md reached the same conclusion for Podman. + - db_data:/var/lib/mysql redis-sidekiq: container_name: df-compose-redis-sidekiq image: redis:7.0 volumes: - redis_sidekiq_data:/data + # doubtfire-sidekiq is gated on this. A plain depends_on only waits for the + # container to exist, not for Redis to accept a connection, and a worker + # that loses that race is down for the rest of the session with nothing in + # the api log to say so. + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + + # Mail catcher. Accepts every email the app sends and shows it in a web inbox + # at http://localhost:8025. Nothing is delivered to the outside world. + mailpit: + container_name: df-compose-mailpit + image: axllent/mailpit:latest + ports: + - "8025:8025" # web inbox, open this in a browser + - "1025:1025" # smtp, what the api sends to + environment: + MP_MAX_MESSAGES: 500 + MP_SMTP_AUTH_ACCEPT_ANY: 1 + MP_SMTP_AUTH_ALLOW_INSECURE: 1 doubtfire-api: container_name: doubtfire-api - image: lmsdoubtfire/doubtfire-api:8.0.x-dev + image: lmsdoubtfire/doubtfire-api:11.0.x-dev build: ../doubtfire-api ports: - "3000:3000" @@ -29,26 +62,55 @@ services: - ../data/student-work:/student-work depends_on: - dev-db - environment: + - redis-sidekiq + - mailpit + environment: &doubtfire-api-environment RAILS_ENV: 'development' + TZ: Australia/Melbourne + TUTORIAL_ENABLED: ${TUTORIAL_ENABLED:-0} + + # Mail catcher. Setting DF_SMTP_ADDRESS is what switches the api from + # writing mail to a file to sending it to mailpit. Unset it and the api + # falls back to files, so running without docker still works. + DF_SMTP_ADDRESS: df-compose-mailpit + DF_SMTP_PORT: 1025 + + # Web push. Without these the push channel is a no-op and the app behaves + # exactly as it did before push existed, so it is safe to blank them out. + # + # This is a throwaway pair generated for local development only, following + # the same convention as DF_SECRET_KEY_BASE below. Production sets its own + # through real secrets and must never reuse these. + # See doubtfire-api/docs/notifications/push-setup.md to generate your own. + DOUBTFIRE_VAPID_PUBLIC_KEY: 'BOs-KbIoHK7gUIX3i2_uEuDoouj-GKxB-mY9CRmLNmd4Wn-SSl254E1g6jR1ukL3e37p8uCpaMjOvfAB0BwzvSI=' + DOUBTFIRE_VAPID_PRIVATE_KEY: '_NFIWSUTdCdLJJFh87pf4ekQLmNYqsweZ4288NpVZaY=' + DOUBTFIRE_VAPID_SUBJECT: 'mailto:noreply@doubtfire.local' + + # Peer Progress Indicator local defaults, aligned with production policy. + DF_PPI_MINIMUM_COHORT_SIZE: '21' + DF_PPI_STALE_AFTER_HOURS: '48' DF_STUDENT_WORK_DIR: /student-work - DF_INSTITUTION_HOST: http://localhost:3000 + # User-facing links in notification emails must open the web app, not the + # Rails API port. + DF_INSTITUTION_HOST: http://localhost:4200 DF_INSTITUTION_PRODUCT_NAME: OnTrack + DF_INSTITUTION_EMAIL_SENDER: noreply@doubtfire.local DF_SECRET_KEY_BASE: test-secret-key-test-secret-key! DF_SECRET_KEY_ATTR: test-secret-key-test-secret-key! DF_SECRET_KEY_DEVISE: test-secret-key-test-secret-key! - # Authentication method - can set to AAF or ldap - DF_AUTH_METHOD: database - DF_AAF_ISSUER_URL: https://rapid.test.aaf.edu.au - DF_AAF_AUDIENCE_URL: http://localhost:3000 - DF_AAF_CALLBACK_URL: http://localhost:3000/api/auth/jwt - DF_AAF_IDENTITY_PROVIDER_URL: https://signon-uat.deakin.edu.au/idp/shibboleth - DF_AAF_UNIQUE_URL: https://rapid.test.aaf.edu.au/jwt/authnrequest/research/Ag4EJJhjf0zXHqlKvKZEbg - DF_AAF_AUTH_SIGNOUT_URL: https://sync-uat.deakin.edu.au/auth/logout - DF_SECRET_KEY_AAF: v4~LMFLzzwRGZdju\5QBa@FiHIN9 + # Database authentication is the safe local default. Optional AAF values + # must come from an ignored development/.env file. + DF_AUTH_METHOD: ${DF_AUTH_METHOD:-database} + DF_AAF_ISSUER_URL: ${DF_AAF_ISSUER_URL:-} + DF_AAF_AUDIENCE_URL: ${DF_AAF_AUDIENCE_URL:-http://localhost:3000} + DF_AAF_CALLBACK_URL: ${DF_AAF_CALLBACK_URL:-http://localhost:3000/api/auth/jwt} + DF_AAF_IDENTITY_PROVIDER_URL: ${DF_AAF_IDENTITY_PROVIDER_URL:-} + DF_AAF_UNIQUE_URL: ${DF_AAF_UNIQUE_URL:-} + DF_AAF_AUTH_SIGNOUT_URL: ${DF_AAF_AUTH_SIGNOUT_URL:-} + DF_SECRET_KEY_AAF: ${DF_SECRET_KEY_AAF:-} # Database settings - for development env DF_DEV_DB_ADAPTER: mysql2 @@ -83,11 +145,62 @@ services: # Redis DF_REDIS_SIDEKIQ_URL: redis://df-compose-redis-sidekiq:6379/0 + doubtfire-sidekiq: + container_name: doubtfire-sidekiq + image: lmsdoubtfire/doubtfire-api:11.0.x-dev + build: ../doubtfire-api + # Deliberately narrow. NotificationEmailJob names `mailers` and + # PushNotificationDeliveryJob names `notifications`, so this worker can + # deliver both user-facing channels without consuming `default`. + # + # Taking `default` here would run AcceptSubmissionJob, which is enqueued on + # every task submission and calls convert_submission_to_pdf. That raises + # "LATEX_CONTAINER_NAME is not set" because development/ has no texlive + # service the way production/ does. The task would drop to "fix" with an + # automated comment on it, so submissions would break for everyone not + # working on notifications. Left unread in Redis, as they are today, those + # jobs are harmless. + # + # `default` needs a texlive service and LATEX_CONTAINER_NAME first, or the + # jobs that need one need a queue of their own. Until then anything we do + # want run here has to name one of the two channel queues. The + # config/schedule.yml cron entries are included: this worker registers them + # at startup and they enqueue on `default`, so they are scheduled but not + # run. + # + # `-q` is set here and not in config/sidekiq.yml because that file lives in + # doubtfire-api, which this repo cannot keep in step. The config file is + # still read for :concurrency and the command line wins over it. + command: ["bundle", "exec", "sidekiq", "-C", "config/sidekiq.yml", "-q", "mailers", "-q", "notifications"] + volumes: + - ../doubtfire-api/:/doubtfire + - ../data/tmp:/doubtfire/tmp + - ../data/student-work:/student-work + # Same policy the sidekiq service in production/docker-compose.yml uses. + # A worker that died on a transient Redis or database hiccup is invisible. + # The rest of the stack still looks up and email just stops. + restart: on-failure:5 + depends_on: + dev-db: + condition: service_started + redis-sidekiq: + condition: service_healthy + mailpit: + condition: service_started + environment: + <<: *doubtfire-api-environment + DF_LOG_TO_STDOUT: '1' + # This is the process that evaluates config/schedule.yml. Three of its + # seven entries name a wall-clock time (5am, 8am, 11:55pm), so an unset + # TZ would run those on the image default of UTC and put them hours out. + # Same value as development/api.env. + TZ: Australia/Melbourne + doubtfire-web: container_name: doubtfire-web - image: lmsdoubtfire/doubtfire-web:8.0.x-dev + image: lmsdoubtfire/doubtfire-web:11.0.x-dev build: ../doubtfire-web - command: /bin/bash -c 'npm install; npm run start-compose' + command: /bin/bash -c 'npm install && npm start' ports: - "4200:4200" depends_on: @@ -95,7 +208,9 @@ services: volumes: - ../doubtfire-web:/doubtfire-web - web_node_modules:/doubtfire-web/node_modules + - ./proxy.conf.docker.json:/doubtfire-web/proxy.conf.json:ro volumes: + db_data: web_node_modules: redis_sidekiq_data: diff --git a/development/notifications-test-db.sql b/development/notifications-test-db.sql new file mode 100644 index 000000000..785bf7a1b --- /dev/null +++ b/development/notifications-test-db.sql @@ -0,0 +1,2 @@ +CREATE DATABASE IF NOT EXISTS `doubtfire-notifications-test`; +GRANT ALL PRIVILEGES ON `doubtfire-notifications-test`.* TO 'dfire'@'%'; diff --git a/development/proxy.conf.docker.json b/development/proxy.conf.docker.json new file mode 100644 index 000000000..ff67b052a --- /dev/null +++ b/development/proxy.conf.docker.json @@ -0,0 +1,4 @@ +{ + "/api": { "target": "http://doubtfire-api:3000", "secure": false }, + "/lti/api": { "target": "http://host.docker.internal:3001", "secure": false } +} diff --git a/development/verify-notifications.sh b/development/verify-notifications.sh new file mode 100755 index 000000000..da017ce44 --- /dev/null +++ b/development/verify-notifications.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Checks the notification stack end to end without opening a browser. +# +# Run from this folder: bash verify-notifications.sh +# +# Checks the stack used by NOTIFICATIONS-INTEGRATION.md. Read that first. +# +# Every check prints PASS or FAIL and the script exits non-zero if any failed, +# so it is safe to run before a demo or after pulling someone else's branch. +# +# Do not run this while `db:populate` is rebuilding the development database. +# Focused tests use the separate `doubtfire-notifications-test` database. + +fails=0 +api_url=${NOTIFICATIONS_API_URL:-http://localhost:3200} +web_url=${NOTIFICATIONS_WEB_URL:-http://localhost:4400} +mailpit_url=${NOTIFICATIONS_MAILPIT_URL:-http://localhost:8225} +pass() { printf ' \033[32mPASS\033[0m %s\n' "$1"; } +fail() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fails=$((fails + 1)); } +check() { # check + if [ "$2" = "$3" ]; then pass "$1"; else fail "$1 (wanted '$2', got '$3')"; fi +} + +http() { curl -s -o /dev/null -w '%{http_code}' "$1"; } + +echo +echo "1. Containers" +for c in notifications-demo-api notifications-demo-sidekiq notifications-demo-web notifications-demo-db notifications-demo-redis notifications-demo-mailpit; do + state=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo missing) + check "$c is running" "true" "$state" +done +worker_queues=$(docker inspect -f '{{json .Config.Cmd}}' notifications-demo-sidekiq 2>/dev/null \ + | python3 -c 'import json,sys; command=json.load(sys.stdin); print(",".join(command[index + 1] for index, value in enumerate(command[:-1]) if value == "-q"))' \ + 2>/dev/null || true) +check "worker consumes only the notification channel queues" "mailers,notifications" "$worker_queues" + +# Compose considers a container started before Puma or the Angular server is +# necessarily ready to answer. A verifier run immediately after `up` or +# `restart` used to sample that short window and report false 000/500 failures. +# Poll all three routes together for up to one minute, then let the ordinary +# checks below report the last observed status if readiness never arrives. +api_status=000 +web_status=000 +proxy_status=000 +for _ in {1..60}; do + api_status=$(http "$api_url/api/settings/public") + web_status=$(http "$web_url/") + proxy_status=$(docker exec notifications-demo-web curl -s -o /dev/null -w '%{http_code}' \ + localhost:4200/api/settings/public 2>/dev/null || true) + if [ "$api_status" = 200 ] && [ "$web_status" = 200 ] && [ "$proxy_status" = 200 ]; then + break + fi + sleep 1 +done + +echo +echo "2. Ports answer" +check "api :3200/api/settings/public" "200" "$api_status" +check "web :4400" "200" "$web_status" +check "mailpit:8225" "200" "$(http "$mailpit_url/")" + +echo +echo "3. Web can reach the api through its proxy" +check "web -> api" "200" "$proxy_status" + +echo +echo "4. Mail goes to the catcher, not to a file (EN-F02)" +check "delivery_method" "smtp" "$(docker exec notifications-demo-api printenv DF_SMTP_ADDRESS >/dev/null 2>&1 && echo smtp || echo file)" +check "smtp host" "mailpit" "$(docker exec notifications-demo-api printenv DF_SMTP_ADDRESS 2>/dev/null | tr -d '\r')" +check "api sender" "noreply@doubtfire.local" "$(docker exec notifications-demo-api printenv DF_INSTITUTION_EMAIL_SENDER 2>/dev/null | tr -d '\r')" +check "worker sender" "noreply@doubtfire.local" "$(docker exec notifications-demo-sidekiq printenv DF_INSTITUTION_EMAIL_SENDER 2>/dev/null | tr -d '\r')" +check "api timezone" "Australia/Melbourne" "$(docker exec notifications-demo-api printenv TZ 2>/dev/null | tr -d '\r')" +check "worker timezone" "Australia/Melbourne" "$(docker exec notifications-demo-sidekiq printenv TZ 2>/dev/null | tr -d '\r')" + +echo +echo "5. Push prerequisites are mounted (MN-F01)" +paths=$(curl -s "$api_url/api/swagger_doc" | python3 -c "import sys,json;print('yes' if '/api/push_subscriptions' in json.load(sys.stdin).get('paths',{}) else 'no')" 2>/dev/null) +check "/api/push_subscriptions in the api docs" "yes" "$paths" +table=$(docker exec notifications-demo-api bash -c "bundle exec rails runner 'puts \"R::\" + PushSubscription.table_exists?.to_s' 2>/dev/null" | grep -o 'R::.*' | cut -d: -f3) +check "push_subscriptions table exists" "true" "$table" + +echo +echo "6. Push keys are loaded (MN-F02)" +cfg=$(docker exec notifications-demo-api bash -c "bundle exec rails runner 'puts \"R::\" + PushNotificationService.configured?.to_s' 2>/dev/null" | grep -o 'R::.*' | cut -d: -f3) +check "PushNotificationService.configured?" "true" "$cfg" + +echo +echo "7. Service worker is served (MN-F03)" +ngsw_json=$(curl -fsS "$web_url/ngsw.json" \ + | python3 -c "import json,sys; json.load(sys.stdin); print('valid')" 2>/dev/null \ + || echo invalid) +ngsw_worker=$(curl -fsS "$web_url/ngsw-worker.js" \ + | python3 -c "import sys; data=sys.stdin.read(); head=data[:1000].lower(); print('valid' if len(data) > 10000 and '/dev/null \ + || echo invalid) +check "ngsw.json is valid JSON" "valid" "$ngsw_json" +check "ngsw-worker.js is a real worker" "valid" "$ngsw_worker" + +echo +echo "8. A comment really does send an email" +before=$(curl -s "$mailpit_url/api/v1/messages" | python3 -c "import sys,json;print(json.load(sys.stdin)['messages_count'])" 2>/dev/null) +token=$(docker exec notifications-demo-api bash -c "bundle exec rails runner \"u=User.find_by(username:'acain'); puts 'R::'+u.generate_authentication_token!.authentication_token\" 2>/dev/null" | grep -o 'R::.*' | cut -d: -f3) +# The text must be different every run. OnTrack drops a comment that duplicates +# the previous one on the same task and answers 403 "Comment duplicates last +# comment, so ignored" (task_comments_api.rb). A fixed string here makes the +# script pass, then fail, then pass, depending on what ran last. No notification +# is raised for a dropped duplicate, which is correct but worth knowing when a +# demo comment produces no email. +body="verify-notifications.sh check $(date +%s) $$" +code=$(curl -s -o /tmp/verify-comment-body -w '%{http_code}' \ + -X POST "$api_url/api/projects/2/task_def_id/1/comments/" \ + -H "Username: acain" -H "Auth-Token: $token" -H "Content-Type: application/json" \ + -d "{\"comment\":\"$body\"}") +check "POST a task comment" "201" "$code" +[ "$code" = "201" ] || echo " response: $(head -c 200 /tmp/verify-comment-body)" +comment_id=$(python3 -c "import json; print(json.load(open('/tmp/verify-comment-body')).get('id', ''))" 2>/dev/null || true) +# Sidekiq polls asynchronously, so a fixed three-second sleep can observe the +# queue just before a healthy worker delivers. Poll for up to 20 seconds and +# finish immediately when Mailpit records the message. +after="$before" +for _ in {1..20}; do + sleep 1 + after=$(curl -s "$mailpit_url/api/v1/messages" | python3 -c "import sys,json;print(json.load(sys.stdin)['messages_count'])" 2>/dev/null) + [ "$after" -gt "$before" ] && break +done +if [ "$after" -gt "$before" ]; then + pass "mailpit received the email ($before -> $after)" +else + fail "mailpit did not receive an email ($before -> $after)" +fi + +# Keep the seeded task visually clean for the recording. The caught email stays +# in Mailpit as verification evidence, but the synthetic comment and its in-app +# notification are removed together after delivery has been observed. +cleanup=$(docker exec -e VERIFY_COMMENT_ID="$comment_id" notifications-demo-api bash -c \ + "bundle exec rails runner 'comment = TaskComment.find_by(id: ENV[\"VERIFY_COMMENT_ID\"]); notification = comment && Notification.where(user_id: comment.recipient_id, event: \"task_comment_created\").where(created_at: (comment.created_at - 2.seconds)..(comment.created_at + 30.seconds)).order(id: :desc).first; notification&.destroy!; comment&.destroy!; puts \"R::\" + (!comment.nil?).to_s' 2>/dev/null" \ + | grep -o 'R::.*' | cut -d: -f3) +check "remove the synthetic comment and in-app notification" "true" "$cleanup" + +echo +echo "9. The API queued cleanly and the worker has no failed channel delivery" +# Queue hand-off failures happen in the API process. SMTP and push delivery +# happen in Sidekiq, so worker logs and retry/dead sets are checked separately +# rather than assuming the API log can report an asynchronous delivery failure. +api_errs=$(docker logs --since 5m notifications-demo-api 2>&1 \ + | grep -Ec "Failed to queue notification (email|push)" \ + || true) +check "notification queue errors in the last 5 minutes" "0" "$api_errs" + +worker_push_errs=$(docker logs --since 5m notifications-demo-sidekiq 2>&1 \ + | grep -Ec "Failed to push to subscription|Refusing to push to subscription" \ + || true) +check "push delivery errors in the last 5 minutes" "0" "$worker_push_errs" + +retry_count=$(docker exec notifications-demo-sidekiq bash -c \ + "bundle exec rails runner 'require \"sidekiq/api\"; puts \"R::\" + Sidekiq::RetrySet.new.count { |job| job.klass == \"NotificationEmailJob\" }.to_s' 2>/dev/null" \ + | grep -o 'R::.*' | cut -d: -f3) +dead_count=$(docker exec notifications-demo-sidekiq bash -c \ + "bundle exec rails runner 'require \"sidekiq/api\"; puts \"R::\" + Sidekiq::DeadSet.new.count { |job| job.klass == \"NotificationEmailJob\" }.to_s' 2>/dev/null" \ + | grep -o 'R::.*' | cut -d: -f3) +push_retry_count=$(docker exec notifications-demo-sidekiq bash -c \ + "bundle exec rails runner 'require \"sidekiq/api\"; puts \"R::\" + Sidekiq::RetrySet.new.count { |job| job.klass == \"PushNotificationDeliveryJob\" }.to_s' 2>/dev/null" \ + | grep -o 'R::.*' | cut -d: -f3) +push_dead_count=$(docker exec notifications-demo-sidekiq bash -c \ + "bundle exec rails runner 'require \"sidekiq/api\"; puts \"R::\" + Sidekiq::DeadSet.new.count { |job| job.klass == \"PushNotificationDeliveryJob\" }.to_s' 2>/dev/null" \ + | grep -o 'R::.*' | cut -d: -f3) +check "notification email jobs waiting to retry" "0" "$retry_count" +check "notification email jobs in the dead set" "0" "$dead_count" +check "push notification jobs waiting to retry" "0" "$push_retry_count" +check "push notification jobs in the dead set" "0" "$push_dead_count" + +echo +if [ "$fails" -eq 0 ]; then + printf '\033[32mAll prerequisite and email checks passed.\033[0m Read the mail at %s\n' "$mailpit_url" + printf 'Real Web Push still needs a signed-in browser/device verification.\n\n' +else + printf '\033[31m%s check(s) failed.\033[0m See doubtfire-deploy/RUNNING-LOCALLY.md\n\n' "$fails" +fi +exit "$fails" diff --git a/development/verify-unit-hub.py b/development/verify-unit-hub.py new file mode 100644 index 000000000..54f0fe5e9 --- /dev/null +++ b/development/verify-unit-hub.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Read-only Unit Hub release smoke check. Never prints credentials or content.""" + +import argparse +import json +import os +import sys +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import HTTPRedirectHandler, Request, build_opener + + +class NoRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("base_url", help="OnTrack origin, for example https://ontrack.example.edu") + parser.add_argument("--expected-unit-id", type=int, required=True) + parser.add_argument("--excluded-unit-id", type=int, required=True) + args = parser.parse_args() + base = urlparse(args.base_url) + if (base.scheme != "https" and not (base.scheme == "http" and base.hostname in ("localhost", "127.0.0.1", "::1"))) or base.username or base.password or base.query or base.fragment or base.path not in ("", "/"): + parser.error("Use an HTTPS origin (HTTP is allowed only on loopback).") + if args.expected_unit_id == args.excluded_unit_id: + parser.error("Expected and excluded unit IDs must differ.") + username = os.environ.get("UNIT_HUB_USERNAME") + token = os.environ.get("UNIT_HUB_AUTH_TOKEN") + if not username or not token: + parser.error("Set UNIT_HUB_USERNAME and UNIT_HUB_AUTH_TOKEN for a student test account.") + opener = build_opener(NoRedirects()) + + def request(path, authenticated=True): + headers = {"Accept": "application/json"} + if authenticated: + headers.update({"Username": username, "Auth-Token": token}) + req = Request(args.base_url.rstrip("/") + "/api" + path, headers=headers) + try: + response = opener.open(req, timeout=30) + except HTTPError as error: + response = error + with response: + payload = response.read(5_000_001) + if len(payload) > 5_000_000: + raise ValueError("Unexpectedly large response") + return response.status, response.headers, payload + + status, _, _ = request("/unit_hub", authenticated=False) + if status not in (401, 403, 419): + raise ValueError("Unauthenticated hub access was not rejected") + print("PASS: sign-in is required") + status, headers, payload = request("/unit_hub") + if status != 200: + raise ValueError(f"Authenticated hub returned HTTP {status}") + if "no-store" not in headers.get("Cache-Control", ""): + raise ValueError("Hub response is missing no-store cache protection") + hub = json.loads(payload) + units = {unit["id"] for unit in hub["units"]} + if args.expected_unit_id not in units or args.excluded_unit_id in units: + raise ValueError("Hub unit list does not match the expected student enrolment boundary") + if any(unit.get("can_manage") for unit in hub["units"]): + raise ValueError("Use a student-only account for this access check") + for records in (hub["announcements"], hub["sessions"]): + if any(record["unit_id"] not in units for record in records): + raise ValueError("Hub content contains a unit outside the authorised list") + print("PASS: feed respects the selected enrolment boundary and disables shared caching") + for unit_id in (args.expected_unit_id, args.excluded_unit_id): + for resource in ("announcements", "sessions"): + status, _, _ = request(f"/units/{unit_id}/{resource}") + if status not in (403, 404): + raise ValueError("Student access to teaching-staff content was not rejected") + print("PASS: student cannot read staff drafts for either unit") + print("Read-only smoke check passed. No records or calendar preferences were changed.") + + +if __name__ == "__main__": + try: + main() + except (ValueError, KeyError, TypeError, URLError, TimeoutError): + # Error response bodies may contain personal data; do not echo them. + print("FAIL: Unit Hub smoke check did not pass. Check the configured origin, student account, unit IDs and server logs.", file=sys.stderr) + sys.exit(1) diff --git a/development/version.sh b/development/version.sh index 4fb36c5e4..1273d0cb5 100644 --- a/development/version.sh +++ b/development/version.sh @@ -4,6 +4,6 @@ # When the configuration of a container changes, the version should be incremented. # Breaking changes should increment the major version. # Non-breaking changes should increment the minor version. -export CURRENT_API_VERSION=8.4 -export CURRENT_WEB_VERSION=8.4 +export CURRENT_API_VERSION=11.0.x +export CURRENT_WEB_VERSION=11.0.x export CURRENT_DEV_VERSION=8.4 diff --git a/docs/ONTRACK_PODMAN_SETUP.md b/docs/ONTRACK_PODMAN_SETUP.md new file mode 100644 index 000000000..6cc4f8729 --- /dev/null +++ b/docs/ONTRACK_PODMAN_SETUP.md @@ -0,0 +1,694 @@ +# Running OnTrack locally with Podman on Bazzite or Fedora + +> **Historical, unsupported troubleshooting transcript.** Its service names, +> ports, and generated override no longer match the tracked three-file Compose +> merge, and its setup commands must not be treated as the current demo or +> production runbook. Use [RUNNING-LOCALLY.md](../RUNNING-LOCALLY.md) or the +> [retained all-features demo](../development/all-features-demo/README.md). +> This record remains only as background for future reviewed Podman support. + +This guide records the changes that were needed to run the existing OnTrack development environment with rootless Podman on Bazzite. + +The normal Docker Compose files were kept. A separate `docker-compose.podman.yml` file was added for the Podman-specific changes. + +This should also be useful for Fedora and other Linux systems where SELinux is enabled. + +## What was different with Podman + +The main issues were not with the OnTrack code itself. They came from differences between Docker and rootless Podman: + +- SELinux blocked some bind-mounted folders. +- MariaDB could not change ownership on the host database folder. +- The frontend container could not write to `package-lock.json` or the Angular cache. +- An old Docker container was already using the Mailpit ports. +- Compose reused an older API image with the wrong Ruby version. +- Podman tried to relabel the frontend `node_modules` folder and failed. + +The final setup keeps the normal Docker files unchanged and handles these problems in a local Podman override. + +## Important notes + +- Run Podman as your normal user. Do not use `sudo podman`. +- Run the Compose commands from `doubtfire-deploy/development`. +- Use all three Compose files in every Podman command. +- Do not run the Docker and Podman OnTrack stacks on the same ports. +- Do not use `podman compose down -v` unless you want to delete the local database. +- Do not disable SELinux globally. +- Do not use `chmod 777` as a workaround. +- Keep `docker-compose.podman.yml` local unless the team decides to support it officially. + +## 1. Check the repository layout + +The three repositories must be beside each other: + +```text +Ontrack Dev/ +|-- doubtfire-api/ +|-- doubtfire-deploy/ +`-- doubtfire-web/ +``` + +Go to the development folder: + +```bash +cd "/var/home/$USER/dev/Ontrack Dev/doubtfire-deploy/development" +``` + +Your path may be under `/home` rather than `/var/home`. Both can point to the same location on Bazzite. + +Check the repository paths: + +```bash +realpath ../../doubtfire-api +realpath ../../doubtfire-web +``` + +Check the main files: + +```bash +[[ -f ../../doubtfire-api/Gemfile ]] && echo "API path is correct" || echo "API Gemfile is missing" +[[ -f ../../doubtfire-web/package.json ]] && echo "Web path is correct" || echo "Web package.json is missing" +``` + +Do not put backslashes before `&&` or `||` when running these as one-line commands. Doing that caused a `binary operator expected` error during our setup. + +## 2. Stop any old Docker version of OnTrack + +We had an old Docker Mailpit container using ports `1025` and `8025`. Podman could not start its own Mailpit container until those ports were released. + +Check the ports used by OnTrack: + +```bash +sudo ss -ltnp | grep -E ':(1025|8025|3000|4200)([[:space:]]|$)' || true +``` + +Check Docker containers using those ports: + +```bash +sudo docker ps \ + --format 'table {{.ID}}\t{{.Names}}\t{{.Ports}}' \ + | grep -E '1025|8025|3000|4200' || true +``` + +If an old Docker OnTrack stack is running, stop it from the development folder: + +```bash +sudo docker compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + down --remove-orphans +``` + +Do not kill `docker-proxy` directly. Stop the Docker container that created it. + +Confirm the ports are free: + +```bash +sudo ss -ltnp | grep -E ':(1025|8025|3000|4200)([[:space:]]|$)' \ + || echo "Required OnTrack ports are free" +``` + +## 3. Create the Podman Compose override + +Create `docker-compose.podman.yml` inside `doubtfire-deploy/development`: + +```bash +cat > docker-compose.podman.yml <<'YAML' +services: + dev-db: + volumes: + - podman_db_data:/var/lib/mysql + + doubtfire-api: + image: localhost/ontrack-doubtfire-api:11.0-local + volumes: + - ../../doubtfire-api/:/doubtfire:z + - ../data/tmp:/doubtfire/tmp:z + - ../data/student-work:/student-work:z + + doubtfire-sidekiq: + image: localhost/ontrack-doubtfire-api:11.0-local + volumes: + - ../../doubtfire-api/:/doubtfire:z + - ../data/tmp:/doubtfire/tmp:z + - ../data/student-work:/student-work:z + + doubtfire-web: + image: localhost/ontrack-doubtfire-web:11.0-local + userns_mode: "keep-id:uid=1000,gid=1000" + security_opt: + - label=disable + command: /bin/bash -c 'npm install && npm start' + volumes: + - ../../doubtfire-web:/doubtfire-web + - ./proxy.conf.docker.json:/doubtfire-web/proxy.conf.json:ro + +volumes: + podman_db_data: +YAML +``` + +### Why these changes are needed + +`podman_db_data` is a Podman-managed volume for MariaDB. The original host bind mount failed because rootless Podman could not change ownership inside `/var/lib/mysql`. + +The local image names stop Compose from pulling or reusing an older public API image. We hit a Bundler exit code `18` because the old image had a Ruby version that did not match the current API Gemfile. + +The API mounts use `:z` so SELinux allows the source folders to be shared with the API containers. + +`doubtfire-sidekiq` is the background worker and runs the same API image, so it repeats the API block exactly. Without it the worker keeps the public `8.0.x-dev` tag from `docker-compose.yml` while the API uses the local build, and `up -d --no-build` pulls that old image instead of failing. It needs no separate build. Building `doubtfire-api` in step 8 produces the image both services use. + +The frontend uses `label=disable` because Podman failed while trying to relabel the full frontend repository, especially `node_modules`. + +The frontend also uses `keep-id` so the Node user inside the container can write to files owned by the local user. + +The command uses `npm install && npm start` so Angular does not start after a failed dependency install. + +## 4. Check the merged Compose configuration + +Run: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + config > /tmp/ontrack-podman-config.yml +``` + +Check the database section: + +```bash +grep -n -A20 '^ dev-db:' /tmp/ontrack-podman-config.yml +``` + +The database should use a named volume at `/var/lib/mysql`. It should not use `../data/database` as the active database mount. + +Check the frontend section: + +```bash +grep -n -A50 '^ doubtfire-web:' /tmp/ontrack-podman-config.yml +``` + +Confirm that it contains: + +```text +localhost/ontrack-doubtfire-web:11.0-local +keep-id:uid=1000,gid=1000 +label=disable +npm install && npm start +``` + +Warnings saying that the Compose `version` field is obsolete are harmless. + +The message saying Podman is executing an external Compose provider is also normal. On this system, `podman compose` used the installed Docker Compose plugin as its Compose provider. + +## 5. Prepare the API writable folders + +SELinux originally blocked the API mount. Later, Podman also failed with an `lsetxattr` error on `doubtfire-api/tmp`. + +Create the writable folders: + +```bash +sudo mkdir -p \ + ../../doubtfire-api/tmp \ + ../data/tmp \ + ../data/student-work +``` + +Return ownership to the current user: + +```bash +sudo chown -R "$(id -u):$(id -g)" \ + ../../doubtfire-api/tmp \ + ../data/tmp \ + ../data/student-work +``` + +Give the owner write access: + +```bash +sudo chmod -R u+rwX \ + ../../doubtfire-api/tmp \ + ../data/tmp \ + ../data/student-work +``` + +Apply the SELinux container label: + +```bash +sudo chcon -R system_u:object_r:container_file_t:s0 \ + ../../doubtfire-api/tmp \ + ../data/tmp \ + ../data/student-work +``` + +Check the labels: + +```bash +ls -ldZ \ + ../../doubtfire-api/tmp \ + ../data/tmp \ + ../data/student-work +``` + +Each path should show `container_file_t`. + +## 6. Prepare the frontend writable files + +The frontend initially failed with permission errors for: + +```text +/doubtfire-web/package-lock.json +/doubtfire-web/.angular/cache +``` + +Fix `package-lock.json` if it exists: + +```bash +if [ -f ../../doubtfire-web/package-lock.json ]; then + sudo chown "$(id -u):$(id -g)" ../../doubtfire-web/package-lock.json + sudo chmod u+rw ../../doubtfire-web/package-lock.json +fi +``` + +The original workaround recursively deleted a privileged cwd-relative cache +path. Do not repeat it. The supported local workflows use their tracked Compose +and named-volume contracts; a future Podman procedure must resolve and validate +an exact disposable cache target before offering any cleanup operation. + +Check the ownership: + +```bash +ls -ldn \ + ../../doubtfire-web \ + ../../doubtfire-web/.angular \ + ../../doubtfire-web/package-lock.json +``` + +The owner should match the result of: + +```bash +id -u +``` + +## 7. Clean up failed containers and the old dependency volume + +Stop the Podman stack without deleting volumes: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + down --remove-orphans +``` + +Remove failed temporary containers if they exist: + +```bash +podman rm -f ontrack-db-populate 2>/dev/null || true +podman rm -f doubtfire-web 2>/dev/null || true +``` + +Find the frontend dependency volume: + +```bash +podman volume ls --format '{{.Name}}' | grep web_node_modules || true +``` + +In our setup, the volume was called: + +```text +development_web_node_modules +``` + +Remove only that dependency volume: + +```bash +podman volume rm development_web_node_modules 2>/dev/null || true +``` + +The project prefix may be different on another computer. Remove the volume ending in `web_node_modules`. + +Do not remove the volume ending in `podman_db_data`. + +## 8. Build the current API and frontend images + +Check the API Dockerfile and current branches: + +```bash +grep -n '^FROM ruby:' ../../doubtfire-api/Dockerfile +git -C ../../doubtfire-api branch --show-current +git -C ../../doubtfire-web branch --show-current +``` + +Build the API from scratch: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + build --pull --no-cache doubtfire-api +``` + +Build the frontend: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + build --pull doubtfire-web +``` + +Confirm that the local images exist: + +```bash +podman images | grep -E 'ontrack-doubtfire-(api|web)' +``` + +Check the API Ruby and Bundler versions: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + run --rm -T \ + --no-deps \ + --entrypoint bash \ + doubtfire-api \ + -lc 'ruby -v; bundle -v' +``` + +The Ruby version must match the requirement in the current API Gemfile. + +Check the frontend image user: + +```bash +podman run --rm \ + --entrypoint id \ + localhost/ontrack-doubtfire-web:11.0-local +``` + +The image used during this setup reported UID and GID `1000`. If a future image uses another UID or GID, update the values in `userns_mode`. + +## 9. Start MariaDB, Redis, and Mailpit + +Start the supporting services first: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + up -d dev-db redis-sidekiq mailpit +``` + +Wait for MariaDB to initialise: + +```bash +sleep 20 +``` + +Check the containers: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + ps -a +``` + +Confirm MariaDB is ready: + +```bash +podman exec df-compose-dev-db \ + mariadb-admin ping \ + -h 127.0.0.1 \ + -uroot \ + -pdb-root-password +``` + +Expected output: + +```text +mysqld is alive +``` + +If the database exits, check its logs: + +```bash +podman logs --tail 200 df-compose-dev-db +``` + +## 10. Populate the database + +Use a named detached container so the logs remain available: + +```bash +podman rm -f ontrack-db-populate 2>/dev/null || true +``` + +Start the population task: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + run -d \ + --no-deps \ + --name ontrack-db-populate \ + --entrypoint bash \ + doubtfire-api \ + -lc 'bundle exec rake --trace db:populate' +``` + +Follow the logs: + +```bash +podman logs -f --tail 100 ontrack-db-populate +``` + +Check the status: + +```bash +podman inspect ontrack-db-populate \ + --format 'status={{.State.Status}} exit={{.State.ExitCode}} error={{.State.Error}}' +``` + +If the status is still `running`, do not try to remove it. Wait for it to finish: + +```bash +podman wait ontrack-db-populate +``` + +A successful task returns: + +```text +0 +``` + +The final inspect result should be: + +```text +status=exited exit=0 error= +``` + +After a successful run, remove the temporary container: + +```bash +podman rm ontrack-db-populate +``` + +If it exits with a non-zero code, keep the container until you have checked the logs: + +```bash +podman logs --tail 300 ontrack-db-populate +``` + +## 11. Start the complete OnTrack environment + +Start all services using the images that were already built: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + up -d --no-build +``` + +Check everything: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + ps -a +``` + +The main containers should be running: + +```text +df-compose-dev-db +df-compose-mailpit +df-compose-redis-sidekiq +doubtfire-api +doubtfire-sidekiq +doubtfire-web +``` + +Check the application logs: + +```bash +podman logs --tail 100 doubtfire-api +podman logs --tail 150 doubtfire-web +``` + +NPM deprecation warnings are not a startup failure. The important errors to look for are `EACCES`, `permission denied`, or `lsetxattr`. + +## 12. Check frontend write access + +Run: + +```bash +podman exec doubtfire-web bash -lc ' + echo "Container identity:" + id + + test -w /doubtfire-web/package-lock.json && + echo "package-lock.json is writable" || + echo "package-lock.json is not writable" + + mkdir -p /doubtfire-web/.angular/cache/podman-write-test && + rmdir /doubtfire-web/.angular/cache/podman-write-test && + echo "Angular cache is writable" +' +``` + +Both write checks should succeed. + +## 13. Open the local services + +```text +OnTrack web: http://localhost:4200 +API documentation: http://localhost:3000/api/docs +Mailpit: http://localhost:8025 +``` + +Common local test accounts use the password `password`: + +```text +student_1 +atutor +aconvenor +aadmin +``` + +## Normal commands after the first setup + +Start the environment: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + up -d --no-build +``` + +Stop the environment: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + down +``` + +Check status: + +```bash +podman compose \ + -f docker-compose.yml \ + -f docker-compose.local-paths.yml \ + -f docker-compose.podman.yml \ + ps -a +``` + +Follow API logs: + +```bash +podman logs -f doubtfire-api +``` + +Follow frontend logs: + +```bash +podman logs -f doubtfire-web +``` + +## Errors we hit + +| Error or message | Cause | Fix | +|---|---|---| +| `binary operator expected` | A Bash test command was pasted with incorrect backslashes | Run the test as one normal line | +| `/doubtfire: Permission denied` | SELinux blocked the API bind mount | Use `:z` on the API mounts | +| `lsetxattr ... doubtfire-api/tmp ... operation not permitted` | API writable folders had unsuitable ownership or SELinux labels | Use `chown`, `chmod`, and `chcon` on the writable folders | +| `bind: address already in use` on port 1025 | An old Docker Mailpit container was still running | Stop the old Docker stack | +| `/var/lib/mysql: Permission denied` | Rootless Podman could not change ownership on the database bind mount | Use the `podman_db_data` named volume | +| Database population exited with code 18 | Compose used an older API image with the wrong Ruby version | Use unique local image names and rebuild the API | +| `ontrack-db-populate` could not be removed | The population job was still running | Follow its logs and wait for it to exit | +| `lsetxattr ... doubtfire-web/node_modules` | Podman tried to relabel the full frontend repository | Use `security_opt: label=disable` for the frontend | +| `EACCES` for `package-lock.json` | The frontend user could not write to the host file | Use `keep-id` and repair the file ownership | +| `EACCES` for `.angular/cache` | The Angular cache had the wrong owner | Delete and recreate `.angular` as the local user | +| Angular started after `npm install` failed | The original command used `;` | Use `npm install && npm start` | +| `version is obsolete` | The Compose files contain an older `version` field | Harmless warning | +| `Executing external compose provider` | `podman compose` is using an installed Compose provider | Normal behaviour | + +## Final Podman override + +The final working `docker-compose.podman.yml` was: + +```yaml +services: + dev-db: + volumes: + - podman_db_data:/var/lib/mysql + + doubtfire-api: + image: localhost/ontrack-doubtfire-api:11.0-local + volumes: + - ../../doubtfire-api/:/doubtfire:z + - ../data/tmp:/doubtfire/tmp:z + - ../data/student-work:/student-work:z + + doubtfire-sidekiq: + image: localhost/ontrack-doubtfire-api:11.0-local + volumes: + - ../../doubtfire-api/:/doubtfire:z + - ../data/tmp:/doubtfire/tmp:z + - ../data/student-work:/student-work:z + + doubtfire-web: + image: localhost/ontrack-doubtfire-web:11.0-local + userns_mode: "keep-id:uid=1000,gid=1000" + security_opt: + - label=disable + command: /bin/bash -c 'npm install && npm start' + volumes: + - ../../doubtfire-web:/doubtfire-web + - ./proxy.conf.docker.json:/doubtfire-web/proxy.conf.json:ro + +volumes: + podman_db_data: +``` + +The existing OnTrack Docker setup did not need to be rewritten. The working solution was a small local override for SELinux, rootless file ownership, MariaDB storage, and the local API and frontend images. diff --git a/docs/vapid-secret-audit.md b/docs/vapid-secret-audit.md new file mode 100644 index 000000000..723d9ee8d --- /dev/null +++ b/docs/vapid-secret-audit.md @@ -0,0 +1,121 @@ +# Finish the VAPID private-key audit (NPR-S03) + +The source controls and rotation procedure do not establish that the actual +production private key is absent from historical commits, image layers and +logs. The release owner can complete that check without sharing the key or +dumping container environments. No staging URL or test account is needed. + +## What the owner needs + +Ask the release owner or operator with secret-manager, registry and log access +for these inputs. Keep the secret itself inside that operator's environment. + +1. The **production VAPID private-key version identifier** and read access to + that version in the approved secret manager. Audit older versions too if + the ticket covers their use. A development key is not evidence about + production. +2. The **exact API and Sidekiq image IDs/digests** that ran with that version, + including retained releases in the audit period. Read only the image ID + from a running container with `docker inspect --format '{{.Image}}' + CONTAINER`; do not dump its environment or full inspection output. Pull + the identified images onto the trusted audit host if they are not local. +3. Full local clones of **api, web and deploy**, with the relevant branches, + tags and PR refs fetched. The tool refuses shallow clones. Submodules are + separate repositories; supply each explicitly. Remote-only or pruned + history cannot be certified from an incomplete local clone. +4. Protected exports of **API, Sidekiq, reverse-proxy and build/CI logs** for + that secret version's lifetime, including retained/rotated logs. Supply + each file with `--file`; gzip exports are supported. Missing retention + periods must be recorded as unknown, not passed. + +## Run on the trusted audit host + +Use Python 3 and Git. Docker is needed only for `--image`. Export the selected +key directly from the existing secret manager to a new owner-only file outside +the repositories. Use the manager's protected file/export facility; do not put +the value in a command argument, shell history, environment dump, screenshot, +chat, ticket, or build argument. The file must be owned by the invoking user, +have no group/other permissions (for example `0600`), and must not be a symlink. + +Set the following variables to **paths or non-secret image references**: + +```sh +umask 077 +: "${VAPID_AUDIT_KEY_FILE:?Protected key-file path}" +: "${VAPID_AUDIT_API_IMAGE:?Exact API image ID or digest}" +: "${VAPID_AUDIT_WORKER_IMAGE:?Exact Sidekiq image ID or digest}" +: "${VAPID_AUDIT_API_LOG:?Protected API log export path}" +: "${VAPID_AUDIT_WORKER_LOG:?Protected Sidekiq log export path}" +: "${VAPID_AUDIT_PROXY_LOG:?Protected reverse-proxy log export path}" +: "${VAPID_AUDIT_BUILD_LOG:?Protected build log export path}" +: "${VAPID_AUDIT_REPORT:?New protected report path outside Git}" + +python3 -B tools/audit-vapid-secret.py \ + --key-file "$VAPID_AUDIT_KEY_FILE" \ + --repo ../doubtfire-api --repo ../doubtfire-web --repo . \ + --image "$VAPID_AUDIT_API_IMAGE" --image "$VAPID_AUDIT_WORKER_IMAGE" \ + --file "$VAPID_AUDIT_API_LOG" --file "$VAPID_AUDIT_WORKER_LOG" \ + --file "$VAPID_AUDIT_PROXY_LOG" --file "$VAPID_AUDIT_BUILD_LOG" \ + > "$VAPID_AUDIT_REPORT" +audit_result=$? +printf 'VAPID audit exit status: %s\n' "$audit_result" +``` + +Alternatively, pipe the secret manager's raw value output directly to +`--key-stdin`. Interactive terminal input is rejected. Existing protected +`docker image save` archives can be supplied with repeated `--image-archive` +instead of invoking Docker. Never use `docker export`: it drops image history +and deleted layers, so it cannot establish this requirement. + +The tool emits JSON with match locations/counts, image identities, scanned +counts and completeness for the selected inputs. It does not print matching +content, a key fingerprint or key values. It suppresses subprocess/parser +diagnostics and redacts secret-bearing paths. For a Git object finding, locate +its commits without displaying file contents: + +```sh +git -C REPOSITORY log --all --find-object=OBJECT_ID --format='%H %cI' +``` + +## Interpret and record the result + +| Exit | Meaning | Required action | +| --- | --- | --- | +| `0` | No matches in all selected inputs | Verify the input inventory and retention coverage, then record the owner, UTC date, secret-manager version ID, Git refs, resolved image IDs, log periods, scanner commit and protected report location. | +| `1` | Matches found | Treat the listed locations as exposure evidence. Follow the existing [VAPID rotation procedure](../DEPLOYING.md#web-push-vapid-contract) and API delivery-operations runbook with the release owner. This tool does not rotate keys, delete subscriptions or rewrite history. | +| `2` | Invalid input or incomplete scan | Resolve missing images/files, permissions, archive format or shallow history and rerun. Even zero matches is not a pass when inputs are incomplete. | + +Inspect the JSON exit result after completion. If the process is interrupted, +no successful audit exists. Keep reports and exported logs under the operator's +normal protected evidence/retention policy; do not commit them. Remove the +temporary key export using the secret manager's approved local-file handling +procedure after the audit. A copied key export must never become a build input. + +## Coverage and limits + +- Git scanning reads **every locally available object**, including blobs for + deleted files, commit/tag messages and unreachable objects. It does not run + checkout filters or write repository files. LFS content and external + submodule objects need separate explicit inputs. +- Docker scanning resolves a local tag to an immutable image ID before saving + it, scans image configuration/history and **each original layer separately**, + and never unpacks paths to the host. A later whiteout/deletion does not hide + an earlier leaked file. Gzip-compressed layers are supported. The temporary + save archive stays in a private directory and is removed after scanning. +- Matching covers the literal base64url private key, padded/standard Base64, + raw 32-byte scalar, hexadecimal scalar and Base64-encoded text. This is an + exact-key audit, not a general secret detector. Arbitrary encrypted files, + split/obfuscated values or nested compressed payloads are not decoded. +- Zero matches says nothing about unprovided production keys, remote logs, + unavailable image releases, expired log periods or copies outside these + inputs. Repository fixtures prove the scanner works; they are not production + attestation. + +The current repository contract supplies `DOUBTFIRE_VAPID_PRIVATE_KEY` only in +API/Sidekiq runtime configuration. Production example env values are +placeholders. API `.dockerignore` excludes local env files, credentials, key +files and logs, and release publishing creates contexts from Git archives. +API PR [#174](https://github.com/ontrack-features-t2-2026/doubtfire-api/pull/174) +adds production paired-key validation, filtered private-key parameters and the +rotation procedure. A matching private key remains necessary at runtime; its +presence in the running process environment is not an image-layer finding. diff --git a/production/.env.production b/production/.env.production deleted file mode 100644 index 9ce785fba..000000000 --- a/production/.env.production +++ /dev/null @@ -1,115 +0,0 @@ -# -# Rails environment -# -RAILS_ENV=production - -# -# Database settings - please update password to align with docker compose or provided database -# -DF_PRODUCTION_DB_ADAPTER=mysql2 -DF_PRODUCTION_DB_HOST=doubtfire-db -DF_PRODUCTION_DB_DATABASE=doubtfire -DF_PRODUCTION_DB_USERNAME=dfire -DF_PRODUCTION_DB_PASSWORD=pwd - -# -# Database and auth related secrets -# -DF_SECRET_KEY_BASE=test-secret-key -DF_SECRET_KEY_ATTR=test-secret-key -DF_SECRET_KEY_DEVISE=test-secret-key -DF_SECRET_KEY_MOSS= - -# Student work location in the containers - should not need to change -DF_STUDENT_WORK_DIR=/student-work - -# -# Latex settings -# -LATEX_CONTAINER_NAME=texlive-container -LATEX_BUILD_PATH=/texlive/shell/latex_build.sh - -# Redis for sidekiq -DF_REDIS_SIDEKIQ_URL=redis://redis-sidekiq:6379/0 - -# -# Institution settings -# -DF_INSTITUTION_HOST=https://localhost -DF_INSTITUTION_PRODUCT_NAME=OnTrack -DF_INSTITUTION_SETTINGS_RB=no_institution_setting.rb -DF_INSTITUTION_EMAIL_DOMAIN=unifoo.edu.au -DF_INSTITUTION_NAME=unifoo - -# Authentication settings. One of: -# aaf -# saml -# ldap -# database (for development only) -DF_AUTH_METHOD=database - -# -# AAF settings -# -DF_AAF_ISSUER_URL=https://rapid.test.aaf.edu.au -DF_AAF_AUDIENCE_URL= -DF_AAF_CALLBACK_URL= -DF_AAF_IDENTITY_PROVIDER_URL= -DF_AAF_UNIQUE_URL= -DF_AAF_AUTH_SIGNOUT_URL= -DF_SECRET_KEY_AAF= - -# -# SAML Settings -# - -# Optional metadata url -DF_SAML_METADATA_URL= -# URL to return the SAML response to -DF_SAML_CONSUMER_SERVICE_URL=https://localhost/api/auth/jwt -# URL of the registered application -DF_SAML_SP_ENTITY_ID=https://localhost -# The IDP SAML login URL -DF_SAML_IDP_TARGET_URL=https://login.microsoftonline.com/xxxx/saml2 -# One of: -# urn:oasis:names:tc:SAML:2.0:nameid-format:persistent, -# urn:oasis:names:tc:SAML:2.0:nameid-format:transient, -# urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress -# urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified, -# urn:oasis:names:tc:SAML:1.1:nameid-format:X509SubjectName, -# urn:oasis:names:tc:SAML:1.1:nameid-format:WindowsDomainQualifiedName -# urn:oasis:names:tc:SAML:2.0:nameid-format:kerberos, -# urn:oasis:names:tc:SAML:2.0:nameid-format:entity -DF_SAML_IDP_SAML_NAME_IDENTIFIER_FORMAT=urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress - -# -# LDAP settings -# -DF_LDAP_HOST= -DF_LDAP_PORT= -DF_LDAP_ATTRIBUTE= -DF_LDAP_BASE= -DF_LDAP_SSL=false -DF_LDAP_USE_ADMIN_TO_BIND=false -DF_LDAP_ADMIN_USER= -DF_LDAP_ADMIN_PWD= - - -# -# Overseer - requires additional configuration -# -OVERSEER_ENABLED=0 - -# Active record encryption -DF_ENCRYPTION_PRIMARY_KEY=AMLOMYA5GV8B4fTK3VKMhVGn8WdvUW8g -DF_ENCRYPTION_DETERMINISTIC_KEY=anlmuJ6cB3bN3biXRbYvmPsC5ALPFqGG -DF_ENCRYPTION_KEY_DERIVATION_SALT=hzPR8D4qpOnAg7VeAhkhWw6JmmzKJB10 - -DF_MAIL_PERFORM_DELIVERIES=no -DF_MAIL_DELIVERY_METHOD=smtp -DF_SMTP_ADDRESS= -DF_SMTP_PORT= -DF_SMTP_DOMAIN= -DF_SMTP_USERNAME= -DF_SMTP_PASSWORD= -DF_SMTP_AUTHENTICATION= diff --git a/production/.env.production.example b/production/.env.production.example new file mode 100644 index 000000000..acb92717f --- /dev/null +++ b/production/.env.production.example @@ -0,0 +1,146 @@ +# Copy this file to .env.production, replace every REPLACE_ME value, and run +# ./validate.sh before deployment. Keep the completed file mode 0600 and never +# commit it. + +# Compose and public endpoint +COMPOSE_PROJECT_NAME=doubtfire-production +SERVER_NAME=ontrack.REPLACE_ME.edu.au +DF_INSTITUTION_HOST=https://ontrack.REPLACE_ME.edu.au +TZ=Australia/Melbourne +# Total HTTP request limit, not the API per-file policy. At least 32m is needed +# for chat attachments plus multipart overhead; see UPLOAD-LIMITS.md. +CLIENT_MAX_BODY_SIZE=1g + +# Immutable container references. Resolve and record the tested sha256 digest +# for every release; mutable tags such as latest or 11.0.x are rejected. +PROXY_IMAGE=nginx:1.30.4-alpine@sha256:REPLACE_ME +MARIADB_IMAGE=mariadb@sha256:REPLACE_ME +REDIS_IMAGE=redis@sha256:REPLACE_ME +DOCKER_SOCKET_PROXY_IMAGE=haproxy:3.2-alpine@sha256:REPLACE_ME +DOUBTFIRE_API_IMAGE=lmsdoubtfire/apiserver@sha256:REPLACE_ME +DOUBTFIRE_APP_IMAGE=lmsdoubtfire/appserver@sha256:REPLACE_ME +DOUBTFIRE_WEB_IMAGE=lmsdoubtfire/doubtfire-web@sha256:REPLACE_ME +TEXLIVE_IMAGE=lmsdoubtfire/formatif-latex@sha256:REPLACE_ME +JPLAG_IMAGE=lmsdoubtfire/doubtfire-jplag@sha256:REPLACE_ME + +# Absolute host paths. Create every directory before deployment and include the +# database and complete student-work tree in the backup policy. +STUDENT_WORK_PATH=/srv/doubtfire/student-work +DOUBTFIRE_LOG_PATH=/srv/doubtfire/log +MARIADB_DATA_PATH=/srv/doubtfire/mariadb +REDIS_DATA_PATH=/srv/doubtfire/redis +JPLAG_REPORT_PATH=/srv/doubtfire/student-work/jplag +JPLAG_ARCHIVE_REPORT_PATH=/srv/doubtfire/student-work/archive/jplag +TLS_CERTIFICATE_PATH=/etc/letsencrypt/live/ontrack.REPLACE_ME.edu.au/fullchain.pem +TLS_PRIVATE_KEY_PATH=/etc/letsencrypt/live/ontrack.REPLACE_ME.edu.au/privkey.pem +ALIASES_PATH=/etc/doubtfire/aliases +MSMTPRC_PATH=/etc/doubtfire/msmtprc +DOCKER_SOCKET_PATH=/var/run/docker.sock + +# Database. Generate independent high-entropy passwords; do not reuse them. +MARIADB_DATABASE=doubtfire +MARIADB_USER=dfire +MARIADB_PASSWORD=REPLACE_ME_WITH_AT_LEAST_20_RANDOM_CHARACTERS +MARIADB_ROOT_PASSWORD=REPLACE_ME_WITH_AT_LEAST_20_RANDOM_CHARACTERS + +# Rails signing and encryption keys. Existing deployments must retain their +# current values or encrypted data and sessions may become unreadable. +DF_SECRET_KEY_BASE=REPLACE_ME_WITH_AT_LEAST_32_RANDOM_CHARACTERS +DF_SECRET_KEY_ATTR=REPLACE_ME_WITH_AT_LEAST_32_RANDOM_CHARACTERS +DF_SECRET_KEY_DEVISE=REPLACE_ME_WITH_AT_LEAST_32_RANDOM_CHARACTERS +DF_SECRET_KEY_MOSS= +DF_ENCRYPTION_PRIMARY_KEY=REPLACE_ME_WITH_AT_LEAST_32_RANDOM_CHARACTERS +DF_ENCRYPTION_DETERMINISTIC_KEY=REPLACE_ME_WITH_AT_LEAST_32_RANDOM_CHARACTERS +DF_ENCRYPTION_KEY_DERIVATION_SALT=REPLACE_ME_WITH_AT_LEAST_32_RANDOM_CHARACTERS + +# Background job threads. Five is the maximum supported by the default Active +# Record connection pool; reduce only after measuring queue latency and load. +DF_SIDEKIQ_CONCURRENCY=5 + +# Peer Progress Indicator privacy and freshness policy for both the compact +# completion view and the advanced status distribution. The minimum applies to +# the remaining peer cohort after the authenticated student is excluded, and +# may be raised above 21. The maximum snapshot age may be reduced below 48 +# hours. Never lower the cohort value to make a suppressed view appear. +DF_PPI_MINIMUM_COHORT_SIZE=21 +DF_PPI_STALE_AFTER_HOURS=48 + +# Optional Teams announcement import. Existing OnTrack SSO is unchanged. +# See ../UNIT-HUB.md before enabling; requires university-approved Microsoft +# app consent and an explicit student-wide channel / approved publisher mapping. +# Keep secrets in the deployment secret store, never in a committed env file. +DF_TEAMS_ANNOUNCEMENTS_ENABLED=false +DF_TEAMS_CHANNEL_MAPPINGS=[] +DF_TEAMS_TENANT_ID= +DF_TEAMS_CLIENT_ID= +DF_TEAMS_CLIENT_SECRET= + +# Web Push. Generate a production-only VAPID P-256 keypair; never reuse the +# checked-in development pair. The private key is a persistent production +# secret. The subject must be an operational mailto: or HTTPS contact. +DOUBTFIRE_VAPID_PUBLIC_KEY=REPLACE_ME_WITH_VAPID_PUBLIC_KEY +DOUBTFIRE_VAPID_PRIVATE_KEY=REPLACE_ME_WITH_VAPID_PRIVATE_KEY +DOUBTFIRE_VAPID_SUBJECT=mailto:ontrack-operations@REPLACE_ME.edu.au + +# Institution +DF_INSTITUTION_PRODUCT_NAME=OnTrack +DF_INSTITUTION_SETTINGS_RB=no_institution_setting.rb +DF_INSTITUTION_EMAIL_DOMAIN=REPLACE_ME.edu.au +DF_INSTITUTION_EMAIL_SENDER=ontrack@REPLACE_ME.edu.au +DF_INSTITUTION_NAME=REPLACE_ME + +# Authentication must use an external production identity provider. Valid +# values are aaf, saml, or ldap. Fill the matching section below. +DF_AUTH_METHOD=saml + +# AAF +DF_AAF_ISSUER_URL= +DF_AAF_AUDIENCE_URL= +DF_AAF_CALLBACK_URL= +DF_AAF_IDENTITY_PROVIDER_URL= +DF_AAF_UNIQUE_URL= +DF_AAF_AUTH_SIGNOUT_URL= +DF_SECRET_KEY_AAF= + +# SAML +DF_SAML_METADATA_URL=https://REPLACE_ME/saml/metadata +DF_SAML_CONSUMER_SERVICE_URL=https://ontrack.REPLACE_ME.edu.au/api/auth/jwt +DF_SAML_SP_ENTITY_ID=https://ontrack.REPLACE_ME.edu.au +DF_SAML_IDP_TARGET_URL=https://REPLACE_ME/saml2 +DF_SAML_IDP_SAML_NAME_IDENTIFIER_FORMAT=urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress + +# LDAP +DF_LDAP_HOST= +DF_LDAP_PORT= +DF_LDAP_ATTRIBUTE= +DF_LDAP_BASE= +DF_LDAP_SSL=true +DF_LDAP_USE_ADMIN_TO_BIND=false +DF_LDAP_ADMIN_USER= +DF_LDAP_ADMIN_PWD= + +# Resource ceilings for student-controlled document and similarity processing. +TEXLIVE_MEMORY_LIMIT=2g +TEXLIVE_CPU_LIMIT=2 +TEXLIVE_PIDS_LIMIT=256 +JPLAG_MEMORY_LIMIT=4g +JPLAG_CPU_LIMIT=2 +JPLAG_PIDS_LIMIT=256 + +# Mail is required for production notifications. +DF_MAIL_PERFORM_DELIVERIES=yes +DF_MAIL_DELIVERY_METHOD=smtp +DF_SMTP_ADDRESS=smtp.REPLACE_ME.edu.au +DF_SMTP_PORT=587 +DF_SMTP_DOMAIN=REPLACE_ME.edu.au +DF_SMTP_USERNAME=REPLACE_ME +DF_SMTP_PASSWORD=REPLACE_ME +DF_SMTP_AUTHENTICATION=plain + +# Overseer is not supported by this single-host core stack. It requires a +# separate runner storage and image-management contract, so keep it disabled. +OVERSEER_ENABLED=0 + +# First-time tutorial rollout. 1 enables it; 0 keeps it off. +# Recreate apiserver after changing this; no web rebuild is required. +TUTORIAL_ENABLED=0 diff --git a/production/CAkeys/RootCA.crt b/production/CAkeys/RootCA.crt deleted file mode 100644 index 4f58f6aa1..000000000 --- a/production/CAkeys/RootCA.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICyjCCAbICCQC6Ht/UfcerKDANBgkqhkiG9w0BAQsFADAnMQswCQYDVQQGEwJV -UzEYMBYGA1UEAwwPRXhhbXBsZS1Sb290LUNBMB4XDTIyMDEyODA1NDY1OVoXDTI0 -MTExNzA1NDY1OVowJzELMAkGA1UEBhMCVVMxGDAWBgNVBAMMD0V4YW1wbGUtUm9v -dC1DQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKqaTmTDFKPej78t -BboHGUiHwvtK+QbSNAJwpcJYc9d3/2/2fS/IaQKQMSmWHfxT3S/1CSMycGpPeVx1 -l6hI2M6YnuUAzF1kbd3d4Z5UeWVeIBjAj20S7Q8d8P45MsYcml+STzAvyfUg3KQc -YAtjTnGpcZPmCfpIMY5PxZkENuMLp7g2MMm0BqTnRJIA7Gm78V+GBTFWs97M6XAU -haYW0fQoJ4Ms18fmH2G83ouccXq8yTYkhS3xC+lWIbM6sunCbbzYPgJn2NhiXZYX -KN4qknhwXsGbhjZQCr94i2nmf9geY+qchu4pNwJqQEWju9UWjFhrsYv6Cad8X9U5 -xdol7dsCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAIjLVdBBhQGh0GUis8Nl7I62q -S8iVIVu0+XP2CWSlzP/abGs7rsa0QZIXeNYxCMiK1LERFMkVxQ7X0SV151kzqsvZ -e8cTB9OV24pkmQhQBBx2Bz3ysRW7WFs/yt9Nn+7lnE2JQYrcrjEibOpYmpmkNq+T -fM4J4P8ncHlwlSSM7WLzSjheOyHo94TckS9DEUzQ5DPecR0BQUaXeOFo4hPYYwPr -UUhErGo7K5a/DRDpZf8R8U/Kge+gowsINV8sBIiRfhXvDETdE7w4YiL0AP6Tx5b/ -x0e3X9aEKKMYUia1Rl4OFINJDIZgFfRT7hDRowfRJ4AwT9PgszrWRG+aIb/rsg== ------END CERTIFICATE----- diff --git a/production/CAkeys/RootCA.key b/production/CAkeys/RootCA.key deleted file mode 100644 index a9ec795ca..000000000 --- a/production/CAkeys/RootCA.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCqmk5kwxSj3o+/ -LQW6BxlIh8L7SvkG0jQCcKXCWHPXd/9v9n0vyGkCkDEplh38U90v9QkjMnBqT3lc -dZeoSNjOmJ7lAMxdZG3d3eGeVHllXiAYwI9tEu0PHfD+OTLGHJpfkk8wL8n1INyk -HGALY05xqXGT5gn6SDGOT8WZBDbjC6e4NjDJtAak50SSAOxpu/FfhgUxVrPezOlw -FIWmFtH0KCeDLNfH5h9hvN6LnHF6vMk2JIUt8QvpViGzOrLpwm282D4CZ9jYYl2W -FyjeKpJ4cF7Bm4Y2UAq/eItp5n/YHmPqnIbuKTcCakBFo7vVFoxYa7GL+gmnfF/V -OcXaJe3bAgMBAAECggEBAIJAVWHw3y1Nmwom4ZxqFw+dN2yMjRx8GGFUh/mddrQn -Vcqxslps9F+UnzRU/IfNr64RsuxAcjt4efvxe9xr4bu6InZM6brUFl2yLbOoYxbV -ikPboKVyMbO89GXkXbOL2wDkv7jWg6a3dD2HvHx1UUrOR3zXwP5paAxzMzXkVdN5 -rlbY24/1O6vAjzJYQGoc3S0TdybF/57rsHw8/e3dQq3SqQrWeNL8VyED4GtTU4a0 -8X6iCS0AVe0Y4SjpXfg0aTBN8zyex/up68b/Ny4P4SNniPVV3c5CQz1YqhP+k274 -3pPJ+VpiXQyA+/JV9xMr33MZZI74D5QoqNQULJuLxuECgYEA4yO5ep1fM+L8zApN -a47gtq3D93GDumbKmu9viKqSa50xeMwi+HHGMgw0MFyXHFDOBQCZjfvcqkK5fOmc -OkG9vcNJx5ys0owFTj2+Dw+5ByBuHXnROJ7tuKoQnA94qmP+j8WUyitoZEpDk/Q3 -byi/R/cArekVlLWZFEAu9yjo8NUCgYEAwEeVMjq8HQVZB5FC46tpTYMyPj2TyTT9 -IclJ4t/p55pejKxwppQrYHb6r1/vIjxelm42q4+ZH+i4hrxDQ2sKRG0bl+LIkcwo -y2VOxnHiSqLjRMvG2U6b9rAP/6kilgDcxsxFASdh4z2YO4mXJYRwYpJy0C9NKh9r -/4eM7Ap8O+8CgYAmG1E72/kj6FoHFVdlyyOyIjb1ZYr3E9W75p2srQlBmq65GG5p -hhT9QoTi9DW532ig+DfK1U1xFx6tcTfvoiXmF0Rig8uOy0T8mmkvd7DQHEW9Fjne -qPMy/v0oYYQZ+gq0Li7HOD8cTEVIJjnCoXBt72eixa8EMu9KsXXvq2JhPQKBgBic -ENhw9Aqy2IZDbVA+bCfjltaogvXJtmxQmwhSqFlgRKG0oGZxqGucTtGLccvr9h9C -0/R5WJvg6V01U1aZqKLBZDq4Z/thKgd+W+rCQLhUyMnRWw4NJaEZP7L1LtwMGDXJ -hztykvDGvWlBXvsKTGgRXmiO5U1y4BvAeaTsYqI5AoGAZ9WJ/nGMrc+G3loCKvkJ -byiaGlF7UwC/tgN9slfABMZ3sMFtJ7+DEGiav1XpnmqHguEPQJT6r7UwpUi18Qvz -0KeZh9riry5a2YMR/XyubbldBzSdjzgmX3LwpiK7uH9Oil0pgtk4gYg9bbaFudFP -idEnB445EE7hY+ZzBp7lHBA= ------END PRIVATE KEY----- diff --git a/production/CAkeys/RootCA.pem b/production/CAkeys/RootCA.pem deleted file mode 100644 index 4f58f6aa1..000000000 --- a/production/CAkeys/RootCA.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICyjCCAbICCQC6Ht/UfcerKDANBgkqhkiG9w0BAQsFADAnMQswCQYDVQQGEwJV -UzEYMBYGA1UEAwwPRXhhbXBsZS1Sb290LUNBMB4XDTIyMDEyODA1NDY1OVoXDTI0 -MTExNzA1NDY1OVowJzELMAkGA1UEBhMCVVMxGDAWBgNVBAMMD0V4YW1wbGUtUm9v -dC1DQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKqaTmTDFKPej78t -BboHGUiHwvtK+QbSNAJwpcJYc9d3/2/2fS/IaQKQMSmWHfxT3S/1CSMycGpPeVx1 -l6hI2M6YnuUAzF1kbd3d4Z5UeWVeIBjAj20S7Q8d8P45MsYcml+STzAvyfUg3KQc -YAtjTnGpcZPmCfpIMY5PxZkENuMLp7g2MMm0BqTnRJIA7Gm78V+GBTFWs97M6XAU -haYW0fQoJ4Ms18fmH2G83ouccXq8yTYkhS3xC+lWIbM6sunCbbzYPgJn2NhiXZYX -KN4qknhwXsGbhjZQCr94i2nmf9geY+qchu4pNwJqQEWju9UWjFhrsYv6Cad8X9U5 -xdol7dsCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAIjLVdBBhQGh0GUis8Nl7I62q -S8iVIVu0+XP2CWSlzP/abGs7rsa0QZIXeNYxCMiK1LERFMkVxQ7X0SV151kzqsvZ -e8cTB9OV24pkmQhQBBx2Bz3ysRW7WFs/yt9Nn+7lnE2JQYrcrjEibOpYmpmkNq+T -fM4J4P8ncHlwlSSM7WLzSjheOyHo94TckS9DEUzQ5DPecR0BQUaXeOFo4hPYYwPr -UUhErGo7K5a/DRDpZf8R8U/Kge+gowsINV8sBIiRfhXvDETdE7w4YiL0AP6Tx5b/ -x0e3X9aEKKMYUia1Rl4OFINJDIZgFfRT7hDRowfRJ4AwT9PgszrWRG+aIb/rsg== ------END CERTIFICATE----- diff --git a/production/CAkeys/RootCA.srl b/production/CAkeys/RootCA.srl deleted file mode 100644 index 1c44b75e7..000000000 --- a/production/CAkeys/RootCA.srl +++ /dev/null @@ -1 +0,0 @@ -C5CAF36E598BDB2A diff --git a/production/CAkeys/domains.ext b/production/CAkeys/domains.ext deleted file mode 100644 index 7572a9161..000000000 --- a/production/CAkeys/domains.ext +++ /dev/null @@ -1,7 +0,0 @@ -authorityKeyIdentifier=keyid,issuer -basicConstraints=CA:FALSE -keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment -subjectAltName = @alt_names -[alt_names] -DNS.1 = localhost -IP.1 = 127.0.0.1 diff --git a/production/CAkeys/localhost.csr b/production/CAkeys/localhost.csr deleted file mode 100644 index 1f6e5de0b..000000000 --- a/production/CAkeys/localhost.csr +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICrTCCAZUCAQAwaDELMAkGA1UEBhMCQVUxDDAKBgNVBAgMA1ZJQzESMBAGA1UE -BwwJTWVsYm91cm5lMR0wGwYDVQQKDBRFeGFtcGxlLUNlcnRpZmljYXRlczEYMBYG -A1UEAwwPbG9jYWxob3N0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA7W68G/dlx/UmwyFT8YprN7kje/ImaIbopoBMSxbFGwSXRykeGjgMe49H -K5pSVab/tqyu3RxgkUtIPixfC93zhFU444CHRumnQoJLmIlvf50plQ2vNoNDxnnj -TDjAsTjNioLCNhPRunw6eeO416GMEcIx+rIX/STieuQy7CdnL7Kl87ZXndZPlP1o -2BbiZeR5EDRm5M9GY5udSV1KpHh801g6CAmnzw5y3RJUF73HwO9ULI1lJMZIA18F -7K5/KZKS1ohPrLpxV7ale182MYmfaLL2ZAXsbUGspMoyP/Lm6fOMd3ath3uMkpE4 -SFBnUqWDdIkPOrTRev2L6QidUyuhgQIDAQABoAAwDQYJKoZIhvcNAQELBQADggEB -AL1X6/esQzFXOl8OkBSH8sm/iAl0g6U5gJu+spxKYCvs8eP+TS0QrcBo070/HmOc -pxWD9o5anBtHg95SRLVoQyZzcyyq+s/QLyzXodJn7UgzewSu5gtY6x4twlQpkdyp -uJJTc1cW1anhGB3aSn/7Nva7iftL+ZKQaxPvsPiN+V07PBcJUrHwSP9yQMtz99TB -+EXzy+BQu+b0YQmm42yM34ZQxCZDXPyVyC6YuYgXC+nItFYoNuQypTpjPpeuuZui -A/A9CEbX+5/+PyIaGSI5ZzL6f9VBHSP5lp0oskvz1O7jo+JpiJ5hwQLeJAzJg7E7 -xR+sNjvVuk9fvrf7V2tKTVA= ------END CERTIFICATE REQUEST----- diff --git a/production/UPLOAD-LIMITS.md b/production/UPLOAD-LIMITS.md new file mode 100644 index 000000000..45a2eb545 --- /dev/null +++ b/production/UPLOAD-LIMITS.md @@ -0,0 +1,94 @@ +# Upload request and file limits + +FILE-DP01 separates the proxy's total HTTP request limit from the API's file +policy. Nginx cannot decide whether a file is an authorized task submission, +chat attachment, valid format or safe document. Rails remains responsible for +all of those checks. + +| Context | API limit | Boundary | +| --- | --- | --- | +| Task submission, per upload requirement | `DF_MAX_FILE_SIZE`, default 10,000,000 bytes | A file at the limit is permitted if its format is valid. | +| Task chat, one attachment per comment | 30,000,000 bytes | Must be nonempty and strictly below the limit. | +| Production proxy, entire request | `CLIENT_MAX_BODY_SIZE`, default `1g` (1,073,741,824 bytes) | Exactly the configured size is permitted; one byte more returns 413 before Rails. | + +The existing production default already allows chat files plus multipart +overhead. No limit increase is needed. The total request budget also serves +multi-file task submissions and other API routes; it is not a new per-file +allowance. The production validator rejects unlimited limits and configurations +below `32m` (33,554,432 bytes), which leaves more than 3 MB of multipart headroom +above the existing chat boundary. A deployment with larger multi-file requests +must retain enough total request capacity for that workload. + +Nginx returns `{"error":"...","code":"request_too_large"}` with HTTP 413. +Clients can display the `error` message without parsing an Nginx HTML page. API +413 responses continue to describe their narrower per-file limit. + +## Repeatable proxy regression + +Run from the repository root with Docker, Python 3 and OpenSSL available: + +```sh +production/tests/validate_test.sh +python3 -B production/tests/nginx_upload_test.py +``` + +The test uses the **production Nginx template**, a disposable TLS certificate, +an isolated Docker network and a port bound only to localhost. It uses `32m` to +exercise the smallest supported configuration without transmitting a gigabyte. +It verifies Nginx syntax, forwards empty/small/29,999,999/30,000,000-byte file +requests including multipart overhead, accepts the exact 32 MiB request boundary, +and rejects one byte more with JSON 413 and the existing security headers. +Containers and the temporary network are removed at completion. + +The upstream fixture only counts bytes. Its acceptance of empty and +30,000,000-byte files demonstrates that the proxy lets Rails make the decision; +it does **not** claim the API accepts them. Run the API upload tests and the +authenticated probe below for application policy. CI runs the configuration and +real Nginx checks on changes to production files. + +## Direct API and proxied application regression + +Use a disposable project/task and test user against the generic CSV attachment +API. Successful test comments are removed immediately. Set credentials in the +environment, then provide the same task's two endpoint URLs: + +```sh +export UPLOAD_TEST_USERNAME=test-student +# Set UPLOAD_TEST_AUTH_TOKEN privately in the shell; do not save it in a report. +python3 -B production/tests/probe_upload_limits.py \ + --direct-url http://127.0.0.1:3000/api/projects/PROJECT/task_def_id/TASK/comments \ + --proxy-url https://ontrack.test.example/api/projects/PROJECT/task_def_id/TASK/comments +``` + +The probe prints sizes and status codes only. It checks empty files (400), small +and 29,999,999-byte files (201), and 30,000,000-byte files (413) both directly and +through the proxy. For an isolated `32m` proxy, add +`--proxy-limit-bytes 33554432` to also check the proxy-specific JSON 413. +TLS verification stays enabled and redirects are rejected to keep credentials +on the chosen endpoint. Record the API/web/deploy commits alongside the output. + +The production request-size default is unchanged. This work does not deploy a +stack or provide production measurements; those require the actual deployment +configuration and its authorized test account. + +## Verified local integration + +The authenticated probe passed on 21 September 2026 against API commit +`6ca598ce` (the safe spreadsheet/chat attachment branch) and deploy commit +`8038777`. It used Rails 8.0.5.1 / Ruby 3.4.10 in test mode, a disposable MariaDB +database and project, and Nginx 1.30.4 with the production template and `32m` +request limit. HTTPS certificate verification remained enabled. No web client +was involved in these transport checks. + +| File/request boundary | Direct API | Through Nginx | +| --- | --- | --- | +| Empty CSV | 400 | 400 | +| 1,024-byte CSV | 201 | 201 | +| 29,999,999-byte CSV | 201 | 201 | +| 30,000,000-byte CSV | 413 | 413 | +| 33,554,433-byte total request | Not an API file-policy case | 413, `request_too_large` | + +The permitted near-limit CSV occupied 30,000,360 bytes as multipart data and +reached Rails through Nginx. Each successful test comment was deleted. The +[raw status/size results](tests/results/upload-boundaries-20260921.jsonl) contain +no credentials, filenames, account identifiers or private endpoint URLs. diff --git a/production/compose.sh b/production/compose.sh new file mode 100755 index 000000000..4e45ab1a1 --- /dev/null +++ b/production/compose.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${SCRIPT_DIR}/.env.production" + +if [[ "${1:-}" == --env-file ]]; then + (( $# >= 2 )) || { + printf 'Usage: %s [--env-file PATH] COMPOSE_ARGUMENTS...\n' "$0" >&2 + exit 2 + } + ENV_FILE="$2" + shift 2 +fi + +(( $# > 0 )) || { + printf 'Usage: %s [--env-file PATH] COMPOSE_ARGUMENTS...\n' "$0" >&2 + exit 2 +} + +[[ -f "${ENV_FILE}" ]] || { + printf 'Production environment file not found: %s\n' "${ENV_FILE}" >&2 + exit 1 +} + +ENV_FILE="$(cd -- "$(dirname -- "${ENV_FILE}")" && pwd)/$(basename -- "${ENV_FILE}")" +docker_cli_config="${DOCKER_CONFIG:-${HOME}/.docker}" + +exec env -i \ + PATH="${PATH}" \ + HOME="${HOME}" \ + DOCKER_CONFIG="${docker_cli_config}" \ + DOCKER_HOST=unix:///var/run/docker.sock \ + docker compose \ + --env-file "${ENV_FILE}" \ + --file "${SCRIPT_DIR}/docker-compose.yml" \ + "$@" diff --git a/production/deploy.sh b/production/deploy.sh new file mode 100755 index 000000000..e359d7a38 --- /dev/null +++ b/production/deploy.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + printf 'Usage: %s [ENV_FILE] [WAIT_TIMEOUT_SECONDS]\n' "$0" >&2 +} + +(( $# <= 2 )) || { + usage + exit 2 +} + +ENV_FILE="${1:-${SCRIPT_DIR}/.env.production}" +WAIT_TIMEOUT="${2:-300}" + +[[ "${WAIT_TIMEOUT}" =~ ^[1-9][0-9]*$ ]] && + (( ${#WAIT_TIMEOUT} <= 5 )) && + (( 10#${WAIT_TIMEOUT} <= 86400 )) || { + printf 'Deployment wait timeout must be between 1 and 86400 seconds.\n' >&2 + exit 2 +} + +command -v timeout >/dev/null 2>&1 || { + printf 'GNU timeout is required for the bounded maintenance stop.\n' >&2 + exit 1 +} + +DOUBTFIRE_CONFIG_ONLY=0 "${SCRIPT_DIR}/validate.sh" "${ENV_FILE}" + +ENV_FILE="$(cd -- "$(dirname -- "${ENV_FILE}")" && pwd)/$(basename -- "${ENV_FILE}")" + +compose=("${SCRIPT_DIR}/compose.sh" --env-file "${ENV_FILE}") + +"${compose[@]}" pull + +# The hardened web accepts both legacy query callbacks and fragment callbacks, +# while the hardened API emits fragments that an older web build cannot read. +# Stage and health-check web first so a forward rollout cannot expose the +# incompatible API-new/web-old combination through the existing proxy. +if "${compose[@]}" up \ + --detach \ + --no-deps \ + --wait \ + --wait-timeout "${WAIT_TIMEOUT}" \ + webserver; then + printf 'Hardened web image is healthy; continuing with API and workers.\n' +else + web_status=$? + printf 'Web compatibility stage failed; API and workers were not updated and public ingress will be stopped.\n' >&2 + "${compose[@]}" ps --all webserver >&2 || true + "${compose[@]}" logs --no-color --tail 100 webserver >&2 || true + timeout --kill-after=10s 120s \ + "${compose[@]}" stop --timeout 60 proxy >&2 || true + exit "${web_status}" +fi + +# Block ingress and quiesce every application writer before the one-shot +# migration. The compatible web stays healthy but is not publicly reachable +# while API/Sidekiq/PDF writers are stopped. Database/Redis remain available. +maintenance_services=(proxy apiserver sidekiq pdfgen migrate) +printf 'Entering bounded migration maintenance window.\n' +if timeout --kill-after=10s 120s \ + "${compose[@]}" stop --timeout 60 "${maintenance_services[@]}"; then + printf 'Public ingress and application writers are stopped; applying release.\n' +else + maintenance_status=$? + printf 'Could not quiesce the running release; migration was not started.\n' >&2 + "${compose[@]}" ps --all "${maintenance_services[@]}" >&2 || true + timeout --kill-after=10s 120s \ + "${compose[@]}" stop --timeout 60 "${maintenance_services[@]}" >&2 || true + exit "${maintenance_status}" +fi + +if "${compose[@]}" up \ + --detach \ + --wait \ + --wait-timeout "${WAIT_TIMEOUT}" \ + --remove-orphans; then + : +else + rollout_status=$? + printf 'Deployment did not become healthy; ingress and writers will remain stopped.\n' >&2 + "${compose[@]}" ps --all >&2 || true + "${compose[@]}" logs \ + --no-color \ + --tail 100 \ + migrate apiserver webserver proxy sidekiq pdfgen >&2 || true + timeout --kill-after=10s 120s \ + "${compose[@]}" stop --timeout 60 "${maintenance_services[@]}" >&2 || true + exit "${rollout_status}" +fi +"${compose[@]}" ps diff --git a/production/docker-compose.yml b/production/docker-compose.yml index 72c10f483..293121869 100644 --- a/production/docker-compose.yml +++ b/production/docker-compose.yml @@ -1,137 +1,408 @@ -version: '3.8' +name: ${COMPOSE_PROJECT_NAME:?Set COMPOSE_PROJECT_NAME in .env.production} + +x-logging: &default-logging + driver: local + options: + max-size: 20m + max-file: "5" + +x-api-environment: &api-environment + RAILS_ENV: production + TZ: ${TZ:?Set TZ in .env.production} + DF_LOG_TO_STDOUT: "true" + DF_PRODUCTION_DB_ADAPTER: mysql2 + DF_PRODUCTION_DB_HOST: doubtfire-db + DF_PRODUCTION_DB_DATABASE: ${MARIADB_DATABASE:?Set MARIADB_DATABASE in .env.production} + DF_PRODUCTION_DB_USERNAME: ${MARIADB_USER:?Set MARIADB_USER in .env.production} + DF_PRODUCTION_DB_PASSWORD: ${MARIADB_PASSWORD:?Set MARIADB_PASSWORD in .env.production} + DF_SECRET_KEY_BASE: ${DF_SECRET_KEY_BASE:?Set DF_SECRET_KEY_BASE in .env.production} + DF_SECRET_KEY_ATTR: ${DF_SECRET_KEY_ATTR:?Set DF_SECRET_KEY_ATTR in .env.production} + DF_SECRET_KEY_DEVISE: ${DF_SECRET_KEY_DEVISE:?Set DF_SECRET_KEY_DEVISE in .env.production} + DF_SECRET_KEY_MOSS: ${DF_SECRET_KEY_MOSS:-} + DF_ENCRYPTION_PRIMARY_KEY: ${DF_ENCRYPTION_PRIMARY_KEY:?Set DF_ENCRYPTION_PRIMARY_KEY in .env.production} + DF_ENCRYPTION_DETERMINISTIC_KEY: ${DF_ENCRYPTION_DETERMINISTIC_KEY:?Set DF_ENCRYPTION_DETERMINISTIC_KEY in .env.production} + DF_ENCRYPTION_KEY_DERIVATION_SALT: ${DF_ENCRYPTION_KEY_DERIVATION_SALT:?Set DF_ENCRYPTION_KEY_DERIVATION_SALT in .env.production} + DF_INSTITUTION_HOST: ${DF_INSTITUTION_HOST:?Set DF_INSTITUTION_HOST in .env.production} + DF_INSTITUTION_PRODUCT_NAME: ${DF_INSTITUTION_PRODUCT_NAME:?Set DF_INSTITUTION_PRODUCT_NAME in .env.production} + DF_INSTITUTION_SETTINGS_RB: ${DF_INSTITUTION_SETTINGS_RB:-no_institution_setting.rb} + DF_INSTITUTION_EMAIL_DOMAIN: ${DF_INSTITUTION_EMAIL_DOMAIN:?Set DF_INSTITUTION_EMAIL_DOMAIN in .env.production} + DF_INSTITUTION_EMAIL_SENDER: ${DF_INSTITUTION_EMAIL_SENDER:?Set DF_INSTITUTION_EMAIL_SENDER in .env.production} + DF_INSTITUTION_NAME: ${DF_INSTITUTION_NAME:?Set DF_INSTITUTION_NAME in .env.production} + DF_AUTH_METHOD: ${DF_AUTH_METHOD:?Set DF_AUTH_METHOD in .env.production} + DF_AAF_ISSUER_URL: ${DF_AAF_ISSUER_URL:-} + DF_AAF_AUDIENCE_URL: ${DF_AAF_AUDIENCE_URL:-} + DF_AAF_CALLBACK_URL: ${DF_AAF_CALLBACK_URL:-} + DF_AAF_IDENTITY_PROVIDER_URL: ${DF_AAF_IDENTITY_PROVIDER_URL:-} + DF_AAF_UNIQUE_URL: ${DF_AAF_UNIQUE_URL:-} + DF_AAF_AUTH_SIGNOUT_URL: ${DF_AAF_AUTH_SIGNOUT_URL:-} + DF_SECRET_KEY_AAF: ${DF_SECRET_KEY_AAF:-} + DF_SAML_METADATA_URL: ${DF_SAML_METADATA_URL:-} + DF_SAML_CONSUMER_SERVICE_URL: ${DF_SAML_CONSUMER_SERVICE_URL:-} + DF_SAML_SP_ENTITY_ID: ${DF_SAML_SP_ENTITY_ID:-} + DF_SAML_IDP_TARGET_URL: ${DF_SAML_IDP_TARGET_URL:-} + DF_SAML_IDP_SAML_NAME_IDENTIFIER_FORMAT: ${DF_SAML_IDP_SAML_NAME_IDENTIFIER_FORMAT:-} + DF_LDAP_HOST: ${DF_LDAP_HOST:-} + DF_LDAP_PORT: ${DF_LDAP_PORT:-} + DF_LDAP_ATTRIBUTE: ${DF_LDAP_ATTRIBUTE:-} + DF_LDAP_BASE: ${DF_LDAP_BASE:-} + DF_LDAP_SSL: ${DF_LDAP_SSL:-false} + DF_LDAP_USE_ADMIN_TO_BIND: ${DF_LDAP_USE_ADMIN_TO_BIND:-false} + DF_LDAP_ADMIN_USER: ${DF_LDAP_ADMIN_USER:-} + DF_LDAP_ADMIN_PWD: ${DF_LDAP_ADMIN_PWD:-} + DF_MAIL_PERFORM_DELIVERIES: ${DF_MAIL_PERFORM_DELIVERIES:?Set DF_MAIL_PERFORM_DELIVERIES in .env.production} + DF_MAIL_DELIVERY_METHOD: ${DF_MAIL_DELIVERY_METHOD:?Set DF_MAIL_DELIVERY_METHOD in .env.production} + DF_SMTP_ADDRESS: ${DF_SMTP_ADDRESS:?Set DF_SMTP_ADDRESS in .env.production} + DF_SMTP_PORT: ${DF_SMTP_PORT:?Set DF_SMTP_PORT in .env.production} + DF_SMTP_DOMAIN: ${DF_SMTP_DOMAIN:?Set DF_SMTP_DOMAIN in .env.production} + DF_SMTP_USERNAME: ${DF_SMTP_USERNAME:-} + DF_SMTP_PASSWORD: ${DF_SMTP_PASSWORD:-} + DF_SMTP_AUTHENTICATION: ${DF_SMTP_AUTHENTICATION:-} + OVERSEER_ENABLED: "0" + TUTORIAL_ENABLED: ${TUTORIAL_ENABLED:-0} + DF_REDIS_SIDEKIQ_URL: redis://redis-sidekiq:6379/0 + DF_STUDENT_WORK_DIR: /student-work + +x-live-feature-environment: &live-feature-environment + # Mapping metadata scopes imported announcements on both API and worker. + # Only the worker receives the Microsoft application credentials below. + DF_TEAMS_ANNOUNCEMENTS_ENABLED: ${DF_TEAMS_ANNOUNCEMENTS_ENABLED:-false} + DF_TEAMS_TENANT_ID: ${DF_TEAMS_TENANT_ID:-} + DF_TEAMS_CHANNEL_MAPPINGS: ${DF_TEAMS_CHANNEL_MAPPINGS:-[]} + DF_PPI_MINIMUM_COHORT_SIZE: ${DF_PPI_MINIMUM_COHORT_SIZE:?Set DF_PPI_MINIMUM_COHORT_SIZE in .env.production} + DF_PPI_STALE_AFTER_HOURS: ${DF_PPI_STALE_AFTER_HOURS:?Set DF_PPI_STALE_AFTER_HOURS in .env.production} + DOUBTFIRE_VAPID_PUBLIC_KEY: ${DOUBTFIRE_VAPID_PUBLIC_KEY:?Set DOUBTFIRE_VAPID_PUBLIC_KEY in .env.production} + DOUBTFIRE_VAPID_PRIVATE_KEY: ${DOUBTFIRE_VAPID_PRIVATE_KEY:?Set DOUBTFIRE_VAPID_PRIVATE_KEY in .env.production} + DOUBTFIRE_VAPID_SUBJECT: ${DOUBTFIRE_VAPID_SUBJECT:?Set DOUBTFIRE_VAPID_SUBJECT in .env.production} networks: - backnet: - frontnet: + edge: + data: + internal: true + egress: + docker-api: + internal: true services: - - # This is the proxy which receives all http requests and forwards to relevant servers. proxy: - image: nginx:mainline-alpine - container_name: proxy + image: ${PROXY_IMAGE:?Set PROXY_IMAGE to an immutable image digest} + environment: + SERVER_NAME: ${SERVER_NAME:?Set SERVER_NAME in .env.production} + CLIENT_MAX_BODY_SIZE: ${CLIENT_MAX_BODY_SIZE:-1g} + NGINX_ENVSUBST_FILTER: ^(SERVER_NAME|CLIENT_MAX_BODY_SIZE)$$ ports: - - 80:80 - - 443:443 + - "80:80" + - "443:443" depends_on: - - webserver - - apiserver + apiserver: + condition: service_healthy + webserver: + condition: service_healthy networks: - - frontnet - - backnet + - edge volumes: - - ./shared-files/proxy-nginx.conf:/etc/nginx/nginx.conf - - ./localhost.crt:/etc/nginx/cert.crt # replace with real certificate - - ./localhost.key:/etc/nginx/key.key # replace with real key - restart: on-failure:5 + - ./proxy-nginx.conf.template:/etc/nginx/templates/default.conf.template:ro + - ${TLS_CERTIFICATE_PATH:?Set TLS_CERTIFICATE_PATH}:/etc/nginx/tls/fullchain.pem:ro + - ${TLS_PRIVATE_KEY_PATH:?Set TLS_PRIVATE_KEY_PATH}:/etc/nginx/tls/private.key:ro + healthcheck: + test: + - CMD-SHELL + - >- + curl --fail --silent --show-error --insecure + --header 'Host: ${SERVER_NAME}' + https://127.0.0.1/healthz + && curl --fail --silent --show-error --insecure + --header 'Host: ${SERVER_NAME}' + https://127.0.0.1/healthz/web + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + restart: unless-stopped + logging: *default-logging - # Web server hosts the static files webserver: - platform: linux/amd64 - image: lmsdoubtfire/doubtfire-web:10.0.0-3 - # ports: - # - 8080:8080 + image: ${DOUBTFIRE_WEB_IMAGE:?Set DOUBTFIRE_WEB_IMAGE to an immutable image digest} + expose: + - "80" networks: - - backnet - restart: on-failure:5 + - edge + healthcheck: + test: ["CMD-SHELL", "wget --quiet --spider http://127.0.0.1/index.html"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + restart: unless-stopped + logging: *default-logging - # API server runs the rails project - apiserver: - platform: linux/amd64 - image: lmsdoubtfire/apiserver:10.0.0-4 - env_file: - - .env.production - # ports: - # - 3000:3000 - networks: - - backnet + migrate: + image: ${DOUBTFIRE_API_IMAGE:?Set DOUBTFIRE_API_IMAGE to an immutable image digest} + environment: *api-environment depends_on: - - doubtfire-db - volumes: - - student_work:/student-work # Adjust path to where student data should be stored - - doubtfire_logs:/doubtfire/log - - ./shared-files:/shared-files - - ./shared-files/aliases:/etc/aliases:ro - command: /bin/bash -c "bundle exec rails s -b 0.0.0.0" - restart: on-failure:5 - - # Database server - could be external to docker - doubtfire-db: - image: mariadb:10 - restart: always + doubtfire-db: + condition: service_healthy networks: - - backnet + data: + egress: + gw_priority: 1 + command: ["bundle", "exec", "rails", "db:migrate"] + restart: "no" + logging: *default-logging + + apiserver: + image: ${DOUBTFIRE_API_IMAGE:?Set DOUBTFIRE_API_IMAGE to an immutable image digest} environment: - MARIADB_RANDOM_ROOT_PASSWORD: true - MARIADB_USER: dfire # Update with mariadb username - MARIADB_PASSWORD: pwd # Update with mariadb password - MARIADB_DATABASE: doubtfire # Update with mariadb database name + <<: [*api-environment, *live-feature-environment] + expose: + - "3000" + depends_on: + migrate: + condition: service_completed_successfully + redis-sidekiq: + condition: service_healthy + networks: + edge: + gw_priority: 1 + data: volumes: - - mysql_db:/var/lib/mysql + - ${STUDENT_WORK_PATH:?Set STUDENT_WORK_PATH}:/student-work + - ${DOUBTFIRE_LOG_PATH:?Set DOUBTFIRE_LOG_PATH}:/doubtfire/log + command: ["bundle", "exec", "rails", "server", "-b", "0.0.0.0"] + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "--show-error", "http://127.0.0.1:3000/readiness"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + init: true + stop_grace_period: 30s + restart: unless-stopped + logging: *default-logging - # Automated generation of PDFs using cron - pdfgen: - platform: linux/amd64 - image: lmsdoubtfire/appserver:10.0.0-4 - env_file: - - .env.production + doubtfire-db: + image: ${MARIADB_IMAGE:?Set MARIADB_IMAGE to an immutable image digest} + environment: + MARIADB_DATABASE: ${MARIADB_DATABASE:?Set MARIADB_DATABASE in .env.production} + MARIADB_USER: ${MARIADB_USER:?Set MARIADB_USER in .env.production} + MARIADB_PASSWORD: ${MARIADB_PASSWORD:?Set MARIADB_PASSWORD in .env.production} + MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD:?Set MARIADB_ROOT_PASSWORD in .env.production} + TZ: ${TZ:?Set TZ in .env.production} + expose: + - "3306" networks: - - backnet - depends_on: - - doubtfire-db + - data volumes: - - student_work:/student-work # Update with path to student work - - ./shared-files:/shared-files - - ./shared-files/aliases:/etc/aliases:ro - - doubtfire_logs:/doubtfire/log - - latex_tmp:/doubtfire/tmp/rails-latex - command: /doubtfire/lib/shell/pdfgen_entry_point.sh - restart: on-failure:5 + - ${MARIADB_DATA_PATH:?Set MARIADB_DATA_PATH}:/var/lib/mysql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + stop_grace_period: 60s + restart: unless-stopped + logging: *default-logging redis-sidekiq: - container_name: redis-sidekiq - image: redis:7.0 + image: ${REDIS_IMAGE:?Set REDIS_IMAGE to an immutable image digest} + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + expose: + - "6379" + networks: + - data volumes: - - redis_data:/data + - ${REDIS_DATA_PATH:?Set REDIS_DATA_PATH}:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 10s + stop_grace_period: 30s + restart: unless-stopped + logging: *default-logging + + pdfgen: + image: ${DOUBTFIRE_APP_IMAGE:?Set DOUBTFIRE_APP_IMAGE to an immutable image digest} + environment: + <<: *api-environment + DOCKER_HOST: tcp://docker-socket-proxy:2375 + LATEX_BUILD_PATH: /texlive/shell/latex_build.sh + LATEX_CONTAINER_NAME: ${COMPOSE_PROJECT_NAME}-texlive-pdfgen + depends_on: + migrate: + condition: service_completed_successfully + apiserver: + condition: service_healthy + docker-socket-proxy: + condition: service_healthy + jplag: + condition: service_healthy + texlive-pdfgen: + condition: service_healthy networks: - - backnet + data: + egress: + gw_priority: 1 + docker-api: + volumes: + - ${STUDENT_WORK_PATH:?Set STUDENT_WORK_PATH}:/student-work + - ${DOUBTFIRE_LOG_PATH:?Set DOUBTFIRE_LOG_PATH}:/doubtfire/log + - ${ALIASES_PATH:?Set ALIASES_PATH}:/etc/aliases:ro + - ${MSMTPRC_PATH:?Set MSMTPRC_PATH}:/shared-files/msmtprc:ro + - latex_pdfgen:/doubtfire/tmp/rails-latex + - jplag_tmp:/doubtfire/tmp/jplag + command: ["/doubtfire/lib/shell/pdfgen_entry_point.sh"] + init: true + stop_grace_period: 30s + restart: unless-stopped + logging: *default-logging sidekiq: - platform: linux/amd64 - image: lmsdoubtfire/appserver:10.0.0-4 - env_file: - - .env.production + image: ${DOUBTFIRE_APP_IMAGE:?Set DOUBTFIRE_APP_IMAGE to an immutable image digest} environment: - DOCKER_HOST: host.docker.internal - networks: - - backnet + <<: [*api-environment, *live-feature-environment] + DF_TEAMS_CLIENT_ID: ${DF_TEAMS_CLIENT_ID:-} + DF_TEAMS_CLIENT_SECRET: ${DF_TEAMS_CLIENT_SECRET:-} + DF_SIDEKIQ_CONCURRENCY: ${DF_SIDEKIQ_CONCURRENCY:?Set DF_SIDEKIQ_CONCURRENCY in .env.production} + DOCKER_HOST: tcp://docker-socket-proxy:2375 + LATEX_BUILD_PATH: /texlive/shell/latex_build.sh + LATEX_CONTAINER_NAME: ${COMPOSE_PROJECT_NAME}-texlive-sidekiq depends_on: - - doubtfire-db - - redis-sidekiq + migrate: + condition: service_completed_successfully + apiserver: + condition: service_healthy + redis-sidekiq: + condition: service_healthy + docker-socket-proxy: + condition: service_healthy + texlive-sidekiq: + condition: service_healthy + networks: + data: + egress: + gw_priority: 1 + docker-api: volumes: - - student_work:/student-work # Update with path to student work - - ./shared-files:/shared-files - - ./shared-files/aliases:/etc/aliases:ro - - doubtfire_logs:/doubtfire/log - - latex_tmp:/doubtfire/tmp/rails-latex - command: /doubtfire/lib/shell/sidekiq_entry_point.sh - restart: on-failure:5 - - texlive: - container_name: texlive-container - image: lmsdoubtfire/formatif-latex:10.0.0-4 - # build: - # context: ../ - # dockerfile: texlive.Dockerfile + - ${STUDENT_WORK_PATH:?Set STUDENT_WORK_PATH}:/student-work + - ${DOUBTFIRE_LOG_PATH:?Set DOUBTFIRE_LOG_PATH}:/doubtfire/log + - ${ALIASES_PATH:?Set ALIASES_PATH}:/etc/aliases:ro + - ${MSMTPRC_PATH:?Set MSMTPRC_PATH}:/shared-files/msmtprc:ro + - latex_sidekiq:/doubtfire/tmp/rails-latex + command: ["/doubtfire/lib/shell/sidekiq_entry_point.sh"] + init: true + stop_grace_period: 30s + restart: unless-stopped + logging: *default-logging + + texlive-sidekiq: + image: ${TEXLIVE_IMAGE:?Set TEXLIVE_IMAGE to an immutable image digest} + container_name: ${COMPOSE_PROJECT_NAME}-texlive-sidekiq + network_mode: none volumes: - - student_work:/student-work - - latex_tmp:/workdir/texlive-latex - command: /bin/sh -c "sleep infinity" + - latex_sidekiq:/workdir/texlive-latex + command: ["sh", "-c", "sleep infinity"] + mem_limit: ${TEXLIVE_MEMORY_LIMIT:?Set TEXLIVE_MEMORY_LIMIT in .env.production} + cpus: ${TEXLIVE_CPU_LIMIT:?Set TEXLIVE_CPU_LIMIT in .env.production} + pids_limit: ${TEXLIVE_PIDS_LIMIT:?Set TEXLIVE_PIDS_LIMIT in .env.production} + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "test", "-x", "/texlive/shell/latex_build.sh"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + restart: unless-stopped + logging: *default-logging -volumes: - latex_tmp: {} - redis_data: {} - doubtfire_logs: {} - mysql_db: {} - student_work: {} + texlive-pdfgen: + image: ${TEXLIVE_IMAGE:?Set TEXLIVE_IMAGE to an immutable image digest} + container_name: ${COMPOSE_PROJECT_NAME}-texlive-pdfgen + network_mode: none + volumes: + - latex_pdfgen:/workdir/texlive-latex + command: ["sh", "-c", "sleep infinity"] + mem_limit: ${TEXLIVE_MEMORY_LIMIT:?Set TEXLIVE_MEMORY_LIMIT in .env.production} + cpus: ${TEXLIVE_CPU_LIMIT:?Set TEXLIVE_CPU_LIMIT in .env.production} + pids_limit: ${TEXLIVE_PIDS_LIMIT:?Set TEXLIVE_PIDS_LIMIT in .env.production} + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "test", "-x", "/texlive/shell/latex_build.sh"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + restart: unless-stopped + logging: *default-logging + + jplag: + image: ${JPLAG_IMAGE:?Set JPLAG_IMAGE to an immutable image digest} + container_name: jplag + network_mode: none + volumes: + - ${JPLAG_REPORT_PATH:?Set JPLAG_REPORT_PATH}:/student-work/jplag + - ${JPLAG_ARCHIVE_REPORT_PATH:?Set JPLAG_ARCHIVE_REPORT_PATH}:/student-work/archive/jplag + - jplag_tmp:/tmp/jplag + command: ["sh", "-c", "sleep infinity"] + mem_limit: ${JPLAG_MEMORY_LIMIT:?Set JPLAG_MEMORY_LIMIT in .env.production} + cpus: ${JPLAG_CPU_LIMIT:?Set JPLAG_CPU_LIMIT in .env.production} + pids_limit: ${JPLAG_PIDS_LIMIT:?Set JPLAG_PIDS_LIMIT in .env.production} + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "test", "-s", "/jplag/jplag-jar-with-dependencies.jar"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + restart: unless-stopped + logging: *default-logging + + docker-socket-proxy: + image: ${DOCKER_SOCKET_PROXY_IMAGE:?Set DOCKER_SOCKET_PROXY_IMAGE to an immutable image digest} + environment: + COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:?Set COMPOSE_PROJECT_NAME in .env.production} + command: + - /bin/sh + - -ec + - >- + sed "s/__PROJECT_NAME__/$${COMPOSE_PROJECT_NAME}/g" + /usr/local/etc/haproxy/docker-api.cfg.template + > /tmp/docker-api.cfg + && exec haproxy -W -db -f /tmp/docker-api.cfg + networks: + - docker-api + volumes: + - ${DOCKER_SOCKET_PATH:?Set DOCKER_SOCKET_PATH}:/var/run/docker.sock:ro + - ./docker-socket-proxy.cfg:/usr/local/etc/haproxy/docker-api.cfg.template:ro + healthcheck: + test: ["CMD", "wget", "--quiet", "--spider", "http://127.0.0.1:2375/_ping"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 10s + read_only: true + tmpfs: + - /run + - /tmp + user: "0:0" + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + restart: unless-stopped + logging: *default-logging +volumes: + latex_pdfgen: + latex_sidekiq: + jplag_tmp: diff --git a/production/docker-socket-proxy.cfg b/production/docker-socket-proxy.cfg new file mode 100644 index 000000000..c2f4ede7d --- /dev/null +++ b/production/docker-socket-proxy.cfg @@ -0,0 +1,40 @@ +global + log stdout format raw local0 + maxconn 64 + +defaults + log global + mode http + option httplog + timeout connect 5s + timeout client 1h + timeout server 1h + timeout tunnel 1h + +frontend docker_api + bind :2375 + + acl safe_read method GET HEAD + acl safe_write method POST + acl ping_path path_reg ^(/v[0-9.]+)?/_ping$ + acl version_path path_reg ^(/v[0-9.]+)?/version$ + acl helper_inspect_path path_reg ^(/v[0-9.]+)?/containers/(jplag|__PROJECT_NAME__-texlive-(pdfgen|sidekiq))/json$ + acl helper_exec_path path_reg ^(/v[0-9.]+)?/containers/(jplag|__PROJECT_NAME__-texlive-(pdfgen|sidekiq))/exec$ + acl exec_inspect_path path_reg ^(/v[0-9.]+)?/exec/[0-9a-f]{64}/json$ + acl exec_action_path path_reg ^(/v[0-9.]+)?/exec/[0-9a-f]{64}/(start|resize)$ + + http-request allow if safe_read ping_path + http-request allow if safe_read version_path + http-request allow if safe_read helper_inspect_path + http-request allow if safe_read exec_inspect_path + http-request allow if safe_write helper_exec_path + http-request allow if safe_write exec_action_path + http-request deny deny_status 403 + + default_backend docker_engine + +backend docker_engine + option httpchk + http-check connect + http-check send meth GET uri /_ping + server docker /var/run/docker.sock check inter 5s diff --git a/production/localhost.crt b/production/localhost.crt deleted file mode 100644 index c0c7581a3..000000000 --- a/production/localhost.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDizCCAnOgAwIBAgIJAMXK825Zi9sqMA0GCSqGSIb3DQEBCwUAMCcxCzAJBgNV -BAYTAlVTMRgwFgYDVQQDDA9FeGFtcGxlLVJvb3QtQ0EwHhcNMjIwMTI4MDU1MTI5 -WhcNMjQxMTE3MDU1MTI5WjBoMQswCQYDVQQGEwJBVTEMMAoGA1UECAwDVklDMRIw -EAYDVQQHDAlNZWxib3VybmUxHTAbBgNVBAoMFEV4YW1wbGUtQ2VydGlmaWNhdGVz -MRgwFgYDVQQDDA9sb2NhbGhvc3QubG9jYWwwggEiMA0GCSqGSIb3DQEBAQUAA4IB -DwAwggEKAoIBAQDtbrwb92XH9SbDIVPxims3uSN78iZohuimgExLFsUbBJdHKR4a -OAx7j0crmlJVpv+2rK7dHGCRS0g+LF8L3fOEVTjjgIdG6adCgkuYiW9/nSmVDa82 -g0PGeeNMOMCxOM2KgsI2E9G6fDp547jXoYwRwjH6shf9JOJ65DLsJ2cvsqXztled -1k+U/WjYFuJl5HkQNGbkz0Zjm51JXUqkeHzTWDoICafPDnLdElQXvcfA71QsjWUk -xkgDXwXsrn8pkpLWiE+sunFXtqV7XzYxiZ9osvZkBextQaykyjI/8ubp84x3dq2H -e4ySkThIUGdSpYN0iQ86tNF6/YvpCJ1TK6GBAgMBAAGjeTB3MEEGA1UdIwQ6MDih -K6QpMCcxCzAJBgNVBAYTAlVTMRgwFgYDVQQDDA9FeGFtcGxlLVJvb3QtQ0GCCQC6 -Ht/UfcerKDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE8DAaBgNVHREEEzARgglsb2Nh -bGhvc3SHBH8AAAEwDQYJKoZIhvcNAQELBQADggEBAAHVVTFaOSHSYm1ylQwapmc4 -lzaLrzm3K4lpNp1RiA9fbY2zDhybdV1dMUR1/BAOx1oWewgN49Gkqw8rx1uvmWGl -ifRfj+yxR5ZSyDpWKiTcEpoRxVYPKDJuwWBL6YOE1cjxx8jo3/64VPVGwyyrCLUM -0WLh7zy3q25IDtkBZtocZmXsN66aqAmeKRM/luhcP0rb3fPaNedtZJn9tSsMgQjo -wf1A4LF69uqQqrBuN35YhqMSwwuNcQ6EqYNnqudE1G7qkqhfyYgu5EciI6DK2qO5 -0luPC/y3p/QVpomO1nseeGDwxEVuLZrcD0/N7bVic9Rr1p1MTkLrFNJBO8sFaAw= ------END CERTIFICATE----- diff --git a/production/localhost.key b/production/localhost.key deleted file mode 100644 index 1a53c3b0d..000000000 --- a/production/localhost.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDtbrwb92XH9SbD -IVPxims3uSN78iZohuimgExLFsUbBJdHKR4aOAx7j0crmlJVpv+2rK7dHGCRS0g+ -LF8L3fOEVTjjgIdG6adCgkuYiW9/nSmVDa82g0PGeeNMOMCxOM2KgsI2E9G6fDp5 -47jXoYwRwjH6shf9JOJ65DLsJ2cvsqXztled1k+U/WjYFuJl5HkQNGbkz0Zjm51J -XUqkeHzTWDoICafPDnLdElQXvcfA71QsjWUkxkgDXwXsrn8pkpLWiE+sunFXtqV7 -XzYxiZ9osvZkBextQaykyjI/8ubp84x3dq2He4ySkThIUGdSpYN0iQ86tNF6/Yvp -CJ1TK6GBAgMBAAECggEBAMdYYbdZT6tCuPi70q7JeJugZZp7BKReIJcqwOOKipLU -l+63Jv78b/6NnUOOnfIr9CZ7dZVPbgSPIGiS+HoOL5a7trfMAcgYBMZifvXi7ipG -2dK7WpB6RT0uuRgVzjL9bhUNE+IzYbMg+je8akUg7Wk7kprZTCrN/RkQ33BG5Ou1 -hFxafGr4ETanCCUHyBboV/AqK5B/M4xfi2q4Lac8vlC+PR77K2UzaI7M8aF/9+w0 -4xKSM34X7wgYrbQMGtqnEpucdZHfPUx2o3jN7sP8HqDiJtDdC43x2l/+4DvOtDWF -zxt41DJrTjUABImjxthTQ5DkrRJPIoRfMQn6kTSLfsECgYEA/S2PPy79GucDkwTu -aXxISJJlXkXWCJbKDcLchys+rVt3wQGU9rUmupOM2n1oln/LhdDBbDu/i89qeARH -z/vwAXb6F2A6Ws4ViKS2jZMU6WMpXzSPywuEG/5OhwDWuE6YrUqym578EczlR5c0 -u8u5Y0VOALRBrD0H/77r8YZRatsCgYEA8BQ+8zAAAcS3XZ9GCdqSOESg+ZvfGBRX -XjK9sPDNjGjIEPh/NstNREyMPz+7R9PgXdg8f2wjCSyMwfTvlcNAPrtrTUx6XCjD -pzm7dPLSDAOZL+w5VFJ+B1+4yIa5/svq92+5pl04+1Q7o9MWriaMtmK0em5ZStFq -1xVKBxFtXdMCgYEA0WejecElu+CjhjCn+TvAlbXwff8JPuVpjeREftB38O8cp9jl -GwFOXTe5YuYjYYemELJPqXPpxHv0NfxkYVSieLOtuzPuUDjweqqyC3tB6qAIEiTp -YinORVEcJnG9J4hqo7qK7EideSOd6I7aIRIp7j/eepqdTxbTzOZfRM+x+80CgYEA -5Jzy4NK5fvWbAszzfdij2m8HtwlF5QEhvTKmTcLQiT8SHI71XPE1b69Q8b23kgth -oRuTaGdJ6aeBlgSuzLb0C4ifAimsbuvme+xO5GuBty0KB4dzhCdVmaH+LP2WSdpW -si6y44fsBfBNJpCRDV7qKZdOczN3VfabPZspNhfTRD8CgYBdncN5W6l9naADSO42 -nyTnIV8VEakiLX/Q7P3NlLeMLjS79F0Mj+Fh3ar81/O7eYhipvlmxZBTvOnjaKxR -m7X3rmwnogRrbq+qkVbnYdKyJj3zAkjZZaaWfgAXQdY1PUnJTtDkvzUkeesEvt+L -Yka/0HSHBh9Hlsj5pgeLFI83BQ== ------END PRIVATE KEY----- diff --git a/production/proxy-nginx.conf.template b/production/proxy-nginx.conf.template new file mode 100644 index 000000000..ea891573d --- /dev/null +++ b/production/proxy-nginx.conf.template @@ -0,0 +1,121 @@ +server_tokens off; + +# The inherited SCORM route includes a reusable credential in its path. Keep +# routing in the common API location, but do not persist that path in proxy +# access logs or allow browsers to send it as a Referer. +map $uri $doubtfire_access_loggable { + default 1; + ~^/api/scorm(?:/|$) 0; +} + +map $uri $doubtfire_referrer_policy { + default strict-origin-when-cross-origin; + ~^/api/scorm(?:/|$) no-referrer; +} + +server { + listen 80 default_server; + server_name _; + access_log /var/log/nginx/access.log main if=$doubtfire_access_loggable; + return 444; +} + +server { + listen 80; + server_name ${SERVER_NAME}; + access_log /var/log/nginx/access.log main if=$doubtfire_access_loggable; + add_header Referrer-Policy $doubtfire_referrer_policy always; + return 301 https://${SERVER_NAME}$request_uri; +} + +server { + listen 443 ssl default_server; + server_name _; + access_log /var/log/nginx/access.log main if=$doubtfire_access_loggable; + + ssl_certificate /etc/nginx/tls/fullchain.pem; + ssl_certificate_key /etc/nginx/tls/private.key; + + return 444; +} + +server { + listen 443 ssl; + http2 on; + server_name ${SERVER_NAME}; + access_log /var/log/nginx/access.log main if=$doubtfire_access_loggable; + + ssl_certificate /etc/nginx/tls/fullchain.pem; + ssl_certificate_key /etc/nginx/tls/private.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_session_tickets off; + + client_max_body_size ${CLIENT_MAX_BODY_SIZE}; + + # Rails cannot format a request rejected before it reaches the API. Keep this + # response compatible with the API error envelope consumed by upload clients. + error_page 413 = @upload_too_large; + + location @upload_too_large { + default_type application/json; + return 413 '{"error":"The upload request exceeds the server limit. Try a smaller file or contact your unit staff.","code":"request_too_large"}'; + } + + # Compress proxied API responses. Only application/json is listed: the static + # bundle (text/css, application/javascript) is already gzipped inside the web + # container by doubtfire-web/nginx.conf, and the notification list is the + # largest per-user JSON the app returns, so it gains the most here. + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_min_length 1024; + gzip_types application/json; + + add_header Strict-Transport-Security "max-age=31536000" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy $doubtfire_referrer_policy always; + + # Resolve service names through Docker DNS for every container lifecycle, + # rather than retaining an address from Nginx startup. + resolver 127.0.0.11 valid=10s ipv6=off; + set $api_upstream apiserver:3000; + set $web_upstream webserver:80; + + location = /healthz { + access_log off; + proxy_set_header Host $host; + proxy_pass http://$api_upstream/readiness; + } + + location = /healthz/web { + access_log off; + proxy_set_header Host $host; + proxy_pass http://$web_upstream/index.html; + } + + location ~ ^/lti/api(?:/|$) { + return 404; + } + + location ~ ^/api(?:/|$) { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_pass http://$api_upstream; + proxy_read_timeout 300s; + } + + location / { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_pass http://$web_upstream; + proxy_read_timeout 90s; + } +} diff --git a/production/publish-release.sh b/production/publish-release.sh new file mode 100755 index 000000000..9e4409da8 --- /dev/null +++ b/production/publish-release.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +umask 077 + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPOSITORY_DIR="$(cd -- "${SCRIPT_DIR}/.." && pwd)" + +usage() { + cat >&2 <<'USAGE' +Usage: + PUBLISH_RELEASE_CONFIRM=1 ./publish-release.sh [--no-cache-web] REGISTRY_NAMESPACE RELEASE_VERSION [PLATFORMS] + +Example: + PUBLISH_RELEASE_CONFIRM=1 ./publish-release.sh \ + registry.example.edu/ontrack "$DF_RELEASE_VERSION" linux/amd64,linux/arm64 + +The command publishes five multi-platform images and prints immutable digest +references. Authenticate Docker to the approved registry before running it. + +For a PWA recovery release from earlier web source, use --no-cache-web and a new +release version. This forces the web image's Angular build to run again, producing +a fresh service-worker manifest. Other images retain normal build caching. +USAGE +} + +fail() { + printf 'Release publication refused: %s\n' "$*" >&2 + exit 1 +} + +NO_CACHE_WEB=0 +POSITIONAL_ARGUMENTS=() +for argument in "$@"; do + case "${argument}" in + --no-cache-web) + [[ "${NO_CACHE_WEB}" == 0 ]] || fail "--no-cache-web may only be specified once" + NO_CACHE_WEB=1 + ;; + --*) + fail "unknown option: ${argument}" + ;; + *) + POSITIONAL_ARGUMENTS+=("${argument}") + ;; + esac +done + +(( ${#POSITIONAL_ARGUMENTS[@]} >= 2 && ${#POSITIONAL_ARGUMENTS[@]} <= 3 )) || { + usage + exit 2 +} +set -- "${POSITIONAL_ARGUMENTS[@]}" + +REGISTRY_NAMESPACE="$1" +RELEASE_VERSION="$2" +RELEASE_PLATFORMS="${3:-linux/amd64,linux/arm64}" + +[[ "${PUBLISH_RELEASE_CONFIRM:-}" == 1 ]] || fail "set PUBLISH_RELEASE_CONFIRM=1 after reviewing RELEASING.md" +[[ "${REGISTRY_NAMESPACE}" =~ ^[a-z0-9][a-z0-9._:/-]*[a-z0-9]$ ]] || fail "registry namespace is malformed or not lowercase" +[[ "${REGISTRY_NAMESPACE}" != *://* && "${REGISTRY_NAMESPACE}" != *@* ]] || fail "registry namespace must not contain a URL scheme or digest" +[[ "${RELEASE_VERSION}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]] || fail "release version is not a valid immutable tag" +[[ "${RELEASE_VERSION}" =~ [0-9] ]] || fail "release version must contain a numeric version or date" +command -v tr >/dev/null 2>&1 || fail "tr is required" +release_version_lower="$(printf '%s' "${RELEASE_VERSION}" | tr '[:upper:]' '[:lower:]')" +case "${release_version_lower}" in + latest|stable|edge|main|master|develop|development|release|*.x|*replace_me*|*changeme*|*placeholder*|*pending*|*approved_version*|*yyyy*) + fail "release version is mutable or still contains placeholder text" + ;; +esac +[[ "${RELEASE_PLATFORMS}" =~ ^linux/(amd64|arm64)(,linux/(amd64|arm64))*$ ]] || fail "platforms must be linux/amd64 and/or linux/arm64" + +for command_name in docker git mkdir mktemp sed tar; do + command -v "${command_name}" >/dev/null 2>&1 || fail "${command_name} is required" +done +docker buildx version >/dev/null 2>&1 || fail "Docker Buildx is required" + +cd "${REPOSITORY_DIR}" + +git diff --quiet || fail "deploy worktree has unstaged changes" +git diff --cached --quiet || fail "deploy worktree has staged changes" +[[ -z "$(git ls-files --others --exclude-standard)" ]] || fail "deploy worktree has untracked files" + +verify_component() { + local component_path="$1" + local expected_revision + local actual_revision + + expected_revision="$(git rev-parse "HEAD:${component_path}")" || fail "cannot read ${component_path} gitlink" + [[ -d "${component_path}/.git" || -f "${component_path}/.git" ]] || fail "${component_path} submodule is not initialised" + actual_revision="$(git -C "${component_path}" rev-parse HEAD)" || fail "cannot read ${component_path} revision" + [[ "${actual_revision}" == "${expected_revision}" ]] || fail "${component_path} does not match its gitlink" + git -C "${component_path}" diff --quiet || fail "${component_path} has unstaged changes" + git -C "${component_path}" diff --cached --quiet || fail "${component_path} has staged changes" + [[ -z "$(git -C "${component_path}" ls-files --others --exclude-standard)" ]] || fail "${component_path} has untracked files" +} + +verify_component doubtfire-api +verify_component doubtfire-web + +if git -C doubtfire-web submodule status --recursive | grep -Eq '^[+-U]'; then + fail "doubtfire-web nested submodules are missing or do not match their gitlinks" +fi +if ! git -C doubtfire-web submodule foreach --quiet --recursive ' + git diff --quiet && + git diff --cached --quiet && + test -z "$(git ls-files --others --exclude-standard)" +'; then + fail "doubtfire-web nested submodules contain staged, unstaged, or untracked changes" +fi + +RELEASE_TEMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/doubtfire-release.XXXXXX")" +METADATA_DIR="${RELEASE_TEMP_DIR}/metadata" +CONTEXT_DIR="${RELEASE_TEMP_DIR}/contexts" +mkdir -p "${METADATA_DIR}" "${CONTEXT_DIR}" +cleanup() { + local publication_status=$? + rm -rf -- "${RELEASE_TEMP_DIR}" + exit "${publication_status}" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +prepare_context() { + local component_path="$1" + local destination_path="$2" + + mkdir -p "${destination_path}" + if ! git -C "${component_path}" archive --format=tar HEAD | \ + tar -xf - -C "${destination_path}"; then + fail "could not create a clean tracked-file context for ${component_path}" + fi +} + +API_CONTEXT="${CONTEXT_DIR}/doubtfire-api" +WEB_CONTEXT="${CONTEXT_DIR}/doubtfire-web" +PUBLISHED_MANIFEST_LINES=() +prepare_context doubtfire-api "${API_CONTEXT}" +prepare_context doubtfire-web "${WEB_CONTEXT}" + +# A superproject archive contains only the nested gitlink directory. Populate +# every nested web submodule from its exact committed tree, never its worktree, +# so ignored/untracked files cannot enter the image build context. +export WEB_CONTEXT +if ! git -C doubtfire-web submodule foreach --quiet --recursive ' + destination="${WEB_CONTEXT}/${displaypath}" + archive="${destination}.git-archive.tar" + mkdir -p "${destination}" + git archive --format=tar --output="${archive}" HEAD && + tar -xf "${archive}" -C "${destination}" && + rm -f -- "${archive}" +'; then + fail "could not populate the clean nested web-submodule build context" +fi +export -n WEB_CONTEXT + +publish_image() { + local manifest_key="$1" + local image_name="$2" + local dockerfile="$3" + local context="$4" + local source_revision="$5" + local image_tag="${REGISTRY_NAMESPACE}/${image_name}:${RELEASE_VERSION}" + local metadata_file="${METADATA_DIR}/${manifest_key}.json" + local digest + local registry_digest + local cache_flag="" + + # A new tag or final-image label does not invalidate the cached Angular build + # stage. Recovery must regenerate ngsw.json, even when the source is unchanged. + if [[ "${manifest_key}" == DOUBTFIRE_WEB_IMAGE && "${NO_CACHE_WEB}" == 1 ]]; then + cache_flag=--no-cache + printf 'Recovery release: bypassing the web image build cache.\n' >&2 + fi + + [[ "${source_revision}" =~ ^[0-9a-f]{40}$ ]] || fail "source revision is invalid for ${image_name}" + + printf 'Publishing %s from %s...\n' "${image_tag}" "${dockerfile}" >&2 + docker buildx build \ + ${cache_flag:+"${cache_flag}"} \ + --file "${dockerfile}" \ + --platform "${RELEASE_PLATFORMS}" \ + --tag "${image_tag}" \ + --label "org.opencontainers.image.revision=${source_revision}" \ + --label "org.opencontainers.image.version=${RELEASE_VERSION}" \ + --sbom=true \ + --provenance=mode=max \ + --push \ + --metadata-file "${metadata_file}" \ + "${context}" >&2 + + digest="$( + sed -nE 's/.*"containerimage\.digest"[[:space:]]*:[[:space:]]*"(sha256:[0-9a-f]{64})".*/\1/p' \ + "${metadata_file}" + )" + [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "Buildx metadata did not contain exactly one valid digest for ${image_tag}" + registry_digest="$( + docker buildx imagetools inspect \ + "${REGISTRY_NAMESPACE}/${image_name}@${digest}" \ + --format '{{.Manifest.Digest}}' + )" + [[ "${registry_digest}" == "${digest}" ]] || fail "registry digest verification failed for ${image_tag}" + PUBLISHED_MANIFEST_LINES+=( + "${manifest_key}=${REGISTRY_NAMESPACE}/${image_name}@${digest}" + ) +} + +DEPLOY_GIT_REVISION="$(git rev-parse HEAD)" +API_GIT_REVISION="$(git rev-parse HEAD:doubtfire-api)" +WEB_GIT_REVISION="$(git rev-parse HEAD:doubtfire-web)" + +publish_image DOUBTFIRE_API_IMAGE apiserver "${API_CONTEXT}/deployApi.Dockerfile" "${API_CONTEXT}" "${API_GIT_REVISION}" +publish_image DOUBTFIRE_APP_IMAGE appserver "${API_CONTEXT}/deployAppSvr.Dockerfile" "${API_CONTEXT}" "${API_GIT_REVISION}" +publish_image DOUBTFIRE_WEB_IMAGE doubtfire-web "${WEB_CONTEXT}/deploy.Dockerfile" "${WEB_CONTEXT}" "${WEB_GIT_REVISION}" +publish_image TEXLIVE_IMAGE formatif-latex "${API_CONTEXT}/texlive.Dockerfile" "${API_CONTEXT}" "${API_GIT_REVISION}" +publish_image JPLAG_IMAGE doubtfire-jplag "${API_CONTEXT}/jplag.Dockerfile" "${API_CONTEXT}" "${API_GIT_REVISION}" + +# Emit a complete manifest only after all five pushes and digest verifications +# succeed. A failed publication therefore cannot look like a complete release. +printf 'DEPLOY_GIT_REVISION=%s\n' "${DEPLOY_GIT_REVISION}" +printf 'API_GIT_REVISION=%s\n' "${API_GIT_REVISION}" +printf 'WEB_GIT_REVISION=%s\n' "${WEB_GIT_REVISION}" +printf 'RELEASE_VERSION=%s\n' "${RELEASE_VERSION}" +printf 'RELEASE_PLATFORMS=%s\n' "${RELEASE_PLATFORMS}" +printf '%s\n' "${PUBLISHED_MANIFEST_LINES[@]}" + +cat >&2 <<'NEXT_STEPS' +Publication completed. Preserve the stdout manifest in the protected change +record, scan every immutable digest, verify BuildKit SBOM/provenance attestations, +sign with the institution-approved identity, and copy only accepted digest +references into .env.production. This script does not publish or choose the +proxy, MariaDB, Redis, or HAProxy images; record their independently reviewed +digests in the same manifest. +NEXT_STEPS diff --git a/production/secrets/.gitkeep b/production/secrets/.gitkeep new file mode 100644 index 000000000..14594a208 --- /dev/null +++ b/production/secrets/.gitkeep @@ -0,0 +1 @@ +# Runtime credentials and TLS material belong outside the repository. diff --git a/production/shared-files/aliases b/production/shared-files/aliases deleted file mode 100644 index d886d2e8b..000000000 --- a/production/shared-files/aliases +++ /dev/null @@ -1 +0,0 @@ -root: support@email.address diff --git a/production/shared-files/aliases.example b/production/shared-files/aliases.example new file mode 100644 index 000000000..3026c0ec7 --- /dev/null +++ b/production/shared-files/aliases.example @@ -0,0 +1,2 @@ +# Copy to the absolute ALIASES_PATH and replace the destination address. +root: operations@REPLACE_ME.edu.au diff --git a/production/shared-files/msmtprc b/production/shared-files/msmtprc deleted file mode 100644 index 1aa8aed9e..000000000 --- a/production/shared-files/msmtprc +++ /dev/null @@ -1,19 +0,0 @@ -# find out more about the configuration here: https://marlam.de/msmtp/msmtprc.txt -# Set default values for all following accounts. -defaults -auth on -auto_from off -tls on -tls_starttls on -logfile /doubtfire/log/msmtp - -# server -account server -host smtp.server # update host -port 587 # update port -from # update from email -user # update email user -password # update email password - -# Set a default account -account default : server diff --git a/production/shared-files/msmtprc.example b/production/shared-files/msmtprc.example new file mode 100644 index 000000000..32bbdca0d --- /dev/null +++ b/production/shared-files/msmtprc.example @@ -0,0 +1,17 @@ +# Copy to the absolute MSMTPRC_PATH, replace all placeholders, and chmod 0600. +# See https://marlam.de/msmtp/msmtprc.txt for other supported configurations. +defaults +auth on +auto_from off +tls on +tls_starttls on +logfile /doubtfire/log/msmtp + +account server +host smtp.REPLACE_ME.edu.au +port 587 +from ontrack@REPLACE_ME.edu.au +user REPLACE_ME +password REPLACE_ME + +account default : server diff --git a/production/shared-files/proxy-nginx.conf b/production/shared-files/proxy-nginx.conf deleted file mode 100644 index b5537ae6e..000000000 --- a/production/shared-files/proxy-nginx.conf +++ /dev/null @@ -1,56 +0,0 @@ -events { - worker_connections 4096; ## Default: 1024 -} - -http { - underscores_in_headers on; - - # Redirect non-ssl to ssl - # and drop www if present - server { - listen 80; - server_name www.localhost localhost; #update to remove localhost name - return 301 https://localhost$request_uri; #replace localhost with domain name of server - } - - # Drop www if present in https connections - server { # This new server will watch for traffic on 443 - listen 443 ssl; - server_name www.localhost; #replace localhost with domain name of server - ssl_certificate /etc/nginx/cert.crt; - ssl_certificate_key /etc/nginx/key.key; - underscores_in_headers on; - return 301 https://localhost$request_uri; #replace localhost with domain name of server - } - - # We have a direct ssl connection to the domain name... - # Route requests to internal servers - server { - listen 443 ssl; - server_name localhost; #replace localhost with domain name of server - ssl_certificate /etc/nginx/cert.crt; - ssl_certificate_key /etc/nginx/key.key; - - # All api requests go to rails server - location /api { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_pass http://apiserver:3000; - proxy_read_timeout 90; - } - - # anything else (the static files) - location / { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_pass http://webserver:80; - proxy_read_timeout 90; - } - } -} diff --git a/production/tests/compose_contract_test.py b/production/tests/compose_contract_test.py new file mode 100755 index 000000000..5b86ad9cb --- /dev/null +++ b/production/tests/compose_contract_test.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 + +import json +import sys + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +with open(sys.argv[1], encoding="utf-8") as compose_file: + compose = json.load(compose_file) + +services = compose["services"] +expected_services = { + "apiserver", + "docker-socket-proxy", + "doubtfire-db", + "jplag", + "migrate", + "pdfgen", + "proxy", + "redis-sidekiq", + "sidekiq", + "texlive-pdfgen", + "texlive-sidekiq", + "webserver", +} +require(set(services) == expected_services, "unexpected production service set") + +published_ports = [] +for service_name, service in services.items(): + for port in service.get("ports", []): + published_ports.append((service_name, int(port["target"]), int(port["published"]))) +require( + sorted(published_ports) == [("proxy", 80, 80), ("proxy", 443, 443)], + "only proxy ports 80 and 443 may be published", +) + +for network_name in ("data", "docker-api"): + require(compose["networks"][network_name]["internal"] is True, f"{network_name} must be internal") + +long_running = expected_services - {"migrate"} +for service_name in long_running: + require( + services[service_name].get("restart") == "unless-stopped", + f"{service_name} must restart unless stopped", + ) + +for service_name in ("apiserver", "migrate", "pdfgen", "sidekiq"): + environment = services[service_name]["environment"] + require(environment["RAILS_ENV"] == "production", f"{service_name} must run Rails in production") + require("MARIADB_ROOT_PASSWORD" not in environment, f"{service_name} received the database root password") + require("TLS_PRIVATE_KEY_PATH" not in environment, f"{service_name} received the TLS key path") + require(environment["OVERSEER_ENABLED"] == "0", f"{service_name} must keep Overseer disabled") + +live_feature_keys = { + "DF_PPI_MINIMUM_COHORT_SIZE", + "DF_PPI_STALE_AFTER_HOURS", + "DOUBTFIRE_VAPID_PUBLIC_KEY", + "DOUBTFIRE_VAPID_PRIVATE_KEY", + "DOUBTFIRE_VAPID_SUBJECT", +} +for service_name in ("apiserver", "sidekiq"): + environment = services[service_name]["environment"] + for key in live_feature_keys: + require( + bool(environment.get(key)), + f"{service_name} must receive the validated {key} value", + ) + +api_environment = services["apiserver"]["environment"] +sidekiq_environment = services["sidekiq"]["environment"] +teams_scope_keys = { + "DF_TEAMS_ANNOUNCEMENTS_ENABLED": "false", + "DF_TEAMS_TENANT_ID": "", + "DF_TEAMS_CHANNEL_MAPPINGS": "[]", +} +teams_credential_keys = {"DF_TEAMS_CLIENT_ID", "DF_TEAMS_CLIENT_SECRET"} +teams_credentials = {key: "" for key in teams_credential_keys} +if len(sys.argv) > 2 and sys.argv[2] == "--teams-configured": + teams_scope_keys.update({ + "DF_TEAMS_ANNOUNCEMENTS_ENABLED": "true", + "DF_TEAMS_TENANT_ID": "11111111-1111-1111-1111-111111111111", + "DF_TEAMS_CHANNEL_MAPPINGS": '[{"unit_id":111,"team_id":"22222222-2222-2222-2222-222222222222","channel_id":"19:fixture@thread.tacv2","publisher_ids":["33333333-3333-3333-3333-333333333333"],"student_visible":true}]', + }) + teams_credentials.update({ + "DF_TEAMS_CLIENT_ID": "44444444-4444-4444-4444-444444444444", + "DF_TEAMS_CLIENT_SECRET": "fixture-only-teams-application-secret", + }) +for key, default in teams_scope_keys.items(): + expected = default + require(api_environment.get(key) == expected, f"API received unexpected {key}") + require(sidekiq_environment.get(key) == expected, f"Sidekiq received unexpected {key}") +for key in teams_credential_keys: + require( + sidekiq_environment.get(key) == teams_credentials[key], + f"Sidekiq received unexpected {key}", + ) +for service_name, service in services.items(): + environment = service.get("environment", {}) + if service_name != "sidekiq": + for key in teams_credential_keys: + require(key not in environment, f"{service_name} must not receive Teams credentials") + if service_name not in {"apiserver", "sidekiq"}: + for key in teams_scope_keys: + require(key not in environment, f"{service_name} must not receive Teams mappings") +require( + sidekiq_environment.get("DF_SIDEKIQ_CONCURRENCY") == "5", + "Sidekiq must receive the validated concurrency value", +) +require( + 2 <= int(sidekiq_environment["DF_SIDEKIQ_CONCURRENCY"]) <= 5, + "Sidekiq concurrency is outside the ActiveRecord pool bound", +) +for service_name, service in services.items(): + if service_name != "sidekiq": + require( + "DF_SIDEKIQ_CONCURRENCY" not in service.get("environment", {}), + f"{service_name} must not receive Sidekiq concurrency", + ) +for key in live_feature_keys: + require( + api_environment[key] == sidekiq_environment[key], + f"API and Sidekiq must receive the same {key} value", + ) +require(int(api_environment["DF_PPI_MINIMUM_COHORT_SIZE"]) >= 21, "PPI cohort privacy floor is unsafe") +require( + 1 <= int(api_environment["DF_PPI_STALE_AFTER_HOURS"]) <= 48, + "PPI freshness window is outside the approved range", +) + +for service_name in ("migrate", "pdfgen"): + environment = services[service_name]["environment"] + for key in live_feature_keys: + require(key not in environment, f"{service_name} must not receive {key}") + +require( + services["apiserver"]["depends_on"]["migrate"]["condition"] == "service_completed_successfully", + "API startup must be gated on migrations", +) +require( + any("/readiness" in argument for argument in services["apiserver"]["healthcheck"]["test"]), + "API health must check database and Redis readiness", +) +for service_name in ("pdfgen", "sidekiq"): + require( + services[service_name]["environment"]["LATEX_BUILD_PATH"] + == "/texlive/shell/latex_build.sh", + f"{service_name} must use the TexLive build entry point", + ) + require( + services[service_name]["environment"]["DOCKER_HOST"] == "tcp://docker-socket-proxy:2375", + f"{service_name} must use the constrained Docker API proxy", + ) + require( + services[service_name]["depends_on"]["docker-socket-proxy"]["condition"] == "service_healthy", + f"{service_name} must wait for the Docker API proxy", + ) + +require(services["migrate"]["networks"]["egress"]["gw_priority"] == 1, "migrations need deterministic egress") +require(services["apiserver"]["networks"]["edge"]["gw_priority"] == 1, "API needs deterministic egress") +for service_name in ("pdfgen", "sidekiq"): + require( + services[service_name]["networks"]["egress"]["gw_priority"] == 1, + f"{service_name} needs deterministic egress", + ) + +require(services["docker-socket-proxy"]["read_only"] is True, "Docker socket proxy filesystem must be read-only") +require("healthcheck" in services["docker-socket-proxy"], "Docker socket proxy must have a health check") +require( + services["docker-socket-proxy"]["environment"] + == {"COMPOSE_PROJECT_NAME": compose["name"]}, + "Docker API proxy must not enable broad category ACLs", +) +require(services["jplag"]["network_mode"] == "none", "JPlag must not have network access") +require(services["texlive-pdfgen"]["network_mode"] == "none", "pdfgen TexLive must not have network access") +require(services["texlive-sidekiq"]["network_mode"] == "none", "Sidekiq TexLive must not have network access") + +for service_name in ("texlive-pdfgen", "texlive-sidekiq", "jplag"): + service = services[service_name] + require(float(service["cpus"]) > 0, f"{service_name} must have a CPU limit") + require(bool(service["mem_limit"]), f"{service_name} must have a memory limit") + require(int(service["pids_limit"]) > 0, f"{service_name} must have a PID limit") + require(service["cap_drop"] == ["ALL"], f"{service_name} must drop Linux capabilities") + require( + "no-new-privileges:true" in service["security_opt"], + f"{service_name} must prevent privilege escalation", + ) + +for service_name in ("texlive-pdfgen", "texlive-sidekiq"): + volume_targets = {volume["target"] for volume in services[service_name]["volumes"]} + require("/student-work" not in volume_targets, f"{service_name} must not mount student work") + +jplag_targets = {volume["target"] for volume in services["jplag"]["volumes"]} +require( + jplag_targets == {"/student-work/jplag", "/student-work/archive/jplag", "/tmp/jplag"}, + "JPlag must only mount report and temporary paths", +) + +for service_name, service in services.items(): + require("@sha256:" in service["image"], f"{service_name} image is not immutable") + +print("Compose contract checks passed.") diff --git a/production/tests/nginx_upload_test.py b/production/tests/nginx_upload_test.py new file mode 100644 index 000000000..029845952 --- /dev/null +++ b/production/tests/nginx_upload_test.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Exercise the real production proxy template with disposable Docker fixtures.""" + +import http.client +import json +import os +from pathlib import Path +import ssl +import subprocess +import tempfile +import time +import unittest +import uuid + + +ROOT = Path(__file__).resolve().parents[1] +NGINX_IMAGE = os.environ.get( + "UPLOAD_TEST_NGINX_IMAGE", + "nginx:1.30.4-alpine@sha256:97d490c12ba55b4946b01546d1c3ed324e8d41ab1c9fcb2a616aa470620e5b46", +) +SINK_IMAGE = os.environ.get("UPLOAD_TEST_SINK_IMAGE", "python:3.13-alpine") +LIMIT = 32 * 1024 * 1024 + + +def docker(*args): + return subprocess.check_output(["docker", *args], text=True, stderr=subprocess.STDOUT).strip() + + +class UploadProxyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.prefix = "upload-boundary-" + uuid.uuid4().hex[:12] + cls.resources = [] + cls.fixture = tempfile.TemporaryDirectory(prefix="upload-boundary-") + cls.addClassCleanup(cls.cleanup) + fixture = Path(cls.fixture.name) + subprocess.run([ + "openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", "/CN=ontrack.test.invalid", "-keyout", str(fixture / "privkey.pem"), + "-out", str(fixture / "fullchain.pem"), + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + docker("network", "create", cls.prefix) + cls.network_created = True + cls.sink = cls.prefix + "-sink" + cls.resources.append(cls.sink) + docker("run", "-d", "--name", cls.sink, "--network", cls.prefix, + "--network-alias", "apiserver", "--network-alias", "webserver", + "-v", f"{ROOT / 'tests/upload_sink.py'}:/sink.py:ro", + SINK_IMAGE, "python", "/sink.py") + cls.proxy = cls.prefix + "-proxy" + cls.resources.append(cls.proxy) + docker("run", "-d", "--name", cls.proxy, "--network", cls.prefix, + "-p", "127.0.0.1::443", "-e", "SERVER_NAME=ontrack.test.invalid", + "-e", "CLIENT_MAX_BODY_SIZE=32m", + "-e", "NGINX_ENVSUBST_FILTER=^(SERVER_NAME|CLIENT_MAX_BODY_SIZE)$", + "-v", f"{ROOT / 'proxy-nginx.conf.template'}:/etc/nginx/templates/default.conf.template:ro", + "-v", f"{fixture / 'fullchain.pem'}:/etc/nginx/tls/fullchain.pem:ro", + "-v", f"{fixture / 'privkey.pem'}:/etc/nginx/tls/private.key:ro", + NGINX_IMAGE) + cls.port = int(docker("port", cls.proxy, "443/tcp").rsplit(":", 1)[1]) + deadline = time.monotonic() + 25 + while time.monotonic() < deadline: + try: + status, _, _ = cls.request("GET", "/healthz") + if status == 200: + break + except (OSError, http.client.HTTPException): + pass + time.sleep(0.2) + else: + raise RuntimeError("Proxy did not start: " + docker("logs", cls.proxy)) + docker("exec", cls.proxy, "nginx", "-t") + + @classmethod + def cleanup(cls): + for name in reversed(cls.resources): + subprocess.run(["docker", "rm", "-f", name], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, check=False) + if getattr(cls, "network_created", False): + subprocess.run(["docker", "network", "rm", cls.prefix], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + cls.fixture.cleanup() + + @classmethod + def request(cls, method, path, body=b"", content_type="application/octet-stream"): + # Only the ephemeral, self-signed localhost fixture disables TLS checking. + connection = http.client.HTTPSConnection( + "127.0.0.1", cls.port, timeout=30, context=ssl._create_unverified_context()) + try: + connection.request(method, path, body, { + "Host": "ontrack.test.invalid", "Content-Type": content_type}) + response = connection.getresponse() + return response.status, dict(response.getheaders()), response.read() + finally: + connection.close() + + def test_permitted_chat_boundary_reaches_api_with_multipart_overhead(self): + boundary = "upload-boundary-fixture" + for size in (0, 1024, 29_999_999, 30_000_000): + with self.subTest(file_size=size): + body = (f"--{boundary}\r\nContent-Disposition: form-data; name=\"attachment\"; " + f"filename=\"boundary.csv\"\r\nContent-Type: text/csv\r\n\r\n").encode() + body += b"x" * size + f"\r\n--{boundary}--\r\n".encode() + status, headers, payload = self.request( + "POST", "/api/projects/1/task_def_id/1/comments", body, + "multipart/form-data; boundary=" + boundary) + self.assertEqual(200, status) + self.assertEqual("true", headers.get("X-Upstream-Reached")) + self.assertEqual(len(body), json.loads(payload)["received_bytes"]) + + def test_total_request_limit_includes_exact_boundary(self): + status, headers, payload = self.request( + "POST", "/api/projects/1/task_def_id/1/comments", b"x" * LIMIT) + self.assertEqual(200, status) + self.assertEqual("true", headers.get("X-Upstream-Reached")) + self.assertEqual(LIMIT, json.loads(payload)["received_bytes"]) + + def test_over_limit_returns_json_without_reaching_api(self): + status, headers, payload = self.request( + "POST", "/api/projects/1/task_def_id/1/comments", b"x" * (LIMIT + 1)) + self.assertEqual(413, status) + self.assertNotIn("X-Upstream-Reached", headers) + self.assertIn("application/json", headers.get("Content-Type", "")) + self.assertEqual("request_too_large", json.loads(payload)["code"]) + self.assertIn("smaller file", json.loads(payload)["error"]) + self.assertEqual("nosniff", headers.get("X-Content-Type-Options")) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/production/tests/probe_upload_limits.py b/production/tests/probe_upload_limits.py new file mode 100644 index 000000000..e86d611f2 --- /dev/null +++ b/production/tests/probe_upload_limits.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Measure chat upload limits against a disposable, authenticated API project. + +Successful test comments are deleted immediately. Requires the generic CSV +attachment API and a test user authorized to comment on the specified task. +Credentials are read from the environment and never written to the report. +""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request +import uuid + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + # Do not forward authentication headers to a redirected endpoint. + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def request(opener, url, headers, method, data=None): + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with opener.open(req, timeout=60) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--direct-url", required=True, help="Full direct API comments endpoint") + parser.add_argument("--proxy-url", required=True, help="Same comments endpoint through Nginx") + parser.add_argument("--proxy-limit-bytes", type=int, + help="Optionally also send one byte above this total request limit") + args = parser.parse_args() + if args.proxy_limit_bytes is not None and not 1 <= args.proxy_limit_bytes <= 128 * 1024 * 1024: + parser.error("Use a disposable proxy with a limit from 1 byte to 128 MiB for this probe") + for url in (args.direct_url, args.proxy_url): + parsed = urllib.parse.urlsplit(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.query or parsed.fragment or parsed.username: + parser.error("Use an HTTP(S) endpoint without credentials, query string or fragment") + headers = {"Username": os.environ["UPLOAD_TEST_USERNAME"], + "Auth-Token": os.environ["UPLOAD_TEST_AUTH_TOKEN"]} + opener = urllib.request.build_opener(NoRedirect()) + failures = [] + for label, endpoint in (("direct", args.direct_url), ("proxy", args.proxy_url)): + for size, expected in ((0, 400), (1024, 201), (29_999_999, 201), (30_000_000, 413)): + boundary = "upload-probe-" + uuid.uuid4().hex + prefix = (f"--{boundary}\r\nContent-Disposition: form-data; name=\"client_request_id\"\r\n\r\n" + f"{uuid.uuid4()}\r\n--{boundary}\r\nContent-Disposition: form-data; " + "name=\"attachment\"; filename=\"upload-limit-probe.csv\"\r\n" + "Content-Type: text/csv\r\n\r\n").encode() + payload = prefix + (b"value\n" + b"x" * size)[:size] + f"\r\n--{boundary}--\r\n".encode() + status, body = request(opener, endpoint, { + **headers, "Content-Type": "multipart/form-data; boundary=" + boundary}, "POST", payload) + print(json.dumps({"path": label, "file_bytes": size, "request_bytes": len(payload), + "status": status, "expected": expected})) + if 200 <= status < 300: + comment_id = json.loads(body)["id"] + if not isinstance(comment_id, int) or isinstance(comment_id, bool): + raise ValueError("API returned no integer comment ID for cleanup") + deleted, _ = request(opener, endpoint.rstrip("/") + "/" + str(comment_id), headers, "DELETE") + if not 200 <= deleted < 300: + failures.append(f"{label}: test comment {comment_id} could not be deleted ({deleted})") + if status != expected: + failures.append(f"{label}: {size} file bytes returned {status}, expected {expected}") + if args.proxy_limit_bytes is not None: + status, body = request(opener, args.proxy_url, { + **headers, "Content-Type": "application/octet-stream"}, "POST", + b"x" * (args.proxy_limit_bytes + 1)) + result = json.loads(body) + print(json.dumps({"path": "proxy", "request_bytes": args.proxy_limit_bytes + 1, + "status": status, "code": result.get("code")})) + if status != 413 or result.get("code") != "request_too_large": + failures.append("Oversized request did not receive the proxy JSON 413") + if failures: + raise SystemExit("\n".join(failures)) + + +if __name__ == "__main__": + try: + main() + except (KeyError, ValueError, OSError) as error: + # Credential values and response bodies must never be emitted on failure. + sys.exit(f"Upload probe could not complete ({type(error).__name__}). Check test credentials and endpoints.") diff --git a/production/tests/publish_release_test.py b/production/tests/publish_release_test.py new file mode 100644 index 000000000..4890821a0 --- /dev/null +++ b/production/tests/publish_release_test.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Exercise publisher arguments with real local Git sources and a fake registry.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest + + +PUBLISHER = Path(__file__).resolve().parents[1] / "publish-release.sh" +REGISTRY = "registry.example.edu/ontrack" +VERSION = "11.0.2-recovery.1" + + +class PublishReleaseTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temporary = tempfile.TemporaryDirectory(prefix="df-publisher-test-") + cls.addClassCleanup(cls.temporary.cleanup) + cls.fixture = Path(cls.temporary.name) + cls.repository = cls.fixture / "release" + cls.binary_directory = cls.fixture / "bin" + cls.binary_directory.mkdir() + cls.docker_calls = cls.fixture / "docker-calls.jsonl" + cls.environment = dict( + os.environ, + GIT_CONFIG_GLOBAL=os.devnull, + GIT_CONFIG_NOSYSTEM="1", + GIT_TERMINAL_PROMPT="0", + PUBLISH_RELEASE_CONFIRM="1", + PUBLISHER_TEST_DOCKER_CALLS=str(cls.docker_calls), + PATH=str(cls.binary_directory) + os.pathsep + os.environ["PATH"], + ) + for name, files in ( + ("api", ("deployApi.Dockerfile", "deployAppSvr.Dockerfile", + "texlive.Dockerfile", "jplag.Dockerfile")), + ("web", ("deploy.Dockerfile",)), + ): + source = cls.fixture / name + source.mkdir() + cls.git(source, "init", "-q") + for filename in files: + (source / filename).write_text("FROM scratch\n", encoding="utf-8") + cls.git(source, "add", ".") + cls.git(source, "commit", "-qm", "Fixture source") + + cls.repository.mkdir() + cls.git(cls.repository, "init", "-q") + (cls.repository / "production").mkdir() + cls.script = cls.repository / "production" / "publish-release.sh" + shutil.copy2(PUBLISHER, cls.script) + for name in ("api", "web"): + cls.git(cls.repository, "submodule", "add", "-q", + str(cls.fixture / name), "doubtfire-" + name) + cls.git(cls.repository, "add", ".") + cls.git(cls.repository, "commit", "-qm", "Pin fixture sources") + + # Every Docker invocation is captured; unsupported commands fail closed. + # The fixture never calls an actual Docker daemon or registry. + docker = cls.binary_directory / "docker" + docker.write_text("#!" + sys.executable + "\n" + textwrap.dedent("""\ + import hashlib + import json + import os + from pathlib import Path + import sys + + arguments = sys.argv[1:] + with open(os.environ["PUBLISHER_TEST_DOCKER_CALLS"], "a") as calls: + calls.write(json.dumps(arguments) + "\\n") + if arguments == ["buildx", "version"]: + print("buildx publisher test fixture") + elif arguments[:2] == ["buildx", "build"]: + image = arguments[arguments.index("--tag") + 1] + if image == os.environ.get("PUBLISHER_TEST_FAIL_IMAGE"): + print("Fixture build failure", file=sys.stderr) + raise SystemExit(23) + digest = "sha256:" + hashlib.sha256(image.encode()).hexdigest() + metadata = Path(arguments[arguments.index("--metadata-file") + 1]) + metadata.write_text(json.dumps({"containerimage.digest": digest})) + elif arguments[:3] == ["buildx", "imagetools", "inspect"]: + print(arguments[3].split("@", 1)[1]) + else: + raise SystemExit("Unexpected Docker command: " + repr(arguments)) + """), encoding="utf-8") + docker.chmod(0o755) + + @classmethod + def git(cls, directory, *arguments): + subprocess.run( + ["git", "-c", "user.name=Publisher Test", "-c", + "user.email=publisher@example.invalid", "-c", "commit.gpgsign=false", + "-c", "protocol.file.allow=always", *arguments], + cwd=directory, env=cls.environment, check=True, capture_output=True, + ) + + def invoke(self, *arguments, confirmed=True, fail_image=None): + self.docker_calls.unlink(missing_ok=True) + environment = dict(self.environment) + if fail_image is not None: + environment["PUBLISHER_TEST_FAIL_IMAGE"] = fail_image + if not confirmed: + environment.pop("PUBLISH_RELEASE_CONFIRM") + result = subprocess.run( + ["bash", str(self.script), *arguments], + env=environment, capture_output=True, text=True, timeout=30, + ) + calls = [json.loads(line) for line in self.docker_calls.read_text().splitlines()] \ + if self.docker_calls.exists() else [] + return result, calls + + def assert_publication(self, result, calls, platforms, recovery): + self.assertEqual(result.returncode, 0, result.stderr) + builds = [call for call in calls if call[:2] == ["buildx", "build"]] + self.assertEqual(len(builds), 5) + self.assertEqual(len([line for line in result.stdout.splitlines() + if "_IMAGE=" in line]), 5) + for build in builds: + image = build[build.index("--tag") + 1] + self.assertEqual(build[build.index("--platform") + 1], platforms) + self.assertIn("--push", build) + self.assertIn("--sbom=true", build) + self.assertIn("--provenance=mode=max", build) + self.assertEqual("--no-cache" in build, + recovery and image == f"{REGISTRY}/doubtfire-web:{VERSION}") + + def test_normal_publication_preserves_cache_and_default_platforms(self): + result, calls = self.invoke(REGISTRY, VERSION) + self.assert_publication(result, calls, "linux/amd64,linux/arm64", recovery=False) + + def test_recovery_bypasses_only_web_cache_and_accepts_option_positions(self): + for arguments in ( + ("--no-cache-web", REGISTRY, VERSION, "linux/amd64"), + (REGISTRY, VERSION, "linux/amd64", "--no-cache-web"), + (REGISTRY, VERSION, "--no-cache-web"), + ): + with self.subTest(arguments=arguments): + result, calls = self.invoke(*arguments) + platforms = "linux/amd64" if "linux/amd64" in arguments \ + else "linux/amd64,linux/arm64" + self.assert_publication(result, calls, platforms, recovery=True) + + def test_invalid_arguments_fail_before_docker(self): + for arguments, message in ( + (("--no-cache", REGISTRY, VERSION), "unknown option"), + (("--no-cache-web", "--no-cache-web", REGISTRY, VERSION), + "may only be specified once"), + (("--no-cache-web",), "Usage:"), + (("--no-cache-web", REGISTRY, VERSION, "linux/amd64", "extra"), "Usage:"), + (("--no-cache-web", REGISTRY, VERSION, "windows/amd64"), "platforms must"), + (("--no-cache-web", REGISTRY, "latest"), "release version"), + ): + with self.subTest(arguments=arguments): + result, calls = self.invoke(*arguments) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + self.assertEqual(calls, []) + + def test_recovery_does_not_bypass_publication_confirmation(self): + result, calls = self.invoke("--no-cache-web", REGISTRY, VERSION, confirmed=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("set PUBLISH_RELEASE_CONFIRM=1", result.stderr) + self.assertEqual(calls, []) + + def test_failed_web_build_preserves_failure_and_emits_no_partial_manifest(self): + result, calls = self.invoke( + "--no-cache-web", REGISTRY, VERSION, + fail_image=f"{REGISTRY}/doubtfire-web:{VERSION}", + ) + self.assertEqual(result.returncode, 23, result.stderr) + self.assertIn("Fixture build failure", result.stderr) + self.assertEqual(result.stdout, "") + self.assertEqual(len([call for call in calls + if call[:2] == ["buildx", "build"]]), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/production/tests/upload_sink.py b/production/tests/upload_sink.py new file mode 100644 index 000000000..585d8d8be --- /dev/null +++ b/production/tests/upload_sink.py @@ -0,0 +1,33 @@ +"""Transport-only fixture: records that Nginx forwarded a request. + +This deliberately does not emulate Rails validation. Application authorization, +format and file-size checks belong to the API suite and the live boundary probe. +""" + +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + size = int(self.headers.get("Content-Length", "0")) + remaining = size + while remaining: + chunk = self.rfile.read(min(remaining, 64 * 1024)) + if not chunk: + break + remaining -= len(chunk) + body = json.dumps({"received_bytes": size - remaining}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Upstream-Reached", "true") + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self.send_response(200) + self.end_headers() + + +ThreadingHTTPServer(("0.0.0.0", 3000), Handler).serve_forever() diff --git a/production/tests/validate_test.sh b/production/tests/validate_test.sh new file mode 100755 index 000000000..0a016f0d6 --- /dev/null +++ b/production/tests/validate_test.sh @@ -0,0 +1,627 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +umask 077 + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +PRODUCTION_DIR="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +REPOSITORY_DIR="$(cd -- "${PRODUCTION_DIR}/.." && pwd)" + +# Prevent a development identity-provider registration from being committed +# again. Values are intentionally not printed because the failure itself may +# represent a credential exposure. +python3 - "${REPOSITORY_DIR}" <<'PY' +from pathlib import Path +import re +import sys + +root = Path(sys.argv[1]) +configuration_files = [ + root / ".devcontainer/devcontainer.env", + root / ".devcontainer/docker-compose.yml", + root / "development/api.env", + root / "development/docker-compose.yml", + root / "development/docker-compose.full.yml", + root / "production/.env.production.example", + root / "production/docker-compose.yml", +] +institution_markers = ( + "rapid.test.aaf.edu.au", + "signon-uat.deakin.edu.au", + "sync-uat.deakin.edu.au", +) +literal_secret = re.compile(r"^\s*DF_SECRET_KEY_AAF\s*(?::|=)\s*(.*)$") +failures = [] + +for path in configuration_files: + text = path.read_text(encoding="utf-8") + lowered = text.lower() + if any(marker in lowered for marker in institution_markers): + failures.append(f"{path.relative_to(root)} contains an institution AAF endpoint") + for line_number, line in enumerate(text.splitlines(), start=1): + match = literal_secret.match(line) + if not match: + continue + value = match.group(1).strip() + if value and not value.startswith("${") and value != "REPLACE_ME": + failures.append( + f"{path.relative_to(root)}:{line_number} contains a literal AAF secret" + ) + +if failures: + raise SystemExit("\n".join(failures)) +print("ok - tracked configuration has no institution AAF credential") +PY + +grep -Fq 'DF_AUTH_METHOD: ${DF_AUTH_METHOD:-database}' \ + "${REPOSITORY_DIR}/.devcontainer/docker-compose.yml" +grep -Fq 'DF_AUTH_METHOD=database' \ + "${REPOSITORY_DIR}/.devcontainer/.env.example" +grep -Fq 'DF_AUTH_METHOD: ${DF_AUTH_METHOD:-database}' \ + "${REPOSITORY_DIR}/development/docker-compose.yml" +grep -Fq 'DF_AUTH_METHOD=database' \ + "${REPOSITORY_DIR}/development/.env.example" +printf 'ok - optional development AAF configuration keeps an explicit safe auth default\n' + +# Unix socket paths are short on some platforms (notably macOS), so keep this +# fixture directly under /tmp instead of a potentially long TMPDIR path. +FIXTURE_DIR="$(mktemp -d "/tmp/df-production-test.XXXXXX")" +cleanup() { + rm -rf -- "${FIXTURE_DIR}" +} +trap cleanup EXIT INT TERM + +HOSTNAME_UNDER_TEST=ontrack.test.edu.au +CERTIFICATE_PATH="${FIXTURE_DIR}/fullchain.pem" +PRIVATE_KEY_PATH="${FIXTURE_DIR}/private.key" +BASE_ENV="${FIXTURE_DIR}/valid.env" + +mkdir -p \ + "${FIXTURE_DIR}/student-work" \ + "${FIXTURE_DIR}/logs" \ + "${FIXTURE_DIR}/mariadb" \ + "${FIXTURE_DIR}/redis" \ + "${FIXTURE_DIR}/student-work/jplag" \ + "${FIXTURE_DIR}/student-work/archive/jplag" + +openssl req -x509 -newkey rsa:2048 -nodes -days 365 \ + -keyout "${PRIVATE_KEY_PATH}" \ + -out "${CERTIFICATE_PATH}" \ + -subj "/CN=${HOSTNAME_UNDER_TEST}" \ + -addext "subjectAltName=DNS:${HOSTNAME_UNDER_TEST}" \ + >/dev/null 2>&1 +chmod 600 "${PRIVATE_KEY_PATH}" + +# Compose rendering does not contact the daemon. CI uses config-only mode for +# its fixture and separately verifies that the live preflight remains +# fail-closed when the standard local daemon socket is unavailable. +export DOUBTFIRE_CONFIG_ONLY=1 + +cat > "${FIXTURE_DIR}/aliases" <<'EOF' +root: operations@test.edu.au +EOF +cat > "${FIXTURE_DIR}/msmtprc" <<'EOF' +defaults +auth on +tls on +account server +host smtp.test.edu.au +port 587 +from ontrack@test.edu.au +user ontrack-service +password fixture-only-random-value +account default : server +EOF +chmod 600 "${FIXTURE_DIR}/msmtprc" + +DIGEST_ONE="sha256:1111111111111111111111111111111111111111111111111111111111111111" +DIGEST_TWO="sha256:2222222222222222222222222222222222222222222222222222222222222222" +DIGEST_THREE="sha256:3333333333333333333333333333333333333333333333333333333333333333" + +cat > "${BASE_ENV}" < "${replacement}" + mv "${replacement}" "${file}" + chmod 600 "${file}" +} + +expect_failure() { + local name="$1" + local key="$2" + local value="$3" + local expected="$4" + local fixture="${FIXTURE_DIR}/${name}.env" + local output + + cp "${BASE_ENV}" "${fixture}" + replace_value "${fixture}" "${key}" "${value}" + + if output="$("${PRODUCTION_DIR}/validate.sh" "${fixture}" 2>&1)"; then + printf 'Expected %s to fail.\n' "${name}" >&2 + exit 1 + fi + printf '%s' "${output}" | grep -Fq "${expected}" || { + printf 'Unexpected %s failure: %s\n' "${name}" "${output}" >&2 + exit 1 + } + printf 'ok - %s\n' "${name}" +} + +"${PRODUCTION_DIR}/validate.sh" "${BASE_ENV}" >/dev/null +printf 'ok - valid production configuration\n' + +MARIADB_PASSWORD=ambient-override \ + "${PRODUCTION_DIR}/compose.sh" --env-file "${BASE_ENV}" config --format json \ + > "${FIXTURE_DIR}/compose.json" +if grep -Fq ambient-override "${FIXTURE_DIR}/compose.json"; then + printf 'Ambient shell values overrode the validated environment file.\n' >&2 + exit 1 +fi +printf 'ok - ambient Compose overrides are ignored\n' +python3 -B "${SCRIPT_DIR}/compose_contract_test.py" "${FIXTURE_DIR}/compose.json" + +teams_env="${FIXTURE_DIR}/teams.env" +cp "${BASE_ENV}" "${teams_env}" +replace_value "${teams_env}" DF_TEAMS_ANNOUNCEMENTS_ENABLED true +replace_value "${teams_env}" DF_TEAMS_TENANT_ID 11111111-1111-1111-1111-111111111111 +replace_value "${teams_env}" DF_TEAMS_CLIENT_ID 44444444-4444-4444-4444-444444444444 +replace_value "${teams_env}" DF_TEAMS_CLIENT_SECRET fixture-only-teams-application-secret +replace_value "${teams_env}" DF_TEAMS_CHANNEL_MAPPINGS '[{"unit_id":111,"team_id":"22222222-2222-2222-2222-222222222222","channel_id":"19:fixture@thread.tacv2","publisher_ids":["33333333-3333-3333-3333-333333333333"],"student_visible":true}]' +"${PRODUCTION_DIR}/validate.sh" "${teams_env}" >/dev/null +"${PRODUCTION_DIR}/compose.sh" --env-file "${teams_env}" config --format json > "${FIXTURE_DIR}/teams-compose.json" +python3 -B "${SCRIPT_DIR}/compose_contract_test.py" "${FIXTURE_DIR}/teams-compose.json" --teams-configured +printf 'ok - Teams metadata is shared only by API and worker; credentials are worker-only\n' + +ordinary_base_env="${BASE_ENV}" +BASE_ENV="${teams_env}" +expect_failure teams-missing-tenant DF_TEAMS_TENANT_ID '' "must be set" +expect_failure teams-invalid-client DF_TEAMS_CLIENT_ID invalid-guid "directory GUID" +expect_failure teams-missing-secret DF_TEAMS_CLIENT_SECRET '' "must be set" +expect_failure teams-no-mappings DF_TEAMS_CHANNEL_MAPPINGS '[]' "Teams channel mappings are invalid" +expect_failure teams-malformed-mapping DF_TEAMS_CHANNEL_MAPPINGS '[invalid]' "Teams channel mappings are invalid" +expect_failure teams-private-channel DF_TEAMS_CHANNEL_MAPPINGS '[{"unit_id":111,"team_id":"22222222-2222-2222-2222-222222222222","channel_id":"19:fixture@thread.tacv2","publisher_ids":["33333333-3333-3333-3333-333333333333"],"student_visible":false}]' "Teams channel mappings are invalid" +expect_failure teams-missing-publisher DF_TEAMS_CHANNEL_MAPPINGS '[{"unit_id":111,"team_id":"22222222-2222-2222-2222-222222222222","channel_id":"19:fixture@thread.tacv2","student_visible":true}]' "Teams channel mappings are invalid" +expect_failure teams-invalid-enabled DF_TEAMS_ANNOUNCEMENTS_ENABLED yes "must be true or false" +BASE_ENV="${ordinary_base_env}" + +grep -Fq 'resolver 127.0.0.11' "${PRODUCTION_DIR}/proxy-nginx.conf.template" +grep -Fq 'proxy_pass http://$api_upstream/readiness' "${PRODUCTION_DIR}/proxy-nginx.conf.template" +grep -Fq 'proxy_pass http://$web_upstream' "${PRODUCTION_DIR}/proxy-nginx.conf.template" +printf 'ok - proxy refreshes service discovery\n' + +proxy_template="${PRODUCTION_DIR}/proxy-nginx.conf.template" +grep -Fq 'map $uri $doubtfire_access_loggable' "${proxy_template}" +grep -Fq 'map $uri $doubtfire_referrer_policy' "${proxy_template}" +grep -Fq '~^/api/scorm(?:/|$) 0;' "${proxy_template}" +grep -Fq '~^/api/scorm(?:/|$) no-referrer;' "${proxy_template}" +server_count="$(grep -Ec '^server \{' "${proxy_template}")" +conditional_access_log_count="$(grep -Fc 'access_log /var/log/nginx/access.log main if=$doubtfire_access_loggable;' "${proxy_template}")" +[[ "${server_count}" -gt 0 && "${conditional_access_log_count}" -eq "${server_count}" ]] || { + printf 'Every proxy virtual server must suppress credential-bearing SCORM access logs.\n' >&2 + exit 1 +} +referrer_policy_header_count="$(grep -Fc 'add_header Referrer-Policy $doubtfire_referrer_policy always;' "${proxy_template}")" +[[ "${referrer_policy_header_count}" -eq 2 ]] || { + printf 'The public HTTP redirect and HTTPS application servers must apply the mapped referrer policy.\n' >&2 + exit 1 +} +if grep -Eiq 'location[^\{]*scorm' "${proxy_template}"; then + printf 'SCORM mitigation must not duplicate or bypass the common API proxy location.\n' >&2 + exit 1 +fi +grep -Fq 'location ~ ^/api(?:/|$)' "${proxy_template}" +printf 'ok - proxy suppresses SCORM credential paths without changing API routing\n' + +grep -Fq 'helper_exec_path' "${PRODUCTION_DIR}/docker-socket-proxy.cfg" +grep -Fq '__PROJECT_NAME__-texlive-' "${PRODUCTION_DIR}/docker-socket-proxy.cfg" +grep -Fq 'http-request deny deny_status 403' "${PRODUCTION_DIR}/docker-socket-proxy.cfg" +printf 'ok - Docker API policy is default-deny and helper-scoped\n' + +grep -Fq -- '--wait-timeout' "${PRODUCTION_DIR}/deploy.sh" +printf 'ok - deployment wait is bounded\n' + +grep -Fq 'Stage and health-check web first' "${PRODUCTION_DIR}/deploy.sh" +grep -Fq -- '--no-deps' "${PRODUCTION_DIR}/deploy.sh" +grep -Fq 'Entering bounded migration maintenance window' "${PRODUCTION_DIR}/deploy.sh" +grep -Fq 'maintenance_services=(proxy apiserver sidekiq pdfgen migrate)' "${PRODUCTION_DIR}/deploy.sh" +grep -Fq 'stop --timeout 60 "${maintenance_services[@]}"' "${PRODUCTION_DIR}/deploy.sh" +grep -Fq '10#${WAIT_TIMEOUT} <= 86400' "${PRODUCTION_DIR}/deploy.sh" +if "${PRODUCTION_DIR}/deploy.sh" one two three >/dev/null 2>&1; then + printf 'deploy.sh accepted extra arguments.\n' >&2 + exit 1 +fi +if "${PRODUCTION_DIR}/deploy.sh" /does/not/exist 86401 >/dev/null 2>&1; then + printf 'deploy.sh accepted an excessive wait timeout.\n' >&2 + exit 1 +fi +printf 'ok - forward rollout stages callback-compatible web before API\n' + +publish_script="${PRODUCTION_DIR}/publish-release.sh" +[[ -x "${publish_script}" ]] || { + printf 'Production release publisher must be executable.\n' >&2 + exit 1 +} +bash -n "${publish_script}" +grep -Fq -- '--sbom=true' "${publish_script}" +grep -Fq -- '--provenance=mode=max' "${publish_script}" +grep -Fq 'org.opencontainers.image.revision=${source_revision}' "${publish_script}" +grep -Fq 'org.opencontainers.image.version=${RELEASE_VERSION}' "${publish_script}" +grep -Fq '"containerimage\.digest"' "${publish_script}" +grep -Fq '"${REGISTRY_NAMESPACE}/${image_name}@${digest}"' "${publish_script}" +grep -Fq 'submodule foreach --quiet --recursive' "${publish_script}" +grep -Fq 'git -C "${component_path}" archive --format=tar HEAD' "${publish_script}" +grep -Fq 'git archive --format=tar --output="${archive}" HEAD' "${publish_script}" +grep -Fq '"${API_CONTEXT}/deployApi.Dockerfile" "${API_CONTEXT}"' "${publish_script}" +grep -Fq 'PUBLISHED_MANIFEST_LINES+=(' "${publish_script}" +grep -Fq 'printf '\''%s\n'\'' "${PUBLISHED_MANIFEST_LINES[@]}"' "${publish_script}" +if grep -Fq 'imagetools inspect "${image_tag}"' "${publish_script}"; then + printf 'Release publisher must resolve the digest from its Buildx metadata, not a mutable tag.\n' >&2 + exit 1 +fi +grep -Fq 'deployApi.Dockerfile' "${publish_script}" +grep -Fq 'deployAppSvr.Dockerfile' "${publish_script}" +for unsafe_release_version in latest 11.0.x RELEASE_OWNER_APPROVED_VERSION; do + if PUBLISH_RELEASE_CONFIRM=1 "${publish_script}" \ + registry.example.edu/ontrack "${unsafe_release_version}" linux/amd64 \ + >/dev/null 2>&1; then + printf 'Release publisher accepted unsafe version: %s\n' "${unsafe_release_version}" >&2 + exit 1 + fi +done +printf 'ok - release publication is revision-locked and emits attested image digests\n' + +verify_script="${PRODUCTION_DIR}/verify.sh" +[[ -x "${verify_script}" ]] || { + printf 'verify.sh must be executable.\n' >&2 + exit 1 +} +for required_source in \ + '/healthz' \ + '/healthz/web' \ + '/index.html' \ + '/ngsw.json' \ + '/ngsw-worker.js' \ + 'http://127.0.0.1:3000/readiness' \ + 'db:abort_if_pending_migrations' \ + '20260824000003' \ + 'ActiveRecord::Base.connection_pool.migration_context.current_version' \ + 'Sidekiq::ProcessSet' \ + 'aggregate_peer_progress' \ + 'poll_communication_set_schedules' \ + 'send_new_task_available_notifications' \ + 'send_due_soon_reminders' \ + 'DF_SIDEKIQ_CONCURRENCY' \ + 'process["concurrency"].to_i == configured_concurrency' \ + 'DF_PPI_MINIMUM_COHORT_SIZE' \ + 'PeerProgressApi::MINIMUM_SAFE_COHORT_SIZE' \ + 'DOUBTFIRE_VAPID_PRIVATE_KEY' \ + 'MANUAL GATES STILL REQUIRED'; do + grep -Fq "${required_source}" "${verify_script}" || { + printf 'verify.sh is missing required check: %s\n' "${required_source}" >&2 + exit 1 + } +done +grep -Fq 'expected final Rails schema version is `20260824000003`' \ + "${REPOSITORY_DIR}/MIGRATING.md" +grep -Fq '| `20260824000003` |' "${REPOSITORY_DIR}/MIGRATING.md" +grep -Fq 'peer_progress_snapshots.submitted_count' "${REPOSITORY_DIR}/MIGRATING.md" +grep -Fq 'new API deliberately withholds compact and detailed output until reaggregation' \ + "${REPOSITORY_DIR}/MIGRATING.md" +printf 'ok - migration dossier matches the release schema gate\n' +grep -Fq 'DOUBTFIRE_CONFIG_ONLY=0 "${SCRIPT_DIR}/validate.sh" "${ENV_FILE}"' "${verify_script}" +if grep -Eq 'printenv|docker (container )?inspect.*Env' "${verify_script}"; then + printf 'verify.sh must not print container environments.\n' >&2 + exit 1 +fi +printf 'ok - post-deploy verification source contract\n' + +if output="$(${verify_script} one two three 2>&1)"; then + printf 'Expected verify.sh to reject extra arguments.\n' >&2 + exit 1 +fi +printf '%s' "${output}" | grep -Fq 'Usage:' +if output="$(${verify_script} "${BASE_ENV}" 0 2>&1)"; then + printf 'Expected verify.sh to reject a zero timeout.\n' >&2 + exit 1 +fi +printf '%s' "${output}" | grep -Fq 'between 1 and 300 seconds' +if output="$(${verify_script} "${BASE_ENV}" 301 2>&1)"; then + printf 'Expected verify.sh to reject an excessive timeout.\n' >&2 + exit 1 +fi +printf '%s' "${output}" | grep -Fq 'between 1 and 300 seconds' +if output="$(${verify_script} "${FIXTURE_DIR}/missing.env" 5 2>&1)"; then + printf 'Expected verify.sh to reject a missing environment file.\n' >&2 + exit 1 +fi +printf '%s' "${output}" | grep -Fq 'environment file not found' +printf 'ok - post-deploy verification rejects invalid invocation\n' + +verify_fixture="${FIXTURE_DIR}/verify-runtime" +mkdir -p "${verify_fixture}/bin" +cp "${verify_script}" "${verify_fixture}/verify.sh" +cat > "${verify_fixture}/validate.sh" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +cat > "${verify_fixture}/compose.sh" <<'EOF' +#!/usr/bin/env bash +case "$*" in + *Sidekiq::ProcessSet*) + case "${VERIFY_STUB_FAIL_AT:-}" in + cron|concurrency) exit 1 ;; + esac + ;; +esac +exit 0 +EOF +cat > "${verify_fixture}/bin/timeout" <<'EOF' +#!/usr/bin/env bash +while [[ "${1:-}" == --* ]]; do + shift +done +shift +exec "$@" +EOF +cat > "${verify_fixture}/bin/curl" <<'EOF' +#!/usr/bin/env bash +output_file=/dev/null +url= +while (( $# > 0 )); do + case "$1" in + --output) + output_file="$2" + shift 2 + ;; + --write-out|--noproxy|--proto|--connect-timeout|--max-time|--max-filesize) + shift 2 + ;; + --disable|--silent|--show-error|--tlsv1.2) + shift + ;; + https://*) + url="$1" + shift + ;; + *) + shift + ;; + esac +done +if [[ "${VERIFY_STUB_FAIL_AT:-}" == public-health && "${url}" == */healthz ]]; then + printf '503' + exit 0 +fi +case "${url}" in + */index.html) + content='' + content_type='text/html' + ;; + */ngsw.json) + content='{"configVersion":1,"assetGroups":[]}' + content_type='application/json' + ;; + */ngsw-worker.js) + if [[ "${VERIFY_STUB_FAIL_AT:-}" == pwa-worker ]]; then + content='' + else + content='const cacheName = "ngsw:test";' + fi + content_type='application/javascript' + ;; + *) + content='ok' + content_type='text/plain' + ;; +esac +if [[ "${output_file}" != /dev/null ]]; then + printf '%s' "${content}" > "${output_file}" +fi +printf '200\n%s' "${content_type}" +EOF +cat > "${verify_fixture}/runtime.env" < "${verify_fixture}/success.out" +grep -Fq 'Automated production verification passed.' "${verify_fixture}/success.out" +grep -Fq 'MANUAL GATES STILL REQUIRED' "${verify_fixture}/success.out" + +if VERIFY_STUB_FAIL_AT=cron PATH="${verify_fixture}/bin:${PATH}" \ + "${verify_fixture}/verify.sh" "${verify_fixture}/runtime.env" 5 \ + > "${verify_fixture}/cron-failure.out" 2>&1; then + printf 'Expected verify.sh to fail when required Sidekiq cron jobs are absent.\n' >&2 + exit 1 +fi +grep -Fq 'Sidekiq process, concurrency, and required cron schedule failed' "${verify_fixture}/cron-failure.out" + +if VERIFY_STUB_FAIL_AT=concurrency PATH="${verify_fixture}/bin:${PATH}" \ + "${verify_fixture}/verify.sh" "${verify_fixture}/runtime.env" 5 \ + > "${verify_fixture}/concurrency-failure.out" 2>&1; then + printf 'Expected verify.sh to fail when live Sidekiq concurrency differs from configuration.\n' >&2 + exit 1 +fi +grep -Fq 'Sidekiq process, concurrency, and required cron schedule failed' "${verify_fixture}/concurrency-failure.out" + +if VERIFY_STUB_FAIL_AT=pwa-worker PATH="${verify_fixture}/bin:${PATH}" \ + "${verify_fixture}/verify.sh" "${verify_fixture}/runtime.env" 5 \ + > "${verify_fixture}/pwa-failure.out" 2>&1; then + printf 'Expected verify.sh to fail when the PWA worker resolves to HTML.\n' >&2 + exit 1 +fi +grep -Fq 'ngsw-worker.js resolved to HTML' "${verify_fixture}/pwa-failure.out" +printf 'ok - post-deploy verification fails closed on runtime regressions\n' + +expect_failure placeholder-secret DF_SECRET_KEY_BASE REPLACE_ME "placeholder value" +expect_failure unlimited-upload CLIENT_MAX_BODY_SIZE 0 "must look like" +expect_failure undersized-upload CLIENT_MAX_BODY_SIZE 31m "at least 32m" +expect_failure undersized-upload-uppercase CLIENT_MAX_BODY_SIZE 1M "at least 32m" + +minimum_upload_env="${FIXTURE_DIR}/minimum-upload.env" +sed 's/^CLIENT_MAX_BODY_SIZE=.*/CLIENT_MAX_BODY_SIZE=32m/' "${BASE_ENV}" > "${minimum_upload_env}" +"${PRODUCTION_DIR}/validate.sh" "${minimum_upload_env}" >/dev/null +printf 'ok - minimum upload limit allows multipart overhead\n' +expect_failure inline-comment-padding DF_SECRET_KEY_BASE 'x # padding-padding-padding-padding' "inline comments are not allowed" +expect_failure compose-interpolation DF_SMTP_PASSWORD 'unsafe$value' "contains a dollar sign" +expect_failure mutable-image DOUBTFIRE_WEB_IMAGE lmsdoubtfire/doubtfire-web:latest "immutable sha256 image digest" +expect_failure development-auth DF_AUTH_METHOD database "development-only" +expect_failure invalid-processor-memory TEXLIVE_MEMORY_LIMIT unlimited "positive whole number" +expect_failure unsupported-overseer OVERSEER_ENABLED 1 "must remain disabled" +expect_failure missing-sidekiq-concurrency DF_SIDEKIQ_CONCURRENCY '' "must be set" +expect_failure low-sidekiq-concurrency DF_SIDEKIQ_CONCURRENCY 1 "integer from 2 to 5" +expect_failure excessive-sidekiq-concurrency DF_SIDEKIQ_CONCURRENCY 6 "integer from 2 to 5" +expect_failure invalid-sidekiq-concurrency DF_SIDEKIQ_CONCURRENCY 2.5 "integer from 2 to 5" +expect_failure missing-ppi-cohort DF_PPI_MINIMUM_COHORT_SIZE '' "must be set" +expect_failure unsafe-ppi-cohort DF_PPI_MINIMUM_COHORT_SIZE 20 "privacy floor of 21" +expect_failure invalid-ppi-cohort DF_PPI_MINIMUM_COHORT_SIZE 21.5 "positive integer" +expect_failure missing-ppi-staleness DF_PPI_STALE_AFTER_HOURS '' "must be set" +expect_failure zero-ppi-staleness DF_PPI_STALE_AFTER_HOURS 0 "positive integer" +expect_failure excessive-ppi-staleness DF_PPI_STALE_AFTER_HOURS 49 "approved maximum of 48 hours" +expect_failure missing-vapid-public DOUBTFIRE_VAPID_PUBLIC_KEY '' "must be set" +expect_failure placeholder-vapid-public DOUBTFIRE_VAPID_PUBLIC_KEY REPLACE_ME_WITH_VAPID_PUBLIC_KEY "placeholder value" +expect_failure malformed-vapid-public DOUBTFIRE_VAPID_PUBLIC_KEY not-a-vapid-key "base64url P-256 public key" +expect_failure missing-vapid-private DOUBTFIRE_VAPID_PRIVATE_KEY '' "must be set" +expect_failure malformed-vapid-private DOUBTFIRE_VAPID_PRIVATE_KEY not-a-vapid-key "base64url P-256 private key" +expect_failure missing-vapid-subject DOUBTFIRE_VAPID_SUBJECT '' "must be set" +expect_failure invalid-vapid-subject DOUBTFIRE_VAPID_SUBJECT ftp://push.test.edu.au "must use mailto: or https://" +expect_failure non-production-vapid-subject DOUBTFIRE_VAPID_SUBJECT mailto:noreply@ontrack-demo.invalid "operational production contact" +expect_failure placeholder-vapid-subject DOUBTFIRE_VAPID_SUBJECT mailto:push@REPLACE_ME.edu.au "placeholder value" +expect_failure development-vapid-public DOUBTFIRE_VAPID_PUBLIC_KEY BOs-KbIoHK7gUIX3i2_uEuDoouj-GKxB-mY9CRmLNmd4Wn-SSl254E1g6jR1ukL3e37p8uCpaMjOvfAB0BwzvSI= "checked-in development key" +expect_failure development-vapid-private DOUBTFIRE_VAPID_PRIVATE_KEY _NFIWSUTdCdLJJFh87pf4ekQLmNYqsweZ4288NpVZaY= "checked-in development key" + +insecure_env="${FIXTURE_DIR}/insecure-mode.env" +cp "${BASE_ENV}" "${insecure_env}" +chmod 644 "${insecure_env}" +if output="$("${PRODUCTION_DIR}/validate.sh" "${insecure_env}" 2>&1)"; then + printf 'Expected insecure environment mode to fail.\n' >&2 + exit 1 +fi +printf '%s' "${output}" | grep -Fq 'must not be accessible by group or other users' +printf 'ok - insecure environment permissions\n' + +expect_failure non-local-daemon DOCKER_SOCKET_PATH "${FIXTURE_DIR}/docker.sock" "must be /var/run/docker.sock" + +if [[ ! -S /var/run/docker.sock || ! -r /var/run/docker.sock || ! -w /var/run/docker.sock ]]; then + if output="$(DOUBTFIRE_CONFIG_ONLY=0 "${PRODUCTION_DIR}/validate.sh" "${BASE_ENV}" 2>&1)"; then + printf 'Expected an inaccessible Docker socket to fail.\n' >&2 + exit 1 + fi + printf '%s' "${output}" | grep -Fq 'not an accessible Unix socket' + printf 'ok - Docker socket preflight is fail-closed\n' +else + printf 'ok - live Docker socket is accessible\n' +fi + +mismatch_key="${FIXTURE_DIR}/mismatch.key" +openssl genpkey -algorithm RSA -out "${mismatch_key}" >/dev/null 2>&1 +chmod 600 "${mismatch_key}" +expect_failure mismatched-tls TLS_PRIVATE_KEY_PATH "${mismatch_key}" "do not match" + +printf 'All production validation tests passed.\n' diff --git a/production/tests/verify_pwa_test.py b/production/tests/verify_pwa_test.py new file mode 100644 index 000000000..6d865706a --- /dev/null +++ b/production/tests/verify_pwa_test.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Exercise the PWA verifier through real local HTTP responses, without Docker.""" + +from contextlib import contextmanager +import copy +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +import json +from pathlib import Path +import struct +import subprocess +import sys +import threading +import unittest +from urllib.parse import urlsplit +import zlib + + +SCRIPT = Path(__file__).resolve().parents[1] / "verify-pwa.py" +SPEC = importlib.util.spec_from_file_location("verify_pwa", SCRIPT) +pwa = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(pwa) + + +def png(size): + def chunk(kind, payload): + return (struct.pack(">I", len(payload)) + kind + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xffffffff)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", size, size, 8, 6, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress((b"\0" + b"\xff\0\x7f\xff" * size) * size)) + + chunk(b"IEND", b"")) + + +def fixtures(): + manifest = { + "id": "/index.html", "start_url": "/index.html", "scope": "/", "name": "OnTrack", + "short_name": "OnTrack", "description": "Learning and feedback", "display": "standalone", + "icons": [{"src": f"/assets/{size}.png", "sizes": f"{size}x{size}", "type": "image/png"} + for size in (192, 512)], + "shortcuts": [{"name": "Home", "url": "/home"}], + } + files = { + "/index.html": ("text/html", b'' + b'' + b'' + b'' + b''), + "/manifest.webmanifest": ("application/manifest+json", json.dumps(manifest).encode()), + "/ngsw-worker.js": ("text/javascript", b"/* ngsw Angular service-worker fixture */ self.onfetch = () => {};"), + "/main.abc.js": ("application/javascript", b"console.log('fixture');"), + "/styles.abc.css": ("text/css", b"body { color: black; }"), + } + for size in (192, 512): + files[f"/assets/{size}.png"] = ("image/png", png(size)) + files["/ngsw.json"] = ("application/json", json.dumps({ + "configVersion": 1, "index": "/index.html", "assetGroups": [{"name": "app"}], + "hashTable": {path: hashlib.sha1(body).hexdigest() for path, (_, body) in files.items() + if path != "/ngsw-worker.js"}, + }).encode()) + return {path: {"status": 200, "body": body, "headers": { + "Content-Type": kind, "Cache-Control": "no-cache, max-age=0, must-revalidate"}} + for path, (kind, body) in files.items()} + + +@contextmanager +def server(files): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + response = files.get(self.path, files.get(urlsplit(self.path).path)) + if response is None: + self.send_error(404) + return + self.send_response(response["status"]) + for key, value in response["headers"].items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(response["body"]) + + def log_message(self, *args): + pass + + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=httpd.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{httpd.server_port}" + finally: + httpd.shutdown() + httpd.server_close() + thread.join() + + +class PwaVerificationTest(unittest.TestCase): + def setUp(self): + self.files = fixtures() + + def check(self): + with server(self.files) as origin: + pwa.Verifier(origin, timeout=2, report=lambda line: None).run() + + def reject(self, expected): + with self.assertRaisesRegex(pwa.VerificationError, expected): + self.check() + + def change_json(self, path, mutate): + obj = json.loads(self.files[path]["body"]) + mutate(obj) + self.files[path]["body"] = json.dumps(obj).encode() + + def test_complete_release_and_cli_pass(self): + self.check() + with server(self.files) as origin: + result = subprocess.run([sys.executable, "-B", str(SCRIPT), origin, "--timeout", "2"], + capture_output=True, text=True) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("public-asset checks passed", result.stdout) + + def test_https_except_loopback_and_root_origin_only(self): + for origin in ("https://ontrack.example.edu", "http://localhost:8080", + "http://127.0.0.1:8080", "http://[::1]:8080"): + with self.subTest(origin=origin): + self.assertTrue(pwa.origin_url(origin).endswith("/")) + for origin in ("http://ontrack.example.edu", "http://192.168.1.1", "file:///tmp/site", + "https://name:password@example.edu", "https://example.edu/app", + "https://example.edu?token=secret", "https://example.edu#fragment", + "https://example.edu:99999", "https://example.edu:0", "https://example.edu\\@other.edu"): + with self.subTest(origin=origin), self.assertRaises(pwa.VerificationError): + pwa.origin_url(origin) + + def test_identity_start_scope_and_display_are_preserved(self): + for field, wrong in (("id", "/"), ("start_url", "/home"), ("scope", "/home/"), + ("display", "browser"), ("id", "https://other.example/id")): + with self.subTest(field=field, value=wrong): + self.files = fixtures() + self.change_json("/manifest.webmanifest", lambda obj: obj.update({field: wrong})) + self.reject("manifest|origin") + + def test_missing_manifest_link_fails(self): + self.files["/index.html"]["body"] = self.files["/index.html"]["body"].replace(b'rel="manifest"', b'rel="alternate"') + self.reject("must link") + + def test_spa_fallback_under_asset_urls_fails(self): + for path in ("/manifest.webmanifest", "/ngsw.json", "/ngsw-worker.js", "/assets/192.png", "/main.abc.js"): + with self.subTest(path=path): + self.files = fixtures() + self.files[path] = copy.deepcopy(self.files["/index.html"]) + self.reject("Content-Type") + + def test_html_disguised_as_javascript_fails(self): + self.files["/ngsw-worker.js"]["body"] = b"ngsw" + self.reject("resolved to HTML") + + def test_missing_worker_http_errors_and_redirects_fail(self): + for status in (301, 302, 404, 503): + with self.subTest(status=status): + self.files = fixtures() + self.files["/ngsw-worker.js"]["status"] = status + self.files["/ngsw-worker.js"]["headers"]["Location"] = "https://other.example/worker.js" + self.reject("redirected|HTTP") + + def test_stale_cache_policy_and_query_bypass_fail(self): + for policy in ("public, max-age=3600", "no-cache, s-maxage=600", "no-store, immutable", "public"): + with self.subTest(policy=policy): + self.files = fixtures() + self.files["/ngsw.json"]["headers"]["Cache-Control"] = policy + self.reject("cache|Cache-Control") + self.files = fixtures() + path = "/ngsw-worker.js?ontrack-pwa-check=1" + self.files[path] = copy.deepcopy(self.files["/ngsw-worker.js"]) + self.files[path]["headers"]["Cache-Control"] = "public, max-age=86400" + self.reject("cache") + + def test_icon_content_not_just_extension_or_header(self): + for bad in (b"not a PNG", png(192)[:33], png(192)[:-1], png(192).replace(b"IDAT", b"JUNK")): + with self.subTest(length=len(bad)): + self.files = fixtures() + self.files["/assets/192.png"]["body"] = bad + self.reject("PNG") + + def test_header_dimensions_without_matching_pixels_fail(self): + data = bytearray(png(192)) + data[16:24] = struct.pack(">II", 512, 512) + data[29:33] = struct.pack(">I", zlib.crc32(data[12:29]) & 0xffffffff) + self.files["/assets/512.png"]["body"] = bytes(data) + self.reject("pixel data does not match") + + def test_declared_dimensions_and_general_purpose_sizes(self): + self.files["/assets/192.png"]["body"] = png(96) + self.reject("dimensions") + self.files = fixtures() + self.change_json("/manifest.webmanifest", lambda obj: obj["icons"][0].update({"purpose": "maskable"})) + self.reject("general-purpose") + + def test_cross_origin_icons_and_shortcuts_fail_before_fetch(self): + self.change_json("/manifest.webmanifest", lambda obj: obj["icons"][0].update({"src": "https://elsewhere.example/icon.png"})) + self.reject("origin") + self.files = fixtures() + self.change_json("/manifest.webmanifest", lambda obj: obj["shortcuts"][0].update({"url": "https://elsewhere.example/home"})) + self.reject("origin") + self.files = fixtures() + self.change_json("/manifest.webmanifest", lambda obj: obj["icons"][0].update({"src": "https://[broken/icon.png"})) + self.reject("invalid asset URL") + + def test_invalid_json_and_control_schema_fail(self): + self.files["/manifest.webmanifest"]["body"] = b"" + self.reject("invalid JSON") + self.files = fixtures() + self.change_json("/ngsw.json", lambda obj: obj.update({"configVersion": 2})) + self.reject("unsupported") + + def test_mixed_release_and_missing_ngsw_hashes_fail(self): + self.files["/main.abc.js"]["body"] = b"console.log('different release');" + self.reject("mismatched ngsw hash") + self.files = fixtures() + self.change_json("/ngsw.json", lambda obj: obj["hashTable"].pop("/manifest.webmanifest")) + self.reject("mismatched ngsw hash") + + def test_invalid_timeout_cli_fails(self): + result = subprocess.run([sys.executable, "-B", str(SCRIPT), "https://example.edu", "--timeout", "0"], + capture_output=True, text=True) + self.assertEqual(2, result.returncode) + self.assertIn("between 1 and 300", result.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/production/validate-teams-mappings.py b/production/validate-teams-mappings.py new file mode 100644 index 000000000..9705bedae --- /dev/null +++ b/production/validate-teams-mappings.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Validate the optional mapping configuration without printing its contents.""" + +import json +import re +import sys + +GUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.I) +CHANNEL = re.compile(r"19:[A-Za-z0-9_-]+@thread\.(?:tacv2|skype)") + + +def valid_guid(value): + return isinstance(value, str) and GUID.fullmatch(value) + + +def validate(): + raw = sys.stdin.read(100_001) + if len(raw.encode("utf-8")) > 100_000: + return False + mappings = json.loads(raw) + minimum = 1 if sys.argv[1] == "true" else 0 + if not isinstance(mappings, list) or not minimum <= len(mappings) <= 20: + return False + identities = set() + for mapping in mappings: + if not isinstance(mapping, dict): + return False + unit = mapping.get("unit_id") + team = mapping.get("team_id") + channel = mapping.get("channel_id") + publishers = mapping.get("publisher_ids") + if not ( + type(unit) is int and unit > 0 + and valid_guid(team) + and isinstance(channel, str) and len(channel) <= 256 + and CHANNEL.fullmatch(channel) + and mapping.get("student_visible") is True + and isinstance(publishers, list) and 1 <= len(publishers) <= 100 + and all(valid_guid(publisher) for publisher in publishers) + ): + return False + identity = (unit, team.lower(), channel) + if identity in identities: + return False + identities.add(identity) + return True + + +try: + accepted = validate() +except (ValueError, TypeError, KeyError, IndexError): + accepted = False +if not accepted: + raise SystemExit("Use up to 20 unique unit/channel mappings, each explicitly student-visible with approved staff publisher IDs; enabled import needs at least one mapping.") diff --git a/production/validate.sh b/production/validate.sh new file mode 100755 index 000000000..55537e0d9 --- /dev/null +++ b/production/validate.sh @@ -0,0 +1,441 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${1:-${SCRIPT_DIR}/.env.production}" +COMPOSE_FILE="${SCRIPT_DIR}/docker-compose.yml" + +fail() { + printf 'Production configuration error: %s\n' "$*" >&2 + exit 1 +} + +for command_name in docker env openssl stat; do + command -v "${command_name}" >/dev/null 2>&1 || fail "${command_name} is required" +done + +compose_version="$(env -i PATH="${PATH}" docker compose version --short 2>/dev/null)" || fail "Docker Compose v2.33.1 or newer is required" +compose_version="${compose_version#v}" +compose_version="${compose_version%%-*}" +IFS=. read -r compose_major compose_minor compose_patch <<< "${compose_version}" +[[ "${compose_major:-}" =~ ^[0-9]+$ && "${compose_minor:-}" =~ ^[0-9]+$ && "${compose_patch:-}" =~ ^[0-9]+$ ]] || fail "cannot parse Docker Compose version: ${compose_version}" +if (( compose_major < 2 || (compose_major == 2 && compose_minor < 33) || (compose_major == 2 && compose_minor == 33 && compose_patch < 1) )); then + fail "Docker Compose v2.33.1 or newer is required (found ${compose_version})" +fi + +[[ -f "${ENV_FILE}" ]] || fail "environment file not found: ${ENV_FILE}" +ENV_FILE="$(cd -- "$(dirname -- "${ENV_FILE}")" && pwd)/$(basename -- "${ENV_FILE}")" + +config_keys=() +config_values=() +line_number=0 +while IFS= read -r line || [[ -n "${line}" ]]; do + line_number=$((line_number + 1)) + line="${line%$'\r'}" + [[ "${line}" =~ ^[[:space:]]*$ ]] && continue + [[ "${line}" =~ ^[[:space:]]*# ]] && continue + [[ "${line}" == *=* ]] || fail "${ENV_FILE}:${line_number} is not KEY=VALUE" + + key="${line%%=*}" + value="${line#*=}" + [[ "${key}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || fail "${ENV_FILE}:${line_number} has an invalid key" + if (( ${#config_keys[@]} > 0 )); then + for existing_key in "${config_keys[@]}"; do + [[ "${existing_key}" != "${key}" ]] || fail "${ENV_FILE}:${line_number} repeats ${key}" + done + fi + + [[ "${value}" != [[:space:]]* && "${value}" != *[[:space:]] ]] || fail "${ENV_FILE}:${line_number} has leading or trailing whitespace" + if [[ "${key}" == DF_TEAMS_CHANNEL_MAPPINGS ]]; then + # This one value is a literal JSON array. Embedded double quotes are JSON, + # not shell syntax; never source or evaluate the environment file. + [[ "${value}" != *\'* ]] || fail "DF_TEAMS_CHANNEL_MAPPINGS must be a JSON array without surrounding quotes" + else + [[ "${value}" != *\"* && "${value}" != *\'* ]] || fail "${ENV_FILE}:${line_number} contains a quote; use an unquoted literal value" + fi + [[ "${value}" != *'#'* ]] || fail "${ENV_FILE}:${line_number} contains #; inline comments are not allowed" + [[ "${value}" != *'\'* ]] || fail "${ENV_FILE}:${line_number} contains a backslash; escaped values are not allowed" + [[ "${value}" != *'$'* ]] || fail "${ENV_FILE}:${line_number} contains a dollar sign; use literal generated values without Compose interpolation" + + config_keys+=("${key}") + config_values+=("${value}") +done < "${ENV_FILE}" + +get_value() { + local wanted_key="$1" + local index + VALUE="" + for ((index = 0; index < ${#config_keys[@]}; index++)); do + if [[ "${config_keys[${index}]}" == "${wanted_key}" ]]; then + VALUE="${config_values[${index}]}" + return 0 + fi + done + return 1 +} + +require_value() { + local key="$1" + get_value "${key}" || true + [[ -n "${VALUE}" ]] || fail "${key} must be set" +} + +reject_placeholder() { + local key="$1" + local lowered + require_value "${key}" + lowered="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" + case "${lowered}" in + *replace_me*|*replace-me*|*changeme*|*change-me*|*test-secret*|*unifoo*|pwd|password|*your-password*) + fail "${key} still contains a placeholder value" + ;; + esac +} + +require_https_url() { + local key="$1" + reject_placeholder "${key}" + [[ "${VALUE}" == https://* ]] || fail "${key} must use https://" +} + +file_mode() { + stat -Lc '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1" +} + +require_private_file() { + local path="$1" + local label="$2" + local mode + [[ -f "${path}" && -r "${path}" ]] || fail "${label} is not a readable file: ${path}" + mode="$(file_mode "${path}")" || fail "cannot inspect permissions for ${label}" + (( (8#${mode} & 077) == 0 )) || fail "${label} must not be accessible by group or other users (mode ${mode})" +} + +mode="$(file_mode "${ENV_FILE}")" || fail "cannot inspect environment file permissions" +(( (8#${mode} & 077) == 0 )) || fail "${ENV_FILE} must not be accessible by group or other users (mode ${mode})" + +require_value COMPOSE_PROJECT_NAME +[[ "${VALUE}" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "COMPOSE_PROJECT_NAME must use lowercase letters, numbers, underscores, or hyphens" + +require_value SERVER_NAME +SERVER_NAME="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" +[[ "${SERVER_NAME}" =~ ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$ ]] || fail "SERVER_NAME must be a production DNS hostname" +[[ "${SERVER_NAME}" != localhost && "${SERVER_NAME}" != *.local ]] || fail "SERVER_NAME cannot be localhost" + +require_value DF_INSTITUTION_HOST +[[ "${VALUE}" == "https://${SERVER_NAME}" ]] || fail "DF_INSTITUTION_HOST must be https://${SERVER_NAME}" + +require_value TZ +[[ -f "/usr/share/zoneinfo/${VALUE}" ]] || fail "TZ is not an installed IANA time zone: ${VALUE}" + +require_value CLIENT_MAX_BODY_SIZE +[[ "${VALUE}" =~ ^[1-9][0-9]*[mMgG]$ ]] || fail "CLIENT_MAX_BODY_SIZE must look like 100m or 1g" +# Task chat permits files below 30,000,000 bytes. Reserve multipart headroom; +# Rails still enforces its separate per-file and per-context limits. +body_size_number="${VALUE%?}" +body_size_unit="${VALUE: -1}" +if [[ "${body_size_unit}" == [mM] && ${#body_size_number} -le 2 ]]; then + (( 10#${body_size_number} >= 32 )) || fail "CLIENT_MAX_BODY_SIZE must be at least 32m for chat attachments and multipart overhead" +fi + +image_keys=( + PROXY_IMAGE + MARIADB_IMAGE + REDIS_IMAGE + DOCKER_SOCKET_PROXY_IMAGE + DOUBTFIRE_API_IMAGE + DOUBTFIRE_APP_IMAGE + DOUBTFIRE_WEB_IMAGE + TEXLIVE_IMAGE + JPLAG_IMAGE +) +for key in "${image_keys[@]}"; do + require_value "${key}" + [[ "${VALUE}" =~ ^[A-Za-z0-9._/:+-]+@sha256:[0-9a-fA-F]{64}$ ]] || fail "${key} must use an immutable sha256 image digest" +done + +get_value DOCKER_SOCKET_PROXY_IMAGE +[[ "${VALUE}" =~ (^|/)haproxy:[A-Za-z0-9._-]*alpine@sha256:[0-9a-fA-F]{64}$ ]] || fail "DOCKER_SOCKET_PROXY_IMAGE must pin an HAProxy Alpine image digest" + +directory_keys=( + STUDENT_WORK_PATH + DOUBTFIRE_LOG_PATH + MARIADB_DATA_PATH + REDIS_DATA_PATH + JPLAG_REPORT_PATH + JPLAG_ARCHIVE_REPORT_PATH +) +resolved_directories=() +for key in "${directory_keys[@]}"; do + require_value "${key}" + [[ "${VALUE}" == /* ]] || fail "${key} must be an absolute path" + [[ "${VALUE}" != / ]] || fail "${key} cannot use the filesystem root" + [[ "${VALUE}" != *:* ]] || fail "${key} cannot contain a colon" + [[ -d "${VALUE}" ]] || fail "${key} directory does not exist: ${VALUE}" + resolved_directory="$(cd -- "${VALUE}" && pwd -P)" + for existing_directory in "${resolved_directories[@]-}"; do + [[ "${existing_directory}" != "${resolved_directory}" ]] || fail "production data paths must use separate directories" + done + resolved_directories+=("${resolved_directory}") +done + +get_value STUDENT_WORK_PATH +student_work_directory="$(cd -- "${VALUE}" && pwd -P)" +for key in JPLAG_REPORT_PATH JPLAG_ARCHIVE_REPORT_PATH; do + get_value "${key}" + resolved_directory="$(cd -- "${VALUE}" && pwd -P)" + case "${resolved_directory}" in + "${student_work_directory}"/*) ;; + *) fail "${key} must be inside STUDENT_WORK_PATH so reports are included in backups" ;; + esac +done + +require_value DOCKER_SOCKET_PATH +[[ "${VALUE}" == /* ]] || fail "DOCKER_SOCKET_PATH must be an absolute path" +[[ "${VALUE}" == /var/run/docker.sock ]] || fail "DOCKER_SOCKET_PATH must be /var/run/docker.sock so deployment targets the validated local daemon" +if [[ "${DOUBTFIRE_CONFIG_ONLY:-0}" != 1 ]]; then + [[ -S "${VALUE}" && -r "${VALUE}" && -w "${VALUE}" ]] || fail "DOCKER_SOCKET_PATH is not an accessible Unix socket: ${VALUE}" +fi + +require_value TLS_CERTIFICATE_PATH +TLS_CERTIFICATE_PATH="${VALUE}" +[[ "${TLS_CERTIFICATE_PATH}" == /* ]] || fail "TLS_CERTIFICATE_PATH must be an absolute path" +[[ -f "${TLS_CERTIFICATE_PATH}" && -r "${TLS_CERTIFICATE_PATH}" ]] || fail "TLS certificate is not readable: ${TLS_CERTIFICATE_PATH}" + +require_value TLS_PRIVATE_KEY_PATH +TLS_PRIVATE_KEY_PATH="${VALUE}" +[[ "${TLS_PRIVATE_KEY_PATH}" == /* ]] || fail "TLS_PRIVATE_KEY_PATH must be an absolute path" +require_private_file "${TLS_PRIVATE_KEY_PATH}" "TLS private key" + +openssl x509 -in "${TLS_CERTIFICATE_PATH}" -noout >/dev/null 2>&1 || fail "TLS_CERTIFICATE_PATH is not a PEM certificate" +openssl x509 -in "${TLS_CERTIFICATE_PATH}" -noout -checkend 2592000 >/dev/null 2>&1 || fail "TLS certificate is expired or expires within 30 days" +openssl x509 -in "${TLS_CERTIFICATE_PATH}" -noout -checkhost "${SERVER_NAME}" >/dev/null 2>&1 || fail "TLS certificate does not cover ${SERVER_NAME}" + +if ! certificate_public_key="$(openssl x509 -in "${TLS_CERTIFICATE_PATH}" -pubkey -noout 2>/dev/null | openssl pkey -pubin -outform DER 2>/dev/null | openssl dgst -sha256)"; then + fail "cannot read the TLS certificate public key" +fi +if ! private_public_key="$(openssl pkey -in "${TLS_PRIVATE_KEY_PATH}" -passin pass: -pubout -outform DER 2>/dev/null | openssl dgst -sha256)"; then + fail "TLS private key must be a readable, unencrypted PEM key" +fi +[[ -n "${certificate_public_key}" && "${certificate_public_key}" == "${private_public_key}" ]] || fail "TLS certificate and private key do not match" + +for key in ALIASES_PATH MSMTPRC_PATH; do + require_value "${key}" + [[ "${VALUE}" == /* ]] || fail "${key} must be an absolute path" + [[ -s "${VALUE}" && -r "${VALUE}" ]] || fail "${key} is not a readable, non-empty file: ${VALUE}" + if grep -Eiq 'REPLACE_ME|support@email\.address|smtp\.server' "${VALUE}"; then + fail "${key} still contains placeholder configuration" + fi +done +require_value MSMTPRC_PATH +require_private_file "${VALUE}" "msmtp configuration" + +for key in MARIADB_DATABASE MARIADB_USER; do + reject_placeholder "${key}" + [[ "${VALUE}" =~ ^[A-Za-z0-9_]+$ ]] || fail "${key} may contain only letters, numbers, and underscores" +done +get_value MARIADB_USER +lowered="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" +[[ "${lowered}" != root ]] || fail "MARIADB_USER cannot be root" +get_value MARIADB_DATABASE +lowered="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" +case "${lowered}" in + information_schema|mysql|performance_schema|sys) + fail "MARIADB_DATABASE cannot use a MariaDB system database" + ;; +esac +for key in MARIADB_PASSWORD MARIADB_ROOT_PASSWORD; do + reject_placeholder "${key}" + (( ${#VALUE} >= 20 )) || fail "${key} must contain at least 20 characters" + [[ "${VALUE}" != *'$'* ]] || fail "${key} cannot contain a dollar sign; use a generated hexadecimal value" +done +get_value MARIADB_PASSWORD +database_password="${VALUE}" +get_value MARIADB_ROOT_PASSWORD +[[ "${database_password}" != "${VALUE}" ]] || fail "database user and root passwords must be different" + +secret_keys=( + DF_SECRET_KEY_BASE + DF_SECRET_KEY_ATTR + DF_SECRET_KEY_DEVISE + DF_ENCRYPTION_PRIMARY_KEY + DF_ENCRYPTION_DETERMINISTIC_KEY + DF_ENCRYPTION_KEY_DERIVATION_SALT +) +secret_values=() +for key in "${secret_keys[@]}"; do + reject_placeholder "${key}" + (( ${#VALUE} >= 32 )) || fail "${key} must contain at least 32 characters" + [[ "${VALUE}" != *'$'* ]] || fail "${key} cannot contain a dollar sign; use a generated hexadecimal value" + for existing_secret in "${secret_values[@]-}"; do + [[ "${existing_secret}" != "${VALUE}" ]] || fail "Rails signing and encryption keys must use independent values" + done + secret_values+=("${VALUE}") +done + +require_value DF_SIDEKIQ_CONCURRENCY +[[ "${VALUE}" =~ ^[2-5]$ ]] || fail "DF_SIDEKIQ_CONCURRENCY must be an integer from 2 to 5" + +require_value DF_PPI_MINIMUM_COHORT_SIZE +[[ "${VALUE}" =~ ^[1-9][0-9]*$ ]] || fail "DF_PPI_MINIMUM_COHORT_SIZE must be a positive integer" +ppi_minimum_cohort_size="${VALUE}" +if (( ${#ppi_minimum_cohort_size} < 2 )) || + (( ${#ppi_minimum_cohort_size} == 2 && 10#${ppi_minimum_cohort_size} < 21 )); then + fail "DF_PPI_MINIMUM_COHORT_SIZE must be at least the privacy floor of 21" +fi + +require_value DF_PPI_STALE_AFTER_HOURS +[[ "${VALUE}" =~ ^[1-9][0-9]*$ ]] || fail "DF_PPI_STALE_AFTER_HOURS must be a positive integer" +(( ${#VALUE} <= 2 )) || fail "DF_PPI_STALE_AFTER_HOURS must not exceed the approved maximum of 48 hours" +(( 10#${VALUE} <= 48 )) || fail "DF_PPI_STALE_AFTER_HOURS must not exceed the approved maximum of 48 hours" + +legacy_vapid_public_key='BOs-KbIoHK7gUIX3i2_uEuDoouj-GKxB-mY9CRmLNmd4Wn-SSl254E1g6jR1ukL3e37p8uCpaMjOvfAB0BwzvSI=' +legacy_vapid_private_key='_NFIWSUTdCdLJJFh87pf4ekQLmNYqsweZ4288NpVZaY=' + +reject_placeholder DOUBTFIRE_VAPID_PUBLIC_KEY +[[ "${VALUE}" =~ ^B[A-Za-z0-9_-]{86}=?$ ]] || fail "DOUBTFIRE_VAPID_PUBLIC_KEY must be a base64url P-256 public key" +[[ "${VALUE}" != "${legacy_vapid_public_key}" ]] || fail "DOUBTFIRE_VAPID_PUBLIC_KEY must not reuse the checked-in development key" + +reject_placeholder DOUBTFIRE_VAPID_PRIVATE_KEY +[[ "${VALUE}" =~ ^[A-Za-z0-9_-]{43}=?$ ]] || fail "DOUBTFIRE_VAPID_PRIVATE_KEY must be a base64url P-256 private key" +[[ "${VALUE}" != "${legacy_vapid_private_key}" ]] || fail "DOUBTFIRE_VAPID_PRIVATE_KEY must not reuse the checked-in development key" + +reject_placeholder DOUBTFIRE_VAPID_SUBJECT +vapid_subject="${VALUE}" +case "${vapid_subject}" in + mailto:*) + [[ "${vapid_subject}" =~ ^mailto:[^[:space:]@]+@([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}$ ]] || fail "DOUBTFIRE_VAPID_SUBJECT mailto: value must contain an operational email address" + ;; + https://*) + [[ "${vapid_subject}" =~ ^https://[^[:space:]]+$ ]] || fail "DOUBTFIRE_VAPID_SUBJECT HTTPS value is invalid" + ;; + *) + fail "DOUBTFIRE_VAPID_SUBJECT must use mailto: or https://" + ;; +esac +lowered="$(printf '%s' "${vapid_subject}" | tr '[:upper:]' '[:lower:]')" +case "${lowered}" in + *localhost*|*.local|*.local/*|*.invalid|*.invalid/*|*.example|*.example/*|*.test|*.test/*) + fail "DOUBTFIRE_VAPID_SUBJECT must use an operational production contact" + ;; +esac + +for key in DF_INSTITUTION_PRODUCT_NAME DF_INSTITUTION_EMAIL_DOMAIN DF_INSTITUTION_EMAIL_SENDER DF_INSTITUTION_NAME; do + reject_placeholder "${key}" +done + +require_value DF_AUTH_METHOD +auth_method="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" +case "${auth_method}" in + aaf) + for key in DF_AAF_ISSUER_URL DF_AAF_AUDIENCE_URL DF_AAF_CALLBACK_URL DF_AAF_IDENTITY_PROVIDER_URL DF_AAF_UNIQUE_URL DF_AAF_AUTH_SIGNOUT_URL; do + require_https_url "${key}" + done + reject_placeholder DF_SECRET_KEY_AAF + (( ${#VALUE} >= 32 )) || fail "DF_SECRET_KEY_AAF must contain at least 32 characters" + ;; + saml) + for key in DF_SAML_METADATA_URL DF_SAML_CONSUMER_SERVICE_URL DF_SAML_SP_ENTITY_ID DF_SAML_IDP_TARGET_URL; do + require_https_url "${key}" + done + reject_placeholder DF_SAML_IDP_SAML_NAME_IDENTIFIER_FORMAT + ;; + ldap) + for key in DF_LDAP_HOST DF_LDAP_PORT DF_LDAP_ATTRIBUTE DF_LDAP_BASE; do + reject_placeholder "${key}" + done + require_value DF_LDAP_SSL + lowered="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" + [[ "${lowered}" == true ]] || fail "DF_LDAP_SSL must be true in production" + require_value DF_LDAP_USE_ADMIN_TO_BIND + lowered="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" + case "${lowered}" in + true) + reject_placeholder DF_LDAP_ADMIN_USER + reject_placeholder DF_LDAP_ADMIN_PWD + ;; + false) ;; + *) fail "DF_LDAP_USE_ADMIN_TO_BIND must be true or false" ;; + esac + ;; + database) + fail "DF_AUTH_METHOD=database is development-only; configure aaf, saml, or ldap" + ;; + *) + fail "DF_AUTH_METHOD must be aaf, saml, or ldap" + ;; +esac + +require_value DF_MAIL_PERFORM_DELIVERIES +lowered="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" +[[ "${lowered}" == yes ]] || fail "DF_MAIL_PERFORM_DELIVERIES must be yes for production notifications" +require_value DF_MAIL_DELIVERY_METHOD +lowered="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" +[[ "${lowered}" == smtp ]] || fail "DF_MAIL_DELIVERY_METHOD must be smtp" +for key in DF_SMTP_ADDRESS DF_SMTP_PORT DF_SMTP_DOMAIN; do + reject_placeholder "${key}" +done +require_value DF_INSTITUTION_EMAIL_SENDER +[[ "${VALUE}" =~ ^[^[:space:]@]+@[^[:space:]@]+\.[^[:space:]@]+$ ]] || fail "DF_INSTITUTION_EMAIL_SENDER must be an email address" +get_value DF_SMTP_PORT +smtp_port="${VALUE}" +[[ "${smtp_port}" =~ ^[0-9]+$ ]] && (( smtp_port >= 1 && smtp_port <= 65535 )) || fail "DF_SMTP_PORT must be between 1 and 65535" +get_value DF_SMTP_AUTHENTICATION || true +smtp_auth="${VALUE}" +lowered="$(printf '%s' "${smtp_auth}" | tr '[:upper:]' '[:lower:]')" +if [[ -n "${smtp_auth}" && "${lowered}" != none ]]; then + reject_placeholder DF_SMTP_USERNAME + reject_placeholder DF_SMTP_PASSWORD +fi + +for key in TEXLIVE_MEMORY_LIMIT JPLAG_MEMORY_LIMIT; do + require_value "${key}" + [[ "${VALUE}" =~ ^[1-9][0-9]*[mMgG]$ ]] || fail "${key} must be a positive whole number of megabytes or gigabytes" +done +for key in TEXLIVE_CPU_LIMIT JPLAG_CPU_LIMIT; do + require_value "${key}" + [[ "${VALUE}" =~ ^(([1-9][0-9]*)(\.[0-9]+)?|0\.[0-9]*[1-9][0-9]*)$ ]] || fail "${key} must be a positive CPU count" +done +for key in TEXLIVE_PIDS_LIMIT JPLAG_PIDS_LIMIT; do + require_value "${key}" + [[ "${VALUE}" =~ ^[1-9][0-9]*$ ]] || fail "${key} must be a positive integer" +done + +get_value OVERSEER_ENABLED || true +lowered="$(printf '%s' "${VALUE}" | tr '[:upper:]' '[:lower:]')" +case "${lowered}" in + ""|0|false|no) ;; + *) fail "OVERSEER_ENABLED must remain disabled; this stack does not provide its runner contract" ;; +esac + +get_value DF_TEAMS_ANNOUNCEMENTS_ENABLED || true +teams_enabled="${VALUE:-false}" +case "${teams_enabled}" in + true|false) ;; + *) fail "DF_TEAMS_ANNOUNCEMENTS_ENABLED must be true or false" ;; +esac +get_value DF_TEAMS_CHANNEL_MAPPINGS || true +teams_mappings="${VALUE:-[]}" +if [[ "${teams_enabled}" == true || "${teams_mappings}" != '[]' ]]; then + command -v python3 >/dev/null 2>&1 || fail "python3 is required for Teams mapping validation" + if ! printf '%s' "${teams_mappings}" | python3 "${SCRIPT_DIR}/validate-teams-mappings.py" "${teams_enabled}"; then + fail "Teams channel mappings are invalid" + fi +fi +if [[ "${teams_enabled}" == true ]]; then + for key in DF_TEAMS_TENANT_ID DF_TEAMS_CLIENT_ID; do + reject_placeholder "${key}" + [[ "${VALUE}" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]] || fail "${key} must be a Microsoft directory GUID" + done + reject_placeholder DF_TEAMS_CLIENT_SECRET + (( ${#VALUE} <= 4096 )) || fail "DF_TEAMS_CLIENT_SECRET is too long" +fi + +[[ -r "${SCRIPT_DIR}/docker-socket-proxy.cfg" ]] || fail "docker-socket-proxy.cfg is missing or unreadable" + +if ! env -i PATH="${PATH}" docker compose --env-file "${ENV_FILE}" --file "${COMPOSE_FILE}" config --quiet; then + fail "Docker Compose rejected the rendered production configuration" +fi + +printf 'Production configuration is valid for %s.\n' "${SERVER_NAME}" diff --git a/production/verify-pwa.py b/production/verify-pwa.py new file mode 100755 index 000000000..38eb21698 --- /dev/null +++ b/production/verify-pwa.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Read-only public-asset checks for OnTrack's root-hosted desktop PWA. + +Uses only Python's standard library. No login, API calls, browser installation, +server changes, or TLS bypass. A passing result still needs browser acceptance. +""" + +import argparse +import hashlib +from html.parser import HTMLParser +import ipaddress +import json +import re +import struct +import sys +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlsplit, urlunsplit +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener +import zlib + + +MAX_RESPONSE = 32 * 1024 * 1024 +JS_TYPES = {"application/javascript", "text/javascript"} + + +class VerificationError(Exception): + pass + + +def require(condition, message): + if not condition: + raise VerificationError(message) + + +def origin_url(value): + """Accept a single root origin; HTTP is limited to local browser testing.""" + try: + parsed = urlsplit(value) + port = parsed.port + host = (parsed.hostname or "").lower().rstrip(".") + require(bool(host) and not parsed.username and not parsed.password, + "provide a URL with a hostname and no credentials") + require(not re.search(r"[\s\\]", value), "URL contains whitespace or a backslash") + require(parsed.path in ("", "/") and not parsed.query and not parsed.fragment, + "provide the root origin, without a path, query, or fragment") + try: + loopback = ipaddress.ip_address(host).is_loopback + except ValueError: + loopback = host == "localhost" + require(parsed.scheme == "https" or (parsed.scheme == "http" and loopback), + "HTTPS is required except for localhost or a loopback IP") + # Access .port above to validate malformed/out-of-range ports. + require(port is None or port > 0, "URL port must be positive") + return urlunsplit((parsed.scheme, parsed.netloc, "/", "", "")) + except ValueError as error: + raise VerificationError("invalid origin URL") from error + + +def same_origin(base, reference): + require(isinstance(reference, str) and bool(reference), "asset URL must be a non-empty string") + require(not re.search(r"[\s\\]", reference), "asset URL contains whitespace or a backslash") + try: + resolved = urljoin(base, reference) + parsed, expected = urlsplit(resolved), urlsplit(base) + parsed.port + except ValueError as error: + raise VerificationError("invalid asset URL") from error + require(not parsed.username and not parsed.password, "asset URL contains credentials") + require((parsed.scheme, parsed.netloc) == (expected.scheme, expected.netloc), + "PWA assets and routes must stay on the supplied origin") + return resolved + + +def check_cache(headers, label): + value = headers.get("Cache-Control", "").lower() + directives = {part.strip().split("=", 1)[0] for part in value.split(",")} + ages = re.findall(r"(?:^|,)\s*(?:s-maxage|max-age)\s*=\s*\"?(\d+)", value) + require("immutable" not in directives and all(int(age) == 0 for age in ages), + f"{label}: control files must not have a positive cache lifetime or immutable") + require("no-store" in directives or "no-cache" in directives or + ("must-revalidate" in directives and bool(ages)), + f"{label}: Cache-Control must prevent stale control files") + + +def png_dimensions(data, label): + """Check PNG chunks, CRCs and decompressed scanlines, not just a file name.""" + require(data.startswith(b"\x89PNG\r\n\x1a\n"), f"{label}: expected PNG signature") + position, chunks, compressed = 8, [], bytearray() + width = height = depth = colour = interlace = None + while position + 12 <= len(data): + length = struct.unpack(">I", data[position:position + 4])[0] + kind = data[position + 4:position + 8] + end = position + 12 + length + require(end <= len(data), f"{label}: truncated PNG chunk") + payload = data[position + 8:end - 4] + checksum = struct.unpack(">I", data[end - 4:end])[0] + require(zlib.crc32(kind + payload) & 0xffffffff == checksum, + f"{label}: invalid PNG chunk checksum") + if kind == b"IHDR": + require(not chunks and length == 13, f"{label}: invalid PNG header") + width, height, depth, colour, compression, filtering, interlace = struct.unpack( + ">IIBBBBB", payload) + valid_depths = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), + 4: (8, 16), 6: (8, 16)} + require(0 < width <= 4096 and 0 < height <= 4096 and + depth in valid_depths.get(colour, ()) and compression == 0 and + filtering == 0 and interlace in (0, 1), f"{label}: unsupported PNG header") + elif kind == b"IDAT": + compressed.extend(payload) + elif kind == b"IEND": + require(length == 0 and end == len(data), f"{label}: invalid PNG end") + chunks.append(kind) + position = end + require(position == len(data) and chunks and chunks[0] == b"IHDR" and + chunks[-1] == b"IEND" and compressed, f"{label}: incomplete PNG") + require(colour != 3 or b"PLTE" in chunks, f"{label}: indexed PNG has no palette") + channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[colour] + passes = [(0, 0, 1, 1)] if not interlace else [ + (0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), + (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)] + rows = [] + for x, y, dx, dy in passes: + columns = max(0, (width - x + dx - 1) // dx) + count = max(0, (height - y + dy - 1) // dy) + if columns: + rows.extend([1 + (columns * channels * depth + 7) // 8] * count) + expected = sum(rows) + require(expected <= MAX_RESPONSE, f"{label}: decoded PNG exceeds 32 MiB") + try: + decoder = zlib.decompressobj() + pixels = decoder.decompress(compressed, expected + 1) + require(len(pixels) == expected and decoder.eof and not decoder.unused_data, + f"{label}: PNG pixel data does not match its dimensions") + except zlib.error as error: + raise VerificationError(f"{label}: invalid PNG pixel data") from error + offset = 0 + for row in rows: + require(pixels[offset] <= 4, f"{label}: invalid PNG scanline filter") + offset += row + return width, height + + +class IndexLinks(HTMLParser): + def __init__(self): + super().__init__() + self.manifests, self.scripts, self.styles, self.bases = [], [], [], [] + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if tag == "link" and "manifest" in attrs.get("rel", "").lower().split(): + self.manifests.append(attrs.get("href")) + if tag == "link" and "stylesheet" in attrs.get("rel", "").lower().split(): + self.styles.append(attrs.get("href")) + if tag == "script" and attrs.get("src"): + self.scripts.append(attrs["src"]) + if tag == "base": + self.bases.append(attrs.get("href")) + + +class NoRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + fp.close() + raise VerificationError("asset redirected; use the final public origin and serve assets directly") + + +class Verifier: + def __init__(self, origin, timeout=30, report=print): + self.origin = origin_url(origin) + self.timeout = timeout + self.report = report + self.downloaded = {} + self.opener = build_opener(ProxyHandler({}), NoRedirects()) + + def fetch(self, reference, types, control=False): + url = same_origin(self.origin, reference) + label = urlsplit(url).path + request = Request(url, headers={"User-Agent": "OnTrack-PWA-Verifier/1.0", + "Accept-Encoding": "identity"}) + try: + with self.opener.open(request, timeout=self.timeout) as response: + require(response.status == 200, f"{label}: expected HTTP 200") + require(response.headers.get_content_type() in types, + f"{label}: incorrect Content-Type (possible SPA fallback)") + require(response.headers.get("Content-Encoding", "identity") == "identity", + f"{label}: server ignored Accept-Encoding: identity") + if control: + check_cache(response.headers, label) + data = response.read(MAX_RESPONSE + 1) + require(0 < len(data) <= MAX_RESPONSE, f"{label}: empty or oversized response") + except HTTPError as error: + error.close() + raise VerificationError(f"{label}: HTTP {error.code}, expected 200") from error + except (URLError, OSError) as error: + raise VerificationError(f"{label}: request failed ({error.reason if isinstance(error, URLError) else error})") from error + if types & JS_TYPES: + require(not re.search(br")", data[:4096], re.I), + f"{label}: JavaScript resolved to HTML") + self.downloaded[label] = data + self.report(f"ok - {label}") + return data + + def json(self, reference, types): + try: + parsed = json.loads(self.fetch(reference, types, control=True)) + except (ValueError, UnicodeError) as error: + raise VerificationError(f"{reference}: invalid JSON") from error + require(isinstance(parsed, dict), f"{reference}: expected JSON object") + return parsed + + def run(self): + index = self.fetch("/index.html", {"text/html"}, control=True) + require(re.search(br")", index, re.I), "index is not HTML") + links = IndexLinks() + try: + links.feed(index.decode("utf-8")) + except UnicodeError as error: + raise VerificationError("index is not UTF-8 HTML") from error + require(links.bases == ["/"], "index must declare one root base href") + require(len(links.manifests) == 1 and + same_origin(self.origin, links.manifests[0]) == self.origin + "manifest.webmanifest", + "index must link /manifest.webmanifest") + manifest = self.json("/manifest.webmanifest", {"application/manifest+json", "application/json"}) + for field in ("name", "short_name", "description"): + require(isinstance(manifest.get(field), str) and bool(manifest[field].strip()), + f"manifest must include {field}") + require(manifest.get("display") == "standalone", "manifest display must be standalone") + for field, path in (("id", "/index.html"), ("start_url", "/index.html"), ("scope", "/")): + require(same_origin(self.origin, manifest.get(field)) == self.origin.rstrip("/") + path, + f"manifest {field} must resolve to {path} to preserve existing installations") + icons = manifest.get("icons") + require(isinstance(icons, list) and 1 <= len(icons) <= 12, "manifest must include 1–12 icons") + general_sizes = set() + for icon in icons: + require(isinstance(icon, dict) and icon.get("type") == "image/png", "icon must declare image/png") + data = self.fetch(icon.get("src"), {"image/png"}) + width, height = png_dimensions(data, "manifest icon") + size = f"{width}x{height}" + require(size in str(icon.get("sizes", "")).split(), "PNG dimensions differ from manifest sizes") + purpose = icon.get("purpose", "any") + require(isinstance(purpose, str), "icon purpose must be a string") + if "any" in purpose.split(): + general_sizes.add(size) + require({"192x192", "512x512"} <= general_sizes, "manifest needs general-purpose 192px and 512px PNGs") + shortcuts = manifest.get("shortcuts", []) + require(isinstance(shortcuts, list) and len(shortcuts) <= 10, "invalid manifest shortcuts") + for shortcut in shortcuts: + require(isinstance(shortcut, dict) and isinstance(shortcut.get("name"), str) and + bool(shortcut["name"].strip()), "shortcut must have a name") + same_origin(self.origin, shortcut.get("url")) + worker = self.fetch("/ngsw-worker.js", JS_TYPES, control=True) + require(b"ngsw" in worker, "worker is missing the Angular service-worker runtime") + control = self.json("/ngsw.json", {"application/json"}) + require(control.get("configVersion") == 1 and control.get("index") == "/index.html" and + isinstance(control.get("assetGroups"), list) and bool(control["assetGroups"]), + "ngsw.json has an unsupported Angular worker configuration") + hashes = control.get("hashTable") + require(isinstance(hashes, dict) and bool(hashes), "ngsw.json has no asset hashes") + require(links.scripts and len(links.scripts) + len(links.styles) <= 24, + "index needs entry scripts and at most 24 entry assets") + for source in links.scripts: + self.fetch(source, JS_TYPES) + for source in links.styles: + parsed = urlsplit(urljoin(self.origin, source)) + if (parsed.scheme, parsed.netloc) == (urlsplit(self.origin).scheme, urlsplit(self.origin).netloc): + self.fetch(source, {"text/css"}) + else: + # Existing Google Fonts stylesheets are outside the app-shell + # hash table. No requests are made to third-party origins. + require(parsed.scheme == "https", "external stylesheet must use HTTPS") + self.report("skip - external stylesheet (browser acceptance required)") + # Check every fetched app-shell asset against the released worker manifest. + # Worker/control JSON intentionally are not members of their own hash table. + for path, data in self.downloaded.items(): + if path not in ("/ngsw-worker.js", "/ngsw.json"): + require(hashes.get(path) == hashlib.sha1(data).hexdigest(), + f"{path}: missing/mismatched ngsw hash; release may contain mixed assets") + for path in ("/index.html", "/manifest.webmanifest", "/ngsw.json", "/ngsw-worker.js"): + kind = {"/index.html": {"text/html"}, "/manifest.webmanifest": {"application/manifest+json", "application/json"}, + "/ngsw.json": {"application/json"}, "/ngsw-worker.js": JS_TYPES}[path] + original = self.downloaded[path] + query_data = self.fetch(path + "?ontrack-pwa-check=1", kind, control=True) + require(query_data == original, f"{path}: query string changed the released control file") + self.report("Desktop PWA public-asset checks passed. Complete the real-browser checks in DESKTOP-PWA.md.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("origin", help="public root origin, e.g. https://ontrack.example.edu") + parser.add_argument("--timeout", type=int, default=30, help="per-request timeout in seconds (1–300)") + args = parser.parse_args() + if not 1 <= args.timeout <= 300: + parser.error("timeout must be between 1 and 300 seconds") + try: + Verifier(args.origin, args.timeout).run() + except VerificationError as error: + print(f"Desktop PWA verification failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/production/verify.sh b/production/verify.sh new file mode 100755 index 000000000..6096ffc4b --- /dev/null +++ b/production/verify.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +umask 077 + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + printf 'Usage: %s [ENV_FILE] [TIMEOUT_SECONDS]\n' "$0" >&2 +} + +fail() { + printf 'Production verification failed: %s\n' "$*" >&2 + exit 1 +} + +(( $# <= 2 )) || { + usage + exit 2 +} + +ENV_FILE="${1:-${SCRIPT_DIR}/.env.production}" +VERIFY_TIMEOUT="${2:-30}" + +[[ "${VERIFY_TIMEOUT}" =~ ^[1-9][0-9]*$ ]] && + (( ${#VERIFY_TIMEOUT} <= 3 )) && + (( 10#${VERIFY_TIMEOUT} <= 300 )) || { + printf 'Verification timeout must be between 1 and 300 seconds.\n' >&2 + exit 2 +} + +[[ -f "${ENV_FILE}" ]] || fail "environment file not found: ${ENV_FILE}" + +for command_name in curl grep mktemp timeout; do + command -v "${command_name}" >/dev/null 2>&1 || fail "${command_name} is required" +done + +# Re-run the complete production preflight so this script cannot verify a live +# stack against an invalid, placeholder-bearing, or differently rendered file. +DOUBTFIRE_CONFIG_ONLY=0 "${SCRIPT_DIR}/validate.sh" "${ENV_FILE}" +ENV_FILE="$(cd -- "$(dirname -- "${ENV_FILE}")" && pwd)/$(basename -- "${ENV_FILE}")" + +read_environment_value() { + local wanted_key="$1" + local line + local key + + while IFS= read -r line || [[ -n "${line}" ]]; do + line="${line%$'\r'}" + [[ "${line}" == *=* ]] || continue + key="${line%%=*}" + if [[ "${key}" == "${wanted_key}" ]]; then + printf '%s' "${line#*=}" + return 0 + fi + done < "${ENV_FILE}" + return 1 +} + +SERVER_NAME="$(read_environment_value SERVER_NAME)" || fail "SERVER_NAME is missing after validation" +[[ -n "${SERVER_NAME}" ]] || fail "SERVER_NAME is empty after validation" + +compose=("${SCRIPT_DIR}/compose.sh" --env-file "${ENV_FILE}") +connect_timeout=10 +if (( VERIFY_TIMEOUT < connect_timeout )); then + connect_timeout="${VERIFY_TIMEOUT}" +fi + +temporary_directory="$(mktemp -d "${TMPDIR:-/tmp}/doubtfire-verify.XXXXXX")" || fail "cannot create a verification workspace" +cleanup() { + rm -rf -- "${temporary_directory}" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +curl_command=( + curl + --disable + --silent + --show-error + --noproxy '*' + --proto '=https' + --tlsv1.2 + --connect-timeout "${connect_timeout}" + --max-time "${VERIFY_TIMEOUT}" + --max-filesize 10485760 +) + +fetch_public() { + local label="$1" + local path="$2" + local output_file="$3" + local expected_content_type="${4:-}" + local metadata + local status + local content_type + + if ! metadata="$( + "${curl_command[@]}" \ + --output "${output_file}" \ + --write-out $'%{http_code}\n%{content_type}' \ + "https://${SERVER_NAME}${path}" + )"; then + fail "${label} request failed" + fi + status="${metadata%%$'\n'*}" + content_type="${metadata#*$'\n'}" + [[ "${status}" == 200 ]] || fail "${label} returned HTTP ${status}, expected 200" + if [[ "${output_file}" != /dev/null ]]; then + [[ -s "${output_file}" ]] || fail "${label} returned an empty response" + fi + case "${expected_content_type}" in + "") ;; + html) + [[ "${content_type}" == text/html* ]] || fail "${label} returned an unsafe Content-Type" + ;; + json) + [[ "${content_type}" == application/json* ]] || fail "${label} returned an unsafe Content-Type" + ;; + javascript) + [[ "${content_type}" == application/javascript* || "${content_type}" == text/javascript* ]] || fail "${label} returned an unsafe Content-Type" + ;; + *) + fail "${label} has an unknown verifier Content-Type contract" + ;; + esac + printf 'ok - %s\n' "${label}" +} + +run_compose_check() { + local label="$1" + shift + + if ! timeout --kill-after=5s "${VERIFY_TIMEOUT}s" "${compose[@]}" "$@" >/dev/null; then + fail "${label} failed" + fi + printf 'ok - %s\n' "${label}" +} + +fetch_public "public API readiness" /healthz /dev/null +fetch_public "public web readiness" /healthz/web /dev/null + +index_file="${temporary_directory}/index.html" +ngsw_file="${temporary_directory}/ngsw.json" +worker_file="${temporary_directory}/ngsw-worker.js" +fetch_public "public web index" /index.html "${index_file}" html +fetch_public "PWA control manifest" /ngsw.json "${ngsw_file}" json +fetch_public "PWA worker script" /ngsw-worker.js "${worker_file}" javascript + +grep -Eiq ' "AggregatePeerProgressJob", "poll_communication_set_schedules" => "PollCommunicationSetSchedulesJob", "send_new_task_available_notifications" => "SendNewTaskAvailableNotificationsJob", "send_due_soon_reminders" => "SendDueSoonRemindersJob" }; jobs = Sidekiq::Cron::Job.all; valid = expected.all? { |name, klass| job = jobs.find { |candidate| candidate.name == name }; job && job.klass == klass && job.status == "enabled" }; abort "Required Sidekiq cron jobs are missing, disabled, or mapped to the wrong class" unless valid' +run_compose_check \ + "Sidekiq process, concurrency, and required cron schedule" \ + exec -T sidekiq bundle exec rails runner "${sidekiq_check}" + +feature_configuration_check='begin; minimum = Integer(ENV.fetch("DF_PPI_MINIMUM_COHORT_SIZE"), 10); stale_after = Integer(ENV.fetch("DF_PPI_STALE_AFTER_HOURS"), 10); rescue KeyError, ArgumentError; abort "PPI configuration is missing or invalid"; end; api_floor = PeerProgressApi::MINIMUM_SAFE_COHORT_SIZE; abort "PPI configuration is below the API privacy floor" unless minimum >= 21 && minimum >= api_floor; abort "PPI freshness configuration is outside the approved production bounds" unless stale_after.between?(1, 48); vapid_keys = %w[DOUBTFIRE_VAPID_PUBLIC_KEY DOUBTFIRE_VAPID_PRIVATE_KEY DOUBTFIRE_VAPID_SUBJECT]; abort "VAPID configuration is missing" unless vapid_keys.all? { |key| !ENV[key].to_s.empty? }; abort "Web Push is not configured" unless PushNotificationService.configured?' +for service_name in apiserver sidekiq; do + run_compose_check \ + "live VAPID/PPI configuration for ${service_name}" \ + exec -T "${service_name}" bundle exec rails runner "${feature_configuration_check}" +done + +printf '\nAutomated production verification passed.\n' +cat <<'MANUAL_GATES' + +MANUAL GATES STILL REQUIRED BEFORE ACCEPTANCE: +- MANUAL-IDP: complete a real production identity-provider login and logout. +- MANUAL-SMTP: trigger a notification and confirm receipt through the approved SMTP service. +- MANUAL-PUSH: opt in on each supported real browser/device, receive a push in the background, and follow its link. +- MANUAL-PPI: verify compact and advanced fresh peer-only views, the default-on profile preference, and remaining-peer/legacy/viewer-stale/vector-suppressed views without recording student-level data. +- MANUAL-PDF-JPLAG: generate a PDF and complete an approved JPlag check. +- MANUAL-SCORM: use a disposable launch token to verify no-referrer and all edge/API/observability log controls, then revoke it. +- MANUAL-RECOVERY: prove the current backup can be restored and the release can be rolled back under the runbook. +MANUAL_GATES diff --git a/releases/v6.0.0/README.md b/releases/v6.0.0/README.md index fb7ce4357..6b19b83d1 100644 --- a/releases/v6.0.0/README.md +++ b/releases/v6.0.0/README.md @@ -1,5 +1,10 @@ # v6.0.0 +> **Historical release note — do not execute the token-export procedure +> below.** It writes reusable bearer tokens as executable plaintext. Current +> operators must follow [MIGRATING.md](../../MIGRATING.md), whose safe default is +> to invalidate the old sessions and require users to sign in again. + This release is a large shift in the deployment approach for Doubtfire, major upgrades to back end libraries, iterative enhancements to the front end, and new branding. The resulting product should be faster for all use cases, easier to deploy, easier to develop, and easier to upgrade going forward. We see this version as providing the foundation for the product going forward, and allowing us to more rapidly integrate new ideas and update backend components. Going forward, the next critical components for the project will be completing the migration of the front end. We aim to release this as components are updated as minor updates to v6. diff --git a/tools/audit-vapid-secret.py b/tools/audit-vapid-secret.py new file mode 100644 index 000000000..3817236be --- /dev/null +++ b/tools/audit-vapid-secret.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Find a supplied VAPID private key without printing its value or matching content.""" + +import argparse +import base64 +import gzip +import io +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys +import tarfile +import tempfile + + +class AuditError(Exception): + """A static, non-sensitive description of an incomplete audit.""" + + +class SafeArgumentParser(argparse.ArgumentParser): + def error(self, _message): + # argparse normally echoes unrecognized arguments, which could include + # a key mistakenly supplied on the command line. + self.exit(2, "Invalid audit arguments; use --help. Never pass a key value as an argument.\n") + + +def read_key(path, stdin): + if path: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as source: + info = os.fstat(source.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077: + raise AuditError("Key file must be a regular, owner-only file owned by this user") + value = source.read(4097).strip() + except OSError: + raise AuditError("Cannot open protected key file (symlinks are not accepted)") from None + else: + if stdin.isatty(): + raise AuditError("Pipe the key from its secret manager; interactive terminal input is disabled") + value = stdin.read(4097).strip() + if not re.fullmatch(rb"[A-Za-z0-9_-]{43}=?", value): + raise AuditError("Expected one base64url-encoded, 32-byte VAPID private key") + raw = base64.urlsafe_b64decode(value.rstrip(b"=") + b"=") + if len(raw) != 32: + raise AuditError("Expected one base64url-encoded, 32-byte VAPID private key") + return value.rstrip(b"="), raw + + +class Scanner: + def __init__(self, value, raw): + forms = {value, value + b"=", raw, raw.hex().encode(), raw.hex().upper().encode(), + base64.b64encode(raw), base64.b64encode(raw).rstrip(b"="), + base64.b64encode(value), base64.b64encode(value + b"=")} + self.patterns = sorted(forms, key=len, reverse=True) + self.overlap = max(map(len, self.patterns)) - 1 + self.report = {"format_version": 1, "complete_for_selected_inputs": True, + "coverage": "Local Git objects, supplied Docker archives/images, and explicit files only", + "targets": [], "images": [], "scanned_items": 0, + "scanned_bytes": 0, "findings": [], "errors": []} + + def redact(self, text): + for pattern in self.patterns: + try: + text = text.replace(pattern.decode("ascii"), "[REDACTED]") + except UnicodeDecodeError: + pass + return text + + def count(self, source, size=None): + tail, count, read = b"", 0, 0 + while size is None or read < size: + chunk = source.read(min(65536, size - read) if size is not None else 65536) + if not chunk: + if size is not None and read != size: + raise AuditError("Input ended before its declared size") + break + read += len(chunk) + data = tail + chunk + limit = max(0, len(data) - self.overlap) + count += self.count_starts(data, limit) + tail = data[limit:] + count += self.count_starts(tail, len(tail)) + return count, read + + def count_starts(self, data, limit): + starts = set() + for pattern in self.patterns: + position = data.find(pattern) + while 0 <= position < limit: + starts.add(position) + position = data.find(pattern, position + 1) + return len(starts) + + def scan(self, source, location, size=None): + matches, length = self.count(source, size) + self.report["scanned_items"] += 1 + self.report["scanned_bytes"] += length + if matches: + self.report["findings"].append({"location": self.redact(location), "matches": matches}) + + def scan_metadata(self, metadata, location): + self.scan(io.BytesIO(json.dumps(metadata, ensure_ascii=False).encode()), location) + + def git(self, repository): + result = subprocess.run(["git", "-C", repository, "rev-parse", "--is-shallow-repository"], + capture_output=True, check=False) + if result.returncode or result.stdout.strip() != b"false": + raise AuditError("Repository unavailable or shallow; obtain full authorized history first") + # All available objects includes deleted historical files, unreachable + # objects, commit/tag messages and alternate object databases. + process = subprocess.Popen(["git", "-C", repository, "cat-file", "--batch-all-objects", "--batch"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + try: + while header := process.stdout.readline(): + match = re.fullmatch(rb"([a-f0-9]{40,64}) (blob|tree|commit|tag) ([0-9]+)\n", header) + if not match: + raise AuditError("Git returned an invalid object stream") + object_id, kind, length = match.groups() + self.scan(process.stdout, f"git:{repository}:{kind.decode()}:{object_id.decode()}", int(length)) + if process.stdout.read(1) != b"\n": + raise AuditError("Git object stream was truncated") + if process.wait(): + raise AuditError("Git did not finish reading all objects") + finally: + if process.poll() is None: + process.kill() + process.wait() + process.stdout.close() + + def layer(self, source, location): + # Never unpack files to the filesystem or apply whiteouts. Each original + # layer is scanned separately, including files deleted by later layers. + with tarfile.open(fileobj=source, mode="r|*") as archive: + for member in archive: + item = f"{location}:{member.name}" + self.scan_metadata({"name": member.name, "link": member.linkname, + "user": member.uname, "group": member.gname, + "pax": member.pax_headers}, item + ":metadata") + if member.isfile(): + with archive.extractfile(member) as content: + self.scan(content, item, member.size) + + def image_archive(self, path, label=None): + location = "image:" + (label or str(path)) + with tarfile.open(path, mode="r:*") as archive: + manifest_info = archive.getmember("manifest.json") + if not manifest_info.isfile() or manifest_info.size > 16 * 1024 * 1024: + raise AuditError("Docker save manifest is missing or invalid") + with archive.extractfile(manifest_info) as source: + manifest = json.load(source) + if not isinstance(manifest, list) or not manifest: + raise AuditError("Expected a non-empty Docker image save archive") + layers, configs = set(), set() + for entry in manifest: + if not isinstance(entry, dict) or not isinstance(entry.get("Layers"), list): + raise AuditError("Docker save manifest has invalid layer references") + configs.add(entry["Config"]) + layers.update(entry["Layers"]) + found = set() + for member in archive: + self.scan_metadata({"name": member.name, "link": member.linkname, + "pax": member.pax_headers}, location + ":archive-metadata") + if not member.isfile(): + continue + with archive.extractfile(member) as source: + if member.name in layers: + self.layer(source, f"{location}:layer:{member.name}") + else: + # Image configuration includes Env and complete history + # (CreatedBy/build instructions), without docker history output. + self.scan(source, f"{location}:metadata:{member.name}", member.size) + found.add(member.name) + if (layers | configs) - found: + raise AuditError("Docker save archive is missing referenced configuration or layers") + + def image(self, reference): + if reference.startswith("-"): + raise AuditError("Invalid Docker image reference") + result = subprocess.run(["docker", "image", "inspect", "--format={{.Id}}", reference], + capture_output=True, check=False) + image_id = result.stdout.strip().decode("ascii") + if result.returncode or not re.fullmatch(r"sha256:[a-f0-9]{64}", image_id): + raise AuditError("Docker could not resolve the selected local image") + self.report["images"].append({"reference": self.redact(reference), "id": image_id}) + with tempfile.TemporaryDirectory(prefix="vapid-image-audit-") as directory: + path = Path(directory) / "image.tar" + result = subprocess.run(["docker", "image", "save", "--output", str(path), image_id], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + if result.returncode: + raise AuditError("Docker could not save the selected local image") + self.image_archive(path, reference) + + def file(self, path): + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise AuditError("Explicit log/source input must be a regular file, not a symlink") + opener = gzip.open if path.suffix == ".gz" else open + with opener(path, "rb") as source: + self.scan(source, f"file:{path}") + + def run_target(self, operation, target): + self.report["targets"].append({"kind": operation.__name__, "target": self.redact(str(target))}) + try: + operation(target) + except Exception: + # Never print exception text: parsers and Docker/Git errors can + # contain secret-bearing file names, metadata, or input fragments. + self.report["complete_for_selected_inputs"] = False + self.report["errors"].append({"target": self.redact(str(target)), + "message": "Input could not be fully scanned; verify format, access and full Git history"}) + + +def main(argv=None, stdin=None): + parser = SafeArgumentParser(description=__doc__) + key = parser.add_mutually_exclusive_group(required=True) + key.add_argument("--key-file", help="Owner-only file containing the private key; never the value itself") + key.add_argument("--key-stdin", action="store_true", help="Read private key from a non-interactive pipe") + parser.add_argument("--repo", action="append", default=[], help="Full local Git repository (repeatable)") + parser.add_argument("--image", action="append", default=[], help="Local Docker image ID/digest (repeatable)") + parser.add_argument("--image-archive", action="append", default=[], help="Archive from docker image save (repeatable)") + parser.add_argument("--file", action="append", default=[], help="Explicit source/log file, optionally gzip (repeatable)") + args = parser.parse_args(argv) + if not any((args.repo, args.image, args.image_archive, args.file)): + parser.error("Select at least one repository, image, image archive or file") + try: + value, raw = read_key(args.key_file, stdin or sys.stdin.buffer) + except (AuditError, OSError) as error: + message = str(error) if isinstance(error, AuditError) else "Cannot read protected key input" + print(json.dumps({"complete_for_selected_inputs": False, "error": message})) + return 2 + scanner = Scanner(value, raw) + for targets, operation in [(args.repo, scanner.git), (args.image, scanner.image), + (args.image_archive, scanner.image_archive), (args.file, scanner.file)]: + for target in targets: + scanner.run_target(operation, target) + report = scanner.report + report["match_count"] = sum(item["matches"] for item in report["findings"]) + print(scanner.redact(json.dumps(report, indent=2))) + return 2 if report["errors"] else 1 if report["findings"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/test_audit_vapid_secret.py b/tools/tests/test_audit_vapid_secret.py new file mode 100644 index 000000000..d3048dda9 --- /dev/null +++ b/tools/tests/test_audit_vapid_secret.py @@ -0,0 +1,188 @@ +import base64 +from contextlib import redirect_stderr, redirect_stdout +import gzip +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "audit-vapid-secret.py" +SPEC = importlib.util.spec_from_file_location("audit_vapid_secret", SCRIPT) +AUDIT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(AUDIT) +RAW = bytes(range(1, 33)) +KEY = base64.urlsafe_b64encode(RAW).rstrip(b"=") + + +def tar_bytes(files): + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w") as archive: + for name, content in files.items(): + info = tarfile.TarInfo(name) + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + return output.getvalue() + + +class AuditTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.key = self.root / "protected-key" + self.key.write_bytes(KEY + b"\n") + self.key.chmod(0o600) + + def cli(self, *args, key_args=None): + output, error = io.StringIO(), io.StringIO() + with redirect_stdout(output), redirect_stderr(error): + code = AUDIT.main([*(key_args or ["--key-file", str(self.key)]), *map(str, args)], + stdin=io.BytesIO(KEY + b"\n")) + text = output.getvalue() + error.getvalue() + self.assertNotIn(KEY.decode(), text) + self.assertNotIn(base64.b64encode(RAW).decode(), text) + return code, json.loads(output.getvalue()) + + def archive(self, name, *, leak=False, history=False, missing=False): + layer1 = tar_bytes({"private.env": b"VAPID=" + KEY if leak else b"public fixture"}) + layer2 = tar_bytes({".wh.private.env": b""}) + config = {"history": [{"created_by": "ENV VAPID=" + KEY.decode() if history else "COPY fixture"}]} + files = {"manifest.json": json.dumps([{"Config": "config.json", "Layers": ["one/layer.tar", "two/layer.tar"]}]).encode(), + "config.json": json.dumps(config).encode(), "one/layer.tar": layer1} + if not missing: + files["two/layer.tar"] = gzip.compress(layer2) + result = self.root / name + result.write_bytes(tar_bytes(files)) + return result + + def test_clean_archive_and_log_pass_with_counts(self): + log = self.root / "clean.log" + log.write_text("Notification delivered without credentials\n") + code, report = self.cli("--image-archive", self.archive("clean.tar"), "--file", log) + self.assertEqual(0, code) + self.assertTrue(report["complete_for_selected_inputs"]) + self.assertEqual(0, report["match_count"]) + self.assertGreater(report["scanned_items"], 0) + + def test_image_detects_deleted_layer_file_and_build_history(self): + code, report = self.cli("--image-archive", self.archive("leaking.tar", leak=True, history=True)) + self.assertEqual(1, code) + locations = [finding["location"] for finding in report["findings"]] + self.assertTrue(any("one/layer.tar:private.env" in path for path in locations)) + self.assertTrue(any("metadata:config.json" in path for path in locations)) + self.assertEqual(2, report["match_count"]) + + def test_missing_image_layer_is_incomplete_not_clean(self): + code, report = self.cli("--image-archive", self.archive("broken.tar", missing=True)) + self.assertEqual(2, code) + self.assertFalse(report["complete_for_selected_inputs"]) + self.assertEqual(1, len(report["errors"])) + + def test_streaming_boundary_counts_a_padded_key_once(self): + scanner = AUDIT.Scanner(KEY, RAW) + data = b"x" * (65536 - 20) + KEY + b"=" + b"end" + matches, length = scanner.count(io.BytesIO(data)) + self.assertEqual(1, matches) + self.assertEqual(len(data), length) + + def test_binary_and_encoded_forms_are_detected(self): + scanner = AUDIT.Scanner(KEY, RAW) + for form in [RAW, RAW.hex().encode(), base64.b64encode(RAW), base64.b64encode(KEY)]: + with self.subTest(length=len(form)): + self.assertEqual(1, scanner.count(io.BytesIO(form))[0]) + + def test_compressed_log_and_secret_bearing_location_are_redacted(self): + log = self.root / (KEY.decode() + ".log.gz") + log.write_bytes(gzip.compress(b"private value: " + KEY)) + code, report = self.cli("--file", log, key_args=["--key-stdin"]) + self.assertEqual(1, code) + self.assertIn("[REDACTED].log.gz", report["findings"][0]["location"]) + + def test_world_readable_key_file_is_rejected(self): + self.key.chmod(0o644) + code, report = self.cli("--file", self.key) + self.assertEqual(2, code) + self.assertFalse(report["complete_for_selected_inputs"]) + + def test_symlink_key_file_is_rejected(self): + link = self.root / "key-link" + link.symlink_to(self.key) + code, _ = self.cli("--file", self.key, key_args=["--key-file", str(link)]) + self.assertEqual(2, code) + + def test_unknown_argument_does_not_echo_a_mistaken_key_value(self): + output = io.StringIO() + with redirect_stderr(output), self.assertRaises(SystemExit) as error: + AUDIT.main(["--key-value", KEY.decode()]) + self.assertEqual(2, error.exception.code) + self.assertNotIn(KEY.decode(), output.getvalue()) + + def test_interactive_key_input_is_rejected(self): + class Terminal(io.BytesIO): + def isatty(self): + return True + with self.assertRaises(AUDIT.AuditError): + AUDIT.read_key(None, Terminal(KEY)) + + def test_malformed_archive_diagnostics_do_not_echo_secret_content(self): + archive = self.root / "invalid.tar" + archive.write_bytes(tar_bytes({"manifest.json": b"invalid " + KEY})) + code, report = self.cli("--image-archive", archive) + self.assertEqual(2, code) + self.assertFalse(report["complete_for_selected_inputs"]) + + def test_layer_paths_are_scanned_without_extracting_files(self): + scanner = AUDIT.Scanner(KEY, RAW) + archive = tar_bytes({"../../outside-audit": KEY}) + scanner.layer(io.BytesIO(archive), "synthetic-layer") + self.assertEqual(1, len(scanner.report["findings"])) + self.assertEqual(1, scanner.report["findings"][0]["matches"]) + self.assertFalse((self.root / "outside-audit").exists()) + + def test_unreadable_or_invalid_input_is_not_reported_clean(self): + code, report = self.cli("--file", self.root / "missing.log") + self.assertEqual(2, code) + self.assertFalse(report["complete_for_selected_inputs"]) + self.key.write_text("invalid " + KEY.decode()) + code, _ = self.cli("--file", self.key) + self.assertEqual(2, code) + + def test_git_scans_deleted_history_and_commit_messages(self): + repository = self.root / "repo" + repository.mkdir() + def git(*args): + return subprocess.check_output(["git", "-C", str(repository), "-c", "user.name=Audit Fixture", + "-c", "user.email=audit@example.invalid", *args], stderr=subprocess.DEVNULL) + git("init", "--quiet") + tracked = repository / "removed.env" + tracked.write_bytes(KEY) + git("add", "removed.env") + git("commit", "--quiet", "-m", "synthetic " + KEY.decode()) + tracked.unlink() + git("add", "-u") + git("commit", "--quiet", "-m", "remove synthetic fixture") + code, report = self.cli("--repo", repository) + self.assertEqual(1, code) + locations = [item["location"] for item in report["findings"]] + self.assertTrue(any(":blob:" in path for path in locations)) + self.assertTrue(any(":commit:" in path for path in locations)) + self.assertEqual(2, report["match_count"]) + + def test_shallow_repository_cannot_pass_the_audit(self): + repository = self.root / "empty-repo" + subprocess.run(["git", "init", "--quiet", str(repository)], check=True, capture_output=True) + # Git recognizes a shallow file as a shallow clone. No object values + # need to be invented or read from a production repository. + (repository / ".git/shallow").write_text(40 * "0" + "\n") + code, report = self.cli("--repo", repository) + self.assertEqual(2, code) + self.assertFalse(report["complete_for_selected_inputs"]) + + +if __name__ == "__main__": + unittest.main()