diff --git a/.github/workflows/portalbackend-integration-tests.yml b/.github/workflows/portalbackend-integration-tests.yml deleted file mode 100644 index 3a4ee1b..0000000 --- a/.github/workflows/portalbackend-integration-tests.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - platform-backend-integration-tests: - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v6 - - - name: Set up python - uses: actions/setup-python@v6 - with: - python-version: "3.10" - - - name: Install docker compose - uses: KengoTODA/actions-setup-docker-compose@v1 - with: - version: '5.1.0' - - - name: Docker compose version - run: docker compose version - - - name: Copy the env file - working-directory: ./deployment/dev/ - run: cp .env.example .env - - - name: Deploy the stack - working-directory: ./deployment/dev/ - run: bash start.sh - - - name: Get deployment status - run: docker ps - - - name: Install python dependencies - working-directory: ./deployment/dev/ - run: pip install -r requirements.txt - - - name: Run the integration tests - working-directory: ./deployment/dev/ - run: pytest diff --git a/deployment/dev/.env.example b/deployment/dev/.env.example index 19ed946..2ea1077 100644 --- a/deployment/dev/.env.example +++ b/deployment/dev/.env.example @@ -1,8 +1,8 @@ # Versions -EXAFLOW=1.1.0 -PLATFORM_BACKEND=9.3.0 -PLATFORM_UI=1.2.0 -MIP=9.1.0 +EXAFLOW=1.3.0 +PLATFORM_BACKEND=10.0.0_candidate +PLATFORM_UI=2.0.0_candidate +MIP=9.2.0 # Toggle authentication AUTHENTICATION=0 @@ -10,6 +10,11 @@ AUTHENTICATION=0 # Frontend base URL used for post-logout redirect. PLATFORM_UI_BASE_URL=http://localhost +# JupyterLab (published on 127.0.0.1:8888 only). +JUPYTER_TOKEN=dev +# Keycloak access token for the notebook client; required when AUTHENTICATION=1. +MIP_TOKEN= + # External Keycloak KEYCLOAK_AUTH_URL=https://iam.ebrains.eu/auth/ KEYCLOAK_REALM=MIP diff --git a/deployment/dev/README.md b/deployment/dev/README.md index 7373abe..212ba2f 100644 --- a/deployment/dev/README.md +++ b/deployment/dev/README.md @@ -34,14 +34,17 @@ ``` ./start.sh ``` - The script waits for `http://172.17.0.1:8080/services/data-models` and verifies 4 data models are loaded. + The script starts the MIP stack plus direct JupyterLab, then checks that the `dementia` data model is available. -4. To test if the MIP stack is properly setup run the 'test.sh': + Open: ``` - ./test.sh + http://localhost + http://localhost:8888/lab/tree/workspace/examples/feres_analysis.ipynb?token=dev ``` - -5. To stop the MIP stack run the 'stop.sh' script to stop all the containers: + + JupyterLab uses the compose backend URL `http://platform-backend:8080/services` inside Docker. It is published on `127.0.0.1` only; change `JUPYTER_TOKEN` in `.env` if the host is shared. With `AUTHENTICATION=1`, set `MIP_TOKEN` in `.env` to a Keycloak access token so the notebook client can call the backend. + +4. To stop the MIP stack run the 'stop.sh' script to stop all the containers: ``` ./stop.sh ``` diff --git a/deployment/dev/config/disabledAlgorithms.json b/deployment/dev/config/disabledAlgorithms.json deleted file mode 100644 index fe51488..0000000 --- a/deployment/dev/config/disabledAlgorithms.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/deployment/dev/docker-compose.yml b/deployment/dev/docker-compose.yml index 8528d5d..129d956 100644 --- a/deployment/dev/docker-compose.yml +++ b/deployment/dev/docker-compose.yml @@ -160,7 +160,7 @@ services: ### Exaflow ### ALGORITHM_UPDATE_INTERVAL: 30 # seconds EXAFLOW_URL: http://exaflow_controller:5000 - PLATFORM_UI_BASE_URL: "${PLATFORM_UI_BASE_URL}" + PLATFORM_UI_BASE_URL: "${PLATFORM_UI_BASE_URL:-http://localhost}" ### Keycloak (external) ### AUTHENTICATION: "${AUTHENTICATION}" KEYCLOAK_AUTH_URL: "${KEYCLOAK_AUTH_URL}" @@ -185,6 +185,20 @@ services: - platform-backend restart: unless-stopped + mip_jupyter: + image: hbpmip/mip-jupyter:0.0.1_candidate + ports: + - "127.0.0.1:8888:8888" + environment: + PLATFORM_BACKEND_URL: http://platform-backend:8080/services + MIP_TOKEN: "${MIP_TOKEN:-}" + JUPYTER_TOKEN: "${JUPYTER_TOKEN:-dev}" + MIP_NOTEBOOK: workspace/examples/feres_analysis.ipynb + JUPYTER_ENABLE_LAB: "yes" + depends_on: + - platform-backend + restart: unless-stopped + networks: default: ipam: diff --git a/deployment/dev/requirements.txt b/deployment/dev/requirements.txt deleted file mode 100644 index 288fdd8..0000000 --- a/deployment/dev/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytest~=9.0 -requests~=2.31 -click~=8.1 \ No newline at end of file diff --git a/deployment/dev/start.sh b/deployment/dev/start.sh index 35fcf61..9611987 100755 --- a/deployment/dev/start.sh +++ b/deployment/dev/start.sh @@ -1,29 +1,20 @@ #!/usr/bin/env bash set -euo pipefail -set -a -source ./.env -set +a - docker compose down -v docker compose up -d endpoint="http://172.17.0.1:8080/services/data-models" -max_attempts=30 - -for attempt in $(seq 1 "${max_attempts}"); do +for _ in {1..30}; do response="$(curl -fsS --max-time 5 "${endpoint}" 2>/dev/null || true)" - if printf '%s' "${response}" | grep -q '"code"[[:space:]]*:[[:space:]]*"dementia_longitudinal"' \ - && printf '%s' "${response}" | grep -q '"code"[[:space:]]*:[[:space:]]*"dementia"' \ - && printf '%s' "${response}" | grep -q '"code"[[:space:]]*:[[:space:]]*"mentalhealth"' \ - && printf '%s' "${response}" | grep -q '"code"[[:space:]]*:[[:space:]]*"tbi"' - then - echo "Data models check passed: expected codes are available." - echo "You can see the MIP at 172.17.0.1" + if printf "%s" "${response}" | grep -q "\"code\"[[:space:]]*:[[:space:]]*\"dementia\""; then + echo "Data model check passed: dementia is available." + echo "MIP UI: http://localhost" + echo "JupyterLab: http://localhost:8888/lab/tree/workspace/examples/feres_analysis.ipynb?token=" exit 0 fi sleep 2 done -echo "Error: expected data model codes were not found at ${endpoint}." >&2 +echo "Error: dementia data model was not found at ${endpoint}." >&2 exit 1 diff --git a/deployment/dev/test.sh b/deployment/dev/test.sh deleted file mode 100755 index 02fc77b..0000000 --- a/deployment/dev/test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "${script_dir}" - -if [[ ! -d .venv ]]; then - python3 -m venv .venv -fi - -.venv/bin/pip install -q -r requirements.txt -.venv/bin/pytest tests/ -v "$@" diff --git a/deployment/dev/tests/test_exaflow_post_experiment.py b/deployment/dev/tests/test_exaflow_post_experiment.py deleted file mode 100644 index 6ba9344..0000000 --- a/deployment/dev/tests/test_exaflow_post_experiment.py +++ /dev/null @@ -1,368 +0,0 @@ -import pytest -import json - -import requests - - -all_success_cases = [ - { - "algorithm": { - "parameters": [ - {"name": "dataset", "value": "dummy_tbi"}, - {"name": "filter", "value": ""}, - {"name": "pathology", "value": "tbi:0.1"}, - { - "name": "y", - "value": "pupil_reactivity_right_eye_result", - }, - ], - "name": "descriptive_stats", - }, - "name": "Descriptive analysis", - }, - { - "algorithm": { - "name": "pca", - "parameters": [ - { - "name": "y", - "value": "rightppplanumpolare,righthippocampus,lefthippocampus,rightamygdala,leftamygdala", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "edsd", - }, - {"name": "filter", "value": None}, - ], - }, - "name": "Principal component algorithm", - }, - { - "algorithm": { - "name": "pearson_correlation", - "parameters": [ - { - "name": "y", - "value": "rightsplsuperiorparietallobule,rightttgtransversetemporalgyrus,leftcaudate,leftocpoccipitalpole", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "edsd", - }, - {"name": "filter", "value": ""}, - {"name": "alpha", "value": "0.9529895484370635"}, - ], - }, - "name": "Pearson Correlation", - }, - { - "algorithm": { - "name": "anova_oneway", - "parameters": [ - {"name": "y", "value": "leftententorhinalarea"}, - {"name": "x", "value": "neurodegenerativescategories"}, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,ppmi", - }, - { - "name": "filter", - "value": '{"condition": "AND", "rules": [{"id": "dataset", "type": "string", "value": ["desd-synthdata", "ppmi"], "operator": "in"}, {"condition": "AND", "rules": [{"id": "neurodegenerativescategories", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftententorhinalarea", "type": "string", "operator": "is_not_null", "value": null}]}], "valid": true}', - }, - ], - }, - "name": "One Way Anova", - }, - { - "algorithm": { - "name": "linear_regression", - "parameters": [ - {"name": "y", "value": "rightcuncuneus"}, - { - "name": "x", - "value": "rightioginferioroccipitalgyrus,leftententorhinalarea,rightamygdala,leftmpogpostcentralgyrusmedialsegment,rightporgposteriororbitalgyrus,leftpoparietaloperculum,righttrifgtriangularpartoftheinferiorfrontalgyrus,rightmpogpostcentralgyrusmedialsegment,rightlateralventricle,rightmfcmedialfrontalcortex,rightorifgorbitalpartoftheinferiorfrontalgyrus,opticchiasm,neurodegenerativescategories,rightpcggposteriorcingulategyrus", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,ppmi,edsd", - }, - { - "name": "filter", - "value": '{"condition": "AND", "rules": [{"id": "dataset", "type": "string", "value": ["desd-synthdata", "ppmi", "edsd"], "operator": "in"}, {"condition": "AND", "rules": [{"id": "rightcuncuneus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightioginferioroccipitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftententorhinalarea", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightamygdala", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightporgposteriororbitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftpoparietaloperculum", "type": "string", "operator": "is_not_null", "value": null}, {"id": "righttrifgtriangularpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightlateralventricle", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmfcmedialfrontalcortex", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightorifgorbitalpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "opticchiasm", "type": "string", "operator": "is_not_null", "value": null}, {"id": "neurodegenerativescategories", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightpcggposteriorcingulategyrus", "type": "string", "operator": "is_not_null", "value": null}]}], "valid": True}', - }, - ], - }, - "name": "Linear Regression", - }, - { - "algorithm": { - "name": "linear_regression_cv", - "parameters": [ - {"name": "y", "value": "leftocpoccipitalpole"}, - { - "name": "x", - "value": "righthippocampus,rightsogsuperioroccipitalgyrus,leftppplanumpolare,leftsmgsupramarginalgyrus,leftgregyrusrectus,rightitginferiortemporalgyrus,leftcalccalcarinecortex", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,ppmi,edsd", - }, - { - "name": "filter", - "value": '{"condition": "AND", "rules": [{"id": "dataset", "type": "string", "value": ["desd-synthdata", "ppmi", "edsd"], "operator": "in"}, {"condition": "AND", "rules": [{"id": "rightcuncuneus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightioginferioroccipitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftententorhinalarea", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightamygdala", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightporgposteriororbitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftpoparietaloperculum", "type": "string", "operator": "is_not_null", "value": null}, {"id": "righttrifgtriangularpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightlateralventricle", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmfcmedialfrontalcortex", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightorifgorbitalpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "opticchiasm", "type": "string", "operator": "is_not_null", "value": null}, {"id": "neurodegenerativescategories", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightpcggposteriorcingulategyrus", "type": "string", "operator": "is_not_null", "value": null}]}], "valid": True}', - }, - { - "name": "n_splits", - "label": "n_splits", - "value": 4, - }, - ], - }, - "name": "Linear Regression CV", - }, - { - "algorithm": { - "name": "linear_regression_cv", - "parameters": [ - {"name": "y", "value": "righthippocampus"}, - { - "name": "x", - "value": "lefthippocampus", - }, - { - "name": "pathology", - "value": "dementia_longitudinal:v1", - }, - { - "name": "dataset", - "value": "desd-synthdata", - }, - { - "name": "filter", - "value": "", - }, - { - "name": "n_splits", - "value": 4, - }, - ], - "preprocessing": [ - { - "name": "longitudinal_transformer", - "parameters": [ - { - "name": "visit1", - "value": "BL", - }, - { - "name": "visit2", - "value": "FL1", - }, - { - "name": "strategies", - "value": '{"righthippocampus": "first", "lefthippocampus": "diff"}', - }, - ], - } - ], - }, - "name": "Linear Regression CV Longitudinal", - }, - { - "algorithm": { - "name": "ttest_independent", - "parameters": [ - {"name": "y", "value": "rightgregyrusrectus"}, - { - "name": "x", - "value": "dataset", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,ppmi,edsd", - }, - { - "name": "filter", - "value": '{"condition": "AND", "rules": [{"id": "dataset", "type": "string", "value": ["desd-synthdata", "ppmi", "edsd"], "operator": "in"}, {"condition": "AND", "rules": [{"id": "rightcuncuneus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightioginferioroccipitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftententorhinalarea", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightamygdala", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightporgposteriororbitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftpoparietaloperculum", "type": "string", "operator": "is_not_null", "value": null}, {"id": "righttrifgtriangularpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightlateralventricle", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmfcmedialfrontalcortex", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightorifgorbitalpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "opticchiasm", "type": "string", "operator": "is_not_null", "value": null}, {"id": "neurodegenerativescategories", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightpcggposteriorcingulategyrus", "type": "string", "operator": "is_not_null", "value": null}]}], "valid": True}', - }, - { - "name": "alt_hypothesis", - "value": "less", - }, - { - "name": "alpha", - "value": 0.5727207100545569, - }, - { - "name": "groupA", - "value": "edsd", - }, - { - "name": "groupB", - "value": "ppmi", - }, - ], - }, - "name": "T-Test Independent", - }, - { - "algorithm": { - "name": "logistic_regression", - "parameters": [ - {"name": "y", "value": "alzheimerbroadcategory"}, - { - "name": "x", - "value": "rightttgtransversetemporalgyrus,leftpinsposteriorinsula,leftpoparietaloperculum,rightptplanumtemporale,leftventraldc", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,ppmi", - }, - { - "name": "filter", - "value": '{"condition": "AND", "rules": [{"id": "dataset", "type": "string", "value": ["desd-synthdata", "ppmi", "edsd"], "operator": "in"}, {"condition": "AND", "rules": [{"id": "rightcuncuneus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightioginferioroccipitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftententorhinalarea", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightamygdala", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightporgposteriororbitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftpoparietaloperculum", "type": "string", "operator": "is_not_null", "value": null}, {"id": "righttrifgtriangularpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightlateralventricle", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmfcmedialfrontalcortex", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightorifgorbitalpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "opticchiasm", "type": "string", "operator": "is_not_null", "value": null}, {"id": "neurodegenerativescategories", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightpcggposteriorcingulategyrus", "type": "string", "operator": "is_not_null", "value": null}]}], "valid": True}', - }, - { - "name": "positive_class", - "value": "Other", - }, - ], - }, - "name": "Logistic Regression", - }, - { - "algorithm": { - "name": "logistic_regression_cv", - "parameters": [ - {"name": "y", "value": "alzheimerbroadcategory"}, - { - "name": "x", - "value": "leftopifgopercularpartoftheinferiorfrontalgyrus,rightmsfgsuperiorfrontalgyrusmedialsegment,leftbasalforebrain,leftinflatvent", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,ppmi,edsd", - }, - { - "name": "filter", - "value": '{"condition": "AND", "rules": [{"id": "dataset", "type": "string", "value": ["desd-synthdata", "ppmi", "edsd"], "operator": "in"}, {"condition": "AND", "rules": [{"id": "rightcuncuneus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightioginferioroccipitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftententorhinalarea", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightamygdala", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightporgposteriororbitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftpoparietaloperculum", "type": "string", "operator": "is_not_null", "value": null}, {"id": "righttrifgtriangularpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightlateralventricle", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmfcmedialfrontalcortex", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightorifgorbitalpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "opticchiasm", "type": "string", "operator": "is_not_null", "value": null}, {"id": "neurodegenerativescategories", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightpcggposteriorcingulategyrus", "type": "string", "operator": "is_not_null", "value": null}]}], "valid": True}', - }, - { - "name": "positive_class", - "value": "AD", - }, - { - "name": "n_splits", - "value": 3, - }, - ], - }, - "name": "Logistic Regression CV", - }, - { - "algorithm": { - "name": "ttest_paired", - "parameters": [ - {"name": "y", "value": "rightppplanumpolare"}, - { - "name": "x", - "value": "rightorifgorbitalpartoftheinferiorfrontalgyrus", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,ppmi,edsd", - }, - { - "name": "filter", - "value": '{"condition": "AND", "rules": [{"id": "dataset", "type": "string", "value": ["desd-synthdata", "ppmi", "edsd"], "operator": "in"}, {"condition": "AND", "rules": [{"id": "rightcuncuneus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightioginferioroccipitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftententorhinalarea", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightamygdala", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightporgposteriororbitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftpoparietaloperculum", "type": "string", "operator": "is_not_null", "value": null}, {"id": "righttrifgtriangularpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightlateralventricle", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmfcmedialfrontalcortex", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightorifgorbitalpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "opticchiasm", "type": "string", "operator": "is_not_null", "value": null}, {"id": "neurodegenerativescategories", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightpcggposteriorcingulategyrus", "type": "string", "operator": "is_not_null", "value": null}]}], "valid": True}', - }, - { - "name": "alt_hypothesis", - "value": "less", - }, - { - "name": "alpha", - "value": 0.06109997172168302, - }, - ], - }, - "name": "Paired t-test", - }, - { - "algorithm": { - "name": "ttest_onesample", - "parameters": [ - {"name": "y", "value": "leftmcggmiddlecingulategyrus"}, - { - "name": "x", - "value": "rightorifgorbitalpartoftheinferiorfrontalgyrus", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,edsd", - }, - { - "name": "filter", - "value": '{"condition": "AND", "rules": [{"id": "dataset", "type": "string", "value": ["desd-synthdata", "ppmi", "edsd"], "operator": "in"}, {"condition": "AND", "rules": [{"id": "rightcuncuneus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightioginferioroccipitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftententorhinalarea", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightamygdala", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightporgposteriororbitalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "leftpoparietaloperculum", "type": "string", "operator": "is_not_null", "value": null}, {"id": "righttrifgtriangularpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmpogpostcentralgyrusmedialsegment", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightlateralventricle", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightmfcmedialfrontalcortex", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightorifgorbitalpartoftheinferiorfrontalgyrus", "type": "string", "operator": "is_not_null", "value": null}, {"id": "opticchiasm", "type": "string", "operator": "is_not_null", "value": null}, {"id": "neurodegenerativescategories", "type": "string", "operator": "is_not_null", "value": null}, {"id": "rightpcggposteriorcingulategyrus", "type": "string", "operator": "is_not_null", "value": null}]}], "valid": True}', - }, - { - "name": "alt_hypothesis", - "value": "greater", - }, - { - "name": "alpha", - "value": 0.6764545707122654, - }, - { - "name": "mu", - "value": -1.7510563394418988, - }, - ], - }, - "name": "T-Test One-Sample", - }, - { - "algorithm": { - "name": "descriptive_stats", - "parameters": [ - { - "name": "y", - "value": "leftttgtransversetemporalgyrus,rightmprgprecentralgyrusmedialsegment", - }, - {"name": "pathology", "value": "dementia:0.1"}, - { - "name": "dataset", - "value": "desd-synthdata,edsd,ppmi", - }, - { - "name": "filter", - "value": "", - }, - ], - }, - "name": "Descriptive stats", - }, -] - -@pytest.mark.parametrize("test_input", all_success_cases) -def test_post_request_exaflow(test_input): - url = "http://127.0.0.1:8080/services/experiments" - request_json = json.dumps(test_input) - - headers = {"Content-type": "application/json", "Accept": "application/json"} - response = requests.post(url, data=request_json, headers=headers) - assert response.status_code == 400 - error = json.loads(response.text) - assert error["title"] == "Bad Request" - assert error["detail"] == "Failed to read request" - assert error["instance"] == "/services/experiments" diff --git a/deployment/dev/tests/test_exaflow_post_experiment_fail.py b/deployment/dev/tests/test_exaflow_post_experiment_fail.py deleted file mode 100644 index b1a0f3f..0000000 --- a/deployment/dev/tests/test_exaflow_post_experiment_fail.py +++ /dev/null @@ -1,101 +0,0 @@ -import pytest -import json - -import requests - - -all_error_cases = [ - ( - "Invalid parameter name", - { - "algorithm": { - "name": "logistic_regression", - "parameters": [ - { - "name": "xyz", - "value": "rightppplanumpolare,righthippocampus,lefthippocampus,rightamygdala,leftamygdala", - }, - {"name": "y", "value": "alzheimerbroadcategory"}, - {"name": "pathology", "value": "dementia"}, - {"name": "dataset", "value": "edsd,ppmi"}, - {"name": "filter", "value": ""}, - {"name": "positive_class", "value": "AD,CN"}, - ], - }, - "name": "Exaflow Invalid parameter name", - } - ), - ( - "Invalid algorithm name", - { - "algorithm": { - "name": "LOGISTIC_REGRESSION", - "parameters": [ - { - "name": "xyz", - "value": "rightppplanumpolare,righthippocampus,lefthippocampus,rightamygdala,leftamygdala", - }, - {"name": "y", "value": "alzheimerbroadcategory"}, - {"name": "pathology", "value": "dementia"}, - {"name": "dataset", "value": "edsd,ppmi"}, - {"name": "filter", "value": ""}, - {"name": "positive_class", "value": "AD,CN"}, - ], - }, - "name": "Exaflow Invalid parameter name", - } - ), - ( - "Invalid parameter value", - { - "algorithm": { - "name": "logistic_regression", - "parameters": [ - {"name": "x", "value": "xyz"}, - {"name": "y", "value": "alzheimerbroadcategory"}, - {"name": "pathology", "value": "dementia"}, - {"name": "dataset", "value": "edsd,ppmi"}, - {"name": "filter", "value": ""}, - {"name": "positive_class", "value": "AD,CN"}, - ], - }, - "name": "Exaflow Invalid parameter value", - } - ), -] - -@pytest.mark.parametrize("test_case,test_input", all_error_cases) -def test_post_request_exaflow(test_case, test_input): - url = "http://127.0.0.1:8080/services/experiments" - - request_json = json.dumps(test_input) - - headers = {"Content-type": "application/json", "Accept": "application/json"} - response = requests.post(url, data=request_json, headers=headers) - assert response.status_code == 400 - error = json.loads(response.text) - assert error["title"] == "Bad Request" - assert error["detail"] == "Failed to read request" - assert error["instance"] == "/services/experiments" - - -def test_post_request_exaflow_invalid_parameters_type(): - url = "http://127.0.0.1:8080/services/experiments" - - request_json = json.dumps( - { - "algorithm": { - "name": "LOGISTIC_REGRESSION", - "parameters": "xyz", - }, - "name": "Error_Logistic_Regression", - } - ) - - headers = {"Content-type": "application/json", "Accept": "application/json"} - response = requests.post(url, data=request_json, headers=headers) - assert response.status_code == 400 - error = json.loads(response.text) - assert error["title"] == "Bad Request" - assert error["detail"] == "Failed to read request" - assert error["instance"] == "/services/experiments" diff --git a/deployment/dev/tests/test_get_algorithms_request.py b/deployment/dev/tests/test_get_algorithms_request.py deleted file mode 100644 index 3b9179f..0000000 --- a/deployment/dev/tests/test_get_algorithms_request.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest -import json -import requests - - -def test_get_algorithms_request(): - url = "http://172.17.0.1:8080/services/algorithms" - headers = {"Content-type": "application/json", "Accept": "application/json"} - response = requests.get(url, headers=headers) - assert response.status_code == 200 - print(f"Algorithms result-> {response.text}") - algorithms = json.loads(response.text) - assert len(algorithms) == 28 diff --git a/deployment/dev/tests/test_get_pathologies_request.py b/deployment/dev/tests/test_get_pathologies_request.py deleted file mode 100644 index 49ba373..0000000 --- a/deployment/dev/tests/test_get_pathologies_request.py +++ /dev/null @@ -1,64 +0,0 @@ -import pytest -import json -import requests - -def test_get_pathologies_request(): - url = "http://172.17.0.1:8080/services/data-models" - headers = {"Content-type": "application/json", "Accept": "application/json"} - response = requests.get(url, headers=headers) - assert response.status_code == 200 - pathologies = json.loads(response.text) - assert len(pathologies) == 4 - - pathology_codes = {pathology["code"] for pathology in pathologies} - assert pathology_codes == {"dementia_longitudinal", "dementia", "mentalhealth", "tbi"} - - datasets_count_by_code = {pathology["code"]: len(pathology["datasets"]) for pathology in pathologies} - assert datasets_count_by_code == { - "dementia": 3, - "dementia_longitudinal": 1, - "mentalhealth": 1, - "tbi": 1, - } - - dataset_enum_count_by_code = {pathology["code"]: count_datasets_from_cdes(pathology) for pathology in pathologies} - assert dataset_enum_count_by_code == { - "dementia": 3, - "dementia_longitudinal": 4, - "mentalhealth": 1, - "tbi": 1, - } - - cdes_count_by_code = {pathology["code"]: count_cdes(pathology) for pathology in pathologies} - assert cdes_count_by_code == { - "dementia": 184, - "dementia_longitudinal": 185, - "mentalhealth": 191, - "tbi": 20, - } - - -def count_cdes(metadata_hierarchy) -> int: - counter = 0 - - if "variables" in metadata_hierarchy: - counter += len(metadata_hierarchy["variables"]) - - if "groups" in metadata_hierarchy: - for cde in metadata_hierarchy["groups"]: - counter += count_cdes(cde) - return counter - - -def count_datasets_from_cdes(metadata_hierarchy) -> int: - if "variables" in metadata_hierarchy: - for variable in metadata_hierarchy["variables"]: - if variable["code"] == "dataset": - return len(variable["enumerations"]) - - if "groups" in metadata_hierarchy: - for cde in metadata_hierarchy["groups"]: - counter = count_datasets_from_cdes(cde) - if counter != 0: - return counter - return 0 diff --git a/deployment/kubernetes/README.md b/deployment/kubernetes/README.md index 6f8318c..68ecb59 100644 --- a/deployment/kubernetes/README.md +++ b/deployment/kubernetes/README.md @@ -56,6 +56,8 @@ Prior to deploying it (on a microk8s K8s cluster of one or more nodes), there ar * `cluster`: storage classes and whether the chart should use the managed storage class or the microk8s local storage class. * `global`: shared public hostname used by the ingress and backend redirects. * `platform-ui`, `platform-backend`, `platformBackendDatabase`: container images and component specific options (including the shared ingress/tls settings and PVC sizes). +* `platform-ui.notebook`: enables the `/notebook` iframe route in platform-ui and wires nginx to the in-cluster JupyterHub service. +* `jupyterhub`: JupyterHub and single-user Jupyter images, hub resources, ingress, storage, crypt key, and notebook resource requests/limits. Rendered only when `platform-ui.notebook.enabled` is true, which also requires `keycloak.enabled`. Notebook traffic for the portal iframe should use platform-ui's `/notebook/` proxy; keep `jupyterhub.ingress.enabled: false` unless you need a separate admin-only host. * `keycloak`: toggles the connection parameters to the external Keycloak instance (`enabled`, `host`, `protocol`, `realm`). Copy `values.yaml` to a new file (for example `my-values.yaml`) and edit it in-place. A few important knobs: @@ -69,6 +71,8 @@ platform-ui: host: platform-backend-service port: 8080 context: services + notebook: + enabled: true ingress: tlsSecretName: platform-ui-tls @@ -77,6 +81,22 @@ keycloak: host: iam.example.org ``` +See `values.yaml` for the full `jupyterhub` block (images, hub and notebook resources, storage). + +JupyterHub encrypts auth state with the key in the `jupyterhub-crypt` secret. Either set `jupyterhub.cryptKey` to a stable value, or leave it empty and create the secret once, like `keycloak-credentials`: + +``` +kubectl create secret generic jupyterhub-crypt -n --from-literal=crypt-key=$(openssl rand -hex 32) +``` + +The chart never generates the key itself, so it stays stable across ArgoCD syncs and `helm template` renders. + +Register a Keycloak redirect URI for the Hub OAuth client: + +`https:///notebook/hub/oauth_callback` + +For Kubernetes, make sure `hbpmip/mip-jupyterhub:0.0.1_candidate` and `hbpmip/mip-jupyter:0.0.1_candidate` are pushed to the configured registry or loaded onto every node that may run the pods. + The reachability diagram from the legacy profiles is still valid as a reference for deciding the correct public URL: ![MIP Reachability Scheme](../docs/MIP_Configuration.png) diff --git a/deployment/kubernetes/templates/_helpers.tpl b/deployment/kubernetes/templates/_helpers.tpl new file mode 100644 index 0000000..5960d7c --- /dev/null +++ b/deployment/kubernetes/templates/_helpers.tpl @@ -0,0 +1,11 @@ +{{- define "mip.keycloakAuthUrl" -}} +{{- $protocol := default "https" .Values.keycloak.protocol -}} +{{- $host := default "" .Values.keycloak.host -}} +{{- if $host -}} +{{- printf "%s://%s/auth/" $protocol $host -}} +{{- end -}} +{{- end -}} + +{{- define "mip.storageClass" -}} +{{- ternary .Values.cluster.storageClasses.managed .Values.cluster.storageClasses.local (ne (toString .Values.cluster.managed) "false") -}} +{{- end -}} diff --git a/deployment/kubernetes/templates/jupyterhub.yaml b/deployment/kubernetes/templates/jupyterhub.yaml new file mode 100644 index 0000000..bf41538 --- /dev/null +++ b/deployment/kubernetes/templates/jupyterhub.yaml @@ -0,0 +1,282 @@ +{{- $namespace := default "default" .Release.Namespace -}} +{{- $jupyterHub := index .Values "jupyterhub" -}} +{{- $platformUi := index .Values "platform-ui" -}} +{{- $notebook := default (dict) $platformUi.notebook -}} +{{- if eq (toString $notebook.enabled) "true" -}} +{{- if eq (toString .Values.keycloak.enabled) "false" -}} +{{- fail "platform-ui.notebook.enabled requires keycloak.enabled: JupyterHub authenticates through Keycloak" -}} +{{- end -}} +{{- $hubImage := printf "%s:%s" $jupyterHub.image.repository $jupyterHub.image.tag -}} +{{- $singleuserImage := printf "%s:%s" $jupyterHub.singleuser.image.repository $jupyterHub.singleuser.image.tag -}} +{{- $storageClass := include "mip.storageClass" . -}} +{{- $keycloakRealm := default "MIP" .Values.keycloak.realm -}} +{{- $keycloakAuthUrl := include "mip.keycloakAuthUrl" . -}} +{{- $publicHost := .Values.global.publicHost -}} +{{- $useOperator := eq (default "kubespawner" $jupyterHub.spawner) "operator" -}} +{{- if $jupyterHub.cryptKey }} +--- +# Without jupyterhub.cryptKey, create the secret out-of-band (see README) so the +# key stays stable across ArgoCD syncs and `helm template` renders. +apiVersion: v1 +kind: Secret +metadata: + name: jupyterhub-crypt + namespace: {{ $namespace }} +type: Opaque +stringData: + crypt-key: {{ $jupyterHub.cryptKey | quote }} +{{- end }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: jupyterhub-claim0 + namespace: {{ $namespace }} + annotations: + argocd.argoproj.io/sync-options: Delete=false,Prune=false +spec: + storageClassName: {{ $storageClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: jupyterhub + namespace: {{ $namespace }} + +{{- if ne (toString $jupyterHub.rbac.create) "false" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: jupyterhub + namespace: {{ $namespace }} +rules: +{{- if $useOperator }} + # The MIP notebook operator builds the pods from the NotebookProfile below; + # the hub only asks for them. secrets: create (no read/update/delete) is for + # the per-server API token Secret, owned by its Notebook. + - apiGroups: ["notebooks.mip.ebrains.eu"] + resources: ["notebooks"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +{{- else }} + - apiGroups: [""] + resources: ["pods", "persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch"] +{{- end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: jupyterhub + namespace: {{ $namespace }} +subjects: + - kind: ServiceAccount + name: jupyterhub + namespace: {{ $namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: jupyterhub +{{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: jupyterhub + namespace: {{ $namespace }} +spec: + selector: + app: jupyterhub + component: hub + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8000 + - name: hub-api + protocol: TCP + port: 8081 + targetPort: 8081 + +{{- if eq (toString $jupyterHub.ingress.enabled) "true" }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: jupyterhub + namespace: {{ $namespace }} +spec: + ingressClassName: {{ default "nginx" (default (dict) $platformUi.ingress).className | quote }} + rules: + - host: notebooks.{{ .Values.global.publicHost }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: jupyterhub + port: + number: 80 +{{- end }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jupyterhub + namespace: {{ $namespace }} + labels: + app: jupyterhub +spec: + replicas: 1 + selector: + matchLabels: + app: jupyterhub + strategy: + type: Recreate + template: + metadata: + labels: + app: jupyterhub + component: hub + spec: + serviceAccountName: jupyterhub +{{- if eq (toString .Values.cluster.managed) "false" }} + nodeSelector: + master: "true" +{{- end }} + volumes: + - name: jupyterhub-claim0 + persistentVolumeClaim: + claimName: jupyterhub-claim0 + containers: + - name: jupyterhub + image: {{ $hubImage | quote }} + ports: + - containerPort: 8000 + - containerPort: 8081 +{{- with $jupyterHub.resources }} + resources: + {{- toYaml . | nindent 12 }} +{{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: JUPYTERHUB_SPAWNER + value: {{ ternary "operator" "kubespawner" $useOperator | quote }} + - name: JUPYTER_SINGLEUSER_IMAGE + value: {{ $singleuserImage | quote }} +{{- if eq (toString .Values.cluster.managed) "false" }} + - name: JUPYTER_NODE_SELECTOR + value: "master=true" +{{- end }} + - name: JUPYTER_STORAGE_CLASS + value: {{ $storageClass | quote }} + - name: JUPYTER_STORAGE_CAPACITY + value: {{ $jupyterHub.singleuser.storageCapacity | quote }} + - name: JUPYTER_CPU_LIMIT + value: {{ $jupyterHub.singleuser.resources.limits.cpu | quote }} + - name: JUPYTER_MEM_LIMIT + value: {{ $jupyterHub.singleuser.resources.limits.memory | quote }} + - name: JUPYTER_CPU_GUARANTEE + value: {{ $jupyterHub.singleuser.resources.requests.cpu | quote }} + - name: JUPYTER_MEM_GUARANTEE + value: {{ $jupyterHub.singleuser.resources.requests.memory | quote }} + - name: JUPYTERHUB_LOGOUT_REDIRECT_URL + value: {{ printf "https://%s" $publicHost | quote }} + # Read by oauthenticator. Without it the hub derives the callback from + # the request, which is http:// behind platform-ui nginx. + - name: OAUTH_CALLBACK_URL + value: {{ printf "https://%s/notebook/hub/oauth_callback" $publicHost | quote }} +{{- with $jupyterHub.codex }} +{{- if .baseUrl }} + # jupyterhub_config.py forwards these to every singleuser pod. + - name: CODEX_VLLM_BASE_URL + value: {{ .baseUrl | quote }} +{{- end }} +{{- if .model }} + - name: CODEX_VLLM_MODEL + value: {{ .model | quote }} +{{- end }} +{{- end }} + - name: KEYCLOAK_AUTH_URL + value: {{ $keycloakAuthUrl | quote }} + - name: KEYCLOAK_REALM + value: {{ $keycloakRealm | quote }} + - name: KEYCLOAK_CLIENT_ID + valueFrom: + secretKeyRef: + name: keycloak-credentials + key: client-id + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: keycloak-credentials + key: client-secret + - name: JUPYTERHUB_CRYPT_KEY + valueFrom: + secretKeyRef: + name: jupyterhub-crypt + key: crypt-key + volumeMounts: + - name: jupyterhub-claim0 + mountPath: /srv/jupyterhub +{{- if $useOperator }} + +--- +# Pod template for the MIP notebook operator (mip-jupyter/operator). The hub +# cannot read or change it; the operator applies the security context itself. +apiVersion: notebooks.mip.ebrains.eu/v1alpha1 +kind: NotebookProfile +metadata: + name: default + namespace: {{ $namespace }} +spec: + image: {{ $singleuserImage | quote }} + imagePullPolicy: Always + resources: + {{- toYaml $jupyterHub.singleuser.resources | nindent 4 }} + storage: + storageClassName: {{ $storageClass | quote }} + size: {{ $jupyterHub.singleuser.storageCapacity | quote }} + # New volumes are root-owned and not every provisioner applies fsGroup; + # the notebook (uid 1000) must own /home/jovyan/work. Same chown init + # container KubeSpawner ran. + fixOwnership: true +{{- if eq (toString .Values.cluster.managed) "false" }} + nodeSelector: + master: "true" +{{- end }} + env: + - name: PLATFORM_BACKEND_URL + value: http://platform-backend-service:8080/services +{{- with $jupyterHub.codex }} +{{- if .baseUrl }} + - name: CODEX_VLLM_BASE_URL + value: {{ .baseUrl | quote }} +{{- end }} +{{- if .model }} + - name: CODEX_VLLM_MODEL + value: {{ .model | quote }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/deployment/kubernetes/templates/microk8s-local-storageclass.yaml b/deployment/kubernetes/templates/microk8s-local-storageclass.yaml index 3e5ffef..a7687e6 100644 --- a/deployment/kubernetes/templates/microk8s-local-storageclass.yaml +++ b/deployment/kubernetes/templates/microk8s-local-storageclass.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.cluster.managed }} +{{- if eq (toString .Values.cluster.managed) "false" }} {{- $localStorageClass := .Values.cluster.storageClasses.local -}} apiVersion: storage.k8s.io/v1 kind: StorageClass diff --git a/deployment/kubernetes/templates/platform-backend.yaml b/deployment/kubernetes/templates/platform-backend.yaml index 7d24fff..e3b5fbd 100644 --- a/deployment/kubernetes/templates/platform-backend.yaml +++ b/deployment/kubernetes/templates/platform-backend.yaml @@ -1,22 +1,15 @@ {{- $namespace := default "default" .Release.Namespace -}} {{- $platformBackend := index .Values "platform-backend" -}} -{{- $authEnabled := default true .Values.keycloak.enabled -}} +{{- $authEnabled := ne (toString .Values.keycloak.enabled) "false" -}} {{- $authFlag := ternary "1" "0" $authEnabled -}} {{- $publicHost := default "" .Values.global.publicHost -}} {{- $platformUiUrl := "" -}} {{- if $publicHost -}} {{- $platformUiUrl = printf "https://%s" $publicHost -}} {{- end -}} -{{- $keycloakProtocol := default "https" .Values.keycloak.protocol -}} -{{- $keycloakHost := default "" .Values.keycloak.host -}} {{- $keycloakRealm := default "MIP" .Values.keycloak.realm -}} -{{- $keycloakAuthUrl := "" -}} -{{- if and $keycloakProtocol $keycloakHost -}} -{{- $keycloakAuthUrl = printf "%s://%s/auth/" $keycloakProtocol $keycloakHost -}} -{{- end -}} -{{- $localStorageClass := .Values.cluster.storageClasses.local -}} -{{- $managedStorageClass := .Values.cluster.storageClasses.managed -}} -{{- $backendStorageClass := ternary $managedStorageClass $localStorageClass .Values.cluster.managed -}} +{{- $keycloakAuthUrl := include "mip.keycloakAuthUrl" . -}} +{{- $backendStorageClass := include "mip.storageClass" . -}} --- # platform-backend PVCs apiVersion: v1 @@ -101,7 +94,7 @@ spec: labels: app: platform-backend spec: -{{- if not .Values.cluster.managed }} +{{- if eq (toString .Values.cluster.managed) "false" }} nodeSelector: master: "true" {{- end }} @@ -202,6 +195,8 @@ spec: value: {{ default "" .Values.engines.exaflow.url | quote }} - name: PLATFORM_UI_BASE_URL value: {{ $platformUiUrl | quote }} + - name: MIP_VERSION + value: {{ default "" .Values.mip.version | quote }} - name: KEYCLOAK_AUTH_URL value: {{ $keycloakAuthUrl | quote }} - name: KEYCLOAK_REALM diff --git a/deployment/kubernetes/templates/platform-ui.yaml b/deployment/kubernetes/templates/platform-ui.yaml index 9cfb555..2d69062 100644 --- a/deployment/kubernetes/templates/platform-ui.yaml +++ b/deployment/kubernetes/templates/platform-ui.yaml @@ -7,8 +7,12 @@ {{- $guide := default (dict) $platformUi.guide -}} {{- $guideCovariate := default "Sex" $guide.covariate -}} {{- $guideVariable := default "Age" $guide.variable -}} -{{- $ingressEnabled := default true $ingress.enabled -}} +{{- $ingressEnabled := ne (toString $ingress.enabled) "false" -}} {{- $publicHost := default "" .Values.global.publicHost -}} +{{- $notebook := default (dict) $platformUi.notebook -}} +{{- $notebookEnabled := eq (toString $notebook.enabled) "true" -}} +{{- /* nginx resolves proxy_pass hosts at startup; without the notebook, jupyterhub is not deployed. */ -}} +{{- $jupyterServer := ternary "jupyterhub:80" "127.0.0.1:80" $notebookEnabled -}} --- apiVersion: apps/v1 kind: Deployment @@ -25,7 +29,7 @@ spec: labels: app: platform-ui spec: -{{- if not .Values.cluster.managed }} +{{- if eq (toString .Values.cluster.managed) "false" }} nodeSelector: master: "true" {{- end }} @@ -47,6 +51,13 @@ spec: - name: PLATFORM_BACKEND_CONTEXT value: {{ $backendContext | quote }} {{- end }} + - name: NOTEBOOK_ENABLED + value: {{ ternary "1" "0" $notebookEnabled | quote }} + - name: JUPYTER_SERVER + value: {{ $jupyterServer | quote }} + # nginx.conf.template hardcodes /notebook in its redirects; envsubst needs it set. + - name: JUPYTER_CONTEXT + value: "notebook" --- apiVersion: v1 kind: Service diff --git a/deployment/kubernetes/values.yaml b/deployment/kubernetes/values.yaml index 1adbba5..772405e 100644 --- a/deployment/kubernetes/values.yaml +++ b/deployment/kubernetes/values.yaml @@ -9,11 +9,11 @@ engines: exaflow: url: "http://exaflow-controller-service:5000" mip: - version: 9.1.0 + version: 9.2.0 platform-ui: image: repository: hbpmip/platform-ui - tag: 1.2.0 + tag: 2.0.0_candidate ingress: enabled: true className: haproxy-public @@ -23,13 +23,15 @@ platform-ui: host: platform-backend-service port: 8080 context: services + notebook: + enabled: true guide: covariate: Sex variable: Age platform-backend: image: repository: hbpmip/platform-backend - tag: 9.3.0 + tag: 10.0.0_candidate config: logLevel: INFO logLevelFramework: INFO @@ -50,3 +52,49 @@ platformBackendDatabase: keycloak: enabled: true host: iam.ebrains.eu + +jupyterhub: + # Leave empty and create the jupyterhub-crypt secret out-of-band (see README), + # or set a stable value such as `openssl rand -hex 32`. + cryptKey: "" + image: + repository: hbpmip/mip-jupyterhub + tag: 0.0.1_candidate + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + ingress: + enabled: false + # kubespawner: the hub creates notebook pods itself (Role on pods/PVCs). + # operator: the MIP notebook operator does (mip-jupyter/operator); the hub + # only creates Notebook resources. Needs the operator and its CRDs installed. + spawner: kubespawner + rbac: + # false: an admin applies the hub Role/RoleBinding (e.g. with + # mip-infra-staging's out-of-band RBAC), so the GitOps tool needs no + # RBAC write. The rules must match the spawner mode. + create: true + # OpenAI-compatible inference server for the Jupyter AI Codex persona. + # baseUrl ends in /v1; empty leaves Codex unconfigured. Point it at a proxy + # (e.g. mip-infra-staging's athena-proxy) to keep the real server URL out of + # notebooks. model must equal a served id from /models; empty keeps + # the image default. + codex: + baseUrl: "" + model: "" + singleuser: + image: + repository: hbpmip/mip-jupyter + tag: 0.0.1_candidate + storageCapacity: 2Gi + resources: + requests: + cpu: 500m + memory: 1G + limits: + cpu: "1" + memory: 4G