From 83555fa5c3c3d11a3cbcb4e05342c0695eab7fb0 Mon Sep 17 00:00:00 2001 From: PRIYANSHI Date: Fri, 18 Sep 2026 23:22:05 +1000 Subject: [PATCH 1/7] Trigger CI/CD pipeline for 8.1P demonstration --- .gitignore | 7 +++++++ course-service/README.md | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 course-service/README.md diff --git a/.gitignore b/.gitignore index 4299cd1a..90cec08c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,13 @@ __pycache__/ # MacOS .DS_Store +# Terraform +terraform/.terraform/ +terraform/*.tfstate +terraform/*.tfstate.* +terraform/tfplan +terraform/.terraform.lock.hcl + # Distribution / packaging .Python build/ diff --git a/course-service/README.md b/course-service/README.md new file mode 100644 index 00000000..1c3856b0 --- /dev/null +++ b/course-service/README.md @@ -0,0 +1,2 @@ + + From 33b2817a1f253f4fe83f402e5cfb7b38caf7685d Mon Sep 17 00:00:00 2001 From: PRIYANSHI Date: Fri, 18 Sep 2026 23:34:58 +1000 Subject: [PATCH 2/7] Retrigger CI via push to chain staging deploy --- course-service/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/course-service/README.md b/course-service/README.md index 1c3856b0..a7ab6f31 100644 --- a/course-service/README.md +++ b/course-service/README.md @@ -1,2 +1,3 @@ + From 60de28394f2b2ed629136e322a311c31da261e79 Mon Sep 17 00:00:00 2001 From: PRIYANSHI Date: Fri, 18 Sep 2026 23:42:44 +1000 Subject: [PATCH 3/7] Retest push trigger after fork settle time --- course-service/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/course-service/README.md b/course-service/README.md index a7ab6f31..efce7d4b 100644 --- a/course-service/README.md +++ b/course-service/README.md @@ -1,3 +1,4 @@ + From 24d7eb97a36829bae06be45e2b99034346392282 Mon Sep 17 00:00:00 2001 From: PRIYANSHI Date: Fri, 25 Sep 2026 23:12:59 +1000 Subject: [PATCH 4/7] Add Continuous Deployment: auto-promote to production after staging tests Trigger 04-deploy-production.yml automatically via workflow_run once 03-staging-test.yml succeeds, deploying the same tested image SHA with no manual step. Keeps workflow_dispatch as a manual fallback for re-deploying a specific tag. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/04-deploy-production.yml | 34 ++++++++++--- README.md | 59 +++++++++++++--------- frontend/src/components/Header.jsx | 2 +- 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/.github/workflows/04-deploy-production.yml b/.github/workflows/04-deploy-production.yml index 969d646b..cf05fecd 100644 --- a/.github/workflows/04-deploy-production.yml +++ b/.github/workflows/04-deploy-production.yml @@ -1,6 +1,17 @@ name: 04 - Deploy to Production on: + # Automatically promote to production once the staging smoke test passes. + # This is the Continuous Deployment step: no manual trigger required. + workflow_run: + workflows: + - "03 - Test Staging" + types: + - completed + branches: + - main + + # Manual fallback to re-deploy a previously tested image tag. workflow_dispatch: inputs: image_tag: @@ -13,12 +24,21 @@ jobs: name: Deploy to Production runs-on: ubuntu-latest + if: > + github.event_name == 'workflow_dispatch' || + github.event.workflow_run.conclusion == 'success' + environment: name: production + env: + IMAGE_TAG: ${{ inputs.image_tag || github.event.workflow_run.head_sha }} + steps: - - name: Checkout repository + - name: Checkout tested commit uses: actions/checkout@v4 + with: + ref: ${{ env.IMAGE_TAG }} - name: Login to Azure uses: azure/login@v3 @@ -69,37 +89,37 @@ jobs: - name: Update frontend image run: | kubectl set image deployment/frontend \ - frontend=${{ vars.ACR_LOGIN_SERVER }}/koalatech-frontend:${{ inputs.image_tag }} \ + frontend=${{ vars.ACR_LOGIN_SERVER }}/koalatech-frontend:${{ env.IMAGE_TAG }} \ -n production - name: Update user-service image run: | kubectl set image deployment/user-service \ - user-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-user-service:${{ inputs.image_tag }} \ + user-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-user-service:${{ env.IMAGE_TAG }} \ -n production - name: Update student-service image run: | kubectl set image deployment/student-service \ - student-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-student-service:${{ inputs.image_tag }} \ + student-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-student-service:${{ env.IMAGE_TAG }} \ -n production - name: Update lecturer-service image run: | kubectl set image deployment/lecturer-service \ - lecturer-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-lecturer-service:${{ inputs.image_tag }} \ + lecturer-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-lecturer-service:${{ env.IMAGE_TAG }} \ -n production - name: Update course-service image run: | kubectl set image deployment/course-service \ - course-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-course-service:${{ inputs.image_tag }} \ + course-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-course-service:${{ env.IMAGE_TAG }} \ -n production - name: Update enrollment-service image run: | kubectl set image deployment/enrollment-service \ - enrollment-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-enrollment-service:${{ inputs.image_tag }} \ + enrollment-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-enrollment-service:${{ env.IMAGE_TAG }} \ -n production - name: Wait for frontend rollout diff --git a/README.md b/README.md index adf1b194..db20c06f 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,32 @@ -# Week 08 – Continuous Delivery with GitHub Actions and Kubernetes +# Week 08 – Continuous Deployment with GitHub Actions and Kubernetes In Week 07, we implemented a Continuous Integration (CI) pipeline using GitHub Actions. The pipeline automatically tested the backend services, built Docker images, and pushed the successfully built images to Azure Container Registry (ACR). -In Week 08, we extend this workflow to implement **Continuous Delivery (CD)**. +In Week 08, we extended this workflow to implement Continuous Delivery: automatic deployment to **staging**, followed by automated smoke tests, with production promoted manually. -The application will first be automatically deployed to a **staging environment**. After deployment, automated tests will verify that the staging application is working correctly. A tested version can then be manually promoted to the **production environment**. +In Task 9.3C, this is extended further to full **Continuous Deployment (CD)**: once the staging smoke test passes, the same tested image is automatically promoted to **production** as well — no manual step is required. The same Docker images that are tested in staging are deployed to production. The application is **not rebuilt** during production deployment. --- -## 1. Continuous Delivery Workflow +## 1. Continuous Deployment Workflow -The Week 08 pipeline consists of four GitHub Actions workflows: +The pipeline consists of four GitHub Actions workflows: ![](./workflow.png) -The first three workflows run automatically. +All four workflows run automatically, chained via `workflow_run`: -Production deployment is intentionally manual. +``` +push to main + -> 01 - CI (test, build, push images) + -> 02 - Deploy to Staging + -> 03 - Test Staging (smoke test) + -> 04 - Deploy to Production (NEW: auto-triggered on staging test success) +``` + +`04-deploy-production.yml` still accepts a manual `workflow_dispatch` with an `image_tag` input as a fallback, so a specific previously-tested SHA can be re-deployed on demand. But the default path is fully automatic: merging a pull request into `main` is enough to carry a change all the way to production. --- @@ -267,6 +275,7 @@ Verify that the following workflows complete successfully: 01 - CI 02 - Deploy to Staging 03 - Staging Test +04 - Deploy to Production Once the deployment is complete, verify the Kubernetes resources in the staging namespace and access the staging application using the frontend external IP. @@ -274,33 +283,35 @@ Confirm that the application is working correctly before proceeding to productio 13. Deploy to Production -Production deployment is performed manually. +Production deployment is automatic. As soon as **03 - Staging Test** completes successfully, GitHub Actions triggers **04 - Deploy to Production** via a `workflow_run` event and deploys the exact same image SHA that was just tested in staging — no manual action is required. -Go to: +To trigger the whole chain end-to-end, push a change to `main` (typically by merging a pull request): -GitHub Repository -→ Actions -→ 04 - Deploy to Production -→ Run workflow - -Provide the image SHA that successfully passed the staging deployment and testing process. - -### Find the Image SHA +```bash +git push origin main +``` -Before running the production workflow, obtain the Git commit SHA of the version that was successfully deployed and tested in staging: +Then watch the run in: -```bash -git rev-parse HEAD +```text +GitHub Repository +→ Actions ``` -Copy the returned SHA and provide it as the `image_tag` when manually running the **04 - Deploy to Production** workflow. +All four workflows should run in sequence and complete successfully. -> Make sure the SHA belongs to the version that successfully passed the staging pipeline. +### Manual override (optional) +If you need to re-deploy a specific previously-tested commit to production without going through staging again, you can still trigger **04 - Deploy to Production** manually: -Run the production workflow and verify that it completes successfully. +```text +GitHub Repository +→ Actions +→ 04 - Deploy to Production +→ Run workflow +``` -Important: Production must use the same image version that was tested in staging. Do not rebuild the Docker images for production. +Provide the image SHA as `image_tag`. This SHA must belong to a version that already passed the staging pipeline — production must use the same image version that was tested in staging. Do not rebuild the Docker images for production. 14. Verify the Production Application diff --git a/frontend/src/components/Header.jsx b/frontend/src/components/Header.jsx index d1a76071..f82352bd 100644 --- a/frontend/src/components/Header.jsx +++ b/frontend/src/components/Header.jsx @@ -43,7 +43,7 @@ const Header = () => { fontWeight: 600, }} > - KoalaTech University + KoalaTech University — Continuous Deployment Demo Date: Fri, 2 Oct 2026 15:15:58 +1000 Subject: [PATCH 5/7] Add Terraform, Docker Scout and monitoring to the pipeline - Terraform job in 01-ci (fmt, validate, plan, apply) with remote state - Docker Scout report, recommendations and critical gate before image push - Prometheus and Grafana on AKS via Helm, with a KoalaTech dashboard - Fix vulnerable dependencies (PyJWT, fastapi, python-multipart) and refresh the frontend nginx base image (15 critical vulnerabilities to 0) - Fix always-true if condition in staging workflows - Manual infrastructure teardown workflow Co-Authored-By: Claude Sonnet 5.5 --- .github/actions/terraform-azure/action.yml | 87 ++++ .github/workflows/01-ci.yml | 104 +++- .github/workflows/02-deploy-staging.yml | 3 +- .github/workflows/03-staging-test.yml | 3 +- .github/workflows/05-deploy-monitoring.yml | 152 ++++++ .../workflows/99-infrastructure-teardown.yml | 39 ++ .gitignore | 1 - course-service/requirements.txt | 8 +- docker-compose.monitoring.yml | 61 +++ enrollment-service/requirements.txt | 8 +- frontend/Dockerfile | 2 +- .../monitoring/koalatech-dashboard.json | 480 ++++++++++++++++++ kubernetes/monitoring/values.yaml | 56 ++ lecturer-service/requirements.txt | 10 +- .../prometheus-self-monitoring.json | 79 +++ .../provisioning/dashboards/dashboard.yml | 12 + .../provisioning/datasources/datasource.yml | 9 + monitoring/prometheus.yml | 19 + student-service/requirements.txt | 10 +- terraform/.terraform.lock.hcl | 23 + terraform/container_registry.tf | 15 + terraform/kubernetes_serivce.tf | 34 ++ terraform/output.tf | 57 +++ terraform/resource_group.tf | 11 + terraform/storage_account.tf | 30 ++ terraform/terraform.tfvars | 13 + terraform/variables.tf | 83 +++ terraform/versions.tf | 18 + user-service/requirements.txt | 10 +- 29 files changed, 1406 insertions(+), 31 deletions(-) create mode 100644 .github/actions/terraform-azure/action.yml create mode 100644 .github/workflows/05-deploy-monitoring.yml create mode 100644 .github/workflows/99-infrastructure-teardown.yml create mode 100644 docker-compose.monitoring.yml create mode 100644 kubernetes/monitoring/koalatech-dashboard.json create mode 100644 kubernetes/monitoring/values.yaml create mode 100644 monitoring/grafana/dashboards/prometheus-self-monitoring.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboard.yml create mode 100644 monitoring/grafana/provisioning/datasources/datasource.yml create mode 100644 monitoring/prometheus.yml create mode 100644 terraform/.terraform.lock.hcl create mode 100644 terraform/container_registry.tf create mode 100644 terraform/kubernetes_serivce.tf create mode 100644 terraform/output.tf create mode 100644 terraform/resource_group.tf create mode 100644 terraform/storage_account.tf create mode 100644 terraform/terraform.tfvars create mode 100644 terraform/variables.tf create mode 100644 terraform/versions.tf diff --git a/.github/actions/terraform-azure/action.yml b/.github/actions/terraform-azure/action.yml new file mode 100644 index 00000000..17a1cd7e --- /dev/null +++ b/.github/actions/terraform-azure/action.yml @@ -0,0 +1,87 @@ +name: Terraform on Azure +description: > + Logs in to Azure, exposes the service principal to Terraform, makes sure the + remote state storage exists and runs terraform init against it. + +inputs: + azure-credentials: + description: Service principal JSON (the AZURE_CREDENTIALS secret) + required: true + working-directory: + description: Folder that holds the Terraform files + default: terraform + +runs: + using: composite + steps: + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Login to Azure + uses: azure/login@v2 + with: + creds: ${{ inputs.azure-credentials }} + + - name: Pass the service principal to Terraform + shell: bash + env: + AZURE_CREDENTIALS: ${{ inputs.azure-credentials }} + run: | + echo "::add-mask::$(echo "$AZURE_CREDENTIALS" | jq -r .clientSecret)" + { + echo "ARM_CLIENT_ID=$(echo "$AZURE_CREDENTIALS" | jq -r .clientId)" + echo "ARM_CLIENT_SECRET=$(echo "$AZURE_CREDENTIALS" | jq -r .clientSecret)" + echo "ARM_TENANT_ID=$(echo "$AZURE_CREDENTIALS" | jq -r .tenantId)" + echo "ARM_SUBSCRIPTION_ID=$(echo "$AZURE_CREDENTIALS" | jq -r .subscriptionId)" + } >> "$GITHUB_ENV" + + # The state lives in its own resource group so that a destroy of the + # application infrastructure never deletes the state it is working from. + # Every command below is idempotent. + - name: Make sure the Terraform state storage exists + shell: bash + run: | + STATE_RG="koalatech-tfstate-rg" + STATE_SA="tfstate$(echo "$ARM_SUBSCRIPTION_ID" | tr -d '-' | cut -c1-12)" + + az group create --name "$STATE_RG" --location australiaeast --output none + az storage account create \ + --name "$STATE_SA" \ + --resource-group "$STATE_RG" \ + --location australiaeast \ + --sku Standard_LRS \ + --min-tls-version TLS1_2 \ + --allow-blob-public-access false \ + --output none + + STATE_KEY="$(az storage account keys list \ + --resource-group "$STATE_RG" \ + --account-name "$STATE_SA" \ + --query '[0].value' --output tsv)" + echo "::add-mask::$STATE_KEY" + + az storage container create \ + --name tfstate \ + --account-name "$STATE_SA" \ + --account-key "$STATE_KEY" \ + --output none + + { + echo "TFSTATE_RG=$STATE_RG" + echo "TFSTATE_SA=$STATE_SA" + echo "ARM_ACCESS_KEY=$STATE_KEY" + } >> "$GITHUB_ENV" + + - name: Terraform init + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + terraform init \ + -input=false \ + -reconfigure \ + -backend-config="resource_group_name=$TFSTATE_RG" \ + -backend-config="storage_account_name=$TFSTATE_SA" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=week08.tfstate" diff --git a/.github/workflows/01-ci.yml b/.github/workflows/01-ci.yml index 8206db43..18fc106d 100644 --- a/.github/workflows/01-ci.yml +++ b/.github/workflows/01-ci.yml @@ -5,13 +5,72 @@ on: push: branches: - main - + + # Pull requests run the Terraform plan and the tests only + pull_request: + branches: + - main + # Manual trigger for the workflow workflow_dispatch: jobs: + # ========================================================= + # Infrastructure as Code (Terraform) + # Runs first so the ACR and AKS cluster exist before images are pushed. + # Pull requests only get a plan, pushes to main also apply it. + # ========================================================= + terraform: + name: Terraform + runs-on: ubuntu-latest + + # Only one run at a time may touch the Terraform state + concurrency: + group: terraform-state + cancel-in-progress: false + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Azure, remote state and Terraform + uses: ./.github/actions/terraform-azure + with: + azure-credentials: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform format check + working-directory: terraform + run: terraform fmt -check -recursive + + - name: Terraform validate + working-directory: terraform + run: terraform validate + + - name: Terraform plan + working-directory: terraform + run: | + terraform plan -input=false -no-color -out=tfplan | tee plan.txt + { + echo '### Terraform plan' + echo '```' + tail -n 25 plan.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Terraform apply + if: github.event_name != 'pull_request' + working-directory: terraform + run: terraform apply -input=false -no-color tfplan + + - name: Show Terraform outputs + if: github.event_name != 'pull_request' + working-directory: terraform + run: terraform output + + # ========================================================= # Backend Tests # ========================================================= @@ -109,9 +168,10 @@ jobs: name: Build and Push ${{ matrix.image }} runs-on: ubuntu-latest - # All backend tests must pass before this job starts + # All backend tests must pass and the infrastructure must be ready needs: - backend-test + - terraform # Build and push when code is pushed to main or manually triggered if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' @@ -160,6 +220,46 @@ jobs: -t ${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.image }}:${{ github.sha }} \ ./${{ matrix.service }} + # Docker Scout analyses the image that was just built, before it is pushed. + - name: Docker Scout - vulnerability report + uses: docker/scout-action@v1 + with: + command: cves + image: local://${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.image }}:${{ github.sha }} + organization: ${{ secrets.DOCKER_HUB_USER }} + dockerhub-user: ${{ secrets.DOCKER_HUB_USER }} + dockerhub-password: ${{ secrets.DOCKER_HUB_PAT }} + only-severities: critical,high + only-fixed: true + summary: true + write-comment: false + exit-code: false + + - name: Docker Scout - recommendations + uses: docker/scout-action@v1 + with: + command: recommendations + image: local://${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.image }}:${{ github.sha }} + organization: ${{ secrets.DOCKER_HUB_USER }} + dockerhub-user: ${{ secrets.DOCKER_HUB_USER }} + dockerhub-password: ${{ secrets.DOCKER_HUB_PAT }} + summary: true + write-comment: false + + # Security gate: critical vulnerabilities stop the pipeline before the + # image is pushed, so a critical image can never reach a deployment. + - name: Docker Scout - fail on critical vulnerabilities + uses: docker/scout-action@v1 + with: + command: cves + image: local://${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.image }}:${{ github.sha }} + organization: ${{ secrets.DOCKER_HUB_USER }} + dockerhub-user: ${{ secrets.DOCKER_HUB_USER }} + dockerhub-password: ${{ secrets.DOCKER_HUB_PAT }} + only-severities: critical + exit-code: true + write-comment: false + - name: Push Docker image with commit SHA run: | docker push \ diff --git a/.github/workflows/02-deploy-staging.yml b/.github/workflows/02-deploy-staging.yml index 006bb70d..83e0c147 100644 --- a/.github/workflows/02-deploy-staging.yml +++ b/.github/workflows/02-deploy-staging.yml @@ -14,8 +14,7 @@ jobs: name: Deploy to Staging runs-on: ubuntu-latest - if: > - ${{ github.event.workflow_run.conclusion == 'success' }} + if: github.event.workflow_run.conclusion == 'success' environment: name: staging diff --git a/.github/workflows/03-staging-test.yml b/.github/workflows/03-staging-test.yml index 01daf5fa..73693aca 100644 --- a/.github/workflows/03-staging-test.yml +++ b/.github/workflows/03-staging-test.yml @@ -14,8 +14,7 @@ jobs: name: Staging Smoke Test runs-on: ubuntu-latest - if: > - ${{ github.event.workflow_run.conclusion == 'success' }} + if: github.event.workflow_run.conclusion == 'success' environment: name: staging diff --git a/.github/workflows/05-deploy-monitoring.yml b/.github/workflows/05-deploy-monitoring.yml new file mode 100644 index 00000000..691dbe43 --- /dev/null +++ b/.github/workflows/05-deploy-monitoring.yml @@ -0,0 +1,152 @@ +name: 05 - Deploy Monitoring + +on: + # Runs after the application has been deployed to production, so the + # dashboards always have the running application to show. + workflow_run: + workflows: + - "04 - Deploy to Production" + types: + - completed + branches: + - main + + # Manual trigger, for example to redeploy after changing the Helm values + workflow_dispatch: + +jobs: + deploy-monitoring: + name: Deploy and validate Prometheus and Grafana + runs-on: ubuntu-latest + + if: > + github.event_name == 'workflow_dispatch' || + github.event.workflow_run.conclusion == 'success' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Azure + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS credentials + run: | + az aks get-credentials \ + --resource-group ${{ vars.AKS_RESOURCE_GROUP }} \ + --name ${{ vars.AKS_CLUSTER_NAME }} \ + --overwrite-existing + + - name: Create monitoring namespace + run: | + kubectl create namespace monitoring \ + --dry-run=client \ + -o yaml | kubectl apply -f - + + - name: Create Grafana admin secret + run: | + kubectl create secret generic grafana-admin \ + --namespace monitoring \ + --from-literal=admin-user="admin" \ + --from-literal=admin-password="${{ secrets.GRAFANA_ADMIN_PASSWORD }}" \ + --dry-run=client \ + -o yaml | kubectl apply -f - + + - name: Add the Prometheus community Helm repository + run: | + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + + # Prometheus, Grafana, kube-state-metrics and node-exporter in one release + - name: Install or upgrade kube-prometheus-stack + run: | + helm upgrade --install kube-prometheus-stack \ + prometheus-community/kube-prometheus-stack \ + --version 91.8.2 \ + --namespace monitoring \ + --values kubernetes/monitoring/values.yaml \ + --wait \ + --timeout 10m + + # Grafana finds this ConfigMap through the grafana_dashboard label + - name: Deploy the KoalaTech dashboard + run: | + kubectl create configmap koalatech-dashboard \ + --namespace monitoring \ + --from-file=koalatech-dashboard.json=kubernetes/monitoring/koalatech-dashboard.json \ + --dry-run=client \ + -o yaml \ + | kubectl label --local -f - grafana_dashboard=1 -o yaml \ + | kubectl apply -f - + + - name: Show monitoring resources + run: | + kubectl get pods -n monitoring + kubectl get servicemonitors -n monitoring + + # --------------------------------------------------------- + # Validation + # --------------------------------------------------------- + - name: Validate Prometheus is collecting metrics + run: | + PROM="/api/v1/namespaces/monitoring/services/http:kube-prometheus-stack-prometheus:9090/proxy/api/v1/query" + + # Targets need a minute or two after install before they are scraped + for _ in {1..20}; do + UP=$(kubectl get --raw "$PROM?query=up" \ + | jq '[.data.result[] | select(.value[1]=="1")] | length') + TOTAL=$(kubectl get --raw "$PROM?query=up" | jq '.data.result | length') + echo "Targets up: $UP of $TOTAL" + if [ "$TOTAL" -gt 0 ] && [ "$UP" -ge 5 ]; then + break + fi + sleep 15 + done + + if [ "$UP" -lt 5 ]; then + echo "Prometheus does not have enough healthy targets." + exit 1 + fi + + # Application metrics: running pods in the production namespace + QUERY='sum(kube_pod_status_phase%7Bnamespace%3D%22production%22%2Cphase%3D%22Running%22%7D)' + PODS=$(kubectl get --raw "$PROM?query=$QUERY" | jq -r '.data.result[0].value[1]') + echo "Running pods in production according to Prometheus: $PODS" + test "$PODS" != "null" && test "${PODS%.*}" -ge 1 + + { + echo '### Prometheus validation' + echo "- Healthy scrape targets: $UP of $TOTAL" + echo "- Running pods in production: $PODS" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Validate Grafana and the dashboard + run: | + kubectl port-forward -n monitoring \ + svc/kube-prometheus-stack-grafana 3000:80 > /dev/null 2>&1 & + sleep 10 + + curl --fail --silent --retry 10 --retry-delay 5 --retry-connrefused \ + http://localhost:3000/api/health | jq . + + # Prometheus must be registered as a Grafana data source + curl --fail --silent \ + -u "admin:${{ secrets.GRAFANA_ADMIN_PASSWORD }}" \ + http://localhost:3000/api/datasources | jq -r '.[].name' + + # The KoalaTech dashboard must have been picked up + FOUND=$(curl --fail --silent \ + -u "admin:${{ secrets.GRAFANA_ADMIN_PASSWORD }}" \ + "http://localhost:3000/api/search?query=KoalaTech" | jq 'length') + echo "KoalaTech dashboards found in Grafana: $FOUND" + test "$FOUND" -ge 1 + + echo "- Grafana healthy, KoalaTech dashboard provisioned" >> "$GITHUB_STEP_SUMMARY" + + - name: Check the application is still operational + run: | + kubectl wait --for=condition=Available deployment --all \ + -n production --timeout=180s + kubectl get pods -n production diff --git a/.github/workflows/99-infrastructure-teardown.yml b/.github/workflows/99-infrastructure-teardown.yml new file mode 100644 index 00000000..09b6db23 --- /dev/null +++ b/.github/workflows/99-infrastructure-teardown.yml @@ -0,0 +1,39 @@ +name: 99 - Infrastructure Teardown + +# Manual only. Removes the Azure resources managed by Terraform so the cloud +# lab does not keep costing credits. The Terraform state storage is kept. +on: + workflow_dispatch: + inputs: + confirm: + description: 'Type "teardown" to confirm' + required: true + type: string + +jobs: + teardown: + name: Terraform teardown + runs-on: ubuntu-latest + + if: ${{ inputs.confirm == 'teardown' }} + + concurrency: + group: terraform-state + cancel-in-progress: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Azure, remote state and Terraform + uses: ./.github/actions/terraform-azure + with: + azure-credentials: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform plan for the teardown + working-directory: terraform + run: terraform plan -destroy -input=false -no-color -out=tfdestroy + + - name: Terraform teardown + working-directory: terraform + run: terraform apply -input=false -no-color tfdestroy diff --git a/.gitignore b/.gitignore index 90cec08c..35382717 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,6 @@ terraform/.terraform/ terraform/*.tfstate terraform/*.tfstate.* terraform/tfplan -terraform/.terraform.lock.hcl # Distribution / packaging .Python diff --git a/course-service/requirements.txt b/course-service/requirements.txt index 58a1178e..6af82b5e 100644 --- a/course-service/requirements.txt +++ b/course-service/requirements.txt @@ -1,9 +1,9 @@ -fastapi==0.116.1 +fastapi==0.142.2 uvicorn[standard]==0.35.0 SQLAlchemy==2.0.43 psycopg2-binary==2.9.10 pydantic==2.11.7 -PyJWT==2.10.1 -python-dotenv==1.0.1 -pytest==8.4.1 +PyJWT==2.15.1 +python-dotenv==1.2.4 +pytest==9.1.1 httpx==0.28.1 \ No newline at end of file diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml new file mode 100644 index 00000000..2564fffe --- /dev/null +++ b/docker-compose.monitoring.yml @@ -0,0 +1,61 @@ +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + restart: unless-stopped + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + ports: + - "9090:9090" + networks: + - koalatech-network + + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + restart: unless-stopped + ports: + - "9100:9100" + networks: + - koalatech-network + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.47.2 + container_name: cadvisor + restart: unless-stopped + privileged: true + ports: + - "8081:8080" + volumes: + - /:/rootfs:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + networks: + - koalatech-network + + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + ports: + - "3001:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - grafana-data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/etc/grafana/dashboards:ro + networks: + - koalatech-network + +volumes: + prometheus-data: + grafana-data: + +networks: + koalatech-network: + driver: bridge diff --git a/enrollment-service/requirements.txt b/enrollment-service/requirements.txt index 58a1178e..6af82b5e 100644 --- a/enrollment-service/requirements.txt +++ b/enrollment-service/requirements.txt @@ -1,9 +1,9 @@ -fastapi==0.116.1 +fastapi==0.142.2 uvicorn[standard]==0.35.0 SQLAlchemy==2.0.43 psycopg2-binary==2.9.10 pydantic==2.11.7 -PyJWT==2.10.1 -python-dotenv==1.0.1 -pytest==8.4.1 +PyJWT==2.15.1 +python-dotenv==1.2.4 +pytest==9.1.1 httpx==0.28.1 \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 6a1bcec6..ae258e47 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -22,7 +22,7 @@ ENV VITE_ENROLLMENT_SERVICE_URL=${VITE_ENROLLMENT_SERVICE_URL} RUN npm run build -FROM nginx:1.27-alpine +FROM nginx:1.31-alpine COPY nginx.conf /etc/nginx/conf.d/default.conf diff --git a/kubernetes/monitoring/koalatech-dashboard.json b/kubernetes/monitoring/koalatech-dashboard.json new file mode 100644 index 00000000..4ec0027d --- /dev/null +++ b/kubernetes/monitoring/koalatech-dashboard.json @@ -0,0 +1,480 @@ +{ + "uid": "koalatech-ops", + "title": "KoalaTech Operations", + "tags": [ + "koalatech" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "Running pods", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\",phase=\"Running\"})", + "legendFormat": "pods", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Pods in the Running phase in the selected namespaces", + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 2, + "title": "Pods not ready", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(kube_pod_status_ready{namespace=~\"$namespace\",condition=\"false\"})", + "legendFormat": "not ready", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 3, + "title": "Container restarts", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"})", + "legendFormat": "restarts", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 4, + "title": "Nodes ready", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", + "legendFormat": "nodes", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 5, + "title": "CPU usage by pod", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"\",pod!=\"\"}[5m]))", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "description": "CPU cores used by each application pod", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 6, + "title": "Memory usage by pod", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",container!=\"\",pod!=\"\"})", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 7, + "title": "Network received by pod", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod!=\"\"}[5m]))", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 8, + "title": "Network transmitted by pod", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod!=\"\"}[5m]))", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 9, + "title": "Node CPU usage", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 21, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "100 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "description": "Infrastructure level CPU from node-exporter", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 10, + "title": "Node memory usage", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 21, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + } + ], + "templating": { + "list": [ + { + "name": "datasource", + "label": "Data source", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0 + }, + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(kube_pod_info, namespace)", + "refId": "ns" + }, + "regex": "/^(staging|production)$/", + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + } + ] + } +} \ No newline at end of file diff --git a/kubernetes/monitoring/values.yaml b/kubernetes/monitoring/values.yaml new file mode 100644 index 00000000..8175a9e2 --- /dev/null +++ b/kubernetes/monitoring/values.yaml @@ -0,0 +1,56 @@ +# Helm values for the kube-prometheus-stack chart (Prometheus, Grafana, +# kube-state-metrics and node-exporter), tuned for the small AKS cluster. + +grafana: + # The admin password comes from a Kubernetes secret created by the pipeline, + # so no password is stored in this repository. + admin: + existingSecret: grafana-admin + userKey: admin-user + passwordKey: admin-password + defaultDashboardsTimezone: browser + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + resources: + requests: + cpu: 50m + memory: 128Mi + +prometheus: + prometheusSpec: + retention: 24h + # Scrape every ServiceMonitor / PodMonitor, not only the ones from this release. + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + resources: + requests: + cpu: 100m + memory: 400Mi + +# Not needed for this practical, keeps the footprint small. +alertmanager: + enabled: false + +# These control plane components are managed by AKS and cannot be scraped. +kubeControllerManager: + enabled: false +kubeScheduler: + enabled: false +kubeEtcd: + enabled: false +kubeProxy: + enabled: false + +kube-state-metrics: + resources: + requests: + cpu: 10m + memory: 64Mi + +prometheus-node-exporter: + resources: + requests: + cpu: 10m + memory: 32Mi diff --git a/lecturer-service/requirements.txt b/lecturer-service/requirements.txt index feeedd2d..030f04f6 100644 --- a/lecturer-service/requirements.txt +++ b/lecturer-service/requirements.txt @@ -1,11 +1,11 @@ -fastapi==0.116.1 +fastapi==0.142.2 uvicorn[standard]==0.35.0 SQLAlchemy==2.0.43 psycopg2-binary==2.9.10 pydantic[email]==2.11.7 -PyJWT==2.10.1 -python-multipart==0.0.20 -python-dotenv==1.0.1 +PyJWT==2.15.1 +python-multipart==0.0.32 +python-dotenv==1.2.4 azure-storage-blob==12.26.0 -pytest==8.4.1 +pytest==9.1.1 httpx==0.28.1 \ No newline at end of file diff --git a/monitoring/grafana/dashboards/prometheus-self-monitoring.json b/monitoring/grafana/dashboards/prometheus-self-monitoring.json new file mode 100644 index 00000000..75100031 --- /dev/null +++ b/monitoring/grafana/dashboards/prometheus-self-monitoring.json @@ -0,0 +1,79 @@ +{ + "title": "Prometheus Self-Monitoring (Application Metrics)", + "uid": "prom-self-monitoring", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "15s", + "time": { "from": "now-30m", "to": "now" }, + "panels": [ + { + "id": 1, + "title": "Scrape Targets Up", + "type": "stat", + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 }, + "targets": [ + { "expr": "sum(up)", "legendFormat": "targets up" } + ], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "id": 2, + "title": "TSDB Head Series", + "type": "stat", + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 }, + "targets": [ + { "expr": "prometheus_tsdb_head_series", "legendFormat": "head series" } + ], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "id": 3, + "title": "Samples Ingested / sec", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])", + "legendFormat": "samples/s" + } + ] + }, + { + "id": 4, + "title": "Prometheus HTTP Request Rate by Endpoint", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "targets": [ + { + "expr": "sum(rate(prometheus_http_requests_total[5m])) by (handler)", + "legendFormat": "{{handler}}" + } + ] + }, + { + "id": 5, + "title": "Scrape Duration by Job", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "targets": [ + { + "expr": "scrape_duration_seconds", + "legendFormat": "{{job}}" + } + ] + }, + { + "id": 6, + "title": "Query Duration (90th percentile)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "targets": [ + { + "expr": "prometheus_engine_query_duration_seconds{quantile=\"0.9\"}", + "legendFormat": "{{slice}}" + } + ] + } + ] +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboard.yml b/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 00000000..86e9cff9 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: "KoalaTech provisioned dashboards" + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /etc/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/datasource.yml b/monitoring/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 00000000..1a57b69c --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 00000000..145aca53 --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + # Prometheus monitoring itself - an example of application-level metrics + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + # cAdvisor - per-container resource usage (container performance) + - job_name: "cadvisor" + static_configs: + - targets: ["cadvisor:8080"] + + # Node Exporter - host/infrastructure metrics (CPU, memory, disk, network) + - job_name: "node-exporter" + static_configs: + - targets: ["node-exporter:9100"] diff --git a/student-service/requirements.txt b/student-service/requirements.txt index feeedd2d..030f04f6 100644 --- a/student-service/requirements.txt +++ b/student-service/requirements.txt @@ -1,11 +1,11 @@ -fastapi==0.116.1 +fastapi==0.142.2 uvicorn[standard]==0.35.0 SQLAlchemy==2.0.43 psycopg2-binary==2.9.10 pydantic[email]==2.11.7 -PyJWT==2.10.1 -python-multipart==0.0.20 -python-dotenv==1.0.1 +PyJWT==2.15.1 +python-multipart==0.0.32 +python-dotenv==1.2.4 azure-storage-blob==12.26.0 -pytest==8.4.1 +pytest==9.1.1 httpx==0.28.1 \ No newline at end of file diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 00000000..676de4ed --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,23 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.81.0" + constraints = "~> 4.0" + hashes = [ + "h1:XhToZua4gtih1Kv8RdStcfND83G4Tmb6GZFT4jEUhDU=", + "h1:olVprUWhmFTARlydw/r7VIx4fata2gy3zsAWfPGRrhg=", + "zh:0732e7b74264ddfa2b90ba69d01c283d3cbae9f72ed3e506c6ac92529fed7fd3", + "zh:12afb524e232fe4e3d6161927724af5dfa4831d71edd9c174917ca9b7377bfae", + "zh:169d619ae202c4145e02fb706fb7c3679445ab3e3ff722edbf89597517a8c92e", + "zh:6beb95a3ef2f2d9c76abaa48e5450e90686a3fb6a47f1cb0ff7c5e94b6960151", + "zh:705e075fb5ffc4bf66fd7cbabf1a65007a41621e80030a2c158a4c83b6046216", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79a8d17fefe647040fcb9ee8821a4f09f395427c4fd49493489b9a93a9a1038e", + "zh:8cc3f900b3774c0ae37ae42365c4579a199cf9e5edf88e476fdf5ab1048f84ea", + "zh:dec373b9390fa95e257291acd018ed65a7d512b428645d35e22cdbe8b245a08b", + "zh:e60f1e9fb45df6defade2855ed6e68547409ea75d30655c556adb0c08579749b", + "zh:f901d12ec82f3f8b5880a27b5cbcd7bd0d97e60c9367a2d7ed82fdd1157b39ff", + "zh:facf68ea5bf0f2b8ba720e7fba5f86492e1d4c591100460bb91c3f79f391f4b6", + ] +} diff --git a/terraform/container_registry.tf b/terraform/container_registry.tf new file mode 100644 index 00000000..1a5f350f --- /dev/null +++ b/terraform/container_registry.tf @@ -0,0 +1,15 @@ +resource "azurerm_container_registry" "acr" { + name = var.acr_name + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + + sku = "Basic" + admin_enabled = true + + tags = merge( + var.tags, + { + Environment = var.environment + } + ) +} \ No newline at end of file diff --git a/terraform/kubernetes_serivce.tf b/terraform/kubernetes_serivce.tf new file mode 100644 index 00000000..ebfd7692 --- /dev/null +++ b/terraform/kubernetes_serivce.tf @@ -0,0 +1,34 @@ +resource "azurerm_kubernetes_cluster" "aks" { + name = var.aks_cluster_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + dns_prefix = var.aks_dns_prefix + kubernetes_version = var.kubernetes_version + + default_node_pool { + name = "default" + node_count = var.aks_node_count + vm_size = var.aks_node_vm_size + } + + identity { + type = "SystemAssigned" + } + + tags = merge( + var.tags, + { + Environment = var.environment + } + ) +} + +# +# Grant AKS permission to pull images from your ACR +# +resource "azurerm_role_assignment" "acr_pull" { + principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + role_definition_name = "AcrPull" + scope = azurerm_container_registry.acr.id + skip_service_principal_aad_check = true +} \ No newline at end of file diff --git a/terraform/output.tf b/terraform/output.tf new file mode 100644 index 00000000..36e88828 --- /dev/null +++ b/terraform/output.tf @@ -0,0 +1,57 @@ +output "resource_group_name" { + description = "Name of the resource group" + value = azurerm_resource_group.rg.name +} + +output "acr_name" { + description = "Name of the Azure Container Registry" + value = azurerm_container_registry.acr.name +} + +output "acr_login_server" { + description = "Login server of the Azure Container Registry" + value = azurerm_container_registry.acr.login_server +} + +output "storage_account_name" { + description = "Name of the Azure Storage Account" + value = azurerm_storage_account.storage_account.name +} + +output "storage_connection_string" { + description = "Connection string used by the application to access Blob Storage" + value = azurerm_storage_account.storage_account.primary_connection_string + sensitive = true +} + +output "student_profile_container" { + description = "Student profile photo Blob container" + value = azurerm_storage_container.student_profile_photo.name +} + +output "lecturer_profile_container" { + description = "Lecturer profile photo Blob container" + value = azurerm_storage_container.lecturer_profile_photo.name +} + +output "aks_cluster_name" { + description = "Name of the AKS cluster" + value = azurerm_kubernetes_cluster.aks.name +} + +output "aks_get_credentials_command" { + description = "Azure CLI command used to configure kubectl" + value = join(" ", [ + "az aks get-credentials", + "--resource-group", + azurerm_resource_group.rg.name, + "--name", + azurerm_kubernetes_cluster.aks.name, + "--overwrite-existing" + ]) +} + +output "acr_login_command" { + description = "Azure CLI command used to log in to ACR" + value = "az acr login --name ${azurerm_container_registry.acr.name}" +} \ No newline at end of file diff --git a/terraform/resource_group.tf b/terraform/resource_group.tf new file mode 100644 index 00000000..0fbe75d3 --- /dev/null +++ b/terraform/resource_group.tf @@ -0,0 +1,11 @@ +resource "azurerm_resource_group" "rg" { + name = var.resource_group_name + location = var.location + + tags = merge( + var.tags, + { + Environment = var.environment + } + ) +} \ No newline at end of file diff --git a/terraform/storage_account.tf b/terraform/storage_account.tf new file mode 100644 index 00000000..8b3a6b6f --- /dev/null +++ b/terraform/storage_account.tf @@ -0,0 +1,30 @@ +resource "azurerm_storage_account" "storage_account" { + name = var.storage_account_name + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + + account_tier = "Standard" + account_replication_type = "LRS" + + min_tls_version = "TLS1_2" + allow_nested_items_to_be_public = false + + tags = merge( + var.tags, + { + Environment = var.environment + } + ) +} + +resource "azurerm_storage_container" "student_profile_photo" { + name = "student-profile-photo" + storage_account_id = azurerm_storage_account.storage_account.id + container_access_type = "private" +} + +resource "azurerm_storage_container" "lecturer_profile_photo" { + name = "lecturer-profile-photo" + storage_account_id = azurerm_storage_account.storage_account.id + container_access_type = "private" +} \ No newline at end of file diff --git a/terraform/terraform.tfvars b/terraform/terraform.tfvars new file mode 100644 index 00000000..86015cfb --- /dev/null +++ b/terraform/terraform.tfvars @@ -0,0 +1,13 @@ +location = "Australia East" +resource_group_name = "koalatech-week08-rg" +acr_name = "koalatechweek08acr30145" +storage_account_name = "koalatechweek08sa" +aks_cluster_name = "koalatech-week08-aks" +aks_dns_prefix = "koalatechweek08" +aks_node_count = 3 +environment = "development" +tags = { + Project = "KoalaTech Course Platform" + ManagedBy = "Terraform" + Practical = "Week08" +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 00000000..d9c95973 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,83 @@ +variable "location" { + description = "Azure region where the resources will be created" + type = string + default = "Australia East" +} + +variable "resource_group_name" { + description = "Name of the Azure Resource Group" + type = string +} + +variable "acr_name" { + description = "Globally unique name of the Azure Container Registry" + type = string + + validation { + condition = can(regex("^[a-zA-Z0-9]+$", var.acr_name)) + error_message = "The ACR name must contain only alphanumeric characters." + } +} + +variable "storage_account_name" { + description = "Globally unique name of the Azure Storage Account" + type = string + + validation { + condition = ( + length(var.storage_account_name) >= 3 && + length(var.storage_account_name) <= 24 && + can(regex("^[a-z0-9]+$", var.storage_account_name)) + ) + + error_message = "The storage account name must contain 3–24 lowercase letters and numbers." + } +} + +variable "aks_cluster_name" { + description = "Name of the Azure Kubernetes Service cluster" + type = string +} + +variable "aks_dns_prefix" { + description = "DNS prefix used by the AKS cluster" + type = string +} + +variable "aks_node_count" { + description = "Number of nodes in the default AKS node pool" + type = number + default = 2 + + validation { + condition = var.aks_node_count >= 1 + error_message = "The AKS node count must be at least 1." + } +} + +variable "aks_node_vm_size" { + description = "Virtual machine size used by the AKS nodes" + type = string + default = "Standard_D2s_v3" +} + +variable "environment" { + description = "Environment name applied to resource tags" + type = string + default = "development" +} + +variable "kubernetes_version" { + default = "1.36.1" +} + +variable "tags" { + description = "Tags applied to Azure resources" + type = map(string) + + default = { + Project = "KoalaTech Course Platform" + ManagedBy = "Terraform" + Practical = "Week06" + } +} \ No newline at end of file diff --git a/terraform/versions.tf b/terraform/versions.tf new file mode 100644 index 00000000..66264a84 --- /dev/null +++ b/terraform/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.7.0" + + # Remote state so the GitHub Actions pipeline and local runs share one state. + # The storage details are passed in with -backend-config at terraform init. + backend "azurerm" {} + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + } +} + +provider "azurerm" { + features {} +} \ No newline at end of file diff --git a/user-service/requirements.txt b/user-service/requirements.txt index 02679ad8..ca65e166 100644 --- a/user-service/requirements.txt +++ b/user-service/requirements.txt @@ -1,11 +1,11 @@ -fastapi==0.116.1 +fastapi==0.142.2 uvicorn[standard]==0.35.0 SQLAlchemy==2.0.43 psycopg2-binary==2.9.10 pydantic[email]==2.11.7 -PyJWT +PyJWT==2.15.1 pwdlib[argon2]==0.2.1 -python-multipart==0.0.20 -pytest==8.4.1 +python-multipart==0.0.32 +pytest==9.1.1 httpx==0.28.1 -python-dotenv==1.0.1 \ No newline at end of file +python-dotenv==1.2.4 \ No newline at end of file From 354e27366e155c35f5e6c0b2f6d8ccc32bcd3a53 Mon Sep 17 00:00:00 2001 From: PRIYANSHI Date: Fri, 2 Oct 2026 15:56:55 +1000 Subject: [PATCH 6/7] Deploy monitoring as a called workflow after production GitHub does not start workflow_run chains deeper than three levels, so 05 never ran after 04. 05 is now a reusable workflow called from 04. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/04-deploy-production.yml | 9 ++++++++- .github/workflows/05-deploy-monitoring.yml | 18 +++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/04-deploy-production.yml b/.github/workflows/04-deploy-production.yml index cf05fecd..0601c1ad 100644 --- a/.github/workflows/04-deploy-production.yml +++ b/.github/workflows/04-deploy-production.yml @@ -162,4 +162,11 @@ jobs: run: | kubectl get pods -n production kubectl get services -n production - kubectl get pvc -n production \ No newline at end of file + kubectl get pvc -n production + + # Prometheus and Grafana are deployed and validated once production is up + monitoring: + name: Deploy Monitoring + needs: deploy-production + uses: ./.github/workflows/05-deploy-monitoring.yml + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/05-deploy-monitoring.yml b/.github/workflows/05-deploy-monitoring.yml index 691dbe43..f010c304 100644 --- a/.github/workflows/05-deploy-monitoring.yml +++ b/.github/workflows/05-deploy-monitoring.yml @@ -1,15 +1,11 @@ name: 05 - Deploy Monitoring on: - # Runs after the application has been deployed to production, so the - # dashboards always have the running application to show. - workflow_run: - workflows: - - "04 - Deploy to Production" - types: - - completed - branches: - - main + # Called by 04 - Deploy to Production once the application is deployed, so + # the dashboards always have the running application to show. GitHub does not + # allow workflow_run chains deeper than three levels, so this is a called + # workflow instead of a fifth workflow in the chain. + workflow_call: # Manual trigger, for example to redeploy after changing the Helm values workflow_dispatch: @@ -19,10 +15,6 @@ jobs: name: Deploy and validate Prometheus and Grafana runs-on: ubuntu-latest - if: > - github.event_name == 'workflow_dispatch' || - github.event.workflow_run.conclusion == 'success' - steps: - name: Checkout repository uses: actions/checkout@v4 From c7e4ba050d9f1321989adfb07dcc7ad3183f9234 Mon Sep 17 00:00:00 2001 From: PRIYANSHI Date: Fri, 2 Oct 2026 17:04:19 +1000 Subject: [PATCH 7/7] Add metrics-driven canary releases with automated rollback for course-service --- .github/workflows/04-deploy-production.yml | 41 +++++- course-service/Dockerfile | 4 + course-service/app/main.py | 28 +++- course-service/requirements.txt | 3 +- course-service/tests/test_courses.py | 9 +- k6/load.js | 15 ++ kubernetes/production/10-course-service.yaml | 58 -------- kubernetes/progressive/analysis-template.yaml | 39 ++++++ .../progressive/course-service-rollout.yaml | 129 ++++++++++++++++++ terraform/.terraform.lock.hcl | 21 +++ terraform/progressive_delivery.tf | 37 +++++ terraform/versions.tf | 17 ++- 12 files changed, 332 insertions(+), 69 deletions(-) create mode 100644 k6/load.js delete mode 100644 kubernetes/production/10-course-service.yaml create mode 100644 kubernetes/progressive/analysis-template.yaml create mode 100644 kubernetes/progressive/course-service-rollout.yaml create mode 100644 terraform/progressive_delivery.tf diff --git a/.github/workflows/04-deploy-production.yml b/.github/workflows/04-deploy-production.yml index 0601c1ad..d84d68e7 100644 --- a/.github/workflows/04-deploy-production.yml +++ b/.github/workflows/04-deploy-production.yml @@ -85,6 +85,12 @@ jobs: run: | kubectl apply \ -f kubernetes/production/ + # course-service is released as an Argo Rollouts canary + # The image is substituted before applying, otherwise re-applying the + # "placeholder" image would itself trigger a canary release. + sed "s|image: placeholder|image: ${{ vars.ACR_LOGIN_SERVER }}/koalatech-course-service:${{ env.IMAGE_TAG }}|" \ + kubernetes/progressive/course-service-rollout.yaml | kubectl apply -f - + kubectl apply -f kubernetes/progressive/analysis-template.yaml - name: Update frontend image run: | @@ -110,11 +116,12 @@ jobs: lecturer-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-lecturer-service:${{ env.IMAGE_TAG }} \ -n production - - name: Update course-service image + - name: Install Argo Rollouts kubectl plugin run: | - kubectl set image deployment/course-service \ - course-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-course-service:${{ env.IMAGE_TAG }} \ - -n production + curl -fsSLo kubectl-argo-rollouts \ + https://github.com/argoproj/argo-rollouts/releases/download/v1.8.3/kubectl-argo-rollouts-linux-amd64 + chmod +x kubectl-argo-rollouts + sudo mv kubectl-argo-rollouts /usr/local/bin/ - name: Update enrollment-service image run: | @@ -146,11 +153,31 @@ jobs: -n production \ --timeout=300s - - name: Wait for course-service rollout + - name: Install Argo Rollouts kubectl plugin + run: | + curl -fsSLo kubectl-argo-rollouts \ + https://github.com/argoproj/argo-rollouts/releases/download/v1.8.3/kubectl-argo-rollouts-linux-amd64 + chmod +x kubectl-argo-rollouts + sudo mv kubectl-argo-rollouts /usr/local/bin/ + + # Exits non-zero if analysis fails and the rollout is aborted, which + # fails this job. The canary is already back at 0% by then. + - name: Wait for course-service canary analysis + id: canary run: | - kubectl rollout status deployment/course-service \ + kubectl argo rollouts status course-service \ -n production \ - --timeout=300s + --timeout 600s + + - name: Alert Discord on automatic rollback + if: failure() && steps.canary.outcome == 'failure' + run: | + kubectl argo rollouts get rollout course-service -n production > rollout.txt || true + FAILED=$(kubectl get analysisrun -n production --sort-by=.metadata.creationTimestamp \ + -o jsonpath='{.items[-1:].status.metricResults[?(@.phase=="Failed")].name}') + BODY=$(jq -n --arg tag "${{ env.IMAGE_TAG }}" --arg m "$FAILED" --arg run "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + '{content: ("Canary ABORTED and rolled back. Revision: `" + $tag + "`, failing metric: `" + $m + "`. " + $run)}') + curl -fsS -H "Content-Type: application/json" -d "$BODY" "${{ secrets.DISCORD_WEBHOOK_URL }}" - name: Wait for enrollment-service rollout run: | diff --git a/course-service/Dockerfile b/course-service/Dockerfile index d750db8d..d6f323da 100644 --- a/course-service/Dockerfile +++ b/course-service/Dockerfile @@ -16,6 +16,10 @@ RUN useradd -m appuser \ USER appuser +# Set only for the deliberately faulty demo image +ARG FAULT_RATE=0 +ENV FAULT_RATE=${FAULT_RATE} + EXPOSE 8000 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/course-service/app/main.py b/course-service/app/main.py index 7c8cbfb2..de0dfa1b 100644 --- a/course-service/app/main.py +++ b/course-service/app/main.py @@ -1,8 +1,12 @@ import logging +import os +import random import time from contextlib import asynccontextmanager -from fastapi import FastAPI +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from prometheus_fastapi_instrumentator import Instrumentator from sqlalchemy.exc import OperationalError from app.db import Base, engine @@ -64,6 +68,28 @@ async def lifespan(_: FastAPI): ) +# Demo fault injection: a "faulty" image is built with FAULT_RATE > 0 so the +# canary analysis has a bad release to catch. Production images use 0. +FAULT_RATE = float(os.getenv("FAULT_RATE", "0")) + + +@app.middleware("http") +async def inject_faults(request: Request, call_next): + if ( + FAULT_RATE > 0 + and request.url.path.startswith("/courses") + and random.random() < FAULT_RATE + ): + return JSONResponse( + {"detail": "Injected fault"}, status_code=500 + ) + return await call_next(request) + + +# Exposes request count and latency histograms at /metrics for Prometheus +Instrumentator().instrument(app).expose(app, include_in_schema=False) + + app.include_router(courses.router) diff --git a/course-service/requirements.txt b/course-service/requirements.txt index 6af82b5e..30e536a9 100644 --- a/course-service/requirements.txt +++ b/course-service/requirements.txt @@ -6,4 +6,5 @@ pydantic==2.11.7 PyJWT==2.15.1 python-dotenv==1.2.4 pytest==9.1.1 -httpx==0.28.1 \ No newline at end of file +httpx==0.28.1 +prometheus-fastapi-instrumentator==7.1.0 \ No newline at end of file diff --git a/course-service/tests/test_courses.py b/course-service/tests/test_courses.py index 93a3d420..6a2b6e07 100644 --- a/course-service/tests/test_courses.py +++ b/course-service/tests/test_courses.py @@ -299,4 +299,11 @@ def test_request_without_token_returns_401( ): response = client.get("/courses") - assert response.status_code == 401 \ No newline at end of file + assert response.status_code == 401 + +def test_metrics_endpoint_exposes_prometheus_metrics(client): + client.get("/health") + response = client.get("/metrics") + + assert response.status_code == 200 + assert "http_requests_total" in response.text diff --git a/k6/load.js b/k6/load.js new file mode 100644 index 00000000..9387ff71 --- /dev/null +++ b/k6/load.js @@ -0,0 +1,15 @@ +// Continuous traffic so the canary has requests to be judged on. +// Run: k6 run -e BASE_URL=http:// k6/load.js +import http from 'k6/http'; +import { check } from 'k6'; + +export const options = { + vus: 10, + duration: '10m', + thresholds: { http_req_failed: ['rate<0.01'] }, +}; + +export default function () { + const res = http.get(`${__ENV.BASE_URL}/courses/`); + check(res, { 'status is 200': (r) => r.status === 200 }); +} diff --git a/kubernetes/production/10-course-service.yaml b/kubernetes/production/10-course-service.yaml deleted file mode 100644 index 3cdc7844..00000000 --- a/kubernetes/production/10-course-service.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: course-service - namespace: production -spec: - replicas: 1 - selector: - matchLabels: - app: course-service - template: - metadata: - labels: - app: course-service - spec: - containers: - - name: course-service - image: placeholder - imagePullPolicy: Always - ports: - - containerPort: 8000 - env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: application-secret - key: POSTGRES_USER - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: application-secret - key: POSTGRES_PASSWORD - - name: POSTGRES_DB - value: courses - - name: POSTGRES_HOST - value: course-db - - name: POSTGRES_PORT - value: "5432" - - name: JWT_SECRET_KEY - valueFrom: - secretKeyRef: - name: application-secret - key: JWT_SECRET_KEY - - name: JWT_ALGORITHM - value: HS256 - ---- -apiVersion: v1 -kind: Service -metadata: - name: course-service - namespace: production -spec: - selector: - app: course-service - ports: - - port: 8000 - targetPort: 8000 \ No newline at end of file diff --git a/kubernetes/progressive/analysis-template.yaml b/kubernetes/progressive/analysis-template.yaml new file mode 100644 index 00000000..b4bd4c76 --- /dev/null +++ b/kubernetes/progressive/analysis-template.yaml @@ -0,0 +1,39 @@ +# Runs at every canary step. Three consecutive bad measurements abort the +# rollout (failureLimit: 2 allows two failures, the third fails the run), so a +# single noisy sample cannot kill a healthy release. +apiVersion: argoproj.io/v1alpha1 +kind: AnalysisTemplate +metadata: + name: course-service-health + namespace: production +spec: + args: + - name: canary-hash + metrics: + - name: error-ratio + interval: 20s + count: 6 + failureLimit: 2 + inconclusiveLimit: 2 + successCondition: result[0] < 0.05 + provider: + prometheus: + address: http://kube-prometheus-stack-prometheus.monitoring.svc:9090 + query: | + ( + sum(rate(http_requests_total{namespace="production",status="5xx",rollouts_pod_template_hash="{{args.canary-hash}}"}[1m])) + / + sum(rate(http_requests_total{namespace="production",rollouts_pod_template_hash="{{args.canary-hash}}"}[1m])) + ) or vector(0) + - name: p95-latency + interval: 20s + count: 6 + failureLimit: 2 + successCondition: result[0] < 0.5 + provider: + prometheus: + address: http://kube-prometheus-stack-prometheus.monitoring.svc:9090 + query: | + histogram_quantile(0.95, + sum by (le) (rate(http_request_duration_seconds_bucket{namespace="production",rollouts_pod_template_hash="{{args.canary-hash}}"}[1m])) + ) or vector(0) diff --git a/kubernetes/progressive/course-service-rollout.yaml b/kubernetes/progressive/course-service-rollout.yaml new file mode 100644 index 00000000..c9aa0c77 --- /dev/null +++ b/kubernetes/progressive/course-service-rollout.yaml @@ -0,0 +1,129 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: course-service + namespace: production +spec: + replicas: 3 + revisionHistoryLimit: 3 + strategy: + canary: + canaryService: course-service-canary + stableService: course-service + trafficRouting: + nginx: + stableIngress: course-service + analysis: + templates: + - templateName: course-service-health + args: + - name: canary-hash + valueFrom: + podTemplateHashValue: Latest + steps: + - setWeight: 10 + - pause: {duration: 90s} + - setWeight: 30 + - pause: {duration: 90s} + - setWeight: 60 + - pause: {duration: 90s} + selector: + matchLabels: + app: course-service + template: + metadata: + labels: + app: course-service + spec: + containers: + - name: course-service + image: placeholder + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + readinessProbe: + httpGet: {path: /health, port: 8000} + periodSeconds: 5 + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: application-secret + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: application-secret + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + value: courses + - name: POSTGRES_HOST + value: course-db + - name: POSTGRES_PORT + value: "5432" + - name: JWT_SECRET_KEY + valueFrom: + secretKeyRef: + name: application-secret + key: JWT_SECRET_KEY + - name: JWT_ALGORITHM + value: HS256 +--- +apiVersion: v1 +kind: Service +metadata: + name: course-service + namespace: production +spec: + selector: + app: course-service + ports: + - {port: 8000, targetPort: 8000, name: http} +--- +apiVersion: v1 +kind: Service +metadata: + name: course-service-canary + namespace: production +spec: + selector: + app: course-service + ports: + - {port: 8000, targetPort: 8000, name: http} +--- +# Stable ingress; Argo Rollouts creates a second "-canary" ingress with the +# nginx canary-weight annotation and adjusts it at each step. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: course-service + namespace: production +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: /courses + pathType: Prefix + backend: + service: + name: course-service + port: {number: 8000} +--- +# Scrape each pod and keep the Rollouts hash label so PromQL can tell the +# canary ReplicaSet apart from the stable one. +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: course-service + namespace: production +spec: + selector: + matchLabels: + app: course-service + podTargetLabels: [rollouts-pod-template-hash] + podMetricsEndpoints: + - port: http + path: /metrics + interval: 10s diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 676de4ed..868d606e 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -21,3 +21,24 @@ provider "registry.terraform.io/hashicorp/azurerm" { "zh:facf68ea5bf0f2b8ba720e7fba5f86492e1d4c591100460bb91c3f79f391f4b6", ] } + +provider "registry.terraform.io/hashicorp/helm" { + version = "3.3.0" + constraints = "~> 3.0" + hashes = [ + "h1:Y9Z+1+RqmCYRCqlncKEKPjYkRqlNIBiK3jdEIbUn1Fc=", + "zh:24a942352667cf49bed467c51a951f17a6c13e0cdf1c505b01b41f75a1a60d0f", + "zh:2c057eafd1e4d8b50acdc676f9a3096e38fb7d30c3fab3d64af82fa04a2f767e", + "zh:44252c60034ca163f2f68eaeece2583411bca25ac83bc58350e24d4f1c2d60a7", + "zh:4628da7c781eebbce1b13d12b498856815bde0e174d7a0d0b85a5d89a1d36d17", + "zh:59b3f4fd8e236c5733032c289a9636ac8bb6c92bc11b540d979b92d005f6cf98", + "zh:5d5f11201b986485ca1d33096f3545c6074b3b4a592864620a8df7f7c399b59f", + "zh:5e656afadb2db3f418eb254b2da02b00851eba8ec63196d1f22b2a2854fe9fb9", + "zh:77515e5576f4ea3d8d9e404237882de62518950ccd7b312371f5c03580f22fb0", + "zh:a4e601eddf9781b5f8678ee9c2b6fb0791fbe6ab85a53707efdf80f714a3011c", + "zh:eaa58edc8bb43aeb0fe6c0f99975cb24e2e9f5441e938a389fe848795a30b825", + "zh:ec97b17b82222d2008256f9b10b88f564d30f717a43438fe425a802ac0ccf2f3", + "zh:f18613d2aa3fd3945d394911e210af7bf58968f64631b6b2e229852b60a2803e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/progressive_delivery.tf b/terraform/progressive_delivery.tf new file mode 100644 index 00000000..79839b96 --- /dev/null +++ b/terraform/progressive_delivery.tf @@ -0,0 +1,37 @@ +# NGINX ingress splits traffic by canary weight and exports per-service metrics +resource "helm_release" "ingress_nginx" { + name = "ingress-nginx" + repository = "https://kubernetes.github.io/ingress-nginx" + chart = "ingress-nginx" + namespace = "ingress-nginx" + create_namespace = true + + set = [ + { + name = "controller.metrics.enabled" + value = "true" + }, + { + name = "controller.metrics.serviceMonitor.enabled" + value = "true" + }, + { + # Health probe path required for the Azure load balancer + name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/azure-load-balancer-health-probe-request-path" + value = "/healthz" + }, + ] + + depends_on = [azurerm_kubernetes_cluster.aks] +} + +# The controller and CRDs for Rollout and AnalysisTemplate +resource "helm_release" "argo_rollouts" { + name = "argo-rollouts" + repository = "https://argoproj.github.io/argo-helm" + chart = "argo-rollouts" + namespace = "argo-rollouts" + create_namespace = true + + depends_on = [azurerm_kubernetes_cluster.aks] +} diff --git a/terraform/versions.tf b/terraform/versions.tf index 66264a84..4c4032da 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -10,9 +10,24 @@ terraform { source = "hashicorp/azurerm" version = "~> 4.0" } + helm = { + source = "hashicorp/helm" + version = "~> 3.0" + } } } provider "azurerm" { features {} -} \ No newline at end of file +} + +# In-cluster components are installed with Helm through the AKS credentials, +# so the whole stack is created by one terraform apply. +provider "helm" { + kubernetes = { + host = azurerm_kubernetes_cluster.aks.kube_config[0].host + client_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].client_certificate) + client_key = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].client_key) + cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].cluster_ca_certificate) + } +}