Feature/progressive delivery

.github/actions/terraform-azure/action.yml

@@ -0,0 +1,87 @@
+name: Terraform on Azure
+description: >
+  Logs in to Azure, exposes the service principal to Terraform, makes sure the
+  remote state storage exists and runs terraform init against it.
+
+inputs:
+  azure-credentials:
+    description: Service principal JSON (the AZURE_CREDENTIALS secret)
+    required: true
+  working-directory:
+    description: Folder that holds the Terraform files
+    default: terraform
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Terraform
+      uses: hashicorp/setup-terraform@v3
+      with:
+        terraform_wrapper: false
+
+    - name: Login to Azure
+      uses: azure/login@v2
+      with:
+        creds: ${{ inputs.azure-credentials }}
+
+    - name: Pass the service principal to Terraform
+      shell: bash
+      env:
+        AZURE_CREDENTIALS: ${{ inputs.azure-credentials }}
+      run: |
+        echo "::add-mask::$(echo "$AZURE_CREDENTIALS" | jq -r .clientSecret)"
+        {
+          echo "ARM_CLIENT_ID=$(echo "$AZURE_CREDENTIALS" | jq -r .clientId)"
+          echo "ARM_CLIENT_SECRET=$(echo "$AZURE_CREDENTIALS" | jq -r .clientSecret)"
+          echo "ARM_TENANT_ID=$(echo "$AZURE_CREDENTIALS" | jq -r .tenantId)"
+          echo "ARM_SUBSCRIPTION_ID=$(echo "$AZURE_CREDENTIALS" | jq -r .subscriptionId)"
+        } >> "$GITHUB_ENV"
+
+    # The state lives in its own resource group so that a destroy of the
+    # application infrastructure never deletes the state it is working from.
+    # Every command below is idempotent.
+    - name: Make sure the Terraform state storage exists
+      shell: bash
+      run: |
+        STATE_RG="koalatech-tfstate-rg"
+        STATE_SA="tfstate$(echo "$ARM_SUBSCRIPTION_ID" | tr -d '-' | cut -c1-12)"
+
+        az group create --name "$STATE_RG" --location australiaeast --output none
+        az storage account create \
+          --name "$STATE_SA" \
+          --resource-group "$STATE_RG" \
+          --location australiaeast \
+          --sku Standard_LRS \
+          --min-tls-version TLS1_2 \
+          --allow-blob-public-access false \
+          --output none
+
+        STATE_KEY="$(az storage account keys list \
+          --resource-group "$STATE_RG" \
+          --account-name "$STATE_SA" \
+          --query '[0].value' --output tsv)"
+        echo "::add-mask::$STATE_KEY"
+
+        az storage container create \
+          --name tfstate \
+          --account-name "$STATE_SA" \
+          --account-key "$STATE_KEY" \
+          --output none
+
+        {
+          echo "TFSTATE_RG=$STATE_RG"
+          echo "TFSTATE_SA=$STATE_SA"
+          echo "ARM_ACCESS_KEY=$STATE_KEY"
+        } >> "$GITHUB_ENV"
+
+    - name: Terraform init
+      shell: bash
+      working-directory: ${{ inputs.working-directory }}
+      run: |
+        terraform init \
+          -input=false \
+          -reconfigure \
+          -backend-config="resource_group_name=$TFSTATE_RG" \
+          -backend-config="storage_account_name=$TFSTATE_SA" \
+          -backend-config="container_name=tfstate" \
+          -backend-config="key=week08.tfstate"

.github/workflows/01-ci.yml

@@ -5,13 +5,72 @@ on:
   push:
     branches:
       - main
-
+
+  # Pull requests run the Terraform plan and the tests only
+  pull_request:
+    branches:
+      - main
+
   # Manual trigger for the workflow
   workflow_dispatch:
 
 
 jobs:
 
+  # =========================================================
+  # Infrastructure as Code (Terraform)
+  # Runs first so the ACR and AKS cluster exist before images are pushed.
+  # Pull requests only get a plan, pushes to main also apply it.
+  # =========================================================
+  terraform:
+    name: Terraform
+    runs-on: ubuntu-latest
+
+    # Only one run at a time may touch the Terraform state
+    concurrency:
+      group: terraform-state
+      cancel-in-progress: false
+
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Azure, remote state and Terraform
+        uses: ./.github/actions/terraform-azure
+        with:
+          azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Terraform format check
+        working-directory: terraform
+        run: terraform fmt -check -recursive
+
+      - name: Terraform validate
+        working-directory: terraform
+        run: terraform validate
+
+      - name: Terraform plan
+        working-directory: terraform
+        run: |
+          terraform plan -input=false -no-color -out=tfplan | tee plan.txt
+          {
+            echo '### Terraform plan'
+            echo '```'
+            tail -n 25 plan.txt
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Terraform apply
+        if: github.event_name != 'pull_request'
+        working-directory: terraform
+        run: terraform apply -input=false -no-color tfplan
+
+      - name: Show Terraform outputs
+        if: github.event_name != 'pull_request'
+        working-directory: terraform
+        run: terraform output
+
+
   # =========================================================
   # Backend Tests
   # =========================================================
@@ -109,9 +168,10 @@ jobs:
     name: Build and Push ${{ matrix.image }}
     runs-on: ubuntu-latest
 
-    # All backend tests must pass before this job starts
+    # All backend tests must pass and the infrastructure must be ready
     needs:
       - backend-test
+      - terraform
 
     # Build and push when code is pushed to main or manually triggered
     if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
@@ -160,6 +220,46 @@ jobs:
             -t ${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.image }}:${{ github.sha }} \
             ./${{ matrix.service }}
 
+      # Docker Scout analyses the image that was just built, before it is pushed.
+      - name: Docker Scout - vulnerability report
+        uses: docker/scout-action@v1
+        with:
+          command: cves
+          image: local://${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.image }}:${{ github.sha }}
+          organization: ${{ secrets.DOCKER_HUB_USER }}
+          dockerhub-user: ${{ secrets.DOCKER_HUB_USER }}
+          dockerhub-password: ${{ secrets.DOCKER_HUB_PAT }}
+          only-severities: critical,high
+          only-fixed: true
+          summary: true
+          write-comment: false
+          exit-code: false
+
+      - name: Docker Scout - recommendations
+        uses: docker/scout-action@v1
+        with:
+          command: recommendations
+          image: local://${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.image }}:${{ github.sha }}
+          organization: ${{ secrets.DOCKER_HUB_USER }}
+          dockerhub-user: ${{ secrets.DOCKER_HUB_USER }}
+          dockerhub-password: ${{ secrets.DOCKER_HUB_PAT }}
+          summary: true
+          write-comment: false
+
+      # Security gate: critical vulnerabilities stop the pipeline before the
+      # image is pushed, so a critical image can never reach a deployment.
+      - name: Docker Scout - fail on critical vulnerabilities
+        uses: docker/scout-action@v1
+        with:
+          command: cves
+          image: local://${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.image }}:${{ github.sha }}
+          organization: ${{ secrets.DOCKER_HUB_USER }}
+          dockerhub-user: ${{ secrets.DOCKER_HUB_USER }}
+          dockerhub-password: ${{ secrets.DOCKER_HUB_PAT }}
+          only-severities: critical
+          exit-code: true
+          write-comment: false
+
       - name: Push Docker image with commit SHA
         run: |
           docker push \

.github/workflows/02-deploy-staging.yml

@@ -14,8 +14,7 @@ jobs:
     name: Deploy to Staging
     runs-on: ubuntu-latest
 
-    if: >
-      ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: github.event.workflow_run.conclusion == 'success'
 
     environment:
       name: staging

.github/workflows/03-staging-test.yml

@@ -14,8 +14,7 @@ jobs:
     name: Staging Smoke Test
     runs-on: ubuntu-latest
 
-    if: >
-      ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: github.event.workflow_run.conclusion == 'success'
 
     environment:
       name: staging

.github/workflows/04-deploy-production.yml

@@ -1,6 +1,17 @@
 name: 04 - Deploy to Production
 
 on:
+  # Automatically promote to production once the staging smoke test passes.
+  # This is the Continuous Deployment step: no manual trigger required.
+  workflow_run:
+    workflows:
+      - "03 - Test Staging"
+    types:
+      - completed
+    branches:
+      - main
+
+  # Manual fallback to re-deploy a previously tested image tag.
   workflow_dispatch:
     inputs:
       image_tag:
@@ -13,12 +24,21 @@ jobs:
     name: Deploy to Production
     runs-on: ubuntu-latest
 
+    if: >
+      github.event_name == 'workflow_dispatch' ||
+      github.event.workflow_run.conclusion == 'success'
+
     environment:
       name: production
 
+    env:
+      IMAGE_TAG: ${{ inputs.image_tag || github.event.workflow_run.head_sha }}
+
     steps:
-      - name: Checkout repository
+      - name: Checkout tested commit
         uses: actions/checkout@v4
+        with:
+          ref: ${{ env.IMAGE_TAG }}
 
       - name: Login to Azure
         uses: azure/login@v3
@@ -65,41 +85,48 @@ jobs:
         run: |
           kubectl apply \
             -f kubernetes/production/
+          # course-service is released as an Argo Rollouts canary
+          # The image is substituted before applying, otherwise re-applying the
+          # "placeholder" image would itself trigger a canary release.
+          sed "s|image: placeholder|image: ${{ vars.ACR_LOGIN_SERVER }}/koalatech-course-service:${{ env.IMAGE_TAG }}|" \
+            kubernetes/progressive/course-service-rollout.yaml | kubectl apply -f -
+          kubectl apply -f kubernetes/progressive/analysis-template.yaml
 
       - name: Update frontend image
         run: |
           kubectl set image deployment/frontend \
-            frontend=${{ vars.ACR_LOGIN_SERVER }}/koalatech-frontend:${{ inputs.image_tag }} \
+            frontend=${{ vars.ACR_LOGIN_SERVER }}/koalatech-frontend:${{ env.IMAGE_TAG }} \
             -n production
 
       - name: Update user-service image
         run: |
           kubectl set image deployment/user-service \
-            user-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-user-service:${{ inputs.image_tag }} \
+            user-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-user-service:${{ env.IMAGE_TAG }} \
             -n production
 
       - name: Update student-service image
         run: |
           kubectl set image deployment/student-service \
-            student-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-student-service:${{ inputs.image_tag }} \
+            student-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-student-service:${{ env.IMAGE_TAG }} \
             -n production
 
       - name: Update lecturer-service image
         run: |
           kubectl set image deployment/lecturer-service \
-            lecturer-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-lecturer-service:${{ inputs.image_tag }} \
+            lecturer-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-lecturer-service:${{ env.IMAGE_TAG }} \
             -n production
 
-      - name: Update course-service image
+      - name: Install Argo Rollouts kubectl plugin
         run: |
-          kubectl set image deployment/course-service \
-            course-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-course-service:${{ inputs.image_tag }} \
-            -n production
+          curl -fsSLo kubectl-argo-rollouts \
+            https://github.com/argoproj/argo-rollouts/releases/download/v1.8.3/kubectl-argo-rollouts-linux-amd64
+          chmod +x kubectl-argo-rollouts
+          sudo mv kubectl-argo-rollouts /usr/local/bin/
 
       - name: Update enrollment-service image
         run: |
           kubectl set image deployment/enrollment-service \
-            enrollment-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-enrollment-service:${{ inputs.image_tag }} \
+            enrollment-service=${{ vars.ACR_LOGIN_SERVER }}/koalatech-enrollment-service:${{ env.IMAGE_TAG }} \
             -n production
 
       - name: Wait for frontend rollout
@@ -126,11 +153,31 @@ jobs:
             -n production \
             --timeout=300s
 
-      - name: Wait for course-service rollout
+      - name: Install Argo Rollouts kubectl plugin
         run: |
-          kubectl rollout status deployment/course-service \
+          curl -fsSLo kubectl-argo-rollouts \
+            https://github.com/argoproj/argo-rollouts/releases/download/v1.8.3/kubectl-argo-rollouts-linux-amd64
+          chmod +x kubectl-argo-rollouts
+          sudo mv kubectl-argo-rollouts /usr/local/bin/
+
+      # Exits non-zero if analysis fails and the rollout is aborted, which
+      # fails this job. The canary is already back at 0% by then.
+      - name: Wait for course-service canary analysis
+        id: canary
+        run: |
+          kubectl argo rollouts status course-service \
             -n production \
-            --timeout=300s
+            --timeout 600s
+
+      - name: Alert Discord on automatic rollback
+        if: failure() && steps.canary.outcome == 'failure'
+        run: |
+          kubectl argo rollouts get rollout course-service -n production > rollout.txt || true
+          FAILED=$(kubectl get analysisrun -n production --sort-by=.metadata.creationTimestamp \
+            -o jsonpath='{.items[-1:].status.metricResults[?(@.phase=="Failed")].name}')
+          BODY=$(jq -n --arg tag "${{ env.IMAGE_TAG }}" --arg m "$FAILED" --arg run "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            '{content: ("Canary ABORTED and rolled back. Revision: `" + $tag + "`, failing metric: `" + $m + "`. " + $run)}')
+          curl -fsS -H "Content-Type: application/json" -d "$BODY" "${{ secrets.DISCORD_WEBHOOK_URL }}"
 
       - name: Wait for enrollment-service rollout
         run: |
@@ -142,4 +189,11 @@ jobs:
         run: |
           kubectl get pods -n production
           kubectl get services -n production
-          kubectl get pvc -n production
+          kubectl get pvc -n production
+
+  # Prometheus and Grafana are deployed and validated once production is up
+  monitoring:
+    name: Deploy Monitoring
+    needs: deploy-production
+    uses: ./.github/workflows/05-deploy-monitoring.yml
+    secrets: inherit