Move remaining STRR services and jobs to Cloud SQL IAM auth

.github/workflows/strr-gcp-iam-deployment-contracts-ci.yaml

@@ -0,0 +1,36 @@
+name: STRR GCP IAM deployment contracts
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - ".github/workflows/strr-gcp-iam-deployment-contracts-ci.yaml"
+      - "tests/deployment/test_gcp_iam_vaults.py"
+      - "queue_services/strr-*/devops/vaults.gcp.env"
+      - "queue_services/strr-*/devops/gcp/clouddeploy.yaml"
+      - "jobs/*/devops/vaults.gcp.env"
+      - "jobs/*/devops/gcp/clouddeploy.yaml"
+  pull_request:
+    branches: [main]
+    paths:
+      - ".github/workflows/strr-gcp-iam-deployment-contracts-ci.yaml"
+      - "tests/deployment/test_gcp_iam_vaults.py"
+      - "queue_services/strr-*/devops/vaults.gcp.env"
+      - "queue_services/strr-*/devops/gcp/clouddeploy.yaml"
+      - "jobs/*/devops/vaults.gcp.env"
+      - "jobs/*/devops/gcp/clouddeploy.yaml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  deployment-contracts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Check Cloud SQL IAM deployment contracts
+        run: python tests/deployment/test_gcp_iam_vaults.py

.sonarcloud.properties

@@ -15,4 +15,5 @@ sonar.exclusions=strr-api/migrations/**/*,strr-api/devops/**/*,strr-api/tests/**
 sonar.sourceEncoding=UTF-8
 
 # Exclusions for copy-paste detection
-sonar.cpd.exclusions=strr-host-pm-web/tests/**, strr-host-pm-web/vitest.config.ts, strr-examiner-web/tests/**
+# Sonar CPD exclusions only work at the file level for this shared database setup.
+sonar.cpd.exclusions=strr-host-pm-web/tests/**, strr-host-pm-web/vitest.config.ts, strr-examiner-web/tests/**, jobs/auto-approval/src/auto_approval/config.py, jobs/batch-permit-validator/src/batch_permit_validator/config.py, jobs/noc_expiry/src/noc_expiry/config.py, jobs/provisional-approval/src/provisional_approval/config.py, jobs/registration_expiry/src/registration_expiry/config.py, jobs/renewal-reminders/src/renewal_reminders/config.py, jobs/strr-backfiller/src/backfiller/config.py

jobs/auto-approval/.env.sample

@@ -7,6 +7,10 @@ DATABASE_NAME=
 DATABASE_HOST=
 DATABASE_PORT=
 
+# Cloud SQL IAM (deployed GCP)
+CLOUDSQL_INSTANCE_CONNECTION_NAME=
+CLOUDSQL_IP_TYPE=PUBLIC
+
 # AUTH API
 AUTH_API_URL=
 AUTH_API_VERSION=

jobs/auto-approval/devops/vaults.gcp.env

@@ -4,11 +4,10 @@ JWT_OIDC_ISSUER="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_ISSUER"
 JWT_OIDC_AUDIENCE="op://keycloak/$APP_ENV/account-services-account/ACCOUNT_SERVICES_SERVICE_ACCOUNT_CLIENT_ID"
 JWT_OIDC_CACHING_ENABLED="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_CACHING_ENABLED"
 JWT_OIDC_JWKS_CACHE_TIMEOUT="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_JWKS_CACHE_TIMEOUT"
-DATABASE_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_USERNAME"
 DATABASE_NAME="op://database/$APP_ENV/strr-db/DATABASE_NAME"
-DATABASE_PASSWORD="op://database/$APP_ENV/strr-db/DATABASE_PASSWORD"
-DATABASE_PORT="op://database/$APP_ENV/strr-db/DATABASE_PORT"
-DATABASE_UNIX_SOCKET="op://database/$APP_ENV/strr-db/DATABASE_UNIX_SOCKET"
+CLOUDSQL_INSTANCE_CONNECTION_NAME="op://database/$APP_ENV/strr-db/DATABASE_INSTANCE_CONNECTION_NAME"
+CLOUDSQL_IP_TYPE="PUBLIC"
+DATABASE_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_JOB_IAM_USERNAME"
 GEOCODER_API_URL="op://API/$APP_ENV/geocoder/GEOCODER_API_URL"
 GEOCODER_API_AUTH_KEY="op://API/$APP_ENV/geocoder/GEOCODER_API_AUTH_KEY"
 KEYCLOAK_AUTH_TOKEN_URL="op://keycloak/$APP_ENV/base/KEYCLOAK_AUTH_TOKEN_URL"

jobs/auto-approval/pyproject.toml

@@ -18,6 +18,7 @@ sentry-sdk = "^1.44.1"
 flask-babel = "^4.0.0"
 gunicorn = "^21.2.0"
 pg8000 = "^1.31.2"
+cloud-sql-connector = { git = "https://github.com/Jacky-Pham/sbc-connect-common.git", subdirectory = "python/cloud-sql-connector", rev = "b8ee1d3568e99c0ac0530ebd839f474737f38680" }
 gcp-queue = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/gcp-queue", branch = "main" }
 structured-logging = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/structured-logging", branch = "main" }
 strr-api = {git = "https://github.com/bcgov/STRR.git", rev = "main", subdirectory = "strr-api"}

jobs/auto-approval/src/auto_approval/config.py

@@ -16,6 +16,7 @@
 import os
 import sys
 
+from cloud_sql_connector import sqlalchemy_settings_from_env
 from dotenv import find_dotenv, load_dotenv
 
 load_dotenv(find_dotenv())
@@ -96,20 +97,7 @@ class _Config:  # pylint: disable=too-few-public-methods
     GEOCODER_SVC_URL = os.getenv("GEOCODER_API_URL", "")
     GEOCODER_SVC_AUTH_KEY = os.getenv("GEOCODER_API_AUTH_KEY", "")
 
-    # DATABASE
-    DB_USER = os.getenv("DATABASE_USERNAME", "")
-    DB_PASSWORD = os.getenv("DATABASE_PASSWORD", "")
-    DB_NAME = os.getenv("DATABASE_NAME", "")
-    DB_HOST = os.getenv("DATABASE_HOST", "")
-    DB_PORT = int(os.getenv("DATABASE_PORT", "5432"))  # POSTGRESQL
-
-    # POSTGRESQL
-    if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None):
-        SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?unix_sock={DB_UNIX_SOCKET}/.s.PGSQL.5432"
-    else:
-        SQLALCHEMY_DATABASE_URI = (
-            f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-        )
+    SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS = sqlalchemy_settings_from_env()
 
     # projects/<project_id-env>/topics/<topic_name>
     GCP_EMAIL_TOPIC = os.getenv("GCP_EMAIL_TOPIC")
@@ -148,6 +136,7 @@ class TestConfig(_Config):  # pylint: disable=too-few-public-methods
         f"postgresql://{DATABASE_TEST_USERNAME}:{DATABASE_TEST_PASSWORD}"
         f"@{DATABASE_TEST_HOST}:{DATABASE_TEST_PORT}/{DATABASE_TEST_NAME}"
     )
+    SQLALCHEMY_ENGINE_OPTIONS = {}
 
     COLIN_URL = os.getenv("COLIN_URL_TEST", "")
     LEGAL_URL = os.getenv("LEGAL_URL_TEST", "")

jobs/auto-approval/tests/unit/test_config_cloudsql_iam.py

@@ -0,0 +1,22 @@
+"""Tests for auto-approval Cloud SQL configuration wiring."""
+
+import runpy
+from pathlib import Path
+from unittest.mock import patch, sentinel
+
+CONFIG_PATH = Path(__file__).resolve().parents[2] / "src/auto_approval/config.py"
+
+
+def test_config_uses_shared_sqlalchemy_settings():
+    with (
+        patch("dotenv.load_dotenv"),
+        patch(
+            "cloud_sql_connector.sqlalchemy_settings_from_env",
+            return_value=(sentinel.database_uri, sentinel.engine_options),
+        ) as settings_from_env,
+    ):
+        config_module = runpy.run_path(str(CONFIG_PATH))
+
+    settings_from_env.assert_called_once_with()
+    assert config_module["_Config"].SQLALCHEMY_DATABASE_URI is sentinel.database_uri
+    assert config_module["_Config"].SQLALCHEMY_ENGINE_OPTIONS is sentinel.engine_options

jobs/batch-permit-validator/.env.sample

@@ -11,6 +11,10 @@ DATABASE_NAME=
 DATABASE_HOST=
 DATABASE_PORT=
 
+# Cloud SQL IAM (deployed GCP)
+CLOUDSQL_INSTANCE_CONNECTION_NAME=
+CLOUDSQL_IP_TYPE=PUBLIC
+
 GCP_CS_PROJECT_ID=
 GCP_CS_SA_SCOPE=
 GCP_AUTH_KEY=

jobs/batch-permit-validator/devops/vaults.gcp.env

@@ -1,8 +1,7 @@
-DATABASE_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_USERNAME"
 DATABASE_NAME="op://database/$APP_ENV/strr-db/DATABASE_NAME"
-DATABASE_PASSWORD="op://database/$APP_ENV/strr-db/DATABASE_PASSWORD"
-DATABASE_PORT="op://database/$APP_ENV/strr-db/DATABASE_PORT"
-DATABASE_UNIX_SOCKET="op://database/$APP_ENV/strr-db/DATABASE_UNIX_SOCKET"
+CLOUDSQL_INSTANCE_CONNECTION_NAME="op://database/$APP_ENV/strr-db/DATABASE_INSTANCE_CONNECTION_NAME"
+CLOUDSQL_IP_TYPE="PUBLIC"
+DATABASE_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_JOB_IAM_USERNAME"
 GCP_CS_PROJECT_ID="op://buckets/$APP_ENV/strr/GCP_CS_PROJECT_ID"
 GCP_CS_SA_SCOPE="op://buckets/$APP_ENV/strr/GCP_CS_SA_SCOPE"
 GCP_AUTH_KEY="op://buckets/$APP_ENV/strr/GCP_AUTH_KEY"