From 02ac63abb14e8caf274386a7312e7fa9e758d8e9 Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Wed, 21 Oct 2026 15:23:26 +0100 Subject: [PATCH] updates compute clusters used - switches to spark 18 (current lts) - switches some workflows to use the medium policy - switches other policies to use single node policy --- databricks_workflows/nhp_data-ecds.yaml | 9 ++++---- .../nhp_data-extract_nhp_for_containers.yaml | 21 ++++++++++--------- databricks_workflows/nhp_data-hes-aae.yaml | 9 ++++---- databricks_workflows/nhp_data-hes-apc.yaml | 9 ++++---- databricks_workflows/nhp_data-hes-opa.yaml | 9 ++++---- databricks_workflows/nhp_data-inpatients.yaml | 9 ++++---- .../nhp_data-inputs-lad23cd.yaml | 9 ++++---- .../nhp_data-inputs-provider.yaml | 9 ++++---- .../nhp_data-outpatients.yaml | 9 ++++---- .../nhp_data-population_projections.yaml | 9 ++++---- .../nhp_data-reference_data.yaml | 21 ++++++++++--------- 11 files changed, 67 insertions(+), 56 deletions(-) diff --git a/databricks_workflows/nhp_data-ecds.yaml b/databricks_workflows/nhp_data-ecds.yaml index b62798e..8351ed0 100644 --- a/databricks_workflows/nhp_data-ecds.yaml +++ b/databricks_workflows/nhp_data-ecds.yaml @@ -48,7 +48,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -56,14 +56,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-extract_nhp_for_containers.yaml b/databricks_workflows/nhp_data-extract_nhp_for_containers.yaml index 4f43bc9..6c1ecc4 100644 --- a/databricks_workflows/nhp_data-extract_nhp_for_containers.yaml +++ b/databricks_workflows/nhp_data-extract_nhp_for_containers.yaml @@ -178,29 +178,30 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: + spark.databricks.sql.initial.catalog.namespace: udal_lake_mart spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" - spark.databricks.sql.initial.catalog.namespace: udal_lake_mart + spark.master: local[*, 4] + spark.databricks.cluster.profile: singleNode azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_E8ads_v7 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium + ResourceClass: SingleNode spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 - data_security_mode: USER_ISOLATION + policy_id: 001D6F598E9C0B76 + data_security_mode: SINGLE_USER runtime_engine: STANDARD kind: CLASSIC_PREVIEW - is_single_node: false - autoscale: - min_workers: 1 - max_workers: 4 + is_single_node: true + num_workers: 0 tags: group: nhp_data_model env: ${bundle.target} diff --git a/databricks_workflows/nhp_data-hes-aae.yaml b/databricks_workflows/nhp_data-hes-aae.yaml index 4e7af6f..a94880c 100644 --- a/databricks_workflows/nhp_data-hes-aae.yaml +++ b/databricks_workflows/nhp_data-hes-aae.yaml @@ -53,7 +53,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -61,14 +61,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-hes-apc.yaml b/databricks_workflows/nhp_data-hes-apc.yaml index 1ac7ba4..0fc19d4 100644 --- a/databricks_workflows/nhp_data-hes-apc.yaml +++ b/databricks_workflows/nhp_data-hes-apc.yaml @@ -53,7 +53,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -61,14 +61,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-hes-opa.yaml b/databricks_workflows/nhp_data-hes-opa.yaml index 05244ff..4aa3349 100644 --- a/databricks_workflows/nhp_data-hes-opa.yaml +++ b/databricks_workflows/nhp_data-hes-opa.yaml @@ -43,7 +43,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -51,14 +51,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-inpatients.yaml b/databricks_workflows/nhp_data-inpatients.yaml index 425d090..529aab8 100644 --- a/databricks_workflows/nhp_data-inpatients.yaml +++ b/databricks_workflows/nhp_data-inpatients.yaml @@ -39,7 +39,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -47,14 +47,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-inputs-lad23cd.yaml b/databricks_workflows/nhp_data-inputs-lad23cd.yaml index c658315..dc5c761 100644 --- a/databricks_workflows/nhp_data-inputs-lad23cd.yaml +++ b/databricks_workflows/nhp_data-inputs-lad23cd.yaml @@ -67,7 +67,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -75,14 +75,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-inputs-provider.yaml b/databricks_workflows/nhp_data-inputs-provider.yaml index 8b54c79..beec520 100644 --- a/databricks_workflows/nhp_data-inputs-provider.yaml +++ b/databricks_workflows/nhp_data-inputs-provider.yaml @@ -100,7 +100,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -108,14 +108,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-outpatients.yaml b/databricks_workflows/nhp_data-outpatients.yaml index b6c997f..785aa59 100644 --- a/databricks_workflows/nhp_data-outpatients.yaml +++ b/databricks_workflows/nhp_data-outpatients.yaml @@ -48,7 +48,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -56,14 +56,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-population_projections.yaml b/databricks_workflows/nhp_data-population_projections.yaml index f8cafd1..4b0cf02 100644 --- a/databricks_workflows/nhp_data-population_projections.yaml +++ b/databricks_workflows/nhp_data-population_projections.yaml @@ -64,7 +64,7 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" @@ -72,14 +72,15 @@ resources: azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_D8ds_v5 + driver_node_type_id: Standard_D8ds_v5 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 + policy_id: 0001CA9D7F08DD46 data_security_mode: USER_ISOLATION runtime_engine: STANDARD kind: CLASSIC_PREVIEW diff --git a/databricks_workflows/nhp_data-reference_data.yaml b/databricks_workflows/nhp_data-reference_data.yaml index 96c9c40..db51db0 100644 --- a/databricks_workflows/nhp_data-reference_data.yaml +++ b/databricks_workflows/nhp_data-reference_data.yaml @@ -97,29 +97,30 @@ resources: job_clusters: - job_cluster_key: nhp_data new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: 18.x-scala2.13 spark_conf: + spark.databricks.sql.initial.catalog.namespace: udal_lake_mart spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true" spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true" - spark.databricks.sql.initial.catalog.namespace: udal_lake_mart + spark.master: local[*, 4] + spark.databricks.cluster.profile: singleNode azure_attributes: availability: ON_DEMAND_AZURE spot_bid_max_price: 100 - node_type_id: Standard_D4ds_v5 + node_type_id: Standard_E8ads_v7 custom_tags: Application: UDAL Workload_Type: Job - Workload_Size: Small + Workload_Size: Medium + ResourceClass: SingleNode spark_env_vars: PYSPARK_PYTHON: /databricks/python3/bin/python3 - policy_id: 000816B2986C0B29 - data_security_mode: USER_ISOLATION + policy_id: 001D6F598E9C0B76 + data_security_mode: SINGLE_USER runtime_engine: STANDARD kind: CLASSIC_PREVIEW - is_single_node: false - autoscale: - min_workers: 1 - max_workers: 4 + is_single_node: true + num_workers: 0 tags: group: nhp_data env: ${bundle.target}