From ac7e9369c6d852c9a5f4e382397adf4f7718da2a Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Mon, 19 Oct 2026 11:39:42 +0100 Subject: [PATCH 1/3] Update references for population provider data Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/nhp/data/table_names/mlcsu.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nhp/data/table_names/mlcsu.py b/src/nhp/data/table_names/mlcsu.py index 45d1571..2c7151f 100644 --- a/src/nhp/data/table_names/mlcsu.py +++ b/src/nhp/data/table_names/mlcsu.py @@ -50,8 +50,8 @@ reference_lsoa11_to_lad23="nhp.reference.lsoa11_to_lad23", reference_lad22_to_lad23="nhp.reference.lad22_to_lad23", reference_theatre_times="nhp.reference.theatre_times", - reference_population_provider_births="unhp.population_projections.provider_births", - reference_population_provider_demographics="unhp.population_projections.provider_demographics", + reference_population_provider_births="nhp.population_projections.provider_births", + reference_population_provider_demographics="nhp.population_projections.provider_demographics", # --- reference_day_procedures_code_list="/Volumes/nhp/reference/files/day_procedures.json", reference_frailty_risk_scores="/Volumes/nhp/reference/files/frailty_risk_scores.csv", From 41349ba2a21e0c059d0fb6538c4d2b93667f81fe Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Mon, 19 Oct 2026 11:40:17 +0100 Subject: [PATCH 2/3] Refine docstring for main function in population_provider.py Updated docstring to clarify the purpose of the main function and the tables generated. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/nhp/data/reference/population_provider.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/nhp/data/reference/population_provider.py b/src/nhp/data/reference/population_provider.py index 0a1756f..209fea5 100644 --- a/src/nhp/data/reference/population_provider.py +++ b/src/nhp/data/reference/population_provider.py @@ -89,16 +89,15 @@ def create_birth_factors(spark: SparkSession) -> None: def main(): - """Main entry point for generating provider catchment population tables. + """Main entry point for generating the provider population reference tables. - This function creates the population by provider table, which includes calculating - provider-to-LAD23 splits based on APC data and applying those splits to population - estimates. The get_pop_by_provider() call will trigger creation of all dependent - tables if they don't already exist. + This function creates the demographic factors and birth factors tables in the + reference layer by applying provider-to-LAD23 catchment splits to the + population projections. Tables generated: - - Provider to LAD23 splits (if not exists) - - Population by provider (if not exists) + - reference_population_provider_demographics + - reference_population_provider_births """ spark = get_spark() From eb0e8559fc709d56087dc6d0ac4d2aad3642508c Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Tue, 13 Oct 2026 20:16:05 +0100 Subject: [PATCH 3/3] reorganise workflow limitations in udal prevent the current compute clusters from having more than two workflows running at once. run the op after the a&e data, as these together take roughly same time as ip data. also run the inputs/tpma explorer for provider/local authorities in parallel --- databricks_workflows/nhp_data.yaml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/databricks_workflows/nhp_data.yaml b/databricks_workflows/nhp_data.yaml index 8cad1fd..2de53b7 100644 --- a/databricks_workflows/nhp_data.yaml +++ b/databricks_workflows/nhp_data.yaml @@ -11,28 +11,34 @@ resources: - task_key: generate_nhp_reference_data run_job_task: job_id: ${resources.jobs.Generate_NHP_Data_AAE_ECDS.id} - - task_key: generate_ip + - task_key: generate_op depends_on: - - task_key: generate_nhp_reference_data + - task_key: generate_ecds run_job_task: - job_id: ${resources.jobs.Generate_NHP_Data_IP.id} - - task_key: generate_op + job_id: ${resources.jobs.Generate_NHP_Data_OP.id} + - task_key: generate_ip depends_on: - task_key: generate_nhp_reference_data run_job_task: - job_id: ${resources.jobs.Generate_NHP_Data_OP.id} + job_id: ${resources.jobs.Generate_NHP_Data_IP.id} - task_key: generate_inputs_data_provider depends_on: - - task_key: generate_ecds - task_key: generate_ip - task_key: generate_op run_job_task: job_id: ${resources.jobs.Generate_NHP_Data_Inputs_provider.id} - task_key: generate_inputs_data_local_authorities depends_on: - - task_key: generate_inputs_data_provider + - task_key: generate_ip + - task_key: generate_op run_job_task: job_id: ${resources.jobs.Generate_NHP_Data_Inputs_lad23cd.id} + - task_key: extract_data + depends_on: + - task_key: generate_inputs_data_provider + - task_key: generate_inputs_data_local_authorities + run_job_task: + job_id: ${resources.jobs.Extract_NHP_for_containers.id} git_source: git_url: https://github.com/The-Strategy-Unit/nhp_data git_provider: gitHub