reorganise workflow

databricks_workflows/nhp_data.yaml

@@ -11,28 +11,34 @@ resources:
             - task_key: generate_nhp_reference_data
           run_job_task:
             job_id: ${resources.jobs.Generate_NHP_Data_AAE_ECDS.id}
-        - task_key: generate_ip
+        - task_key: generate_op
           depends_on:
-            - task_key: generate_nhp_reference_data
+            - task_key: generate_ecds
           run_job_task:
-            job_id: ${resources.jobs.Generate_NHP_Data_IP.id}
-        - task_key: generate_op
+            job_id: ${resources.jobs.Generate_NHP_Data_OP.id}
+        - task_key: generate_ip
           depends_on:
             - task_key: generate_nhp_reference_data
           run_job_task:
-            job_id: ${resources.jobs.Generate_NHP_Data_OP.id}
+            job_id: ${resources.jobs.Generate_NHP_Data_IP.id}
         - task_key: generate_inputs_data_provider
           depends_on:
-            - task_key: generate_ecds
             - task_key: generate_ip
             - task_key: generate_op
           run_job_task:
             job_id: ${resources.jobs.Generate_NHP_Data_Inputs_provider.id}
         - task_key: generate_inputs_data_local_authorities
           depends_on:
-            - task_key: generate_inputs_data_provider
+            - task_key: generate_ip
+            - task_key: generate_op
           run_job_task:
             job_id: ${resources.jobs.Generate_NHP_Data_Inputs_lad23cd.id}
+        - task_key: extract_data
+          depends_on:
+            - task_key: generate_inputs_data_provider
+            - task_key: generate_inputs_data_local_authorities
+          run_job_task:
+            job_id: ${resources.jobs.Extract_NHP_for_containers.id}
       git_source:
         git_url: https://github.com/The-Strategy-Unit/nhp_data
         git_provider: gitHub

src/nhp/data/reference/population_provider.py

@@ -89,16 +89,15 @@ def create_birth_factors(spark: SparkSession) -> None:
 
 
 def main():
-    """Main entry point for generating provider catchment population tables.
+    """Main entry point for generating the provider population reference tables.
 
-    This function creates the population by provider table, which includes calculating
-    provider-to-LAD23 splits based on APC data and applying those splits to population
-    estimates. The get_pop_by_provider() call will trigger creation of all dependent
-    tables if they don't already exist.
+    This function creates the demographic factors and birth factors tables in the
+    reference layer by applying provider-to-LAD23 catchment splits to the
+    population projections.
 
     Tables generated:
-        - Provider to LAD23 splits (if not exists)
-        - Population by provider (if not exists)
+        - reference_population_provider_demographics
+        - reference_population_provider_births
     """
     spark = get_spark()
 

src/nhp/data/table_names/mlcsu.py

@@ -50,8 +50,8 @@
     reference_lsoa11_to_lad23="nhp.reference.lsoa11_to_lad23",
     reference_lad22_to_lad23="nhp.reference.lad22_to_lad23",
     reference_theatre_times="nhp.reference.theatre_times",
-    reference_population_provider_births="unhp.population_projections.provider_births",
-    reference_population_provider_demographics="unhp.population_projections.provider_demographics",
+    reference_population_provider_births="nhp.population_projections.provider_births",
+    reference_population_provider_demographics="nhp.population_projections.provider_demographics",
     # ---
     reference_day_procedures_code_list="/Volumes/nhp/reference/files/day_procedures.json",
     reference_frailty_risk_scores="/Volumes/nhp/reference/files/frailty_risk_scores.csv",