diff --git a/src/nhp/data/raw_data/aae.py b/src/nhp/data/raw_data/aae.py index c83eb35..0c5c4ff 100644 --- a/src/nhp/data/raw_data/aae.py +++ b/src/nhp/data/raw_data/aae.py @@ -63,13 +63,14 @@ def get_aae_data(spark: SparkSession) -> DataFrame: df_treatments = ( spark.read.table(table_names.hes_aae_treatments) - .filter(F.col("treatment") != "24") + .filter(~F.col("treatment").startswith("22")) + .filter(F.col("treatment") != "99") .select("aekey", "fyear", "procode3") ) df_investigations = ( spark.read.table(table_names.hes_aae_investigations) - .filter(~F.col("investigation").isin(["22", "99"])) + .filter(F.col("investigation") != "24") .select("aekey", "fyear", "procode3") ) @@ -144,9 +145,14 @@ def get_aae_data(spark: SparkSession) -> DataFrame: .fillna(True, ["is_discharged_no_treatment"]) .withColumn( "is_discharged_no_treatment", - F.when(F.col("aeattenddisp") != "03", False).otherwise( - F.col("is_discharged_no_treatment") - ), + # Only consider discharged no treatment for attendances with disposition + # - 02: Discharged – follow-up treatment to be provided by general practitioner + # - 03: Discharged – did not require any follow-up treatment + # otherwise, assume that there was some treatment or investigation + F.when( + F.col("aeattenddisp").isin(["02", "03"]), + F.col("is_discharged_no_treatment"), + ).otherwise(False), ) .join(df_pri_diag, ["procode3", "fyear", "aekey"], how="left") .join(df_pri_treat, ["procode3", "fyear", "aekey"], how="left") diff --git a/src/nhp/data/raw_data/ecds.py b/src/nhp/data/raw_data/ecds.py index 770df0a..9a16051 100644 --- a/src/nhp/data/raw_data/ecds.py +++ b/src/nhp/data/raw_data/ecds.py @@ -134,6 +134,23 @@ def get_ecds_data(spark: SparkSession) -> DataFrame: "1066321000000107", # Left care setting before treatment completed (finding) ] + discharged_no_treatment = { + "destinations": [ + "306694006", # Discharge to nursing home + "306691003", # Discharge to residential home + "306689006", # Discharge to home + "306705005", # Discharge to police custody + "50861005", # Patient discharge, to legal custody (procedure) + ], + "investigations": [ + "1088291000000101", # Clinical investigation not indicated (situation) + ], + "treatments": [ + "183964008", # Treatment not indicated (situation) + "413334001", # Patient given written advice (situation) + ], + } + # add main icb column df = add_main_icb(spark, df) # add age and age_group columns @@ -175,16 +192,27 @@ def get_ecds_data(spark: SparkSession) -> DataFrame: ) .withColumn( "is_discharged_no_treatment", - ( + F.when( + F.col("Discharge_Destination_SNOMED_CT").isin( + discharged_no_treatment["destinations"] + ), ( F.isnull("Der_EC_Investigation_All") - | (F.col("Der_EC_Investigation_All") == "1088291000000101") + | ( + F.col("Der_EC_Investigation_All").isin( + discharged_no_treatment["investigations"] + ) + ) ) & ( F.isnull("Der_EC_Treatment_All") - | (F.col("Der_EC_Treatment_All") == "183964008") - ) - ), + | ( + F.col("Der_EC_Treatment_All").isin( + discharged_no_treatment["treatments"] + ) + ) + ), + ).otherwise(False), ) # for the boolean columns, default to False if null .fillna(