fix discharged no treatment tpma

src/nhp/data/raw_data/aae.py

@@ -63,13 +63,14 @@ def get_aae_data(spark: SparkSession) -> DataFrame:
 
     df_treatments = (
         spark.read.table(table_names.hes_aae_treatments)
-        .filter(F.col("treatment") != "24")
+        .filter(~F.col("treatment").startswith("22"))
+        .filter(F.col("treatment") != "99")
         .select("aekey", "fyear", "procode3")
     )
 
     df_investigations = (
         spark.read.table(table_names.hes_aae_investigations)
-        .filter(~F.col("investigation").isin(["22", "99"]))
+        .filter(F.col("investigation") != "24")
         .select("aekey", "fyear", "procode3")
     )
 
@@ -144,9 +145,14 @@ def get_aae_data(spark: SparkSession) -> DataFrame:
         .fillna(True, ["is_discharged_no_treatment"])
         .withColumn(
             "is_discharged_no_treatment",
-            F.when(F.col("aeattenddisp") != "03", False).otherwise(
-                F.col("is_discharged_no_treatment")
-            ),
+            # Only consider discharged no treatment for attendances with disposition
+            # - 02: Discharged – follow-up treatment to be provided by general practitioner
+            # - 03: Discharged – did not require any follow-up treatment
+            # otherwise, assume that there was some treatment or investigation
+            F.when(
+                F.col("aeattenddisp").isin(["02", "03"]),
+                F.col("is_discharged_no_treatment"),
+            ).otherwise(False),
         )
         .join(df_pri_diag, ["procode3", "fyear", "aekey"], how="left")
         .join(df_pri_treat, ["procode3", "fyear", "aekey"], how="left")

src/nhp/data/raw_data/ecds.py

@@ -134,6 +134,23 @@ def get_ecds_data(spark: SparkSession) -> DataFrame:
         "1066321000000107",  # Left care setting before treatment completed (finding)
     ]
 
+    discharged_no_treatment = {
+        "destinations": [
+            "306694006",  # Discharge to nursing home
+            "306691003",  # Discharge to residential home
+            "306689006",  # Discharge to home
+            "306705005",  # Discharge to police custody
+            "50861005",  # Patient discharge, to legal custody (procedure)
+        ],
+        "investigations": [
+            "1088291000000101",  # Clinical investigation not indicated (situation)
+        ],
+        "treatments": [
+            "183964008",  # Treatment not indicated (situation)
+            "413334001",  # Patient given written advice (situation)
+        ],
+    }
+
     # add main icb column
     df = add_main_icb(spark, df)
     # add age and age_group columns
@@ -175,16 +192,27 @@ def get_ecds_data(spark: SparkSession) -> DataFrame:
         )
         .withColumn(
             "is_discharged_no_treatment",
-            (
+            F.when(
+                F.col("Discharge_Destination_SNOMED_CT").isin(
+                    discharged_no_treatment["destinations"]
+                ),
                 (
                     F.isnull("Der_EC_Investigation_All")
-                    | (F.col("Der_EC_Investigation_All") == "1088291000000101")
+                    | (
+                        F.col("Der_EC_Investigation_All").isin(
+                            discharged_no_treatment["investigations"]
+                        )
+                    )
                 )
                 & (
                     F.isnull("Der_EC_Treatment_All")
-                    | (F.col("Der_EC_Treatment_All") == "183964008")
-                )
-            ),
+                    | (
+                        F.col("Der_EC_Treatment_All").isin(
+                            discharged_no_treatment["treatments"]
+                        )
+                    )
+                ),
+            ).otherwise(False),
         )
         # for the boolean columns, default to False if null
         .fillna(