From b42e2704e8d79ddbad2d577b9b99eadc23ce83d8 Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Tue, 25 Aug 2026 09:54:17 +0100 Subject: [PATCH 1/6] fixes #276 --- src/nhp/data/raw_data/aae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nhp/data/raw_data/aae.py b/src/nhp/data/raw_data/aae.py index c83eb35..8a3f7ae 100644 --- a/src/nhp/data/raw_data/aae.py +++ b/src/nhp/data/raw_data/aae.py @@ -63,13 +63,13 @@ def get_aae_data(spark: SparkSession) -> DataFrame: df_treatments = ( spark.read.table(table_names.hes_aae_treatments) - .filter(F.col("treatment") != "24") + .filter(~F.col("treatment").isin(["22", "99"])) .select("aekey", "fyear", "procode3") ) df_investigations = ( spark.read.table(table_names.hes_aae_investigations) - .filter(~F.col("investigation").isin(["22", "99"])) + .filter(F.col("investigation") != "24") .select("aekey", "fyear", "procode3") ) From ad118ca44d48ead725e6e111975fa77b715afb4e Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Tue, 25 Aug 2026 10:00:11 +0100 Subject: [PATCH 2/6] fixes #277 --- src/nhp/data/raw_data/ecds.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/nhp/data/raw_data/ecds.py b/src/nhp/data/raw_data/ecds.py index 770df0a..a5f65a8 100644 --- a/src/nhp/data/raw_data/ecds.py +++ b/src/nhp/data/raw_data/ecds.py @@ -134,6 +134,16 @@ def get_ecds_data(spark: SparkSession) -> DataFrame: "1066321000000107", # Left care setting before treatment completed (finding) ] + discharged_no_treatment = { + "investigations": [ + "1088291000000101", # Clinical investigation not indicated (situation) + ], + "treatments": [ + "183964008", # Treatment not indicated (situation) + "413334001", # Patient given written advice (situation) + ], + } + # add main icb column df = add_main_icb(spark, df) # add age and age_group columns @@ -178,11 +188,19 @@ def get_ecds_data(spark: SparkSession) -> DataFrame: ( ( F.isnull("Der_EC_Investigation_All") - | (F.col("Der_EC_Investigation_All") == "1088291000000101") + | ( + F.col("Der_EC_Investigation_All").isin( + discharged_no_treatment["investigations"] + ) + ) ) & ( F.isnull("Der_EC_Treatment_All") - | (F.col("Der_EC_Treatment_All") == "183964008") + | ( + F.col("Der_EC_Treatment_All").isin( + discharged_no_treatment["treatments"] + ) + ) ) ), ) From 973d3cd40d2be336586cd7250af805dcbbe1601a Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Fri, 25 Sep 2026 11:46:48 +0100 Subject: [PATCH 3/6] fixes treatment filter should be either 22, 221, or 222. uses startswith to handle this, and pushes 99 into a separate filter --- src/nhp/data/raw_data/aae.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nhp/data/raw_data/aae.py b/src/nhp/data/raw_data/aae.py index 8a3f7ae..454c30b 100644 --- a/src/nhp/data/raw_data/aae.py +++ b/src/nhp/data/raw_data/aae.py @@ -63,7 +63,8 @@ def get_aae_data(spark: SparkSession) -> DataFrame: df_treatments = ( spark.read.table(table_names.hes_aae_treatments) - .filter(~F.col("treatment").isin(["22", "99"])) + .filter(~F.col("treatment").startswith("22")) + .filter(F.col("treatment") != "99") .select("aekey", "fyear", "procode3") ) From edc844e62b2635c04288743a76c2ba927a3ecf5d Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Fri, 25 Sep 2026 11:59:41 +0100 Subject: [PATCH 4/6] updates logic for is_discharged_no_treatment to include follow-up needed by GP --- src/nhp/data/raw_data/aae.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/nhp/data/raw_data/aae.py b/src/nhp/data/raw_data/aae.py index 454c30b..0c5c4ff 100644 --- a/src/nhp/data/raw_data/aae.py +++ b/src/nhp/data/raw_data/aae.py @@ -145,9 +145,14 @@ def get_aae_data(spark: SparkSession) -> DataFrame: .fillna(True, ["is_discharged_no_treatment"]) .withColumn( "is_discharged_no_treatment", - F.when(F.col("aeattenddisp") != "03", False).otherwise( - F.col("is_discharged_no_treatment") - ), + # Only consider discharged no treatment for attendances with disposition + # - 02: Discharged – follow-up treatment to be provided by general practitioner + # - 03: Discharged – did not require any follow-up treatment + # otherwise, assume that there was some treatment or investigation + F.when( + F.col("aeattenddisp").isin(["02", "03"]), + F.col("is_discharged_no_treatment"), + ).otherwise(False), ) .join(df_pri_diag, ["procode3", "fyear", "aekey"], how="left") .join(df_pri_treat, ["procode3", "fyear", "aekey"], how="left") From 8dff8c10b40272d0f8447a1c203814d2d30af8d4 Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Fri, 25 Sep 2026 13:41:13 +0100 Subject: [PATCH 5/6] adds additional checks for discharge destination to discharged_no_treatment (ecds) --- src/nhp/data/raw_data/ecds.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/nhp/data/raw_data/ecds.py b/src/nhp/data/raw_data/ecds.py index a5f65a8..be3fdd9 100644 --- a/src/nhp/data/raw_data/ecds.py +++ b/src/nhp/data/raw_data/ecds.py @@ -135,6 +135,14 @@ def get_ecds_data(spark: SparkSession) -> DataFrame: ] discharged_no_treatment = { + "destinations": [ + "306694006", # Discharge to nursing home + "306691003", # Discharge to residential home + "306689006", # Discharge to home + "306705005", # Discharge to police custody + "306706006", # Discharge to ward + "50861005", # Patient discharge, to legal custody (procedure) + ], "investigations": [ "1088291000000101", # Clinical investigation not indicated (situation) ], @@ -185,7 +193,10 @@ def get_ecds_data(spark: SparkSession) -> DataFrame: ) .withColumn( "is_discharged_no_treatment", - ( + F.when( + F.col("Discharge_Destination_SNOMED_CT").isin( + discharged_no_treatment["destinations"] + ), ( F.isnull("Der_EC_Investigation_All") | ( @@ -201,8 +212,8 @@ def get_ecds_data(spark: SparkSession) -> DataFrame: discharged_no_treatment["treatments"] ) ) - ) - ), + ), + ).otherwise(False), ) # for the boolean columns, default to False if null .fillna( From 7bc5fe07216a2ef8873d27a307a86eb92bd487c4 Mon Sep 17 00:00:00 2001 From: Tom Jemmett Date: Mon, 28 Sep 2026 18:48:05 +0100 Subject: [PATCH 6/6] Update src/nhp/data/raw_data/ecds.py Co-authored-by: ghobro --- src/nhp/data/raw_data/ecds.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nhp/data/raw_data/ecds.py b/src/nhp/data/raw_data/ecds.py index be3fdd9..9a16051 100644 --- a/src/nhp/data/raw_data/ecds.py +++ b/src/nhp/data/raw_data/ecds.py @@ -140,7 +140,6 @@ def get_ecds_data(spark: SparkSession) -> DataFrame: "306691003", # Discharge to residential home "306689006", # Discharge to home "306705005", # Discharge to police custody - "306706006", # Discharge to ward "50861005", # Patient discharge, to legal custody (procedure) ], "investigations": [