From e7334619a0f77d5aefdbfdaa625d6a05a96039e7 Mon Sep 17 00:00:00 2001 From: Ksonar262 Date: Wed, 25 Mar 2026 11:03:44 +0000 Subject: [PATCH 01/21] restructured and renamed the repo --- 2D_Classifier/train/requirements.txt | 10 ---------- .../train => Classifier_2D}/Dockerfile | 0 {2D_Classifier => Classifier_2D}/README.md | 0 .../app => Classifier_2D}/__init__.py | 0 .../tests => Classifier_2D/app}/__init__.py | 0 .../config}/__init__.py | 0 .../train => Classifier_2D}/config/config.cfg | 0 .../config/local_config.cfg | 0 .../train => Classifier_2D/mlops}/README.md | 0 .../mlops}/__init__.py | 0 .../scripts => Classifier_2D/mlops}/train.py | 2 +- .../scripts => Classifier_2D/mlops}/tune.py | 20 +++++++++++++------ .../shared}/XNATDataImport.py | 2 +- .../shared}/__init__.py | 0 .../train => Classifier_2D}/src/DataModule.py | 0 .../train => Classifier_2D}/src/Network.py | 0 .../train => Classifier_2D}/src/__init__.py | 0 .../src/transforms/LoadImageXNATd.py | 0 .../src/transforms/SafeWrapper.py | 0 .../src/transforms/__init__.py | 0 Classifier_2D/src/utils/__init__.py | 0 .../src/utils/tools.py | 0 Classifier_2D/tests/__init__.py | 0 Classifier_2D/tests/app_tests/__init__.py | 0 Classifier_2D/tests/train_tests/__init__.py | 0 25 files changed, 16 insertions(+), 18 deletions(-) delete mode 100644 2D_Classifier/train/requirements.txt rename {2D_Classifier/train => Classifier_2D}/Dockerfile (100%) rename {2D_Classifier => Classifier_2D}/README.md (100%) rename {2D_Classifier/app => Classifier_2D}/__init__.py (100%) rename {2D_Classifier/tests => Classifier_2D/app}/__init__.py (100%) rename {2D_Classifier/tests/app_tests => Classifier_2D/config}/__init__.py (100%) rename {2D_Classifier/train => Classifier_2D}/config/config.cfg (100%) rename {2D_Classifier/train => Classifier_2D}/config/local_config.cfg (100%) rename {2D_Classifier/train => Classifier_2D/mlops}/README.md (100%) rename {2D_Classifier/tests/train_tests => Classifier_2D/mlops}/__init__.py (100%) rename {2D_Classifier/train/scripts => Classifier_2D/mlops}/train.py (99%) rename {2D_Classifier/train/scripts => Classifier_2D/mlops}/tune.py (92%) rename {2D_Classifier/train/src => Classifier_2D/shared}/XNATDataImport.py (98%) rename {2D_Classifier/train/scripts => Classifier_2D/shared}/__init__.py (100%) rename {2D_Classifier/train => Classifier_2D}/src/DataModule.py (100%) rename {2D_Classifier/train => Classifier_2D}/src/Network.py (100%) rename {2D_Classifier/train => Classifier_2D}/src/__init__.py (100%) rename {2D_Classifier/train => Classifier_2D}/src/transforms/LoadImageXNATd.py (100%) rename {2D_Classifier/train => Classifier_2D}/src/transforms/SafeWrapper.py (100%) rename {2D_Classifier/train => Classifier_2D}/src/transforms/__init__.py (100%) create mode 100644 Classifier_2D/src/utils/__init__.py rename {2D_Classifier/train => Classifier_2D}/src/utils/tools.py (100%) create mode 100644 Classifier_2D/tests/__init__.py create mode 100644 Classifier_2D/tests/app_tests/__init__.py create mode 100644 Classifier_2D/tests/train_tests/__init__.py diff --git a/2D_Classifier/train/requirements.txt b/2D_Classifier/train/requirements.txt deleted file mode 100644 index d24b654..0000000 --- a/2D_Classifier/train/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -csc-mlops -torch -torchmetrics -torchvision -pytorch_lightning -ray -timm -pytest -pytest-cov -flake8 diff --git a/2D_Classifier/train/Dockerfile b/Classifier_2D/Dockerfile similarity index 100% rename from 2D_Classifier/train/Dockerfile rename to Classifier_2D/Dockerfile diff --git a/2D_Classifier/README.md b/Classifier_2D/README.md similarity index 100% rename from 2D_Classifier/README.md rename to Classifier_2D/README.md diff --git a/2D_Classifier/app/__init__.py b/Classifier_2D/__init__.py similarity index 100% rename from 2D_Classifier/app/__init__.py rename to Classifier_2D/__init__.py diff --git a/2D_Classifier/tests/__init__.py b/Classifier_2D/app/__init__.py similarity index 100% rename from 2D_Classifier/tests/__init__.py rename to Classifier_2D/app/__init__.py diff --git a/2D_Classifier/tests/app_tests/__init__.py b/Classifier_2D/config/__init__.py similarity index 100% rename from 2D_Classifier/tests/app_tests/__init__.py rename to Classifier_2D/config/__init__.py diff --git a/2D_Classifier/train/config/config.cfg b/Classifier_2D/config/config.cfg similarity index 100% rename from 2D_Classifier/train/config/config.cfg rename to Classifier_2D/config/config.cfg diff --git a/2D_Classifier/train/config/local_config.cfg b/Classifier_2D/config/local_config.cfg similarity index 100% rename from 2D_Classifier/train/config/local_config.cfg rename to Classifier_2D/config/local_config.cfg diff --git a/2D_Classifier/train/README.md b/Classifier_2D/mlops/README.md similarity index 100% rename from 2D_Classifier/train/README.md rename to Classifier_2D/mlops/README.md diff --git a/2D_Classifier/tests/train_tests/__init__.py b/Classifier_2D/mlops/__init__.py similarity index 100% rename from 2D_Classifier/tests/train_tests/__init__.py rename to Classifier_2D/mlops/__init__.py diff --git a/2D_Classifier/train/scripts/train.py b/Classifier_2D/mlops/train.py similarity index 99% rename from 2D_Classifier/train/scripts/train.py rename to Classifier_2D/mlops/train.py index c38fbc1..1f518a3 100644 --- a/2D_Classifier/train/scripts/train.py +++ b/Classifier_2D/mlops/train.py @@ -15,7 +15,7 @@ from src.DataModule import DataModule from src.Network import Network from src.DataModule import label_dict -from src.XNATDataImport import XNATDataImport +from shared.XNATDataImport import XNATDataImport logger = logging.getLogger(__name__) diff --git a/2D_Classifier/train/scripts/tune.py b/Classifier_2D/mlops/tune.py similarity index 92% rename from 2D_Classifier/train/scripts/tune.py rename to Classifier_2D/mlops/tune.py index 623e5cc..9a727f8 100644 --- a/2D_Classifier/train/scripts/tune.py +++ b/Classifier_2D/mlops/tune.py @@ -1,4 +1,3 @@ - import configparser import logging import multiprocessing @@ -11,10 +10,11 @@ from pytorch_lightning.callbacks import ModelCheckpoint from torch.cuda import is_available as cuda_available -from project.DataModule import DataModule -from project.DataModule import label_dict -from project.Network import Network -from project.XNATDataImport import XNATDataImport +from src.DataModule import DataModule +from src.DataModule import label_dict +from src.Network import Network +from shared.XNATDataImport import XNATDataImport +from pytorch_lightning.callbacks import EarlyStopping import optuna logger = logging.getLogger(__name__) @@ -105,6 +105,14 @@ def objective(trial,data,config): ) callbacks.append(checkpoint_callback) + early_stopping_callback = EarlyStopping( + monitor="val_loss", + patience=10, + mode="min", + verbose=True, + ) + callbacks.append(early_stopping_callback) + # configure trainer trainer = pl.Trainer( precision="32" if cuda_available() else "16", @@ -171,7 +179,7 @@ def tune(config): mlflow.pytorch.autolog(log_models=False) # Create optuna study (hyperparameter tuning framework) - study = optuna.create_study(study_name="scaphx-tune", direction="minimize") + study = optuna.create_study(study_name="project-tune", direction="minimize") study.optimize(lambda trial: objective(trial, data, config), n_trials=50) with open(('tune_log.txt'), 'w') as f: diff --git a/2D_Classifier/train/src/XNATDataImport.py b/Classifier_2D/shared/XNATDataImport.py similarity index 98% rename from 2D_Classifier/train/src/XNATDataImport.py rename to Classifier_2D/shared/XNATDataImport.py index 58ce51c..05c6004 100644 --- a/2D_Classifier/train/src/XNATDataImport.py +++ b/Classifier_2D/shared/XNATDataImport.py @@ -2,7 +2,7 @@ from tqdm import tqdm from typing import List -from utils.tools import DataBuilderXNAT +from src.utils.tools import DataBuilderXNAT from xnat.mixin import ImageScanData, SubjectData from src.transforms import load_xnat diff --git a/2D_Classifier/train/scripts/__init__.py b/Classifier_2D/shared/__init__.py similarity index 100% rename from 2D_Classifier/train/scripts/__init__.py rename to Classifier_2D/shared/__init__.py diff --git a/2D_Classifier/train/src/DataModule.py b/Classifier_2D/src/DataModule.py similarity index 100% rename from 2D_Classifier/train/src/DataModule.py rename to Classifier_2D/src/DataModule.py diff --git a/2D_Classifier/train/src/Network.py b/Classifier_2D/src/Network.py similarity index 100% rename from 2D_Classifier/train/src/Network.py rename to Classifier_2D/src/Network.py diff --git a/2D_Classifier/train/src/__init__.py b/Classifier_2D/src/__init__.py similarity index 100% rename from 2D_Classifier/train/src/__init__.py rename to Classifier_2D/src/__init__.py diff --git a/2D_Classifier/train/src/transforms/LoadImageXNATd.py b/Classifier_2D/src/transforms/LoadImageXNATd.py similarity index 100% rename from 2D_Classifier/train/src/transforms/LoadImageXNATd.py rename to Classifier_2D/src/transforms/LoadImageXNATd.py diff --git a/2D_Classifier/train/src/transforms/SafeWrapper.py b/Classifier_2D/src/transforms/SafeWrapper.py similarity index 100% rename from 2D_Classifier/train/src/transforms/SafeWrapper.py rename to Classifier_2D/src/transforms/SafeWrapper.py diff --git a/2D_Classifier/train/src/transforms/__init__.py b/Classifier_2D/src/transforms/__init__.py similarity index 100% rename from 2D_Classifier/train/src/transforms/__init__.py rename to Classifier_2D/src/transforms/__init__.py diff --git a/Classifier_2D/src/utils/__init__.py b/Classifier_2D/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/2D_Classifier/train/src/utils/tools.py b/Classifier_2D/src/utils/tools.py similarity index 100% rename from 2D_Classifier/train/src/utils/tools.py rename to Classifier_2D/src/utils/tools.py diff --git a/Classifier_2D/tests/__init__.py b/Classifier_2D/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Classifier_2D/tests/app_tests/__init__.py b/Classifier_2D/tests/app_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Classifier_2D/tests/train_tests/__init__.py b/Classifier_2D/tests/train_tests/__init__.py new file mode 100644 index 0000000..e69de29 From 433cf7587bdf22f510256f05501e749e632b3a4d Mon Sep 17 00:00:00 2001 From: Ksonar262 Date: Wed, 25 Mar 2026 12:11:23 +0000 Subject: [PATCH 02/21] PEP naming --- Classifier_2D/mlops/train.py | 8 ++++---- Classifier_2D/mlops/tune.py | 8 ++++---- .../shared/{XNATDataImport.py => XNAT_data_import.py} | 0 Classifier_2D/src/Network.py | 2 +- Classifier_2D/src/{DataModule.py => data_module.py} | 2 +- Classifier_2D/src/transforms/__init__.py | 2 +- .../transforms/{LoadImageXNATd.py => load_image_XNATd.py} | 0 .../src/transforms/{SafeWrapper.py => safe_wrapper.py} | 0 8 files changed, 11 insertions(+), 11 deletions(-) rename Classifier_2D/shared/{XNATDataImport.py => XNAT_data_import.py} (100%) rename Classifier_2D/src/{DataModule.py => data_module.py} (98%) rename Classifier_2D/src/transforms/{LoadImageXNATd.py => load_image_XNATd.py} (100%) rename Classifier_2D/src/transforms/{SafeWrapper.py => safe_wrapper.py} (100%) diff --git a/Classifier_2D/mlops/train.py b/Classifier_2D/mlops/train.py index 1f518a3..0d9e104 100644 --- a/Classifier_2D/mlops/train.py +++ b/Classifier_2D/mlops/train.py @@ -12,10 +12,10 @@ from ray.air.integrations.mlflow import setup_mlflow from torch.cuda import is_available as cuda_available -from src.DataModule import DataModule -from src.Network import Network -from src.DataModule import label_dict -from shared.XNATDataImport import XNATDataImport +from Classifier_2D.src.data_module import DataModule +from src.network import Network +from Classifier_2D.src.data_module import label_dict +from Classifier_2D.shared.XNAT_data_import import XNATDataImport logger = logging.getLogger(__name__) diff --git a/Classifier_2D/mlops/tune.py b/Classifier_2D/mlops/tune.py index 9a727f8..aae140d 100644 --- a/Classifier_2D/mlops/tune.py +++ b/Classifier_2D/mlops/tune.py @@ -10,10 +10,10 @@ from pytorch_lightning.callbacks import ModelCheckpoint from torch.cuda import is_available as cuda_available -from src.DataModule import DataModule -from src.DataModule import label_dict -from src.Network import Network -from shared.XNATDataImport import XNATDataImport +from Classifier_2D.src.data_module import DataModule +from Classifier_2D.src.data_module import label_dict +from src.network import Network +from Classifier_2D.shared.XNAT_data_import import XNATDataImport from pytorch_lightning.callbacks import EarlyStopping import optuna diff --git a/Classifier_2D/shared/XNATDataImport.py b/Classifier_2D/shared/XNAT_data_import.py similarity index 100% rename from Classifier_2D/shared/XNATDataImport.py rename to Classifier_2D/shared/XNAT_data_import.py diff --git a/Classifier_2D/src/Network.py b/Classifier_2D/src/Network.py index 5058ba7..6025665 100644 --- a/Classifier_2D/src/Network.py +++ b/Classifier_2D/src/Network.py @@ -14,7 +14,7 @@ from timm.data import Mixup from torch.nn import CrossEntropyLoss from torchmetrics import Accuracy, F1Score -from src.DataModule import label_dict +from Classifier_2D.src.data_module import label_dict from torchmetrics.classification import MulticlassAUROC import numpy as np diff --git a/Classifier_2D/src/DataModule.py b/Classifier_2D/src/data_module.py similarity index 98% rename from Classifier_2D/src/DataModule.py rename to Classifier_2D/src/data_module.py index e06a2a8..c7bb463 100644 --- a/Classifier_2D/src/DataModule.py +++ b/Classifier_2D/src/data_module.py @@ -15,7 +15,7 @@ from torch.utils.data import DataLoader from src.transforms import normalise, train_augment, output -from src.transforms.SafeWrapper import SafeWrapperTransform +from Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform logger = logging.getLogger(__name__) diff --git a/Classifier_2D/src/transforms/__init__.py b/Classifier_2D/src/transforms/__init__.py index 385d338..b2f9ac4 100644 --- a/Classifier_2D/src/transforms/__init__.py +++ b/Classifier_2D/src/transforms/__init__.py @@ -23,7 +23,7 @@ Spacingd, ) -from src.transforms.LoadImageXNATd import LoadImageXNATd +from Classifier_2D.src.transforms.load_image_XNATd import LoadImageXNATd def load_xnat(xnat_configuration: dict): """ diff --git a/Classifier_2D/src/transforms/LoadImageXNATd.py b/Classifier_2D/src/transforms/load_image_XNATd.py similarity index 100% rename from Classifier_2D/src/transforms/LoadImageXNATd.py rename to Classifier_2D/src/transforms/load_image_XNATd.py diff --git a/Classifier_2D/src/transforms/SafeWrapper.py b/Classifier_2D/src/transforms/safe_wrapper.py similarity index 100% rename from Classifier_2D/src/transforms/SafeWrapper.py rename to Classifier_2D/src/transforms/safe_wrapper.py From c304a439946912f23f565320ab1691aa3910d757 Mon Sep 17 00:00:00 2001 From: Ksonar262 Date: Wed, 25 Mar 2026 13:53:14 +0000 Subject: [PATCH 03/21] fixed imports and early stopping in train.py, removed ray.mlflow --- Classifier_2D/mlops/train.py | 25 +++++++++++++----------- Classifier_2D/mlops/tune.py | 16 ++------------- Classifier_2D/shared/XNAT_data_import.py | 4 ++-- Classifier_2D/src/data_module.py | 2 +- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/Classifier_2D/mlops/train.py b/Classifier_2D/mlops/train.py index 0d9e104..a99d5fd 100644 --- a/Classifier_2D/mlops/train.py +++ b/Classifier_2D/mlops/train.py @@ -8,12 +8,11 @@ import mlflow import pytorch_lightning as pl import torch -from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint -from ray.air.integrations.mlflow import setup_mlflow +from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping from torch.cuda import is_available as cuda_available from Classifier_2D.src.data_module import DataModule -from src.network import Network +from Classifier_2D.src.network import Network from Classifier_2D.src.data_module import label_dict from Classifier_2D.shared.XNAT_data_import import XNATDataImport @@ -48,14 +47,6 @@ def train(config): data = importer.xnat_image_download(raw_data) # Set up mflow experiment - setup_mlflow( - tracking_uri=mlflow.get_tracking_uri(), - experiment_id=mlflow.get_experiment_by_name( - config["project"]["name"] - ).experiment_id - if mlflow.get_experiment_by_name(config["project"]["name"]) - else mlflow.create_experiment(config["project"]["name"]), - ) with mlflow.start_run(nested=True): save_best_model = True @@ -93,6 +84,10 @@ def train(config): label_smoothing = float(config['params']['label_smoothing']), ) + # Callbacks + checkpoint_metric = config['params']['checkpoint_metric'] + checkpoint_mode = "min" if checkpoint_metric == "val_loss" else "max" + # Callbacks callbacks = [] callbacks.append(LearningRateMonitor(logging_interval="step")) @@ -104,6 +99,14 @@ def train(config): ) callbacks.append(checkpoint_callback) + early_stopping_callback = EarlyStopping( + monitor=checkpoint_metric, + patience=10, + mode=checkpoint_mode, + verbose=True, + ) + callbacks.append(early_stopping_callback) + # configure trainer trainer = pl.Trainer( precision="32" if cuda_available() else "16", diff --git a/Classifier_2D/mlops/tune.py b/Classifier_2D/mlops/tune.py index aae140d..8133fa0 100644 --- a/Classifier_2D/mlops/tune.py +++ b/Classifier_2D/mlops/tune.py @@ -5,16 +5,13 @@ import mlflow import pytorch_lightning as pl -from ray.air.integrations.mlflow import setup_mlflow -from pytorch_lightning.callbacks import LearningRateMonitor -from pytorch_lightning.callbacks import ModelCheckpoint +from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping from torch.cuda import is_available as cuda_available from Classifier_2D.src.data_module import DataModule from Classifier_2D.src.data_module import label_dict -from src.network import Network +from Classifier_2D.src.network import Network from Classifier_2D.shared.XNAT_data_import import XNATDataImport -from pytorch_lightning.callbacks import EarlyStopping import optuna logger = logging.getLogger(__name__) @@ -166,15 +163,6 @@ def tune(config): # Download images from XNAT data = importer.xnat_image_download(raw_data) - # Set up mflow experiment - setup_mlflow( - tracking_uri=mlflow.get_tracking_uri(), - experiment_id=mlflow.get_experiment_by_name( - config["project"]["name"] - ).experiment_id - if mlflow.get_experiment_by_name(config["project"]["name"]) - else mlflow.create_experiment(config["project"]["name"]), - ) mlflow.pytorch.autolog(log_models=False) diff --git a/Classifier_2D/shared/XNAT_data_import.py b/Classifier_2D/shared/XNAT_data_import.py index 05c6004..691970b 100644 --- a/Classifier_2D/shared/XNAT_data_import.py +++ b/Classifier_2D/shared/XNAT_data_import.py @@ -2,10 +2,10 @@ from tqdm import tqdm from typing import List -from src.utils.tools import DataBuilderXNAT +from Classifier_2D.src.utils.tools import DataBuilderXNAT from xnat.mixin import ImageScanData, SubjectData -from src.transforms import load_xnat +from Classifier_2D.src.transforms import load_xnat from monai.data import Dataset from torch.utils.data import DataLoader diff --git a/Classifier_2D/src/data_module.py b/Classifier_2D/src/data_module.py index c7bb463..e5627e0 100644 --- a/Classifier_2D/src/data_module.py +++ b/Classifier_2D/src/data_module.py @@ -14,7 +14,7 @@ from torch.cuda import is_available from torch.utils.data import DataLoader -from src.transforms import normalise, train_augment, output +from Classifier_2D.src.transforms import normalise, train_augment, output from Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform logger = logging.getLogger(__name__) From b1fa71e6c6dc43117bb7a81534061a0a7b07b599 Mon Sep 17 00:00:00 2001 From: Ksonar262 Date: Wed, 15 Apr 2026 16:38:13 +0100 Subject: [PATCH 04/21] made corrections according to new format --- Project/Classifier_2D/Dockerfile | 16 ++ Project/Classifier_2D/README.md | 29 +++ Project/Classifier_2D/__init__.py | 0 Project/Classifier_2D/config/__init__.py | 0 Project/Classifier_2D/config/config.cfg | 34 +++ Project/Classifier_2D/config/local_config.cfg | 34 +++ Project/Classifier_2D/mlops/README.md | 60 +++++ Project/Classifier_2D/mlops/__init__.py | 0 Project/Classifier_2D/mlops/train.py | 177 ++++++++++++++ Project/Classifier_2D/mlops/tune.py | 198 +++++++++++++++ Project/Classifier_2D/scripts/__init__.py | 0 Project/Classifier_2D/src/__init__.py | 0 Project/Classifier_2D/src/data_module.py | 153 ++++++++++++ Project/Classifier_2D/src/network.py | 226 ++++++++++++++++++ .../Classifier_2D/src/transforms/__init__.py | 92 +++++++ .../src/transforms/load_image_XNATd.py | 123 ++++++++++ .../src/transforms/safe_wrapper.py | 47 ++++ .../src/utils/XNAT_data_import.py | 83 +++++++ Project/Classifier_2D/src/utils/__init__.py | 0 Project/Classifier_2D/src/utils/tools.py | 209 ++++++++++++++++ Project/app/__init__.py | 0 Project/app/app_tests/__init__.py | 0 Project/shared/__init__.py | 0 Project/tests/__init__.py | 0 Project/tests/train_tests/__init__.py | 0 25 files changed, 1481 insertions(+) create mode 100644 Project/Classifier_2D/Dockerfile create mode 100644 Project/Classifier_2D/README.md create mode 100644 Project/Classifier_2D/__init__.py create mode 100644 Project/Classifier_2D/config/__init__.py create mode 100644 Project/Classifier_2D/config/config.cfg create mode 100644 Project/Classifier_2D/config/local_config.cfg create mode 100644 Project/Classifier_2D/mlops/README.md create mode 100644 Project/Classifier_2D/mlops/__init__.py create mode 100644 Project/Classifier_2D/mlops/train.py create mode 100644 Project/Classifier_2D/mlops/tune.py create mode 100644 Project/Classifier_2D/scripts/__init__.py create mode 100644 Project/Classifier_2D/src/__init__.py create mode 100644 Project/Classifier_2D/src/data_module.py create mode 100644 Project/Classifier_2D/src/network.py create mode 100644 Project/Classifier_2D/src/transforms/__init__.py create mode 100644 Project/Classifier_2D/src/transforms/load_image_XNATd.py create mode 100644 Project/Classifier_2D/src/transforms/safe_wrapper.py create mode 100644 Project/Classifier_2D/src/utils/XNAT_data_import.py create mode 100644 Project/Classifier_2D/src/utils/__init__.py create mode 100644 Project/Classifier_2D/src/utils/tools.py create mode 100644 Project/app/__init__.py create mode 100644 Project/app/app_tests/__init__.py create mode 100644 Project/shared/__init__.py create mode 100644 Project/tests/__init__.py create mode 100644 Project/tests/train_tests/__init__.py diff --git a/Project/Classifier_2D/Dockerfile b/Project/Classifier_2D/Dockerfile new file mode 100644 index 0000000..2a7e158 --- /dev/null +++ b/Project/Classifier_2D/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.10 + +WORKDIR /train + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y --no-install-recommends build-essential git rsync software-properties-common ffmpeg libsm6 libxext6 && \ + rm -rf /var/lib/apt/lists/* + +ENV PYTHONPATH="/mlflow/projects/code/:$PYTHONPATH" + +COPY . . + +RUN python -m pip install --upgrade pip && \ + python -m pip install --no-cache-dir -r requirements.txt diff --git a/Project/Classifier_2D/README.md b/Project/Classifier_2D/README.md new file mode 100644 index 0000000..d3f076d --- /dev/null +++ b/Project/Classifier_2D/README.md @@ -0,0 +1,29 @@ +# 2D Classifier + +## Introduction +The template provides a starting point with helper classes and functions to facilitate rapid development and deployment of applications for 2D Classifiers. + +## Layout + +### There are 3 sections: + +#### 1. app +This area is used to work on the deployment of the application and provides helper frameworks. + +#### 2. tests +This area holds unit tests for both app and train. + +#### 3. train +This area is used to work on the training and creation of AI models + +*Further information can be found within each folder.* + + +## Examples + +https://github.com/GSTT-CSC/wrist-fracture-x-ray + +For further information on MLOps please refer to the MLOps repo: + +https://github.com/GSTT-CSC/MLOps + diff --git a/Project/Classifier_2D/__init__.py b/Project/Classifier_2D/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/Classifier_2D/config/__init__.py b/Project/Classifier_2D/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/Classifier_2D/config/config.cfg b/Project/Classifier_2D/config/config.cfg new file mode 100644 index 0000000..b847369 --- /dev/null +++ b/Project/Classifier_2D/config/config.cfg @@ -0,0 +1,34 @@ +[server] +MLFLOW_S3_ENDPOINT_URL = +MLFLOW_TRACKING_URI = +ARTIFACT_PATH = s3://mlflow + +[project] +NAME = example_mlflow_project +;VOLUME_MOUNT = < data path>:/DATA + +[system] +USE_GPU = True +CUDA_VISIBLE_DEVICES = 0 + +[xnat] +USER = +PASSWORD = +PROJECT = example_xnat_project +VERIFY = False +SERVER = + +[params] +RANDOM_SEED = 42 +DM_BATCH_SIZE = +DROPOUT = +LR = +MODEL = +NW_BATCH_SIZE = +TEST_FRACTION = +PRETRAINED = +WEIGHTED_LOSS = +LABEL_SMOOTHING = +MAX_EPOCHS = +GRAD_BATCHES = +IMAGE_SIZE = diff --git a/Project/Classifier_2D/config/local_config.cfg b/Project/Classifier_2D/config/local_config.cfg new file mode 100644 index 0000000..0d00679 --- /dev/null +++ b/Project/Classifier_2D/config/local_config.cfg @@ -0,0 +1,34 @@ +[server] +MLFLOW_S3_ENDPOINT_URL = http://localhost:8002 +MLFLOW_TRACKING_URI = http://localhost:85 +ARTIFACT_PATH = s3://mlflow + +[project] +NAME = example_mlflow_project +;VOLUME_MOUNT = < data path>:/DATA + +[system] +USE_GPU = false +CUDA_VISIBLE_DEVICES = 0 + +[xnat] +USER = +PASSWORD = +PROJECT = example_xnat_project +VERIFY = False +SERVER = http://localhost + +[params] +RANDOM_SEED = 42 +DM_BATCH_SIZE = +DROPOUT = +LR = +MODEL = +NW_BATCH_SIZE = +TEST_FRACTION = +PRETRAINED = +WEIGHTED_LOSS = +LABEL_SMOOTHING = +MAX_EPOCHS = +GRAD_BATCHES = +IMAGE_SIZE = diff --git a/Project/Classifier_2D/mlops/README.md b/Project/Classifier_2D/mlops/README.md new file mode 100644 index 0000000..a8304db --- /dev/null +++ b/Project/Classifier_2D/mlops/README.md @@ -0,0 +1,60 @@ +## Example Workflow: + +#### 1. Adapt XNATDataImport.py for your data +#### 2. Adapt Datamodule.py and Network.py if necessary +#### 3. Perform hyperparameter tuning using tune.py +#### 4. Utilise these hyperparameter values with train.py and threshold tune. +#### 5. Success! Model is stored in mlflow and ready for app development. + +More information can be found below. + +## There are 7 main components for training 2D Classifiers: + +### 1. `config/config.cfg` and `config/local_config.cfg` +The config file contains all the information that is used for configuring the project, experiment, and tracking server. This includes training parameters and XNAT configurations. + +The config file path is also passed as an argument to the MLOps `Experiment` class where the experiment and tracking are configured. + +As there will be differences between local development and running on DGX (for example XNAT configurations), it is highly encouraged to make use of `local_config` when developing locally. + +Note: The values present in the template config files are examples, you can remove any except those in `[server]` and `[project]` which are necessary for MLOps. Outside of these you are encouraged to add and modify the config files as relevant to your project. + +### 2. `src/XNATDataImport.py` +This file is used to define and pull the required data from XNAT. It utilises DataBuilderXNAT to do so as shown in the example, if you require additional or different data from XNAT additional actions can be added. + +If your data is not stored in XNAT this can be replaced by any method that accesses your data. + +### 3. `src/Network.py` +This file is used to define the PyTorch `LightningModule` class. + +This is where you set the Network architecture and flow that you will use for training, validation, and testing. + +Here you can set up which metrics are calculated and at which stage in the flow these occur, along with the model and optimiser. + +The example has numerous metrics and steps that are not always necessary, feel free to delete or add as relevant to your project. + +### 4. `src/DataModule.py` +This file is used to define the PyTorch `LightningDataModule` class. + +This is where you define the data that is used for training, validation, and testing. + +The example involves additional data validation steps that might not be relevant, feel free to delete or add as relevant to your project. + +### 5. `scripts/tune.py` +This file is used to define hyperparameter tuning runs. + +This is where the 'Datamodule' and' Network' are pulled together and uses optuna to perform the trials. +Starting hyperparameters are included near the top of the file. + +### 6. `scripts/train.py` +This file is used to define the training run. + +This is where the `Datamodule` and `Network` are pulled together. + +The example includes callbacks to retrieve the best model parameters, feel free to delete or add as relevant to your project. + +### 7. `Dockerfile` +This dockerfile sets up the Docker image that the MLOps run will utilise. + +In the example this is just a simple environment running python version 3.10. +You will most likely need to adapt this for your project. diff --git a/Project/Classifier_2D/mlops/__init__.py b/Project/Classifier_2D/mlops/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/Classifier_2D/mlops/train.py b/Project/Classifier_2D/mlops/train.py new file mode 100644 index 0000000..2c3fb56 --- /dev/null +++ b/Project/Classifier_2D/mlops/train.py @@ -0,0 +1,177 @@ +import sys +import configparser +import logging +import os +import multiprocessing +import json + +import mlflow +import pytorch_lightning as pl +import torch +from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping +from torch.cuda import is_available as cuda_available + +from Project.Classifier_2D.src.data_module import DataModule +from Project.Classifier_2D.src.network import Network +from Project.Classifier_2D.src.data_module import label_dict +from Project.Classifier_2D.src.utils.XNAT_data_import import XNATDataImport + +logger = logging.getLogger(__name__) + +def train(config): + + os.environ["CUDA_VISIBLE_DEVICES"] = config["system"]["cuda_visible_devices"] + + xnat_configuration = {'server': config['xnat']['SERVER'], + 'user': config['xnat']['USER'], + 'password': config['xnat']['PASSWORD'], + 'project': config['xnat']['PROJECT'], + 'verify': config.getboolean('xnat', 'VERIFY')} + + max_workers = 32 + num_workers = ( + max_workers + if max_workers < multiprocessing.cpu_count() + else multiprocessing.cpu_count() + ) + + importer = XNATDataImport( + xnat_configuration = xnat_configuration, + num_workers = num_workers + ) + + # Import raw data + raw_data = importer.import_xnat_data() + + # Download images from XNAT + data = importer.xnat_image_download(raw_data) + + # Set up mflow experiment + with mlflow.start_run(nested=True): + save_best_model = True + + mlflow.pytorch.autolog(log_models=False) + + # initialise network and datamodule + dm = DataModule( + data = data, + dm_batch_size = int(config['params']['dm_batch_size']), + test_fraction = float(config['params']['test_fraction']), + num_workers = num_workers, + random_seed = int(config['params']['random_seed']), + image_size = int(config['params']['image_size']) + ) + + dm.setup() + + n_classes = len(set([x for x in label_dict.values() if x is not None])) + mlflow.log_param('n_classes', n_classes) + + train_class_weights = dm.data_manifest["train"]["class_weights"] + validation_class_weights = dm.data_manifest["validation"]["class_weights"] + + net = Network( + model_name = config['params']['model'], + pretrained = config['params']['pretrained'], + n_classes = n_classes, + dropout = float(config['params']['dropout']), + weighted_loss = config['params']['weighted_loss'], + train_class_weights = train_class_weights, + validation_class_weights = validation_class_weights, + learning_rate = float(config['params']['lr']), + max_lr = float(config['params']['max_lr']), + nw_batch_size = int(config['params']['nw_batch_size']), + label_smoothing = float(config['params']['label_smoothing']), + ) + + # Callbacks + checkpoint_metric = config['params']['checkpoint_metric'] + checkpoint_mode = "min" if checkpoint_metric == "val_loss" else "max" + + # Callbacks + callbacks = [] + callbacks.append(LearningRateMonitor(logging_interval="step")) + checkpoint_callback = ModelCheckpoint( + save_top_k=1, + monitor="val_loss", + mode="min", + dirpath="./checkpoint/", + ) + callbacks.append(checkpoint_callback) + + early_stopping_callback = EarlyStopping( + monitor=checkpoint_metric, + patience=10, + mode=checkpoint_mode, + verbose=True, + ) + callbacks.append(early_stopping_callback) + + # configure trainer + trainer = pl.Trainer( + precision="32" if cuda_available() else "16", + callbacks=callbacks, + devices= 1 if cuda_available() else "auto", + accelerator="gpu" if cuda_available() else "auto", + log_every_n_steps=1, + num_sanity_val_steps=0, + max_epochs=int(config['params']['max_epochs']), + accumulate_grad_batches= int(config['params']['grad_batches']) + ) + + trainer.fit(net, dm) + + if save_best_model: + checkpoint = torch.load(checkpoint_callback.best_model_path) + net.load_state_dict(checkpoint['state_dict']) + file_path = f"model-{config['project']['name']}-{mlflow.active_run().info.run_name}.pt" + script = net.to_torchscript(file_path=file_path) + + checkpoint_info = {} + checkpoint_info["monitored_metric"] = checkpoint_callback.monitor + checkpoint_info["metric_value"] = checkpoint_callback.best_model_score.item() + checkpoint_info["mode"] = checkpoint_callback.mode + checkpoint_info["epoch"] = checkpoint["epoch"] + + with open("checkpoint_info.json", "w") as f: + json.dump(checkpoint_info, f, indent=2) + + mlflow.pytorch.log_model( + script, file_path, extra_files=["checkpoint_info.json"] + ) + + _ = [ + os.remove(fp) for fp in ["checkpoint_info.json"] + ] # remove the temporary files after logging to mlflow + + # Prepare config for mlflow logging + useful_keys = ['system', + 'project', + 'params', + ] + + with open(('config_log.txt'), 'w') as f: + for section in useful_keys: + for key, value in config.items(section): + f.write(f'{key} = {value}\n') + + mlflow.log_artifact('config_log.txt') + + logger.info('Finding best threshold') + best_model_path = checkpoint_callback.best_model_path + best_model = net.load_from_checkpoint(best_model_path) + net.evaluate_best_model(best_model, threshold_tune=True) + + mlflow.log_artifact('val_transform_failures.csv') + mlflow.log_artifact('train_transform_failures.csv') + + logger.info('Training complete') + + +if __name__ == '__main__': + + config_path = 'config/config.cfg' + + config = configparser.ConfigParser() + config.read(config_path) + train(config) \ No newline at end of file diff --git a/Project/Classifier_2D/mlops/tune.py b/Project/Classifier_2D/mlops/tune.py new file mode 100644 index 0000000..55b3066 --- /dev/null +++ b/Project/Classifier_2D/mlops/tune.py @@ -0,0 +1,198 @@ +import configparser +import logging +import multiprocessing +import os + +import mlflow +import pytorch_lightning as pl +from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping +from torch.cuda import is_available as cuda_available + +from Project.Classifier_2D.src.data_module import DataModule +from Project.Classifier_2D.src.data_module import label_dict +from Project.Classifier_2D.src.network import Network +from Project.Classifier_2D.src.utils.XNAT_data_import import XNATDataImport + +import optuna +logger = logging.getLogger(__name__) + +# Obtain hyperparameters for this trial +def suggest_hyperparameters(trial): + + dropout = trial.suggest_float("dropout", 0.0, 0.3, step=0.1) + lr = trial.suggest_float("lr", 1e-4, 5e-4, log=True) + max_lr = trial.suggest_categorical("max_lr",[5e-4,1e-3]) + model = trial.suggest_categorical("model", ["convnextv2_tiny.fcmae_ft_in22k_in1k","convnextv2_base.fcmae_ft_in22k_in1k"]) + batch_size = trial.suggest_int("batch_size", 4, 16, step=4) + pretrained = trial.suggest_categorical("pretrained", ["TRUE"]) + label_smoothing = trial.suggest_float("label_smoothing", 0.0, 0.1) + grad_batches = trial.suggest_int('grad_batches',1,4) + image_size = trial.suggest_categorical("image_size", [224]) + + params = { + "dropout": dropout, + "lr": lr, + "max_lr": max_lr, + "model": model, + "batch_size": batch_size, + "pretrained": pretrained, + "label_smoothing": label_smoothing, + "grad_batches": grad_batches, + "image_size": image_size + } + return params + +def objective(trial,data,config): + + max_workers = 32 + num_workers = ( + max_workers + if max_workers < multiprocessing.cpu_count() + else multiprocessing.cpu_count() + ) + + best_val_loss = float('Inf') + + # Start a new mlflow run + with mlflow.start_run(nested=True): + + params = suggest_hyperparameters(trial) + mlflow.log_params(params) + + # initialise network and datamodule + dm = DataModule( + data = data, + dm_batch_size = int(config['params']['dm_batch_size']), + test_fraction = float(config['params']['test_fraction']), + num_workers = num_workers, + random_seed = int(config['params']['random_seed']), + image_size = params['image_size'] + ) + + dm.setup() + + n_classes = len(set([x for x in label_dict.values() if x is not None])) + mlflow.log_param('n_classes', n_classes) + + train_class_weights = dm.data_manifest["train"]["class_weights"] + validation_class_weights = dm.data_manifest["validation"]["class_weights"] + + net = Network( + model_name = params['model'], + pretrained = params['pretrained'], + n_classes = n_classes, + dropout = params['dropout'], + weighted_loss = config['params']['weighted_loss'], + train_class_weights = train_class_weights, + validation_class_weights = validation_class_weights, + learning_rate = params['lr'], + max_lr = params['max_lr'], + nw_batch_size = params['batch_size'], + label_smoothing = params['label_smoothing'] + ) + + # Callbacks + callbacks = [] + callbacks.append(LearningRateMonitor(logging_interval="step")) + checkpoint_callback = ModelCheckpoint( + save_top_k=1, + monitor="val_loss", + mode="min", + dirpath= f"./checkpoint_trial_{trial.number}/", + ) + callbacks.append(checkpoint_callback) + + early_stopping_callback = EarlyStopping( + monitor="val_loss", + patience=10, + mode="min", + verbose=True, + ) + callbacks.append(early_stopping_callback) + + # configure trainer + trainer = pl.Trainer( + precision="32" if cuda_available() else "16", + callbacks=callbacks, + devices= 1 if cuda_available() else "auto", + accelerator="gpu" if cuda_available() else "auto", + log_every_n_steps=1, + num_sanity_val_steps=0, + max_epochs=int(config['params']['max_epochs']), + accumulate_grad_batches= params['grad_batches'] + ) + + trainer.fit(net, dm) + + best_model_path = checkpoint_callback.best_model_path + best_model = net.load_from_checkpoint(best_model_path) + net.evaluate_best_model(best_model) + + val_loss = checkpoint_callback.best_model_score.item() + + if val_loss <= best_val_loss: + best_val_loss = val_loss + + return best_val_loss + +def tune(config): + + os.environ["CUDA_VISIBLE_DEVICES"] = config["system"]["cuda_visible_devices"] + + xnat_configuration = {'server': config['xnat']['SERVER'], + 'user': config['xnat']['USER'], + 'password': config['xnat']['PASSWORD'], + 'project': config['xnat']['PROJECT'], + 'verify': config.getboolean('xnat', 'VERIFY')} + + max_workers = 32 + num_workers = ( + max_workers + if max_workers < multiprocessing.cpu_count() + else multiprocessing.cpu_count() + ) + + importer = XNATDataImport( + xnat_configuration = xnat_configuration, + num_workers = num_workers + ) + + # Import raw data + raw_data = importer.import_xnat_data() + + # Download images from XNAT + data = importer.xnat_image_download(raw_data) + + + mlflow.pytorch.autolog(log_models=False) + + # Create optuna study (hyperparameter tuning framework) + study = optuna.create_study(study_name="project-tune", direction="minimize") + study.optimize(lambda trial: objective(trial, data, config), n_trials=50) + + with open(('tune_log.txt'), 'w') as f: + f.write("Study statistics: \n") + f.write(f" Number of finished trials: {len(study.trials)}\n") + f.write("Best trial:\n") + + trial = study.best_trial + + f.write(f" Trial number: {trial.number}\n") + f.write(f" Loss (trial value): {trial.value}\n") + + f.write("Params:\n") + for key, value in trial.params.items(): + f.write(f" {key} = {value}\n") + + mlflow.log_artifact('tune_log.txt') + mlflow.log_artifact('val_transform_failures.csv') + mlflow.log_artifact('train_transform_failures.csv') + +if __name__ == '__main__': + + + config_path = 'config/config.cfg' + + config = configparser.ConfigParser() + config.read(config_path) + tune(config) diff --git a/Project/Classifier_2D/scripts/__init__.py b/Project/Classifier_2D/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/Classifier_2D/src/__init__.py b/Project/Classifier_2D/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/Classifier_2D/src/data_module.py b/Project/Classifier_2D/src/data_module.py new file mode 100644 index 0000000..4ba94e4 --- /dev/null +++ b/Project/Classifier_2D/src/data_module.py @@ -0,0 +1,153 @@ +import logging +from collections import Counter +from typing import List, Optional +import mlflow +import numpy as np +import pytorch_lightning +import torch + +from monai.data import CacheDataset, Dataset +from monai.data import pad_list_data_collate +from monai.transforms import Compose + +from sklearn.model_selection import train_test_split +from torch.cuda import is_available +from torch.utils.data import DataLoader + + +from Project.Classifier_2D.src.transforms import normalise, train_augment, output +from Project.Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform + +logger = logging.getLogger(__name__) + +label_dict = { + 'NEGATIVE': 0, + 'POSITIVE': 1, +} + +class DataModule(pytorch_lightning.LightningDataModule): + + def __init__(self, data, dm_batch_size: int = 1, num_workers: int = 16, + test_fraction: float = 0.2, cache_dataset=False, + random_seed: int = 42, image_size: int = 224): + super().__init__() + self.data = data + self.xnat_data_list = None + self.num_workers = num_workers + self.batch_size = dm_batch_size + self.test_fraction = test_fraction + self.cache_dataset = cache_dataset + self.random_seed = random_seed + self.image_size = image_size + + self.torch_device = self.trainer.model.device if self.trainer else torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') + + self.train_transforms = Compose( + normalise(self.image_size) + + train_augment(self.image_size) + + output(self.image_size) + ) + + self.val_transforms = Compose( + normalise(self.image_size) + + output(self.image_size) + ) + + def calculate_class_weights(self, data) -> list: + """Calculate class weights for unbalanced datasets + + :param data: input data + :type data: list + :return: class weights + :rtype: list + """ + labels = self.get_labels(data) + return [ + len(labels) / (len(set(labels)) * labels.count(l)) + for l in sorted(set(labels)) + ] + + def get_labels(self, data) -> list: + """Get list of sample labels + + :param data: input data + :type data: list + :return: list of labels + :rtype: list + """ + return [ + int(sample["action_data"]) + for subject in data + for sample in subject["data"] + if sample["data_label"] == "label" + ] + + def setup(self, stage: Optional[str] = None): + """ + Use the setup method to set up your data and define your Dataset objects + :param stage: + :return: + """ + data = self.data + + self.labels = [int(sample['action_data']) for subject in data for sample in subject['data'] if + sample['data_label'] == 'label'] + + self.train_data, self.validation_data = train_test_split(data, + test_size=self.test_fraction, + stratify=self.labels, + random_state=self.random_seed, + ) + + self.data_manifest = { + 'N_total': len(data), + 'train': self.dataset_stats(self.train_data), + 'validation': self.dataset_stats(self.validation_data), + 'labels': label_dict + } + + mlflow.log_dict(self.data_manifest, "data_manifest.json") + + if self.cache_dataset: + self.val_dataset = CacheDataset(data=self.validation_data, + transform=SafeWrapperTransform(transform = self.val_transforms, + log_file = "val_transform_failures.csv", + image_size = self.image_size), + num_workers=self.num_workers) + self.train_dataset = CacheDataset(data=self.train_data, + transform=SafeWrapperTransform(transform = self.train_transforms, + log_file = "train_transform_failures.csv", + image_size = self.image_size), + num_workers=self.num_workers) + else: + self.val_dataset = Dataset(data=self.validation_data, transform=SafeWrapperTransform(transform = self.val_transforms, + log_file = "val_transform_failures.csv", + image_size = self.image_size) + ) + self.train_dataset = Dataset(data=self.train_data, transform=SafeWrapperTransform(transform = self.train_transforms, + log_file = "train_transform_failures.csv", + image_size = self.image_size) + ) + + def train_dataloader(self): + """ + Define train dataloader + :return: + """ + + return DataLoader(self.train_dataset, batch_size=self.batch_size, + num_workers=self.num_workers, collate_fn=pad_list_data_collate, + pin_memory=is_available()) + + def val_dataloader(self): + """ + Define validation dataloader + :return: + """ + return DataLoader(self.val_dataset, batch_size=1, num_workers=self.num_workers, + pin_memory=is_available(), collate_fn=pad_list_data_collate, ) + + def dataset_stats(self, dataset): + stats = {'n_samples': len(dataset)} + stats["class_weights"] = self.calculate_class_weights(dataset) + return stats \ No newline at end of file diff --git a/Project/Classifier_2D/src/network.py b/Project/Classifier_2D/src/network.py new file mode 100644 index 0000000..d264c1a --- /dev/null +++ b/Project/Classifier_2D/src/network.py @@ -0,0 +1,226 @@ +import logging +import pytorch_lightning +import mlflow +import torch +from abc import ABC +from monai.data import decollate_batch +from monai.transforms import ( + AsDiscrete, + Compose, + Activations, +) +from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay, recall_score +from timm import create_model +from timm.data import Mixup +from torch.nn import CrossEntropyLoss +from torchmetrics import Accuracy, F1Score +from Project.Classifier_2D.src.data_module import label_dict +from torchmetrics.classification import MulticlassAUROC +import numpy as np + +logger = logging.getLogger(__name__) + + +class Network(pytorch_lightning.LightningModule, ABC): + """ + Network object defines the model architecture, inherits from LightningModule. + + https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html + """ + + def __init__(self, + model_name, + pretrained, + n_classes, + dropout, + weighted_loss, + train_class_weights, + validation_class_weights, + learning_rate, + max_lr, + nw_batch_size, + label_smoothing, + **kwargs): + super().__init__() + + self.save_hyperparameters() + self.n_classes = n_classes + self.dropout = dropout + self.report_interval = 5 + self.label_smoothing = label_smoothing + + self.y_pred_trans = Compose([Activations(softmax=True)]) + self.y_trans = Compose([AsDiscrete(to_onehot=self.n_classes)]) + + self.learning_rate = learning_rate + self.max_lr = max_lr + self.batch_size = nw_batch_size + + self.val_acc = Accuracy(task="multiclass", num_classes=self.n_classes, top_k=1) + self.val_f1 = F1Score(task="multiclass", num_classes=self.n_classes, top_k=1) + self.val_auroc = MulticlassAUROC(num_classes=self.n_classes, average='macro', thresholds=None) + + self.targets, self.labels = list( + map(list, zip(*[(target, label) for target, label in label_dict.items() if label is not None]))) + + self.model_name = model_name + self.pretrained = pretrained + self.model = create_model(self.model_name, pretrained=self.pretrained, in_chans=1, + num_classes=self.n_classes, drop_rate=self.dropout) + + self.weighted_loss = weighted_loss + self.train_class_weights = train_class_weights + self.validation_class_weights = validation_class_weights + + self.train_loss_function = CrossEntropyLoss( + weight=torch.tensor(self.train_class_weights) + if self.weighted_loss + else None + ) + + self.validation_loss_function = CrossEntropyLoss( + weight=torch.tensor(self.validation_class_weights) + if self.weighted_loss + else None + ) + + self.mixup_fn = Mixup( + mixup_alpha = 0.2, + cutmix_alpha = 0.2, + prob=0.5, + switch_prob=0.5, + mode='batch', + label_smoothing=0.1, + num_classes=self.n_classes, + ) + + def forward(self, x): + return self.model(x) + + def training_step(self, batch, batch_idx): + """ + Training step + :param batch: + :param batch_idx: + :return: loss + """ + valid_mask = batch['valid'].bool() + if valid_mask.sum() == 0: + return None + + x = batch['image'][valid_mask] + y = batch['label'][valid_mask] + if x.size(0) % 2 == 0: + x, y = self.mixup_fn(x, y) + + y_hat = self(x) + loss = self.train_loss_function(y_hat, y) + self.log('train_loss', loss, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + return loss + + def validation_step(self, batch, batch_idx): + """ + Validation step + :param batch: + :param batch_idx: + :return: loss + """ + valid_mask = batch['valid'].bool() + if valid_mask.sum() == 0: + return None + + x = batch['image'][valid_mask] + y = batch['label'][valid_mask] + y_hat = self(x) + + loss = self.validation_loss_function(y_hat, y) + self.log('val_loss', loss, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + # additional metrics + self.val_acc(y_hat, y) + self.log('val_acc', self.val_acc, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + y_onehot = [self.y_trans(i) for i in decollate_batch(y, detach=True)] + y_pred_act = [self.y_pred_trans(i) for i in decollate_batch(y_hat, detach=True)] + + self.val_f1(y_hat, y) + self.log('val_f1', self.val_f1, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + self.val_auroc(y_hat, y) + self.log('val_auroc', self.val_auroc, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + return {"loss": loss, 'y_onehot': y_onehot, 'y_pred_act': y_pred_act} + + def configure_optimizers(self): + optimizer = torch.optim.AdamW(self.parameters(), lr=self.learning_rate) + scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, max_lr=self.max_lr, total_steps=self.trainer.estimated_stepping_batches + ) + + return {"optimizer": optimizer, "lr_scheduler": scheduler} + + def evaluate_best_model(self, model, threshold_tune = False, epoch=0): + model.eval() + model.freeze() + + val_loader = self.trainer.datamodule.val_dataloader() + + preds, labels = [], [] + if threshold_tune == False: + for batch in val_loader: + images = batch['image'].to(self.device) + label = batch['label'] + y_hat = model(images) + pred = torch.argmax(y_hat, dim=1).cpu() + + preds.append(pred) + labels.append(label) + + self._create_classification_report(preds, labels) + self._create_confusion_matrix(preds, labels) + else: + for batch in val_loader: + images = batch['image'].to(self.device) + label = batch['label'] + y_hat = model(images) + pred = torch.softmax(y_hat, dim=1)[:, 1].cpu().numpy() + + preds.extend(pred) + labels.extend(label) + + best_thresh = 0.5 + best_sensitivity = 0 + + for t in np.linspace(0,1,100): + t_preds = (np.array(preds) >= t).astype(int) + + sens = recall_score(labels, t_preds) + if sens > best_sensitivity: + best_sensitivity = sens + best_thresh = t + + d_preds = (np.array(preds) >= 0.5).astype(int) + d_sens = recall_score(labels,d_preds) + + threshold_text = f"Default Sensitivity: {d_sens}\nBest Threshold: {best_thresh}\nBest Sensitivity: {best_sensitivity}" + with open("threshold.txt", "w") as f: + f.write(threshold_text) + mlflow.log_artifact('threshold.txt') + preds = (np.array(preds) >= best_thresh).astype(int) + self._create_classification_report(preds, labels) + self._create_confusion_matrix(preds, labels) + + def validation_epoch_end(self, validation_step_outputs) -> None: + if self.current_epoch % self.report_interval == 0: + self._attribute(n_samples_plot=4, step_type='validation') + + def _create_classification_report(self, preds, labels): + val_report = classification_report(labels, preds, labels=self.labels, target_names=self.targets) + mlflow.log_text(val_report, f'classification_report.txt') + + def _create_confusion_matrix(self, preds, labels): + confmat = confusion_matrix(labels, preds, normalize='true') + disp = ConfusionMatrixDisplay(confusion_matrix=confmat) + disp.plot() + mlflow.log_figure(disp.figure_, 'confusion_matrix.jpg') \ No newline at end of file diff --git a/Project/Classifier_2D/src/transforms/__init__.py b/Project/Classifier_2D/src/transforms/__init__.py new file mode 100644 index 0000000..def4dfd --- /dev/null +++ b/Project/Classifier_2D/src/transforms/__init__.py @@ -0,0 +1,92 @@ +import torch +from monai.transforms import ( + LoadImage, + SqueezeDimd, + EnsureChannelFirstd, + CropForegroundd, + Resized, + ScaleIntensityd, + CastToTyped, + RandFlipd, + RandZoomd, + RandRotated, + RandAffined, + RandGaussianNoised, + RandGaussianSmoothd, + RandScaleIntensityd, + RandAdjustContrastd, + RandCoarseDropoutd, + ResizeWithPadOrCropd, + ToTensord, + SelectItemsd, + EnsureTyped, + Spacingd, +) + +from Project.Classifier_2D.src.transforms.load_image_XNATd import LoadImageXNATd + +def load_xnat(xnat_configuration: dict): + """ + This transform is used by the DataModule to load images from XNAT + """ + return [ + LoadImageXNATd( + keys=['data'], + xnat_configuration=xnat_configuration, + expected_filetype_ext='.dcm', + image_loader=LoadImage(image_only=True, prune_meta_pattern="^0|^2") + ), + ] + +def normalise(image_size): + """ + This transform list is used to prepare tensors for training or inference + """ + return [ + SqueezeDimd(keys=['image'], dim=2), + EnsureChannelFirstd(keys=['image']), + CropForegroundd(keys=['image'], source_key='image'), + Resized(keys=['image'], size_mode='longest', spatial_size=image_size+20), + #Maybe limit top intensity in case of big spikes? + ScaleIntensityd(keys=["image"], minv=0.0, maxv=255.0), + CastToTyped(keys=["image"], dtype=torch.uint8), + ] + +def train_augment(image_size): + """ + This transform list is used to augment images for training. + Aim here is to improve generalisation. + """ + return [ + RandFlipd(keys=['image'], spatial_axis=0, prob=0.5), + RandZoomd(keys=['image'], prob=0.2, min_zoom=1.05,max_zoom=1.1), + RandRotated(keys=['image'], prob=0.2, range_x=0.4), + RandAffined(keys=['image'], prob=0.2, padding_mode='zeros'), + RandGaussianNoised(keys=['image'], prob=0.1, mean=0.0, std=0.1), + RandGaussianSmoothd(keys=['image'], prob=0.2, sigma_x=(0.5,1.0)), + RandScaleIntensityd(keys=['image'], prob=0.15, factors=(0.75,1.25)), + RandAdjustContrastd(keys=['image'], prob=0.1, gamma=(0.5,2), retain_stats=True, invert_image=True), + RandAdjustContrastd(keys=['image'], prob=0.3, gamma=(0.5,2), retain_stats=True, invert_image=False), + ResizeWithPadOrCropd( + keys=["image"], + spatial_size=(image_size,image_size), + mode='replicate' + ), + RandCoarseDropoutd(keys=['image'], prob=0.5, fill_value=0, holes=8, max_holes=16, spatial_size=(10,10), max_spatial_size=(36,36)), + ] + +def output(image_size): + """ + This transform list is used for final normalisation and feature selection. + """ + return [ + ResizeWithPadOrCropd( + keys=["image"], + spatial_size=(image_size,image_size), + mode='replicate' + ), + ScaleIntensityd(keys=["image"], minv=0.0, maxv=1), + ToTensord(keys=['image', 'label']), + SelectItemsd(keys=['subject_id', 'image', 'label']), + EnsureTyped(keys=['image', 'label'], track_meta=False), + ] \ No newline at end of file diff --git a/Project/Classifier_2D/src/transforms/load_image_XNATd.py b/Project/Classifier_2D/src/transforms/load_image_XNATd.py new file mode 100644 index 0000000..d91eb39 --- /dev/null +++ b/Project/Classifier_2D/src/transforms/load_image_XNATd.py @@ -0,0 +1,123 @@ +""" +MONAI MapTransform for importing image data from XNAT +""" +import glob +import logging +import os +import tempfile +import time + +import xnat +from monai.config import KeysCollection +from monai.transforms import MapTransform, LoadImage +from monai.transforms import Transform + +logger = logging.getLogger(__name__) + + +class LoadImageXNATd(MapTransform): + """ + MapTransform for importing image data from XNAT + """ + + def __init__(self, keys: KeysCollection, xnat_configuration: dict = None, + image_loader: Transform = LoadImage(), validate_data: bool = False, + expected_filetype_ext: str = '.dcm', + image_series_option: str = 'error', + verbose=False): + super().__init__(keys) + self.image_loader = image_loader + self.xnat_configuration = xnat_configuration + self.expected_filetype = expected_filetype_ext + self.validate_data = validate_data + self.image_series_option = image_series_option + self.verbose = verbose + + def __call__(self, data): + """ + Checks for the requested keys in the input data dictionary. + + If specified key is found then will loop over actions in action list and apply each to the value of the + requested key, if no action is triggered then raise a warning actions arefunctions that return any of projects, + subjects, experiments, scans, or resources XNAT object along with a key to be used in the data dictionary + + Each action function should locate a single image object in XNAT. This image object is then downloaded to a + temporary directory and loaded into memory as the value defined by key set by the actions' data_label. + + If validate_data is true then NO data will be downloaded. In this case the transform will loop over the actions + but will instead return a true/false value for each data sample. This can be used to remove samples where the + data is not present in XNAT. + + :param data: dictionary of data + :return: + + """ + + d = dict(data) + + for key in self.keys: + + if key in data: + + with xnat.connect(server=self.xnat_configuration['server'], + user=self.xnat_configuration['user'], + password=self.xnat_configuration['password'], + verify=self.xnat_configuration['verify'], + loglevel='ERROR', + ) as session: + + "Check data list has no duplicate keys" + if len(set([x['data_label'] for x in d[key]])) != len([x['data_label'] for x in d[key]]): + logger.warning('Multiple images with identical labels found') + raise + + "Download image from XNAT" + for item in d[key]: + data_label = item['data_label'] + if item['data_type'] == 'value': + d[data_label] = item['action_data'] + continue + + attempts = 0 + while attempts < 3: + try: + with tempfile.TemporaryDirectory() as tmpdirname: + session_obj = session.create_object(item['action_data']) + session_obj.download_dir(tmpdirname, verbose=self.verbose) + + images_path = glob.glob(os.path.join(tmpdirname, '**/*' + self.expected_filetype), + recursive=True) + + # image loader needs full path to load single images + # logger.info(f"Downloading images: {images_path}") + if not images_path: + raise Exception + + if len(images_path) == 1: + image = self.image_loader(images_path) + + # image loader needs directory path to load 3D images + else: + "find unique directories in list of image paths" + image_dirs = list( + set(os.path.dirname(image_path) for image_path in images_path)) + if len(image_dirs) > 1: + if self.image_series_option == 'error': + raise ValueError(f'More than one image series found in {images_path}') + elif self.image_series_option == 'keepfirst': + image = self.image_loader(image_dirs[0]) + elif self.image_series_option == 'keeplast': + image = self.image_loader(image_dirs[-1]) + else: + raise ValueError(f"More than one image series found in {images_path} and no option specified") + break + + except Exception as e: + attempts += 1 + time.sleep(1.0) + if attempts == 3: + raise Exception(f'Image loader failed on {item} after 3 retries due to {e}') + + d[data_label] = image + + return d diff --git a/Project/Classifier_2D/src/transforms/safe_wrapper.py b/Project/Classifier_2D/src/transforms/safe_wrapper.py new file mode 100644 index 0000000..0436d64 --- /dev/null +++ b/Project/Classifier_2D/src/transforms/safe_wrapper.py @@ -0,0 +1,47 @@ +# Wrapper for Monai Transforms so failures are logged instead of run ending + +import os +import csv +import torch + +class SafeWrapperTransform: + def __init__(self, transform, image_size, log_file="transform_failures.csv"): + self.transform = transform + self.image_size = image_size + self.log_file = log_file + self.logged_ids = set() + + # Create the log file + if not os.path.exists(self.log_file): + with open(self.log_file, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["subject_id", "data", "error_message"]) + + def subject_already_logged(self, subject_id): + if not os.path.exists(self.log_file): + return False + with open(self.log_file, "r") as f: + return any(subject_id == row.split(",")[0] for row in f.readlines()[1:]) + + def __call__(self, data): + try: + result = self.transform(data) + result["valid"] = True + return result + except Exception as e: + subject_id = data.get("subject_id", "UNKNOWN") + + # Check if already logged + if not self.subject_already_logged(subject_id): + subject_data = data.get("data", "N/A") + with open(self.log_file, "a", newline="") as f: + writer = csv.writer(f) + writer.writerow([subject_id, subject_data, str(e)]) + + # Return dummy image, label, and data + keys =['subject_id'] + result = {k: v for k, v in data.items() if k in keys} + result["image"] = torch.zeros((1, self.image_size, self.image_size), dtype=torch.float32) + result["label"] = torch.tensor(-1) + result["valid"] = False + return result diff --git a/Project/Classifier_2D/src/utils/XNAT_data_import.py b/Project/Classifier_2D/src/utils/XNAT_data_import.py new file mode 100644 index 0000000..691e08f --- /dev/null +++ b/Project/Classifier_2D/src/utils/XNAT_data_import.py @@ -0,0 +1,83 @@ +import logging +from tqdm import tqdm +from typing import List + +from Project.Classifier_2D.src.utils.tools import DataBuilderXNAT +from xnat.mixin import ImageScanData, SubjectData + +from Project.Classifier_2D.src.transforms import load_xnat +from monai.data import Dataset +from torch.utils.data import DataLoader + +logger = logging.getLogger(__name__) + +class XNATDataImport(): + + def __init__(self, xnat_configuration: dict = None, num_workers: int = 4, test_batch: int = 0, + n_month_data_window=9999, run_type: str='train'): + self.xnat_configuration = xnat_configuration + self.num_workers = 4 + self.test_batch = test_batch + self.run_type = run_type + + def import_xnat_data(self): + + if self.run_type == 'train': + actions = [ + (self.fetch_xr, 'image'), + (self.fetch_label, 'label'), + ] # list of tuples defining action functions and their data keys + + data_builder = DataBuilderXNAT(self.xnat_configuration, + actions=actions, + test_batch=self.test_batch, + num_workers=self.num_workers) + + data_builder.fetch_data() + return(data_builder.dataset) + + def xnat_image_download(self, data): + + raw_dataset = Dataset( + data=data, + transform=load_xnat(self.xnat_configuration) + ) + + raw_dataloader = DataLoader(raw_dataset, batch_size=None, + num_workers=self.num_workers,shuffle=False, pin_memory=False) + + extracted_data = [] + for sample in tqdm(raw_dataloader, "Downloading Image Data"): + extracted_data.append(sample) + + return(extracted_data) + + @staticmethod + def fetch_xr(subject_data: SubjectData = None) -> List[ImageScanData]: + """ + Function that identifies and returns the required xnat ImageData object from a xnat SubjectData object + along with the 'key' that it will be used to access it. + """ + + scan_objects = [] + + for exp in subject_data.experiments: + for scan in subject_data.experiments[exp].scans: + if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): + scan_objects.append(subject_data.experiments[exp].scans[scan].uri) + + return scan_objects + + @staticmethod + def fetch_label(subject_data: SubjectData = None): + """ + Function that identifies and returns the label from a SubjectData object + """ + for exp in subject_data.experiments: + for scan in subject_data.experiments[exp].scans: + if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): + try: + full_label = subject_data.experiments[exp].label + return full_label + except Exception as e: + logger.warning(f"Unable to fetch {subject_data.experiments[exp]}'s label due to exception: {e}") \ No newline at end of file diff --git a/Project/Classifier_2D/src/utils/__init__.py b/Project/Classifier_2D/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/Classifier_2D/src/utils/tools.py b/Project/Classifier_2D/src/utils/tools.py new file mode 100644 index 0000000..8df0b4e --- /dev/null +++ b/Project/Classifier_2D/src/utils/tools.py @@ -0,0 +1,209 @@ +import asyncio +import logging +from concurrent.futures import ThreadPoolExecutor +from itertools import chain + +import mlflow +import pandas as pd +import tqdm +import xnat + +logger = logging.getLogger(__name__) + + +class DataBuilderXNAT: + + def __init__(self, xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1, + num_workers: int = 1, validate_data=True): + self.xnat_configuration = xnat_configuration + self.actions = actions + self.flatten_output = flatten_output + self.test_batch = test_batch + self.missing_data_log = [] + self.num_workers = num_workers + self.validate_data = validate_data + + self.dataset = [] + + def fetch_data(self): + asyncio.run(self.start_async_process()) + + async def start_async_process(self): + with ThreadPoolExecutor(max_workers=self.num_workers) as executor: + # with requests.Session() as session: + with xnat.connect(server=self.xnat_configuration['server'], + user=self.xnat_configuration['user'], + password=self.xnat_configuration['password'], + verify=self.xnat_configuration['verify'], + loglevel='ERROR', + ) as session: + + logger.info(f"Collecting XNAT project: {self.xnat_configuration['project']}") + project = session.projects[self.xnat_configuration["project"]] + + if 0 < self.test_batch < len(project.subjects): + from random import sample + project_subjects = sample(project.subjects[:], self.test_batch) + else: + project_subjects = project.subjects[:] + + loop = asyncio.get_event_loop() + tasks = [ + loop.run_in_executor( + executor, + self.process_subject, + *(project, subject_i) + ) + for subject_i in project_subjects + ] + + responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(tasks))] + + for response in await asyncio.gather(*tasks): + pass + + # remove any items where not all actions returned a value + if self.validate_data: + self.dataset = [item for item in self.dataset if len(item['data']) >= len(self.actions)] + + def process_subject(self, project, subject_i): + subject = project.subjects.data[subject_i.id] + data_sample = {'subject_uri': project.subjects[subject.id].uri, + 'subject_id': project.subjects[subject.id].label, + 'data': []} + + logger.debug(f'checking subject {subject_i}') + if self.actions: + try: + data = [] + for action, data_label in self.actions: + action_data = [] + + # logger.debug(f"Running action: {action.__name__} on {subject.id}") + xnat_obj = action(project.subjects[subject.id]) + + if xnat_obj is None or type(xnat_obj) == list and len(xnat_obj) == 0: + self.missing_data_log.append({'subject_id': subject_i.id, + 'action_data': subject_i.label, + 'failed_action': action}) + logger.debug(f'No data found for {subject_i}: action {action} removing sample') + raise Exception + + elif type(xnat_obj) == list: + for obj in xnat_obj: + if obj.cache_id[0] == 'ResourceCatalog': + action_data.append({'source_action': action.__name__, + 'action_data': obj.uri, + 'data_type': 'xnat_uri', + 'data_label': data_label, + 'resource_files': list(obj.files)}) # retrieve filenames if xnat_obj contains Resource + else: + action_data.append({'source_action': action.__name__, + 'action_data': obj.uri, + 'data_type': 'xnat_uri', + 'data_label': data_label}) + + else: + action_data.append({'source_action': action.__name__, + 'action_data': xnat_obj, + 'data_type': 'value', + 'data_label': data_label}) + + data.append(action_data) + + except Exception as e: + logger.debug(f'No data found for {subject_i}; removing sample. Exception {e}') + pass + + if self.flatten_output: + data = list(chain(*data)) + + data_sample['data'] = data + + self.dataset.append((data_sample)) + + +def xnat_build_dataset(xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1): + """ + ***NOTE THE ASYNC VERSION ABOVE WILL BE MUCH FASTER FOR LARGE DATASETS*** + + Builds a dictionary that describes the XNAT project dataset using XNAT data hierarchy: project/subject/experiment/scan + + structured output returned flattened dataset. Can set structured_output to True to perform custom flattening. + + """ + + with xnat.connect(server=xnat_configuration['server'], + user=xnat_configuration['user'], + password=xnat_configuration['password'], + verify=xnat_configuration['verify'], + loglevel='ERROR', + ) as session: + + logger.info(f"Collecting XNAT project: {xnat_configuration['project']}") + project = session.projects[xnat_configuration["project"]] + + dataset = [] + + if 0 < test_batch < len(project.subjects): + from random import sample + project_subjects = sample(project.subjects[:], test_batch) + else: + project_subjects = project.subjects[:] + + missing_data_log = [] + for subject_i in project_subjects: + subject = project.subjects.data[subject_i.id] + data_sample = {'subject_uri': project.subjects[subject.id].uri, + 'subject_id': project.subjects[subject.id].label, + 'data': []} + + if actions: + try: + data = [] + for action, data_label in actions: + action_data = [] + + # logger.debug(f"Running action: {action.__name__} on {subject.id}") + xnat_obj = action(project.subjects[subject.id]) + + if type(xnat_obj) == list: + if len(xnat_obj) == 0: + missing_data_log.append({'subject_id': subject_i.id, + 'action_data': subject_i.label, + 'failed_action': action}) + logger.debug(f'No data found for {subject_i}: action {action} removing sample') + raise Exception + + for obj in xnat_obj: + action_data.append({'source_action': action.__name__, + 'action_data': obj.uri, + 'data_type': 'xnat_uri', + 'data_label': data_label}) + + elif type(xnat_obj) == str: + action_data.append({'source_action': action.__name__, + 'action_data': xnat_obj, + 'data_type': 'value', + 'data_label': data_label}) + + data.append(action_data) + + except Exception as e: + logger.debug(f'No data found for {subject_i}; removing sample. Exception {e}') + continue + + if flatten_output: + data = list(chain(*data)) + + data_sample['data'] = data + + dataset.append((data_sample)) + + if missing_data_log: + df = pd.DataFrame(missing_data_log) + df.to_csv('missing_data_log.csv') + mlflow.log_artifact('missing_data_log.csv') + mlflow.log_param('N_failed_xnat_samples', len(df)) + + return dataset diff --git a/Project/app/__init__.py b/Project/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/app/app_tests/__init__.py b/Project/app/app_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/shared/__init__.py b/Project/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/tests/__init__.py b/Project/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Project/tests/train_tests/__init__.py b/Project/tests/train_tests/__init__.py new file mode 100644 index 0000000..e69de29 From a244faf22c8fdb703e025e2dd5c6f37884a522f0 Mon Sep 17 00:00:00 2001 From: mikewoodward94 <105747447+mikewoodward94@users.noreply.github.com> Date: Wed, 6 May 2026 14:53:00 +0100 Subject: [PATCH 05/21] Delete Classifier_2D directory --- Classifier_2D/Dockerfile | 16 -- Classifier_2D/README.md | 29 --- Classifier_2D/__init__.py | 0 Classifier_2D/app/__init__.py | 0 Classifier_2D/config/__init__.py | 0 Classifier_2D/config/config.cfg | 34 --- Classifier_2D/config/local_config.cfg | 34 --- Classifier_2D/mlops/README.md | 60 ----- Classifier_2D/mlops/__init__.py | 0 Classifier_2D/mlops/train.py | 177 -------------- Classifier_2D/mlops/tune.py | 198 --------------- Classifier_2D/shared/XNAT_data_import.py | 83 ------- Classifier_2D/shared/__init__.py | 0 Classifier_2D/src/Network.py | 226 ------------------ Classifier_2D/src/__init__.py | 0 Classifier_2D/src/data_module.py | 152 ------------ Classifier_2D/src/transforms/__init__.py | 92 ------- .../src/transforms/load_image_XNATd.py | 123 ---------- Classifier_2D/src/transforms/safe_wrapper.py | 47 ---- Classifier_2D/src/utils/__init__.py | 0 Classifier_2D/src/utils/tools.py | 209 ---------------- Classifier_2D/tests/__init__.py | 0 Classifier_2D/tests/app_tests/__init__.py | 0 Classifier_2D/tests/train_tests/__init__.py | 0 24 files changed, 1480 deletions(-) delete mode 100644 Classifier_2D/Dockerfile delete mode 100644 Classifier_2D/README.md delete mode 100644 Classifier_2D/__init__.py delete mode 100644 Classifier_2D/app/__init__.py delete mode 100644 Classifier_2D/config/__init__.py delete mode 100644 Classifier_2D/config/config.cfg delete mode 100644 Classifier_2D/config/local_config.cfg delete mode 100644 Classifier_2D/mlops/README.md delete mode 100644 Classifier_2D/mlops/__init__.py delete mode 100644 Classifier_2D/mlops/train.py delete mode 100644 Classifier_2D/mlops/tune.py delete mode 100644 Classifier_2D/shared/XNAT_data_import.py delete mode 100644 Classifier_2D/shared/__init__.py delete mode 100644 Classifier_2D/src/Network.py delete mode 100644 Classifier_2D/src/__init__.py delete mode 100644 Classifier_2D/src/data_module.py delete mode 100644 Classifier_2D/src/transforms/__init__.py delete mode 100644 Classifier_2D/src/transforms/load_image_XNATd.py delete mode 100644 Classifier_2D/src/transforms/safe_wrapper.py delete mode 100644 Classifier_2D/src/utils/__init__.py delete mode 100644 Classifier_2D/src/utils/tools.py delete mode 100644 Classifier_2D/tests/__init__.py delete mode 100644 Classifier_2D/tests/app_tests/__init__.py delete mode 100644 Classifier_2D/tests/train_tests/__init__.py diff --git a/Classifier_2D/Dockerfile b/Classifier_2D/Dockerfile deleted file mode 100644 index 2a7e158..0000000 --- a/Classifier_2D/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.10 - -WORKDIR /train - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y --no-install-recommends build-essential git rsync software-properties-common ffmpeg libsm6 libxext6 && \ - rm -rf /var/lib/apt/lists/* - -ENV PYTHONPATH="/mlflow/projects/code/:$PYTHONPATH" - -COPY . . - -RUN python -m pip install --upgrade pip && \ - python -m pip install --no-cache-dir -r requirements.txt diff --git a/Classifier_2D/README.md b/Classifier_2D/README.md deleted file mode 100644 index d3f076d..0000000 --- a/Classifier_2D/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# 2D Classifier - -## Introduction -The template provides a starting point with helper classes and functions to facilitate rapid development and deployment of applications for 2D Classifiers. - -## Layout - -### There are 3 sections: - -#### 1. app -This area is used to work on the deployment of the application and provides helper frameworks. - -#### 2. tests -This area holds unit tests for both app and train. - -#### 3. train -This area is used to work on the training and creation of AI models - -*Further information can be found within each folder.* - - -## Examples - -https://github.com/GSTT-CSC/wrist-fracture-x-ray - -For further information on MLOps please refer to the MLOps repo: - -https://github.com/GSTT-CSC/MLOps - diff --git a/Classifier_2D/__init__.py b/Classifier_2D/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/app/__init__.py b/Classifier_2D/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/config/__init__.py b/Classifier_2D/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/config/config.cfg b/Classifier_2D/config/config.cfg deleted file mode 100644 index b847369..0000000 --- a/Classifier_2D/config/config.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[server] -MLFLOW_S3_ENDPOINT_URL = -MLFLOW_TRACKING_URI = -ARTIFACT_PATH = s3://mlflow - -[project] -NAME = example_mlflow_project -;VOLUME_MOUNT = < data path>:/DATA - -[system] -USE_GPU = True -CUDA_VISIBLE_DEVICES = 0 - -[xnat] -USER = -PASSWORD = -PROJECT = example_xnat_project -VERIFY = False -SERVER = - -[params] -RANDOM_SEED = 42 -DM_BATCH_SIZE = -DROPOUT = -LR = -MODEL = -NW_BATCH_SIZE = -TEST_FRACTION = -PRETRAINED = -WEIGHTED_LOSS = -LABEL_SMOOTHING = -MAX_EPOCHS = -GRAD_BATCHES = -IMAGE_SIZE = diff --git a/Classifier_2D/config/local_config.cfg b/Classifier_2D/config/local_config.cfg deleted file mode 100644 index 0d00679..0000000 --- a/Classifier_2D/config/local_config.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[server] -MLFLOW_S3_ENDPOINT_URL = http://localhost:8002 -MLFLOW_TRACKING_URI = http://localhost:85 -ARTIFACT_PATH = s3://mlflow - -[project] -NAME = example_mlflow_project -;VOLUME_MOUNT = < data path>:/DATA - -[system] -USE_GPU = false -CUDA_VISIBLE_DEVICES = 0 - -[xnat] -USER = -PASSWORD = -PROJECT = example_xnat_project -VERIFY = False -SERVER = http://localhost - -[params] -RANDOM_SEED = 42 -DM_BATCH_SIZE = -DROPOUT = -LR = -MODEL = -NW_BATCH_SIZE = -TEST_FRACTION = -PRETRAINED = -WEIGHTED_LOSS = -LABEL_SMOOTHING = -MAX_EPOCHS = -GRAD_BATCHES = -IMAGE_SIZE = diff --git a/Classifier_2D/mlops/README.md b/Classifier_2D/mlops/README.md deleted file mode 100644 index a8304db..0000000 --- a/Classifier_2D/mlops/README.md +++ /dev/null @@ -1,60 +0,0 @@ -## Example Workflow: - -#### 1. Adapt XNATDataImport.py for your data -#### 2. Adapt Datamodule.py and Network.py if necessary -#### 3. Perform hyperparameter tuning using tune.py -#### 4. Utilise these hyperparameter values with train.py and threshold tune. -#### 5. Success! Model is stored in mlflow and ready for app development. - -More information can be found below. - -## There are 7 main components for training 2D Classifiers: - -### 1. `config/config.cfg` and `config/local_config.cfg` -The config file contains all the information that is used for configuring the project, experiment, and tracking server. This includes training parameters and XNAT configurations. - -The config file path is also passed as an argument to the MLOps `Experiment` class where the experiment and tracking are configured. - -As there will be differences between local development and running on DGX (for example XNAT configurations), it is highly encouraged to make use of `local_config` when developing locally. - -Note: The values present in the template config files are examples, you can remove any except those in `[server]` and `[project]` which are necessary for MLOps. Outside of these you are encouraged to add and modify the config files as relevant to your project. - -### 2. `src/XNATDataImport.py` -This file is used to define and pull the required data from XNAT. It utilises DataBuilderXNAT to do so as shown in the example, if you require additional or different data from XNAT additional actions can be added. - -If your data is not stored in XNAT this can be replaced by any method that accesses your data. - -### 3. `src/Network.py` -This file is used to define the PyTorch `LightningModule` class. - -This is where you set the Network architecture and flow that you will use for training, validation, and testing. - -Here you can set up which metrics are calculated and at which stage in the flow these occur, along with the model and optimiser. - -The example has numerous metrics and steps that are not always necessary, feel free to delete or add as relevant to your project. - -### 4. `src/DataModule.py` -This file is used to define the PyTorch `LightningDataModule` class. - -This is where you define the data that is used for training, validation, and testing. - -The example involves additional data validation steps that might not be relevant, feel free to delete or add as relevant to your project. - -### 5. `scripts/tune.py` -This file is used to define hyperparameter tuning runs. - -This is where the 'Datamodule' and' Network' are pulled together and uses optuna to perform the trials. -Starting hyperparameters are included near the top of the file. - -### 6. `scripts/train.py` -This file is used to define the training run. - -This is where the `Datamodule` and `Network` are pulled together. - -The example includes callbacks to retrieve the best model parameters, feel free to delete or add as relevant to your project. - -### 7. `Dockerfile` -This dockerfile sets up the Docker image that the MLOps run will utilise. - -In the example this is just a simple environment running python version 3.10. -You will most likely need to adapt this for your project. diff --git a/Classifier_2D/mlops/__init__.py b/Classifier_2D/mlops/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/mlops/train.py b/Classifier_2D/mlops/train.py deleted file mode 100644 index a99d5fd..0000000 --- a/Classifier_2D/mlops/train.py +++ /dev/null @@ -1,177 +0,0 @@ -import sys -import configparser -import logging -import os -import multiprocessing -import json - -import mlflow -import pytorch_lightning as pl -import torch -from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping -from torch.cuda import is_available as cuda_available - -from Classifier_2D.src.data_module import DataModule -from Classifier_2D.src.network import Network -from Classifier_2D.src.data_module import label_dict -from Classifier_2D.shared.XNAT_data_import import XNATDataImport - -logger = logging.getLogger(__name__) - -def train(config): - - os.environ["CUDA_VISIBLE_DEVICES"] = config["system"]["cuda_visible_devices"] - - xnat_configuration = {'server': config['xnat']['SERVER'], - 'user': config['xnat']['USER'], - 'password': config['xnat']['PASSWORD'], - 'project': config['xnat']['PROJECT'], - 'verify': config.getboolean('xnat', 'VERIFY')} - - max_workers = 32 - num_workers = ( - max_workers - if max_workers < multiprocessing.cpu_count() - else multiprocessing.cpu_count() - ) - - importer = XNATDataImport( - xnat_configuration = xnat_configuration, - num_workers = num_workers - ) - - # Import raw data - raw_data = importer.import_xnat_data() - - # Download images from XNAT - data = importer.xnat_image_download(raw_data) - - # Set up mflow experiment - with mlflow.start_run(nested=True): - save_best_model = True - - mlflow.pytorch.autolog(log_models=False) - - # initialise network and datamodule - dm = DataModule( - data = data, - dm_batch_size = int(config['params']['dm_batch_size']), - test_fraction = float(config['params']['test_fraction']), - num_workers = num_workers, - random_seed = int(config['params']['random_seed']), - image_size = int(config['params']['image_size']) - ) - - dm.setup() - - n_classes = len(set([x for x in label_dict.values() if x is not None])) - mlflow.log_param('n_classes', n_classes) - - train_class_weights = dm.data_manifest["train"]["class_weights"] - validation_class_weights = dm.data_manifest["validation"]["class_weights"] - - net = Network( - model_name = config['params']['model'], - pretrained = config['params']['pretrained'], - n_classes = n_classes, - dropout = float(config['params']['dropout']), - weighted_loss = config['params']['weighted_loss'], - train_class_weights = train_class_weights, - validation_class_weights = validation_class_weights, - learning_rate = float(config['params']['lr']), - max_lr = float(config['params']['max_lr']), - nw_batch_size = int(config['params']['nw_batch_size']), - label_smoothing = float(config['params']['label_smoothing']), - ) - - # Callbacks - checkpoint_metric = config['params']['checkpoint_metric'] - checkpoint_mode = "min" if checkpoint_metric == "val_loss" else "max" - - # Callbacks - callbacks = [] - callbacks.append(LearningRateMonitor(logging_interval="step")) - checkpoint_callback = ModelCheckpoint( - save_top_k=1, - monitor="val_loss", - mode="min", - dirpath="./checkpoint/", - ) - callbacks.append(checkpoint_callback) - - early_stopping_callback = EarlyStopping( - monitor=checkpoint_metric, - patience=10, - mode=checkpoint_mode, - verbose=True, - ) - callbacks.append(early_stopping_callback) - - # configure trainer - trainer = pl.Trainer( - precision="32" if cuda_available() else "16", - callbacks=callbacks, - devices= 1 if cuda_available() else "auto", - accelerator="gpu" if cuda_available() else "auto", - log_every_n_steps=1, - num_sanity_val_steps=0, - max_epochs=int(config['params']['max_epochs']), - accumulate_grad_batches= int(config['params']['grad_batches']) - ) - - trainer.fit(net, dm) - - if save_best_model: - checkpoint = torch.load(checkpoint_callback.best_model_path) - net.load_state_dict(checkpoint['state_dict']) - file_path = f"model-{config['project']['name']}-{mlflow.active_run().info.run_name}.pt" - script = net.to_torchscript(file_path=file_path) - - checkpoint_info = {} - checkpoint_info["monitored_metric"] = checkpoint_callback.monitor - checkpoint_info["metric_value"] = checkpoint_callback.best_model_score.item() - checkpoint_info["mode"] = checkpoint_callback.mode - checkpoint_info["epoch"] = checkpoint["epoch"] - - with open("checkpoint_info.json", "w") as f: - json.dump(checkpoint_info, f, indent=2) - - mlflow.pytorch.log_model( - script, file_path, extra_files=["checkpoint_info.json"] - ) - - _ = [ - os.remove(fp) for fp in ["checkpoint_info.json"] - ] # remove the temporary files after logging to mlflow - - # Prepare config for mlflow logging - useful_keys = ['system', - 'project', - 'params', - ] - - with open(('config_log.txt'), 'w') as f: - for section in useful_keys: - for key, value in config.items(section): - f.write(f'{key} = {value}\n') - - mlflow.log_artifact('config_log.txt') - - logger.info('Finding best threshold') - best_model_path = checkpoint_callback.best_model_path - best_model = net.load_from_checkpoint(best_model_path) - net.evaluate_best_model(best_model, threshold_tune=True) - - mlflow.log_artifact('val_transform_failures.csv') - mlflow.log_artifact('train_transform_failures.csv') - - logger.info('Training complete') - - -if __name__ == '__main__': - - config_path = 'config/config.cfg' - - config = configparser.ConfigParser() - config.read(config_path) - train(config) \ No newline at end of file diff --git a/Classifier_2D/mlops/tune.py b/Classifier_2D/mlops/tune.py deleted file mode 100644 index 8133fa0..0000000 --- a/Classifier_2D/mlops/tune.py +++ /dev/null @@ -1,198 +0,0 @@ -import configparser -import logging -import multiprocessing -import os - -import mlflow -import pytorch_lightning as pl -from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping -from torch.cuda import is_available as cuda_available - -from Classifier_2D.src.data_module import DataModule -from Classifier_2D.src.data_module import label_dict -from Classifier_2D.src.network import Network -from Classifier_2D.shared.XNAT_data_import import XNATDataImport - -import optuna -logger = logging.getLogger(__name__) - -# Obtain hyperparameters for this trial -def suggest_hyperparameters(trial): - - dropout = trial.suggest_float("dropout", 0.0, 0.3, step=0.1) - lr = trial.suggest_float("lr", 1e-4, 5e-4, log=True) - max_lr = trial.suggest_categorical("max_lr",[5e-4,1e-3]) - model = trial.suggest_categorical("model", ["convnextv2_tiny.fcmae_ft_in22k_in1k","convnextv2_base.fcmae_ft_in22k_in1k"]) - batch_size = trial.suggest_int("batch_size", 4, 16, step=4) - pretrained = trial.suggest_categorical("pretrained", ["TRUE"]) - label_smoothing = trial.suggest_float("label_smoothing", 0.0, 0.1) - grad_batches = trial.suggest_int('grad_batches',1,4) - image_size = trial.suggest_categorical("image_size", [224]) - - params = { - "dropout": dropout, - "lr": lr, - "max_lr": max_lr, - "model": model, - "batch_size": batch_size, - "pretrained": pretrained, - "label_smoothing": label_smoothing, - "grad_batches": grad_batches, - "image_size": image_size - } - return params - -def objective(trial,data,config): - - max_workers = 32 - num_workers = ( - max_workers - if max_workers < multiprocessing.cpu_count() - else multiprocessing.cpu_count() - ) - - best_val_loss = float('Inf') - - # Start a new mlflow run - with mlflow.start_run(nested=True): - - params = suggest_hyperparameters(trial) - mlflow.log_params(params) - - # initialise network and datamodule - dm = DataModule( - data = data, - dm_batch_size = int(config['params']['dm_batch_size']), - test_fraction = float(config['params']['test_fraction']), - num_workers = num_workers, - random_seed = int(config['params']['random_seed']), - image_size = params['image_size'] - ) - - dm.setup() - - n_classes = len(set([x for x in label_dict.values() if x is not None])) - mlflow.log_param('n_classes', n_classes) - - train_class_weights = dm.data_manifest["train"]["class_weights"] - validation_class_weights = dm.data_manifest["validation"]["class_weights"] - - net = Network( - model_name = params['model'], - pretrained = params['pretrained'], - n_classes = n_classes, - dropout = params['dropout'], - weighted_loss = config['params']['weighted_loss'], - train_class_weights = train_class_weights, - validation_class_weights = validation_class_weights, - learning_rate = params['lr'], - max_lr = params['max_lr'], - nw_batch_size = params['batch_size'], - label_smoothing = params['label_smoothing'] - ) - - # Callbacks - callbacks = [] - callbacks.append(LearningRateMonitor(logging_interval="step")) - checkpoint_callback = ModelCheckpoint( - save_top_k=1, - monitor="val_loss", - mode="min", - dirpath= f"./checkpoint_trial_{trial.number}/", - ) - callbacks.append(checkpoint_callback) - - early_stopping_callback = EarlyStopping( - monitor="val_loss", - patience=10, - mode="min", - verbose=True, - ) - callbacks.append(early_stopping_callback) - - # configure trainer - trainer = pl.Trainer( - precision="32" if cuda_available() else "16", - callbacks=callbacks, - devices= 1 if cuda_available() else "auto", - accelerator="gpu" if cuda_available() else "auto", - log_every_n_steps=1, - num_sanity_val_steps=0, - max_epochs=int(config['params']['max_epochs']), - accumulate_grad_batches= params['grad_batches'] - ) - - trainer.fit(net, dm) - - best_model_path = checkpoint_callback.best_model_path - best_model = net.load_from_checkpoint(best_model_path) - net.evaluate_best_model(best_model) - - val_loss = checkpoint_callback.best_model_score.item() - - if val_loss <= best_val_loss: - best_val_loss = val_loss - - return best_val_loss - -def tune(config): - - os.environ["CUDA_VISIBLE_DEVICES"] = config["system"]["cuda_visible_devices"] - - xnat_configuration = {'server': config['xnat']['SERVER'], - 'user': config['xnat']['USER'], - 'password': config['xnat']['PASSWORD'], - 'project': config['xnat']['PROJECT'], - 'verify': config.getboolean('xnat', 'VERIFY')} - - max_workers = 32 - num_workers = ( - max_workers - if max_workers < multiprocessing.cpu_count() - else multiprocessing.cpu_count() - ) - - importer = XNATDataImport( - xnat_configuration = xnat_configuration, - num_workers = num_workers - ) - - # Import raw data - raw_data = importer.import_xnat_data() - - # Download images from XNAT - data = importer.xnat_image_download(raw_data) - - - mlflow.pytorch.autolog(log_models=False) - - # Create optuna study (hyperparameter tuning framework) - study = optuna.create_study(study_name="project-tune", direction="minimize") - study.optimize(lambda trial: objective(trial, data, config), n_trials=50) - - with open(('tune_log.txt'), 'w') as f: - f.write("Study statistics: \n") - f.write(f" Number of finished trials: {len(study.trials)}\n") - f.write("Best trial:\n") - - trial = study.best_trial - - f.write(f" Trial number: {trial.number}\n") - f.write(f" Loss (trial value): {trial.value}\n") - - f.write("Params:\n") - for key, value in trial.params.items(): - f.write(f" {key} = {value}\n") - - mlflow.log_artifact('tune_log.txt') - mlflow.log_artifact('val_transform_failures.csv') - mlflow.log_artifact('train_transform_failures.csv') - -if __name__ == '__main__': - - - config_path = 'config/config.cfg' - - config = configparser.ConfigParser() - config.read(config_path) - tune(config) diff --git a/Classifier_2D/shared/XNAT_data_import.py b/Classifier_2D/shared/XNAT_data_import.py deleted file mode 100644 index 691970b..0000000 --- a/Classifier_2D/shared/XNAT_data_import.py +++ /dev/null @@ -1,83 +0,0 @@ -import logging -from tqdm import tqdm -from typing import List - -from Classifier_2D.src.utils.tools import DataBuilderXNAT -from xnat.mixin import ImageScanData, SubjectData - -from Classifier_2D.src.transforms import load_xnat -from monai.data import Dataset -from torch.utils.data import DataLoader - -logger = logging.getLogger(__name__) - -class XNATDataImport(): - - def __init__(self, xnat_configuration: dict = None, num_workers: int = 4, test_batch: int = 0, - n_month_data_window=9999, run_type: str='train'): - self.xnat_configuration = xnat_configuration - self.num_workers = 4 - self.test_batch = test_batch - self.run_type = run_type - - def import_xnat_data(self): - - if self.run_type == 'train': - actions = [ - (self.fetch_xr, 'image'), - (self.fetch_label, 'label'), - ] # list of tuples defining action functions and their data keys - - data_builder = DataBuilderXNAT(self.xnat_configuration, - actions=actions, - test_batch=self.test_batch, - num_workers=self.num_workers) - - data_builder.fetch_data() - return(data_builder.dataset) - - def xnat_image_download(self, data): - - raw_dataset = Dataset( - data=data, - transform=load_xnat(self.xnat_configuration) - ) - - raw_dataloader = DataLoader(raw_dataset, batch_size=None, - num_workers=self.num_workers,shuffle=False, pin_memory=False) - - extracted_data = [] - for sample in tqdm(raw_dataloader, "Downloading Image Data"): - extracted_data.append(sample) - - return(extracted_data) - - @staticmethod - def fetch_xr(subject_data: SubjectData = None) -> List[ImageScanData]: - """ - Function that identifies and returns the required xnat ImageData object from a xnat SubjectData object - along with the 'key' that it will be used to access it. - """ - - scan_objects = [] - - for exp in subject_data.experiments: - for scan in subject_data.experiments[exp].scans: - if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): - scan_objects.append(subject_data.experiments[exp].scans[scan].uri) - - return scan_objects - - @staticmethod - def fetch_label(subject_data: SubjectData = None): - """ - Function that identifies and returns the label from a SubjectData object - """ - for exp in subject_data.experiments: - for scan in subject_data.experiments[exp].scans: - if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): - try: - full_label = subject_data.experiments[exp].label - return full_label - except Exception as e: - logger.warning(f"Unable to fetch {subject_data.experiments[exp]}'s label due to exception: {e}") \ No newline at end of file diff --git a/Classifier_2D/shared/__init__.py b/Classifier_2D/shared/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/src/Network.py b/Classifier_2D/src/Network.py deleted file mode 100644 index 6025665..0000000 --- a/Classifier_2D/src/Network.py +++ /dev/null @@ -1,226 +0,0 @@ -import logging -import pytorch_lightning -import mlflow -import torch -from abc import ABC -from monai.data import decollate_batch -from monai.transforms import ( - AsDiscrete, - Compose, - Activations, -) -from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay, recall_score -from timm import create_model -from timm.data import Mixup -from torch.nn import CrossEntropyLoss -from torchmetrics import Accuracy, F1Score -from Classifier_2D.src.data_module import label_dict -from torchmetrics.classification import MulticlassAUROC -import numpy as np - -logger = logging.getLogger(__name__) - - -class Network(pytorch_lightning.LightningModule, ABC): - """ - Network object defines the model architecture, inherits from LightningModule. - - https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html - """ - - def __init__(self, - model_name, - pretrained, - n_classes, - dropout, - weighted_loss, - train_class_weights, - validation_class_weights, - learning_rate, - max_lr, - nw_batch_size, - label_smoothing, - **kwargs): - super().__init__() - - self.save_hyperparameters() - self.n_classes = n_classes - self.dropout = dropout - self.report_interval = 5 - self.label_smoothing = label_smoothing - - self.y_pred_trans = Compose([Activations(softmax=True)]) - self.y_trans = Compose([AsDiscrete(to_onehot=self.n_classes)]) - - self.learning_rate = learning_rate - self.max_lr = max_lr - self.batch_size = nw_batch_size - - self.val_acc = Accuracy(task="multiclass", num_classes=self.n_classes, top_k=1) - self.val_f1 = F1Score(task="multiclass", num_classes=self.n_classes, top_k=1) - self.val_auroc = MulticlassAUROC(num_classes=self.n_classes, average='macro', thresholds=None) - - self.targets, self.labels = list( - map(list, zip(*[(target, label) for target, label in label_dict.items() if label is not None]))) - - self.model_name = model_name - self.pretrained = pretrained - self.model = create_model(self.model_name, pretrained=self.pretrained, in_chans=1, - num_classes=self.n_classes, drop_rate=self.dropout) - - self.weighted_loss = weighted_loss - self.train_class_weights = train_class_weights - self.validation_class_weights = validation_class_weights - - self.train_loss_function = CrossEntropyLoss( - weight=torch.tensor(self.train_class_weights) - if self.weighted_loss - else None - ) - - self.validation_loss_function = CrossEntropyLoss( - weight=torch.tensor(self.validation_class_weights) - if self.weighted_loss - else None - ) - - self.mixup_fn = Mixup( - mixup_alpha = 0.2, - cutmix_alpha = 0.2, - prob=0.5, - switch_prob=0.5, - mode='batch', - label_smoothing=0.1, - num_classes=self.n_classes, - ) - - def forward(self, x): - return self.model(x) - - def training_step(self, batch, batch_idx): - """ - Training step - :param batch: - :param batch_idx: - :return: loss - """ - valid_mask = batch['valid'].bool() - if valid_mask.sum() == 0: - return None - - x = batch['image'][valid_mask] - y = batch['label'][valid_mask] - if x.size(0) % 2 == 0: - x, y = self.mixup_fn(x, y) - - y_hat = self(x) - loss = self.train_loss_function(y_hat, y) - self.log('train_loss', loss, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - return loss - - def validation_step(self, batch, batch_idx): - """ - Validation step - :param batch: - :param batch_idx: - :return: loss - """ - valid_mask = batch['valid'].bool() - if valid_mask.sum() == 0: - return None - - x = batch['image'][valid_mask] - y = batch['label'][valid_mask] - y_hat = self(x) - - loss = self.validation_loss_function(y_hat, y) - self.log('val_loss', loss, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - # additional metrics - self.val_acc(y_hat, y) - self.log('val_acc', self.val_acc, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - y_onehot = [self.y_trans(i) for i in decollate_batch(y, detach=True)] - y_pred_act = [self.y_pred_trans(i) for i in decollate_batch(y_hat, detach=True)] - - self.val_f1(y_hat, y) - self.log('val_f1', self.val_f1, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - self.val_auroc(y_hat, y) - self.log('val_auroc', self.val_auroc, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - return {"loss": loss, 'y_onehot': y_onehot, 'y_pred_act': y_pred_act} - - def configure_optimizers(self): - optimizer = torch.optim.AdamW(self.parameters(), lr=self.learning_rate) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, max_lr=self.max_lr, total_steps=self.trainer.estimated_stepping_batches - ) - - return {"optimizer": optimizer, "lr_scheduler": scheduler} - - def evaluate_best_model(self, model, threshold_tune = False, epoch=0): - model.eval() - model.freeze() - - val_loader = self.trainer.datamodule.val_dataloader() - - preds, labels = [], [] - if threshold_tune == False: - for batch in val_loader: - images = batch['image'].to(self.device) - label = batch['label'] - y_hat = model(images) - pred = torch.argmax(y_hat, dim=1).cpu() - - preds.append(pred) - labels.append(label) - - self._create_classification_report(preds, labels) - self._create_confusion_matrix(preds, labels) - else: - for batch in val_loader: - images = batch['image'].to(self.device) - label = batch['label'] - y_hat = model(images) - pred = torch.softmax(y_hat, dim=1)[:, 1].cpu().numpy() - - preds.extend(pred) - labels.extend(label) - - best_thresh = 0.5 - best_sensitivity = 0 - - for t in np.linspace(0,1,100): - t_preds = (np.array(preds) >= t).astype(int) - - sens = recall_score(labels, t_preds) - if sens > best_sensitivity: - best_sensitivity = sens - best_thresh = t - - d_preds = (np.array(preds) >= 0.5).astype(int) - d_sens = recall_score(labels,d_preds) - - threshold_text = f"Default Sensitivity: {d_sens}\nBest Threshold: {best_thresh}\nBest Sensitivity: {best_sensitivity}" - with open("threshold.txt", "w") as f: - f.write(threshold_text) - mlflow.log_artifact('threshold.txt') - preds = (np.array(preds) >= best_thresh).astype(int) - self._create_classification_report(preds, labels) - self._create_confusion_matrix(preds, labels) - - def validation_epoch_end(self, validation_step_outputs) -> None: - if self.current_epoch % self.report_interval == 0: - self._attribute(n_samples_plot=4, step_type='validation') - - def _create_classification_report(self, preds, labels): - val_report = classification_report(labels, preds, labels=self.labels, target_names=self.targets) - mlflow.log_text(val_report, f'classification_report.txt') - - def _create_confusion_matrix(self, preds, labels): - confmat = confusion_matrix(labels, preds, normalize='true') - disp = ConfusionMatrixDisplay(confusion_matrix=confmat) - disp.plot() - mlflow.log_figure(disp.figure_, 'confusion_matrix.jpg') \ No newline at end of file diff --git a/Classifier_2D/src/__init__.py b/Classifier_2D/src/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/src/data_module.py b/Classifier_2D/src/data_module.py deleted file mode 100644 index e5627e0..0000000 --- a/Classifier_2D/src/data_module.py +++ /dev/null @@ -1,152 +0,0 @@ -import logging -from collections import Counter -from typing import List, Optional -import mlflow -import numpy as np -import pytorch_lightning -import torch - -from monai.data import CacheDataset, Dataset -from monai.data import pad_list_data_collate -from monai.transforms import Compose - -from sklearn.model_selection import train_test_split -from torch.cuda import is_available -from torch.utils.data import DataLoader - -from Classifier_2D.src.transforms import normalise, train_augment, output -from Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform - -logger = logging.getLogger(__name__) - -label_dict = { - 'NEGATIVE': 0, - 'POSITIVE': 1, -} - -class DataModule(pytorch_lightning.LightningDataModule): - - def __init__(self, data, dm_batch_size: int = 1, num_workers: int = 16, - test_fraction: float = 0.2, cache_dataset=False, - random_seed: int = 42, image_size: int = 224): - super().__init__() - self.data = data - self.xnat_data_list = None - self.num_workers = num_workers - self.batch_size = dm_batch_size - self.test_fraction = test_fraction - self.cache_dataset = cache_dataset - self.random_seed = random_seed - self.image_size = image_size - - self.torch_device = self.trainer.model.device if self.trainer else torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') - - self.train_transforms = Compose( - normalise(self.image_size) - + train_augment(self.image_size) - + output(self.image_size) - ) - - self.val_transforms = Compose( - normalise(self.image_size) - + output(self.image_size) - ) - - def calculate_class_weights(self, data) -> list: - """Calculate class weights for unbalanced datasets - - :param data: input data - :type data: list - :return: class weights - :rtype: list - """ - labels = self.get_labels(data) - return [ - len(labels) / (len(set(labels)) * labels.count(l)) - for l in sorted(set(labels)) - ] - - def get_labels(self, data) -> list: - """Get list of sample labels - - :param data: input data - :type data: list - :return: list of labels - :rtype: list - """ - return [ - int(sample["action_data"]) - for subject in data - for sample in subject["data"] - if sample["data_label"] == "label" - ] - - def setup(self, stage: Optional[str] = None): - """ - Use the setup method to set up your data and define your Dataset objects - :param stage: - :return: - """ - data = self.data - - self.labels = [int(sample['action_data']) for subject in data for sample in subject['data'] if - sample['data_label'] == 'label'] - - self.train_data, self.validation_data = train_test_split(data, - test_size=self.test_fraction, - stratify=self.labels, - random_state=self.random_seed, - ) - - self.data_manifest = { - 'N_total': len(data), - 'train': self.dataset_stats(self.train_data), - 'validation': self.dataset_stats(self.validation_data), - 'labels': label_dict - } - - mlflow.log_dict(self.data_manifest, "data_manifest.json") - - if self.cache_dataset: - self.val_dataset = CacheDataset(data=self.validation_data, - transform=SafeWrapperTransform(transform = self.val_transforms, - log_file = "val_transform_failures.csv", - image_size = self.image_size), - num_workers=self.num_workers) - self.train_dataset = CacheDataset(data=self.train_data, - transform=SafeWrapperTransform(transform = self.train_transforms, - log_file = "train_transform_failures.csv", - image_size = self.image_size), - num_workers=self.num_workers) - else: - self.val_dataset = Dataset(data=self.validation_data, transform=SafeWrapperTransform(transform = self.val_transforms, - log_file = "val_transform_failures.csv", - image_size = self.image_size) - ) - self.train_dataset = Dataset(data=self.train_data, transform=SafeWrapperTransform(transform = self.train_transforms, - log_file = "train_transform_failures.csv", - image_size = self.image_size) - ) - - def train_dataloader(self): - """ - Define train dataloader - :return: - """ - - return DataLoader(self.train_dataset, batch_size=self.batch_size, - num_workers=self.num_workers, collate_fn=pad_list_data_collate, - pin_memory=is_available()) - - def val_dataloader(self): - """ - Define validation dataloader - :return: - """ - return DataLoader(self.val_dataset, batch_size=1, num_workers=self.num_workers, - pin_memory=is_available(), collate_fn=pad_list_data_collate, ) - - def dataset_stats(self, dataset): - stats = {'n_samples': len(dataset)} - stats["class_weights"] = self.calculate_class_weights(dataset) - return stats \ No newline at end of file diff --git a/Classifier_2D/src/transforms/__init__.py b/Classifier_2D/src/transforms/__init__.py deleted file mode 100644 index b2f9ac4..0000000 --- a/Classifier_2D/src/transforms/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -import torch -from monai.transforms import ( - LoadImage, - SqueezeDimd, - EnsureChannelFirstd, - CropForegroundd, - Resized, - ScaleIntensityd, - CastToTyped, - RandFlipd, - RandZoomd, - RandRotated, - RandAffined, - RandGaussianNoised, - RandGaussianSmoothd, - RandScaleIntensityd, - RandAdjustContrastd, - RandCoarseDropoutd, - ResizeWithPadOrCropd, - ToTensord, - SelectItemsd, - EnsureTyped, - Spacingd, -) - -from Classifier_2D.src.transforms.load_image_XNATd import LoadImageXNATd - -def load_xnat(xnat_configuration: dict): - """ - This transform is used by the DataModule to load images from XNAT - """ - return [ - LoadImageXNATd( - keys=['data'], - xnat_configuration=xnat_configuration, - expected_filetype_ext='.dcm', - image_loader=LoadImage(image_only=True, prune_meta_pattern="^0|^2") - ), - ] - -def normalise(image_size): - """ - This transform list is used to prepare tensors for training or inference - """ - return [ - SqueezeDimd(keys=['image'], dim=2), - EnsureChannelFirstd(keys=['image']), - CropForegroundd(keys=['image'], source_key='image'), - Resized(keys=['image'], size_mode='longest', spatial_size=image_size+20), - #Maybe limit top intensity in case of big spikes? - ScaleIntensityd(keys=["image"], minv=0.0, maxv=255.0), - CastToTyped(keys=["image"], dtype=torch.uint8), - ] - -def train_augment(image_size): - """ - This transform list is used to augment images for training. - Aim here is to improve generalisation. - """ - return [ - RandFlipd(keys=['image'], spatial_axis=0, prob=0.5), - RandZoomd(keys=['image'], prob=0.2, min_zoom=1.05,max_zoom=1.1), - RandRotated(keys=['image'], prob=0.2, range_x=0.4), - RandAffined(keys=['image'], prob=0.2, padding_mode='zeros'), - RandGaussianNoised(keys=['image'], prob=0.1, mean=0.0, std=0.1), - RandGaussianSmoothd(keys=['image'], prob=0.2, sigma_x=(0.5,1.0)), - RandScaleIntensityd(keys=['image'], prob=0.15, factors=(0.75,1.25)), - RandAdjustContrastd(keys=['image'], prob=0.1, gamma=(0.5,2), retain_stats=True, invert_image=True), - RandAdjustContrastd(keys=['image'], prob=0.3, gamma=(0.5,2), retain_stats=True, invert_image=False), - ResizeWithPadOrCropd( - keys=["image"], - spatial_size=(image_size,image_size), - mode='replicate' - ), - RandCoarseDropoutd(keys=['image'], prob=0.5, fill_value=0, holes=8, max_holes=16, spatial_size=(10,10), max_spatial_size=(36,36)), - ] - -def output(image_size): - """ - This transform list is used for final normalisation and feature selection. - """ - return [ - ResizeWithPadOrCropd( - keys=["image"], - spatial_size=(image_size,image_size), - mode='replicate' - ), - ScaleIntensityd(keys=["image"], minv=0.0, maxv=1), - ToTensord(keys=['image', 'label']), - SelectItemsd(keys=['subject_id', 'image', 'label']), - EnsureTyped(keys=['image', 'label'], track_meta=False), - ] \ No newline at end of file diff --git a/Classifier_2D/src/transforms/load_image_XNATd.py b/Classifier_2D/src/transforms/load_image_XNATd.py deleted file mode 100644 index d91eb39..0000000 --- a/Classifier_2D/src/transforms/load_image_XNATd.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -MONAI MapTransform for importing image data from XNAT -""" -import glob -import logging -import os -import tempfile -import time - -import xnat -from monai.config import KeysCollection -from monai.transforms import MapTransform, LoadImage -from monai.transforms import Transform - -logger = logging.getLogger(__name__) - - -class LoadImageXNATd(MapTransform): - """ - MapTransform for importing image data from XNAT - """ - - def __init__(self, keys: KeysCollection, xnat_configuration: dict = None, - image_loader: Transform = LoadImage(), validate_data: bool = False, - expected_filetype_ext: str = '.dcm', - image_series_option: str = 'error', - verbose=False): - super().__init__(keys) - self.image_loader = image_loader - self.xnat_configuration = xnat_configuration - self.expected_filetype = expected_filetype_ext - self.validate_data = validate_data - self.image_series_option = image_series_option - self.verbose = verbose - - def __call__(self, data): - """ - Checks for the requested keys in the input data dictionary. - - If specified key is found then will loop over actions in action list and apply each to the value of the - requested key, if no action is triggered then raise a warning actions arefunctions that return any of projects, - subjects, experiments, scans, or resources XNAT object along with a key to be used in the data dictionary - - Each action function should locate a single image object in XNAT. This image object is then downloaded to a - temporary directory and loaded into memory as the value defined by key set by the actions' data_label. - - If validate_data is true then NO data will be downloaded. In this case the transform will loop over the actions - but will instead return a true/false value for each data sample. This can be used to remove samples where the - data is not present in XNAT. - - :param data: dictionary of data - :return: - - """ - - d = dict(data) - - for key in self.keys: - - if key in data: - - with xnat.connect(server=self.xnat_configuration['server'], - user=self.xnat_configuration['user'], - password=self.xnat_configuration['password'], - verify=self.xnat_configuration['verify'], - loglevel='ERROR', - ) as session: - - "Check data list has no duplicate keys" - if len(set([x['data_label'] for x in d[key]])) != len([x['data_label'] for x in d[key]]): - logger.warning('Multiple images with identical labels found') - raise - - "Download image from XNAT" - for item in d[key]: - data_label = item['data_label'] - if item['data_type'] == 'value': - d[data_label] = item['action_data'] - continue - - attempts = 0 - while attempts < 3: - try: - with tempfile.TemporaryDirectory() as tmpdirname: - session_obj = session.create_object(item['action_data']) - session_obj.download_dir(tmpdirname, verbose=self.verbose) - - images_path = glob.glob(os.path.join(tmpdirname, '**/*' + self.expected_filetype), - recursive=True) - - # image loader needs full path to load single images - # logger.info(f"Downloading images: {images_path}") - if not images_path: - raise Exception - - if len(images_path) == 1: - image = self.image_loader(images_path) - - # image loader needs directory path to load 3D images - else: - "find unique directories in list of image paths" - image_dirs = list( - set(os.path.dirname(image_path) for image_path in images_path)) - if len(image_dirs) > 1: - if self.image_series_option == 'error': - raise ValueError(f'More than one image series found in {images_path}') - elif self.image_series_option == 'keepfirst': - image = self.image_loader(image_dirs[0]) - elif self.image_series_option == 'keeplast': - image = self.image_loader(image_dirs[-1]) - else: - raise ValueError(f"More than one image series found in {images_path} and no option specified") - break - - except Exception as e: - attempts += 1 - time.sleep(1.0) - if attempts == 3: - raise Exception(f'Image loader failed on {item} after 3 retries due to {e}') - - d[data_label] = image - - return d diff --git a/Classifier_2D/src/transforms/safe_wrapper.py b/Classifier_2D/src/transforms/safe_wrapper.py deleted file mode 100644 index 0436d64..0000000 --- a/Classifier_2D/src/transforms/safe_wrapper.py +++ /dev/null @@ -1,47 +0,0 @@ -# Wrapper for Monai Transforms so failures are logged instead of run ending - -import os -import csv -import torch - -class SafeWrapperTransform: - def __init__(self, transform, image_size, log_file="transform_failures.csv"): - self.transform = transform - self.image_size = image_size - self.log_file = log_file - self.logged_ids = set() - - # Create the log file - if not os.path.exists(self.log_file): - with open(self.log_file, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(["subject_id", "data", "error_message"]) - - def subject_already_logged(self, subject_id): - if not os.path.exists(self.log_file): - return False - with open(self.log_file, "r") as f: - return any(subject_id == row.split(",")[0] for row in f.readlines()[1:]) - - def __call__(self, data): - try: - result = self.transform(data) - result["valid"] = True - return result - except Exception as e: - subject_id = data.get("subject_id", "UNKNOWN") - - # Check if already logged - if not self.subject_already_logged(subject_id): - subject_data = data.get("data", "N/A") - with open(self.log_file, "a", newline="") as f: - writer = csv.writer(f) - writer.writerow([subject_id, subject_data, str(e)]) - - # Return dummy image, label, and data - keys =['subject_id'] - result = {k: v for k, v in data.items() if k in keys} - result["image"] = torch.zeros((1, self.image_size, self.image_size), dtype=torch.float32) - result["label"] = torch.tensor(-1) - result["valid"] = False - return result diff --git a/Classifier_2D/src/utils/__init__.py b/Classifier_2D/src/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/src/utils/tools.py b/Classifier_2D/src/utils/tools.py deleted file mode 100644 index 8df0b4e..0000000 --- a/Classifier_2D/src/utils/tools.py +++ /dev/null @@ -1,209 +0,0 @@ -import asyncio -import logging -from concurrent.futures import ThreadPoolExecutor -from itertools import chain - -import mlflow -import pandas as pd -import tqdm -import xnat - -logger = logging.getLogger(__name__) - - -class DataBuilderXNAT: - - def __init__(self, xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1, - num_workers: int = 1, validate_data=True): - self.xnat_configuration = xnat_configuration - self.actions = actions - self.flatten_output = flatten_output - self.test_batch = test_batch - self.missing_data_log = [] - self.num_workers = num_workers - self.validate_data = validate_data - - self.dataset = [] - - def fetch_data(self): - asyncio.run(self.start_async_process()) - - async def start_async_process(self): - with ThreadPoolExecutor(max_workers=self.num_workers) as executor: - # with requests.Session() as session: - with xnat.connect(server=self.xnat_configuration['server'], - user=self.xnat_configuration['user'], - password=self.xnat_configuration['password'], - verify=self.xnat_configuration['verify'], - loglevel='ERROR', - ) as session: - - logger.info(f"Collecting XNAT project: {self.xnat_configuration['project']}") - project = session.projects[self.xnat_configuration["project"]] - - if 0 < self.test_batch < len(project.subjects): - from random import sample - project_subjects = sample(project.subjects[:], self.test_batch) - else: - project_subjects = project.subjects[:] - - loop = asyncio.get_event_loop() - tasks = [ - loop.run_in_executor( - executor, - self.process_subject, - *(project, subject_i) - ) - for subject_i in project_subjects - ] - - responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(tasks))] - - for response in await asyncio.gather(*tasks): - pass - - # remove any items where not all actions returned a value - if self.validate_data: - self.dataset = [item for item in self.dataset if len(item['data']) >= len(self.actions)] - - def process_subject(self, project, subject_i): - subject = project.subjects.data[subject_i.id] - data_sample = {'subject_uri': project.subjects[subject.id].uri, - 'subject_id': project.subjects[subject.id].label, - 'data': []} - - logger.debug(f'checking subject {subject_i}') - if self.actions: - try: - data = [] - for action, data_label in self.actions: - action_data = [] - - # logger.debug(f"Running action: {action.__name__} on {subject.id}") - xnat_obj = action(project.subjects[subject.id]) - - if xnat_obj is None or type(xnat_obj) == list and len(xnat_obj) == 0: - self.missing_data_log.append({'subject_id': subject_i.id, - 'action_data': subject_i.label, - 'failed_action': action}) - logger.debug(f'No data found for {subject_i}: action {action} removing sample') - raise Exception - - elif type(xnat_obj) == list: - for obj in xnat_obj: - if obj.cache_id[0] == 'ResourceCatalog': - action_data.append({'source_action': action.__name__, - 'action_data': obj.uri, - 'data_type': 'xnat_uri', - 'data_label': data_label, - 'resource_files': list(obj.files)}) # retrieve filenames if xnat_obj contains Resource - else: - action_data.append({'source_action': action.__name__, - 'action_data': obj.uri, - 'data_type': 'xnat_uri', - 'data_label': data_label}) - - else: - action_data.append({'source_action': action.__name__, - 'action_data': xnat_obj, - 'data_type': 'value', - 'data_label': data_label}) - - data.append(action_data) - - except Exception as e: - logger.debug(f'No data found for {subject_i}; removing sample. Exception {e}') - pass - - if self.flatten_output: - data = list(chain(*data)) - - data_sample['data'] = data - - self.dataset.append((data_sample)) - - -def xnat_build_dataset(xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1): - """ - ***NOTE THE ASYNC VERSION ABOVE WILL BE MUCH FASTER FOR LARGE DATASETS*** - - Builds a dictionary that describes the XNAT project dataset using XNAT data hierarchy: project/subject/experiment/scan - - structured output returned flattened dataset. Can set structured_output to True to perform custom flattening. - - """ - - with xnat.connect(server=xnat_configuration['server'], - user=xnat_configuration['user'], - password=xnat_configuration['password'], - verify=xnat_configuration['verify'], - loglevel='ERROR', - ) as session: - - logger.info(f"Collecting XNAT project: {xnat_configuration['project']}") - project = session.projects[xnat_configuration["project"]] - - dataset = [] - - if 0 < test_batch < len(project.subjects): - from random import sample - project_subjects = sample(project.subjects[:], test_batch) - else: - project_subjects = project.subjects[:] - - missing_data_log = [] - for subject_i in project_subjects: - subject = project.subjects.data[subject_i.id] - data_sample = {'subject_uri': project.subjects[subject.id].uri, - 'subject_id': project.subjects[subject.id].label, - 'data': []} - - if actions: - try: - data = [] - for action, data_label in actions: - action_data = [] - - # logger.debug(f"Running action: {action.__name__} on {subject.id}") - xnat_obj = action(project.subjects[subject.id]) - - if type(xnat_obj) == list: - if len(xnat_obj) == 0: - missing_data_log.append({'subject_id': subject_i.id, - 'action_data': subject_i.label, - 'failed_action': action}) - logger.debug(f'No data found for {subject_i}: action {action} removing sample') - raise Exception - - for obj in xnat_obj: - action_data.append({'source_action': action.__name__, - 'action_data': obj.uri, - 'data_type': 'xnat_uri', - 'data_label': data_label}) - - elif type(xnat_obj) == str: - action_data.append({'source_action': action.__name__, - 'action_data': xnat_obj, - 'data_type': 'value', - 'data_label': data_label}) - - data.append(action_data) - - except Exception as e: - logger.debug(f'No data found for {subject_i}; removing sample. Exception {e}') - continue - - if flatten_output: - data = list(chain(*data)) - - data_sample['data'] = data - - dataset.append((data_sample)) - - if missing_data_log: - df = pd.DataFrame(missing_data_log) - df.to_csv('missing_data_log.csv') - mlflow.log_artifact('missing_data_log.csv') - mlflow.log_param('N_failed_xnat_samples', len(df)) - - return dataset diff --git a/Classifier_2D/tests/__init__.py b/Classifier_2D/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/tests/app_tests/__init__.py b/Classifier_2D/tests/app_tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Classifier_2D/tests/train_tests/__init__.py b/Classifier_2D/tests/train_tests/__init__.py deleted file mode 100644 index e69de29..0000000 From fbbe5a7d26ed051cc09c943980fce6307b023f3b Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:00:08 +0100 Subject: [PATCH 06/21] Ensure sticking to lowercase snake_case for .py files --- Project/Classifier_2D/src/datamodule.py | 153 ++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 Project/Classifier_2D/src/datamodule.py diff --git a/Project/Classifier_2D/src/datamodule.py b/Project/Classifier_2D/src/datamodule.py new file mode 100644 index 0000000..4ba94e4 --- /dev/null +++ b/Project/Classifier_2D/src/datamodule.py @@ -0,0 +1,153 @@ +import logging +from collections import Counter +from typing import List, Optional +import mlflow +import numpy as np +import pytorch_lightning +import torch + +from monai.data import CacheDataset, Dataset +from monai.data import pad_list_data_collate +from monai.transforms import Compose + +from sklearn.model_selection import train_test_split +from torch.cuda import is_available +from torch.utils.data import DataLoader + + +from Project.Classifier_2D.src.transforms import normalise, train_augment, output +from Project.Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform + +logger = logging.getLogger(__name__) + +label_dict = { + 'NEGATIVE': 0, + 'POSITIVE': 1, +} + +class DataModule(pytorch_lightning.LightningDataModule): + + def __init__(self, data, dm_batch_size: int = 1, num_workers: int = 16, + test_fraction: float = 0.2, cache_dataset=False, + random_seed: int = 42, image_size: int = 224): + super().__init__() + self.data = data + self.xnat_data_list = None + self.num_workers = num_workers + self.batch_size = dm_batch_size + self.test_fraction = test_fraction + self.cache_dataset = cache_dataset + self.random_seed = random_seed + self.image_size = image_size + + self.torch_device = self.trainer.model.device if self.trainer else torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') + + self.train_transforms = Compose( + normalise(self.image_size) + + train_augment(self.image_size) + + output(self.image_size) + ) + + self.val_transforms = Compose( + normalise(self.image_size) + + output(self.image_size) + ) + + def calculate_class_weights(self, data) -> list: + """Calculate class weights for unbalanced datasets + + :param data: input data + :type data: list + :return: class weights + :rtype: list + """ + labels = self.get_labels(data) + return [ + len(labels) / (len(set(labels)) * labels.count(l)) + for l in sorted(set(labels)) + ] + + def get_labels(self, data) -> list: + """Get list of sample labels + + :param data: input data + :type data: list + :return: list of labels + :rtype: list + """ + return [ + int(sample["action_data"]) + for subject in data + for sample in subject["data"] + if sample["data_label"] == "label" + ] + + def setup(self, stage: Optional[str] = None): + """ + Use the setup method to set up your data and define your Dataset objects + :param stage: + :return: + """ + data = self.data + + self.labels = [int(sample['action_data']) for subject in data for sample in subject['data'] if + sample['data_label'] == 'label'] + + self.train_data, self.validation_data = train_test_split(data, + test_size=self.test_fraction, + stratify=self.labels, + random_state=self.random_seed, + ) + + self.data_manifest = { + 'N_total': len(data), + 'train': self.dataset_stats(self.train_data), + 'validation': self.dataset_stats(self.validation_data), + 'labels': label_dict + } + + mlflow.log_dict(self.data_manifest, "data_manifest.json") + + if self.cache_dataset: + self.val_dataset = CacheDataset(data=self.validation_data, + transform=SafeWrapperTransform(transform = self.val_transforms, + log_file = "val_transform_failures.csv", + image_size = self.image_size), + num_workers=self.num_workers) + self.train_dataset = CacheDataset(data=self.train_data, + transform=SafeWrapperTransform(transform = self.train_transforms, + log_file = "train_transform_failures.csv", + image_size = self.image_size), + num_workers=self.num_workers) + else: + self.val_dataset = Dataset(data=self.validation_data, transform=SafeWrapperTransform(transform = self.val_transforms, + log_file = "val_transform_failures.csv", + image_size = self.image_size) + ) + self.train_dataset = Dataset(data=self.train_data, transform=SafeWrapperTransform(transform = self.train_transforms, + log_file = "train_transform_failures.csv", + image_size = self.image_size) + ) + + def train_dataloader(self): + """ + Define train dataloader + :return: + """ + + return DataLoader(self.train_dataset, batch_size=self.batch_size, + num_workers=self.num_workers, collate_fn=pad_list_data_collate, + pin_memory=is_available()) + + def val_dataloader(self): + """ + Define validation dataloader + :return: + """ + return DataLoader(self.val_dataset, batch_size=1, num_workers=self.num_workers, + pin_memory=is_available(), collate_fn=pad_list_data_collate, ) + + def dataset_stats(self, dataset): + stats = {'n_samples': len(dataset)} + stats["class_weights"] = self.calculate_class_weights(dataset) + return stats \ No newline at end of file From d6b62da76ead95b38b8dcfb536d817c30b339231 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:02:48 +0100 Subject: [PATCH 07/21] move data import to src out of tools and rename class --- Project/Classifier_2D/src/data_import_xnat.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 Project/Classifier_2D/src/data_import_xnat.py diff --git a/Project/Classifier_2D/src/data_import_xnat.py b/Project/Classifier_2D/src/data_import_xnat.py new file mode 100644 index 0000000..487cd16 --- /dev/null +++ b/Project/Classifier_2D/src/data_import_xnat.py @@ -0,0 +1,83 @@ +import logging +from tqdm import tqdm +from typing import List + +from Project.Classifier_2D.src.utils.tools import DataBuilderXNAT +from xnat.mixin import ImageScanData, SubjectData + +from Project.Classifier_2D.src.transforms import load_xnat +from monai.data import Dataset +from torch.utils.data import DataLoader + +logger = logging.getLogger(__name__) + +class DataImportXNAT(): + + def __init__(self, xnat_configuration: dict = None, num_workers: int = 4, test_batch: int = 0, + n_month_data_window=9999, run_type: str='train'): + self.xnat_configuration = xnat_configuration + self.num_workers = 4 + self.test_batch = test_batch + self.run_type = run_type + + def import_xnat_data(self): + + if self.run_type == 'train': + actions = [ + (self.fetch_xr, 'image'), + (self.fetch_label, 'label'), + ] # list of tuples defining action functions and their data keys + + data_builder = DataBuilderXNAT(self.xnat_configuration, + actions=actions, + test_batch=self.test_batch, + num_workers=self.num_workers) + + data_builder.fetch_data() + return(data_builder.dataset) + + def xnat_image_download(self, data): + + raw_dataset = Dataset( + data=data, + transform=load_xnat(self.xnat_configuration) + ) + + raw_dataloader = DataLoader(raw_dataset, batch_size=None, + num_workers=self.num_workers,shuffle=False, pin_memory=False) + + extracted_data = [] + for sample in tqdm(raw_dataloader, "Downloading Image Data"): + extracted_data.append(sample) + + return(extracted_data) + + @staticmethod + def fetch_xr(subject_data: SubjectData = None) -> List[ImageScanData]: + """ + Function that identifies and returns the required xnat ImageData object from a xnat SubjectData object + along with the 'key' that it will be used to access it. + """ + + scan_objects = [] + + for exp in subject_data.experiments: + for scan in subject_data.experiments[exp].scans: + if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): + scan_objects.append(subject_data.experiments[exp].scans[scan].uri) + + return scan_objects + + @staticmethod + def fetch_label(subject_data: SubjectData = None): + """ + Function that identifies and returns the label from a SubjectData object + """ + for exp in subject_data.experiments: + for scan in subject_data.experiments[exp].scans: + if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): + try: + full_label = subject_data.experiments[exp].label + return full_label + except Exception as e: + logger.warning(f"Unable to fetch {subject_data.experiments[exp]}'s label due to exception: {e}") \ No newline at end of file From 1826be2e94c6e4941b0b46a1e5366cf2bbe8694f Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:19:28 +0100 Subject: [PATCH 08/21] Casing and train.py imports --- project/app/__init__.py | 0 project/app/app_tests/__init__.py | 0 project/classifier_2d/Dockerfile | 16 ++ project/classifier_2d/README.md | 29 +++ project/classifier_2d/__init__.py | 0 project/classifier_2d/config/__init__.py | 0 project/classifier_2d/config/config.cfg | 34 +++ project/classifier_2d/config/local_config.cfg | 34 +++ project/classifier_2d/mlops/README.md | 60 +++++ project/classifier_2d/mlops/__init__.py | 0 project/classifier_2d/mlops/train.py | 176 ++++++++++++++ project/classifier_2d/mlops/tune.py | 198 +++++++++++++++ project/classifier_2d/scripts/__init__.py | 0 project/classifier_2d/src/__init__.py | 0 project/classifier_2d/src/data_import_xnat.py | 83 +++++++ project/classifier_2d/src/datamodule.py | 153 ++++++++++++ project/classifier_2d/src/network.py | 226 ++++++++++++++++++ .../classifier_2d/src/transforms/__init__.py | 92 +++++++ .../src/transforms/load_image_xnatd.py | 123 ++++++++++ .../src/transforms/safe_wrapper.py | 47 ++++ project/classifier_2d/src/utils/__init__.py | 0 project/classifier_2d/src/utils/tools.py | 209 ++++++++++++++++ project/shared/__init__.py | 0 project/tests/__init__.py | 0 project/tests/train_tests/__init__.py | 0 25 files changed, 1480 insertions(+) create mode 100644 project/app/__init__.py create mode 100644 project/app/app_tests/__init__.py create mode 100644 project/classifier_2d/Dockerfile create mode 100644 project/classifier_2d/README.md create mode 100644 project/classifier_2d/__init__.py create mode 100644 project/classifier_2d/config/__init__.py create mode 100644 project/classifier_2d/config/config.cfg create mode 100644 project/classifier_2d/config/local_config.cfg create mode 100644 project/classifier_2d/mlops/README.md create mode 100644 project/classifier_2d/mlops/__init__.py create mode 100644 project/classifier_2d/mlops/train.py create mode 100644 project/classifier_2d/mlops/tune.py create mode 100644 project/classifier_2d/scripts/__init__.py create mode 100644 project/classifier_2d/src/__init__.py create mode 100644 project/classifier_2d/src/data_import_xnat.py create mode 100644 project/classifier_2d/src/datamodule.py create mode 100644 project/classifier_2d/src/network.py create mode 100644 project/classifier_2d/src/transforms/__init__.py create mode 100644 project/classifier_2d/src/transforms/load_image_xnatd.py create mode 100644 project/classifier_2d/src/transforms/safe_wrapper.py create mode 100644 project/classifier_2d/src/utils/__init__.py create mode 100644 project/classifier_2d/src/utils/tools.py create mode 100644 project/shared/__init__.py create mode 100644 project/tests/__init__.py create mode 100644 project/tests/train_tests/__init__.py diff --git a/project/app/__init__.py b/project/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/app/app_tests/__init__.py b/project/app/app_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/classifier_2d/Dockerfile b/project/classifier_2d/Dockerfile new file mode 100644 index 0000000..2a7e158 --- /dev/null +++ b/project/classifier_2d/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.10 + +WORKDIR /train + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y --no-install-recommends build-essential git rsync software-properties-common ffmpeg libsm6 libxext6 && \ + rm -rf /var/lib/apt/lists/* + +ENV PYTHONPATH="/mlflow/projects/code/:$PYTHONPATH" + +COPY . . + +RUN python -m pip install --upgrade pip && \ + python -m pip install --no-cache-dir -r requirements.txt diff --git a/project/classifier_2d/README.md b/project/classifier_2d/README.md new file mode 100644 index 0000000..d3f076d --- /dev/null +++ b/project/classifier_2d/README.md @@ -0,0 +1,29 @@ +# 2D Classifier + +## Introduction +The template provides a starting point with helper classes and functions to facilitate rapid development and deployment of applications for 2D Classifiers. + +## Layout + +### There are 3 sections: + +#### 1. app +This area is used to work on the deployment of the application and provides helper frameworks. + +#### 2. tests +This area holds unit tests for both app and train. + +#### 3. train +This area is used to work on the training and creation of AI models + +*Further information can be found within each folder.* + + +## Examples + +https://github.com/GSTT-CSC/wrist-fracture-x-ray + +For further information on MLOps please refer to the MLOps repo: + +https://github.com/GSTT-CSC/MLOps + diff --git a/project/classifier_2d/__init__.py b/project/classifier_2d/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/classifier_2d/config/__init__.py b/project/classifier_2d/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/classifier_2d/config/config.cfg b/project/classifier_2d/config/config.cfg new file mode 100644 index 0000000..b847369 --- /dev/null +++ b/project/classifier_2d/config/config.cfg @@ -0,0 +1,34 @@ +[server] +MLFLOW_S3_ENDPOINT_URL = +MLFLOW_TRACKING_URI = +ARTIFACT_PATH = s3://mlflow + +[project] +NAME = example_mlflow_project +;VOLUME_MOUNT = < data path>:/DATA + +[system] +USE_GPU = True +CUDA_VISIBLE_DEVICES = 0 + +[xnat] +USER = +PASSWORD = +PROJECT = example_xnat_project +VERIFY = False +SERVER = + +[params] +RANDOM_SEED = 42 +DM_BATCH_SIZE = +DROPOUT = +LR = +MODEL = +NW_BATCH_SIZE = +TEST_FRACTION = +PRETRAINED = +WEIGHTED_LOSS = +LABEL_SMOOTHING = +MAX_EPOCHS = +GRAD_BATCHES = +IMAGE_SIZE = diff --git a/project/classifier_2d/config/local_config.cfg b/project/classifier_2d/config/local_config.cfg new file mode 100644 index 0000000..0d00679 --- /dev/null +++ b/project/classifier_2d/config/local_config.cfg @@ -0,0 +1,34 @@ +[server] +MLFLOW_S3_ENDPOINT_URL = http://localhost:8002 +MLFLOW_TRACKING_URI = http://localhost:85 +ARTIFACT_PATH = s3://mlflow + +[project] +NAME = example_mlflow_project +;VOLUME_MOUNT = < data path>:/DATA + +[system] +USE_GPU = false +CUDA_VISIBLE_DEVICES = 0 + +[xnat] +USER = +PASSWORD = +PROJECT = example_xnat_project +VERIFY = False +SERVER = http://localhost + +[params] +RANDOM_SEED = 42 +DM_BATCH_SIZE = +DROPOUT = +LR = +MODEL = +NW_BATCH_SIZE = +TEST_FRACTION = +PRETRAINED = +WEIGHTED_LOSS = +LABEL_SMOOTHING = +MAX_EPOCHS = +GRAD_BATCHES = +IMAGE_SIZE = diff --git a/project/classifier_2d/mlops/README.md b/project/classifier_2d/mlops/README.md new file mode 100644 index 0000000..a8304db --- /dev/null +++ b/project/classifier_2d/mlops/README.md @@ -0,0 +1,60 @@ +## Example Workflow: + +#### 1. Adapt XNATDataImport.py for your data +#### 2. Adapt Datamodule.py and Network.py if necessary +#### 3. Perform hyperparameter tuning using tune.py +#### 4. Utilise these hyperparameter values with train.py and threshold tune. +#### 5. Success! Model is stored in mlflow and ready for app development. + +More information can be found below. + +## There are 7 main components for training 2D Classifiers: + +### 1. `config/config.cfg` and `config/local_config.cfg` +The config file contains all the information that is used for configuring the project, experiment, and tracking server. This includes training parameters and XNAT configurations. + +The config file path is also passed as an argument to the MLOps `Experiment` class where the experiment and tracking are configured. + +As there will be differences between local development and running on DGX (for example XNAT configurations), it is highly encouraged to make use of `local_config` when developing locally. + +Note: The values present in the template config files are examples, you can remove any except those in `[server]` and `[project]` which are necessary for MLOps. Outside of these you are encouraged to add and modify the config files as relevant to your project. + +### 2. `src/XNATDataImport.py` +This file is used to define and pull the required data from XNAT. It utilises DataBuilderXNAT to do so as shown in the example, if you require additional or different data from XNAT additional actions can be added. + +If your data is not stored in XNAT this can be replaced by any method that accesses your data. + +### 3. `src/Network.py` +This file is used to define the PyTorch `LightningModule` class. + +This is where you set the Network architecture and flow that you will use for training, validation, and testing. + +Here you can set up which metrics are calculated and at which stage in the flow these occur, along with the model and optimiser. + +The example has numerous metrics and steps that are not always necessary, feel free to delete or add as relevant to your project. + +### 4. `src/DataModule.py` +This file is used to define the PyTorch `LightningDataModule` class. + +This is where you define the data that is used for training, validation, and testing. + +The example involves additional data validation steps that might not be relevant, feel free to delete or add as relevant to your project. + +### 5. `scripts/tune.py` +This file is used to define hyperparameter tuning runs. + +This is where the 'Datamodule' and' Network' are pulled together and uses optuna to perform the trials. +Starting hyperparameters are included near the top of the file. + +### 6. `scripts/train.py` +This file is used to define the training run. + +This is where the `Datamodule` and `Network` are pulled together. + +The example includes callbacks to retrieve the best model parameters, feel free to delete or add as relevant to your project. + +### 7. `Dockerfile` +This dockerfile sets up the Docker image that the MLOps run will utilise. + +In the example this is just a simple environment running python version 3.10. +You will most likely need to adapt this for your project. diff --git a/project/classifier_2d/mlops/__init__.py b/project/classifier_2d/mlops/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/classifier_2d/mlops/train.py b/project/classifier_2d/mlops/train.py new file mode 100644 index 0000000..da42d0f --- /dev/null +++ b/project/classifier_2d/mlops/train.py @@ -0,0 +1,176 @@ +import configparser +import json +import logging +import multiprocessing +import os +import sys + +import mlflow +import pytorch_lightning as pl +import torch +from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor, ModelCheckpoint +from torch.cuda import is_available as cuda_available + +from src.data_import_xnat import DataImportXNAT +from src.datamodule import DataModule, label_dict +from src.network import Network + +logger = logging.getLogger(__name__) + +def train(config): + + os.environ["CUDA_VISIBLE_DEVICES"] = config["system"]["cuda_visible_devices"] + + xnat_configuration = {'server': config['xnat']['SERVER'], + 'user': config['xnat']['USER'], + 'password': config['xnat']['PASSWORD'], + 'project': config['xnat']['PROJECT'], + 'verify': config.getboolean('xnat', 'VERIFY')} + + max_workers = 32 + num_workers = ( + max_workers + if max_workers < multiprocessing.cpu_count() + else multiprocessing.cpu_count() + ) + + importer = DataImportXNAT( + xnat_configuration = xnat_configuration, + num_workers = num_workers + ) + + # Import raw data + raw_data = importer.import_xnat_data() + + # Download images from XNAT + data = importer.xnat_image_download(raw_data) + + # Set up mflow experiment + with mlflow.start_run(nested=True): + save_best_model = True + + mlflow.pytorch.autolog(log_models=False) + + # initialise network and datamodule + dm = DataModule( + data = data, + dm_batch_size = int(config['params']['dm_batch_size']), + test_fraction = float(config['params']['test_fraction']), + num_workers = num_workers, + random_seed = int(config['params']['random_seed']), + image_size = int(config['params']['image_size']) + ) + + dm.setup() + + n_classes = len(set([x for x in label_dict.values() if x is not None])) + mlflow.log_param('n_classes', n_classes) + + train_class_weights = dm.data_manifest["train"]["class_weights"] + validation_class_weights = dm.data_manifest["validation"]["class_weights"] + + net = Network( + model_name = config['params']['model'], + pretrained = config['params']['pretrained'], + n_classes = n_classes, + dropout = float(config['params']['dropout']), + weighted_loss = config['params']['weighted_loss'], + train_class_weights = train_class_weights, + validation_class_weights = validation_class_weights, + learning_rate = float(config['params']['lr']), + max_lr = float(config['params']['max_lr']), + nw_batch_size = int(config['params']['nw_batch_size']), + label_smoothing = float(config['params']['label_smoothing']), + ) + + # Callbacks + checkpoint_metric = config['params']['checkpoint_metric'] + checkpoint_mode = "min" if checkpoint_metric == "val_loss" else "max" + + # Callbacks + callbacks = [] + callbacks.append(LearningRateMonitor(logging_interval="step")) + checkpoint_callback = ModelCheckpoint( + save_top_k=1, + monitor="val_loss", + mode="min", + dirpath="./checkpoint/", + ) + callbacks.append(checkpoint_callback) + + early_stopping_callback = EarlyStopping( + monitor=checkpoint_metric, + patience=10, + mode=checkpoint_mode, + verbose=True, + ) + callbacks.append(early_stopping_callback) + + # configure trainer + trainer = pl.Trainer( + precision="32" if cuda_available() else "16", + callbacks=callbacks, + devices= 1 if cuda_available() else "auto", + accelerator="gpu" if cuda_available() else "auto", + log_every_n_steps=1, + num_sanity_val_steps=0, + max_epochs=int(config['params']['max_epochs']), + accumulate_grad_batches= int(config['params']['grad_batches']) + ) + + trainer.fit(net, dm) + + if save_best_model: + checkpoint = torch.load(checkpoint_callback.best_model_path) + net.load_state_dict(checkpoint['state_dict']) + file_path = f"model-{config['project']['name']}-{mlflow.active_run().info.run_name}.pt" + script = net.to_torchscript(file_path=file_path) + + checkpoint_info = {} + checkpoint_info["monitored_metric"] = checkpoint_callback.monitor + checkpoint_info["metric_value"] = checkpoint_callback.best_model_score.item() + checkpoint_info["mode"] = checkpoint_callback.mode + checkpoint_info["epoch"] = checkpoint["epoch"] + + with open("checkpoint_info.json", "w") as f: + json.dump(checkpoint_info, f, indent=2) + + mlflow.pytorch.log_model( + script, file_path, extra_files=["checkpoint_info.json"] + ) + + _ = [ + os.remove(fp) for fp in ["checkpoint_info.json"] + ] # remove the temporary files after logging to mlflow + + # Prepare config for mlflow logging + useful_keys = ['system', + 'project', + 'params', + ] + + with open(('config_log.txt'), 'w') as f: + for section in useful_keys: + for key, value in config.items(section): + f.write(f'{key} = {value}\n') + + mlflow.log_artifact('config_log.txt') + + logger.info('Finding best threshold') + best_model_path = checkpoint_callback.best_model_path + best_model = net.load_from_checkpoint(best_model_path) + net.evaluate_best_model(best_model, threshold_tune=True) + + mlflow.log_artifact('val_transform_failures.csv') + mlflow.log_artifact('train_transform_failures.csv') + + logger.info('Training complete') + + +if __name__ == '__main__': + + config_path = 'config/config.cfg' + + config = configparser.ConfigParser() + config.read(config_path) + train(config) \ No newline at end of file diff --git a/project/classifier_2d/mlops/tune.py b/project/classifier_2d/mlops/tune.py new file mode 100644 index 0000000..55b3066 --- /dev/null +++ b/project/classifier_2d/mlops/tune.py @@ -0,0 +1,198 @@ +import configparser +import logging +import multiprocessing +import os + +import mlflow +import pytorch_lightning as pl +from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping +from torch.cuda import is_available as cuda_available + +from Project.Classifier_2D.src.data_module import DataModule +from Project.Classifier_2D.src.data_module import label_dict +from Project.Classifier_2D.src.network import Network +from Project.Classifier_2D.src.utils.XNAT_data_import import XNATDataImport + +import optuna +logger = logging.getLogger(__name__) + +# Obtain hyperparameters for this trial +def suggest_hyperparameters(trial): + + dropout = trial.suggest_float("dropout", 0.0, 0.3, step=0.1) + lr = trial.suggest_float("lr", 1e-4, 5e-4, log=True) + max_lr = trial.suggest_categorical("max_lr",[5e-4,1e-3]) + model = trial.suggest_categorical("model", ["convnextv2_tiny.fcmae_ft_in22k_in1k","convnextv2_base.fcmae_ft_in22k_in1k"]) + batch_size = trial.suggest_int("batch_size", 4, 16, step=4) + pretrained = trial.suggest_categorical("pretrained", ["TRUE"]) + label_smoothing = trial.suggest_float("label_smoothing", 0.0, 0.1) + grad_batches = trial.suggest_int('grad_batches',1,4) + image_size = trial.suggest_categorical("image_size", [224]) + + params = { + "dropout": dropout, + "lr": lr, + "max_lr": max_lr, + "model": model, + "batch_size": batch_size, + "pretrained": pretrained, + "label_smoothing": label_smoothing, + "grad_batches": grad_batches, + "image_size": image_size + } + return params + +def objective(trial,data,config): + + max_workers = 32 + num_workers = ( + max_workers + if max_workers < multiprocessing.cpu_count() + else multiprocessing.cpu_count() + ) + + best_val_loss = float('Inf') + + # Start a new mlflow run + with mlflow.start_run(nested=True): + + params = suggest_hyperparameters(trial) + mlflow.log_params(params) + + # initialise network and datamodule + dm = DataModule( + data = data, + dm_batch_size = int(config['params']['dm_batch_size']), + test_fraction = float(config['params']['test_fraction']), + num_workers = num_workers, + random_seed = int(config['params']['random_seed']), + image_size = params['image_size'] + ) + + dm.setup() + + n_classes = len(set([x for x in label_dict.values() if x is not None])) + mlflow.log_param('n_classes', n_classes) + + train_class_weights = dm.data_manifest["train"]["class_weights"] + validation_class_weights = dm.data_manifest["validation"]["class_weights"] + + net = Network( + model_name = params['model'], + pretrained = params['pretrained'], + n_classes = n_classes, + dropout = params['dropout'], + weighted_loss = config['params']['weighted_loss'], + train_class_weights = train_class_weights, + validation_class_weights = validation_class_weights, + learning_rate = params['lr'], + max_lr = params['max_lr'], + nw_batch_size = params['batch_size'], + label_smoothing = params['label_smoothing'] + ) + + # Callbacks + callbacks = [] + callbacks.append(LearningRateMonitor(logging_interval="step")) + checkpoint_callback = ModelCheckpoint( + save_top_k=1, + monitor="val_loss", + mode="min", + dirpath= f"./checkpoint_trial_{trial.number}/", + ) + callbacks.append(checkpoint_callback) + + early_stopping_callback = EarlyStopping( + monitor="val_loss", + patience=10, + mode="min", + verbose=True, + ) + callbacks.append(early_stopping_callback) + + # configure trainer + trainer = pl.Trainer( + precision="32" if cuda_available() else "16", + callbacks=callbacks, + devices= 1 if cuda_available() else "auto", + accelerator="gpu" if cuda_available() else "auto", + log_every_n_steps=1, + num_sanity_val_steps=0, + max_epochs=int(config['params']['max_epochs']), + accumulate_grad_batches= params['grad_batches'] + ) + + trainer.fit(net, dm) + + best_model_path = checkpoint_callback.best_model_path + best_model = net.load_from_checkpoint(best_model_path) + net.evaluate_best_model(best_model) + + val_loss = checkpoint_callback.best_model_score.item() + + if val_loss <= best_val_loss: + best_val_loss = val_loss + + return best_val_loss + +def tune(config): + + os.environ["CUDA_VISIBLE_DEVICES"] = config["system"]["cuda_visible_devices"] + + xnat_configuration = {'server': config['xnat']['SERVER'], + 'user': config['xnat']['USER'], + 'password': config['xnat']['PASSWORD'], + 'project': config['xnat']['PROJECT'], + 'verify': config.getboolean('xnat', 'VERIFY')} + + max_workers = 32 + num_workers = ( + max_workers + if max_workers < multiprocessing.cpu_count() + else multiprocessing.cpu_count() + ) + + importer = XNATDataImport( + xnat_configuration = xnat_configuration, + num_workers = num_workers + ) + + # Import raw data + raw_data = importer.import_xnat_data() + + # Download images from XNAT + data = importer.xnat_image_download(raw_data) + + + mlflow.pytorch.autolog(log_models=False) + + # Create optuna study (hyperparameter tuning framework) + study = optuna.create_study(study_name="project-tune", direction="minimize") + study.optimize(lambda trial: objective(trial, data, config), n_trials=50) + + with open(('tune_log.txt'), 'w') as f: + f.write("Study statistics: \n") + f.write(f" Number of finished trials: {len(study.trials)}\n") + f.write("Best trial:\n") + + trial = study.best_trial + + f.write(f" Trial number: {trial.number}\n") + f.write(f" Loss (trial value): {trial.value}\n") + + f.write("Params:\n") + for key, value in trial.params.items(): + f.write(f" {key} = {value}\n") + + mlflow.log_artifact('tune_log.txt') + mlflow.log_artifact('val_transform_failures.csv') + mlflow.log_artifact('train_transform_failures.csv') + +if __name__ == '__main__': + + + config_path = 'config/config.cfg' + + config = configparser.ConfigParser() + config.read(config_path) + tune(config) diff --git a/project/classifier_2d/scripts/__init__.py b/project/classifier_2d/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/classifier_2d/src/__init__.py b/project/classifier_2d/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/classifier_2d/src/data_import_xnat.py b/project/classifier_2d/src/data_import_xnat.py new file mode 100644 index 0000000..487cd16 --- /dev/null +++ b/project/classifier_2d/src/data_import_xnat.py @@ -0,0 +1,83 @@ +import logging +from tqdm import tqdm +from typing import List + +from Project.Classifier_2D.src.utils.tools import DataBuilderXNAT +from xnat.mixin import ImageScanData, SubjectData + +from Project.Classifier_2D.src.transforms import load_xnat +from monai.data import Dataset +from torch.utils.data import DataLoader + +logger = logging.getLogger(__name__) + +class DataImportXNAT(): + + def __init__(self, xnat_configuration: dict = None, num_workers: int = 4, test_batch: int = 0, + n_month_data_window=9999, run_type: str='train'): + self.xnat_configuration = xnat_configuration + self.num_workers = 4 + self.test_batch = test_batch + self.run_type = run_type + + def import_xnat_data(self): + + if self.run_type == 'train': + actions = [ + (self.fetch_xr, 'image'), + (self.fetch_label, 'label'), + ] # list of tuples defining action functions and their data keys + + data_builder = DataBuilderXNAT(self.xnat_configuration, + actions=actions, + test_batch=self.test_batch, + num_workers=self.num_workers) + + data_builder.fetch_data() + return(data_builder.dataset) + + def xnat_image_download(self, data): + + raw_dataset = Dataset( + data=data, + transform=load_xnat(self.xnat_configuration) + ) + + raw_dataloader = DataLoader(raw_dataset, batch_size=None, + num_workers=self.num_workers,shuffle=False, pin_memory=False) + + extracted_data = [] + for sample in tqdm(raw_dataloader, "Downloading Image Data"): + extracted_data.append(sample) + + return(extracted_data) + + @staticmethod + def fetch_xr(subject_data: SubjectData = None) -> List[ImageScanData]: + """ + Function that identifies and returns the required xnat ImageData object from a xnat SubjectData object + along with the 'key' that it will be used to access it. + """ + + scan_objects = [] + + for exp in subject_data.experiments: + for scan in subject_data.experiments[exp].scans: + if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): + scan_objects.append(subject_data.experiments[exp].scans[scan].uri) + + return scan_objects + + @staticmethod + def fetch_label(subject_data: SubjectData = None): + """ + Function that identifies and returns the label from a SubjectData object + """ + for exp in subject_data.experiments: + for scan in subject_data.experiments[exp].scans: + if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): + try: + full_label = subject_data.experiments[exp].label + return full_label + except Exception as e: + logger.warning(f"Unable to fetch {subject_data.experiments[exp]}'s label due to exception: {e}") \ No newline at end of file diff --git a/project/classifier_2d/src/datamodule.py b/project/classifier_2d/src/datamodule.py new file mode 100644 index 0000000..4ba94e4 --- /dev/null +++ b/project/classifier_2d/src/datamodule.py @@ -0,0 +1,153 @@ +import logging +from collections import Counter +from typing import List, Optional +import mlflow +import numpy as np +import pytorch_lightning +import torch + +from monai.data import CacheDataset, Dataset +from monai.data import pad_list_data_collate +from monai.transforms import Compose + +from sklearn.model_selection import train_test_split +from torch.cuda import is_available +from torch.utils.data import DataLoader + + +from Project.Classifier_2D.src.transforms import normalise, train_augment, output +from Project.Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform + +logger = logging.getLogger(__name__) + +label_dict = { + 'NEGATIVE': 0, + 'POSITIVE': 1, +} + +class DataModule(pytorch_lightning.LightningDataModule): + + def __init__(self, data, dm_batch_size: int = 1, num_workers: int = 16, + test_fraction: float = 0.2, cache_dataset=False, + random_seed: int = 42, image_size: int = 224): + super().__init__() + self.data = data + self.xnat_data_list = None + self.num_workers = num_workers + self.batch_size = dm_batch_size + self.test_fraction = test_fraction + self.cache_dataset = cache_dataset + self.random_seed = random_seed + self.image_size = image_size + + self.torch_device = self.trainer.model.device if self.trainer else torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') + + self.train_transforms = Compose( + normalise(self.image_size) + + train_augment(self.image_size) + + output(self.image_size) + ) + + self.val_transforms = Compose( + normalise(self.image_size) + + output(self.image_size) + ) + + def calculate_class_weights(self, data) -> list: + """Calculate class weights for unbalanced datasets + + :param data: input data + :type data: list + :return: class weights + :rtype: list + """ + labels = self.get_labels(data) + return [ + len(labels) / (len(set(labels)) * labels.count(l)) + for l in sorted(set(labels)) + ] + + def get_labels(self, data) -> list: + """Get list of sample labels + + :param data: input data + :type data: list + :return: list of labels + :rtype: list + """ + return [ + int(sample["action_data"]) + for subject in data + for sample in subject["data"] + if sample["data_label"] == "label" + ] + + def setup(self, stage: Optional[str] = None): + """ + Use the setup method to set up your data and define your Dataset objects + :param stage: + :return: + """ + data = self.data + + self.labels = [int(sample['action_data']) for subject in data for sample in subject['data'] if + sample['data_label'] == 'label'] + + self.train_data, self.validation_data = train_test_split(data, + test_size=self.test_fraction, + stratify=self.labels, + random_state=self.random_seed, + ) + + self.data_manifest = { + 'N_total': len(data), + 'train': self.dataset_stats(self.train_data), + 'validation': self.dataset_stats(self.validation_data), + 'labels': label_dict + } + + mlflow.log_dict(self.data_manifest, "data_manifest.json") + + if self.cache_dataset: + self.val_dataset = CacheDataset(data=self.validation_data, + transform=SafeWrapperTransform(transform = self.val_transforms, + log_file = "val_transform_failures.csv", + image_size = self.image_size), + num_workers=self.num_workers) + self.train_dataset = CacheDataset(data=self.train_data, + transform=SafeWrapperTransform(transform = self.train_transforms, + log_file = "train_transform_failures.csv", + image_size = self.image_size), + num_workers=self.num_workers) + else: + self.val_dataset = Dataset(data=self.validation_data, transform=SafeWrapperTransform(transform = self.val_transforms, + log_file = "val_transform_failures.csv", + image_size = self.image_size) + ) + self.train_dataset = Dataset(data=self.train_data, transform=SafeWrapperTransform(transform = self.train_transforms, + log_file = "train_transform_failures.csv", + image_size = self.image_size) + ) + + def train_dataloader(self): + """ + Define train dataloader + :return: + """ + + return DataLoader(self.train_dataset, batch_size=self.batch_size, + num_workers=self.num_workers, collate_fn=pad_list_data_collate, + pin_memory=is_available()) + + def val_dataloader(self): + """ + Define validation dataloader + :return: + """ + return DataLoader(self.val_dataset, batch_size=1, num_workers=self.num_workers, + pin_memory=is_available(), collate_fn=pad_list_data_collate, ) + + def dataset_stats(self, dataset): + stats = {'n_samples': len(dataset)} + stats["class_weights"] = self.calculate_class_weights(dataset) + return stats \ No newline at end of file diff --git a/project/classifier_2d/src/network.py b/project/classifier_2d/src/network.py new file mode 100644 index 0000000..d264c1a --- /dev/null +++ b/project/classifier_2d/src/network.py @@ -0,0 +1,226 @@ +import logging +import pytorch_lightning +import mlflow +import torch +from abc import ABC +from monai.data import decollate_batch +from monai.transforms import ( + AsDiscrete, + Compose, + Activations, +) +from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay, recall_score +from timm import create_model +from timm.data import Mixup +from torch.nn import CrossEntropyLoss +from torchmetrics import Accuracy, F1Score +from Project.Classifier_2D.src.data_module import label_dict +from torchmetrics.classification import MulticlassAUROC +import numpy as np + +logger = logging.getLogger(__name__) + + +class Network(pytorch_lightning.LightningModule, ABC): + """ + Network object defines the model architecture, inherits from LightningModule. + + https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html + """ + + def __init__(self, + model_name, + pretrained, + n_classes, + dropout, + weighted_loss, + train_class_weights, + validation_class_weights, + learning_rate, + max_lr, + nw_batch_size, + label_smoothing, + **kwargs): + super().__init__() + + self.save_hyperparameters() + self.n_classes = n_classes + self.dropout = dropout + self.report_interval = 5 + self.label_smoothing = label_smoothing + + self.y_pred_trans = Compose([Activations(softmax=True)]) + self.y_trans = Compose([AsDiscrete(to_onehot=self.n_classes)]) + + self.learning_rate = learning_rate + self.max_lr = max_lr + self.batch_size = nw_batch_size + + self.val_acc = Accuracy(task="multiclass", num_classes=self.n_classes, top_k=1) + self.val_f1 = F1Score(task="multiclass", num_classes=self.n_classes, top_k=1) + self.val_auroc = MulticlassAUROC(num_classes=self.n_classes, average='macro', thresholds=None) + + self.targets, self.labels = list( + map(list, zip(*[(target, label) for target, label in label_dict.items() if label is not None]))) + + self.model_name = model_name + self.pretrained = pretrained + self.model = create_model(self.model_name, pretrained=self.pretrained, in_chans=1, + num_classes=self.n_classes, drop_rate=self.dropout) + + self.weighted_loss = weighted_loss + self.train_class_weights = train_class_weights + self.validation_class_weights = validation_class_weights + + self.train_loss_function = CrossEntropyLoss( + weight=torch.tensor(self.train_class_weights) + if self.weighted_loss + else None + ) + + self.validation_loss_function = CrossEntropyLoss( + weight=torch.tensor(self.validation_class_weights) + if self.weighted_loss + else None + ) + + self.mixup_fn = Mixup( + mixup_alpha = 0.2, + cutmix_alpha = 0.2, + prob=0.5, + switch_prob=0.5, + mode='batch', + label_smoothing=0.1, + num_classes=self.n_classes, + ) + + def forward(self, x): + return self.model(x) + + def training_step(self, batch, batch_idx): + """ + Training step + :param batch: + :param batch_idx: + :return: loss + """ + valid_mask = batch['valid'].bool() + if valid_mask.sum() == 0: + return None + + x = batch['image'][valid_mask] + y = batch['label'][valid_mask] + if x.size(0) % 2 == 0: + x, y = self.mixup_fn(x, y) + + y_hat = self(x) + loss = self.train_loss_function(y_hat, y) + self.log('train_loss', loss, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + return loss + + def validation_step(self, batch, batch_idx): + """ + Validation step + :param batch: + :param batch_idx: + :return: loss + """ + valid_mask = batch['valid'].bool() + if valid_mask.sum() == 0: + return None + + x = batch['image'][valid_mask] + y = batch['label'][valid_mask] + y_hat = self(x) + + loss = self.validation_loss_function(y_hat, y) + self.log('val_loss', loss, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + # additional metrics + self.val_acc(y_hat, y) + self.log('val_acc', self.val_acc, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + y_onehot = [self.y_trans(i) for i in decollate_batch(y, detach=True)] + y_pred_act = [self.y_pred_trans(i) for i in decollate_batch(y_hat, detach=True)] + + self.val_f1(y_hat, y) + self.log('val_f1', self.val_f1, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + self.val_auroc(y_hat, y) + self.log('val_auroc', self.val_auroc, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) + + return {"loss": loss, 'y_onehot': y_onehot, 'y_pred_act': y_pred_act} + + def configure_optimizers(self): + optimizer = torch.optim.AdamW(self.parameters(), lr=self.learning_rate) + scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, max_lr=self.max_lr, total_steps=self.trainer.estimated_stepping_batches + ) + + return {"optimizer": optimizer, "lr_scheduler": scheduler} + + def evaluate_best_model(self, model, threshold_tune = False, epoch=0): + model.eval() + model.freeze() + + val_loader = self.trainer.datamodule.val_dataloader() + + preds, labels = [], [] + if threshold_tune == False: + for batch in val_loader: + images = batch['image'].to(self.device) + label = batch['label'] + y_hat = model(images) + pred = torch.argmax(y_hat, dim=1).cpu() + + preds.append(pred) + labels.append(label) + + self._create_classification_report(preds, labels) + self._create_confusion_matrix(preds, labels) + else: + for batch in val_loader: + images = batch['image'].to(self.device) + label = batch['label'] + y_hat = model(images) + pred = torch.softmax(y_hat, dim=1)[:, 1].cpu().numpy() + + preds.extend(pred) + labels.extend(label) + + best_thresh = 0.5 + best_sensitivity = 0 + + for t in np.linspace(0,1,100): + t_preds = (np.array(preds) >= t).astype(int) + + sens = recall_score(labels, t_preds) + if sens > best_sensitivity: + best_sensitivity = sens + best_thresh = t + + d_preds = (np.array(preds) >= 0.5).astype(int) + d_sens = recall_score(labels,d_preds) + + threshold_text = f"Default Sensitivity: {d_sens}\nBest Threshold: {best_thresh}\nBest Sensitivity: {best_sensitivity}" + with open("threshold.txt", "w") as f: + f.write(threshold_text) + mlflow.log_artifact('threshold.txt') + preds = (np.array(preds) >= best_thresh).astype(int) + self._create_classification_report(preds, labels) + self._create_confusion_matrix(preds, labels) + + def validation_epoch_end(self, validation_step_outputs) -> None: + if self.current_epoch % self.report_interval == 0: + self._attribute(n_samples_plot=4, step_type='validation') + + def _create_classification_report(self, preds, labels): + val_report = classification_report(labels, preds, labels=self.labels, target_names=self.targets) + mlflow.log_text(val_report, f'classification_report.txt') + + def _create_confusion_matrix(self, preds, labels): + confmat = confusion_matrix(labels, preds, normalize='true') + disp = ConfusionMatrixDisplay(confusion_matrix=confmat) + disp.plot() + mlflow.log_figure(disp.figure_, 'confusion_matrix.jpg') \ No newline at end of file diff --git a/project/classifier_2d/src/transforms/__init__.py b/project/classifier_2d/src/transforms/__init__.py new file mode 100644 index 0000000..def4dfd --- /dev/null +++ b/project/classifier_2d/src/transforms/__init__.py @@ -0,0 +1,92 @@ +import torch +from monai.transforms import ( + LoadImage, + SqueezeDimd, + EnsureChannelFirstd, + CropForegroundd, + Resized, + ScaleIntensityd, + CastToTyped, + RandFlipd, + RandZoomd, + RandRotated, + RandAffined, + RandGaussianNoised, + RandGaussianSmoothd, + RandScaleIntensityd, + RandAdjustContrastd, + RandCoarseDropoutd, + ResizeWithPadOrCropd, + ToTensord, + SelectItemsd, + EnsureTyped, + Spacingd, +) + +from Project.Classifier_2D.src.transforms.load_image_XNATd import LoadImageXNATd + +def load_xnat(xnat_configuration: dict): + """ + This transform is used by the DataModule to load images from XNAT + """ + return [ + LoadImageXNATd( + keys=['data'], + xnat_configuration=xnat_configuration, + expected_filetype_ext='.dcm', + image_loader=LoadImage(image_only=True, prune_meta_pattern="^0|^2") + ), + ] + +def normalise(image_size): + """ + This transform list is used to prepare tensors for training or inference + """ + return [ + SqueezeDimd(keys=['image'], dim=2), + EnsureChannelFirstd(keys=['image']), + CropForegroundd(keys=['image'], source_key='image'), + Resized(keys=['image'], size_mode='longest', spatial_size=image_size+20), + #Maybe limit top intensity in case of big spikes? + ScaleIntensityd(keys=["image"], minv=0.0, maxv=255.0), + CastToTyped(keys=["image"], dtype=torch.uint8), + ] + +def train_augment(image_size): + """ + This transform list is used to augment images for training. + Aim here is to improve generalisation. + """ + return [ + RandFlipd(keys=['image'], spatial_axis=0, prob=0.5), + RandZoomd(keys=['image'], prob=0.2, min_zoom=1.05,max_zoom=1.1), + RandRotated(keys=['image'], prob=0.2, range_x=0.4), + RandAffined(keys=['image'], prob=0.2, padding_mode='zeros'), + RandGaussianNoised(keys=['image'], prob=0.1, mean=0.0, std=0.1), + RandGaussianSmoothd(keys=['image'], prob=0.2, sigma_x=(0.5,1.0)), + RandScaleIntensityd(keys=['image'], prob=0.15, factors=(0.75,1.25)), + RandAdjustContrastd(keys=['image'], prob=0.1, gamma=(0.5,2), retain_stats=True, invert_image=True), + RandAdjustContrastd(keys=['image'], prob=0.3, gamma=(0.5,2), retain_stats=True, invert_image=False), + ResizeWithPadOrCropd( + keys=["image"], + spatial_size=(image_size,image_size), + mode='replicate' + ), + RandCoarseDropoutd(keys=['image'], prob=0.5, fill_value=0, holes=8, max_holes=16, spatial_size=(10,10), max_spatial_size=(36,36)), + ] + +def output(image_size): + """ + This transform list is used for final normalisation and feature selection. + """ + return [ + ResizeWithPadOrCropd( + keys=["image"], + spatial_size=(image_size,image_size), + mode='replicate' + ), + ScaleIntensityd(keys=["image"], minv=0.0, maxv=1), + ToTensord(keys=['image', 'label']), + SelectItemsd(keys=['subject_id', 'image', 'label']), + EnsureTyped(keys=['image', 'label'], track_meta=False), + ] \ No newline at end of file diff --git a/project/classifier_2d/src/transforms/load_image_xnatd.py b/project/classifier_2d/src/transforms/load_image_xnatd.py new file mode 100644 index 0000000..d91eb39 --- /dev/null +++ b/project/classifier_2d/src/transforms/load_image_xnatd.py @@ -0,0 +1,123 @@ +""" +MONAI MapTransform for importing image data from XNAT +""" +import glob +import logging +import os +import tempfile +import time + +import xnat +from monai.config import KeysCollection +from monai.transforms import MapTransform, LoadImage +from monai.transforms import Transform + +logger = logging.getLogger(__name__) + + +class LoadImageXNATd(MapTransform): + """ + MapTransform for importing image data from XNAT + """ + + def __init__(self, keys: KeysCollection, xnat_configuration: dict = None, + image_loader: Transform = LoadImage(), validate_data: bool = False, + expected_filetype_ext: str = '.dcm', + image_series_option: str = 'error', + verbose=False): + super().__init__(keys) + self.image_loader = image_loader + self.xnat_configuration = xnat_configuration + self.expected_filetype = expected_filetype_ext + self.validate_data = validate_data + self.image_series_option = image_series_option + self.verbose = verbose + + def __call__(self, data): + """ + Checks for the requested keys in the input data dictionary. + + If specified key is found then will loop over actions in action list and apply each to the value of the + requested key, if no action is triggered then raise a warning actions arefunctions that return any of projects, + subjects, experiments, scans, or resources XNAT object along with a key to be used in the data dictionary + + Each action function should locate a single image object in XNAT. This image object is then downloaded to a + temporary directory and loaded into memory as the value defined by key set by the actions' data_label. + + If validate_data is true then NO data will be downloaded. In this case the transform will loop over the actions + but will instead return a true/false value for each data sample. This can be used to remove samples where the + data is not present in XNAT. + + :param data: dictionary of data + :return: + + """ + + d = dict(data) + + for key in self.keys: + + if key in data: + + with xnat.connect(server=self.xnat_configuration['server'], + user=self.xnat_configuration['user'], + password=self.xnat_configuration['password'], + verify=self.xnat_configuration['verify'], + loglevel='ERROR', + ) as session: + + "Check data list has no duplicate keys" + if len(set([x['data_label'] for x in d[key]])) != len([x['data_label'] for x in d[key]]): + logger.warning('Multiple images with identical labels found') + raise + + "Download image from XNAT" + for item in d[key]: + data_label = item['data_label'] + if item['data_type'] == 'value': + d[data_label] = item['action_data'] + continue + + attempts = 0 + while attempts < 3: + try: + with tempfile.TemporaryDirectory() as tmpdirname: + session_obj = session.create_object(item['action_data']) + session_obj.download_dir(tmpdirname, verbose=self.verbose) + + images_path = glob.glob(os.path.join(tmpdirname, '**/*' + self.expected_filetype), + recursive=True) + + # image loader needs full path to load single images + # logger.info(f"Downloading images: {images_path}") + if not images_path: + raise Exception + + if len(images_path) == 1: + image = self.image_loader(images_path) + + # image loader needs directory path to load 3D images + else: + "find unique directories in list of image paths" + image_dirs = list( + set(os.path.dirname(image_path) for image_path in images_path)) + if len(image_dirs) > 1: + if self.image_series_option == 'error': + raise ValueError(f'More than one image series found in {images_path}') + elif self.image_series_option == 'keepfirst': + image = self.image_loader(image_dirs[0]) + elif self.image_series_option == 'keeplast': + image = self.image_loader(image_dirs[-1]) + else: + raise ValueError(f"More than one image series found in {images_path} and no option specified") + break + + except Exception as e: + attempts += 1 + time.sleep(1.0) + if attempts == 3: + raise Exception(f'Image loader failed on {item} after 3 retries due to {e}') + + d[data_label] = image + + return d diff --git a/project/classifier_2d/src/transforms/safe_wrapper.py b/project/classifier_2d/src/transforms/safe_wrapper.py new file mode 100644 index 0000000..0436d64 --- /dev/null +++ b/project/classifier_2d/src/transforms/safe_wrapper.py @@ -0,0 +1,47 @@ +# Wrapper for Monai Transforms so failures are logged instead of run ending + +import os +import csv +import torch + +class SafeWrapperTransform: + def __init__(self, transform, image_size, log_file="transform_failures.csv"): + self.transform = transform + self.image_size = image_size + self.log_file = log_file + self.logged_ids = set() + + # Create the log file + if not os.path.exists(self.log_file): + with open(self.log_file, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["subject_id", "data", "error_message"]) + + def subject_already_logged(self, subject_id): + if not os.path.exists(self.log_file): + return False + with open(self.log_file, "r") as f: + return any(subject_id == row.split(",")[0] for row in f.readlines()[1:]) + + def __call__(self, data): + try: + result = self.transform(data) + result["valid"] = True + return result + except Exception as e: + subject_id = data.get("subject_id", "UNKNOWN") + + # Check if already logged + if not self.subject_already_logged(subject_id): + subject_data = data.get("data", "N/A") + with open(self.log_file, "a", newline="") as f: + writer = csv.writer(f) + writer.writerow([subject_id, subject_data, str(e)]) + + # Return dummy image, label, and data + keys =['subject_id'] + result = {k: v for k, v in data.items() if k in keys} + result["image"] = torch.zeros((1, self.image_size, self.image_size), dtype=torch.float32) + result["label"] = torch.tensor(-1) + result["valid"] = False + return result diff --git a/project/classifier_2d/src/utils/__init__.py b/project/classifier_2d/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/classifier_2d/src/utils/tools.py b/project/classifier_2d/src/utils/tools.py new file mode 100644 index 0000000..8df0b4e --- /dev/null +++ b/project/classifier_2d/src/utils/tools.py @@ -0,0 +1,209 @@ +import asyncio +import logging +from concurrent.futures import ThreadPoolExecutor +from itertools import chain + +import mlflow +import pandas as pd +import tqdm +import xnat + +logger = logging.getLogger(__name__) + + +class DataBuilderXNAT: + + def __init__(self, xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1, + num_workers: int = 1, validate_data=True): + self.xnat_configuration = xnat_configuration + self.actions = actions + self.flatten_output = flatten_output + self.test_batch = test_batch + self.missing_data_log = [] + self.num_workers = num_workers + self.validate_data = validate_data + + self.dataset = [] + + def fetch_data(self): + asyncio.run(self.start_async_process()) + + async def start_async_process(self): + with ThreadPoolExecutor(max_workers=self.num_workers) as executor: + # with requests.Session() as session: + with xnat.connect(server=self.xnat_configuration['server'], + user=self.xnat_configuration['user'], + password=self.xnat_configuration['password'], + verify=self.xnat_configuration['verify'], + loglevel='ERROR', + ) as session: + + logger.info(f"Collecting XNAT project: {self.xnat_configuration['project']}") + project = session.projects[self.xnat_configuration["project"]] + + if 0 < self.test_batch < len(project.subjects): + from random import sample + project_subjects = sample(project.subjects[:], self.test_batch) + else: + project_subjects = project.subjects[:] + + loop = asyncio.get_event_loop() + tasks = [ + loop.run_in_executor( + executor, + self.process_subject, + *(project, subject_i) + ) + for subject_i in project_subjects + ] + + responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(tasks))] + + for response in await asyncio.gather(*tasks): + pass + + # remove any items where not all actions returned a value + if self.validate_data: + self.dataset = [item for item in self.dataset if len(item['data']) >= len(self.actions)] + + def process_subject(self, project, subject_i): + subject = project.subjects.data[subject_i.id] + data_sample = {'subject_uri': project.subjects[subject.id].uri, + 'subject_id': project.subjects[subject.id].label, + 'data': []} + + logger.debug(f'checking subject {subject_i}') + if self.actions: + try: + data = [] + for action, data_label in self.actions: + action_data = [] + + # logger.debug(f"Running action: {action.__name__} on {subject.id}") + xnat_obj = action(project.subjects[subject.id]) + + if xnat_obj is None or type(xnat_obj) == list and len(xnat_obj) == 0: + self.missing_data_log.append({'subject_id': subject_i.id, + 'action_data': subject_i.label, + 'failed_action': action}) + logger.debug(f'No data found for {subject_i}: action {action} removing sample') + raise Exception + + elif type(xnat_obj) == list: + for obj in xnat_obj: + if obj.cache_id[0] == 'ResourceCatalog': + action_data.append({'source_action': action.__name__, + 'action_data': obj.uri, + 'data_type': 'xnat_uri', + 'data_label': data_label, + 'resource_files': list(obj.files)}) # retrieve filenames if xnat_obj contains Resource + else: + action_data.append({'source_action': action.__name__, + 'action_data': obj.uri, + 'data_type': 'xnat_uri', + 'data_label': data_label}) + + else: + action_data.append({'source_action': action.__name__, + 'action_data': xnat_obj, + 'data_type': 'value', + 'data_label': data_label}) + + data.append(action_data) + + except Exception as e: + logger.debug(f'No data found for {subject_i}; removing sample. Exception {e}') + pass + + if self.flatten_output: + data = list(chain(*data)) + + data_sample['data'] = data + + self.dataset.append((data_sample)) + + +def xnat_build_dataset(xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1): + """ + ***NOTE THE ASYNC VERSION ABOVE WILL BE MUCH FASTER FOR LARGE DATASETS*** + + Builds a dictionary that describes the XNAT project dataset using XNAT data hierarchy: project/subject/experiment/scan + + structured output returned flattened dataset. Can set structured_output to True to perform custom flattening. + + """ + + with xnat.connect(server=xnat_configuration['server'], + user=xnat_configuration['user'], + password=xnat_configuration['password'], + verify=xnat_configuration['verify'], + loglevel='ERROR', + ) as session: + + logger.info(f"Collecting XNAT project: {xnat_configuration['project']}") + project = session.projects[xnat_configuration["project"]] + + dataset = [] + + if 0 < test_batch < len(project.subjects): + from random import sample + project_subjects = sample(project.subjects[:], test_batch) + else: + project_subjects = project.subjects[:] + + missing_data_log = [] + for subject_i in project_subjects: + subject = project.subjects.data[subject_i.id] + data_sample = {'subject_uri': project.subjects[subject.id].uri, + 'subject_id': project.subjects[subject.id].label, + 'data': []} + + if actions: + try: + data = [] + for action, data_label in actions: + action_data = [] + + # logger.debug(f"Running action: {action.__name__} on {subject.id}") + xnat_obj = action(project.subjects[subject.id]) + + if type(xnat_obj) == list: + if len(xnat_obj) == 0: + missing_data_log.append({'subject_id': subject_i.id, + 'action_data': subject_i.label, + 'failed_action': action}) + logger.debug(f'No data found for {subject_i}: action {action} removing sample') + raise Exception + + for obj in xnat_obj: + action_data.append({'source_action': action.__name__, + 'action_data': obj.uri, + 'data_type': 'xnat_uri', + 'data_label': data_label}) + + elif type(xnat_obj) == str: + action_data.append({'source_action': action.__name__, + 'action_data': xnat_obj, + 'data_type': 'value', + 'data_label': data_label}) + + data.append(action_data) + + except Exception as e: + logger.debug(f'No data found for {subject_i}; removing sample. Exception {e}') + continue + + if flatten_output: + data = list(chain(*data)) + + data_sample['data'] = data + + dataset.append((data_sample)) + + if missing_data_log: + df = pd.DataFrame(missing_data_log) + df.to_csv('missing_data_log.csv') + mlflow.log_artifact('missing_data_log.csv') + mlflow.log_param('N_failed_xnat_samples', len(df)) + + return dataset diff --git a/project/shared/__init__.py b/project/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/tests/__init__.py b/project/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/tests/train_tests/__init__.py b/project/tests/train_tests/__init__.py new file mode 100644 index 0000000..e69de29 From 25ec7344f86c4d2ce2c81a4d14792239b4eeb8be Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:22:59 +0100 Subject: [PATCH 09/21] tune.py imports --- project/classifier_2d/mlops/tune.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/project/classifier_2d/mlops/tune.py b/project/classifier_2d/mlops/tune.py index 55b3066..b8d7ec7 100644 --- a/project/classifier_2d/mlops/tune.py +++ b/project/classifier_2d/mlops/tune.py @@ -4,16 +4,15 @@ import os import mlflow +import optuna import pytorch_lightning as pl from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping from torch.cuda import is_available as cuda_available -from Project.Classifier_2D.src.data_module import DataModule -from Project.Classifier_2D.src.data_module import label_dict -from Project.Classifier_2D.src.network import Network -from Project.Classifier_2D.src.utils.XNAT_data_import import XNATDataImport +from src.data_import_xnat import DataImportXNAT +from src.datamodule import DataModule, label_dict +from src.network import Network -import optuna logger = logging.getLogger(__name__) # Obtain hyperparameters for this trial @@ -152,7 +151,7 @@ def tune(config): else multiprocessing.cpu_count() ) - importer = XNATDataImport( + importer = DataImportXNAT( xnat_configuration = xnat_configuration, num_workers = num_workers ) From c66ef521ebb0758c6b0f8fe9edc94ffffd33b25b Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:23:18 +0100 Subject: [PATCH 10/21] remove import sys --- project/classifier_2d/mlops/train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/project/classifier_2d/mlops/train.py b/project/classifier_2d/mlops/train.py index da42d0f..3e360ab 100644 --- a/project/classifier_2d/mlops/train.py +++ b/project/classifier_2d/mlops/train.py @@ -3,7 +3,6 @@ import logging import multiprocessing import os -import sys import mlflow import pytorch_lightning as pl From c05ef782684066a3b1822a9a00c0c434ea3ba5db Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:24:51 +0100 Subject: [PATCH 11/21] small tidy --- Project/Classifier_2D/mlops/train.py | 16 +- Project/Classifier_2D/mlops/tune.py | 11 +- Project/Classifier_2D/src/data_module.py | 153 ------------------ .../src/utils/XNAT_data_import.py | 83 ---------- 4 files changed, 12 insertions(+), 251 deletions(-) delete mode 100644 Project/Classifier_2D/src/data_module.py delete mode 100644 Project/Classifier_2D/src/utils/XNAT_data_import.py diff --git a/Project/Classifier_2D/mlops/train.py b/Project/Classifier_2D/mlops/train.py index 2c3fb56..3e360ab 100644 --- a/Project/Classifier_2D/mlops/train.py +++ b/Project/Classifier_2D/mlops/train.py @@ -1,20 +1,18 @@ -import sys import configparser +import json import logging -import os import multiprocessing -import json +import os import mlflow import pytorch_lightning as pl import torch -from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping +from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor, ModelCheckpoint from torch.cuda import is_available as cuda_available -from Project.Classifier_2D.src.data_module import DataModule -from Project.Classifier_2D.src.network import Network -from Project.Classifier_2D.src.data_module import label_dict -from Project.Classifier_2D.src.utils.XNAT_data_import import XNATDataImport +from src.data_import_xnat import DataImportXNAT +from src.datamodule import DataModule, label_dict +from src.network import Network logger = logging.getLogger(__name__) @@ -35,7 +33,7 @@ def train(config): else multiprocessing.cpu_count() ) - importer = XNATDataImport( + importer = DataImportXNAT( xnat_configuration = xnat_configuration, num_workers = num_workers ) diff --git a/Project/Classifier_2D/mlops/tune.py b/Project/Classifier_2D/mlops/tune.py index 55b3066..b8d7ec7 100644 --- a/Project/Classifier_2D/mlops/tune.py +++ b/Project/Classifier_2D/mlops/tune.py @@ -4,16 +4,15 @@ import os import mlflow +import optuna import pytorch_lightning as pl from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping from torch.cuda import is_available as cuda_available -from Project.Classifier_2D.src.data_module import DataModule -from Project.Classifier_2D.src.data_module import label_dict -from Project.Classifier_2D.src.network import Network -from Project.Classifier_2D.src.utils.XNAT_data_import import XNATDataImport +from src.data_import_xnat import DataImportXNAT +from src.datamodule import DataModule, label_dict +from src.network import Network -import optuna logger = logging.getLogger(__name__) # Obtain hyperparameters for this trial @@ -152,7 +151,7 @@ def tune(config): else multiprocessing.cpu_count() ) - importer = XNATDataImport( + importer = DataImportXNAT( xnat_configuration = xnat_configuration, num_workers = num_workers ) diff --git a/Project/Classifier_2D/src/data_module.py b/Project/Classifier_2D/src/data_module.py deleted file mode 100644 index 4ba94e4..0000000 --- a/Project/Classifier_2D/src/data_module.py +++ /dev/null @@ -1,153 +0,0 @@ -import logging -from collections import Counter -from typing import List, Optional -import mlflow -import numpy as np -import pytorch_lightning -import torch - -from monai.data import CacheDataset, Dataset -from monai.data import pad_list_data_collate -from monai.transforms import Compose - -from sklearn.model_selection import train_test_split -from torch.cuda import is_available -from torch.utils.data import DataLoader - - -from Project.Classifier_2D.src.transforms import normalise, train_augment, output -from Project.Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform - -logger = logging.getLogger(__name__) - -label_dict = { - 'NEGATIVE': 0, - 'POSITIVE': 1, -} - -class DataModule(pytorch_lightning.LightningDataModule): - - def __init__(self, data, dm_batch_size: int = 1, num_workers: int = 16, - test_fraction: float = 0.2, cache_dataset=False, - random_seed: int = 42, image_size: int = 224): - super().__init__() - self.data = data - self.xnat_data_list = None - self.num_workers = num_workers - self.batch_size = dm_batch_size - self.test_fraction = test_fraction - self.cache_dataset = cache_dataset - self.random_seed = random_seed - self.image_size = image_size - - self.torch_device = self.trainer.model.device if self.trainer else torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') - - self.train_transforms = Compose( - normalise(self.image_size) - + train_augment(self.image_size) - + output(self.image_size) - ) - - self.val_transforms = Compose( - normalise(self.image_size) - + output(self.image_size) - ) - - def calculate_class_weights(self, data) -> list: - """Calculate class weights for unbalanced datasets - - :param data: input data - :type data: list - :return: class weights - :rtype: list - """ - labels = self.get_labels(data) - return [ - len(labels) / (len(set(labels)) * labels.count(l)) - for l in sorted(set(labels)) - ] - - def get_labels(self, data) -> list: - """Get list of sample labels - - :param data: input data - :type data: list - :return: list of labels - :rtype: list - """ - return [ - int(sample["action_data"]) - for subject in data - for sample in subject["data"] - if sample["data_label"] == "label" - ] - - def setup(self, stage: Optional[str] = None): - """ - Use the setup method to set up your data and define your Dataset objects - :param stage: - :return: - """ - data = self.data - - self.labels = [int(sample['action_data']) for subject in data for sample in subject['data'] if - sample['data_label'] == 'label'] - - self.train_data, self.validation_data = train_test_split(data, - test_size=self.test_fraction, - stratify=self.labels, - random_state=self.random_seed, - ) - - self.data_manifest = { - 'N_total': len(data), - 'train': self.dataset_stats(self.train_data), - 'validation': self.dataset_stats(self.validation_data), - 'labels': label_dict - } - - mlflow.log_dict(self.data_manifest, "data_manifest.json") - - if self.cache_dataset: - self.val_dataset = CacheDataset(data=self.validation_data, - transform=SafeWrapperTransform(transform = self.val_transforms, - log_file = "val_transform_failures.csv", - image_size = self.image_size), - num_workers=self.num_workers) - self.train_dataset = CacheDataset(data=self.train_data, - transform=SafeWrapperTransform(transform = self.train_transforms, - log_file = "train_transform_failures.csv", - image_size = self.image_size), - num_workers=self.num_workers) - else: - self.val_dataset = Dataset(data=self.validation_data, transform=SafeWrapperTransform(transform = self.val_transforms, - log_file = "val_transform_failures.csv", - image_size = self.image_size) - ) - self.train_dataset = Dataset(data=self.train_data, transform=SafeWrapperTransform(transform = self.train_transforms, - log_file = "train_transform_failures.csv", - image_size = self.image_size) - ) - - def train_dataloader(self): - """ - Define train dataloader - :return: - """ - - return DataLoader(self.train_dataset, batch_size=self.batch_size, - num_workers=self.num_workers, collate_fn=pad_list_data_collate, - pin_memory=is_available()) - - def val_dataloader(self): - """ - Define validation dataloader - :return: - """ - return DataLoader(self.val_dataset, batch_size=1, num_workers=self.num_workers, - pin_memory=is_available(), collate_fn=pad_list_data_collate, ) - - def dataset_stats(self, dataset): - stats = {'n_samples': len(dataset)} - stats["class_weights"] = self.calculate_class_weights(dataset) - return stats \ No newline at end of file diff --git a/Project/Classifier_2D/src/utils/XNAT_data_import.py b/Project/Classifier_2D/src/utils/XNAT_data_import.py deleted file mode 100644 index 691e08f..0000000 --- a/Project/Classifier_2D/src/utils/XNAT_data_import.py +++ /dev/null @@ -1,83 +0,0 @@ -import logging -from tqdm import tqdm -from typing import List - -from Project.Classifier_2D.src.utils.tools import DataBuilderXNAT -from xnat.mixin import ImageScanData, SubjectData - -from Project.Classifier_2D.src.transforms import load_xnat -from monai.data import Dataset -from torch.utils.data import DataLoader - -logger = logging.getLogger(__name__) - -class XNATDataImport(): - - def __init__(self, xnat_configuration: dict = None, num_workers: int = 4, test_batch: int = 0, - n_month_data_window=9999, run_type: str='train'): - self.xnat_configuration = xnat_configuration - self.num_workers = 4 - self.test_batch = test_batch - self.run_type = run_type - - def import_xnat_data(self): - - if self.run_type == 'train': - actions = [ - (self.fetch_xr, 'image'), - (self.fetch_label, 'label'), - ] # list of tuples defining action functions and their data keys - - data_builder = DataBuilderXNAT(self.xnat_configuration, - actions=actions, - test_batch=self.test_batch, - num_workers=self.num_workers) - - data_builder.fetch_data() - return(data_builder.dataset) - - def xnat_image_download(self, data): - - raw_dataset = Dataset( - data=data, - transform=load_xnat(self.xnat_configuration) - ) - - raw_dataloader = DataLoader(raw_dataset, batch_size=None, - num_workers=self.num_workers,shuffle=False, pin_memory=False) - - extracted_data = [] - for sample in tqdm(raw_dataloader, "Downloading Image Data"): - extracted_data.append(sample) - - return(extracted_data) - - @staticmethod - def fetch_xr(subject_data: SubjectData = None) -> List[ImageScanData]: - """ - Function that identifies and returns the required xnat ImageData object from a xnat SubjectData object - along with the 'key' that it will be used to access it. - """ - - scan_objects = [] - - for exp in subject_data.experiments: - for scan in subject_data.experiments[exp].scans: - if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): - scan_objects.append(subject_data.experiments[exp].scans[scan].uri) - - return scan_objects - - @staticmethod - def fetch_label(subject_data: SubjectData = None): - """ - Function that identifies and returns the label from a SubjectData object - """ - for exp in subject_data.experiments: - for scan in subject_data.experiments[exp].scans: - if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): - try: - full_label = subject_data.experiments[exp].label - return full_label - except Exception as e: - logger.warning(f"Unable to fetch {subject_data.experiments[exp]}'s label due to exception: {e}") \ No newline at end of file From 4bc2fefc4be7f33969d1de0c84af6c9485886283 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:32:38 +0100 Subject: [PATCH 12/21] load_image_xnatd.py imports + removed cached files --- Project/Classifier_2D/Dockerfile | 16 -- Project/Classifier_2D/README.md | 29 --- Project/Classifier_2D/__init__.py | 0 Project/Classifier_2D/config/__init__.py | 0 Project/Classifier_2D/config/config.cfg | 34 --- Project/Classifier_2D/config/local_config.cfg | 34 --- Project/Classifier_2D/mlops/README.md | 60 ----- Project/Classifier_2D/mlops/__init__.py | 0 Project/Classifier_2D/mlops/train.py | 175 -------------- Project/Classifier_2D/mlops/tune.py | 197 --------------- Project/Classifier_2D/scripts/__init__.py | 0 Project/Classifier_2D/src/__init__.py | 0 Project/Classifier_2D/src/data_import_xnat.py | 83 ------- Project/Classifier_2D/src/datamodule.py | 153 ------------ Project/Classifier_2D/src/network.py | 226 ------------------ .../Classifier_2D/src/transforms/__init__.py | 92 ------- .../src/transforms/load_image_XNATd.py | 123 ---------- .../src/transforms/safe_wrapper.py | 47 ---- Project/Classifier_2D/src/utils/__init__.py | 0 Project/Classifier_2D/src/utils/tools.py | 209 ---------------- Project/app/__init__.py | 0 Project/app/app_tests/__init__.py | 0 Project/shared/__init__.py | 0 Project/tests/__init__.py | 0 Project/tests/train_tests/__init__.py | 0 .../src/transforms/load_image_xnatd.py | 4 +- 26 files changed, 1 insertion(+), 1481 deletions(-) delete mode 100644 Project/Classifier_2D/Dockerfile delete mode 100644 Project/Classifier_2D/README.md delete mode 100644 Project/Classifier_2D/__init__.py delete mode 100644 Project/Classifier_2D/config/__init__.py delete mode 100644 Project/Classifier_2D/config/config.cfg delete mode 100644 Project/Classifier_2D/config/local_config.cfg delete mode 100644 Project/Classifier_2D/mlops/README.md delete mode 100644 Project/Classifier_2D/mlops/__init__.py delete mode 100644 Project/Classifier_2D/mlops/train.py delete mode 100644 Project/Classifier_2D/mlops/tune.py delete mode 100644 Project/Classifier_2D/scripts/__init__.py delete mode 100644 Project/Classifier_2D/src/__init__.py delete mode 100644 Project/Classifier_2D/src/data_import_xnat.py delete mode 100644 Project/Classifier_2D/src/datamodule.py delete mode 100644 Project/Classifier_2D/src/network.py delete mode 100644 Project/Classifier_2D/src/transforms/__init__.py delete mode 100644 Project/Classifier_2D/src/transforms/load_image_XNATd.py delete mode 100644 Project/Classifier_2D/src/transforms/safe_wrapper.py delete mode 100644 Project/Classifier_2D/src/utils/__init__.py delete mode 100644 Project/Classifier_2D/src/utils/tools.py delete mode 100644 Project/app/__init__.py delete mode 100644 Project/app/app_tests/__init__.py delete mode 100644 Project/shared/__init__.py delete mode 100644 Project/tests/__init__.py delete mode 100644 Project/tests/train_tests/__init__.py diff --git a/Project/Classifier_2D/Dockerfile b/Project/Classifier_2D/Dockerfile deleted file mode 100644 index 2a7e158..0000000 --- a/Project/Classifier_2D/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.10 - -WORKDIR /train - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y --no-install-recommends build-essential git rsync software-properties-common ffmpeg libsm6 libxext6 && \ - rm -rf /var/lib/apt/lists/* - -ENV PYTHONPATH="/mlflow/projects/code/:$PYTHONPATH" - -COPY . . - -RUN python -m pip install --upgrade pip && \ - python -m pip install --no-cache-dir -r requirements.txt diff --git a/Project/Classifier_2D/README.md b/Project/Classifier_2D/README.md deleted file mode 100644 index d3f076d..0000000 --- a/Project/Classifier_2D/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# 2D Classifier - -## Introduction -The template provides a starting point with helper classes and functions to facilitate rapid development and deployment of applications for 2D Classifiers. - -## Layout - -### There are 3 sections: - -#### 1. app -This area is used to work on the deployment of the application and provides helper frameworks. - -#### 2. tests -This area holds unit tests for both app and train. - -#### 3. train -This area is used to work on the training and creation of AI models - -*Further information can be found within each folder.* - - -## Examples - -https://github.com/GSTT-CSC/wrist-fracture-x-ray - -For further information on MLOps please refer to the MLOps repo: - -https://github.com/GSTT-CSC/MLOps - diff --git a/Project/Classifier_2D/__init__.py b/Project/Classifier_2D/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/Classifier_2D/config/__init__.py b/Project/Classifier_2D/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/Classifier_2D/config/config.cfg b/Project/Classifier_2D/config/config.cfg deleted file mode 100644 index b847369..0000000 --- a/Project/Classifier_2D/config/config.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[server] -MLFLOW_S3_ENDPOINT_URL = -MLFLOW_TRACKING_URI = -ARTIFACT_PATH = s3://mlflow - -[project] -NAME = example_mlflow_project -;VOLUME_MOUNT = < data path>:/DATA - -[system] -USE_GPU = True -CUDA_VISIBLE_DEVICES = 0 - -[xnat] -USER = -PASSWORD = -PROJECT = example_xnat_project -VERIFY = False -SERVER = - -[params] -RANDOM_SEED = 42 -DM_BATCH_SIZE = -DROPOUT = -LR = -MODEL = -NW_BATCH_SIZE = -TEST_FRACTION = -PRETRAINED = -WEIGHTED_LOSS = -LABEL_SMOOTHING = -MAX_EPOCHS = -GRAD_BATCHES = -IMAGE_SIZE = diff --git a/Project/Classifier_2D/config/local_config.cfg b/Project/Classifier_2D/config/local_config.cfg deleted file mode 100644 index 0d00679..0000000 --- a/Project/Classifier_2D/config/local_config.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[server] -MLFLOW_S3_ENDPOINT_URL = http://localhost:8002 -MLFLOW_TRACKING_URI = http://localhost:85 -ARTIFACT_PATH = s3://mlflow - -[project] -NAME = example_mlflow_project -;VOLUME_MOUNT = < data path>:/DATA - -[system] -USE_GPU = false -CUDA_VISIBLE_DEVICES = 0 - -[xnat] -USER = -PASSWORD = -PROJECT = example_xnat_project -VERIFY = False -SERVER = http://localhost - -[params] -RANDOM_SEED = 42 -DM_BATCH_SIZE = -DROPOUT = -LR = -MODEL = -NW_BATCH_SIZE = -TEST_FRACTION = -PRETRAINED = -WEIGHTED_LOSS = -LABEL_SMOOTHING = -MAX_EPOCHS = -GRAD_BATCHES = -IMAGE_SIZE = diff --git a/Project/Classifier_2D/mlops/README.md b/Project/Classifier_2D/mlops/README.md deleted file mode 100644 index a8304db..0000000 --- a/Project/Classifier_2D/mlops/README.md +++ /dev/null @@ -1,60 +0,0 @@ -## Example Workflow: - -#### 1. Adapt XNATDataImport.py for your data -#### 2. Adapt Datamodule.py and Network.py if necessary -#### 3. Perform hyperparameter tuning using tune.py -#### 4. Utilise these hyperparameter values with train.py and threshold tune. -#### 5. Success! Model is stored in mlflow and ready for app development. - -More information can be found below. - -## There are 7 main components for training 2D Classifiers: - -### 1. `config/config.cfg` and `config/local_config.cfg` -The config file contains all the information that is used for configuring the project, experiment, and tracking server. This includes training parameters and XNAT configurations. - -The config file path is also passed as an argument to the MLOps `Experiment` class where the experiment and tracking are configured. - -As there will be differences between local development and running on DGX (for example XNAT configurations), it is highly encouraged to make use of `local_config` when developing locally. - -Note: The values present in the template config files are examples, you can remove any except those in `[server]` and `[project]` which are necessary for MLOps. Outside of these you are encouraged to add and modify the config files as relevant to your project. - -### 2. `src/XNATDataImport.py` -This file is used to define and pull the required data from XNAT. It utilises DataBuilderXNAT to do so as shown in the example, if you require additional or different data from XNAT additional actions can be added. - -If your data is not stored in XNAT this can be replaced by any method that accesses your data. - -### 3. `src/Network.py` -This file is used to define the PyTorch `LightningModule` class. - -This is where you set the Network architecture and flow that you will use for training, validation, and testing. - -Here you can set up which metrics are calculated and at which stage in the flow these occur, along with the model and optimiser. - -The example has numerous metrics and steps that are not always necessary, feel free to delete or add as relevant to your project. - -### 4. `src/DataModule.py` -This file is used to define the PyTorch `LightningDataModule` class. - -This is where you define the data that is used for training, validation, and testing. - -The example involves additional data validation steps that might not be relevant, feel free to delete or add as relevant to your project. - -### 5. `scripts/tune.py` -This file is used to define hyperparameter tuning runs. - -This is where the 'Datamodule' and' Network' are pulled together and uses optuna to perform the trials. -Starting hyperparameters are included near the top of the file. - -### 6. `scripts/train.py` -This file is used to define the training run. - -This is where the `Datamodule` and `Network` are pulled together. - -The example includes callbacks to retrieve the best model parameters, feel free to delete or add as relevant to your project. - -### 7. `Dockerfile` -This dockerfile sets up the Docker image that the MLOps run will utilise. - -In the example this is just a simple environment running python version 3.10. -You will most likely need to adapt this for your project. diff --git a/Project/Classifier_2D/mlops/__init__.py b/Project/Classifier_2D/mlops/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/Classifier_2D/mlops/train.py b/Project/Classifier_2D/mlops/train.py deleted file mode 100644 index 3e360ab..0000000 --- a/Project/Classifier_2D/mlops/train.py +++ /dev/null @@ -1,175 +0,0 @@ -import configparser -import json -import logging -import multiprocessing -import os - -import mlflow -import pytorch_lightning as pl -import torch -from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor, ModelCheckpoint -from torch.cuda import is_available as cuda_available - -from src.data_import_xnat import DataImportXNAT -from src.datamodule import DataModule, label_dict -from src.network import Network - -logger = logging.getLogger(__name__) - -def train(config): - - os.environ["CUDA_VISIBLE_DEVICES"] = config["system"]["cuda_visible_devices"] - - xnat_configuration = {'server': config['xnat']['SERVER'], - 'user': config['xnat']['USER'], - 'password': config['xnat']['PASSWORD'], - 'project': config['xnat']['PROJECT'], - 'verify': config.getboolean('xnat', 'VERIFY')} - - max_workers = 32 - num_workers = ( - max_workers - if max_workers < multiprocessing.cpu_count() - else multiprocessing.cpu_count() - ) - - importer = DataImportXNAT( - xnat_configuration = xnat_configuration, - num_workers = num_workers - ) - - # Import raw data - raw_data = importer.import_xnat_data() - - # Download images from XNAT - data = importer.xnat_image_download(raw_data) - - # Set up mflow experiment - with mlflow.start_run(nested=True): - save_best_model = True - - mlflow.pytorch.autolog(log_models=False) - - # initialise network and datamodule - dm = DataModule( - data = data, - dm_batch_size = int(config['params']['dm_batch_size']), - test_fraction = float(config['params']['test_fraction']), - num_workers = num_workers, - random_seed = int(config['params']['random_seed']), - image_size = int(config['params']['image_size']) - ) - - dm.setup() - - n_classes = len(set([x for x in label_dict.values() if x is not None])) - mlflow.log_param('n_classes', n_classes) - - train_class_weights = dm.data_manifest["train"]["class_weights"] - validation_class_weights = dm.data_manifest["validation"]["class_weights"] - - net = Network( - model_name = config['params']['model'], - pretrained = config['params']['pretrained'], - n_classes = n_classes, - dropout = float(config['params']['dropout']), - weighted_loss = config['params']['weighted_loss'], - train_class_weights = train_class_weights, - validation_class_weights = validation_class_weights, - learning_rate = float(config['params']['lr']), - max_lr = float(config['params']['max_lr']), - nw_batch_size = int(config['params']['nw_batch_size']), - label_smoothing = float(config['params']['label_smoothing']), - ) - - # Callbacks - checkpoint_metric = config['params']['checkpoint_metric'] - checkpoint_mode = "min" if checkpoint_metric == "val_loss" else "max" - - # Callbacks - callbacks = [] - callbacks.append(LearningRateMonitor(logging_interval="step")) - checkpoint_callback = ModelCheckpoint( - save_top_k=1, - monitor="val_loss", - mode="min", - dirpath="./checkpoint/", - ) - callbacks.append(checkpoint_callback) - - early_stopping_callback = EarlyStopping( - monitor=checkpoint_metric, - patience=10, - mode=checkpoint_mode, - verbose=True, - ) - callbacks.append(early_stopping_callback) - - # configure trainer - trainer = pl.Trainer( - precision="32" if cuda_available() else "16", - callbacks=callbacks, - devices= 1 if cuda_available() else "auto", - accelerator="gpu" if cuda_available() else "auto", - log_every_n_steps=1, - num_sanity_val_steps=0, - max_epochs=int(config['params']['max_epochs']), - accumulate_grad_batches= int(config['params']['grad_batches']) - ) - - trainer.fit(net, dm) - - if save_best_model: - checkpoint = torch.load(checkpoint_callback.best_model_path) - net.load_state_dict(checkpoint['state_dict']) - file_path = f"model-{config['project']['name']}-{mlflow.active_run().info.run_name}.pt" - script = net.to_torchscript(file_path=file_path) - - checkpoint_info = {} - checkpoint_info["monitored_metric"] = checkpoint_callback.monitor - checkpoint_info["metric_value"] = checkpoint_callback.best_model_score.item() - checkpoint_info["mode"] = checkpoint_callback.mode - checkpoint_info["epoch"] = checkpoint["epoch"] - - with open("checkpoint_info.json", "w") as f: - json.dump(checkpoint_info, f, indent=2) - - mlflow.pytorch.log_model( - script, file_path, extra_files=["checkpoint_info.json"] - ) - - _ = [ - os.remove(fp) for fp in ["checkpoint_info.json"] - ] # remove the temporary files after logging to mlflow - - # Prepare config for mlflow logging - useful_keys = ['system', - 'project', - 'params', - ] - - with open(('config_log.txt'), 'w') as f: - for section in useful_keys: - for key, value in config.items(section): - f.write(f'{key} = {value}\n') - - mlflow.log_artifact('config_log.txt') - - logger.info('Finding best threshold') - best_model_path = checkpoint_callback.best_model_path - best_model = net.load_from_checkpoint(best_model_path) - net.evaluate_best_model(best_model, threshold_tune=True) - - mlflow.log_artifact('val_transform_failures.csv') - mlflow.log_artifact('train_transform_failures.csv') - - logger.info('Training complete') - - -if __name__ == '__main__': - - config_path = 'config/config.cfg' - - config = configparser.ConfigParser() - config.read(config_path) - train(config) \ No newline at end of file diff --git a/Project/Classifier_2D/mlops/tune.py b/Project/Classifier_2D/mlops/tune.py deleted file mode 100644 index b8d7ec7..0000000 --- a/Project/Classifier_2D/mlops/tune.py +++ /dev/null @@ -1,197 +0,0 @@ -import configparser -import logging -import multiprocessing -import os - -import mlflow -import optuna -import pytorch_lightning as pl -from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint, EarlyStopping -from torch.cuda import is_available as cuda_available - -from src.data_import_xnat import DataImportXNAT -from src.datamodule import DataModule, label_dict -from src.network import Network - -logger = logging.getLogger(__name__) - -# Obtain hyperparameters for this trial -def suggest_hyperparameters(trial): - - dropout = trial.suggest_float("dropout", 0.0, 0.3, step=0.1) - lr = trial.suggest_float("lr", 1e-4, 5e-4, log=True) - max_lr = trial.suggest_categorical("max_lr",[5e-4,1e-3]) - model = trial.suggest_categorical("model", ["convnextv2_tiny.fcmae_ft_in22k_in1k","convnextv2_base.fcmae_ft_in22k_in1k"]) - batch_size = trial.suggest_int("batch_size", 4, 16, step=4) - pretrained = trial.suggest_categorical("pretrained", ["TRUE"]) - label_smoothing = trial.suggest_float("label_smoothing", 0.0, 0.1) - grad_batches = trial.suggest_int('grad_batches',1,4) - image_size = trial.suggest_categorical("image_size", [224]) - - params = { - "dropout": dropout, - "lr": lr, - "max_lr": max_lr, - "model": model, - "batch_size": batch_size, - "pretrained": pretrained, - "label_smoothing": label_smoothing, - "grad_batches": grad_batches, - "image_size": image_size - } - return params - -def objective(trial,data,config): - - max_workers = 32 - num_workers = ( - max_workers - if max_workers < multiprocessing.cpu_count() - else multiprocessing.cpu_count() - ) - - best_val_loss = float('Inf') - - # Start a new mlflow run - with mlflow.start_run(nested=True): - - params = suggest_hyperparameters(trial) - mlflow.log_params(params) - - # initialise network and datamodule - dm = DataModule( - data = data, - dm_batch_size = int(config['params']['dm_batch_size']), - test_fraction = float(config['params']['test_fraction']), - num_workers = num_workers, - random_seed = int(config['params']['random_seed']), - image_size = params['image_size'] - ) - - dm.setup() - - n_classes = len(set([x for x in label_dict.values() if x is not None])) - mlflow.log_param('n_classes', n_classes) - - train_class_weights = dm.data_manifest["train"]["class_weights"] - validation_class_weights = dm.data_manifest["validation"]["class_weights"] - - net = Network( - model_name = params['model'], - pretrained = params['pretrained'], - n_classes = n_classes, - dropout = params['dropout'], - weighted_loss = config['params']['weighted_loss'], - train_class_weights = train_class_weights, - validation_class_weights = validation_class_weights, - learning_rate = params['lr'], - max_lr = params['max_lr'], - nw_batch_size = params['batch_size'], - label_smoothing = params['label_smoothing'] - ) - - # Callbacks - callbacks = [] - callbacks.append(LearningRateMonitor(logging_interval="step")) - checkpoint_callback = ModelCheckpoint( - save_top_k=1, - monitor="val_loss", - mode="min", - dirpath= f"./checkpoint_trial_{trial.number}/", - ) - callbacks.append(checkpoint_callback) - - early_stopping_callback = EarlyStopping( - monitor="val_loss", - patience=10, - mode="min", - verbose=True, - ) - callbacks.append(early_stopping_callback) - - # configure trainer - trainer = pl.Trainer( - precision="32" if cuda_available() else "16", - callbacks=callbacks, - devices= 1 if cuda_available() else "auto", - accelerator="gpu" if cuda_available() else "auto", - log_every_n_steps=1, - num_sanity_val_steps=0, - max_epochs=int(config['params']['max_epochs']), - accumulate_grad_batches= params['grad_batches'] - ) - - trainer.fit(net, dm) - - best_model_path = checkpoint_callback.best_model_path - best_model = net.load_from_checkpoint(best_model_path) - net.evaluate_best_model(best_model) - - val_loss = checkpoint_callback.best_model_score.item() - - if val_loss <= best_val_loss: - best_val_loss = val_loss - - return best_val_loss - -def tune(config): - - os.environ["CUDA_VISIBLE_DEVICES"] = config["system"]["cuda_visible_devices"] - - xnat_configuration = {'server': config['xnat']['SERVER'], - 'user': config['xnat']['USER'], - 'password': config['xnat']['PASSWORD'], - 'project': config['xnat']['PROJECT'], - 'verify': config.getboolean('xnat', 'VERIFY')} - - max_workers = 32 - num_workers = ( - max_workers - if max_workers < multiprocessing.cpu_count() - else multiprocessing.cpu_count() - ) - - importer = DataImportXNAT( - xnat_configuration = xnat_configuration, - num_workers = num_workers - ) - - # Import raw data - raw_data = importer.import_xnat_data() - - # Download images from XNAT - data = importer.xnat_image_download(raw_data) - - - mlflow.pytorch.autolog(log_models=False) - - # Create optuna study (hyperparameter tuning framework) - study = optuna.create_study(study_name="project-tune", direction="minimize") - study.optimize(lambda trial: objective(trial, data, config), n_trials=50) - - with open(('tune_log.txt'), 'w') as f: - f.write("Study statistics: \n") - f.write(f" Number of finished trials: {len(study.trials)}\n") - f.write("Best trial:\n") - - trial = study.best_trial - - f.write(f" Trial number: {trial.number}\n") - f.write(f" Loss (trial value): {trial.value}\n") - - f.write("Params:\n") - for key, value in trial.params.items(): - f.write(f" {key} = {value}\n") - - mlflow.log_artifact('tune_log.txt') - mlflow.log_artifact('val_transform_failures.csv') - mlflow.log_artifact('train_transform_failures.csv') - -if __name__ == '__main__': - - - config_path = 'config/config.cfg' - - config = configparser.ConfigParser() - config.read(config_path) - tune(config) diff --git a/Project/Classifier_2D/scripts/__init__.py b/Project/Classifier_2D/scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/Classifier_2D/src/__init__.py b/Project/Classifier_2D/src/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/Classifier_2D/src/data_import_xnat.py b/Project/Classifier_2D/src/data_import_xnat.py deleted file mode 100644 index 487cd16..0000000 --- a/Project/Classifier_2D/src/data_import_xnat.py +++ /dev/null @@ -1,83 +0,0 @@ -import logging -from tqdm import tqdm -from typing import List - -from Project.Classifier_2D.src.utils.tools import DataBuilderXNAT -from xnat.mixin import ImageScanData, SubjectData - -from Project.Classifier_2D.src.transforms import load_xnat -from monai.data import Dataset -from torch.utils.data import DataLoader - -logger = logging.getLogger(__name__) - -class DataImportXNAT(): - - def __init__(self, xnat_configuration: dict = None, num_workers: int = 4, test_batch: int = 0, - n_month_data_window=9999, run_type: str='train'): - self.xnat_configuration = xnat_configuration - self.num_workers = 4 - self.test_batch = test_batch - self.run_type = run_type - - def import_xnat_data(self): - - if self.run_type == 'train': - actions = [ - (self.fetch_xr, 'image'), - (self.fetch_label, 'label'), - ] # list of tuples defining action functions and their data keys - - data_builder = DataBuilderXNAT(self.xnat_configuration, - actions=actions, - test_batch=self.test_batch, - num_workers=self.num_workers) - - data_builder.fetch_data() - return(data_builder.dataset) - - def xnat_image_download(self, data): - - raw_dataset = Dataset( - data=data, - transform=load_xnat(self.xnat_configuration) - ) - - raw_dataloader = DataLoader(raw_dataset, batch_size=None, - num_workers=self.num_workers,shuffle=False, pin_memory=False) - - extracted_data = [] - for sample in tqdm(raw_dataloader, "Downloading Image Data"): - extracted_data.append(sample) - - return(extracted_data) - - @staticmethod - def fetch_xr(subject_data: SubjectData = None) -> List[ImageScanData]: - """ - Function that identifies and returns the required xnat ImageData object from a xnat SubjectData object - along with the 'key' that it will be used to access it. - """ - - scan_objects = [] - - for exp in subject_data.experiments: - for scan in subject_data.experiments[exp].scans: - if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): - scan_objects.append(subject_data.experiments[exp].scans[scan].uri) - - return scan_objects - - @staticmethod - def fetch_label(subject_data: SubjectData = None): - """ - Function that identifies and returns the label from a SubjectData object - """ - for exp in subject_data.experiments: - for scan in subject_data.experiments[exp].scans: - if 'cr' in subject_data.experiments[exp].scans[scan].modality.lower() or 'dx' in subject_data.experiments[exp].scans[scan].modality.lower(): - try: - full_label = subject_data.experiments[exp].label - return full_label - except Exception as e: - logger.warning(f"Unable to fetch {subject_data.experiments[exp]}'s label due to exception: {e}") \ No newline at end of file diff --git a/Project/Classifier_2D/src/datamodule.py b/Project/Classifier_2D/src/datamodule.py deleted file mode 100644 index 4ba94e4..0000000 --- a/Project/Classifier_2D/src/datamodule.py +++ /dev/null @@ -1,153 +0,0 @@ -import logging -from collections import Counter -from typing import List, Optional -import mlflow -import numpy as np -import pytorch_lightning -import torch - -from monai.data import CacheDataset, Dataset -from monai.data import pad_list_data_collate -from monai.transforms import Compose - -from sklearn.model_selection import train_test_split -from torch.cuda import is_available -from torch.utils.data import DataLoader - - -from Project.Classifier_2D.src.transforms import normalise, train_augment, output -from Project.Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform - -logger = logging.getLogger(__name__) - -label_dict = { - 'NEGATIVE': 0, - 'POSITIVE': 1, -} - -class DataModule(pytorch_lightning.LightningDataModule): - - def __init__(self, data, dm_batch_size: int = 1, num_workers: int = 16, - test_fraction: float = 0.2, cache_dataset=False, - random_seed: int = 42, image_size: int = 224): - super().__init__() - self.data = data - self.xnat_data_list = None - self.num_workers = num_workers - self.batch_size = dm_batch_size - self.test_fraction = test_fraction - self.cache_dataset = cache_dataset - self.random_seed = random_seed - self.image_size = image_size - - self.torch_device = self.trainer.model.device if self.trainer else torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') - - self.train_transforms = Compose( - normalise(self.image_size) - + train_augment(self.image_size) - + output(self.image_size) - ) - - self.val_transforms = Compose( - normalise(self.image_size) - + output(self.image_size) - ) - - def calculate_class_weights(self, data) -> list: - """Calculate class weights for unbalanced datasets - - :param data: input data - :type data: list - :return: class weights - :rtype: list - """ - labels = self.get_labels(data) - return [ - len(labels) / (len(set(labels)) * labels.count(l)) - for l in sorted(set(labels)) - ] - - def get_labels(self, data) -> list: - """Get list of sample labels - - :param data: input data - :type data: list - :return: list of labels - :rtype: list - """ - return [ - int(sample["action_data"]) - for subject in data - for sample in subject["data"] - if sample["data_label"] == "label" - ] - - def setup(self, stage: Optional[str] = None): - """ - Use the setup method to set up your data and define your Dataset objects - :param stage: - :return: - """ - data = self.data - - self.labels = [int(sample['action_data']) for subject in data for sample in subject['data'] if - sample['data_label'] == 'label'] - - self.train_data, self.validation_data = train_test_split(data, - test_size=self.test_fraction, - stratify=self.labels, - random_state=self.random_seed, - ) - - self.data_manifest = { - 'N_total': len(data), - 'train': self.dataset_stats(self.train_data), - 'validation': self.dataset_stats(self.validation_data), - 'labels': label_dict - } - - mlflow.log_dict(self.data_manifest, "data_manifest.json") - - if self.cache_dataset: - self.val_dataset = CacheDataset(data=self.validation_data, - transform=SafeWrapperTransform(transform = self.val_transforms, - log_file = "val_transform_failures.csv", - image_size = self.image_size), - num_workers=self.num_workers) - self.train_dataset = CacheDataset(data=self.train_data, - transform=SafeWrapperTransform(transform = self.train_transforms, - log_file = "train_transform_failures.csv", - image_size = self.image_size), - num_workers=self.num_workers) - else: - self.val_dataset = Dataset(data=self.validation_data, transform=SafeWrapperTransform(transform = self.val_transforms, - log_file = "val_transform_failures.csv", - image_size = self.image_size) - ) - self.train_dataset = Dataset(data=self.train_data, transform=SafeWrapperTransform(transform = self.train_transforms, - log_file = "train_transform_failures.csv", - image_size = self.image_size) - ) - - def train_dataloader(self): - """ - Define train dataloader - :return: - """ - - return DataLoader(self.train_dataset, batch_size=self.batch_size, - num_workers=self.num_workers, collate_fn=pad_list_data_collate, - pin_memory=is_available()) - - def val_dataloader(self): - """ - Define validation dataloader - :return: - """ - return DataLoader(self.val_dataset, batch_size=1, num_workers=self.num_workers, - pin_memory=is_available(), collate_fn=pad_list_data_collate, ) - - def dataset_stats(self, dataset): - stats = {'n_samples': len(dataset)} - stats["class_weights"] = self.calculate_class_weights(dataset) - return stats \ No newline at end of file diff --git a/Project/Classifier_2D/src/network.py b/Project/Classifier_2D/src/network.py deleted file mode 100644 index d264c1a..0000000 --- a/Project/Classifier_2D/src/network.py +++ /dev/null @@ -1,226 +0,0 @@ -import logging -import pytorch_lightning -import mlflow -import torch -from abc import ABC -from monai.data import decollate_batch -from monai.transforms import ( - AsDiscrete, - Compose, - Activations, -) -from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay, recall_score -from timm import create_model -from timm.data import Mixup -from torch.nn import CrossEntropyLoss -from torchmetrics import Accuracy, F1Score -from Project.Classifier_2D.src.data_module import label_dict -from torchmetrics.classification import MulticlassAUROC -import numpy as np - -logger = logging.getLogger(__name__) - - -class Network(pytorch_lightning.LightningModule, ABC): - """ - Network object defines the model architecture, inherits from LightningModule. - - https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html - """ - - def __init__(self, - model_name, - pretrained, - n_classes, - dropout, - weighted_loss, - train_class_weights, - validation_class_weights, - learning_rate, - max_lr, - nw_batch_size, - label_smoothing, - **kwargs): - super().__init__() - - self.save_hyperparameters() - self.n_classes = n_classes - self.dropout = dropout - self.report_interval = 5 - self.label_smoothing = label_smoothing - - self.y_pred_trans = Compose([Activations(softmax=True)]) - self.y_trans = Compose([AsDiscrete(to_onehot=self.n_classes)]) - - self.learning_rate = learning_rate - self.max_lr = max_lr - self.batch_size = nw_batch_size - - self.val_acc = Accuracy(task="multiclass", num_classes=self.n_classes, top_k=1) - self.val_f1 = F1Score(task="multiclass", num_classes=self.n_classes, top_k=1) - self.val_auroc = MulticlassAUROC(num_classes=self.n_classes, average='macro', thresholds=None) - - self.targets, self.labels = list( - map(list, zip(*[(target, label) for target, label in label_dict.items() if label is not None]))) - - self.model_name = model_name - self.pretrained = pretrained - self.model = create_model(self.model_name, pretrained=self.pretrained, in_chans=1, - num_classes=self.n_classes, drop_rate=self.dropout) - - self.weighted_loss = weighted_loss - self.train_class_weights = train_class_weights - self.validation_class_weights = validation_class_weights - - self.train_loss_function = CrossEntropyLoss( - weight=torch.tensor(self.train_class_weights) - if self.weighted_loss - else None - ) - - self.validation_loss_function = CrossEntropyLoss( - weight=torch.tensor(self.validation_class_weights) - if self.weighted_loss - else None - ) - - self.mixup_fn = Mixup( - mixup_alpha = 0.2, - cutmix_alpha = 0.2, - prob=0.5, - switch_prob=0.5, - mode='batch', - label_smoothing=0.1, - num_classes=self.n_classes, - ) - - def forward(self, x): - return self.model(x) - - def training_step(self, batch, batch_idx): - """ - Training step - :param batch: - :param batch_idx: - :return: loss - """ - valid_mask = batch['valid'].bool() - if valid_mask.sum() == 0: - return None - - x = batch['image'][valid_mask] - y = batch['label'][valid_mask] - if x.size(0) % 2 == 0: - x, y = self.mixup_fn(x, y) - - y_hat = self(x) - loss = self.train_loss_function(y_hat, y) - self.log('train_loss', loss, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - return loss - - def validation_step(self, batch, batch_idx): - """ - Validation step - :param batch: - :param batch_idx: - :return: loss - """ - valid_mask = batch['valid'].bool() - if valid_mask.sum() == 0: - return None - - x = batch['image'][valid_mask] - y = batch['label'][valid_mask] - y_hat = self(x) - - loss = self.validation_loss_function(y_hat, y) - self.log('val_loss', loss, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - # additional metrics - self.val_acc(y_hat, y) - self.log('val_acc', self.val_acc, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - y_onehot = [self.y_trans(i) for i in decollate_batch(y, detach=True)] - y_pred_act = [self.y_pred_trans(i) for i in decollate_batch(y_hat, detach=True)] - - self.val_f1(y_hat, y) - self.log('val_f1', self.val_f1, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - self.val_auroc(y_hat, y) - self.log('val_auroc', self.val_auroc, on_step=False, on_epoch=True, batch_size=self.batch_size, sync_dist=True) - - return {"loss": loss, 'y_onehot': y_onehot, 'y_pred_act': y_pred_act} - - def configure_optimizers(self): - optimizer = torch.optim.AdamW(self.parameters(), lr=self.learning_rate) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, max_lr=self.max_lr, total_steps=self.trainer.estimated_stepping_batches - ) - - return {"optimizer": optimizer, "lr_scheduler": scheduler} - - def evaluate_best_model(self, model, threshold_tune = False, epoch=0): - model.eval() - model.freeze() - - val_loader = self.trainer.datamodule.val_dataloader() - - preds, labels = [], [] - if threshold_tune == False: - for batch in val_loader: - images = batch['image'].to(self.device) - label = batch['label'] - y_hat = model(images) - pred = torch.argmax(y_hat, dim=1).cpu() - - preds.append(pred) - labels.append(label) - - self._create_classification_report(preds, labels) - self._create_confusion_matrix(preds, labels) - else: - for batch in val_loader: - images = batch['image'].to(self.device) - label = batch['label'] - y_hat = model(images) - pred = torch.softmax(y_hat, dim=1)[:, 1].cpu().numpy() - - preds.extend(pred) - labels.extend(label) - - best_thresh = 0.5 - best_sensitivity = 0 - - for t in np.linspace(0,1,100): - t_preds = (np.array(preds) >= t).astype(int) - - sens = recall_score(labels, t_preds) - if sens > best_sensitivity: - best_sensitivity = sens - best_thresh = t - - d_preds = (np.array(preds) >= 0.5).astype(int) - d_sens = recall_score(labels,d_preds) - - threshold_text = f"Default Sensitivity: {d_sens}\nBest Threshold: {best_thresh}\nBest Sensitivity: {best_sensitivity}" - with open("threshold.txt", "w") as f: - f.write(threshold_text) - mlflow.log_artifact('threshold.txt') - preds = (np.array(preds) >= best_thresh).astype(int) - self._create_classification_report(preds, labels) - self._create_confusion_matrix(preds, labels) - - def validation_epoch_end(self, validation_step_outputs) -> None: - if self.current_epoch % self.report_interval == 0: - self._attribute(n_samples_plot=4, step_type='validation') - - def _create_classification_report(self, preds, labels): - val_report = classification_report(labels, preds, labels=self.labels, target_names=self.targets) - mlflow.log_text(val_report, f'classification_report.txt') - - def _create_confusion_matrix(self, preds, labels): - confmat = confusion_matrix(labels, preds, normalize='true') - disp = ConfusionMatrixDisplay(confusion_matrix=confmat) - disp.plot() - mlflow.log_figure(disp.figure_, 'confusion_matrix.jpg') \ No newline at end of file diff --git a/Project/Classifier_2D/src/transforms/__init__.py b/Project/Classifier_2D/src/transforms/__init__.py deleted file mode 100644 index def4dfd..0000000 --- a/Project/Classifier_2D/src/transforms/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -import torch -from monai.transforms import ( - LoadImage, - SqueezeDimd, - EnsureChannelFirstd, - CropForegroundd, - Resized, - ScaleIntensityd, - CastToTyped, - RandFlipd, - RandZoomd, - RandRotated, - RandAffined, - RandGaussianNoised, - RandGaussianSmoothd, - RandScaleIntensityd, - RandAdjustContrastd, - RandCoarseDropoutd, - ResizeWithPadOrCropd, - ToTensord, - SelectItemsd, - EnsureTyped, - Spacingd, -) - -from Project.Classifier_2D.src.transforms.load_image_XNATd import LoadImageXNATd - -def load_xnat(xnat_configuration: dict): - """ - This transform is used by the DataModule to load images from XNAT - """ - return [ - LoadImageXNATd( - keys=['data'], - xnat_configuration=xnat_configuration, - expected_filetype_ext='.dcm', - image_loader=LoadImage(image_only=True, prune_meta_pattern="^0|^2") - ), - ] - -def normalise(image_size): - """ - This transform list is used to prepare tensors for training or inference - """ - return [ - SqueezeDimd(keys=['image'], dim=2), - EnsureChannelFirstd(keys=['image']), - CropForegroundd(keys=['image'], source_key='image'), - Resized(keys=['image'], size_mode='longest', spatial_size=image_size+20), - #Maybe limit top intensity in case of big spikes? - ScaleIntensityd(keys=["image"], minv=0.0, maxv=255.0), - CastToTyped(keys=["image"], dtype=torch.uint8), - ] - -def train_augment(image_size): - """ - This transform list is used to augment images for training. - Aim here is to improve generalisation. - """ - return [ - RandFlipd(keys=['image'], spatial_axis=0, prob=0.5), - RandZoomd(keys=['image'], prob=0.2, min_zoom=1.05,max_zoom=1.1), - RandRotated(keys=['image'], prob=0.2, range_x=0.4), - RandAffined(keys=['image'], prob=0.2, padding_mode='zeros'), - RandGaussianNoised(keys=['image'], prob=0.1, mean=0.0, std=0.1), - RandGaussianSmoothd(keys=['image'], prob=0.2, sigma_x=(0.5,1.0)), - RandScaleIntensityd(keys=['image'], prob=0.15, factors=(0.75,1.25)), - RandAdjustContrastd(keys=['image'], prob=0.1, gamma=(0.5,2), retain_stats=True, invert_image=True), - RandAdjustContrastd(keys=['image'], prob=0.3, gamma=(0.5,2), retain_stats=True, invert_image=False), - ResizeWithPadOrCropd( - keys=["image"], - spatial_size=(image_size,image_size), - mode='replicate' - ), - RandCoarseDropoutd(keys=['image'], prob=0.5, fill_value=0, holes=8, max_holes=16, spatial_size=(10,10), max_spatial_size=(36,36)), - ] - -def output(image_size): - """ - This transform list is used for final normalisation and feature selection. - """ - return [ - ResizeWithPadOrCropd( - keys=["image"], - spatial_size=(image_size,image_size), - mode='replicate' - ), - ScaleIntensityd(keys=["image"], minv=0.0, maxv=1), - ToTensord(keys=['image', 'label']), - SelectItemsd(keys=['subject_id', 'image', 'label']), - EnsureTyped(keys=['image', 'label'], track_meta=False), - ] \ No newline at end of file diff --git a/Project/Classifier_2D/src/transforms/load_image_XNATd.py b/Project/Classifier_2D/src/transforms/load_image_XNATd.py deleted file mode 100644 index d91eb39..0000000 --- a/Project/Classifier_2D/src/transforms/load_image_XNATd.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -MONAI MapTransform for importing image data from XNAT -""" -import glob -import logging -import os -import tempfile -import time - -import xnat -from monai.config import KeysCollection -from monai.transforms import MapTransform, LoadImage -from monai.transforms import Transform - -logger = logging.getLogger(__name__) - - -class LoadImageXNATd(MapTransform): - """ - MapTransform for importing image data from XNAT - """ - - def __init__(self, keys: KeysCollection, xnat_configuration: dict = None, - image_loader: Transform = LoadImage(), validate_data: bool = False, - expected_filetype_ext: str = '.dcm', - image_series_option: str = 'error', - verbose=False): - super().__init__(keys) - self.image_loader = image_loader - self.xnat_configuration = xnat_configuration - self.expected_filetype = expected_filetype_ext - self.validate_data = validate_data - self.image_series_option = image_series_option - self.verbose = verbose - - def __call__(self, data): - """ - Checks for the requested keys in the input data dictionary. - - If specified key is found then will loop over actions in action list and apply each to the value of the - requested key, if no action is triggered then raise a warning actions arefunctions that return any of projects, - subjects, experiments, scans, or resources XNAT object along with a key to be used in the data dictionary - - Each action function should locate a single image object in XNAT. This image object is then downloaded to a - temporary directory and loaded into memory as the value defined by key set by the actions' data_label. - - If validate_data is true then NO data will be downloaded. In this case the transform will loop over the actions - but will instead return a true/false value for each data sample. This can be used to remove samples where the - data is not present in XNAT. - - :param data: dictionary of data - :return: - - """ - - d = dict(data) - - for key in self.keys: - - if key in data: - - with xnat.connect(server=self.xnat_configuration['server'], - user=self.xnat_configuration['user'], - password=self.xnat_configuration['password'], - verify=self.xnat_configuration['verify'], - loglevel='ERROR', - ) as session: - - "Check data list has no duplicate keys" - if len(set([x['data_label'] for x in d[key]])) != len([x['data_label'] for x in d[key]]): - logger.warning('Multiple images with identical labels found') - raise - - "Download image from XNAT" - for item in d[key]: - data_label = item['data_label'] - if item['data_type'] == 'value': - d[data_label] = item['action_data'] - continue - - attempts = 0 - while attempts < 3: - try: - with tempfile.TemporaryDirectory() as tmpdirname: - session_obj = session.create_object(item['action_data']) - session_obj.download_dir(tmpdirname, verbose=self.verbose) - - images_path = glob.glob(os.path.join(tmpdirname, '**/*' + self.expected_filetype), - recursive=True) - - # image loader needs full path to load single images - # logger.info(f"Downloading images: {images_path}") - if not images_path: - raise Exception - - if len(images_path) == 1: - image = self.image_loader(images_path) - - # image loader needs directory path to load 3D images - else: - "find unique directories in list of image paths" - image_dirs = list( - set(os.path.dirname(image_path) for image_path in images_path)) - if len(image_dirs) > 1: - if self.image_series_option == 'error': - raise ValueError(f'More than one image series found in {images_path}') - elif self.image_series_option == 'keepfirst': - image = self.image_loader(image_dirs[0]) - elif self.image_series_option == 'keeplast': - image = self.image_loader(image_dirs[-1]) - else: - raise ValueError(f"More than one image series found in {images_path} and no option specified") - break - - except Exception as e: - attempts += 1 - time.sleep(1.0) - if attempts == 3: - raise Exception(f'Image loader failed on {item} after 3 retries due to {e}') - - d[data_label] = image - - return d diff --git a/Project/Classifier_2D/src/transforms/safe_wrapper.py b/Project/Classifier_2D/src/transforms/safe_wrapper.py deleted file mode 100644 index 0436d64..0000000 --- a/Project/Classifier_2D/src/transforms/safe_wrapper.py +++ /dev/null @@ -1,47 +0,0 @@ -# Wrapper for Monai Transforms so failures are logged instead of run ending - -import os -import csv -import torch - -class SafeWrapperTransform: - def __init__(self, transform, image_size, log_file="transform_failures.csv"): - self.transform = transform - self.image_size = image_size - self.log_file = log_file - self.logged_ids = set() - - # Create the log file - if not os.path.exists(self.log_file): - with open(self.log_file, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(["subject_id", "data", "error_message"]) - - def subject_already_logged(self, subject_id): - if not os.path.exists(self.log_file): - return False - with open(self.log_file, "r") as f: - return any(subject_id == row.split(",")[0] for row in f.readlines()[1:]) - - def __call__(self, data): - try: - result = self.transform(data) - result["valid"] = True - return result - except Exception as e: - subject_id = data.get("subject_id", "UNKNOWN") - - # Check if already logged - if not self.subject_already_logged(subject_id): - subject_data = data.get("data", "N/A") - with open(self.log_file, "a", newline="") as f: - writer = csv.writer(f) - writer.writerow([subject_id, subject_data, str(e)]) - - # Return dummy image, label, and data - keys =['subject_id'] - result = {k: v for k, v in data.items() if k in keys} - result["image"] = torch.zeros((1, self.image_size, self.image_size), dtype=torch.float32) - result["label"] = torch.tensor(-1) - result["valid"] = False - return result diff --git a/Project/Classifier_2D/src/utils/__init__.py b/Project/Classifier_2D/src/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/Classifier_2D/src/utils/tools.py b/Project/Classifier_2D/src/utils/tools.py deleted file mode 100644 index 8df0b4e..0000000 --- a/Project/Classifier_2D/src/utils/tools.py +++ /dev/null @@ -1,209 +0,0 @@ -import asyncio -import logging -from concurrent.futures import ThreadPoolExecutor -from itertools import chain - -import mlflow -import pandas as pd -import tqdm -import xnat - -logger = logging.getLogger(__name__) - - -class DataBuilderXNAT: - - def __init__(self, xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1, - num_workers: int = 1, validate_data=True): - self.xnat_configuration = xnat_configuration - self.actions = actions - self.flatten_output = flatten_output - self.test_batch = test_batch - self.missing_data_log = [] - self.num_workers = num_workers - self.validate_data = validate_data - - self.dataset = [] - - def fetch_data(self): - asyncio.run(self.start_async_process()) - - async def start_async_process(self): - with ThreadPoolExecutor(max_workers=self.num_workers) as executor: - # with requests.Session() as session: - with xnat.connect(server=self.xnat_configuration['server'], - user=self.xnat_configuration['user'], - password=self.xnat_configuration['password'], - verify=self.xnat_configuration['verify'], - loglevel='ERROR', - ) as session: - - logger.info(f"Collecting XNAT project: {self.xnat_configuration['project']}") - project = session.projects[self.xnat_configuration["project"]] - - if 0 < self.test_batch < len(project.subjects): - from random import sample - project_subjects = sample(project.subjects[:], self.test_batch) - else: - project_subjects = project.subjects[:] - - loop = asyncio.get_event_loop() - tasks = [ - loop.run_in_executor( - executor, - self.process_subject, - *(project, subject_i) - ) - for subject_i in project_subjects - ] - - responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(tasks))] - - for response in await asyncio.gather(*tasks): - pass - - # remove any items where not all actions returned a value - if self.validate_data: - self.dataset = [item for item in self.dataset if len(item['data']) >= len(self.actions)] - - def process_subject(self, project, subject_i): - subject = project.subjects.data[subject_i.id] - data_sample = {'subject_uri': project.subjects[subject.id].uri, - 'subject_id': project.subjects[subject.id].label, - 'data': []} - - logger.debug(f'checking subject {subject_i}') - if self.actions: - try: - data = [] - for action, data_label in self.actions: - action_data = [] - - # logger.debug(f"Running action: {action.__name__} on {subject.id}") - xnat_obj = action(project.subjects[subject.id]) - - if xnat_obj is None or type(xnat_obj) == list and len(xnat_obj) == 0: - self.missing_data_log.append({'subject_id': subject_i.id, - 'action_data': subject_i.label, - 'failed_action': action}) - logger.debug(f'No data found for {subject_i}: action {action} removing sample') - raise Exception - - elif type(xnat_obj) == list: - for obj in xnat_obj: - if obj.cache_id[0] == 'ResourceCatalog': - action_data.append({'source_action': action.__name__, - 'action_data': obj.uri, - 'data_type': 'xnat_uri', - 'data_label': data_label, - 'resource_files': list(obj.files)}) # retrieve filenames if xnat_obj contains Resource - else: - action_data.append({'source_action': action.__name__, - 'action_data': obj.uri, - 'data_type': 'xnat_uri', - 'data_label': data_label}) - - else: - action_data.append({'source_action': action.__name__, - 'action_data': xnat_obj, - 'data_type': 'value', - 'data_label': data_label}) - - data.append(action_data) - - except Exception as e: - logger.debug(f'No data found for {subject_i}; removing sample. Exception {e}') - pass - - if self.flatten_output: - data = list(chain(*data)) - - data_sample['data'] = data - - self.dataset.append((data_sample)) - - -def xnat_build_dataset(xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1): - """ - ***NOTE THE ASYNC VERSION ABOVE WILL BE MUCH FASTER FOR LARGE DATASETS*** - - Builds a dictionary that describes the XNAT project dataset using XNAT data hierarchy: project/subject/experiment/scan - - structured output returned flattened dataset. Can set structured_output to True to perform custom flattening. - - """ - - with xnat.connect(server=xnat_configuration['server'], - user=xnat_configuration['user'], - password=xnat_configuration['password'], - verify=xnat_configuration['verify'], - loglevel='ERROR', - ) as session: - - logger.info(f"Collecting XNAT project: {xnat_configuration['project']}") - project = session.projects[xnat_configuration["project"]] - - dataset = [] - - if 0 < test_batch < len(project.subjects): - from random import sample - project_subjects = sample(project.subjects[:], test_batch) - else: - project_subjects = project.subjects[:] - - missing_data_log = [] - for subject_i in project_subjects: - subject = project.subjects.data[subject_i.id] - data_sample = {'subject_uri': project.subjects[subject.id].uri, - 'subject_id': project.subjects[subject.id].label, - 'data': []} - - if actions: - try: - data = [] - for action, data_label in actions: - action_data = [] - - # logger.debug(f"Running action: {action.__name__} on {subject.id}") - xnat_obj = action(project.subjects[subject.id]) - - if type(xnat_obj) == list: - if len(xnat_obj) == 0: - missing_data_log.append({'subject_id': subject_i.id, - 'action_data': subject_i.label, - 'failed_action': action}) - logger.debug(f'No data found for {subject_i}: action {action} removing sample') - raise Exception - - for obj in xnat_obj: - action_data.append({'source_action': action.__name__, - 'action_data': obj.uri, - 'data_type': 'xnat_uri', - 'data_label': data_label}) - - elif type(xnat_obj) == str: - action_data.append({'source_action': action.__name__, - 'action_data': xnat_obj, - 'data_type': 'value', - 'data_label': data_label}) - - data.append(action_data) - - except Exception as e: - logger.debug(f'No data found for {subject_i}; removing sample. Exception {e}') - continue - - if flatten_output: - data = list(chain(*data)) - - data_sample['data'] = data - - dataset.append((data_sample)) - - if missing_data_log: - df = pd.DataFrame(missing_data_log) - df.to_csv('missing_data_log.csv') - mlflow.log_artifact('missing_data_log.csv') - mlflow.log_param('N_failed_xnat_samples', len(df)) - - return dataset diff --git a/Project/app/__init__.py b/Project/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/app/app_tests/__init__.py b/Project/app/app_tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/shared/__init__.py b/Project/shared/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/tests/__init__.py b/Project/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Project/tests/train_tests/__init__.py b/Project/tests/train_tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/project/classifier_2d/src/transforms/load_image_xnatd.py b/project/classifier_2d/src/transforms/load_image_xnatd.py index d91eb39..ac88d3d 100644 --- a/project/classifier_2d/src/transforms/load_image_xnatd.py +++ b/project/classifier_2d/src/transforms/load_image_xnatd.py @@ -9,12 +9,10 @@ import xnat from monai.config import KeysCollection -from monai.transforms import MapTransform, LoadImage -from monai.transforms import Transform +from monai.transforms import MapTransform, LoadImage, Transform logger = logging.getLogger(__name__) - class LoadImageXNATd(MapTransform): """ MapTransform for importing image data from XNAT From 2f2f49c2f7bbf9a8375065a7f32bfc746a4439f2 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:36:50 +0100 Subject: [PATCH 13/21] Formatting of import and comments for transforms with files. --- .../classifier_2d/src/transforms/load_image_xnatd.py | 10 +++------- project/classifier_2d/src/transforms/safe_wrapper.py | 5 ++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/project/classifier_2d/src/transforms/load_image_xnatd.py b/project/classifier_2d/src/transforms/load_image_xnatd.py index ac88d3d..251cf69 100644 --- a/project/classifier_2d/src/transforms/load_image_xnatd.py +++ b/project/classifier_2d/src/transforms/load_image_xnatd.py @@ -1,6 +1,4 @@ -""" -MONAI MapTransform for importing image data from XNAT -""" +"""MONAI MapTransform for importing image data from XNAT.""" import glob import logging import os @@ -9,14 +7,12 @@ import xnat from monai.config import KeysCollection -from monai.transforms import MapTransform, LoadImage, Transform +from monai.transforms import LoadImage, MapTransform, Transform logger = logging.getLogger(__name__) class LoadImageXNATd(MapTransform): - """ - MapTransform for importing image data from XNAT - """ + """MapTransform for importing image data from XNAT.""" def __init__(self, keys: KeysCollection, xnat_configuration: dict = None, image_loader: Transform = LoadImage(), validate_data: bool = False, diff --git a/project/classifier_2d/src/transforms/safe_wrapper.py b/project/classifier_2d/src/transforms/safe_wrapper.py index 0436d64..d3d65a0 100644 --- a/project/classifier_2d/src/transforms/safe_wrapper.py +++ b/project/classifier_2d/src/transforms/safe_wrapper.py @@ -1,7 +1,6 @@ -# Wrapper for Monai Transforms so failures are logged instead of run ending - -import os +"""Wrapper for Monai Transforms so failures are logged instead of run ending.""" import csv +import os import torch class SafeWrapperTransform: From a8226e2c9c283b55a81ee8c8f8704598d2a39e61 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:40:57 +0100 Subject: [PATCH 14/21] Update transforms imports --- .../classifier_2d/src/transforms/__init__.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/project/classifier_2d/src/transforms/__init__.py b/project/classifier_2d/src/transforms/__init__.py index def4dfd..5097ed7 100644 --- a/project/classifier_2d/src/transforms/__init__.py +++ b/project/classifier_2d/src/transforms/__init__.py @@ -1,29 +1,29 @@ import torch from monai.transforms import ( - LoadImage, - SqueezeDimd, - EnsureChannelFirstd, - CropForegroundd, - Resized, - ScaleIntensityd, CastToTyped, - RandFlipd, - RandZoomd, - RandRotated, + CropForegroundd, + EnsureChannelFirstd, + EnsureTyped, + LoadImage, + RandAdjustContrastd, RandAffined, + RandCoarseDropoutd, + RandFlipd, RandGaussianNoised, RandGaussianSmoothd, + RandRotated, RandScaleIntensityd, - RandAdjustContrastd, - RandCoarseDropoutd, + RandZoomd, ResizeWithPadOrCropd, - ToTensord, + Resized, + ScaleIntensityd, SelectItemsd, - EnsureTyped, Spacingd, + SqueezeDimd, + ToTensord, ) -from Project.Classifier_2D.src.transforms.load_image_XNATd import LoadImageXNATd +from src.transforms.load_image_xnatd import LoadImageXNATd def load_xnat(xnat_configuration: dict): """ From 2f2d457353f6a88c202be0f14f33ab3c6668b518 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:42:55 +0100 Subject: [PATCH 15/21] Tidy tools.py imports area --- project/classifier_2d/src/utils/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/project/classifier_2d/src/utils/tools.py b/project/classifier_2d/src/utils/tools.py index 8df0b4e..5dd988b 100644 --- a/project/classifier_2d/src/utils/tools.py +++ b/project/classifier_2d/src/utils/tools.py @@ -10,7 +10,6 @@ logger = logging.getLogger(__name__) - class DataBuilderXNAT: def __init__(self, xnat_configuration: dict, actions: list = None, flatten_output=True, test_batch: int = -1, From cda071573bb3bf6b63ea236f601a4f2b2fc46702 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:44:59 +0100 Subject: [PATCH 16/21] Tidy module imports for data_import_xnat.py --- project/classifier_2d/src/data_import_xnat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project/classifier_2d/src/data_import_xnat.py b/project/classifier_2d/src/data_import_xnat.py index 487cd16..599d691 100644 --- a/project/classifier_2d/src/data_import_xnat.py +++ b/project/classifier_2d/src/data_import_xnat.py @@ -2,12 +2,12 @@ from tqdm import tqdm from typing import List -from Project.Classifier_2D.src.utils.tools import DataBuilderXNAT -from xnat.mixin import ImageScanData, SubjectData - -from Project.Classifier_2D.src.transforms import load_xnat from monai.data import Dataset from torch.utils.data import DataLoader +from xnat.mixin import ImageScanData, SubjectData + +from src.transforms import load_xnat +from src.utils.tools import DataBuilderXNAT logger = logging.getLogger(__name__) From 1fc546c27ae44a2197462a761225aec713a2cc84 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:47:19 +0100 Subject: [PATCH 17/21] Tidy imports for datamodule.py --- project/classifier_2d/src/datamodule.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/project/classifier_2d/src/datamodule.py b/project/classifier_2d/src/datamodule.py index 4ba94e4..7d649cc 100644 --- a/project/classifier_2d/src/datamodule.py +++ b/project/classifier_2d/src/datamodule.py @@ -1,22 +1,19 @@ import logging from collections import Counter from typing import List, Optional + import mlflow import numpy as np import pytorch_lightning import torch - -from monai.data import CacheDataset, Dataset -from monai.data import pad_list_data_collate +from monai.data import CacheDataset, Dataset, pad_list_data_collate from monai.transforms import Compose - from sklearn.model_selection import train_test_split from torch.cuda import is_available from torch.utils.data import DataLoader - -from Project.Classifier_2D.src.transforms import normalise, train_augment, output -from Project.Classifier_2D.src.transforms.safe_wrapper import SafeWrapperTransform +from src.transforms import normalise, output, train_augment +from src.transforms.safe_wrapper import SafeWrapperTransform logger = logging.getLogger(__name__) From a6825a7f3d3abc3cdcd11efdb058bc576c585af3 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:49:23 +0100 Subject: [PATCH 18/21] Tidy imports for network.py --- project/classifier_2d/src/network.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/project/classifier_2d/src/network.py b/project/classifier_2d/src/network.py index d264c1a..a452915 100644 --- a/project/classifier_2d/src/network.py +++ b/project/classifier_2d/src/network.py @@ -1,25 +1,22 @@ import logging -import pytorch_lightning +from abc import ABC + import mlflow +import numpy as np +import pytorch_lightning import torch -from abc import ABC from monai.data import decollate_batch -from monai.transforms import ( - AsDiscrete, - Compose, - Activations, -) -from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay, recall_score +from monai.transforms import Activations, AsDiscrete, Compose +from sklearn.metrics import ClassificationReport, ConfusionMatrixDisplay, confusion_matrix, recall_score from timm import create_model from timm.data import Mixup from torch.nn import CrossEntropyLoss from torchmetrics import Accuracy, F1Score -from Project.Classifier_2D.src.data_module import label_dict from torchmetrics.classification import MulticlassAUROC -import numpy as np -logger = logging.getLogger(__name__) +from src.datamodule import label_dict +logger = logging.getLogger(__name__) class Network(pytorch_lightning.LightningModule, ABC): """ From 6f93e081a9d37edb274a9ceb5c9ab95c25efd418 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:50:01 +0100 Subject: [PATCH 19/21] Remove software-properties-common from Dockerfile --- project/classifier_2d/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/classifier_2d/Dockerfile b/project/classifier_2d/Dockerfile index 2a7e158..0d1d35c 100644 --- a/project/classifier_2d/Dockerfile +++ b/project/classifier_2d/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /train ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get install -y --no-install-recommends build-essential git rsync software-properties-common ffmpeg libsm6 libxext6 && \ + apt-get install -y --no-install-recommends build-essential git rsync ffmpeg libsm6 libxext6 && \ rm -rf /var/lib/apt/lists/* ENV PYTHONPATH="/mlflow/projects/code/:$PYTHONPATH" From 9f975e979f06eb7b89c79bd1fd30c3bbf100e2fc Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:56:39 +0100 Subject: [PATCH 20/21] Update standard transform list --- .../classifier_2d/src/transforms/__init__.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/project/classifier_2d/src/transforms/__init__.py b/project/classifier_2d/src/transforms/__init__.py index 5097ed7..33b8881 100644 --- a/project/classifier_2d/src/transforms/__init__.py +++ b/project/classifier_2d/src/transforms/__init__.py @@ -5,6 +5,7 @@ EnsureChannelFirstd, EnsureTyped, LoadImage, + NormalizeIntensityd, RandAdjustContrastd, RandAffined, RandCoarseDropoutd, @@ -17,6 +18,7 @@ ResizeWithPadOrCropd, Resized, ScaleIntensityd, + ScaleIntensityRangePercentilesd, SelectItemsd, Spacingd, SqueezeDimd, @@ -47,9 +49,8 @@ def normalise(image_size): EnsureChannelFirstd(keys=['image']), CropForegroundd(keys=['image'], source_key='image'), Resized(keys=['image'], size_mode='longest', spatial_size=image_size+20), - #Maybe limit top intensity in case of big spikes? - ScaleIntensityd(keys=["image"], minv=0.0, maxv=255.0), - CastToTyped(keys=["image"], dtype=torch.uint8), + ScaleIntensityRangePercentilesd(keys=["image"], lower=0, upper=99, b_min=0.0, b_max=255.0, clip=True), + CastToTyped(keys=["image"], dtype=torch.float32), ] def train_augment(image_size): @@ -59,20 +60,20 @@ def train_augment(image_size): """ return [ RandFlipd(keys=['image'], spatial_axis=0, prob=0.5), - RandZoomd(keys=['image'], prob=0.2, min_zoom=1.05,max_zoom=1.1), - RandRotated(keys=['image'], prob=0.2, range_x=0.4), - RandAffined(keys=['image'], prob=0.2, padding_mode='zeros'), - RandGaussianNoised(keys=['image'], prob=0.1, mean=0.0, std=0.1), - RandGaussianSmoothd(keys=['image'], prob=0.2, sigma_x=(0.5,1.0)), - RandScaleIntensityd(keys=['image'], prob=0.15, factors=(0.75,1.25)), - RandAdjustContrastd(keys=['image'], prob=0.1, gamma=(0.5,2), retain_stats=True, invert_image=True), - RandAdjustContrastd(keys=['image'], prob=0.3, gamma=(0.5,2), retain_stats=True, invert_image=False), + RandZoomd(keys=['image'], prob=0.4, min_zoom=1.05,max_zoom=1.1), + RandRotated(keys=['image'], prob=0.4, range_x=0.4), + RandAffined(keys=['image'], prob=0.3, padding_mode='zeros'), + RandGaussianNoised(keys=['image'], prob=0.3, mean=0.0, std=10.0), + RandGaussianSmoothd(keys=['image'], prob=0.35, sigma_x=(0.5,1.0), sigma_y=(0.5,1.0)), + RandScaleIntensityd(keys=['image'], prob=0.3, factors=(0.75,1.25)), + RandAdjustContrastd(keys=['image'], prob=0.2, gamma=(0.5,2), retain_stats=True, invert_image=True), + RandAdjustContrastd(keys=['image'], prob=0.4, gamma=(0.5,2), retain_stats=True, invert_image=False), ResizeWithPadOrCropd( keys=["image"], spatial_size=(image_size,image_size), mode='replicate' ), - RandCoarseDropoutd(keys=['image'], prob=0.5, fill_value=0, holes=8, max_holes=16, spatial_size=(10,10), max_spatial_size=(36,36)), + RandCoarseDropoutd(keys=['image'], prob=0.35, fill_value=0, holes=8, max_holes=16, spatial_size=(10,10), max_spatial_size=(15,15)), ] def output(image_size): @@ -86,6 +87,9 @@ def output(image_size): mode='replicate' ), ScaleIntensityd(keys=["image"], minv=0.0, maxv=1), + # Normalize with grayscale-averaged ImageNet stats (mean=0.449, std=0.226) + # Required for pretrained ImageNet models + NormalizeIntensityd(keys=["image"], subtrahend=0.449, divisor=0.226), ToTensord(keys=['image', 'label']), SelectItemsd(keys=['subject_id', 'image', 'label']), EnsureTyped(keys=['image', 'label'], track_meta=False), From 9677449d419b7ada8265b448d427b1b9d4bab918 Mon Sep 17 00:00:00 2001 From: Michael Woodward Date: Wed, 6 May 2026 15:58:20 +0100 Subject: [PATCH 21/21] Move readme, this needs updating --- project/classifier_2d/README.md | 67 ++++++++++++++++++++------- project/classifier_2d/mlops/README.md | 60 ------------------------ 2 files changed, 51 insertions(+), 76 deletions(-) delete mode 100644 project/classifier_2d/mlops/README.md diff --git a/project/classifier_2d/README.md b/project/classifier_2d/README.md index d3f076d..0364a0f 100644 --- a/project/classifier_2d/README.md +++ b/project/classifier_2d/README.md @@ -1,29 +1,64 @@ -# 2D Classifier +# 2D Classifier THIS NEEDS UPDATING -## Introduction -The template provides a starting point with helper classes and functions to facilitate rapid development and deployment of applications for 2D Classifiers. +## Example Workflow: -## Layout +#### 1. Adapt XNATDataImport.py for your data +#### 2. Adapt Datamodule.py and Network.py if necessary +#### 3. Perform hyperparameter tuning using tune.py +#### 4. Utilise these hyperparameter values with train.py and threshold tune. +#### 5. Success! Model is stored in mlflow and ready for app development. -### There are 3 sections: +More information can be found below. -#### 1. app -This area is used to work on the deployment of the application and provides helper frameworks. +## There are 7 main components for training 2D Classifiers: -#### 2. tests -This area holds unit tests for both app and train. +### 1. `config/config.cfg` and `config/local_config.cfg` +The config file contains all the information that is used for configuring the project, experiment, and tracking server. This includes training parameters and XNAT configurations. -#### 3. train -This area is used to work on the training and creation of AI models +The config file path is also passed as an argument to the MLOps `Experiment` class where the experiment and tracking are configured. -*Further information can be found within each folder.* +As there will be differences between local development and running on DGX (for example XNAT configurations), it is highly encouraged to make use of `local_config` when developing locally. +Note: The values present in the template config files are examples, you can remove any except those in `[server]` and `[project]` which are necessary for MLOps. Outside of these you are encouraged to add and modify the config files as relevant to your project. -## Examples +### 2. `src/XNATDataImport.py` +This file is used to define and pull the required data from XNAT. It utilises DataBuilderXNAT to do so as shown in the example, if you require additional or different data from XNAT additional actions can be added. -https://github.com/GSTT-CSC/wrist-fracture-x-ray +If your data is not stored in XNAT this can be replaced by any method that accesses your data. -For further information on MLOps please refer to the MLOps repo: +### 3. `src/Network.py` +This file is used to define the PyTorch `LightningModule` class. + +This is where you set the Network architecture and flow that you will use for training, validation, and testing. + +Here you can set up which metrics are calculated and at which stage in the flow these occur, along with the model and optimiser. + +The example has numerous metrics and steps that are not always necessary, feel free to delete or add as relevant to your project. + +### 4. `src/DataModule.py` +This file is used to define the PyTorch `LightningDataModule` class. + +This is where you define the data that is used for training, validation, and testing. + +The example involves additional data validation steps that might not be relevant, feel free to delete or add as relevant to your project. + +### 5. `scripts/tune.py` +This file is used to define hyperparameter tuning runs. + +This is where the 'Datamodule' and' Network' are pulled together and uses optuna to perform the trials. +Starting hyperparameters are included near the top of the file. + +### 6. `scripts/train.py` +This file is used to define the training run. + +This is where the `Datamodule` and `Network` are pulled together. + +The example includes callbacks to retrieve the best model parameters, feel free to delete or add as relevant to your project. + +### 7. `Dockerfile` +This dockerfile sets up the Docker image that the MLOps run will utilise. + +In the example this is just a simple environment running python version 3.10. +You will most likely need to adapt this for your project. -https://github.com/GSTT-CSC/MLOps diff --git a/project/classifier_2d/mlops/README.md b/project/classifier_2d/mlops/README.md deleted file mode 100644 index a8304db..0000000 --- a/project/classifier_2d/mlops/README.md +++ /dev/null @@ -1,60 +0,0 @@ -## Example Workflow: - -#### 1. Adapt XNATDataImport.py for your data -#### 2. Adapt Datamodule.py and Network.py if necessary -#### 3. Perform hyperparameter tuning using tune.py -#### 4. Utilise these hyperparameter values with train.py and threshold tune. -#### 5. Success! Model is stored in mlflow and ready for app development. - -More information can be found below. - -## There are 7 main components for training 2D Classifiers: - -### 1. `config/config.cfg` and `config/local_config.cfg` -The config file contains all the information that is used for configuring the project, experiment, and tracking server. This includes training parameters and XNAT configurations. - -The config file path is also passed as an argument to the MLOps `Experiment` class where the experiment and tracking are configured. - -As there will be differences between local development and running on DGX (for example XNAT configurations), it is highly encouraged to make use of `local_config` when developing locally. - -Note: The values present in the template config files are examples, you can remove any except those in `[server]` and `[project]` which are necessary for MLOps. Outside of these you are encouraged to add and modify the config files as relevant to your project. - -### 2. `src/XNATDataImport.py` -This file is used to define and pull the required data from XNAT. It utilises DataBuilderXNAT to do so as shown in the example, if you require additional or different data from XNAT additional actions can be added. - -If your data is not stored in XNAT this can be replaced by any method that accesses your data. - -### 3. `src/Network.py` -This file is used to define the PyTorch `LightningModule` class. - -This is where you set the Network architecture and flow that you will use for training, validation, and testing. - -Here you can set up which metrics are calculated and at which stage in the flow these occur, along with the model and optimiser. - -The example has numerous metrics and steps that are not always necessary, feel free to delete or add as relevant to your project. - -### 4. `src/DataModule.py` -This file is used to define the PyTorch `LightningDataModule` class. - -This is where you define the data that is used for training, validation, and testing. - -The example involves additional data validation steps that might not be relevant, feel free to delete or add as relevant to your project. - -### 5. `scripts/tune.py` -This file is used to define hyperparameter tuning runs. - -This is where the 'Datamodule' and' Network' are pulled together and uses optuna to perform the trials. -Starting hyperparameters are included near the top of the file. - -### 6. `scripts/train.py` -This file is used to define the training run. - -This is where the `Datamodule` and `Network` are pulled together. - -The example includes callbacks to retrieve the best model parameters, feel free to delete or add as relevant to your project. - -### 7. `Dockerfile` -This dockerfile sets up the Docker image that the MLOps run will utilise. - -In the example this is just a simple environment running python version 3.10. -You will most likely need to adapt this for your project.