From 69e8f9c6b76617a99f64fdda06d42509d340e2f3 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Sat, 24 Oct 2026 01:27:23 +0200 Subject: [PATCH] Evaluation tab performance --- birdnet_analyzer/evaluation/__init__.py | 5 +- .../preprocessing/data_processor.py | 229 +++++++++----- .../evaluation/preprocessing/utils.py | 8 +- .../preprocessing/test_data_processor.py | 291 +++++++++++++++++- tests/evaluation/preprocessing/test_utils.py | 25 ++ tests/evaluation/test_process_data.py | 144 ++++++++- 6 files changed, 597 insertions(+), 105 deletions(-) diff --git a/birdnet_analyzer/evaluation/__init__.py b/birdnet_analyzer/evaluation/__init__.py index f6652a9e4..307f8aeaf 100644 --- a/birdnet_analyzer/evaluation/__init__.py +++ b/birdnet_analyzer/evaluation/__init__.py @@ -19,6 +19,7 @@ PerformanceAssessor, ) from birdnet_analyzer.evaluation.preprocessing.data_processor import DataProcessor +from birdnet_analyzer.utils import runtime_error_handler if typing.TYPE_CHECKING: import numpy as np @@ -197,6 +198,7 @@ def _threshold_arg(value: str) -> float: return number +@runtime_error_handler def main(): """ Entry point for the script. Parses command-line arguments and orchestrates the @@ -205,9 +207,6 @@ def main(): import matplotlib.pyplot as plt from birdnet_analyzer.cli import verbosity_args - from birdnet_analyzer.logs import setup_logging - - setup_logging() parser = argparse.ArgumentParser( description="Performance Assessor Core Script", parents=[verbosity_args()] diff --git a/birdnet_analyzer/evaluation/preprocessing/data_processor.py b/birdnet_analyzer/evaluation/preprocessing/data_processor.py index e3e82cd4c..3e19ad127 100644 --- a/birdnet_analyzer/evaluation/preprocessing/data_processor.py +++ b/birdnet_analyzer/evaluation/preprocessing/data_processor.py @@ -7,8 +7,8 @@ training or evaluation. """ +import logging import os -import warnings from collections.abc import Sequence from typing import ClassVar @@ -21,6 +21,16 @@ read_and_concatenate_files_in_directory, ) +logger = logging.getLogger(__name__) + + +def _by_recording(table: pd.DataFrame) -> dict: + return dict(list(table.groupby("recording_filename", sort=False))) + + +def _is_sorted(values: np.ndarray) -> bool: + return bool(np.all(values[1:] >= values[:-1])) + class DataProcessor: """ @@ -207,6 +217,7 @@ def load_data(self) -> None: Raises: ValueError: If file reading fails or data preparation encounters issues. + KeyError: If the predictions have rows but no confidence column. """ self.unmatched_prediction_files = set() @@ -243,10 +254,9 @@ def load_data(self) -> None: if not self.prediction_file_name.startswith( os.path.splitext(self.annotation_file_name)[0] ): - warnings.warn( + logger.warning( "Prediction file name and annotation file name do not fully match, " - "but proceeding anyway.", - stacklevel=2, + "but proceeding anyway." ) prediction_file = os.path.join( @@ -275,6 +285,13 @@ def load_data(self) -> None: class_col_pred ].apply(lambda x: self.class_mapping.get(x, x)) # ty:ignore[unresolved-attribute] + confidence_col = self.get_column_name("Confidence", prediction=True) + if ( + not self.predictions_df.empty + and self.predictions_df[confidence_col].isna().all() + ): + raise ValueError(f"Confidence column {confidence_col!r} has no values.") + # Reconcile the two recording sets. A recording that only has predictions has no # ground truth, so its predictions cannot be scored. By default those recordings # are dropped; only when the caller opts in (exhaustive annotations) are they @@ -298,17 +315,17 @@ def load_data(self) -> None: if self.unmatched_prediction_files: names = ", ".join(sorted(map(str, self.unmatched_prediction_files))) if self.score_unannotated_as_empty: - warnings.warn( - f"Prediction files without matching annotation file: {names}. " - "Their annotations are assumed empty, so every prediction on " - "them counts as a false positive.", - stacklevel=2, + logger.warning( + "Prediction files without matching annotation file: %s. Their " + "annotations are assumed empty, so every prediction on them " + "counts as a false positive.", + names, ) else: - warnings.warn( - f"Prediction files without matching annotation file: {names}. " - "These recordings are dropped and not scored.", - stacklevel=2, + logger.warning( + "Prediction files without matching annotation file: %s. These " + "recordings are dropped and not scored.", + names, ) self.predictions_df = self.predictions_df[ self.predictions_df["recording_filename"].isin( @@ -316,6 +333,15 @@ def load_data(self) -> None: ) ] + unpredicted_recordings = annotation_recordings - prediction_recordings + if unpredicted_recordings: + logger.warning( + "Annotation files without matching prediction file: %s. Their " + "predictions are assumed empty, so every annotation on them counts " + "as a false negative.", + ", ".join(sorted(map(str, unpredicted_recordings))), + ) + class_col_pred = self.get_column_name("Class", prediction=True) class_col_annot = self.get_column_name("Class", prediction=False) @@ -360,28 +386,25 @@ def process_data(self) -> None: Processes the loaded data, aligns predictions and annotations with sample intervals, and updates the samples DataFrame. - This method iterates through all recording filenames, processes each recording, - and aggregates the results into the `samples_df` attribute. + Every recording found in either table is processed once, in sorted order, and + the per-recording results are concatenated into the `samples_df` attribute. """ - self.samples_df = pd.DataFrame() - - recording_filenames = set( - self.predictions_df["recording_filename"].unique() - ).union(set(self.annotations_df["recording_filename"].unique())) - - for recording_filename in recording_filenames: - pred_df = self.predictions_df[ - self.predictions_df["recording_filename"] == recording_filename - ] - annot_df = self.annotations_df[ - self.annotations_df["recording_filename"] == recording_filename - ] - - samples_df = self.process_recording(recording_filename, pred_df, annot_df) - - self.samples_df = pd.concat( - [self.samples_df, samples_df], ignore_index=True + predictions = _by_recording(self.predictions_df) + annotations = _by_recording(self.annotations_df) + no_predictions = self.predictions_df.iloc[:0] + no_annotations = self.annotations_df.iloc[:0] + + frames = [ + self.process_recording( + recording, + predictions.get(recording, no_predictions), + annotations.get(recording, no_annotations), ) + for recording in sorted(predictions.keys() | annotations.keys(), key=str) + ] + self.samples_df = ( + pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() + ) def process_recording( self, recording_filename: str, pred_df: pd.DataFrame, annot_df: pd.DataFrame @@ -471,9 +494,11 @@ def determine_file_duration( if end_time_col_annot in annot_df.columns else 0.0 ) - duration = max(max_end_pred, max_end_annot) + duration = max( + filter(pd.notna, (max_end_pred, max_end_annot)), default=0.0 + ) - if pd.isna(duration) or duration < 0: + if duration < 0: duration = 0.0 return duration @@ -507,19 +532,14 @@ def initialize_samples( samples = { "filename": recording_filename, - "sample_index": [], - "start_time": [], - "end_time": [], + "sample_index": np.arange(len(intervals)), + "start_time": intervals, + "end_time": np.minimum(intervals + self.sample_duration, file_duration), } - for idx, start in enumerate(intervals): - samples["sample_index"].append(idx) - samples["start_time"].append(start) - samples["end_time"].append(min(start + self.sample_duration, file_duration)) - for label in self.classes: - samples[f"{label}_confidence"] = [0.0] * len(samples["sample_index"]) - samples[f"{label}_annotation"] = [0] * len(samples["sample_index"]) + samples[f"{label}_confidence"] = np.zeros(len(intervals)) + samples[f"{label}_annotation"] = np.zeros(len(intervals), dtype=np.int64) return pd.DataFrame(samples) @@ -545,31 +565,26 @@ def update_samples_with_predictions( pred_df (pd.DataFrame): DataFrame containing prediction information. samples_df (pd.DataFrame): DataFrame of samples to be updated with confidence scores. - """ - class_col = self.get_column_name("Class", prediction=True) - start_time_col = self.get_column_name("Start Time", prediction=True) - end_time_col = self.get_column_name("End Time", prediction=True) - confidence_col = self.get_column_name("Confidence", prediction=True) - for _, row in pred_df.iterrows(): - class_name = row[class_col] - if class_name not in self.classes: - continue - - begin_time = row[start_time_col] - end_time = row[end_time_col] - confidence = row.get(confidence_col, 0.0) + Raises: + KeyError: If `pred_df` has rows but no confidence column. + """ + if pred_df.empty: + return - sample_indices = samples_df[ - (samples_df["start_time"] <= end_time - self.min_overlap) - & (samples_df["end_time"] >= begin_time + self.min_overlap) - ].index + sample_rows, class_cols, event_rows = self._overlapping_pairs( + pred_df, samples_df, prediction=True + ) + confidence_col = self.get_column_name("Confidence", prediction=True) + confidence = pred_df[confidence_col].to_numpy()[event_rows].astype(float) + if sample_rows.size == 0: + return - for i in sample_indices: - current_confidence = samples_df.loc[i, f"{class_name}_confidence"] - samples_df.loc[i, f"{class_name}_confidence"] = max( - current_confidence, confidence - ) + hit_classes, hit_cols = np.unique(class_cols, return_inverse=True) + columns = [f"{self.classes[cls]}_confidence" for cls in hit_classes] + block = samples_df[columns].to_numpy(dtype=float, copy=True) + np.fmax.at(block, (sample_rows, hit_cols), confidence) + samples_df[columns] = block def update_samples_with_annotations( self, annot_df: pd.DataFrame, samples_df: pd.DataFrame @@ -586,25 +601,75 @@ def update_samples_with_annotations( samples_df (pd.DataFrame): DataFrame of samples to be updated with annotations. """ - class_col = self.get_column_name("Class", prediction=False) - start_time_col = self.get_column_name("Start Time", prediction=False) - end_time_col = self.get_column_name("End Time", prediction=False) + if annot_df.empty: + return + + sample_rows, class_cols, _ = self._overlapping_pairs( + annot_df, samples_df, prediction=False + ) + if sample_rows.size == 0: + return + + hit_classes, hit_cols = np.unique(class_cols, return_inverse=True) + columns = [f"{self.classes[cls]}_annotation" for cls in hit_classes] + block = samples_df[columns].to_numpy(copy=True) + block[sample_rows, hit_cols] = 1 + samples_df[columns] = block + + def _overlapping_pairs( + self, events: pd.DataFrame, samples_df: pd.DataFrame, prediction: bool + ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """ + Pairs every event of a known class with each sample it overlaps. + + On a grid of consecutive samples the samples an event overlaps form one index + range, found with two binary searches; irregular sample intervals fall back to + comparing every event with every sample. - for _, row in annot_df.iterrows(): - class_name = row[class_col] - if class_name not in self.classes: - continue + Returns: + Three equally long arrays: the sample positions, the class positions (into + `self.classes`) and the event positions (into `events`) of the pairs. + """ + class_col = self.get_column_name("Class", prediction=prediction) + start_time_col = self.get_column_name("Start Time", prediction=prediction) + end_time_col = self.get_column_name("End Time", prediction=prediction) - begin_time = row[start_time_col] - end_time = row[end_time_col] + class_cols = pd.Index(self.classes).get_indexer(events[class_col]) + known = np.flatnonzero(class_cols >= 0) + if known.size == 0: + return known, known, known - sample_indices = samples_df[ - (samples_df["start_time"] <= end_time - self.min_overlap) - & (samples_df["end_time"] >= begin_time + self.min_overlap) - ].index + latest_start = ( + events[end_time_col].to_numpy()[known].astype(float) - self.min_overlap + ) + earliest_end = ( + events[start_time_col].to_numpy()[known].astype(float) + self.min_overlap + ) + start_time = samples_df["start_time"].to_numpy(dtype=float) + end_time = samples_df["end_time"].to_numpy(dtype=float) + + order = np.argsort(start_time, kind="stable") + if _is_sorted(end_time[order]): + first = np.searchsorted(end_time[order], earliest_end, side="left") + stop = np.searchsorted(start_time[order], latest_start, side="right") + counts = np.where( + np.isnan(latest_start) | np.isnan(earliest_end), + 0, + np.maximum(stop - first, 0), + ) + pair = np.repeat(np.arange(known.size), counts) + offsets = np.arange(counts.sum()) - np.repeat( + np.cumsum(counts) - counts, counts + ) + sample_rows = order[np.repeat(first, counts) + offsets] + else: + hit = (start_time <= latest_start[:, None]) & ( + end_time >= earliest_end[:, None] + ) + pair, sample_rows = np.nonzero(hit) - for i in sample_indices: - samples_df.loc[i, f"{class_name}_annotation"] = 1 + event_rows = known[pair] + return sample_rows, class_cols[event_rows], event_rows def create_tensors(self) -> None: """ diff --git a/birdnet_analyzer/evaluation/preprocessing/utils.py b/birdnet_analyzer/evaluation/preprocessing/utils.py index c68ad7dea..8df6e4517 100644 --- a/birdnet_analyzer/evaluation/preprocessing/utils.py +++ b/birdnet_analyzer/evaluation/preprocessing/utils.py @@ -120,6 +120,8 @@ def read_and_concatenate_files_in_directory(directory_path: str) -> pd.DataFrame df_list.append(df) - if df_list: - return pd.concat(df_list, ignore_index=True) - return pd.DataFrame() + if not df_list: + return pd.DataFrame() + + non_empty = [df for df in df_list if not df.empty] + return pd.concat(non_empty or df_list[:1], ignore_index=True) diff --git a/tests/evaluation/preprocessing/test_data_processor.py b/tests/evaluation/preprocessing/test_data_processor.py index c9995863e..1fed3e8b9 100644 --- a/tests/evaluation/preprocessing/test_data_processor.py +++ b/tests/evaluation/preprocessing/test_data_processor.py @@ -1,3 +1,5 @@ +import logging +import tracemalloc from unittest.mock import MagicMock, patch import numpy as np @@ -338,6 +340,7 @@ def test_load_data_with_none_filenames(self, mock_read_concat): "Class": ["A", "B"], "Start Time": [0, 1], "End Time": [1, 2], + "Confidence": [0.9, 0.8], "source_file": ["file1.txt", "file2.txt"], } ) @@ -381,6 +384,7 @@ def test_load_data_with_specific_filenames(self, mock_read_csv): "Class": ["A", "B"], "Start Time": [0, 1], "End Time": [1, 2], + "Confidence": [0.9, 0.8], } ) mock_annotations_df = pd.DataFrame( @@ -527,13 +531,14 @@ def test_load_data_with_empty_directories(self, mock_read_concat): assert dp.annotations_df.empty @patch("pandas.read_csv") - def test_load_data_filenames_do_not_match(self, mock_read_csv): + def test_load_data_filenames_do_not_match(self, mock_read_csv, caplog): """Test load_data with mismatched filenames.""" mock_predictions_df = pd.DataFrame( { "Class": ["A"], "Start Time": [0], "End Time": [1], + "Confidence": [0.9], } ) mock_annotations_df = pd.DataFrame( @@ -545,7 +550,7 @@ def test_load_data_filenames_do_not_match(self, mock_read_csv): ) mock_read_csv.side_effect = [mock_predictions_df, mock_annotations_df] - with pytest.warns(UserWarning, match="do not fully match"): + with caplog.at_level(logging.WARNING, logger=DataProcessor.__module__): DataProcessor( prediction_directory_path="dummy_pred_path", prediction_file_name="prediction_file.txt", @@ -564,6 +569,108 @@ def test_load_data_filenames_do_not_match(self, mock_read_csv): recording_duration=10, ) + assert "do not fully match" in caplog.text + + @patch.object(DataProcessor, "process_data") + @patch( + "birdnet_analyzer.evaluation.preprocessing.data_processor.read_and_concatenate_files_in_directory" + ) + def test_load_data_rejects_missing_confidence_before_binning( + self, mock_read_concat, mock_process_data + ): + mock_read_concat.return_value = pd.DataFrame( + { + "Class": ["A"], + "Start Time": [0], + "End Time": [1], + "source_file": ["file.txt"], + } + ) + + with pytest.raises(KeyError, match="Confidence"): + DataProcessor( + prediction_directory_path="dummy_pred_path", + annotation_directory_path="dummy_annot_path", + ) + + mock_process_data.assert_not_called() + + @patch( + "birdnet_analyzer.evaluation.preprocessing.data_processor.read_and_concatenate_files_in_directory" + ) + def test_load_data_rejects_missing_confidence_of_unannotated_predictions( + self, mock_read_concat + ): + mock_read_concat.side_effect = [ + pd.DataFrame( + { + "Class": ["A"], + "Start Time": [0], + "End Time": [1], + "source_file": ["rec2.txt"], + } + ), + pd.DataFrame( + { + "Class": ["A"], + "Start Time": [0], + "End Time": [1], + "source_file": ["rec1.txt"], + } + ), + ] + + with pytest.raises(KeyError, match="Confidence"): + DataProcessor( + prediction_directory_path="dummy_pred_path", + annotation_directory_path="dummy_annot_path", + recording_duration=10, + ) + + @patch( + "birdnet_analyzer.evaluation.preprocessing.data_processor.read_and_concatenate_files_in_directory" + ) + def test_load_data_accepts_partially_missing_confidence(self, mock_read_concat): + mock_read_concat.return_value = pd.DataFrame( + { + "Class": ["A", "B"], + "Start Time": [0, 1], + "End Time": [1, 2], + "Confidence": [None, 0.5], + "source_file": ["file.txt", "file.txt"], + } + ) + + dp = DataProcessor( + prediction_directory_path="dummy_pred_path", + annotation_directory_path="dummy_annot_path", + recording_duration=10, + ) + + assert dp.samples_df["B_confidence"].max() == 0.5 + + @patch( + "birdnet_analyzer.evaluation.preprocessing.data_processor.read_and_concatenate_files_in_directory" + ) + def test_load_data_rejects_confidence_column_without_values( + self, mock_read_concat + ): + mock_read_concat.return_value = pd.DataFrame( + { + "Class": ["A", "B"], + "Start Time": [0, 1], + "End Time": [1, 2], + "Confidence": [None, None], + "source_file": ["file.txt", "file.txt"], + } + ) + + with pytest.raises(ValueError, match="no values"): + DataProcessor( + prediction_directory_path="dummy_pred_path", + annotation_directory_path="dummy_annot_path", + ) + @patch( "birdnet_analyzer.evaluation.preprocessing.data_processor.read_and_concatenate_files_in_directory" ) @@ -602,6 +709,7 @@ def test_load_data_handle_different_encodings(self, mock_read_concat): "Class": ["A"], "Start Time": [0], "End Time": [1], + "Confidence": [0.9], "source_file": ["file.txt"], } ) @@ -633,6 +741,7 @@ def test_load_data_with_class_mapping(self, mock_read_concat): "Class": ["A", "B", "C"], "Start Time": [0, 1, 2], "End Time": [1, 2, 3], + "Confidence": [0.9, 0.8, 0.7], "source_file": ["file1.txt", "file2.txt", "file3.txt"], } ) @@ -679,6 +788,7 @@ def test_load_data_without_class_mapping(self, mock_read_concat): "Class": ["A", "B", "C"], "Start Time": [0, 1, 2], "End Time": [1, 2, 3], + "Confidence": [0.9, 0.8, 0.7], "source_file": ["file1.txt", "file2.txt", "file3.txt"], } ) @@ -1370,7 +1480,12 @@ def teardown_method(self): def test_valid_predictions_and_annotations(self): """Test with valid predictions and annotations.""" pred_df = pd.DataFrame( - {"Class": ["A", "B"], "Start Time": [0, 5], "End Time": [5, 10]} + { + "Class": ["A", "B"], + "Start Time": [0, 5], + "End Time": [5, 10], + "Confidence": [0.9, 0.8], + } ) annot_df = pd.DataFrame( {"Class": ["A", "C"], "Start Time": [2, 7], "End Time": [7, 12]} @@ -1397,14 +1512,11 @@ def test_empty_predictions_and_annotations(self): ) def test_only_predictions_present(self): - """Test with only predictions present.""" + """Predictions without a confidence column are an error, not 0.0.""" pred_df = pd.DataFrame({"Class": ["A"], "Start Time": [0], "End Time": [5]}) annot_df = pd.DataFrame() - samples_df = self.dp.process_recording("rec1", pred_df, annot_df) - assert ( - samples_df["A_confidence"].iloc[0] == 0.0 - ) # Default confidence since 'Confidence' column missing - assert samples_df["A_annotation"].iloc[0] == 0 # No annotations + with pytest.raises(KeyError, match="Confidence"): + self.dp.process_recording("rec1", pred_df, annot_df) def test_only_annotations_present(self): """Test with only annotations present.""" @@ -1416,17 +1528,21 @@ def test_only_annotations_present(self): def test_no_overlap_between_pred_and_annot(self): """Test with predictions and annotations having no overlap.""" - pred_df = pd.DataFrame({"Class": ["A"], "Start Time": [0], "End Time": [5]}) + pred_df = pd.DataFrame( + {"Class": ["A"], "Start Time": [0], "End Time": [5], "Confidence": [0.9]} + ) annot_df = pd.DataFrame({"Class": ["A"], "Start Time": [10], "End Time": [15]}) samples_df = self.dp.process_recording("rec1", pred_df, annot_df) - assert samples_df["A_confidence"].iloc[0] == 0.0 # Default confidence + assert samples_df["A_confidence"].iloc[0] == 0.9 assert samples_df["A_annotation"].iloc[0] == 0 # No overlap in first sample assert samples_df["A_annotation"].iloc[1] == 0 # No overlap in second sample assert samples_df["A_annotation"].iloc[2] == 1 # Overlap in third sample def test_invalid_times_in_predictions(self): """Test with invalid times in predictions.""" - pred_df = pd.DataFrame({"Class": ["A"], "Start Time": [-5], "End Time": [-1]}) + pred_df = pd.DataFrame( + {"Class": ["A"], "Start Time": [-5], "End Time": [-1], "Confidence": [0.9]} + ) annot_df = pd.DataFrame() samples_df = self.dp.process_recording("rec1", pred_df, annot_df) assert not samples_df.empty @@ -1443,7 +1559,9 @@ def test_custom_sample_duration_and_min_overlap(self): """Test with custom sample_duration and min_overlap.""" self.dp.sample_duration = 3 self.dp.min_overlap = 0.1 - pred_df = pd.DataFrame({"Class": ["A"], "Start Time": [1], "End Time": [2]}) + pred_df = pd.DataFrame( + {"Class": ["A"], "Start Time": [1], "End Time": [2], "Confidence": [0.9]} + ) annot_df = pd.DataFrame( {"Class": ["A"], "Start Time": [1.5], "End Time": [2.5]} ) @@ -1452,7 +1570,9 @@ def test_custom_sample_duration_and_min_overlap(self): def test_classes_not_in_self_classes(self): """Test with classes not in self.classes.""" - pred_df = pd.DataFrame({"Class": ["D"], "Start Time": [0], "End Time": [5]}) + pred_df = pd.DataFrame( + {"Class": ["D"], "Start Time": [0], "End Time": [5], "Confidence": [0.9]} + ) annot_df = pd.DataFrame({"Class": ["D"], "Start Time": [0], "End Time": [5]}) samples_df = self.dp.process_recording("rec1", pred_df, annot_df) # Since 'D' is not in self.classes, it should be skipped @@ -1546,6 +1666,13 @@ def test_max_end_time(self): duration = self.dp.determine_file_duration(pred_df, annot_df) assert duration == 100.0 + def test_end_time_from_one_side_only(self): + self.dp.recording_duration = None + pred_df = pd.DataFrame({"Start Time": [], "End Time": []}) + annot_df = pd.DataFrame({"Start Time": [20], "End Time": [100]}) + duration = self.dp.determine_file_duration(pred_df, annot_df) + assert duration == 100.0 + def test_empty_dataframes(self): """Test when pred_df and annot_df are empty; duration should be zero.""" self.dp.recording_duration = None @@ -1674,6 +1801,12 @@ def test_multiple_classes(self): for col in ["A_confidence", "B_confidence", "A_annotation", "B_annotation"] ) + def test_column_dtypes(self): + samples_df = self.dp.initialize_samples("rec1", 10) + assert samples_df["sample_index"].dtype == np.int64 + assert samples_df["A_confidence"].dtype == np.float64 + assert samples_df["A_annotation"].dtype == np.int64 + def test_empty_recording_filename(self): """Test when recording_filename is an empty string.""" samples_df = self.dp.initialize_samples("", 10) @@ -1809,8 +1942,129 @@ def test_predictions_missing_confidence(self): # 'Confidence' missing } ) + with pytest.raises(KeyError, match="Confidence"): + self.dp.update_samples_with_predictions(pred_df, self.samples_df) + + def test_nan_confidence_is_ignored(self): + pred_df = pd.DataFrame( + { + "Class": ["A", "A", "B"], + "Start Time": [1, 1, 1], + "End Time": [4, 4, 4], + "Confidence": [np.nan, 0.6, np.nan], + } + ) + self.dp.update_samples_with_predictions(pred_df, self.samples_df) + assert self.samples_df.loc[0, "A_confidence"] == 0.6 + assert self.samples_df.loc[0, "B_confidence"] == 0.0 + + def test_sample_aligned_prediction_with_full_min_overlap(self): + self.dp.min_overlap = 5 + pred_df = pd.DataFrame( + {"Class": ["A"], "Start Time": [0], "End Time": [5], "Confidence": [0.7]} + ) + self.dp.update_samples_with_predictions(pred_df, self.samples_df) + assert self.samples_df["A_confidence"].tolist() == [0.7, 0.0, 0.0] + + def test_unsorted_samples_df(self): + self.samples_df = self.samples_df.iloc[::-1].reset_index(drop=True) + pred_df = pd.DataFrame( + {"Class": ["A"], "Start Time": [1], "End Time": [4], "Confidence": [0.6]} + ) + self.dp.update_samples_with_predictions(pred_df, self.samples_df) + assert self.samples_df.set_index("start_time")["A_confidence"].to_dict() == { + 10: 0.0, + 5: 0.0, + 0: 0.6, + } + + def test_unknown_class_with_empty_samples_df(self): + pred_df = pd.DataFrame( + {"Class": ["D"], "Start Time": [0], "End Time": [5], "Confidence": [0.9]} + ) + samples_df = pd.DataFrame() + self.dp.update_samples_with_predictions(pred_df, samples_df) + assert samples_df.empty + + def test_unknown_class_row_with_unparseable_times_is_skipped(self): + pred_df = pd.DataFrame( + { + "Class": ["D", "A"], + "Start Time": ["n/a", 1], + "End Time": ["n/a", 4], + "Confidence": [0.9, 0.6], + } + ) + self.dp.update_samples_with_predictions(pred_df, self.samples_df) + assert self.samples_df["A_confidence"].tolist() == [0.6, 0.0, 0.0] + + def test_nested_sample_intervals(self): + self.samples_df["start_time"] = [0, 2, 4] + self.samples_df["end_time"] = [9, 7, 5] + pred_df = pd.DataFrame( + { + "Class": ["A", "B"], + "Start Time": [0, 6], + "End Time": [3, 8], + "Confidence": [0.8, 0.4], + } + ) + self.dp.update_samples_with_predictions(pred_df, self.samples_df) + assert self.samples_df["A_confidence"].tolist() == [0.8, 0.8, 0.0] + assert self.samples_df["B_confidence"].tolist() == [0.4, 0.4, 0.0] + + def test_prediction_for_the_last_class_only(self): + pred_df = pd.DataFrame( + {"Class": ["C"], "Start Time": [6], "End Time": [9], "Confidence": [0.3]} + ) + self.dp.update_samples_with_predictions(pred_df, self.samples_df) + assert self.samples_df["C_confidence"].tolist() == [0.0, 0.3, 0.0] + assert (self.samples_df[["A_confidence", "B_confidence"]] == 0.0).all().all() + + def test_columns_of_classes_without_hits_are_untouched(self): + self.samples_df["B_confidence"] = self.samples_df["B_confidence"].astype( + np.float32 + ) + pred_df = pd.DataFrame( + {"Class": ["A"], "Start Time": [1], "End Time": [4], "Confidence": [0.6]} + ) + self.dp.update_samples_with_predictions(pred_df, self.samples_df) + assert self.samples_df["A_confidence"].tolist() == [0.6, 0.0, 0.0] + assert self.samples_df["B_confidence"].dtype == np.float32 + + def test_long_recording_with_many_predictions_stays_small_in_memory(self): + self.dp.sample_duration = 3 + samples_df = self.dp.initialize_samples("rec1", 86_400) + rng = np.random.default_rng(0) + start = rng.integers(0, 86_400 // 3, 5_000) * 3.0 + pred_df = pd.DataFrame( + { + "Class": "A", + "Start Time": start, + "End Time": start + 3, + "Confidence": rng.random(5_000), + } + ) + + tracemalloc.start() + self.dp.update_samples_with_predictions(pred_df, samples_df) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + assert peak < 50_000_000 + assert (samples_df["A_confidence"] > 0).sum() == len(np.unique(start)) + + def test_prediction_with_nan_time_is_ignored(self): + pred_df = pd.DataFrame( + { + "Class": ["A", "A"], + "Start Time": [1, np.nan], + "End Time": [np.nan, 4], + "Confidence": [0.9, 0.9], + } + ) self.dp.update_samples_with_predictions(pred_df, self.samples_df) - assert self.samples_df.loc[0, "A_confidence"] == 0.0 # Default confidence + assert (self.samples_df["A_confidence"] == 0.0).all() def test_predictions_with_negative_times(self): """Test predictions with negative times.""" @@ -1958,6 +2212,13 @@ def test_single_annotation_overlapping_multiple_samples(self): assert self.samples_df.loc[0, "A_annotation"] == 1 assert self.samples_df.loc[1, "A_annotation"] == 1 + def test_missing_columns_of_classes_without_hits_are_tolerated(self): + samples_df = self.samples_df.drop(columns=["C_annotation"]) + annot_df = pd.DataFrame({"Class": ["B"], "Start Time": [6], "End Time": [9]}) + self.dp.update_samples_with_annotations(annot_df, samples_df) + assert samples_df["B_annotation"].tolist() == [0, 1, 0] + assert samples_df["A_annotation"].tolist() == [0, 0, 0] + def test_annotations_classes_not_in_self_classes(self): """Test annotations with classes not in self.classes.""" annot_df = pd.DataFrame({"Class": ["D"], "Start Time": [0], "End Time": [5]}) diff --git a/tests/evaluation/preprocessing/test_utils.py b/tests/evaluation/preprocessing/test_utils.py index 016385481..f03be7468 100644 --- a/tests/evaluation/preprocessing/test_utils.py +++ b/tests/evaluation/preprocessing/test_utils.py @@ -1,3 +1,5 @@ +import warnings + import numpy as np import pandas as pd import pytest @@ -289,6 +291,29 @@ def test_read_and_concatenate_files_multiple_txt(tmp_path): pd.testing.assert_frame_equal(result_df, expected_df) +def test_read_and_concatenate_files_skips_header_only_tables(tmp_path): + df1 = pd.DataFrame({"A": [1, 2], "B": [3.5, 4.5]}) + df1.to_csv(tmp_path / "file1.txt", sep="\t", index=False) + df1.iloc[:0].to_csv(tmp_path / "file2.txt", sep="\t", index=False) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + result_df = read_and_concatenate_files_in_directory(str(tmp_path)) + + pd.testing.assert_frame_equal(result_df, df1.assign(source_file="file1.txt")) + + +def test_read_and_concatenate_files_all_header_only_keeps_columns(tmp_path): + pd.DataFrame({"A": [], "B": []}).to_csv( + tmp_path / "file1.txt", sep="\t", index=False + ) + + result_df = read_and_concatenate_files_in_directory(str(tmp_path)) + + assert list(result_df.columns) == ["A", "B", "source_file"] + assert result_df.empty + + def test_read_and_concatenate_files_no_txt(tmp_path): """ Test read_and_concatenate_files_in_directory when there are no .txt files. diff --git a/tests/evaluation/test_process_data.py b/tests/evaluation/test_process_data.py index a9b412120..fb211fbc4 100644 --- a/tests/evaluation/test_process_data.py +++ b/tests/evaluation/test_process_data.py @@ -5,8 +5,13 @@ dragging down the aggregate score. """ +import logging +import sys + +import numpy as np import pytest +from birdnet_analyzer import evaluation, logs, utils from birdnet_analyzer.evaluation import process_data from birdnet_analyzer.evaluation.preprocessing.data_processor import DataProcessor @@ -29,8 +34,9 @@ def _dirs(tmp_path): return ann, pred -def test_unmatched_prediction_file_dropped_by_default(tmp_path): +def test_unmatched_prediction_file_dropped_by_default(tmp_path, caplog): """A recording with predictions but no annotation is dropped, not scored.""" + caplog.set_level(logging.WARNING, logger=DataProcessor.__module__) ann, pred = _dirs(tmp_path) _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) _write_table(pred / "rec1.txt", PRED_HEADER, [[0, 3, "X", 0.9]]) @@ -51,10 +57,13 @@ def test_unmatched_prediction_file_dropped_by_default(tmp_path): assert result.predictions.shape[0] == 10 # rec2's predictions are not counted as false positives. assert result.metrics_df.loc["Precision", "Overall"] == 1.0 + assert "rec2" in caplog.text + assert "dropped" in caplog.text -def test_unmatched_prediction_file_scored_as_empty_when_opted_in(tmp_path): +def test_unmatched_prediction_file_scored_as_empty_when_opted_in(tmp_path, caplog): """Opting in keeps unmatched recordings and scores their predictions as FPs.""" + caplog.set_level(logging.WARNING, logger=DataProcessor.__module__) ann, pred = _dirs(tmp_path) _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) _write_table(pred / "rec1.txt", PRED_HEADER, [[0, 3, "X", 0.9]]) @@ -75,6 +84,8 @@ def test_unmatched_prediction_file_scored_as_empty_when_opted_in(tmp_path): assert result.predictions.shape[0] == 20 # rec2's two predictions become false positives -> precision 1 / (1 + 2). assert result.metrics_df.loc["Precision", "Overall"] == 1 / 3 + assert "rec2" in caplog.text + assert "false positive" in caplog.text def test_dotted_recording_names_do_not_collide(tmp_path): @@ -223,3 +234,132 @@ def test_raven_annotation_matches_birdnet_prediction_table(tmp_path): assert result.empty_classes == () assert result.metrics_df.loc["Precision", "gibbon.female"] == 1.0 assert result.metrics_df.loc["Recall", "noise"] == 0.0 + + +def test_samples_take_the_max_confidence_of_overlapping_predictions(tmp_path): + ann, pred = _dirs(tmp_path) + _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "A"], [4.0, 4.3, "B"]]) + _write_table( + pred / "rec1.txt", + PRED_HEADER, + [[0, 9, "Z", 0.99], [1, 4, "A", 0.9], [0, 3, "A", 0.4], [6, 9, "B", 0.2]], + ) + + processor = DataProcessor( + prediction_directory_path=str(pred), + annotation_directory_path=str(ann), + sample_duration=3, + min_overlap=0.5, + recording_duration=9, + ) + processor.classes = ("A", "B") + processor.process_data() + + samples = processor.samples_df + assert list(samples.columns) == [ + "filename", + "sample_index", + "start_time", + "end_time", + "A_confidence", + "A_annotation", + "B_confidence", + "B_annotation", + ] + assert samples["filename"].tolist() == ["rec1"] * 3 + assert samples["A_confidence"].tolist() == [0.9, 0.9, 0.0] + assert samples["B_confidence"].tolist() == [0.0, 0.0, 0.2] + assert samples["A_annotation"].tolist() == [1, 0, 0] + assert samples["B_annotation"].tolist() == [0, 1, 0] + assert "Z_confidence" not in samples.columns + assert samples["A_confidence"].dtype == np.float64 + assert samples["A_annotation"].dtype == np.int64 + + +def test_samples_are_ordered_by_recording_name(tmp_path): + ann, pred = _dirs(tmp_path) + names = [f"rec_{letter}" for letter in "hcfadgbe"] + _write_table( + ann / "all.txt", [*ANN_HEADER, "Recording"], [[0, 3, "X", n] for n in names] + ) + _write_table( + pred / "all.txt", + [*PRED_HEADER, "Recording"], + [[0, 3, "X", 0.9, n] for n in names], + ) + + processor = DataProcessor( + prediction_directory_path=str(pred), + annotation_directory_path=str(ann), + recording_duration=3, + ) + + assert processor.samples_df["filename"].tolist() == sorted(names) + + +def test_predictions_without_the_confidence_column_are_rejected(tmp_path): + ann, pred = _dirs(tmp_path) + _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) + _write_table(pred / "rec1.txt", PRED_HEADER, [[0, 3, "X", 0.9]]) + + with pytest.raises(KeyError, match="Score"): + DataProcessor( + prediction_directory_path=str(pred), + annotation_directory_path=str(ann), + recording_duration=3, + columns_predictions={ + **DataProcessor.DEFAULT_COLUMNS_PREDICTIONS, + "Confidence": "Score", + }, + ) + + +def test_annotation_only_recording_is_kept_without_recording_duration( + tmp_path, caplog +): + caplog.set_level(logging.WARNING, logger=DataProcessor.__module__) + ann, pred = _dirs(tmp_path) + _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) + _write_table(pred / "rec1.txt", PRED_HEADER, [[0, 3, "X", 0.9]]) + _write_table(ann / "rec2.txt", ANN_HEADER, [[3, 6, "X"]]) + _write_table(pred / "rec2.txt", PRED_HEADER, []) + + processor = DataProcessor( + prediction_directory_path=str(pred), annotation_directory_path=str(ann) + ) + + rec2 = processor.samples_df[processor.samples_df["filename"] == "rec2"] + assert rec2["X_annotation"].tolist() == [0, 1] + assert rec2["X_confidence"].tolist() == [0.0, 0.0] + assert "rec2" in caplog.text + assert "false negative" in caplog.text + + +def test_cli_writes_uncaught_errors_to_the_error_log(tmp_path, monkeypatch): + ann, pred = _dirs(tmp_path) + _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) + _write_table(pred / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) + logged = [] + monkeypatch.setattr(utils, "write_error_log", logged.append) + monkeypatch.setattr( + sys, + "argv", + [ + "birdnet-evaluate", + "--annotation_path", + str(ann), + "--prediction_path", + str(pred), + "--recording_duration", + "3", + ], + ) + + try: + with pytest.raises(KeyError, match="Confidence"): + evaluation.main() + finally: + logs._remove_installed_handlers() + + assert len(logged) == 1 + assert isinstance(logged[0], KeyError)