diff --git a/docs/models-and-metrics/just-bird.md b/docs/models-and-metrics/just-bird.md
index 2c5a9fd..71b8285 100644
--- a/docs/models-and-metrics/just-bird.md
+++ b/docs/models-and-metrics/just-bird.md
@@ -38,7 +38,7 @@ The graph shows precision, recall and F1 at IoU 0.5 for every confidence thresho
-- **Confidence threshold:** around **0.15**. The best threshold on the Hawai'i validation recordings is 0.13 to 0.15, and it carries over to the unseen site with almost no loss (0.384 against the best 0.385).
+- **Confidence threshold:** **0.15**, the default in `detect_birds.py`, the Streamlit app and the ONNX demo. The best threshold on the Hawai'i validation recordings is 0.13 to 0.15, and it carries over to the unseen site with almost no loss (0.384 against the best 0.385).
- **Phrase gap:** some Southern Sierra Nevada labels box a whole phrase where the model boxes single syllables, which lowers the scores at IoU 0.5 and above. The default phrase gap of 0.05 s raises F1 at IoU 0.5 there from 0.385 to 0.396, and on song-labelled Western-US validation minutes from 0.170 to 0.197. On call-level labels it costs about 0.05 (Hawai'i val: 0.581 to 0.531). If your annotations box single calls, pass `--phrase-gap 0`. [Phrase boxes](../cli/detect-birds.md#window-fusion-just-bird-default) has the numbers for other gaps.
- **In-domain:** on the full Hawai'i validation recordings, the released model reaches F1 0.590 at IoU 0.5 (threshold 0.14, phrase gap off).
diff --git a/docs/models/Just-Bird_fp16.onnx b/docs/models/Just-Bird_fp16.onnx
index dc9eab7..4a705b1 100644
Binary files a/docs/models/Just-Bird_fp16.onnx and b/docs/models/Just-Bird_fp16.onnx differ
diff --git a/run_pipeline.bat b/run_pipeline.bat
index f0728ac..4b97bc8 100644
--- a/run_pipeline.bat
+++ b/run_pipeline.bat
@@ -113,13 +113,13 @@ python src\evaluation\f_beta_score_analysis.py ^
if errorlevel 1 goto :fail
-REM Step 3: From raw detections, filter at conf=0.25 and merge.
-echo Filtering raw detections at conf=0.25 and merging for confusion matrix...
+REM Step 3: From raw detections, filter at the model's default confidence
+REM (0.15 for Just-Bird, 0.2 for the other models; add --conf to change it) and merge.
+echo Filtering raw detections at the model's default confidence and merging for confusion matrix...
python src\evaluation\filter_and_merge_detections.py ^
--raw-detections "%OUTPUT_PATH%" ^
--output-path "%OUTPUT_PATH%" ^
--output-format all ^
- --conf 0.2 ^
%MERGE_FLAGS%
if errorlevel 1 goto :fail
diff --git a/run_pipeline.ps1 b/run_pipeline.ps1
index 13e5961..63a216d 100644
--- a/run_pipeline.ps1
+++ b/run_pipeline.ps1
@@ -107,13 +107,13 @@ python src\evaluation\f_beta_score_analysis.py `
@SINGLE_CLS_FLAG
-# Step 3: From raw detections, filter at conf=0.25 and merge.
-Write-Host "Filtering raw detections at conf=0.25 and merging for confusion matrix..."
+# Step 3: From raw detections, filter at the model's default confidence
+# (0.15 for Just-Bird, 0.2 for the other models; add --conf to change it) and merge.
+Write-Host "Filtering raw detections at the model's default confidence and merging for confusion matrix..."
python src\evaluation\filter_and_merge_detections.py `
--raw-detections "$OUTPUT_PATH" `
--output-path "$OUTPUT_PATH" `
--output-format all `
- --conf 0.2 `
@MERGE_FLAGS
diff --git a/run_pipeline.sh b/run_pipeline.sh
index 8eada7a..5c570b8 100755
--- a/run_pipeline.sh
+++ b/run_pipeline.sh
@@ -115,13 +115,13 @@ python src/evaluation/f_beta_score_analysis.py \
"${SINGLE_CLS_FLAG[@]}"
-# Step 3: From raw detections, filter at conf=0.25 and merge.
-echo "Filtering raw detections at conf=0.25 and merging for confusion matrix..."
+# Step 3: From raw detections, filter at the model's default confidence
+# (0.15 for Just-Bird, 0.2 for the other models; add --conf to change it) and merge.
+echo "Filtering raw detections at the model's default confidence and merging for confusion matrix..."
python src/evaluation/filter_and_merge_detections.py \
--raw-detections "${OUTPUT_PATH}" \
--output-path "${OUTPUT_PATH}" \
--output-format all \
- --conf 0.2 \
"${MERGE_FLAGS[@]}"
diff --git a/src/evaluation/filter_and_merge_detections.py b/src/evaluation/filter_and_merge_detections.py
index aa07b38..e35ee97 100644
--- a/src/evaluation/filter_and_merge_detections.py
+++ b/src/evaluation/filter_and_merge_detections.py
@@ -29,6 +29,7 @@
describe_merge_settings,
file_ends_from_raw,
merge_detections,
+ resolve_conf,
resolve_merge_settings,
)
from inference.utils.output_paths import (
@@ -323,8 +324,11 @@ def main():
parser.add_argument(
'--conf',
type=float,
- default=0.2,
- help='Confidence threshold for filtering (0.0-1.0, default: 0.2)'
+ default=None,
+ help=(
+ 'Confidence threshold for filtering (0.0-1.0). Default: 0.15 for Just-Bird, '
+ '0.2 for the other models (species_mapping in the JSON model_config)'
+ )
)
parser.add_argument(
@@ -393,7 +397,7 @@ def main():
if not Path(args.raw_detections).exists():
print(f"Error: Raw detections file not found: {args.raw_detections}", file=sys.stderr)
sys.exit(1)
- if not (0.0 <= args.conf <= 1.0):
+ if args.conf is not None and not (0.0 <= args.conf <= 1.0):
print(f"Error: Confidence threshold must be between 0.0 and 1.0, got: {args.conf}", file=sys.stderr)
sys.exit(1)
if not ensure_output_directory(args.output_path):
@@ -403,6 +407,7 @@ def main():
data = filter_obj.load_detections(args.raw_detections)
raw_list = data.get('detections', [])
model_config = data.get('model_config', {})
+ args.conf = resolve_conf(model_config.get('species_mapping'), args.conf)
settings = resolve_merge_settings(model_config, args.song_gap, args.freq_iou, args.merge, args.phrase_gap)
song_gap = settings['song_gap_threshold']
diff --git a/src/inference/detect_birds.py b/src/inference/detect_birds.py
index d38b470..c81075e 100755
--- a/src/inference/detect_birds.py
+++ b/src/inference/detect_birds.py
@@ -356,6 +356,12 @@ def reconstruct_songs(
# (call labels) 0.581 -> 0.531. The Hawaii and Southern-Sierra-Nevada models
# are trained on and used for call labels, so they keep call-level boxes.
FUSION_DEFAULT_PHRASE_GAP = {JUST_BIRD_SPECIES_MAPPING: 0.05}
+# Default confidence threshold per mapping for direct use. Just-Bird: the best
+# IoU 0.5 micro F1 is at 0.13 to 0.14 on Hawaii val and 0.16 on Western-US val;
+# 0.15 is within 0.005 of the best there and on the Southern Sierra Nevada
+# hold-out (best at 0.08 to 0.09, not used for tuning).
+DEFAULT_CONF = 0.2
+MAPPING_DEFAULT_CONF = {JUST_BIRD_SPECIES_MAPPING: 0.15}
def resolve_merge_method(species_mapping: Optional[str], requested: Optional[str]) -> str:
@@ -368,6 +374,14 @@ def resolve_merge_method(species_mapping: Optional[str], requested: Optional[str
return requested
+def resolve_conf(species_mapping: Optional[str], requested: Optional[float] = None) -> float:
+ """Confidence threshold: the requested value, else the mapping's default
+ (MAPPING_DEFAULT_CONF, DEFAULT_CONF for other mappings)."""
+ if requested is None:
+ requested = MAPPING_DEFAULT_CONF.get(species_mapping, DEFAULT_CONF)
+ return float(requested)
+
+
def resolve_phrase_gap(
species_mapping: Optional[str],
merge_method: str,
@@ -1934,12 +1948,11 @@ def main():
)
)
- # the default value should work perfectly
parser.add_argument(
'--conf',
type=float,
- default=0.2,
- help='Confidence threshold for detections'
+ default=None,
+ help='Confidence threshold for detections (default: 0.15 for Just-Bird, 0.2 for the other models)'
)
# this value can be further explored
@@ -2029,6 +2042,7 @@ def main():
)
args = parser.parse_args()
+ args.conf = resolve_conf(args.species_mapping, args.conf)
if args.no_merge and args.output_format != ['json-with-algorithm-metadata']:
print(
diff --git a/src/inference/onnx_export.py b/src/inference/onnx_export.py
index fa06137..db33fbc 100644
--- a/src/inference/onnx_export.py
+++ b/src/inference/onnx_export.py
@@ -22,8 +22,9 @@
1.5 s grid is complete. Detections that fall only in
that silence are removed.
Input ``conf`` float32, shape ``(1,)``
- Confidence threshold. Required. Suggested default 0.18
- (also stored in ONNX metadata as ``default_conf``).
+ Confidence threshold. Required. Suggested default 0.15
+ for Just-Bird, 0.2 for the other models (also stored in
+ ONNX metadata as ``default_conf``).
Input ``song_gap`` float32, shape ``(1,)``
Required. Song-merge graphs: max gap in seconds between
detections to merge into the same song, suggested 0.1.
@@ -151,7 +152,8 @@
MAX_FREQ = 15000
MIN_FREQ = 50
-DEFAULT_CONF = 0.18
+# The suggested conf comes from detect_birds.resolve_conf (0.15 for Just-Bird,
+# 0.2 otherwise).
DEFAULT_NMS_IOU = 0.7
DEFAULT_SONG_GAP = 0.1
# Fusion graphs: the song_gap input is the phrase gap (0 = off); its default
@@ -1942,11 +1944,11 @@ def main():
parser.add_argument(
"--conf",
type=float,
- default=DEFAULT_CONF,
+ default=None,
help=(
"Suggested confidence threshold stored in ONNX metadata "
- f"(default: {DEFAULT_CONF}). Hosts must pass a conf tensor on "
- "every run. This value is the usual starting point."
+ "(default: 0.15 for Just-Bird, 0.2 for the other models). Hosts must "
+ "pass a conf tensor on every run. This value is the usual starting point."
),
)
parser.add_argument(
@@ -2028,10 +2030,14 @@ def main():
try:
from inference.detect_birds import (
- resolve_freq_iou_threshold, resolve_merge_method, resolve_phrase_gap,
+ resolve_conf, resolve_freq_iou_threshold, resolve_merge_method, resolve_phrase_gap,
)
except ImportError:
- from detect_birds import resolve_freq_iou_threshold, resolve_merge_method, resolve_phrase_gap
+ from detect_birds import (
+ resolve_conf, resolve_freq_iou_threshold, resolve_merge_method, resolve_phrase_gap,
+ )
+
+ args.conf = resolve_conf(mapping_name, args.conf)
resolved_freq_iou = resolve_freq_iou_threshold(mapping_name, None)
merge_method = resolve_merge_method(mapping_name, args.merge)
diff --git a/src/streamlit/app.py b/src/streamlit/app.py
index 794f02e..25d4918 100644
--- a/src/streamlit/app.py
+++ b/src/streamlit/app.py
@@ -45,7 +45,7 @@
sys.path.insert(0, str(Path(__file__).parent))
import config
-from inference.detect_birds import BirdCallDetector, resolve_merge_method, resolve_phrase_gap
+from inference.detect_birds import BirdCallDetector, resolve_conf, resolve_merge_method, resolve_phrase_gap
from inference.utils import pcen_inference
from inference.utils.raven_export import build_raven_rows
from inference.utils.xeno_canto_export import build_xeno_canto_json
@@ -849,14 +849,14 @@ def main():
"Confidence Threshold",
min_value=0.01,
max_value=0.8,
- value=0.18,
+ value=resolve_conf(species_mapping_name),
step=0.01,
format="%.2f",
help="""
Minimum confidence score for individual detections.\\
Decrease to retrieve more detections and therefore emphasize Recall.\\
Increase to retrieve less detections and therefore emphasize Precision.\\
- Recommended: 0.25
+ Default: 0.15 for Just-Bird, 0.2 for the other models.
"""
)
diff --git a/tests/java/test_model.java b/tests/java/test_model.java
index 0b78737..9963562 100644
--- a/tests/java/test_model.java
+++ b/tests/java/test_model.java
@@ -42,7 +42,7 @@
* java -cp tests/java:tests/java/onnxruntime.jar test_model
* java -cp tests/java:tests/java/onnxruntime.jar test_model tests/test.wav
* java -cp tests/java:tests/java/onnxruntime.jar test_model audio.wav model.onnx out.txt
- * java -cp tests/java:tests/java/onnxruntime.jar test_model audio.wav model.onnx out.txt 0.18 0.1
+ * java -cp tests/java:tests/java/onnxruntime.jar test_model audio.wav model.onnx out.txt 0.15 0.05
*
* Args: [wav] [onnx] [out.txt] [conf] [song_gap]
* Defaults: tests/test.wav, docs/models/Just-Bird_fp32.onnx,
@@ -69,7 +69,7 @@ public static void main(String[] args) throws Exception {
OrtEnvironment env = OrtEnvironment.getEnvironment();
try (OrtSession session = env.createSession(onnx.toString())) {
Map
meta = session.getMetadata().getCustomMetadata();
- float conf = floatArg(args, 3, meta, "default_conf", 0.18f);
+ float conf = floatArg(args, 3, meta, "default_conf", 0.15f);
float gap = floatArg(args, 4, meta, "default_song_gap", 0.1f);
Map names = parseNames(meta.get("names"));