feat(stream-density)!: judge pass/fail using time-weighted FPS

benchmark-scripts/benchmark.py

@@ -73,9 +73,16 @@ def parse_args(print=False):
                             help='time in seconds, not needed when ' +
                                  '--target_fps is specified')  
 
-    parser.add_argument('--init_duration', type=int, default=20,
+    parser.add_argument('--init_duration', type=int, default=10,
                         help='initial time in seconds before ' +
                              'starting metric data collection')
+    parser.add_argument('--measurement_window_seconds', type=int, default=100,
+                        help='duration in seconds to collect FPS samples ' +
+                             'after INIT_DURATION for the stream density ' +
+                             'pass/fail decision; minimum 100, because ' +
+                             'each pipeline log needs 100 samples at about ' +
+                             'one per second and collection extends to at ' +
+                             'most twice this duration')
     # TODO: change target_device to an env variable in docker compose
     try:
         default_target_device = resolve_target_device_default('CPU')
@@ -103,9 +110,10 @@ def parse_args(print=False):
         parser.print_help()
         return
     args = parser.parse_args()
-    if args.density_increment and not args.target_fps:
+    if args.density_increment and not args.target_fps and not args.container_names:
         parser.error(
-            '--density_increment needs to have --target_fps be specified')
+            '--density_increment needs --target_fps or --container_names '
+            'to be specified')
     if args.compose_file is None:
         parser.error(
             '--compose_file is empty, please provide compose files')
@@ -153,6 +161,37 @@ def docker_compose_containers(command, compose_files=[], compose_pre_args="",
 
 
 
+def print_stream_density_result(status, num_pipelines, streams_sustained,
+                                results_dir):
+    if status == stream_density.STATUS_PASS:
+        print("Result: Pass")
+        print(
+            f"Stream density completed: use case density {num_pipelines} lanes, "
+            f"stream density {streams_sustained} streams. "
+            f"For the detailed report, see {results_dir}/stream_density.log")
+    elif status == stream_density.STATUS_INCONCLUSIVE:
+        print("Result: Inconclusive")
+        print(
+            f"Measurement intervals did not collect "
+            f"{stream_density.MIN_SAMPLES_PER_STREAM} samples from every "
+            f"pipeline log, so they were not used for a pass/fail decision.")
+        if num_pipelines:
+            print(
+                f"Last passing density: use case density {num_pipelines} lanes, "
+                f"stream density {streams_sustained} streams, not confirmed by "
+                f"the run acceptance criterion.")
+        else:
+            print("No measurement interval had passed.")
+        print(
+            f"Per-log sample counts are in {results_dir}/stream_density.log")
+    else:
+        print("Result: Fail")
+        print(
+            "Stream density did not meet the target. Check "
+            f"{results_dir}/stream_density.log and "
+            f"{results_dir}/gst-launch*.log for further analysis.")
+
+
 def main():
     '''
     runs benchmarking using docker compose for the specified pipeline
@@ -165,6 +204,18 @@ def main():
         if my_args.container_names else []
     )
 
+    # Distinguishes a user-supplied target from the synthesized fallback so
+    # the fallback can never be treated as an explicit TARGET_FPS override.
+    explicit_target_fps = bool(target_fps_list)
+
+    # With no explicit target FPS, container names alone drive stream
+    # density; per-camera targetFps/fps (or env TARGET_FPS) resolves the
+    # actual per-stream targets, with DEFAULT_TARGET_FPS as final fallback.
+    if not target_fps_list and container_names_list:
+        target_fps_list = [
+            stream_density.DEFAULT_TARGET_FPS
+        ] * len(container_names_list)
+
     if (len(target_fps_list) > 1
             and len(target_fps_list) != len(container_names_list)):
         raise ValueError(
@@ -203,39 +254,38 @@ def main():
     retail_use_case_root = os.path.abspath(my_args.retail_use_case_root)
     env_vars["RETAIL_USE_CASE_ROOT"] = retail_use_case_root
     env_vars["VLM_WORKLOAD_ENABLED"] = str(os.getenv("LP_VLM_WORKLOAD_ENABLED"))
+    env_vars["MEASUREMENT_WINDOW_SECONDS"] = str(my_args.measurement_window_seconds)
+    env_vars["INIT_DURATION"] = str(my_args.init_duration)
     if my_args.density_increment:
         env_vars["PIPELINE_INC"] = str(my_args.density_increment)
     if len(target_fps_list) > 1 and container_names_list:
         # stream density for multiple target FPS values and containers
         print('starting stream density for multiple running pipelines...')
         results = stream_density.run_stream_density(env_vars, compose_files,
                                                     target_fps_list,
-                                                    container_names_list)
+                                                    container_names_list,
+                                                    explicit_target_fps)
         for result in results:
-            target_fps, container_name, num_pipelines, met_fps = result
-            print(
-                f"Completed stream density for target FPS: {target_fps} in "
-                f"container: {container_name}. "
-                f"Max pipelines: {num_pipelines}, "
-                f"Met target FPS? {met_fps}")
+            target_fps, container_name, num_pipelines, status, streams_sustained = result
+            print_stream_density_result(
+                status, num_pipelines, streams_sustained, results_dir)
     elif len(target_fps_list) == 1:
         # single target_fps stream density mode:
         print('starting stream density...')
-        env_vars["TARGET_FPS"] = str(target_fps_list[0])
+        if explicit_target_fps:
+            env_vars["TARGET_FPS"] = str(target_fps_list[0])
         if my_args.density_increment:
             env_vars["PIPELINE_INC"] = str(my_args.density_increment)
-        env_vars["INIT_DURATION"] = str(my_args.init_duration)
         # use a default name since there is no
         # --container_names provided in this case
         container_name = (container_names_list[0]
                           if container_names_list else "default_container")
         results = stream_density.run_stream_density(
-            env_vars, compose_files, [target_fps_list[0]], [container_name])
-        target_fps, container_name, num_pipelines, met_fps = results[0]
-        print(
-            f"Max number of pipelines in stream density found for target "
-            f"FPS = {target_fps} is {num_pipelines}. "
-            f"Met target FPS? {met_fps}")
+            env_vars, compose_files, [target_fps_list[0]], [container_name],
+            explicit_target_fps)
+        target_fps, container_name, num_pipelines, status, streams_sustained = results[0]
+        print_stream_density_result(
+            status, num_pipelines, streams_sustained, results_dir)
     else:
         # regular --pipelines mode:
         stream_density.clean_up_pipeline_logs(results_dir)

benchmark-scripts/benchmark_test.py

@@ -8,7 +8,10 @@
 import subprocess  # nosec B404
 import unittest
 import benchmark
+import stream_density
+import io
 import os
+from contextlib import redirect_stdout
 
 
 class Testing(unittest.TestCase):
@@ -55,5 +58,38 @@ def test_docker_compose_containers_fail(self):
         mock_popen.communicate.assert_called_once_with()
         mock_returncode.assert_called()
 
+    def _result_output(self, status, num_pipelines, streams):
+        output = io.StringIO()
+        with redirect_stdout(output):
+            benchmark.print_stream_density_result(
+                status, num_pipelines, streams, '/results')
+        return output.getvalue()
+
+    def test_print_stream_density_result_pass(self):
+        output = self._result_output(stream_density.STATUS_PASS, 11, 22)
+        self.assertIn('Result: Pass', output)
+        self.assertIn(
+            'use case density 11 lanes, stream density 22 streams', output)
+        self.assertNotIn('sustained', output)
+
+    def test_print_stream_density_result_fail(self):
+        output = self._result_output(stream_density.STATUS_FAIL, 1, 2)
+        self.assertIn('Result: Fail', output)
+        self.assertIn('did not meet the target', output)
+
+    def test_print_stream_density_result_inconclusive(self):
+        output = self._result_output(stream_density.STATUS_INCONCLUSIVE, 10, 20)
+        self.assertIn('Result: Inconclusive', output)
+        self.assertNotIn('Result: Fail', output)
+        self.assertIn(
+            'Last passing density: use case density 10 lanes, '
+            'stream density 20 streams', output)
+
+    def test_print_stream_density_result_inconclusive_without_pass(self):
+        output = self._result_output(stream_density.STATUS_INCONCLUSIVE, 0, 0)
+        self.assertIn('Result: Inconclusive', output)
+        self.assertIn('No measurement interval had passed.', output)
+        self.assertNotIn('Last passing density', output)
+
 if __name__ == '__main__':
     unittest.main()

benchmark-scripts/consolidate_multiple_run_of_metrics.py

@@ -607,8 +607,9 @@ def extract_data(self, log_file_path):
         cam = re.findall(r'\d+', os.path.basename(log_file_path))
         camera_key = "Camera_{} {}".format(cam[0], AVG_FPS_CONSTANT)
         with open(log_file_path) as f:
-            for line in f:
-              average_fps_list.append(float(line))
+            for row in csv.reader(f):
+                if row and row != ['fps', 'duration_seconds']:
+                    average_fps_list.append(float(row[0]))
 
         if len(average_fps_list) > 0:
             camera_fps[camera_key] = round(mean(average_fps_list), 2)
@@ -841,7 +842,7 @@ def return_blank(self):
 
 KPIExtractor_OPTION = {"meta_summary.txt":MetaExtractor,
                        "camera":FPSExtractor,
-                       "pipeline":PIPELINEFPSExtractor,
+                       r"^pipeline.*\.log$":PIPELINEFPSExtractor,
                        r"(?:^r).*\.jsonl$": PIPELINLastModifiedExtractor,
                        "gst-launch":PipelineLatencyExtractor,
                        "cpu_usage.log":CPUUsageExtractor,