diff --git a/benchmark-scripts/stream_density.py b/benchmark-scripts/stream_density.py index 7491e58..b8310f9 100644 --- a/benchmark-scripts/stream_density.py +++ b/benchmark-scripts/stream_density.py @@ -283,6 +283,43 @@ def build_per_stream_camera_meta(stream_fps_dict, env_vars=None): per_stream_meta[stream_name] = meta return per_stream_meta + +def get_camera_config_about(env_vars=None): + """Load the optional scenario description from the selected camera config.""" + env_vars = env_vars if env_vars is not None else os.environ + camera_stream = env_vars.get(CAMERA_STREAM_KEY, "camera_to_workload.json") + config_path = camera_stream if os.path.isabs(camera_stream) else os.path.normpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "configs", camera_stream) + ) + try: + with open(config_path, "r", encoding="utf-8") as f: + about = json.load(f).get("_about", {}) + return about if isinstance(about, dict) else {} + except (IOError, ValueError): + return {} + + +def print_camera_config_about(env_vars=None): + """Print the selected camera config's scenario description, if present.""" + env_vars = env_vars if env_vars is not None else os.environ + about = get_camera_config_about(env_vars) + about_fields = ( + ("name", "Scenario"), + ("summary", "Summary"), + ("cameras", "Cameras"), + ("why_run_it", "Why run it"), + ) + if not any(about.get(key) for key, _ in about_fields): + return + + camera_stream = env_vars.get(CAMERA_STREAM_KEY, "camera_to_workload.json") + print(f"Camera config _about ({camera_stream}):") + for key, label in about_fields: + value = about.get(key) + if value: + print(f"{label}: {value}") + + def get_mean_target_fps(stream_target_fps, fallback_target_fps): target_fps_values = [float(v) for v in stream_target_fps.values() if float(v) > 0] return statistics.mean(target_fps_values) if target_fps_values else float(fallback_target_fps) @@ -771,7 +808,8 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, init_duration_seconds=None, stream_samples=None, actual_measurement_seconds=None, - sample_counts=None): + sample_counts=None, + about=None): """Print the per-stream stream density result summary.""" stream_meta = stream_meta or {} stream_samples = stream_samples or {} @@ -791,6 +829,19 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, for name in stream_fps_dict ) + about = about or {} + about_fields = ( + ("name", "Scenario"), + ("summary", "Summary"), + ("cameras", "Cameras"), + ("why_run_it", "Why run it"), + ) + if any(about.get(key) for key, _ in about_fields): + print("") + for key, label in about_fields: + value = about.get(key) + if value: + print(f"{label}: {value}") print("") print("Stream density result") print("-" * 47) @@ -1168,7 +1219,8 @@ def run_pipeline_iterations( consecutive_pass_windows, consecutive_fail_windows, pass_tolerance_ratio, stream_camera_meta, INIT_DURATION, stream_samples_dict, - actual_measurement_seconds, sample_counts) + actual_measurement_seconds, sample_counts, + about=get_camera_config_about(env_vars)) else: increments = 0 print( @@ -1198,7 +1250,8 @@ def run_pipeline_iterations( consecutive_pass_windows, consecutive_fail_windows, pass_tolerance_ratio, stream_camera_meta, INIT_DURATION, stream_samples_dict, - actual_measurement_seconds, sample_counts) + actual_measurement_seconds, sample_counts, + about=get_camera_config_about(env_vars)) meet_target_fps = False break else: @@ -1266,6 +1319,7 @@ def run_stream_density(env_vars, compose_files, target_fps_list, validate_and_setup_env(env_vars, target_fps_list) results_dir = env_vars[RESULTS_DIR_KEY] log_file_path = os.path.join(results_dir, 'stream_density.log') + print_camera_config_about(env_vars) orig_stdout = sys.stdout orig_stderr = sys.stderr try: @@ -1273,6 +1327,7 @@ def run_stream_density(env_vars, compose_files, target_fps_list, logger.reconfigure(line_buffering=True, write_through=True) sys.stdout = logger sys.stderr = logger + print_camera_config_about(env_vars) # loop through the target_fps list and find out the stream density: for target_fps, container_name in zip( diff --git a/benchmark-scripts/stream_density_test.py b/benchmark-scripts/stream_density_test.py index 98a9a3f..010fbb0 100644 --- a/benchmark-scripts/stream_density_test.py +++ b/benchmark-scripts/stream_density_test.py @@ -94,7 +94,10 @@ def _run_with_manifest(self, cameras, manifest_streams, stream_count, func): config_path = os.path.join(tmpdir, 'cameras.json') manifest_path = os.path.join(tmpdir, 'pipeline_streams.json') with open(config_path, 'w') as f: - json.dump({'lane_config': {'cameras': cameras}}, f) + json.dump({ + '_about': {'name': 'Test scenario'}, + 'lane_config': {'cameras': cameras}, + }, f) with open(manifest_path, 'w') as f: json.dump({'streams': manifest_streams}, f) stream_fps_dict = {f'pipeline_stream{i}': 10.0 for i in range(stream_count)} @@ -142,6 +145,44 @@ def test_camera_meta_handles_multi_branch_and_skipped_cameras(self): self.assertEqual([targets[f'pipeline_stream{i}'] for i in range(6)], [15.0, 5.0, 5.0, 15.0, 5.0, 5.0]) + def test_get_camera_config_about_reads_selected_config(self): + about = { + 'name': 'Test scenario', + 'summary': 'Test summary', + 'cameras': 'One camera at 15 fps', + 'why_run_it': 'Test purpose', + } + with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file: + json.dump({'_about': about, 'lane_config': {'cameras': []}}, temp_file) + temp_file.flush() + config_path = temp_file.name + + try: + self.assertEqual( + stream_density.get_camera_config_about({'CAMERA_STREAM': config_path}), + about) + finally: + os.remove(config_path) + + def test_print_camera_config_about_shows_selected_config_and_fields(self): + about = { + 'name': 'Test scenario', + 'summary': 'Test summary', + 'cameras': 'One camera at 15 fps', + 'why_run_it': 'Test purpose', + } + with patch('stream_density.get_camera_config_about', return_value=about), \ + patch('builtins.print') as print_line: + stream_density.print_camera_config_about( + {'CAMERA_STREAM': 'camera_to_workload_test.json'}) + + self.assertEqual(print_line.call_args_list[0].args[0], + 'Camera config _about (camera_to_workload_test.json):') + self.assertEqual( + [call.args[0] for call in print_line.call_args_list[1:]], + ['Scenario: Test scenario', 'Summary: Test summary', + 'Cameras: One camera at 15 fps', 'Why run it: Test purpose']) + def test_manifest_log_count_mismatch_raises(self): cameras = [{'camera_id': 'cam1', 'fps': 15}] manifest = self._manifest([(0, 0), (1, 0)]) @@ -479,9 +520,20 @@ def test_print_stream_density_report_summary(self): 18, stream_fps, targets, pass_marks, 100, 2, 2, 0.95, meta, 10, samples, 103.4, {'pipeline_stream0_lane1_gst.log': 100, - 'pipeline_stream1_lane1_gst.log': 103}) + 'pipeline_stream1_lane1_gst.log': 103}, + { + 'name': 'Example lane', + 'summary': 'Detects and classifies products', + 'cameras': 'One camera at 15 fps', + 'why_run_it': 'Compares complete product scanning performance', + }) report = output.getvalue() + self.assertLess(report.index('Scenario: Example lane'), + report.index('Stream density result')) + self.assertIn('Summary: Detects and classifies products', report) + self.assertIn('Cameras: One camera at 15 fps', report) + self.assertIn('Why run it: Compares complete product scanning performance', report) self.assertIn('Use case density (lanes) 18', report) self.assertIn('Stream density (streams) 2', report) self.assertIn('Warmup period 10 s', report) @@ -723,6 +775,7 @@ def test_pipeline_iterations( { "env_vars": { "INIT_DURATION": "10", + "PIPELINE_INC": "1", "CONSECUTIVE_FAIL_WINDOWS": "1", }, "compose_files": ["docker-compose.yml"], @@ -733,6 +786,7 @@ def test_pipeline_iterations( "expected_status": stream_density.STATUS_FAIL, "expected_streams_sustained": 1, "calculate_fps_side_effect": [ + (15.0, 15.0, {"pipeline_stream0": 15.0}, {}), (10.0, 10.0, {"pipeline_stream0": 10.0}, {}), (10.0, 10.0, {"pipeline_stream0": 10.0}, {}), ] @@ -774,11 +828,18 @@ def test_pipeline_iterations( "measurement_window_result", ({}, {}, 100.0, {'pipeline_stream0_lane_gst.log': 100}, True)) - num_pipelines, status, streams_sustained = ( - stream_density.run_pipeline_iterations( - env_vars, compose_files, results_dir, - container_name, target_fps) - ) + about = { + 'name': 'Test lane', + 'summary': 'Runs a test workload', + } + with patch('stream_density.get_camera_config_about', + return_value=about), \ + patch('stream_density.print_stream_density_report') as report: + num_pipelines, status, streams_sustained = ( + stream_density.run_pipeline_iterations( + env_vars, compose_files, results_dir, + container_name, target_fps) + ) self.assertEqual( num_pipelines, test_case["expected_num_pipelines"]) @@ -790,6 +851,9 @@ def test_pipeline_iterations( if "expected_window_calls" in test_case: self.assertEqual(mock_collect_window.call_count, test_case["expected_window_calls"]) + if i < 2: + report.assert_called_once() + self.assertEqual(report.call_args.kwargs['about'], about) def _run_iterations_with_windows(self, window_results, fps_responses): env_vars = {'INIT_DURATION': '0', 'PIPELINE_INC': '1',