From 8b2b39ed7469a66995c5e7b02707af847ecaaa93 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 24 Sep 2026 09:41:11 +0530 Subject: [PATCH 01/12] Update benchmark.py --- benchmark-scripts/benchmark.py | 66 +++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/benchmark-scripts/benchmark.py b/benchmark-scripts/benchmark.py index 52568cb..7c499be 100644 --- a/benchmark-scripts/benchmark.py +++ b/benchmark-scripts/benchmark.py @@ -76,6 +76,10 @@ def parse_args(print=False): parser.add_argument('--init_duration', type=int, default=20, help='initial time in seconds before ' + 'starting metric data collection') + parser.add_argument('--measurement_window_seconds', type=int, default=30, + help='duration in seconds to collect FPS samples ' + + 'after INIT_DURATION for the stream density ' + + 'pass/fail decision') # TODO: change target_device to an env variable in docker compose try: default_target_device = resolve_target_device_default('CPU') @@ -103,9 +107,10 @@ def parse_args(print=False): parser.print_help() return args = parser.parse_args() - if args.density_increment and not args.target_fps: + if args.density_increment and not args.target_fps and not args.container_names: parser.error( - '--density_increment needs to have --target_fps be specified') + '--density_increment needs --target_fps or --container_names ' + 'to be specified') if args.compose_file is None: parser.error( '--compose_file is empty, please provide compose files') @@ -165,6 +170,18 @@ def main(): if my_args.container_names else [] ) + # Distinguishes a user-supplied target from the synthesized fallback so + # the fallback can never be treated as an explicit TARGET_FPS override. + explicit_target_fps = bool(target_fps_list) + + # With no explicit target FPS, container names alone drive stream + # density; per-camera targetFps/fps (or env TARGET_FPS) resolves the + # actual per-stream targets, with DEFAULT_TARGET_FPS as final fallback. + if not target_fps_list and container_names_list: + target_fps_list = [ + stream_density.DEFAULT_TARGET_FPS + ] * len(container_names_list) + if (len(target_fps_list) > 1 and len(target_fps_list) != len(container_names_list)): raise ValueError( @@ -203,6 +220,7 @@ def main(): retail_use_case_root = os.path.abspath(my_args.retail_use_case_root) env_vars["RETAIL_USE_CASE_ROOT"] = retail_use_case_root env_vars["VLM_WORKLOAD_ENABLED"] = str(os.getenv("LP_VLM_WORKLOAD_ENABLED")) + env_vars["MEASUREMENT_WINDOW_SECONDS"] = str(my_args.measurement_window_seconds) if my_args.density_increment: env_vars["PIPELINE_INC"] = str(my_args.density_increment) if len(target_fps_list) > 1 and container_names_list: @@ -210,18 +228,26 @@ def main(): print('starting stream density for multiple running pipelines...') results = stream_density.run_stream_density(env_vars, compose_files, target_fps_list, - container_names_list) + container_names_list, + explicit_target_fps) for result in results: - target_fps, container_name, num_pipelines, met_fps = result - print( - f"Completed stream density for target FPS: {target_fps} in " - f"container: {container_name}. " - f"Max pipelines: {num_pipelines}, " - f"Met target FPS? {met_fps}") + target_fps, container_name, num_pipelines, met_fps, streams_sustained = result + print(f"Result: {'Pass' if met_fps else 'Fail'}") + if met_fps: + print( + f"Stream density completed: {num_pipelines} pipelines, " + f"{streams_sustained} streams sustained. " + f"For the detailed report, see {results_dir}/stream_density.log") + else: + print( + f"Stream density did not meet the target. Check " + f"{results_dir}/stream_density.log and " + f"{results_dir}/gst-launch*.log for further analysis.") elif len(target_fps_list) == 1: # single target_fps stream density mode: print('starting stream density...') - env_vars["TARGET_FPS"] = str(target_fps_list[0]) + if explicit_target_fps: + env_vars["TARGET_FPS"] = str(target_fps_list[0]) if my_args.density_increment: env_vars["PIPELINE_INC"] = str(my_args.density_increment) env_vars["INIT_DURATION"] = str(my_args.init_duration) @@ -230,12 +256,20 @@ def main(): container_name = (container_names_list[0] if container_names_list else "default_container") results = stream_density.run_stream_density( - env_vars, compose_files, [target_fps_list[0]], [container_name]) - target_fps, container_name, num_pipelines, met_fps = results[0] - print( - f"Max number of pipelines in stream density found for target " - f"FPS = {target_fps} is {num_pipelines}. " - f"Met target FPS? {met_fps}") + env_vars, compose_files, [target_fps_list[0]], [container_name], + explicit_target_fps) + target_fps, container_name, num_pipelines, met_fps, streams_sustained = results[0] + print(f"Result: {'Pass' if met_fps else 'Fail'}") + if met_fps: + print( + f"Stream density completed: {num_pipelines} pipelines, " + f"{streams_sustained} streams sustained. " + f"For the detailed report, see {results_dir}/stream_density.log") + else: + print( + "Stream density did not meet the target. Check " + f"{results_dir}/stream_density.log and " + f"{results_dir}/gst-launch*.log for further analysis.") else: # regular --pipelines mode: stream_density.clean_up_pipeline_logs(results_dir) From 0803ddd972109eeb66d2e38e70a20af6f95c714e Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 24 Sep 2026 09:42:57 +0530 Subject: [PATCH 02/12] Update stream_density.py --- benchmark-scripts/stream_density.py | 522 ++++++++++++++++++---------- 1 file changed, 343 insertions(+), 179 deletions(-) diff --git a/benchmark-scripts/stream_density.py b/benchmark-scripts/stream_density.py index 3fd45b8..133b29b 100644 --- a/benchmark-scripts/stream_density.py +++ b/benchmark-scripts/stream_density.py @@ -28,10 +28,10 @@ CONSECUTIVE_PASS_WINDOWS_KEY = "CONSECUTIVE_PASS_WINDOWS" DEFAULT_CONSECUTIVE_FAIL_WINDOWS = 2 DEFAULT_CONSECUTIVE_PASS_WINDOWS = 2 -HYSTERESIS_FPS_KEY = "HYSTERESIS_FPS" PASS_TOLERANCE_RATIO_KEY = "PASS_TOLERANCE_RATIO" -DEFAULT_HYSTERESIS_FPS = 0.10 DEFAULT_PASS_TOLERANCE_RATIO = 0.95 +MEASUREMENT_WINDOW_SECONDS_KEY = "MEASUREMENT_WINDOW_SECONDS" +DEFAULT_MEASUREMENT_WINDOW_SECONDS = 30 CAMERA_STREAM_KEY = "CAMERA_STREAM" @@ -39,7 +39,11 @@ class ArgumentError(Exception): pass def build_per_stream_target_fps(stream_fps_dict, default_target_fps): - """Build stream-level FPS targets from camera config.""" + """Resolve per-stream FPS targets. + + Precedence: env TARGET_FPS (applies to every camera) > per-camera + targetFps > per-camera fps > default_target_fps fallback. + """ # Get camera config path camera_stream = os.getenv(CAMERA_STREAM_KEY, "camera_to_workload.json") config_path = camera_stream if os.path.isabs(camera_stream) else os.path.normpath( @@ -66,12 +70,22 @@ def build_per_stream_target_fps(stream_fps_dict, default_target_fps): total_cameras = len(cameras) for idx, cam in enumerate(cameras): if isinstance(cam, dict): + resolved_fps = None try: - fps_value = float(cam.get("targetFps", 0)) - if fps_value > 0: - stream_idx_to_target_fps[idx] = fps_value + target_fps_value = float(cam.get("targetFps", 0)) + if target_fps_value > 0: + resolved_fps = target_fps_value except (TypeError, ValueError): pass + if resolved_fps is None: + try: + fps_value = float(cam.get("fps", 0)) + if fps_value > 0: + resolved_fps = fps_value + except (TypeError, ValueError): + pass + if resolved_fps is not None: + stream_idx_to_target_fps[idx] = resolved_fps cache[cache_key] = (dict(stream_idx_to_target_fps), int(total_cameras)) build_per_stream_target_fps._camera_config_cache = cache except (IOError, ValueError) as e: @@ -89,7 +103,21 @@ def build_per_stream_target_fps(stream_fps_dict, default_target_fps): stream_pattern = re.compile(r"pipeline_stream(\d+)") per_stream_targets = {} + # env TARGET_FPS, when set, overrides every camera's config value. + forced_target_fps = None + env_target_fps = os.getenv(TARGET_FPS_KEY, "").strip() + if env_target_fps: + try: + candidate = float(env_target_fps) + if candidate > 0: + forced_target_fps = candidate + except ValueError: + pass + for stream_name in stream_fps_dict: + if forced_target_fps is not None: + per_stream_targets[stream_name] = forced_target_fps + continue match = stream_pattern.search(stream_name) if match and total_cameras > 0: camera_idx = int(match.group(1)) % total_cameras @@ -102,6 +130,105 @@ def build_per_stream_target_fps(stream_fps_dict, default_target_fps): print(f"INFO: per-stream target FPS map: {per_stream_targets}") return per_stream_targets + +def describe_target_fps_configuration(default_target_fps): + """Describe the configured target FPS values and their sources.""" + env_target_fps = os.getenv(TARGET_FPS_KEY, "").strip() + if env_target_fps: + try: + forced_target_fps = float(env_target_fps) + if forced_target_fps > 0: + return f"TARGET_FPS environment override: {forced_target_fps:.2f} FPS for every stream" + except ValueError: + pass + + camera_stream = os.getenv(CAMERA_STREAM_KEY, "camera_to_workload.json") + config_path = camera_stream if os.path.isabs(camera_stream) else os.path.normpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "configs", camera_stream) + ) + try: + with open(config_path, "r", encoding="utf-8") as f: + cameras = json.load(f).get("lane_config", {}).get("cameras", []) + except (IOError, ValueError): + return f"default target FPS: {default_target_fps:.2f} FPS" + + descriptions = [] + for index, camera in enumerate(cameras): + if not isinstance(camera, dict): + continue + camera_id = camera.get("camera_id", f"camera{index}") + if camera.get("targetFps", 0): + try: + target_fps = float(camera["targetFps"]) + if target_fps > 0: + descriptions.append(f"{camera_id}={target_fps:.2f} FPS from targetFps") + continue + except (TypeError, ValueError): + pass + if camera.get("fps", 0): + try: + target_fps = float(camera["fps"]) + if target_fps > 0: + descriptions.append(f"{camera_id}={target_fps:.2f} FPS from fps") + continue + except (TypeError, ValueError): + pass + descriptions.append(f"{camera_id}={default_target_fps:.2f} FPS from default") + + if descriptions: + return "per-camera target FPS configuration: " + ", ".join(descriptions) + return f"default target FPS: {default_target_fps:.2f} FPS" + +def build_per_stream_camera_meta(stream_fps_dict): + """Build per-stream camera_id and workload labels from camera config.""" + camera_stream = os.getenv(CAMERA_STREAM_KEY, "camera_to_workload.json") + config_path = camera_stream if os.path.isabs(camera_stream) else os.path.normpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "configs", camera_stream) + ) + + cache = getattr(build_per_stream_camera_meta, "_camera_meta_cache", {}) + file_mtime = os.path.getmtime(config_path) if os.path.isfile(config_path) else 0 + cache_key = (config_path, file_mtime) + + cached = cache.get(cache_key) + if cached is not None: + idx_to_meta, total_cameras = cached + else: + idx_to_meta = {} + total_cameras = 0 + if os.path.isfile(config_path): + try: + with open(config_path, "r", encoding="utf-8") as f: + cameras = json.load(f).get("lane_config", {}).get("cameras", []) + total_cameras = len(cameras) + for idx, cam in enumerate(cameras): + if isinstance(cam, dict): + camera_id = cam.get("camera_id", f"camera{idx}") + workloads = cam.get("workloads", []) + workload = workloads[0] if isinstance(workloads, list) and workloads else "-" + idx_to_meta[idx] = {"camera": str(camera_id), "workload": str(workload)} + cache[cache_key] = (dict(idx_to_meta), int(total_cameras)) + build_per_stream_camera_meta._camera_meta_cache = cache + except (IOError, ValueError) as e: + print(f"WARN: Failed to load camera config {config_path}: {e}") + cache[cache_key] = ({}, 0) + build_per_stream_camera_meta._camera_meta_cache = cache + else: + cache[cache_key] = ({}, 0) + build_per_stream_camera_meta._camera_meta_cache = cache + + stream_pattern = re.compile(r"pipeline_stream(\d+)") + per_stream_meta = {} + for stream_name in stream_fps_dict: + match = stream_pattern.search(stream_name) + if match and total_cameras > 0: + camera_idx = int(match.group(1)) % total_cameras + meta = idx_to_meta.get(camera_idx, {"camera": stream_name, "workload": "-"}) + else: + meta = {"camera": stream_name, "workload": "-"} + per_stream_meta[stream_name] = meta + return per_stream_meta + def get_mean_target_fps(stream_target_fps, fallback_target_fps): target_fps_values = [float(v) for v in stream_target_fps.values() if float(v) > 0] return statistics.mean(target_fps_values) if target_fps_values else float(fallback_target_fps) @@ -356,61 +483,15 @@ def get_latest_pipeline_logs(num_pipelines, pipeline_log_files): return latest_files -def detect_steady_state(numeric_fps, target_cv_ratio=0.05, rolling_window=20, min_consecutive=3): - """ - Detect steady-state in FPS samples using rolling coefficient of variation (CV). - - Steady-state = period where CV < target_cv_ratio consistently. - - Args: - numeric_fps: List of FPS samples in time order - target_cv_ratio: Coefficient of Variation threshold (lower = more stable) - rolling_window: Window size for rolling CV calculation - min_consecutive: Require N consecutive windows below threshold - - Returns: - Tuple: (steady_state_start_idx, confidence_score, is_detected) - """ - if len(numeric_fps) < rolling_window: - return 0, 0.0, False - - rolling_cvs = [] - for i in range(len(numeric_fps) - rolling_window + 1): - window = numeric_fps[i:i+rolling_window] - mean_val = statistics.mean(window) - if mean_val <= 0: - rolling_cvs.append(float('inf')) - continue - stdev_val = statistics.stdev(window) if len(window) > 1 else 0 - cv = stdev_val / mean_val - rolling_cvs.append(cv) - - # Find first window where CV stays below threshold for min_consecutive windows - for idx in range(len(rolling_cvs) - min_consecutive + 1): - window_cvs = rolling_cvs[idx:idx+min_consecutive] - if all(c < target_cv_ratio for c in window_cvs): - steady_idx = idx - avg_cv = statistics.mean(window_cvs) - confidence = max(0.0, 1.0 - (avg_cv / target_cv_ratio)) - return steady_idx, confidence, True - - return 0, 0.0, False - - -def extract_numeric_fps(file_path, discard_warm_up_ratio=0.30, detect_steady=True): - """ - Extract numeric FPS from log file with warm-up discard and optional steady-state detection. - - Args: - file_path: Path to FPS log file - discard_warm_up_ratio: Fraction of early samples to discard as warm-up (default 30%) - detect_steady: Use variance-based steady-state detection (industry standard) - - Returns: - Tuple: (numeric_fps_filtered, discard_count, steady_idx, is_steady_detected) - """ +def extract_numeric_fps(file_path, start_offset=0, end_offset=None): + """Extract valid numeric FPS samples within an optional byte range.""" with open(file_path, "r") as file: - lines = file.readlines() + if start_offset: + file.seek(start_offset) + if end_offset is not None: + lines = file.read(max(0, end_offset - start_offset)).splitlines() + else: + lines = file.readlines() numeric_fps = [] for line in lines: @@ -422,27 +503,7 @@ def extract_numeric_fps(file_path, discard_warm_up_ratio=0.30, detect_steady=Tru except ValueError: print(f"DEBUG: Skipping non-numeric line '{stripped_line}' in {file_path}") - if not numeric_fps: - return [], 0, 0, False - - # --- Warm-up discard: remove first 30% --- - warm_up_count = max(1, int(len(numeric_fps) * discard_warm_up_ratio)) - fps_after_discard = numeric_fps[warm_up_count:] - - if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": - print(f"DEBUG: Discarded {warm_up_count}/{len(numeric_fps)} warm-up samples ({discard_warm_up_ratio*100}%)") - - # --- Steady-state detection: find where variance stabilizes --- - steady_idx = 0 - is_steady_detected = False - if detect_steady and len(fps_after_discard) >= 20: - steady_idx, confidence, is_detected = detect_steady_state(fps_after_discard) - is_steady_detected = is_detected - if is_detected: - if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": - print(f"DEBUG: Steady-state detected at sample {steady_idx} (confidence={confidence:.2f})") - - return fps_after_discard, warm_up_count, steady_idx, is_steady_detected + return numeric_fps def filter_logs_for_current_timestamp(log_files): @@ -605,10 +666,10 @@ def validate_and_setup_env(env_vars, target_fps_list): raise ArgumentError( 'ERROR: consecutive pass windows should be greater than 0') - if is_env_non_empty(env_vars, HYSTERESIS_FPS_KEY) and float( - env_vars[HYSTERESIS_FPS_KEY]) < 0.0: + if is_env_non_empty(env_vars, MEASUREMENT_WINDOW_SECONDS_KEY) and int( + env_vars[MEASUREMENT_WINDOW_SECONDS_KEY]) <= 0: raise ArgumentError( - 'ERROR: hysteresis FPS should be greater than or equal to 0') + 'ERROR: measurement window seconds should be greater than 0') if is_env_non_empty(env_vars, PASS_TOLERANCE_RATIO_KEY): pass_tolerance_ratio = float(env_vars[PASS_TOLERANCE_RATIO_KEY]) @@ -620,9 +681,109 @@ def validate_and_setup_env(env_vars, target_fps_list): env_vars[INIT_DURATION_KEY] = "120" +def print_stream_density_report(num_pipelines, stream_fps_dict, + stream_target_fps, pass_thresholds, + measurement_window_seconds, + consecutive_pass_windows, + consecutive_fail_windows, + pass_tolerance_ratio, stream_meta=None, + init_duration_seconds=None): + """Print the per-stream stream density result summary.""" + stream_meta = stream_meta or {} + binding_name = None + binding_headroom = None + for name in stream_fps_dict: + pass_mark = pass_thresholds.get(name, 0.0) + measured = stream_fps_dict.get(name, 0.0) + headroom = ((measured - pass_mark) / pass_mark + if pass_mark > 0 else float('inf')) + if binding_headroom is None or headroom < binding_headroom: + binding_headroom = headroom + binding_name = name + + overall_pass = all( + stream_fps_dict[name] >= pass_thresholds.get(name, 0.0) + for name in stream_fps_dict + ) + + print("") + print("Stream density result") + print("-" * 47) + camera_stream_count = sum( + 1 for measured in stream_fps_dict.values() if float(measured) > 0 + ) + print(f"Streams sustained {camera_stream_count}") + if init_duration_seconds is not None: + print( + f"Settle time {init_duration_seconds} s " + f"(INIT_DURATION before measuring)") + print( + f"Measurement window {measurement_window_seconds} s per window; " + f"{consecutive_pass_windows} consecutive windows must agree") + print( + " (each window is measured on its own; the " + "count moves") + print( + f" only after {consecutive_pass_windows} " + f"windows in a row agree - windows are not added together)") + print( + f"Pass mark target x {pass_tolerance_ratio:g} " + f"(pass tolerance ratio)") + print("") + print( + f"{'stream':<18}{'camera':<8}{'workload':<50}{'target':>8}{'pass mark':>11}" + f"{'measured':>10}{'result':>8}") + for name in sorted( + stream_fps_dict, + key=lambda n: int(re.search(r'(\d+)', n).group(1)) if re.search(r'(\d+)', n) else 0): + meta = stream_meta.get(name, {}) + camera = meta.get("camera", name) + workload = meta.get("workload", "-") + target = stream_target_fps.get(name, 0.0) + pass_mark = pass_thresholds.get(name, 0.0) + measured = stream_fps_dict.get(name, 0.0) + result = "pass" if measured >= pass_mark else "fail" + print( + f"{name:<18}{camera:<8}{workload:<50}{target:>8.2f}{pass_mark:>11.2f}" + f"{measured:>10.2f}{result:>8}") + print("") + print( + f"Result: {'Pass' if overall_pass else 'Fail'} - " + f"{camera_stream_count} concurrent streams sustained") + if binding_name is not None: + bt = stream_target_fps.get(binding_name, 0.0) + bp = pass_thresholds.get(binding_name, 0.0) + bm = stream_fps_dict.get(binding_name, 0.0) + binding_camera = stream_meta.get(binding_name, {}).get("camera", binding_name) + binding_workload = stream_meta.get(binding_name, {}).get("workload", "") + target_values = [float(v) for v in stream_target_fps.values()] + uniform_targets = (max(target_values) - min(target_values) < 1e-9) if target_values else True + if uniform_targets: + targets_clause = f"All streams met the {bp:.2f} FPS minimum requirement." + stream_descriptor = "lowest-throughput stream" + else: + targets_clause = "All streams met their individual minimum FPS requirements." + stream_descriptor = "lowest-headroom stream" + workload_clause = f" ({binding_workload})" if binding_workload else "" + absolute_headroom = bm - bp + relative_headroom = binding_headroom * 100 + settle_clause = ( + f"after a {init_duration_seconds} second settle time (INIT_DURATION), " + if init_duration_seconds is not None else "") + print( + f"The run was evaluated {settle_clause}over {consecutive_pass_windows} " + f"consecutive measurement windows of {measurement_window_seconds} " + f"seconds each. " + f"{targets_clause} The {stream_descriptor} was {binding_name} " + f"({binding_camera}{workload_clause}), targeting {bt:.2f} FPS and " + f"measuring {bm:.2f} FPS against its {bp:.2f} FPS minimum, leaving " + f"{absolute_headroom:.2f} FPS " + f"({relative_headroom:.1f}% relative headroom).") + + def run_pipeline_iterations( env_vars, compose_files, results_dir, - container_name, target_fps): + container_name, target_fps, explicit_target_fps=True): ''' runs an iteration of stream density benchmarking for a given container name and target FPS. @@ -653,20 +814,21 @@ def run_pipeline_iterations( DEFAULT_CONSECUTIVE_PASS_WINDOWS, ) ) - hysteresis_fps = float( - env_vars.get( - HYSTERESIS_FPS_KEY, - DEFAULT_HYSTERESIS_FPS, - ) - ) pass_tolerance_ratio = float( env_vars.get( PASS_TOLERANCE_RATIO_KEY, DEFAULT_PASS_TOLERANCE_RATIO, ) ) + measurement_window_seconds = int( + env_vars.get( + MEASUREMENT_WINDOW_SECONDS_KEY, + DEFAULT_MEASUREMENT_WINDOW_SECONDS, + ) + ) fail_window_count = 0 pass_window_count = 0 + streams_sustained = 0 # Measure memory usage of a single pipeline per_pipeline_memory_mb = measure_pipeline_memory( @@ -676,10 +838,14 @@ def run_pipeline_iterations( # clean up any residual pipeline log files before starts: clean_up_pipeline_logs(results_dir) + target_fps_label = describe_target_fps_configuration(target_fps) print( - f"INFO: Stream density TARGET_FPS set for {target_fps} " + f"INFO: Stream density {target_fps_label} " f"with container_name {container_name} " - f"and INIT_DURATION set for {INIT_DURATION} seconds") + f"and INIT_DURATION set for {INIT_DURATION} seconds; " + f"measurement window {measurement_window_seconds} seconds; " + f"required agreement {consecutive_pass_windows} pass / " + f"{consecutive_fail_windows} fail windows") while not meet_target_fps: # --- Memory check before scaling up --- @@ -693,7 +859,7 @@ def run_pipeline_iterations( num_pipelines = num_pipelines - increments if num_pipelines < 1: num_pipelines = 1 - return num_pipelines, False + return num_pipelines, False, streams_sustained # Bring down previous iteration's containers before starting new ones print("Stopping previous containers before scaling...") @@ -724,12 +890,29 @@ def run_pipeline_iterations( num_pipelines = num_pipelines - increments if num_pipelines < 1: num_pipelines = 1 - return num_pipelines, False + return num_pipelines, False, streams_sustained # once we have all non-empty pipeline log files - # we then can calculate the average fps + # capture where each stream log ends so only samples produced + # during the measurement window feed the decision + window_start_offsets = snapshot_stream_log_offsets( + results_dir, container_name) + print( + f"INFO: SWEEPING-START (Measurement Window Start): " + f"{time.strftime('%Y-%m-%dT%H:%M:%S')}") + time.sleep(measurement_window_seconds) + window_end_offsets = snapshot_stream_log_offsets( + results_dir, container_name) + print( + f"INFO: SWEEPING-STOP (Measurement Window Stop): " + f"{time.strftime('%Y-%m-%dT%H:%M:%S')}") # --- Calculate FPS and latency metrics --- total_fps, min_p90_across_streams, stream_fps_dict = calculate_multi_stream_fps( - num_pipelines, results_dir, container_name, env_vars) + num_pipelines, results_dir, container_name, env_vars, + start_offsets=window_start_offsets, + end_offsets=window_end_offsets) + streams_sustained = sum( + 1 for measured in stream_fps_dict.values() if float(measured) > 0 + ) print('container name:', container_name) print('Total FPS:', total_fps) @@ -757,25 +940,14 @@ def run_pipeline_iterations( for name in stream_fps_dict } - fail_thresholds = { - name: pass_thresholds[name] - hysteresis_fps - for name in stream_fps_dict - } - passing_streams = { name: fps for name, fps in stream_fps_dict.items() if fps >= pass_thresholds[name] } - failing_streams = { - name: fps for name, fps in stream_fps_dict.items() - if fps < fail_thresholds[name] - } all_streams_meet_target = len(passing_streams) == len(stream_fps_dict) if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": print('pass_thresholds:', pass_thresholds) - print('fail_thresholds:', fail_thresholds) print('passing_streams:', passing_streams) - print('failing_streams:', failing_streams) if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": print("INFO: All streams meet target" if all_streams_meet_target else "INFO: Not all streams meet target") @@ -819,20 +991,11 @@ def run_pipeline_iterations( ) else: increments = 0 - if failing_streams: - print( - f"⚠️ Below hysteresis fail thresholds in streams: " - f"observed={failing_streams}, fail_thresholds={fail_thresholds}. " - f"Fail window {fail_window_count}/{consecutive_fail_windows}; " - f"holding pipeline count at {num_pipelines} for confirmation." - ) - else: - print( - f"INFO: Streams are below pass thresholds ({pass_thresholds}) but " - f"above fail thresholds ({fail_thresholds}). " - f"Fail window {fail_window_count}/{consecutive_fail_windows}; " - f"holding pipeline count at {num_pipelines} for confirmation." - ) + print( + f"INFO: Streams are below pass thresholds ({pass_thresholds}). " + f"Fail window {fail_window_count}/{consecutive_fail_windows}; " + f"holding pipeline count at {num_pipelines} for confirmation." + ) else: # --- In decrement phase --- if all_streams_meet_target: @@ -849,6 +1012,14 @@ def run_pipeline_iterations( f"{stream_target_fps} is {num_pipelines}" ) increments = 0 + stream_camera_meta = build_per_stream_camera_meta( + stream_fps_dict) + print_stream_density_report( + num_pipelines, stream_fps_dict, stream_target_fps, + pass_thresholds, measurement_window_seconds, + consecutive_pass_windows, consecutive_fail_windows, + pass_tolerance_ratio, stream_camera_meta, + INIT_DURATION) else: increments = 0 print( @@ -871,34 +1042,18 @@ def run_pipeline_iterations( if fail_window_count >= consecutive_fail_windows: increments = -1 fail_window_count = 0 - if failing_streams: - print( - f"decrementing number of pipelines {num_pipelines} by 1 " - f"because streams are below hysteresis fail thresholds. " - f"observed={failing_streams}, fail_thresholds={fail_thresholds}" - ) - else: - print( - f"decrementing number of pipelines {num_pipelines} by 1 " - f"because streams stayed below pass thresholds " - f"({pass_thresholds}) for {consecutive_fail_windows} windows." - ) + print( + f"decrementing number of pipelines {num_pipelines} by 1 " + f"because streams stayed below pass thresholds " + f"({pass_thresholds}) for {consecutive_fail_windows} windows." + ) else: increments = 0 - if failing_streams: - print( - f"INFO: Below fail thresholds in streams. " - f"observed={failing_streams}, fail_thresholds={fail_thresholds}. " - f"Fail window {fail_window_count}/{consecutive_fail_windows}; " - f"holding pipeline count at {num_pipelines} for confirmation." - ) - else: - print( - f"INFO: Below pass thresholds ({pass_thresholds}) but above " - f"fail thresholds ({fail_thresholds}). " - f"Fail window {fail_window_count}/{consecutive_fail_windows}; " - f"holding pipeline count at {num_pipelines} for confirmation." - ) + print( + f"INFO: Below pass thresholds ({pass_thresholds}). " + f"Fail window {fail_window_count}/{consecutive_fail_windows}; " + f"holding pipeline count at {num_pipelines} for confirmation." + ) # --- Update pipeline count --- num_pipelines += increments @@ -910,17 +1065,17 @@ def run_pipeline_iterations( # end of while - print( - f"pipeline iterations done for " - f"container_name: {container_name} " - f"with input target_fps = {target_fps}" - ) + #print( + # f"pipeline iterations done for " + # f"container_name: {container_name} " + # f"with input target_fps = {target_fps}" + #) - return num_pipelines, meet_target_fps + return num_pipelines, meet_target_fps, streams_sustained def run_stream_density(env_vars, compose_files, target_fps_list, - container_names_list): + container_names_list, explicit_target_fps=True): ''' runs stream density using docker compose for the specified target FPS values and the corresponding container names @@ -957,22 +1112,24 @@ def run_stream_density(env_vars, compose_files, target_fps_list, target_fps_list, container_names_list ): print( - f"DEBUG: in for-loop, target_fps={target_fps} " + f"DEBUG: in for-loop, Starting stream density" f"container_name={container_name}") - env_vars[TARGET_FPS_KEY] = str(target_fps) + if explicit_target_fps: + env_vars[TARGET_FPS_KEY] = str(target_fps) env_vars[CONTAINER_NAME_KEY] = container_name # stream density main logic: try: - num_pipelines, meet_target_fps = run_pipeline_iterations( + num_pipelines, meet_target_fps, streams_sustained = run_pipeline_iterations( env_vars, compose_files, results_dir, - container_name, target_fps + container_name, target_fps, explicit_target_fps ) results.append( ( target_fps, container_name, num_pipelines, - meet_target_fps + meet_target_fps, + streams_sustained ) ) finally: @@ -1003,7 +1160,23 @@ def run_stream_density(env_vars, compose_files, target_fps_list, return results -def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_vars=None): +def snapshot_stream_log_offsets(results_dir, container_name): + """Record current end-of-file byte offsets for the run's stream logs.""" + offsets = {} + stream_count = get_pipeline_stream_count() + for idx in range(stream_count): + pattern = os.path.join( + results_dir, f'pipeline_stream{idx}_*_{container_name}.log') + matching = filter_logs_for_current_timestamp(glob.glob(pattern)) + for log_file in matching: + try: + offsets[log_file] = os.path.getsize(log_file) + except OSError: + offsets[log_file] = 0 + return offsets + + +def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_vars=None, start_offsets=None, end_offsets=None): """ Calculates reporting and decision FPS metrics per stream from log files named pipeline_stream*.log. @@ -1027,8 +1200,7 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v # --- Initialize accumulators --- total_fps = 0.0 stream_fps_dict = {} - time.sleep(10) # Ensure logs are fully written - + # --- Loop over all streams --- for idx in range(stream_count): pattern = os.path.join(results_dir, f'pipeline_stream{idx}_*_{container_name}.log') @@ -1056,25 +1228,17 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v for pipeline_file in latest_pipeline_logs: print(f"DEBUG: Processing file: {pipeline_file}") try: - # Extract FPS with warm-up discard and steady-state detection - fps_data, warm_up_discarded, steady_idx, is_steady = extract_numeric_fps( - pipeline_file, discard_warm_up_ratio=0.30, detect_steady=True - ) + fps_data = extract_numeric_fps( + pipeline_file, + start_offset=(start_offsets or {}).get(pipeline_file, 0), + end_offset=(end_offsets or {}).get(pipeline_file)) if not fps_data: print(f"WARN: No numeric FPS entries for {pipeline_file}") continue - - # Use only steady-state portion (or all if steady not detected) - if is_steady: - measurement_fps = fps_data[steady_idx:] - else: - measurement_fps = fps_data - - if not measurement_fps: - print(f"WARN: No steady-state data for {pipeline_file}") - continue - + + measurement_fps = fps_data + stream_fps_sum += sum(measurement_fps) stream_sample_count += len(measurement_fps) sorted_fps = sorted(measurement_fps) @@ -1086,7 +1250,7 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v stream_min_p90 = stream_fps_p90 if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": - print(f"INFO: P90 FPS for {pipeline_file}: {stream_fps_p90} (samples={len(measurement_fps)}, warm_up_discarded={warm_up_discarded}, steady_detected={is_steady})") + print(f"INFO: P90 FPS for {pipeline_file}: {stream_fps_p90} (samples={len(measurement_fps)})") else: print(f"INFO: P90 FPS for {pipeline_file}: {stream_fps_p90}") From e32c0219a1d39ce66dbcbdb3a8ae773781f43788 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 24 Sep 2026 09:44:09 +0530 Subject: [PATCH 03/12] Update stream_density_test.py --- benchmark-scripts/stream_density_test.py | 175 ++++++++--------------- 1 file changed, 63 insertions(+), 112 deletions(-) diff --git a/benchmark-scripts/stream_density_test.py b/benchmark-scripts/stream_density_test.py index af7e1d3..5c4e969 100644 --- a/benchmark-scripts/stream_density_test.py +++ b/benchmark-scripts/stream_density_test.py @@ -164,20 +164,22 @@ def test_check_non_empty_result_logs_success(self): if not os.listdir(test_results_dir): os.rmdir(test_results_dir) - def test_calculate_total_fps_success(self): - test_results_dir = './test_stream_density_results' - try: - fps, avg_fps = stream_density.calculate_total_fps( - 2, test_results_dir, 'gst') - self.assertTrue( - fps > 0.0, - f"total_fps is expected > 0.0 but found {fps}") - self.assertTrue( - avg_fps > 0.0, - f"total_fps_per_stream is expected > 0.0 but found " - f"{avg_fps}") - except Exception as ex: - self.fail(f'ERROR: got exception {type(ex).__name__}') + def test_calculate_multi_stream_fps_success(self): + with tempfile.TemporaryDirectory() as test_results_dir: + log_file = os.path.join( + test_results_dir, 'pipeline_stream0_123_gst.log') + with open(log_file, 'w') as file: + file.write('10\n20\n30\n40\n50\n') + + with patch('stream_density.get_pipeline_stream_count', + return_value=1): + total_fps, min_p90, stream_fps = ( + stream_density.calculate_multi_stream_fps( + 1, test_results_dir, 'gst')) + + self.assertEqual(total_fps, 30.0) + self.assertEqual(min_p90, 50) + self.assertEqual(stream_fps, {'pipeline_stream0': 50}) def test_clean_up_pipeline_logs(self): test_results_dir = './test_results_clean' @@ -296,86 +298,66 @@ def test_validate_and_setup_env(self): @patch('time.sleep', return_value=None) @patch('benchmark.docker_compose_containers') - @patch('stream_density.calculate_total_fps') + @patch('stream_density.calculate_multi_stream_fps') + @patch('stream_density.calculate_pipeline_latency', + return_value=(0.0, 0.0)) + @patch('stream_density.snapshot_stream_log_offsets', return_value={}) + @patch('stream_density.check_can_add_pipelines', return_value=True) + @patch('stream_density.measure_pipeline_memory', return_value=100.0) @patch('stream_density.check_non_empty_result_logs') @patch('stream_density.clean_up_pipeline_logs') def test_pipeline_iterations( self, mock_clean_logs, mock_check_logs, + mock_measure_memory, + mock_check_can_add, + mock_snapshot_offsets, + mock_calculate_latency, mock_calculate_fps, mock_docker_compose, mock_sleep ): test_cases = [ - # Test case 1: Succeed on the first iteration - # with target_fps achieved + # Test case 1: fail at two pipelines, then pass at one pipeline. { - "env_vars": {"INIT_DURATION": "10"}, + "env_vars": { + "INIT_DURATION": "10", + "PIPELINE_INC": "1", + "CONSECUTIVE_PASS_WINDOWS": "1", + "CONSECUTIVE_FAIL_WINDOWS": "1", + }, "compose_files": ["docker-compose.yml"], "results_dir": "/path/to/results", "container_name": "above_fps_target", "target_fps": 14.0, - "expected_num_pipelines": 10, + "expected_num_pipelines": 1, "expected_meet_target_fps": True, - # Mock returns FPS below target + "expected_streams_sustained": 1, "calculate_fps_side_effect": [ - (50, 20.0), (70, 16.0), (100, 12.0), (120, 14.8) + (15.0, 15.0, {"pipeline_stream0": 15.0}), + (10.0, 10.0, {"pipeline_stream0": 10.0}), + (15.0, 15.0, {"pipeline_stream0": 15.0}), ] }, - # Test case 2: Reach minimum pipeline count - # without achieving target_fps + # Test case 2: fail at the minimum pipeline count. { - "env_vars": {"INIT_DURATION": "10"}, + "env_vars": { + "INIT_DURATION": "10", + "CONSECUTIVE_FAIL_WINDOWS": "1", + }, "compose_files": ["docker-compose.yml"], "results_dir": "/path/to/results", "container_name": "below_fps_target", "target_fps": 15.0, "expected_num_pipelines": 1, "expected_meet_target_fps": False, - # Mock returns FPS below target - "calculate_fps_side_effect": [(100, 10.0), (50, 5.0)] - }, - # Test case 3: Below target_fps and come back - { - "env_vars": {"INIT_DURATION": "10", "PIPELINE_INC": "1"}, - "compose_files": ["docker-compose.yml"], - "results_dir": "/path/to/results", - "container_name": "below_comeback_target", - "target_fps": 14.0, - "expected_num_pipelines": 1, - "expected_meet_target_fps": True, - # Mock returns FPS below target - "calculate_fps_side_effect": [ - (100, 60.0), (190, 10.0), (320, 14.2) - ] - }, - # Test case 4: Below target_fps and reach minimum - { - "env_vars": {"INIT_DURATION": "10", "PIPELINE_INC": "1"}, - "compose_files": ["docker-compose.yml"], - "results_dir": "/path/to/results", - "container_name": "below_comeback_target", - "target_fps": 14.0, - "expected_num_pipelines": 1, - "expected_meet_target_fps": False, - # Mock returns FPS below target + "expected_streams_sustained": 1, "calculate_fps_side_effect": [ - (100, 60.0), (190, 10.0), (220, 13.2) + (10.0, 10.0, {"pipeline_stream0": 10.0}), + (10.0, 10.0, {"pipeline_stream0": 10.0}), ] }, - # Test case 5: check_non_empty_result_logs raises ValueError - { - "env_vars": {"INIT_DURATION": "10"}, - "compose_files": ["docker-compose.yml"], - "results_dir": "/path/to/results", - "container_name": "value_error", - "target_fps": 14.0, - "check_logs_side_effect": ValueError( - "Expecting ValueError for check_non_empty_result_logs"), - "expected_num_pipelines": 1, - "expected_meet_target_fps": False - }, ] for i, test_case in enumerate(test_cases): @@ -386,51 +368,20 @@ def test_pipeline_iterations( container_name = test_case["container_name"] target_fps = test_case["target_fps"] - # Set up the side effect for - # check_non_empty_result_logs if specified - if "check_logs_side_effect" in test_case: - mock_check_logs.side_effect = test_case[ - "check_logs_side_effect"] - - # Set up the side effect for - # calculate_total_fps using a finite list - if "calculate_fps_side_effect" in test_case: - mock_calculate_fps.side_effect = test_case[ - "calculate_fps_side_effect"] - - # Run the function with the test case inputs - if "check_logs_side_effect" in test_case: - num_pipelines, meet_target_fps = ( - stream_density.run_pipeline_iterations( - env_vars, compose_files, results_dir, - container_name, target_fps) - ) - - # Verify the returned values after ValueError is handled - self.assertEqual( - num_pipelines, - test_case["expected_num_pipelines"]) - self.assertEqual( - meet_target_fps, - test_case["expected_meet_target_fps"]) - - # Ensure that check_non_empty_result_logs was - # called and raised ValueError - mock_check_logs.assert_called() - else: - num_pipelines, meet_target_fps = ( - stream_density.run_pipeline_iterations( - env_vars, compose_files, results_dir, - container_name, target_fps) - ) + mock_calculate_fps.side_effect = test_case[ + "calculate_fps_side_effect"] + num_pipelines, meet_target_fps, streams_sustained = ( + stream_density.run_pipeline_iterations( + env_vars, compose_files, results_dir, + container_name, target_fps) + ) - # Verify the output - self.assertEqual( - num_pipelines, - test_case["expected_num_pipelines"]) - self.assertEqual( - meet_target_fps, - test_case["expected_meet_target_fps"]) + self.assertEqual( + num_pipelines, test_case["expected_num_pipelines"]) + self.assertEqual( + meet_target_fps, test_case["expected_meet_target_fps"]) + self.assertEqual( + streams_sustained, test_case["expected_streams_sustained"]) @patch('time.sleep', return_value=None) @patch('stream_density.validate_and_setup_env') @@ -453,12 +404,12 @@ def test_run_stream_density( "target_fps_list": [15.0, 25.0], "container_names_list": ["container1", "container2"], "run_pipeline_side_effect": [ - (5, True), # For container1 - (7, False) # For container2 + (5, True, 10), # For container1 + (7, False, 12) # For container2 ], "expected_results": [ - (15.0, "container1", 5, True), - (25.0, "container2", 7, False) + (15.0, "container1", 5, True, 10), + (25.0, "container2", 7, False, 12) ], # Expected number of compose down calls "expected_down_call_count": 2 From 3e986511b95a526f82423f7c41a0838051ce8092 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 24 Sep 2026 19:01:13 +0530 Subject: [PATCH 04/12] Update benchmark.py --- benchmark-scripts/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/benchmark.py b/benchmark-scripts/benchmark.py index 7c499be..b3bd850 100644 --- a/benchmark-scripts/benchmark.py +++ b/benchmark-scripts/benchmark.py @@ -221,6 +221,7 @@ def main(): env_vars["RETAIL_USE_CASE_ROOT"] = retail_use_case_root env_vars["VLM_WORKLOAD_ENABLED"] = str(os.getenv("LP_VLM_WORKLOAD_ENABLED")) env_vars["MEASUREMENT_WINDOW_SECONDS"] = str(my_args.measurement_window_seconds) + env_vars["INIT_DURATION"] = str(my_args.init_duration) if my_args.density_increment: env_vars["PIPELINE_INC"] = str(my_args.density_increment) if len(target_fps_list) > 1 and container_names_list: @@ -250,7 +251,6 @@ def main(): env_vars["TARGET_FPS"] = str(target_fps_list[0]) if my_args.density_increment: env_vars["PIPELINE_INC"] = str(my_args.density_increment) - env_vars["INIT_DURATION"] = str(my_args.init_duration) # use a default name since there is no # --container_names provided in this case container_name = (container_names_list[0] From a4047790ae94c5c10f1d88629e67e06c3efbd436 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 24 Sep 2026 19:02:40 +0530 Subject: [PATCH 05/12] Update stream_density.py --- benchmark-scripts/stream_density.py | 79 ++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/benchmark-scripts/stream_density.py b/benchmark-scripts/stream_density.py index 133b29b..28feec3 100644 --- a/benchmark-scripts/stream_density.py +++ b/benchmark-scripts/stream_density.py @@ -38,14 +38,15 @@ class ArgumentError(Exception): pass -def build_per_stream_target_fps(stream_fps_dict, default_target_fps): +def build_per_stream_target_fps(stream_fps_dict, default_target_fps, env_vars=None): """Resolve per-stream FPS targets. - Precedence: env TARGET_FPS (applies to every camera) > per-camera + Precedence: TARGET_FPS from env_vars (applies to every camera) > per-camera targetFps > per-camera fps > default_target_fps fallback. """ + env_vars = env_vars if env_vars is not None else os.environ # Get camera config path - camera_stream = os.getenv(CAMERA_STREAM_KEY, "camera_to_workload.json") + camera_stream = env_vars.get(CAMERA_STREAM_KEY, "camera_to_workload.json") config_path = camera_stream if os.path.isabs(camera_stream) else os.path.normpath( os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "configs", camera_stream) ) @@ -103,9 +104,9 @@ def build_per_stream_target_fps(stream_fps_dict, default_target_fps): stream_pattern = re.compile(r"pipeline_stream(\d+)") per_stream_targets = {} - # env TARGET_FPS, when set, overrides every camera's config value. + # TARGET_FPS from env_vars, when set, overrides every camera's config value. forced_target_fps = None - env_target_fps = os.getenv(TARGET_FPS_KEY, "").strip() + env_target_fps = str(env_vars.get(TARGET_FPS_KEY, "")).strip() if env_target_fps: try: candidate = float(env_target_fps) @@ -131,9 +132,10 @@ def build_per_stream_target_fps(stream_fps_dict, default_target_fps): return per_stream_targets -def describe_target_fps_configuration(default_target_fps): +def describe_target_fps_configuration(default_target_fps, env_vars=None): """Describe the configured target FPS values and their sources.""" - env_target_fps = os.getenv(TARGET_FPS_KEY, "").strip() + env_vars = env_vars if env_vars is not None else os.environ + env_target_fps = str(env_vars.get(TARGET_FPS_KEY, "")).strip() if env_target_fps: try: forced_target_fps = float(env_target_fps) @@ -142,7 +144,7 @@ def describe_target_fps_configuration(default_target_fps): except ValueError: pass - camera_stream = os.getenv(CAMERA_STREAM_KEY, "camera_to_workload.json") + camera_stream = env_vars.get(CAMERA_STREAM_KEY, "camera_to_workload.json") config_path = camera_stream if os.path.isabs(camera_stream) else os.path.normpath( os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "configs", camera_stream) ) @@ -179,9 +181,10 @@ def describe_target_fps_configuration(default_target_fps): return "per-camera target FPS configuration: " + ", ".join(descriptions) return f"default target FPS: {default_target_fps:.2f} FPS" -def build_per_stream_camera_meta(stream_fps_dict): +def build_per_stream_camera_meta(stream_fps_dict, env_vars=None): """Build per-stream camera_id and workload labels from camera config.""" - camera_stream = os.getenv(CAMERA_STREAM_KEY, "camera_to_workload.json") + env_vars = env_vars if env_vars is not None else os.environ + camera_stream = env_vars.get(CAMERA_STREAM_KEY, "camera_to_workload.json") config_path = camera_stream if os.path.isabs(camera_stream) else os.path.normpath( os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "configs", camera_stream) ) @@ -681,6 +684,20 @@ def validate_and_setup_env(env_vars, target_fps_list): env_vars[INIT_DURATION_KEY] = "120" +def count_pipeline_scaled_streams(num_pipelines, stream_fps_dict): + """Return the number of concurrent streams actually in flight. + + The FPS map is keyed by stream index, and each index aggregates the latest + num_pipelines log files. The report must therefore scale the number of valid + stream indices by the active pipeline count so it matches the returned + pipeline-scaled result. + """ + valid_streams = sum( + 1 for measured in stream_fps_dict.values() if float(measured) > 0 + ) + return int(num_pipelines) * valid_streams + + def print_stream_density_report(num_pipelines, stream_fps_dict, stream_target_fps, pass_thresholds, measurement_window_seconds, @@ -709,8 +726,8 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, print("") print("Stream density result") print("-" * 47) - camera_stream_count = sum( - 1 for measured in stream_fps_dict.values() if float(measured) > 0 + camera_stream_count = count_pipeline_scaled_streams( + num_pipelines, stream_fps_dict ) print(f"Streams sustained {camera_stream_count}") if init_duration_seconds is not None: @@ -719,13 +736,14 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, f"(INIT_DURATION before measuring)") print( f"Measurement window {measurement_window_seconds} s per window; " - f"{consecutive_pass_windows} consecutive windows must agree") + f"ramp-up is immediate, confirmation uses {consecutive_pass_windows} pass windows") print( " (each window is measured on its own; the " - "count moves") + "count is not accumulated)") print( - f" only after {consecutive_pass_windows} " - f"windows in a row agree - windows are not added together)") + f" passing windows increase the count during ramp-up, " + f"and the final density is accepted only after {consecutive_pass_windows} " + f"consecutive pass windows agree") print( f"Pass mark target x {pass_tolerance_ratio:g} " f"(pass tolerance ratio)") @@ -838,7 +856,7 @@ def run_pipeline_iterations( # clean up any residual pipeline log files before starts: clean_up_pipeline_logs(results_dir) - target_fps_label = describe_target_fps_configuration(target_fps) + target_fps_label = describe_target_fps_configuration(target_fps, env_vars=env_vars) print( f"INFO: Stream density {target_fps_label} " f"with container_name {container_name} " @@ -910,8 +928,8 @@ def run_pipeline_iterations( num_pipelines, results_dir, container_name, env_vars, start_offsets=window_start_offsets, end_offsets=window_end_offsets) - streams_sustained = sum( - 1 for measured in stream_fps_dict.values() if float(measured) > 0 + streams_sustained = count_pipeline_scaled_streams( + num_pipelines, stream_fps_dict ) print('container name:', container_name) @@ -932,7 +950,7 @@ def run_pipeline_iterations( # --- Decide scaling logic (per-stream thresholds) --- stream_target_fps = build_per_stream_target_fps( - stream_fps_dict, target_fps + stream_fps_dict, target_fps, env_vars=env_vars ) pass_thresholds = { @@ -1013,7 +1031,7 @@ def run_pipeline_iterations( ) increments = 0 stream_camera_meta = build_per_stream_camera_meta( - stream_fps_dict) + stream_fps_dict, env_vars=env_vars) print_stream_density_report( num_pipelines, stream_fps_dict, stream_target_fps, pass_thresholds, measurement_window_seconds, @@ -1030,10 +1048,25 @@ def run_pipeline_iterations( elif num_pipelines <= 1: pass_window_count = 0 fail_window_count = 0 + per_stream_fps = [ + float(value) for value in stream_fps_dict.values() if float(value) > 0 + ] + avg_stream_fps = ( + statistics.mean(per_stream_fps) + if per_stream_fps else 0.0 + ) print( - f"already reached num. pipeline 1, and the fps per stream is " - f"{total_fps_per_stream} but target FPS map is {stream_target_fps}" + f"already reached num. pipeline 1, and the average fps per stream is " + f"{avg_stream_fps:.2f} but target FPS map is {stream_target_fps}" ) + stream_camera_meta = build_per_stream_camera_meta( + stream_fps_dict, env_vars=env_vars) + print_stream_density_report( + num_pipelines, stream_fps_dict, stream_target_fps, + pass_thresholds, measurement_window_seconds, + consecutive_pass_windows, consecutive_fail_windows, + pass_tolerance_ratio, stream_camera_meta, + INIT_DURATION) meet_target_fps = False break else: From efcb1b64c11528c42b97ff71e72b173a4eb30d09 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 24 Sep 2026 19:03:52 +0530 Subject: [PATCH 06/12] Update stream_density_test.py --- benchmark-scripts/stream_density_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/benchmark-scripts/stream_density_test.py b/benchmark-scripts/stream_density_test.py index 5c4e969..3d80e5b 100644 --- a/benchmark-scripts/stream_density_test.py +++ b/benchmark-scripts/stream_density_test.py @@ -181,6 +181,18 @@ def test_calculate_multi_stream_fps_success(self): self.assertEqual(min_p90, 50) self.assertEqual(stream_fps, {'pipeline_stream0': 50}) + def test_count_pipeline_scaled_streams(self): + stream_fps_dict = { + 'pipeline_stream0': 15.0, + 'pipeline_stream1': 12.5, + 'pipeline_stream2': 0.0, + } + + self.assertEqual( + stream_density.count_pipeline_scaled_streams(36, stream_fps_dict), + 72 + ) + def test_clean_up_pipeline_logs(self): test_results_dir = './test_results_clean' testFile1 = os.path.join( From 8c60c62e9c3c22fcc3b82e5c27c0ebed2c49b0e8 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 29 Sep 2026 12:56:58 +0530 Subject: [PATCH 07/12] feat(stream-density): report p90 and per-stream samples below pass mark Return per-stream FPS samples from calculate_multi_stream_fps and surface them in the density report: add a p90 column and a seconds-below-pass-mark column so a stream's stability is visible, not just its average, plus a Lanes sustained line alongside Streams sustained. Replace count_pipeline_scaled_streams with count_valid_streams, dropping the num_pipelines multiplier that inflated the sustained-stream count; pipeline indices already span every lane. Update unit tests to match the new return arity and report format. Signed-off-by: sumanaga --- benchmark-scripts/stream_density.py | 52 ++++++------ benchmark-scripts/stream_density_test.py | 101 +++++++++++++++++++---- 2 files changed, 115 insertions(+), 38 deletions(-) diff --git a/benchmark-scripts/stream_density.py b/benchmark-scripts/stream_density.py index 28feec3..e4f71e4 100644 --- a/benchmark-scripts/stream_density.py +++ b/benchmark-scripts/stream_density.py @@ -684,18 +684,12 @@ def validate_and_setup_env(env_vars, target_fps_list): env_vars[INIT_DURATION_KEY] = "120" -def count_pipeline_scaled_streams(num_pipelines, stream_fps_dict): - """Return the number of concurrent streams actually in flight. - - The FPS map is keyed by stream index, and each index aggregates the latest - num_pipelines log files. The report must therefore scale the number of valid - stream indices by the active pipeline count so it matches the returned - pipeline-scaled result. - """ - valid_streams = sum( +def count_valid_streams(stream_fps_dict): + """Return the number of concurrent streams with valid FPS data.""" + # pipeline.sh is generated with every lane's sources, so indices already span all lanes. + return sum( 1 for measured in stream_fps_dict.values() if float(measured) > 0 ) - return int(num_pipelines) * valid_streams def print_stream_density_report(num_pipelines, stream_fps_dict, @@ -704,9 +698,11 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, consecutive_pass_windows, consecutive_fail_windows, pass_tolerance_ratio, stream_meta=None, - init_duration_seconds=None): + init_duration_seconds=None, + stream_samples=None): """Print the per-stream stream density result summary.""" stream_meta = stream_meta or {} + stream_samples = stream_samples or {} binding_name = None binding_headroom = None for name in stream_fps_dict: @@ -726,9 +722,8 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, print("") print("Stream density result") print("-" * 47) - camera_stream_count = count_pipeline_scaled_streams( - num_pipelines, stream_fps_dict - ) + camera_stream_count = count_valid_streams(stream_fps_dict) + print(f"Lanes sustained {num_pipelines}") print(f"Streams sustained {camera_stream_count}") if init_duration_seconds is not None: print( @@ -750,7 +745,7 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, print("") print( f"{'stream':<18}{'camera':<8}{'workload':<50}{'target':>8}{'pass mark':>11}" - f"{'measured':>10}{'result':>8}") + f"{'p90':>10}{'seconds below pass mark':>26}{'result':>8}") for name in sorted( stream_fps_dict, key=lambda n: int(re.search(r'(\d+)', n).group(1)) if re.search(r'(\d+)', n) else 0): @@ -761,9 +756,15 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, pass_mark = pass_thresholds.get(name, 0.0) measured = stream_fps_dict.get(name, 0.0) result = "pass" if measured >= pass_mark else "fail" + samples = stream_samples.get(name, []) + if samples: + below = sum(1 for fps in samples if fps < pass_mark) + below_text = f"{below}/{len(samples)} ({below / len(samples) * 100:.0f}%)" + else: + below_text = "-" print( f"{name:<18}{camera:<8}{workload:<50}{target:>8.2f}{pass_mark:>11.2f}" - f"{measured:>10.2f}{result:>8}") + f"{measured:>10.2f}{below_text:>26}{result:>8}") print("") print( f"Result: {'Pass' if overall_pass else 'Fail'} - " @@ -794,7 +795,7 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, f"seconds each. " f"{targets_clause} The {stream_descriptor} was {binding_name} " f"({binding_camera}{workload_clause}), targeting {bt:.2f} FPS and " - f"measuring {bm:.2f} FPS against its {bp:.2f} FPS minimum, leaving " + f"with a p90 of {bm:.2f} FPS against its {bp:.2f} FPS minimum, leaving " f"{absolute_headroom:.2f} FPS " f"({relative_headroom:.1f}% relative headroom).") @@ -924,13 +925,11 @@ def run_pipeline_iterations( f"INFO: SWEEPING-STOP (Measurement Window Stop): " f"{time.strftime('%Y-%m-%dT%H:%M:%S')}") # --- Calculate FPS and latency metrics --- - total_fps, min_p90_across_streams, stream_fps_dict = calculate_multi_stream_fps( + total_fps, min_p90_across_streams, stream_fps_dict, stream_samples_dict = calculate_multi_stream_fps( num_pipelines, results_dir, container_name, env_vars, start_offsets=window_start_offsets, end_offsets=window_end_offsets) - streams_sustained = count_pipeline_scaled_streams( - num_pipelines, stream_fps_dict - ) + streams_sustained = count_valid_streams(stream_fps_dict) print('container name:', container_name) print('Total FPS:', total_fps) @@ -1037,7 +1036,7 @@ def run_pipeline_iterations( pass_thresholds, measurement_window_seconds, consecutive_pass_windows, consecutive_fail_windows, pass_tolerance_ratio, stream_camera_meta, - INIT_DURATION) + INIT_DURATION, stream_samples_dict) else: increments = 0 print( @@ -1066,7 +1065,7 @@ def run_pipeline_iterations( pass_thresholds, measurement_window_seconds, consecutive_pass_windows, consecutive_fail_windows, pass_tolerance_ratio, stream_camera_meta, - INIT_DURATION) + INIT_DURATION, stream_samples_dict) meet_target_fps = False break else: @@ -1226,6 +1225,7 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v total_fps: sum of per-stream average FPS (reporting) min_p90_across_streams: minimum p90 across all streams (decision) stream_fps_dict: stream -> minimum p90 FPS across matching files + stream_samples_dict: stream -> FPS samples from the file that gave that p90 """ stream_count = get_pipeline_stream_count() @@ -1233,6 +1233,7 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v # --- Initialize accumulators --- total_fps = 0.0 stream_fps_dict = {} + stream_samples_dict = {} # --- Loop over all streams --- for idx in range(stream_count): @@ -1257,6 +1258,7 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v stream_fps_sum = 0.0 stream_sample_count = 0 stream_min_p90 = None + stream_min_p90_samples = [] for pipeline_file in latest_pipeline_logs: print(f"DEBUG: Processing file: {pipeline_file}") @@ -1281,6 +1283,7 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v stream_fps_p90 = sorted_fps[p90_idx] if stream_min_p90 is None or stream_fps_p90 < stream_min_p90: stream_min_p90 = stream_fps_p90 + stream_min_p90_samples = measurement_fps if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": print(f"INFO: P90 FPS for {pipeline_file}: {stream_fps_p90} (samples={len(measurement_fps)})") @@ -1295,6 +1298,7 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v report_stream_avg = stream_fps_sum / stream_sample_count total_fps += report_stream_avg stream_fps_dict[f'pipeline_stream{idx}'] = round(stream_min_p90, 2) + stream_samples_dict[f'pipeline_stream{idx}'] = stream_min_p90_samples else: stream_fps_dict[f'pipeline_stream{idx}'] = 0.0 print(f"WARN: No valid FPS data for stream index {idx}") @@ -1305,7 +1309,7 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v ] min_p90_across_streams = min(decision_stream_p90) if decision_stream_p90 else 0.0 - return total_fps, min_p90_across_streams, stream_fps_dict + return total_fps, min_p90_across_streams, stream_fps_dict, stream_samples_dict def get_pipeline_stream_count(base_dir=None): diff --git a/benchmark-scripts/stream_density_test.py b/benchmark-scripts/stream_density_test.py index 3d80e5b..50c40fe 100644 --- a/benchmark-scripts/stream_density_test.py +++ b/benchmark-scripts/stream_density_test.py @@ -19,6 +19,8 @@ DEFAULT_TARGET_FPS ) import os +import io +from contextlib import redirect_stdout class Testing(unittest.TestCase): @@ -165,23 +167,96 @@ def test_check_non_empty_result_logs_success(self): os.rmdir(test_results_dir) def test_calculate_multi_stream_fps_success(self): + samples = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] with tempfile.TemporaryDirectory() as test_results_dir: log_file = os.path.join( test_results_dir, 'pipeline_stream0_123_gst.log') with open(log_file, 'w') as file: - file.write('10\n20\n30\n40\n50\n') + file.write(''.join(f'{s}\n' for s in samples)) with patch('stream_density.get_pipeline_stream_count', return_value=1): - total_fps, min_p90, stream_fps = ( + total_fps, min_p90, stream_fps, stream_samples = ( stream_density.calculate_multi_stream_fps( 1, test_results_dir, 'gst')) - self.assertEqual(total_fps, 30.0) - self.assertEqual(min_p90, 50) - self.assertEqual(stream_fps, {'pipeline_stream0': 50}) + self.assertEqual(total_fps, 5.5) + # Nearest-rank p90 of 10 samples is the 9th value, not the top sample. + self.assertEqual(min_p90, 9) + self.assertEqual(stream_fps, {'pipeline_stream0': 9}) + self.assertEqual(stream_samples, {'pipeline_stream0': samples}) + + def test_calculate_multi_stream_fps_measurement_window(self): + with tempfile.TemporaryDirectory() as test_results_dir, \ + patch.dict(os.environ, {'TIMESTAMP': ''}), \ + patch('stream_density.get_pipeline_stream_count', + return_value=1): + log_file = os.path.join( + test_results_dir, 'pipeline_stream0_123_gst.log') + with open(log_file, 'w') as file: + file.write('1\n2\n3\n') + start_offsets = stream_density.snapshot_stream_log_offsets( + test_results_dir, 'gst') + with open(log_file, 'a') as file: + file.write('14\n15\n16\n') + end_offsets = stream_density.snapshot_stream_log_offsets( + test_results_dir, 'gst') + with open(log_file, 'a') as file: + file.write('99\n100\n') + + total_fps, min_p90, stream_fps, stream_samples = ( + stream_density.calculate_multi_stream_fps( + 1, test_results_dir, 'gst', + start_offsets=start_offsets, + end_offsets=end_offsets)) + + self.assertEqual(stream_samples, {'pipeline_stream0': [14, 15, 16]}) + self.assertEqual(total_fps, 15.0) + self.assertEqual(stream_fps, {'pipeline_stream0': 16}) + self.assertEqual(min_p90, 16) + + def test_print_stream_density_report_summary(self): + stream_fps = {'pipeline_stream0': 16.0, 'pipeline_stream1': 15.0} + targets = {'pipeline_stream0': 15.0, 'pipeline_stream1': 15.0} + pass_marks = {'pipeline_stream0': 14.25, 'pipeline_stream1': 14.25} + samples = { + 'pipeline_stream0': [16, 14, 15, 13, 17, 14, 16, 12, 15, 14], + 'pipeline_stream1': [15] * 10, + } + meta = { + 'pipeline_stream0': {'camera': 'cam1', 'workload': 'wl'}, + 'pipeline_stream1': {'camera': 'cam2', 'workload': 'wl'}, + } - def test_count_pipeline_scaled_streams(self): + output = io.StringIO() + with redirect_stdout(output): + stream_density.print_stream_density_report( + 18, stream_fps, targets, pass_marks, 100, 2, 2, 0.95, + meta, 10, samples) + report = output.getvalue() + + self.assertIn('Lanes sustained 18', report) + self.assertIn('Streams sustained 2', report) + header = next(line for line in report.splitlines() + if line.startswith('stream')) + for column in ('target', 'pass mark', 'p90', + 'seconds below pass mark', 'result'): + self.assertIn(column, header) + rows = {line.split()[0]: line.split() for line in report.splitlines() + if line.startswith('pipeline_stream')} + self.assertEqual( + rows['pipeline_stream0'][3:], + ['15.00', '14.25', '16.00', '5/10', '(50%)', 'pass']) + self.assertEqual( + rows['pipeline_stream1'][3:], + ['15.00', '14.25', '15.00', '0/10', '(0%)', 'pass']) + self.assertIn( + 'The lowest-throughput stream was pipeline_stream1', report) + self.assertIn('targeting 15.00 FPS', report) + self.assertIn('p90 of 15.00 FPS', report) + self.assertIn('against its 14.25 FPS minimum', report) + + def test_count_valid_streams(self): stream_fps_dict = { 'pipeline_stream0': 15.0, 'pipeline_stream1': 12.5, @@ -189,9 +264,7 @@ def test_count_pipeline_scaled_streams(self): } self.assertEqual( - stream_density.count_pipeline_scaled_streams(36, stream_fps_dict), - 72 - ) + stream_density.count_valid_streams(stream_fps_dict), 2) def test_clean_up_pipeline_logs(self): test_results_dir = './test_results_clean' @@ -347,9 +420,9 @@ def test_pipeline_iterations( "expected_meet_target_fps": True, "expected_streams_sustained": 1, "calculate_fps_side_effect": [ - (15.0, 15.0, {"pipeline_stream0": 15.0}), - (10.0, 10.0, {"pipeline_stream0": 10.0}), - (15.0, 15.0, {"pipeline_stream0": 15.0}), + (15.0, 15.0, {"pipeline_stream0": 15.0}, {}), + (10.0, 10.0, {"pipeline_stream0": 10.0}, {}), + (15.0, 15.0, {"pipeline_stream0": 15.0}, {}), ] }, # Test case 2: fail at the minimum pipeline count. @@ -366,8 +439,8 @@ def test_pipeline_iterations( "expected_meet_target_fps": False, "expected_streams_sustained": 1, "calculate_fps_side_effect": [ - (10.0, 10.0, {"pipeline_stream0": 10.0}), - (10.0, 10.0, {"pipeline_stream0": 10.0}), + (10.0, 10.0, {"pipeline_stream0": 10.0}, {}), + (10.0, 10.0, {"pipeline_stream0": 10.0}, {}), ] }, ] From 91e17315a61de9618391386a347f411af9f2c3b8 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 29 Sep 2026 13:25:20 +0530 Subject: [PATCH 08/12] feat(stream-density): default 10s settle and 100s measurement window Align the stream_density fallback defaults with the argparse defaults and the Makefile: INIT_DURATION falls back to 10 seconds instead of 120, and DEFAULT_MEASUREMENT_WINDOW_SECONDS is 100 instead of 30. Update the validate_and_setup_env unit tests to expect the 10 second fallback. Signed-off-by: sumanaga --- benchmark-scripts/stream_density.py | 4 ++-- benchmark-scripts/stream_density_test.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmark-scripts/stream_density.py b/benchmark-scripts/stream_density.py index e4f71e4..390a07a 100644 --- a/benchmark-scripts/stream_density.py +++ b/benchmark-scripts/stream_density.py @@ -31,7 +31,7 @@ PASS_TOLERANCE_RATIO_KEY = "PASS_TOLERANCE_RATIO" DEFAULT_PASS_TOLERANCE_RATIO = 0.95 MEASUREMENT_WINDOW_SECONDS_KEY = "MEASUREMENT_WINDOW_SECONDS" -DEFAULT_MEASUREMENT_WINDOW_SECONDS = 30 +DEFAULT_MEASUREMENT_WINDOW_SECONDS = 100 CAMERA_STREAM_KEY = "CAMERA_STREAM" @@ -681,7 +681,7 @@ def validate_and_setup_env(env_vars, target_fps_list): 'ERROR: pass tolerance ratio should be in (0, 1]') if not is_env_non_empty(env_vars, INIT_DURATION_KEY): - env_vars[INIT_DURATION_KEY] = "120" + env_vars[INIT_DURATION_KEY] = "10" def count_valid_streams(stream_fps_dict): diff --git a/benchmark-scripts/stream_density_test.py b/benchmark-scripts/stream_density_test.py index 50c40fe..c812c65 100644 --- a/benchmark-scripts/stream_density_test.py +++ b/benchmark-scripts/stream_density_test.py @@ -306,7 +306,7 @@ def test_validate_and_setup_env(self): "expected_target_fps_list": [20.0], "expected_env_vars": { RESULTS_DIR_KEY: "/some/path", - INIT_DURATION_KEY: "120" + INIT_DURATION_KEY: "10" }, }, # Test case 2: Missing RESULTS_DIR_KEY in env_vars @@ -324,7 +324,7 @@ def test_validate_and_setup_env(self): "expected_target_fps_list": [DEFAULT_TARGET_FPS], "expected_env_vars": { RESULTS_DIR_KEY: "/some/path", - INIT_DURATION_KEY: "120" + INIT_DURATION_KEY: "10" }, }, # Test case 4: Negative target_fps value in target_fps_list @@ -335,7 +335,7 @@ def test_validate_and_setup_env(self): "exception_type": ArgumentError, }, # Test case 5: Missing INIT_DURATION_KEY in env_vars- - # should default to "120" + # should default to "10" { "env_vars": {RESULTS_DIR_KEY: "/some/path"}, "target_fps_list": [20.0], @@ -343,7 +343,7 @@ def test_validate_and_setup_env(self): "expected_target_fps_list": [20.0], "expected_env_vars": { RESULTS_DIR_KEY: "/some/path", - INIT_DURATION_KEY: "120" + INIT_DURATION_KEY: "10" }, }, # Test case 6: PIPELINE_INCR_KEY <= 0 (should raise exception) From 5461f69dea4b38ae4c2640ca8ea68ba92a9c339d Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 29 Sep 2026 13:29:23 +0530 Subject: [PATCH 09/12] feat(stream-density): default 10s settle and 100s measurement window Align the stream_density fallback defaults with the argparse defaults and the Makefile: INIT_DURATION falls back to 10 seconds instead of 120, and DEFAULT_MEASUREMENT_WINDOW_SECONDS is 100 instead of 30. Update the validate_and_setup_env unit tests to expect the 10 second fallback. Signed-off-by: sumanaga --- benchmark-scripts/benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark-scripts/benchmark.py b/benchmark-scripts/benchmark.py index b3bd850..342b2ce 100644 --- a/benchmark-scripts/benchmark.py +++ b/benchmark-scripts/benchmark.py @@ -73,10 +73,10 @@ def parse_args(print=False): help='time in seconds, not needed when ' + '--target_fps is specified') - parser.add_argument('--init_duration', type=int, default=20, + parser.add_argument('--init_duration', type=int, default=10, help='initial time in seconds before ' + 'starting metric data collection') - parser.add_argument('--measurement_window_seconds', type=int, default=30, + parser.add_argument('--measurement_window_seconds', type=int, default=100, help='duration in seconds to collect FPS samples ' + 'after INIT_DURATION for the stream density ' + 'pass/fail decision') From e85f4189531bc26b8a0e7de8eeb5831446c0a3b8 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 6 Oct 2026 21:00:44 +0530 Subject: [PATCH 10/12] feat(stream-density)!: judge pass/fail on time-weighted measured FPS p90 FPS reflects a stream's best moments, not what it sustains, so it could overstate how many lanes a box holds. Pass/fail now uses each stream's time-weighted average FPS, with p10, p90 and time below threshold reported. An interval needs 100 samples per pipeline log or it is inconclusive. BREAKING CHANGE: Densities may be lower than with p90. Report and log labels use the documented terms, and DEFAULT_TARGET_FPS is now 15. Signed-off-by: sumanaga --- .../consolidate_multiple_run_of_metrics.py | 5 +- benchmark-scripts/stream_density.py | 365 ++++++++++++------ benchmark-scripts/stream_density_rate_test.py | 114 ++++++ benchmark-scripts/stream_density_test.py | 169 ++++++-- 4 files changed, 497 insertions(+), 156 deletions(-) create mode 100644 benchmark-scripts/stream_density_rate_test.py diff --git a/benchmark-scripts/consolidate_multiple_run_of_metrics.py b/benchmark-scripts/consolidate_multiple_run_of_metrics.py index fbfad0e..9323c8e 100644 --- a/benchmark-scripts/consolidate_multiple_run_of_metrics.py +++ b/benchmark-scripts/consolidate_multiple_run_of_metrics.py @@ -607,8 +607,9 @@ def extract_data(self, log_file_path): cam = re.findall(r'\d+', os.path.basename(log_file_path)) camera_key = "Camera_{} {}".format(cam[0], AVG_FPS_CONSTANT) with open(log_file_path) as f: - for line in f: - average_fps_list.append(float(line)) + for row in csv.reader(f): + if row and row != ['fps', 'duration_seconds']: + average_fps_list.append(float(row[0])) if len(average_fps_list) > 0: camera_fps[camera_key] = round(mean(average_fps_list), 2) diff --git a/benchmark-scripts/stream_density.py b/benchmark-scripts/stream_density.py index 390a07a..ab780e1 100644 --- a/benchmark-scripts/stream_density.py +++ b/benchmark-scripts/stream_density.py @@ -15,6 +15,7 @@ import psutil import json import math +import csv # Constants: TARGET_FPS_KEY = "TARGET_FPS" @@ -22,7 +23,7 @@ PIPELINE_INCR_KEY = "PIPELINE_INC" INIT_DURATION_KEY = "INIT_DURATION" RESULTS_DIR_KEY = "RESULTS_DIR" -DEFAULT_TARGET_FPS = 14.95 +DEFAULT_TARGET_FPS = 15 MAX_GUESS_INCREMENTS = 5 CONSECUTIVE_FAIL_WINDOWS_KEY = "CONSECUTIVE_FAIL_WINDOWS" CONSECUTIVE_PASS_WINDOWS_KEY = "CONSECUTIVE_PASS_WINDOWS" @@ -32,6 +33,8 @@ DEFAULT_PASS_TOLERANCE_RATIO = 0.95 MEASUREMENT_WINDOW_SECONDS_KEY = "MEASUREMENT_WINDOW_SECONDS" DEFAULT_MEASUREMENT_WINDOW_SECONDS = 100 +MIN_SAMPLES_PER_STREAM = 100 +MEASUREMENT_SAMPLE_POLL_SECONDS = 1 CAMERA_STREAM_KEY = "CAMERA_STREAM" @@ -486,8 +489,8 @@ def get_latest_pipeline_logs(num_pipelines, pipeline_log_files): return latest_files -def extract_numeric_fps(file_path, start_offset=0, end_offset=None): - """Extract valid numeric FPS samples within an optional byte range.""" +def extract_fps_samples(file_path, start_offset=0, end_offset=None): + """Extract (FPS, seconds) samples within an optional byte range.""" with open(file_path, "r") as file: if start_offset: file.seek(start_offset) @@ -496,17 +499,27 @@ def extract_numeric_fps(file_path, start_offset=0, end_offset=None): else: lines = file.readlines() - numeric_fps = [] - for line in lines: - stripped_line = line.strip() - if not stripped_line or 'na' in stripped_line.lower(): + samples = [] + for row in csv.reader(lines): + if not row or row == ['fps', 'duration_seconds']: continue try: - numeric_fps.append(float(stripped_line)) + if len(row) not in (1, 2): + continue + fps = float(row[0]) + seconds = float(row[1]) if len(row) == 2 else 1.0 + if not math.isfinite(fps) or not math.isfinite(seconds) or fps < 0 or seconds <= 0: + continue + samples.append((fps, seconds)) except ValueError: - print(f"DEBUG: Skipping non-numeric line '{stripped_line}' in {file_path}") + print(f"DEBUG: Skipping invalid FPS row {row} in {file_path}") + + return samples - return numeric_fps + +def extract_numeric_fps(file_path, start_offset=0, end_offset=None): + """Extract FPS values from legacy logs or FPS/duration rows.""" + return [fps for fps, _ in extract_fps_samples(file_path, start_offset, end_offset)] def filter_logs_for_current_timestamp(log_files): @@ -662,17 +675,17 @@ def validate_and_setup_env(env_vars, target_fps_list): if is_env_non_empty(env_vars, CONSECUTIVE_FAIL_WINDOWS_KEY) and int( env_vars[CONSECUTIVE_FAIL_WINDOWS_KEY]) <= 0: raise ArgumentError( - 'ERROR: consecutive fail windows should be greater than 0') + 'ERROR: consecutive fail intervals should be greater than 0') if is_env_non_empty(env_vars, CONSECUTIVE_PASS_WINDOWS_KEY) and int( env_vars[CONSECUTIVE_PASS_WINDOWS_KEY]) <= 0: raise ArgumentError( - 'ERROR: consecutive pass windows should be greater than 0') + 'ERROR: run acceptance criterion (consecutive passes) should be greater than 0') if is_env_non_empty(env_vars, MEASUREMENT_WINDOW_SECONDS_KEY) and int( env_vars[MEASUREMENT_WINDOW_SECONDS_KEY]) <= 0: raise ArgumentError( - 'ERROR: measurement window seconds should be greater than 0') + 'ERROR: measurement interval seconds should be greater than 0') if is_env_non_empty(env_vars, PASS_TOLERANCE_RATIO_KEY): pass_tolerance_ratio = float(env_vars[PASS_TOLERANCE_RATIO_KEY]) @@ -684,6 +697,11 @@ def validate_and_setup_env(env_vars, target_fps_list): env_vars[INIT_DURATION_KEY] = "10" +def describe_run_acceptance_criterion(consecutive_pass_windows): + passes = "two" if consecutive_pass_windows == 2 else str(consecutive_pass_windows) + return f"run acceptance criterion ({passes} consecutive passes)" + + def count_valid_streams(stream_fps_dict): """Return the number of concurrent streams with valid FPS data.""" # pipeline.sh is generated with every lane's sources, so indices already span all lanes. @@ -699,7 +717,9 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, consecutive_fail_windows, pass_tolerance_ratio, stream_meta=None, init_duration_seconds=None, - stream_samples=None): + stream_samples=None, + actual_measurement_seconds=None, + sample_counts=None): """Print the per-stream stream density result summary.""" stream_meta = stream_meta or {} stream_samples = stream_samples or {} @@ -714,7 +734,7 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, binding_headroom = headroom binding_name = name - overall_pass = all( + overall_pass = bool(stream_fps_dict) and all( stream_fps_dict[name] >= pass_thresholds.get(name, 0.0) for name in stream_fps_dict ) @@ -723,29 +743,44 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, print("Stream density result") print("-" * 47) camera_stream_count = count_valid_streams(stream_fps_dict) - print(f"Lanes sustained {num_pipelines}") - print(f"Streams sustained {camera_stream_count}") + acceptance_criterion = describe_run_acceptance_criterion(consecutive_pass_windows) + print(f"Use case density (lanes) {num_pipelines}") + print(f"Stream density (streams) {camera_stream_count}") if init_duration_seconds is not None: print( - f"Settle time {init_duration_seconds} s " + f"Warmup period {init_duration_seconds} s " f"(INIT_DURATION before measuring)") print( - f"Measurement window {measurement_window_seconds} s per window; " - f"ramp-up is immediate, confirmation uses {consecutive_pass_windows} pass windows") + f"Measurement interval {measurement_window_seconds} s minimum; " + f"ramp-up is immediate, confirmation uses the {acceptance_criterion}") + if actual_measurement_seconds is not None: + print( + f"Actual measurement time {actual_measurement_seconds:.1f} s " + f"(minimum {measurement_window_seconds} s)") + if sample_counts: + print( + f"Samples per pipeline log {min(sample_counts.values())}-" + f"{max(sample_counts.values())} across {len(sample_counts)} logs " + f"(minimum {MIN_SAMPLES_PER_STREAM})") print( - " (each window is measured on its own; the " + " (each interval is measured on its own; the " "count is not accumulated)") print( - f" passing windows increase the count during ramp-up, " - f"and the final density is accepted only after {consecutive_pass_windows} " - f"consecutive pass windows agree") + f" passing intervals increase the count during ramp-up, " + f"and the final density is accepted only after the {acceptance_criterion} is met") print( - f"Pass mark target x {pass_tolerance_ratio:g} " + f"Throughput threshold target x {pass_tolerance_ratio:g} " f"(pass tolerance ratio)") print("") + stream_w = max([len('stream')] + [len(n) for n in stream_fps_dict]) + 2 + camera_w = max([len('camera')] + [len(str(stream_meta.get(n, {}).get('camera', n))) + for n in stream_fps_dict]) + 2 + workload_w = max([len('workload')] + [len(str(stream_meta.get(n, {}).get('workload', '-'))) + for n in stream_fps_dict]) + 2 print( - f"{'stream':<18}{'camera':<8}{'workload':<50}{'target':>8}{'pass mark':>11}" - f"{'p90':>10}{'seconds below pass mark':>26}{'result':>8}") + f"{'stream':<{stream_w}}{'camera':<{camera_w}}{'workload':<{workload_w}}{'target':>8}" + f"{'throughput threshold':>22}{'measured (avg)':>16}{'p10':>9}{'p90':>9}" + f"{'seconds below throughput threshold':>36}{'result':>8}") for name in sorted( stream_fps_dict, key=lambda n: int(re.search(r'(\d+)', n).group(1)) if re.search(r'(\d+)', n) else 0): @@ -758,17 +793,32 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, result = "pass" if measured >= pass_mark else "fail" samples = stream_samples.get(name, []) if samples: - below = sum(1 for fps in samples if fps < pass_mark) - below_text = f"{below}/{len(samples)} ({below / len(samples) * 100:.0f}%)" + sample_pairs = [ + (float(sample[0]), float(sample[1])) + if isinstance(sample, (tuple, list)) else (float(sample), 1.0) + for sample in samples + ] + fps_values = [fps for fps, _ in sample_pairs] + sorted_samples = sorted(fps_values) + p10 = sorted_samples[math.ceil(0.1 * len(sorted_samples)) - 1] + p90 = sorted_samples[math.ceil(0.9 * len(sorted_samples)) - 1] + total_sample_duration = sum(seconds for _, seconds in sample_pairs) + below_duration = sum( + seconds for fps, seconds in sample_pairs if fps < pass_mark) + below_text = ( + f"{below_duration:.2f} s " + f"({below_duration / total_sample_duration * 100:.0f}%)") else: + p10 = p90 = 0.0 below_text = "-" print( - f"{name:<18}{camera:<8}{workload:<50}{target:>8.2f}{pass_mark:>11.2f}" - f"{measured:>10.2f}{below_text:>26}{result:>8}") + f"{name:<{stream_w}}{camera:<{camera_w}}{workload:<{workload_w}}{target:>8.2f}" + f"{pass_mark:>22.2f}{measured:>16.6f}{p10:>9.2f}{p90:>9.2f}" + f"{below_text:>36}{result:>8}") print("") print( f"Result: {'Pass' if overall_pass else 'Fail'} - " - f"{camera_stream_count} concurrent streams sustained") + f"stream density {camera_stream_count} streams") if binding_name is not None: bt = stream_target_fps.get(binding_name, 0.0) bp = pass_thresholds.get(binding_name, 0.0) @@ -778,24 +828,27 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, target_values = [float(v) for v in stream_target_fps.values()] uniform_targets = (max(target_values) - min(target_values) < 1e-9) if target_values else True if uniform_targets: - targets_clause = f"All streams met the {bp:.2f} FPS minimum requirement." + targets_clause = ( + f"All streams met the {bp:.2f} FPS throughput threshold." + if overall_pass else f"Not all streams met the {bp:.2f} FPS throughput threshold.") stream_descriptor = "lowest-throughput stream" else: - targets_clause = "All streams met their individual minimum FPS requirements." + targets_clause = ( + "All streams met their individual throughput thresholds." + if overall_pass else "Not all streams met their individual throughput thresholds.") stream_descriptor = "lowest-headroom stream" workload_clause = f" ({binding_workload})" if binding_workload else "" absolute_headroom = bm - bp relative_headroom = binding_headroom * 100 settle_clause = ( - f"after a {init_duration_seconds} second settle time (INIT_DURATION), " + f"after a {init_duration_seconds} second warmup period (INIT_DURATION), " if init_duration_seconds is not None else "") print( - f"The run was evaluated {settle_clause}over {consecutive_pass_windows} " - f"consecutive measurement windows of {measurement_window_seconds} " - f"seconds each. " + f"The run was evaluated {settle_clause}using the {acceptance_criterion} " + f"over consecutive measurement intervals. " f"{targets_clause} The {stream_descriptor} was {binding_name} " f"({binding_camera}{workload_clause}), targeting {bt:.2f} FPS and " - f"with a p90 of {bm:.2f} FPS against its {bp:.2f} FPS minimum, leaving " + f"measuring {bm:.6f} FPS against its {bp:.2f} FPS throughput threshold, leaving " f"{absolute_headroom:.2f} FPS " f"({relative_headroom:.1f}% relative headroom).") @@ -862,9 +915,9 @@ def run_pipeline_iterations( f"INFO: Stream density {target_fps_label} " f"with container_name {container_name} " f"and INIT_DURATION set for {INIT_DURATION} seconds; " - f"measurement window {measurement_window_seconds} seconds; " - f"required agreement {consecutive_pass_windows} pass / " - f"{consecutive_fail_windows} fail windows") + f"measurement interval {measurement_window_seconds} seconds; " + f"{describe_run_acceptance_criterion(consecutive_pass_windows)}; " + f"{consecutive_fail_windows} consecutive failing intervals to decrement") while not meet_target_fps: # --- Memory check before scaling up --- @@ -892,7 +945,7 @@ def run_pipeline_iterations( benchmark.docker_compose_containers( "up", compose_files=compose_files, compose_post_args="-d", env_vars=env_vars) - print("waiting for pipelines to settle...") + print(f"waiting for warmup period ({INIT_DURATION} s)...") time.sleep(INIT_DURATION) # note: before reading the pipeline log files @@ -912,20 +965,23 @@ def run_pipeline_iterations( return num_pipelines, False, streams_sustained # once we have all non-empty pipeline log files # capture where each stream log ends so only samples produced - # during the measurement window feed the decision - window_start_offsets = snapshot_stream_log_offsets( - results_dir, container_name) - print( - f"INFO: SWEEPING-START (Measurement Window Start): " - f"{time.strftime('%Y-%m-%dT%H:%M:%S')}") - time.sleep(measurement_window_seconds) - window_end_offsets = snapshot_stream_log_offsets( - results_dir, container_name) - print( - f"INFO: SWEEPING-STOP (Measurement Window Stop): " - f"{time.strftime('%Y-%m-%dT%H:%M:%S')}") + # during the measurement interval feed the decision + (window_start_offsets, window_end_offsets, + actual_measurement_seconds, sample_counts, + enough_samples) = collect_measurement_window( + num_pipelines, results_dir, container_name, + measurement_window_seconds) + if not enough_samples: + print( + "INCONCLUSIVE: Measurement interval did not collect " + f"{MIN_SAMPLES_PER_STREAM} samples from every pipeline log " + f"after {actual_measurement_seconds:.1f} s " + f"(limit {measurement_window_seconds * 2} s).") + for log_file, count in sorted(sample_counts.items()): + print(f" {os.path.basename(log_file)}: {count} samples") + return num_pipelines, False, streams_sustained # --- Calculate FPS and latency metrics --- - total_fps, min_p90_across_streams, stream_fps_dict, stream_samples_dict = calculate_multi_stream_fps( + total_fps, min_stream_fps, stream_fps_dict, stream_samples_dict = calculate_multi_stream_fps( num_pipelines, results_dir, container_name, env_vars, start_offsets=window_start_offsets, end_offsets=window_end_offsets) @@ -934,9 +990,8 @@ def run_pipeline_iterations( print('container name:', container_name) print('Total FPS:', total_fps) print('stream_fps_dict:', stream_fps_dict) - # Calculate average p90 across all streams - avg_p90_per_stream = statistics.mean([v for v in stream_fps_dict.values() if float(v) > 0]) if stream_fps_dict.values() else 0.0 - print(f"Averaged FPS per stream (mean p90): {avg_p90_per_stream} " + avg_fps_per_stream = statistics.mean(stream_fps_dict.values()) if stream_fps_dict else 0.0 + print(f"Averaged FPS per stream (time-weighted): {avg_fps_per_stream} " f"for {num_pipelines} pipeline(s)") total_pipeline_latency, total_pipeline_latency_per_stream = calculate_pipeline_latency( @@ -961,7 +1016,7 @@ def run_pipeline_iterations( name: fps for name, fps in stream_fps_dict.items() if fps >= pass_thresholds[name] } - all_streams_meet_target = len(passing_streams) == len(stream_fps_dict) + all_streams_meet_target = bool(stream_fps_dict) and len(passing_streams) == len(stream_fps_dict) if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": print('pass_thresholds:', pass_thresholds) print('passing_streams:', passing_streams) @@ -976,8 +1031,8 @@ def run_pipeline_iterations( increments = int(env_vars[PIPELINE_INCR_KEY]) else: per_stream_values = list(stream_fps_dict.values()) - robust_per_stream = statistics.median(per_stream_values) if per_stream_values else min_p90_across_streams - conservative_per_stream = min(min_p90_across_streams, robust_per_stream) + robust_per_stream = statistics.median(per_stream_values) + conservative_per_stream = min(min_stream_fps, robust_per_stream) average_target_fps = get_mean_target_fps(stream_target_fps, target_fps) if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": print('mean target fps:', average_target_fps) @@ -989,7 +1044,7 @@ def run_pipeline_iterations( else: increments = min(increments, max_increment) print( - f"✅ All streams meet pass thresholds {pass_thresholds} " + f"✅ All streams meet throughput thresholds {pass_thresholds} " f"for target FPS map {stream_target_fps}. " f"Incrementing pipeline no. by {increments}" ) @@ -1001,16 +1056,16 @@ def run_pipeline_iterations( in_decrement = True fail_window_count = 0 print( - f"⚠️ Pass thresholds not met in streams: observed={stream_fps_dict}, " - f"pass_thresholds={pass_thresholds}. " + f"⚠️ throughput thresholds not met in streams: observed={stream_fps_dict}, " + f"throughput thresholds={pass_thresholds}. " f"Observed {consecutive_fail_windows} consecutive " - f"below-threshold windows; starting to decrement pipelines by 1..." + f"below throughput threshold intervals; starting to decrement pipelines by 1..." ) else: increments = 0 print( - f"INFO: Streams are below pass thresholds ({pass_thresholds}). " - f"Fail window {fail_window_count}/{consecutive_fail_windows}; " + f"INFO: Streams are below throughput threshold ({pass_thresholds}). " + f"Fail interval {fail_window_count}/{consecutive_fail_windows}; " f"holding pipeline count at {num_pipelines} for confirmation." ) else: @@ -1036,11 +1091,12 @@ def run_pipeline_iterations( pass_thresholds, measurement_window_seconds, consecutive_pass_windows, consecutive_fail_windows, pass_tolerance_ratio, stream_camera_meta, - INIT_DURATION, stream_samples_dict) + INIT_DURATION, stream_samples_dict, + actual_measurement_seconds, sample_counts) else: increments = 0 print( - f"✅ Target FPS met again. Pass window " + f"✅ Target FPS met again. Pass interval " f"{pass_window_count}/{consecutive_pass_windows}; " f"holding pipeline count at {num_pipelines} for confirmation." ) @@ -1065,7 +1121,8 @@ def run_pipeline_iterations( pass_thresholds, measurement_window_seconds, consecutive_pass_windows, consecutive_fail_windows, pass_tolerance_ratio, stream_camera_meta, - INIT_DURATION, stream_samples_dict) + INIT_DURATION, stream_samples_dict, + actual_measurement_seconds, sample_counts) meet_target_fps = False break else: @@ -1076,14 +1133,14 @@ def run_pipeline_iterations( fail_window_count = 0 print( f"decrementing number of pipelines {num_pipelines} by 1 " - f"because streams stayed below pass thresholds " - f"({pass_thresholds}) for {consecutive_fail_windows} windows." + f"because streams stayed below throughput thresholds " + f"({pass_thresholds}) for {consecutive_fail_windows} intervals." ) else: increments = 0 print( - f"INFO: Below pass thresholds ({pass_thresholds}). " - f"Fail window {fail_window_count}/{consecutive_fail_windows}; " + f"INFO: Below throughput thresholds ({pass_thresholds}). " + f"Fail interval {fail_window_count}/{consecutive_fail_windows}; " f"holding pipeline count at {num_pipelines} for confirmation." ) @@ -1208,24 +1265,94 @@ def snapshot_stream_log_offsets(results_dir, container_name): return offsets -def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_vars=None, start_offsets=None, end_offsets=None): - """ - Calculates reporting and decision FPS metrics per stream from - log files named pipeline_stream*.log. +def count_measurement_samples(num_pipelines, results_dir, container_name, + start_offsets, end_offsets): + """Count valid samples in each expected pipeline log for this interval.""" + sample_counts = {} + stream_count = get_pipeline_stream_count() + expected_log_count = stream_count + for stream_index in range(stream_count): + pattern = os.path.join( + results_dir, + f'pipeline_stream{stream_index}_*_{container_name}.log') + matching = filter_logs_for_current_timestamp(glob.glob(pattern)) + pipeline_logs = get_latest_pipeline_stream_logs( + 1, matching) + for pipeline_file in pipeline_logs: + try: + start_offset = start_offsets.get(pipeline_file, 0) + end_offset = end_offsets.get(pipeline_file) + if end_offset is None: + sample_counts[pipeline_file] = 0 + continue + samples = extract_numeric_fps( + pipeline_file, + start_offset=start_offset, + end_offset=end_offset) + sample_counts[pipeline_file] = len(samples) + except (IOError, OSError) as error: + print(f"WARN: Could not count samples in {pipeline_file}: {error}") + sample_counts[pipeline_file] = 0 + return sample_counts, expected_log_count + + +def collect_measurement_window(num_pipelines, results_dir, container_name, + minimum_duration_seconds, + minimum_samples=MIN_SAMPLES_PER_STREAM): + """Collect one interval, extending it only as needed to meet the sample floor.""" + start_offsets = snapshot_stream_log_offsets(results_dir, container_name) + start_time = time.monotonic() + max_extension_seconds = minimum_duration_seconds + deadline = start_time + minimum_duration_seconds + max_extension_seconds + print( + f"INFO: SWEEPING-START (Measurement Interval Start): " + f"{time.strftime('%Y-%m-%dT%H:%M:%S')}") - Reporting metric: - - total_fps: sum of per-stream average FPS values. + while True: + elapsed = time.monotonic() - start_time + if elapsed < minimum_duration_seconds: + time.sleep(min(MEASUREMENT_SAMPLE_POLL_SECONDS, + minimum_duration_seconds - elapsed)) + continue - Decision metrics: - - stream_fps_dict: per-stream p90 FPS (minimum p90 across files for each stream). - - min_p90_across_streams: minimum p90 across all streams (bottleneck performance). + end_offsets = snapshot_stream_log_offsets(results_dir, container_name) + sample_counts, expected_log_count = count_measurement_samples( + num_pipelines, results_dir, container_name, + start_offsets, end_offsets) + elapsed = time.monotonic() - start_time + enough_samples = ( + expected_log_count > 0 + and len(sample_counts) == expected_log_count + and all(count >= minimum_samples + for count in sample_counts.values()) + ) + if enough_samples or elapsed >= deadline - start_time: + print( + f"INFO: SWEEPING-STOP (Measurement Interval Stop): " + f"{time.strftime('%Y-%m-%dT%H:%M:%S')}") + return (start_offsets, end_offsets, elapsed, + sample_counts, enough_samples) + + print( + f"INFO: Measurement interval has {len(sample_counts)}/" + f"{expected_log_count} pipeline logs; minimum samples so far: " + f"{min(sample_counts.values()) if sample_counts else 0}/" + f"{minimum_samples}. Extending interval.") + remaining = max(0.0, deadline - time.monotonic()) + if remaining: + time.sleep(min(MEASUREMENT_SAMPLE_POLL_SECONDS, remaining)) + + +def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_vars=None, start_offsets=None, end_offsets=None): + """ + Calculate per-stream FPS from the selected measurement-log interval. Each stream index is handled independently. Returns: - total_fps: sum of per-stream average FPS (reporting) - min_p90_across_streams: minimum p90 across all streams (decision) - stream_fps_dict: stream -> minimum p90 FPS across matching files - stream_samples_dict: stream -> FPS samples from the file that gave that p90 + total_fps: sum of per-stream time-weighted FPS + min_stream_fps: lowest per-stream time-weighted FPS + stream_fps_dict: stream -> time-weighted FPS used for pass/fail + stream_samples_dict: stream -> FPS samples used for p10/p90 reporting """ stream_count = get_pipeline_stream_count() @@ -1243,73 +1370,65 @@ def calculate_multi_stream_fps(num_pipelines, results_dir, container_name, env_v if not matching: print(f"[WARN] No log file found for stream {idx} (container: {container_name}). Skipping...") + stream_fps_dict[f'pipeline_stream{idx}'] = 0.0 continue print(f"DEBUG: idx={idx}, match_count={len(matching)}, pattern={pattern}") - latest_pipeline_logs = get_latest_pipeline_stream_logs(num_pipelines, matching) + latest_pipeline_logs = get_latest_pipeline_stream_logs( + 1, [path for path in matching if start_offsets is None or path in start_offsets]) if not latest_pipeline_logs: print(f"WARN: No log file for stream index {idx}") stream_fps_dict[f'pipeline_stream{idx}'] = 0.0 continue - # --- Process all matching log files for this stream --- - stream_fps_sum = 0.0 - stream_sample_count = 0 - stream_min_p90 = None - stream_min_p90_samples = [] + stream_rate = None + selected_samples = [] for pipeline_file in latest_pipeline_logs: print(f"DEBUG: Processing file: {pipeline_file}") try: - fps_data = extract_numeric_fps( + samples = extract_fps_samples( pipeline_file, start_offset=(start_offsets or {}).get(pipeline_file, 0), end_offset=(end_offsets or {}).get(pipeline_file)) - if not fps_data: - print(f"WARN: No numeric FPS entries for {pipeline_file}") + if not samples: + print(f"WARN: No valid FPS entries for {pipeline_file}") continue - measurement_fps = fps_data - - stream_fps_sum += sum(measurement_fps) - stream_sample_count += len(measurement_fps) - sorted_fps = sorted(measurement_fps) - # Industry-standard p90 (90th percentile) using nearest-rank method with proper rounding - # Fixes small-sample bias in floor-based calculation - p90_idx = min(math.ceil(0.9 * len(sorted_fps)) - 1, len(sorted_fps) - 1) - stream_fps_p90 = sorted_fps[p90_idx] - if stream_min_p90 is None or stream_fps_p90 < stream_min_p90: - stream_min_p90 = stream_fps_p90 - stream_min_p90_samples = measurement_fps + duration = sum(seconds for _, seconds in samples) + weighted_frames = sum(fps * seconds for fps, seconds in samples) + rate = round(weighted_frames / duration, 6) + print( + f"RATE-CALC {os.path.basename(pipeline_file)}: " + f"weighted_frames_estimate={weighted_frames:.6f} " + f"total_duration_seconds={duration:.6f} " + f"samples={len(samples)} measured_fps={rate:.6f}") + if stream_rate is None or rate < stream_rate: + stream_rate = rate + selected_samples = samples if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": - print(f"INFO: P90 FPS for {pipeline_file}: {stream_fps_p90} (samples={len(measurement_fps)})") + print(f"INFO: Time-weighted FPS for {pipeline_file}: {rate} (samples={len(samples)}, duration={duration})") else: - print(f"INFO: P90 FPS for {pipeline_file}: {stream_fps_p90}") + print(f"INFO: Time-weighted FPS for {pipeline_file}: {rate}") except (IOError, OSError) as e: print(f"WARN: Read error on {pipeline_file}: {e}") - # --- Compute reporting average and decision p90 for this stream index --- - if num_pipelines > 0 and stream_sample_count > 0: - report_stream_avg = stream_fps_sum / stream_sample_count - total_fps += report_stream_avg - stream_fps_dict[f'pipeline_stream{idx}'] = round(stream_min_p90, 2) - stream_samples_dict[f'pipeline_stream{idx}'] = stream_min_p90_samples + if stream_rate is not None: + total_fps += stream_rate + stream_fps_dict[f'pipeline_stream{idx}'] = stream_rate + stream_samples_dict[f'pipeline_stream{idx}'] = selected_samples else: stream_fps_dict[f'pipeline_stream{idx}'] = 0.0 print(f"WARN: No valid FPS data for stream index {idx}") - # --- Decision metric: bottleneck p90 across all streams --- - decision_stream_p90 = [ - float(v) for v in stream_fps_dict.values() if float(v) > 0 - ] - min_p90_across_streams = min(decision_stream_p90) if decision_stream_p90 else 0.0 + min_stream_fps = min(stream_fps_dict.values()) if stream_fps_dict else 0.0 - return total_fps, min_p90_across_streams, stream_fps_dict, stream_samples_dict + return total_fps, min_stream_fps, stream_fps_dict, stream_samples_dict def get_pipeline_stream_count(base_dir=None): diff --git a/benchmark-scripts/stream_density_rate_test.py b/benchmark-scripts/stream_density_rate_test.py new file mode 100644 index 0000000..50df0ce --- /dev/null +++ b/benchmark-scripts/stream_density_rate_test.py @@ -0,0 +1,114 @@ +import io +import os +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest.mock import patch + +import stream_density + + +class StreamDensityRateTest(unittest.TestCase): + def test_weighted_rate_uses_only_measurement_rows(self): + samples = [(14.97, 1.00), (14.99, 1.00), (14.94, 1.00), + (15.76, 1.02), (14.97, 1.00), (14.42, 1.04), + (14.97, 1.00), (14.96, 1.00)] + with tempfile.TemporaryDirectory() as results_dir, \ + patch.dict(os.environ, {'TIMESTAMP': ''}), \ + patch('stream_density.get_pipeline_stream_count', return_value=1): + log_file = os.path.join(results_dir, 'pipeline_stream0_123_gst.log') + with open(log_file, 'w') as output: + output.write('fps,duration_seconds\n1.00,1.00\n') + start_offsets = stream_density.snapshot_stream_log_offsets(results_dir, 'gst') + with open(log_file, 'a') as output: + for fps, seconds in samples: + output.write(f'{fps:.2f},{seconds:.2f}\n') + end_offsets = stream_density.snapshot_stream_log_offsets(results_dir, 'gst') + with open(log_file, 'a') as output: + output.write('99.00,1.00\n') + + calculation_log = io.StringIO() + with redirect_stdout(calculation_log): + total, minimum, rates, fps_samples = stream_density.calculate_multi_stream_fps( + 1, results_dir, 'gst', start_offsets=start_offsets, end_offsets=end_offsets) + + expected_rate = sum(fps * seconds for fps, seconds in samples) / sum( + seconds for _, seconds in samples) + self.assertAlmostEqual(expected_rate, 120.872 / 8.06) + self.assertAlmostEqual(rates['pipeline_stream0'], expected_rate, places=6) + self.assertAlmostEqual(total, expected_rate, places=6) + self.assertAlmostEqual(minimum, expected_rate, places=6) + self.assertEqual(fps_samples['pipeline_stream0'], samples) + self.assertIn('weighted_frames_estimate=120.872000', calculation_log.getvalue()) + self.assertIn('total_duration_seconds=8.060000', calculation_log.getvalue()) + self.assertIn('measured_fps=14.996526', calculation_log.getvalue()) + + def test_report_uses_weighted_rate_and_displays_p10_and_p90(self): + fps_samples = [10.0, 10.0] + [15.0] * 8 + with tempfile.TemporaryDirectory() as results_dir, \ + patch.dict(os.environ, {'TIMESTAMP': ''}), \ + patch('stream_density.get_pipeline_stream_count', return_value=1): + log_file = os.path.join(results_dir, 'pipeline_stream0_123_gst.log') + with open(log_file, 'w') as output: + output.write('fps,duration_seconds\n') + for fps in fps_samples: + output.write(f'{fps:.2f},1.00\n') + _, _, rates, samples = stream_density.calculate_multi_stream_fps( + 1, results_dir, 'gst') + + output = io.StringIO() + with redirect_stdout(output): + stream_density.print_stream_density_report( + 1, rates, {'pipeline_stream0': 15.0}, + {'pipeline_stream0': 14.25}, 100, 2, 2, 0.95, + stream_samples=samples) + + report = output.getvalue() + self.assertEqual(rates['pipeline_stream0'], 14.0) + self.assertIn('throughput threshold', report) + self.assertIn('measured (avg)', report) + self.assertIn('p10', report) + self.assertIn('p90', report) + self.assertIn('14.0000', report) + self.assertIn('10.00', report) + self.assertIn('15.00', report) + self.assertIn('seconds below throughput threshold', report) + self.assertIn('2.00 s (20%)', report) + self.assertIn('Result: Fail', report) + self.assertIn('measuring 14.000000 FPS', report) + + def test_sweep_fails_on_weighted_rate_even_when_p90_passes(self): + responses = [ + (15.0, 15.0, {'pipeline_stream0': 15.0}, {'pipeline_stream0': [15.0] * 100}), + (14.0, 14.0, {'pipeline_stream0': 14.0}, + {'pipeline_stream0': [10.0] * 20 + [15.0] * 80}), + (15.0, 15.0, {'pipeline_stream0': 15.0}, {'pipeline_stream0': [15.0] * 100}), + ] + env_vars = {'INIT_DURATION': '0', 'PIPELINE_INC': '1', + 'CONSECUTIVE_FAIL_WINDOWS': '1', + 'CONSECUTIVE_PASS_WINDOWS': '1'} + with patch('stream_density.measure_pipeline_memory', return_value=100), \ + patch('stream_density.clean_up_pipeline_logs'), \ + patch('stream_density.check_can_add_pipelines', return_value=True), \ + patch('stream_density.benchmark.docker_compose_containers'), \ + patch('stream_density.time.sleep'), \ + patch('stream_density.check_non_empty_result_logs'), \ + patch('stream_density.collect_measurement_window', + return_value=({}, {}, 100.0, {'pipeline_stream0.log': 100}, True)), \ + patch('stream_density.calculate_multi_stream_fps', side_effect=responses) as calculate, \ + patch('stream_density.calculate_pipeline_latency', return_value=(0.0, 0.0)), \ + patch('stream_density.build_per_stream_target_fps', + return_value={'pipeline_stream0': 15.0}), \ + patch('stream_density.build_per_stream_camera_meta', return_value={}), \ + patch('stream_density.print_stream_density_report') as report: + with redirect_stdout(io.StringIO()): + num_pipelines, passed, _ = stream_density.run_pipeline_iterations( + env_vars, ['docker-compose.yml'], '/tmp/results', 'gst', 15.0) + + self.assertEqual((num_pipelines, passed), (1, True)) + self.assertEqual(calculate.call_count, 3) + self.assertEqual(report.call_args.args[1], {'pipeline_stream0': 15.0}) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/benchmark-scripts/stream_density_test.py b/benchmark-scripts/stream_density_test.py index c812c65..d18a877 100644 --- a/benchmark-scripts/stream_density_test.py +++ b/benchmark-scripts/stream_density_test.py @@ -52,15 +52,15 @@ def test_build_per_stream_target_fps_mixed_values(self): try: with patch.dict(os.environ, {'CAMERA_STREAM': temp_config_path}, clear=False): result = stream_density.build_per_stream_target_fps( - stream_fps_dict, default_target_fps=14.95 + stream_fps_dict, default_target_fps=15 ) expected = { 'pipeline_stream0': 30.0, 'pipeline_stream1': 15.5, - 'pipeline_stream2': 14.95, - 'pipeline_stream3': 14.95, - 'pipeline_stream4': 14.95, + 'pipeline_stream2': 15, + 'pipeline_stream3': 15, + 'pipeline_stream4': 15, } self.assertEqual(result, expected) finally: @@ -176,15 +176,17 @@ def test_calculate_multi_stream_fps_success(self): with patch('stream_density.get_pipeline_stream_count', return_value=1): - total_fps, min_p90, stream_fps, stream_samples = ( + total_fps, min_stream_fps, stream_fps, stream_samples = ( stream_density.calculate_multi_stream_fps( 1, test_results_dir, 'gst')) self.assertEqual(total_fps, 5.5) - # Nearest-rank p90 of 10 samples is the 9th value, not the top sample. - self.assertEqual(min_p90, 9) - self.assertEqual(stream_fps, {'pipeline_stream0': 9}) - self.assertEqual(stream_samples, {'pipeline_stream0': samples}) + # Legacy single-value rows weigh each sample as 1 second. + self.assertEqual(min_stream_fps, 5.5) + self.assertEqual(stream_fps, {'pipeline_stream0': 5.5}) + self.assertEqual( + stream_samples, + {'pipeline_stream0': [(float(s), 1.0) for s in samples]}) def test_calculate_multi_stream_fps_measurement_window(self): with tempfile.TemporaryDirectory() as test_results_dir, \ @@ -204,24 +206,93 @@ def test_calculate_multi_stream_fps_measurement_window(self): with open(log_file, 'a') as file: file.write('99\n100\n') - total_fps, min_p90, stream_fps, stream_samples = ( + total_fps, min_stream_fps, stream_fps, stream_samples = ( stream_density.calculate_multi_stream_fps( 1, test_results_dir, 'gst', start_offsets=start_offsets, end_offsets=end_offsets)) - self.assertEqual(stream_samples, {'pipeline_stream0': [14, 15, 16]}) + self.assertEqual( + stream_samples, + {'pipeline_stream0': [(14.0, 1.0), (15.0, 1.0), (16.0, 1.0)]}) self.assertEqual(total_fps, 15.0) - self.assertEqual(stream_fps, {'pipeline_stream0': 16}) - self.assertEqual(min_p90, 16) + self.assertEqual(stream_fps, {'pipeline_stream0': 15.0}) + self.assertEqual(min_stream_fps, 15.0) + + def test_count_measurement_samples_counts_generated_stream_logs_once(self): + with tempfile.TemporaryDirectory() as test_results_dir, \ + patch.dict(os.environ, {'TIMESTAMP': ''}), \ + patch('stream_density.get_pipeline_stream_count', + return_value=2): + first_log = os.path.join( + test_results_dir, 'pipeline_stream0_lane1_gst.log') + second_log = os.path.join( + test_results_dir, 'pipeline_stream1_lane2_gst.log') + with open(first_log, 'w') as file: + file.write('1\n' * 99) + with open(second_log, 'w') as file: + file.write('1\n' * 100) + + end_offsets = { + first_log: os.path.getsize(first_log), + second_log: os.path.getsize(second_log), + } + counts, expected_count = stream_density.count_measurement_samples( + 2, test_results_dir, 'gst', {}, end_offsets) + + self.assertEqual(expected_count, 2) + self.assertEqual(sorted(counts.values()), [99, 100]) + + def test_collect_measurement_window_extends_to_sample_floor(self): + elapsed = [0.0] + + def advance_time(seconds): + elapsed[0] += seconds + + with patch('stream_density.time.monotonic', + side_effect=lambda: elapsed[0]), \ + patch('stream_density.time.sleep', side_effect=advance_time), \ + patch('stream_density.snapshot_stream_log_offsets', + side_effect=[{}, {'log': 99}, {'log': 100}]), \ + patch('stream_density.count_measurement_samples', + side_effect=[({'log': 99}, 1), ({'log': 100}, 1)]): + result = stream_density.collect_measurement_window( + 1, '/results', 'gst', minimum_duration_seconds=2) + + start_offsets, end_offsets, duration, counts, enough_samples = result + self.assertEqual(start_offsets, {}) + self.assertEqual(end_offsets, {'log': 100}) + self.assertEqual(duration, 3) + self.assertEqual(counts, {'log': 100}) + self.assertTrue(enough_samples) + + def test_collect_measurement_window_caps_extension_at_configured_duration(self): + elapsed = [0.0] + + def advance_time(seconds): + elapsed[0] += seconds + + with patch('stream_density.time.monotonic', + side_effect=lambda: elapsed[0]), \ + patch('stream_density.time.sleep', side_effect=advance_time), \ + patch('stream_density.snapshot_stream_log_offsets', + return_value={}), \ + patch('stream_density.count_measurement_samples', + return_value=({'log': 99}, 1)): + result = stream_density.collect_measurement_window( + 1, '/results', 'gst', minimum_duration_seconds=2) + + self.assertEqual(result[2], 4) + self.assertFalse(result[4]) def test_print_stream_density_report_summary(self): - stream_fps = {'pipeline_stream0': 16.0, 'pipeline_stream1': 15.0} + stream_fps = {'pipeline_stream0': 14.6, 'pipeline_stream1': 15.0} targets = {'pipeline_stream0': 15.0, 'pipeline_stream1': 15.0} pass_marks = {'pipeline_stream0': 14.25, 'pipeline_stream1': 14.25} samples = { - 'pipeline_stream0': [16, 14, 15, 13, 17, 14, 16, 12, 15, 14], - 'pipeline_stream1': [15] * 10, + 'pipeline_stream0': [(fps, 1.0) for fps in + [16, 14, 15, 13, 17, 14, 16, 12, 15, 14]], + 'pipeline_stream1': [(15, 1.0)] * 10, } meta = { 'pipeline_stream0': {'camera': 'cam1', 'workload': 'wl'}, @@ -232,29 +303,39 @@ def test_print_stream_density_report_summary(self): with redirect_stdout(output): stream_density.print_stream_density_report( 18, stream_fps, targets, pass_marks, 100, 2, 2, 0.95, - meta, 10, samples) + meta, 10, samples, 103.4, + {'pipeline_stream0_lane1_gst.log': 100, + 'pipeline_stream1_lane1_gst.log': 103}) report = output.getvalue() - self.assertIn('Lanes sustained 18', report) - self.assertIn('Streams sustained 2', report) + self.assertIn('Use case density (lanes) 18', report) + self.assertIn('Stream density (streams) 2', report) + self.assertIn('Warmup period 10 s', report) + self.assertIn('Measurement interval 100 s minimum', report) + self.assertIn('run acceptance criterion (two consecutive passes)', report) + self.assertIn('Actual measurement time 103.4 s (minimum 100 s)', report) + self.assertIn( + 'Samples per pipeline log 100-103 across 2 logs (minimum 100)', + report) header = next(line for line in report.splitlines() if line.startswith('stream')) - for column in ('target', 'pass mark', 'p90', - 'seconds below pass mark', 'result'): + for column in ('target', 'throughput threshold', 'measured (avg)', + 'p10', 'p90', 'seconds below throughput threshold', 'result'): self.assertIn(column, header) - rows = {line.split()[0]: line.split() for line in report.splitlines() + rows = {line.split()[0]: line for line in report.splitlines() if line.startswith('pipeline_stream')} - self.assertEqual( - rows['pipeline_stream0'][3:], - ['15.00', '14.25', '16.00', '5/10', '(50%)', 'pass']) - self.assertEqual( - rows['pipeline_stream1'][3:], - ['15.00', '14.25', '15.00', '0/10', '(0%)', 'pass']) + self.assertIn('14.25', rows['pipeline_stream0']) + self.assertIn('14.600000', rows['pipeline_stream0']) + self.assertIn('12.00', rows['pipeline_stream0']) + self.assertIn('16.00', rows['pipeline_stream0']) + self.assertIn('5.00 s (50%)', rows['pipeline_stream0']) + self.assertIn('pass', rows['pipeline_stream0']) + self.assertIn('0.00 s (0%)', rows['pipeline_stream1']) self.assertIn( - 'The lowest-throughput stream was pipeline_stream1', report) + 'The lowest-throughput stream was pipeline_stream0', report) self.assertIn('targeting 15.00 FPS', report) - self.assertIn('p90 of 15.00 FPS', report) - self.assertIn('against its 14.25 FPS minimum', report) + self.assertIn('measuring 14.600000 FPS', report) + self.assertIn('against its 14.25 FPS throughput threshold', report) def test_count_valid_streams(self): stream_fps_dict = { @@ -382,6 +463,9 @@ def test_validate_and_setup_env(self): self.fail(f"Unexpected exception raised: {ex}") @patch('time.sleep', return_value=None) + @patch('stream_density.collect_measurement_window', + return_value=({}, {}, 100.0, + {'pipeline_stream0_lane_gst.log': 100}, True)) @patch('benchmark.docker_compose_containers') @patch('stream_density.calculate_multi_stream_fps') @patch('stream_density.calculate_pipeline_latency', @@ -401,6 +485,7 @@ def test_pipeline_iterations( mock_calculate_latency, mock_calculate_fps, mock_docker_compose, + mock_collect_window, mock_sleep ): test_cases = [ @@ -443,6 +528,21 @@ def test_pipeline_iterations( (10.0, 10.0, {"pipeline_stream0": 10.0}, {}), ] }, + { + "env_vars": {"INIT_DURATION": "10"}, + "compose_files": ["docker-compose.yml"], + "results_dir": "/path/to/results", + "container_name": "insufficient_samples", + "target_fps": 15.0, + "expected_num_pipelines": 1, + "expected_meet_target_fps": False, + "expected_streams_sustained": 0, + "measurement_window_result": ( + {}, {}, 200.0, {"pipeline_stream0_lane1_gst.log": 99}, + False), + "calculate_fps_side_effect": [], + "expect_no_fps_calculation": True, + }, ] for i, test_case in enumerate(test_cases): @@ -453,8 +553,13 @@ def test_pipeline_iterations( container_name = test_case["container_name"] target_fps = test_case["target_fps"] + mock_calculate_fps.reset_mock() mock_calculate_fps.side_effect = test_case[ "calculate_fps_side_effect"] + mock_collect_window.return_value = test_case.get( + "measurement_window_result", + ({}, {}, 100.0, + {'pipeline_stream0_lane_gst.log': 100}, True)) num_pipelines, meet_target_fps, streams_sustained = ( stream_density.run_pipeline_iterations( env_vars, compose_files, results_dir, @@ -467,6 +572,8 @@ def test_pipeline_iterations( meet_target_fps, test_case["expected_meet_target_fps"]) self.assertEqual( streams_sustained, test_case["expected_streams_sustained"]) + if test_case.get("expect_no_fps_calculation"): + mock_calculate_fps.assert_not_called() @patch('time.sleep', return_value=None) @patch('stream_density.validate_and_setup_env') From f326aba108c2454252445d282ca69a071c28b3ad Mon Sep 17 00:00:00 2001 From: sumanaga Date: Wed, 7 Oct 2026 13:05:57 +0530 Subject: [PATCH 11/12] fix(stream-density)!: require durations for FPS samples FPS-only rows lack timing data, so treating them as one-second samples can skew weighted throughput and density pass/fail results. Make the duration requirement explicit and keep related regression coverage together, including the five-lane, six-camera stream count. BREAKING CHANGE: FPS-only pipeline log rows are ignored; consumers must provide both FPS and duration fields. Signed-off-by: sumanaga --- benchmark-scripts/stream_density.py | 7 +- benchmark-scripts/stream_density_rate_test.py | 114 --------------- benchmark-scripts/stream_density_test.py | 137 +++++++++++++++++- 3 files changed, 133 insertions(+), 125 deletions(-) delete mode 100644 benchmark-scripts/stream_density_rate_test.py diff --git a/benchmark-scripts/stream_density.py b/benchmark-scripts/stream_density.py index ab780e1..66f4611 100644 --- a/benchmark-scripts/stream_density.py +++ b/benchmark-scripts/stream_density.py @@ -504,10 +504,10 @@ def extract_fps_samples(file_path, start_offset=0, end_offset=None): if not row or row == ['fps', 'duration_seconds']: continue try: - if len(row) not in (1, 2): + if len(row) != 2: continue fps = float(row[0]) - seconds = float(row[1]) if len(row) == 2 else 1.0 + seconds = float(row[1]) if not math.isfinite(fps) or not math.isfinite(seconds) or fps < 0 or seconds <= 0: continue samples.append((fps, seconds)) @@ -518,7 +518,7 @@ def extract_fps_samples(file_path, start_offset=0, end_offset=None): def extract_numeric_fps(file_path, start_offset=0, end_offset=None): - """Extract FPS values from legacy logs or FPS/duration rows.""" + """Extract FPS values from FPS/duration rows.""" return [fps for fps, _ in extract_fps_samples(file_path, start_offset, end_offset)] @@ -795,7 +795,6 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, if samples: sample_pairs = [ (float(sample[0]), float(sample[1])) - if isinstance(sample, (tuple, list)) else (float(sample), 1.0) for sample in samples ] fps_values = [fps for fps, _ in sample_pairs] diff --git a/benchmark-scripts/stream_density_rate_test.py b/benchmark-scripts/stream_density_rate_test.py deleted file mode 100644 index 50df0ce..0000000 --- a/benchmark-scripts/stream_density_rate_test.py +++ /dev/null @@ -1,114 +0,0 @@ -import io -import os -import tempfile -import unittest -from contextlib import redirect_stdout -from unittest.mock import patch - -import stream_density - - -class StreamDensityRateTest(unittest.TestCase): - def test_weighted_rate_uses_only_measurement_rows(self): - samples = [(14.97, 1.00), (14.99, 1.00), (14.94, 1.00), - (15.76, 1.02), (14.97, 1.00), (14.42, 1.04), - (14.97, 1.00), (14.96, 1.00)] - with tempfile.TemporaryDirectory() as results_dir, \ - patch.dict(os.environ, {'TIMESTAMP': ''}), \ - patch('stream_density.get_pipeline_stream_count', return_value=1): - log_file = os.path.join(results_dir, 'pipeline_stream0_123_gst.log') - with open(log_file, 'w') as output: - output.write('fps,duration_seconds\n1.00,1.00\n') - start_offsets = stream_density.snapshot_stream_log_offsets(results_dir, 'gst') - with open(log_file, 'a') as output: - for fps, seconds in samples: - output.write(f'{fps:.2f},{seconds:.2f}\n') - end_offsets = stream_density.snapshot_stream_log_offsets(results_dir, 'gst') - with open(log_file, 'a') as output: - output.write('99.00,1.00\n') - - calculation_log = io.StringIO() - with redirect_stdout(calculation_log): - total, minimum, rates, fps_samples = stream_density.calculate_multi_stream_fps( - 1, results_dir, 'gst', start_offsets=start_offsets, end_offsets=end_offsets) - - expected_rate = sum(fps * seconds for fps, seconds in samples) / sum( - seconds for _, seconds in samples) - self.assertAlmostEqual(expected_rate, 120.872 / 8.06) - self.assertAlmostEqual(rates['pipeline_stream0'], expected_rate, places=6) - self.assertAlmostEqual(total, expected_rate, places=6) - self.assertAlmostEqual(minimum, expected_rate, places=6) - self.assertEqual(fps_samples['pipeline_stream0'], samples) - self.assertIn('weighted_frames_estimate=120.872000', calculation_log.getvalue()) - self.assertIn('total_duration_seconds=8.060000', calculation_log.getvalue()) - self.assertIn('measured_fps=14.996526', calculation_log.getvalue()) - - def test_report_uses_weighted_rate_and_displays_p10_and_p90(self): - fps_samples = [10.0, 10.0] + [15.0] * 8 - with tempfile.TemporaryDirectory() as results_dir, \ - patch.dict(os.environ, {'TIMESTAMP': ''}), \ - patch('stream_density.get_pipeline_stream_count', return_value=1): - log_file = os.path.join(results_dir, 'pipeline_stream0_123_gst.log') - with open(log_file, 'w') as output: - output.write('fps,duration_seconds\n') - for fps in fps_samples: - output.write(f'{fps:.2f},1.00\n') - _, _, rates, samples = stream_density.calculate_multi_stream_fps( - 1, results_dir, 'gst') - - output = io.StringIO() - with redirect_stdout(output): - stream_density.print_stream_density_report( - 1, rates, {'pipeline_stream0': 15.0}, - {'pipeline_stream0': 14.25}, 100, 2, 2, 0.95, - stream_samples=samples) - - report = output.getvalue() - self.assertEqual(rates['pipeline_stream0'], 14.0) - self.assertIn('throughput threshold', report) - self.assertIn('measured (avg)', report) - self.assertIn('p10', report) - self.assertIn('p90', report) - self.assertIn('14.0000', report) - self.assertIn('10.00', report) - self.assertIn('15.00', report) - self.assertIn('seconds below throughput threshold', report) - self.assertIn('2.00 s (20%)', report) - self.assertIn('Result: Fail', report) - self.assertIn('measuring 14.000000 FPS', report) - - def test_sweep_fails_on_weighted_rate_even_when_p90_passes(self): - responses = [ - (15.0, 15.0, {'pipeline_stream0': 15.0}, {'pipeline_stream0': [15.0] * 100}), - (14.0, 14.0, {'pipeline_stream0': 14.0}, - {'pipeline_stream0': [10.0] * 20 + [15.0] * 80}), - (15.0, 15.0, {'pipeline_stream0': 15.0}, {'pipeline_stream0': [15.0] * 100}), - ] - env_vars = {'INIT_DURATION': '0', 'PIPELINE_INC': '1', - 'CONSECUTIVE_FAIL_WINDOWS': '1', - 'CONSECUTIVE_PASS_WINDOWS': '1'} - with patch('stream_density.measure_pipeline_memory', return_value=100), \ - patch('stream_density.clean_up_pipeline_logs'), \ - patch('stream_density.check_can_add_pipelines', return_value=True), \ - patch('stream_density.benchmark.docker_compose_containers'), \ - patch('stream_density.time.sleep'), \ - patch('stream_density.check_non_empty_result_logs'), \ - patch('stream_density.collect_measurement_window', - return_value=({}, {}, 100.0, {'pipeline_stream0.log': 100}, True)), \ - patch('stream_density.calculate_multi_stream_fps', side_effect=responses) as calculate, \ - patch('stream_density.calculate_pipeline_latency', return_value=(0.0, 0.0)), \ - patch('stream_density.build_per_stream_target_fps', - return_value={'pipeline_stream0': 15.0}), \ - patch('stream_density.build_per_stream_camera_meta', return_value={}), \ - patch('stream_density.print_stream_density_report') as report: - with redirect_stdout(io.StringIO()): - num_pipelines, passed, _ = stream_density.run_pipeline_iterations( - env_vars, ['docker-compose.yml'], '/tmp/results', 'gst', 15.0) - - self.assertEqual((num_pipelines, passed), (1, True)) - self.assertEqual(calculate.call_count, 3) - self.assertEqual(report.call_args.args[1], {'pipeline_stream0': 15.0}) - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/benchmark-scripts/stream_density_test.py b/benchmark-scripts/stream_density_test.py index d18a877..b9cf448 100644 --- a/benchmark-scripts/stream_density_test.py +++ b/benchmark-scripts/stream_density_test.py @@ -172,7 +172,8 @@ def test_calculate_multi_stream_fps_success(self): log_file = os.path.join( test_results_dir, 'pipeline_stream0_123_gst.log') with open(log_file, 'w') as file: - file.write(''.join(f'{s}\n' for s in samples)) + file.write('fps,duration_seconds\n') + file.write(''.join(f'{s},1.00\n' for s in samples)) with patch('stream_density.get_pipeline_stream_count', return_value=1): @@ -181,7 +182,6 @@ def test_calculate_multi_stream_fps_success(self): 1, test_results_dir, 'gst')) self.assertEqual(total_fps, 5.5) - # Legacy single-value rows weigh each sample as 1 second. self.assertEqual(min_stream_fps, 5.5) self.assertEqual(stream_fps, {'pipeline_stream0': 5.5}) self.assertEqual( @@ -196,15 +196,15 @@ def test_calculate_multi_stream_fps_measurement_window(self): log_file = os.path.join( test_results_dir, 'pipeline_stream0_123_gst.log') with open(log_file, 'w') as file: - file.write('1\n2\n3\n') + file.write('fps,duration_seconds\n1,1.00\n2,1.00\n3,1.00\n') start_offsets = stream_density.snapshot_stream_log_offsets( test_results_dir, 'gst') with open(log_file, 'a') as file: - file.write('14\n15\n16\n') + file.write('14,1.00\n15,1.00\n16,1.00\n') end_offsets = stream_density.snapshot_stream_log_offsets( test_results_dir, 'gst') with open(log_file, 'a') as file: - file.write('99\n100\n') + file.write('99,1.00\n100,1.00\n') total_fps, min_stream_fps, stream_fps, stream_samples = ( stream_density.calculate_multi_stream_fps( @@ -219,6 +219,118 @@ def test_calculate_multi_stream_fps_measurement_window(self): self.assertEqual(stream_fps, {'pipeline_stream0': 15.0}) self.assertEqual(min_stream_fps, 15.0) + def test_extract_fps_samples_requires_duration_column(self): + with tempfile.TemporaryDirectory() as test_results_dir: + log_file = os.path.join(test_results_dir, 'pipeline_stream0.log') + with open(log_file, 'w') as file: + file.write('fps,duration_seconds\n') + file.write('15\n') + file.write('14.5,1.25\n') + + self.assertEqual( + stream_density.extract_fps_samples(log_file), + [(14.5, 1.25)]) + + def test_weighted_rate_uses_only_measurement_rows(self): + samples = [(14.97, 1.00), (14.99, 1.00), (14.94, 1.00), + (15.76, 1.02), (14.97, 1.00), (14.42, 1.04), + (14.97, 1.00), (14.96, 1.00)] + with tempfile.TemporaryDirectory() as results_dir, \ + patch.dict(os.environ, {'TIMESTAMP': ''}), \ + patch('stream_density.get_pipeline_stream_count', return_value=1): + log_file = os.path.join(results_dir, 'pipeline_stream0_123_gst.log') + with open(log_file, 'w') as output: + output.write('fps,duration_seconds\n1.00,1.00\n') + start_offsets = stream_density.snapshot_stream_log_offsets(results_dir, 'gst') + with open(log_file, 'a') as output: + for fps, seconds in samples: + output.write(f'{fps:.2f},{seconds:.2f}\n') + end_offsets = stream_density.snapshot_stream_log_offsets(results_dir, 'gst') + with open(log_file, 'a') as output: + output.write('99.00,1.00\n') + + calculation_log = io.StringIO() + with redirect_stdout(calculation_log): + total, minimum, rates, fps_samples = stream_density.calculate_multi_stream_fps( + 1, results_dir, 'gst', start_offsets=start_offsets, end_offsets=end_offsets) + + expected_rate = sum(fps * seconds for fps, seconds in samples) / sum( + seconds for _, seconds in samples) + self.assertAlmostEqual(expected_rate, 120.872 / 8.06) + self.assertAlmostEqual(rates['pipeline_stream0'], expected_rate, places=6) + self.assertAlmostEqual(total, expected_rate, places=6) + self.assertAlmostEqual(minimum, expected_rate, places=6) + self.assertEqual(fps_samples['pipeline_stream0'], samples) + self.assertIn('weighted_frames_estimate=120.872000', calculation_log.getvalue()) + self.assertIn('total_duration_seconds=8.060000', calculation_log.getvalue()) + self.assertIn('measured_fps=14.996526', calculation_log.getvalue()) + + def test_report_uses_weighted_rate_and_displays_p10_and_p90(self): + fps_samples = [10.0, 10.0] + [15.0] * 8 + with tempfile.TemporaryDirectory() as results_dir, \ + patch.dict(os.environ, {'TIMESTAMP': ''}), \ + patch('stream_density.get_pipeline_stream_count', return_value=1): + log_file = os.path.join(results_dir, 'pipeline_stream0_123_gst.log') + with open(log_file, 'w') as output: + output.write('fps,duration_seconds\n') + for fps in fps_samples: + output.write(f'{fps:.2f},1.00\n') + _, _, rates, samples = stream_density.calculate_multi_stream_fps( + 1, results_dir, 'gst') + + output = io.StringIO() + with redirect_stdout(output): + stream_density.print_stream_density_report( + 1, rates, {'pipeline_stream0': 15.0}, + {'pipeline_stream0': 14.25}, 100, 2, 2, 0.95, + stream_samples=samples) + + report = output.getvalue() + self.assertEqual(rates['pipeline_stream0'], 14.0) + self.assertIn('throughput threshold', report) + self.assertIn('measured (avg)', report) + self.assertIn('p10', report) + self.assertIn('p90', report) + self.assertIn('14.0000', report) + self.assertIn('10.00', report) + self.assertIn('15.00', report) + self.assertIn('seconds below throughput threshold', report) + self.assertIn('2.00 s (20%)', report) + self.assertIn('Result: Fail', report) + self.assertIn('measuring 14.000000 FPS', report) + + def test_sweep_fails_on_weighted_rate_even_when_p90_passes(self): + responses = [ + (15.0, 15.0, {'pipeline_stream0': 15.0}, {'pipeline_stream0': [15.0] * 100}), + (14.0, 14.0, {'pipeline_stream0': 14.0}, + {'pipeline_stream0': [10.0] * 20 + [15.0] * 80}), + (15.0, 15.0, {'pipeline_stream0': 15.0}, {'pipeline_stream0': [15.0] * 100}), + ] + env_vars = {'INIT_DURATION': '0', 'PIPELINE_INC': '1', + 'CONSECUTIVE_FAIL_WINDOWS': '1', + 'CONSECUTIVE_PASS_WINDOWS': '1'} + with patch('stream_density.measure_pipeline_memory', return_value=100), \ + patch('stream_density.clean_up_pipeline_logs'), \ + patch('stream_density.check_can_add_pipelines', return_value=True), \ + patch('stream_density.benchmark.docker_compose_containers'), \ + patch('stream_density.time.sleep'), \ + patch('stream_density.check_non_empty_result_logs'), \ + patch('stream_density.collect_measurement_window', + return_value=({}, {}, 100.0, {'pipeline_stream0.log': 100}, True)), \ + patch('stream_density.calculate_multi_stream_fps', side_effect=responses) as calculate, \ + patch('stream_density.calculate_pipeline_latency', return_value=(0.0, 0.0)), \ + patch('stream_density.build_per_stream_target_fps', + return_value={'pipeline_stream0': 15.0}), \ + patch('stream_density.build_per_stream_camera_meta', return_value={}), \ + patch('stream_density.print_stream_density_report') as report: + with redirect_stdout(io.StringIO()): + num_pipelines, passed, _ = stream_density.run_pipeline_iterations( + env_vars, ['docker-compose.yml'], '/tmp/results', 'gst', 15.0) + + self.assertEqual((num_pipelines, passed), (1, True)) + self.assertEqual(calculate.call_count, 3) + self.assertEqual(report.call_args.args[1], {'pipeline_stream0': 15.0}) + def test_count_measurement_samples_counts_generated_stream_logs_once(self): with tempfile.TemporaryDirectory() as test_results_dir, \ patch.dict(os.environ, {'TIMESTAMP': ''}), \ @@ -229,9 +341,9 @@ def test_count_measurement_samples_counts_generated_stream_logs_once(self): second_log = os.path.join( test_results_dir, 'pipeline_stream1_lane2_gst.log') with open(first_log, 'w') as file: - file.write('1\n' * 99) + file.write('fps,duration_seconds\n' + '1,1.00\n' * 99) with open(second_log, 'w') as file: - file.write('1\n' * 100) + file.write('fps,duration_seconds\n' + '1,1.00\n' * 100) end_offsets = { first_log: os.path.getsize(first_log), @@ -347,6 +459,17 @@ def test_count_valid_streams(self): self.assertEqual( stream_density.count_valid_streams(stream_fps_dict), 2) + def test_count_valid_streams_five_lanes_of_six_cameras(self): + lane_count = 5 + cameras_per_lane = 6 + stream_fps_dict = { + f'pipeline_stream{stream_index}': 15.0 + for stream_index in range(lane_count * cameras_per_lane) + } + + self.assertEqual( + stream_density.count_valid_streams(stream_fps_dict), 30) + def test_clean_up_pipeline_logs(self): test_results_dir = './test_results_clean' testFile1 = os.path.join( From 296e6c3995fa8b9b0408d8fac12436e5ea5daf34 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Fri, 9 Oct 2026 21:30:56 +0530 Subject: [PATCH 12/12] stream_density: map streams via manifest, report inconclusive runs Per-stream targets and report labels used `stream index % cameras` to find a stream's camera. That only holds when pipeline_stream logs are in lane order and every camera has one branch, so above one lane, cameras with different fps/targetFps got the wrong thresholds and labels. - Read src/pipelines/pipeline_streams.json, written by the pipeline generator, to map each pipeline_stream log to its camera config index and branch workloads. This handles multi-branch cameras and skipped lp_vlm cameras. Fail if the manifest and the logs disagree. Fall back to `% cameras` with a warning when no manifest exists. - Label report rows with the branch's own workloads. An interval below the 100-sample floor used to end the sweep as a Fail and report a lane count that was never measured: - Retry an inconclusive interval once without restarting the pipelines. - If it is still inconclusive, end with status "inconclusive" and report the last passing density as not confirmed. - run_pipeline_iterations / run_stream_density now return a status (pass | fail | inconclusive) instead of a boolean. - benchmark.py prints Result: Pass / Fail / Inconclusive through one helper, using "use case density N lanes, stream density M streams". - Reject MEASUREMENT_WINDOW_SECONDS below 100. At about 1 sample/s with collection capped at 2x the interval, shorter intervals can never reach the floor. The --measurement_window_seconds help text says so. Other changes: - Print measured FPS with two decimals in the report table and summary. - consolidate_multiple_run_of_metrics: match ^pipeline.*\.log$ so the pipeline_streams_*.json manifest copies are not parsed as FPS logs. - clean_up_pipeline_logs also removes pipeline_streams_*.json copies. Tests: multi-lane target mapping with camera-ordered logs (2 and 5 lanes), multi-branch and skipped cameras, manifest/log mismatch, inconclusive retry and last-passing density, measurement interval validation, result printing for each status, manifest cleanup. Removed unused imports, so the tests no longer need the `mock` package. Signed-off-by: sumanaga --- benchmark-scripts/benchmark.py | 66 +++--- benchmark-scripts/benchmark_test.py | 36 ++++ .../consolidate_multiple_run_of_metrics.py | 2 +- benchmark-scripts/stream_density.py | 145 ++++++++++--- benchmark-scripts/stream_density_test.py | 203 ++++++++++++++++-- 5 files changed, 371 insertions(+), 81 deletions(-) diff --git a/benchmark-scripts/benchmark.py b/benchmark-scripts/benchmark.py index 342b2ce..ed2f5ba 100644 --- a/benchmark-scripts/benchmark.py +++ b/benchmark-scripts/benchmark.py @@ -79,7 +79,10 @@ def parse_args(print=False): parser.add_argument('--measurement_window_seconds', type=int, default=100, help='duration in seconds to collect FPS samples ' + 'after INIT_DURATION for the stream density ' + - 'pass/fail decision') + 'pass/fail decision; minimum 100, because ' + + 'each pipeline log needs 100 samples at about ' + + 'one per second and collection extends to at ' + + 'most twice this duration') # TODO: change target_device to an env variable in docker compose try: default_target_device = resolve_target_device_default('CPU') @@ -158,6 +161,37 @@ def docker_compose_containers(command, compose_files=[], compose_pre_args="", +def print_stream_density_result(status, num_pipelines, streams_sustained, + results_dir): + if status == stream_density.STATUS_PASS: + print("Result: Pass") + print( + f"Stream density completed: use case density {num_pipelines} lanes, " + f"stream density {streams_sustained} streams. " + f"For the detailed report, see {results_dir}/stream_density.log") + elif status == stream_density.STATUS_INCONCLUSIVE: + print("Result: Inconclusive") + print( + f"Measurement intervals did not collect " + f"{stream_density.MIN_SAMPLES_PER_STREAM} samples from every " + f"pipeline log, so they were not used for a pass/fail decision.") + if num_pipelines: + print( + f"Last passing density: use case density {num_pipelines} lanes, " + f"stream density {streams_sustained} streams, not confirmed by " + f"the run acceptance criterion.") + else: + print("No measurement interval had passed.") + print( + f"Per-log sample counts are in {results_dir}/stream_density.log") + else: + print("Result: Fail") + print( + "Stream density did not meet the target. Check " + f"{results_dir}/stream_density.log and " + f"{results_dir}/gst-launch*.log for further analysis.") + + def main(): ''' runs benchmarking using docker compose for the specified pipeline @@ -232,18 +266,9 @@ def main(): container_names_list, explicit_target_fps) for result in results: - target_fps, container_name, num_pipelines, met_fps, streams_sustained = result - print(f"Result: {'Pass' if met_fps else 'Fail'}") - if met_fps: - print( - f"Stream density completed: {num_pipelines} pipelines, " - f"{streams_sustained} streams sustained. " - f"For the detailed report, see {results_dir}/stream_density.log") - else: - print( - f"Stream density did not meet the target. Check " - f"{results_dir}/stream_density.log and " - f"{results_dir}/gst-launch*.log for further analysis.") + target_fps, container_name, num_pipelines, status, streams_sustained = result + print_stream_density_result( + status, num_pipelines, streams_sustained, results_dir) elif len(target_fps_list) == 1: # single target_fps stream density mode: print('starting stream density...') @@ -258,18 +283,9 @@ def main(): results = stream_density.run_stream_density( env_vars, compose_files, [target_fps_list[0]], [container_name], explicit_target_fps) - target_fps, container_name, num_pipelines, met_fps, streams_sustained = results[0] - print(f"Result: {'Pass' if met_fps else 'Fail'}") - if met_fps: - print( - f"Stream density completed: {num_pipelines} pipelines, " - f"{streams_sustained} streams sustained. " - f"For the detailed report, see {results_dir}/stream_density.log") - else: - print( - "Stream density did not meet the target. Check " - f"{results_dir}/stream_density.log and " - f"{results_dir}/gst-launch*.log for further analysis.") + target_fps, container_name, num_pipelines, status, streams_sustained = results[0] + print_stream_density_result( + status, num_pipelines, streams_sustained, results_dir) else: # regular --pipelines mode: stream_density.clean_up_pipeline_logs(results_dir) diff --git a/benchmark-scripts/benchmark_test.py b/benchmark-scripts/benchmark_test.py index b05f63d..63a62fd 100644 --- a/benchmark-scripts/benchmark_test.py +++ b/benchmark-scripts/benchmark_test.py @@ -8,7 +8,10 @@ import subprocess # nosec B404 import unittest import benchmark +import stream_density +import io import os +from contextlib import redirect_stdout class Testing(unittest.TestCase): @@ -55,5 +58,38 @@ def test_docker_compose_containers_fail(self): mock_popen.communicate.assert_called_once_with() mock_returncode.assert_called() + def _result_output(self, status, num_pipelines, streams): + output = io.StringIO() + with redirect_stdout(output): + benchmark.print_stream_density_result( + status, num_pipelines, streams, '/results') + return output.getvalue() + + def test_print_stream_density_result_pass(self): + output = self._result_output(stream_density.STATUS_PASS, 11, 22) + self.assertIn('Result: Pass', output) + self.assertIn( + 'use case density 11 lanes, stream density 22 streams', output) + self.assertNotIn('sustained', output) + + def test_print_stream_density_result_fail(self): + output = self._result_output(stream_density.STATUS_FAIL, 1, 2) + self.assertIn('Result: Fail', output) + self.assertIn('did not meet the target', output) + + def test_print_stream_density_result_inconclusive(self): + output = self._result_output(stream_density.STATUS_INCONCLUSIVE, 10, 20) + self.assertIn('Result: Inconclusive', output) + self.assertNotIn('Result: Fail', output) + self.assertIn( + 'Last passing density: use case density 10 lanes, ' + 'stream density 20 streams', output) + + def test_print_stream_density_result_inconclusive_without_pass(self): + output = self._result_output(stream_density.STATUS_INCONCLUSIVE, 0, 0) + self.assertIn('Result: Inconclusive', output) + self.assertIn('No measurement interval had passed.', output) + self.assertNotIn('Last passing density', output) + if __name__ == '__main__': unittest.main() diff --git a/benchmark-scripts/consolidate_multiple_run_of_metrics.py b/benchmark-scripts/consolidate_multiple_run_of_metrics.py index 9323c8e..9717c36 100644 --- a/benchmark-scripts/consolidate_multiple_run_of_metrics.py +++ b/benchmark-scripts/consolidate_multiple_run_of_metrics.py @@ -842,7 +842,7 @@ def return_blank(self): KPIExtractor_OPTION = {"meta_summary.txt":MetaExtractor, "camera":FPSExtractor, - "pipeline":PIPELINEFPSExtractor, + r"^pipeline.*\.log$":PIPELINEFPSExtractor, r"(?:^r).*\.jsonl$": PIPELINLastModifiedExtractor, "gst-launch":PipelineLatencyExtractor, "cpu_usage.log":CPUUsageExtractor, diff --git a/benchmark-scripts/stream_density.py b/benchmark-scripts/stream_density.py index 66f4611..7491e58 100644 --- a/benchmark-scripts/stream_density.py +++ b/benchmark-scripts/stream_density.py @@ -35,12 +35,60 @@ DEFAULT_MEASUREMENT_WINDOW_SECONDS = 100 MIN_SAMPLES_PER_STREAM = 100 MEASUREMENT_SAMPLE_POLL_SECONDS = 1 +MAX_INCONCLUSIVE_RETRIES = 1 +STATUS_PASS = "pass" +STATUS_FAIL = "fail" +STATUS_INCONCLUSIVE = "inconclusive" CAMERA_STREAM_KEY = "CAMERA_STREAM" +STREAM_MANIFEST_PATH_KEY = "STREAM_MANIFEST_PATH" class ArgumentError(Exception): pass + +def load_stream_manifest(env_vars=None): + """Return {log_index: (camera_index, branch_workloads)} from the generator's stream manifest, or None if absent.""" + env_vars = env_vars if env_vars is not None else os.environ + manifest_path = env_vars.get(STREAM_MANIFEST_PATH_KEY) or os.path.normpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..", "src", "pipelines", "pipeline_streams.json")) + if not os.path.isfile(manifest_path): + return None + try: + with open(manifest_path, "r", encoding="utf-8") as f: + streams = json.load(f).get("streams", []) + return { + int(entry["log_index"]): (int(entry["camera_index"]), + [str(w) for w in entry.get("workloads", [])]) + for entry in streams + } + except (IOError, ValueError, KeyError, TypeError, AttributeError) as e: + raise ValueError(f"ERROR: invalid stream manifest {manifest_path}: {e}") + + +def resolve_stream_entries(stream_fps_dict, total_cameras, env_vars=None): + """Map each pipeline_stream name to (camera config index, branch workloads or None).""" + stream_pattern = re.compile(r"pipeline_stream(\d+)") + log_indices = {} + for stream_name in stream_fps_dict: + match = stream_pattern.search(stream_name) + if match: + log_indices[stream_name] = int(match.group(1)) + + log_to_entry = load_stream_manifest(env_vars) + if log_to_entry is not None: + missing = sorted(i for i in log_indices.values() if i not in log_to_entry) + if missing or len(log_to_entry) != len(log_indices): + raise ValueError( + f"ERROR: stream manifest lists {len(log_to_entry)} streams but " + f"{len(log_indices)} pipeline logs were found (unmapped: {missing})") + return {name: log_to_entry[i] for name, i in log_indices.items()} + + if total_cameras <= 0: + return {} + print("WARN: stream manifest not found; assuming one log per camera in lane order") + return {name: (i % total_cameras, None) for name, i in log_indices.items()} + def build_per_stream_target_fps(stream_fps_dict, default_target_fps, env_vars=None): """Resolve per-stream FPS targets. @@ -103,8 +151,6 @@ def build_per_stream_target_fps(stream_fps_dict, default_target_fps, env_vars=No cache[cache_key] = ({}, 0) build_per_stream_target_fps._camera_config_cache = cache - # Compile regex once (if needed for stream name parsing) - stream_pattern = re.compile(r"pipeline_stream(\d+)") per_stream_targets = {} # TARGET_FPS from env_vars, when set, overrides every camera's config value. @@ -118,13 +164,14 @@ def build_per_stream_target_fps(stream_fps_dict, default_target_fps, env_vars=No except ValueError: pass + stream_to_entry = {} if forced_target_fps is not None else resolve_stream_entries( + stream_fps_dict, total_cameras, env_vars) for stream_name in stream_fps_dict: if forced_target_fps is not None: per_stream_targets[stream_name] = forced_target_fps continue - match = stream_pattern.search(stream_name) - if match and total_cameras > 0: - camera_idx = int(match.group(1)) % total_cameras + camera_idx = stream_to_entry.get(stream_name, (None, None))[0] + if camera_idx is not None: target_fps = stream_idx_to_target_fps.get(camera_idx, default_target_fps) else: target_fps = default_target_fps @@ -223,13 +270,14 @@ def build_per_stream_camera_meta(stream_fps_dict, env_vars=None): cache[cache_key] = ({}, 0) build_per_stream_camera_meta._camera_meta_cache = cache - stream_pattern = re.compile(r"pipeline_stream(\d+)") + stream_to_entry = resolve_stream_entries(stream_fps_dict, total_cameras, env_vars) per_stream_meta = {} for stream_name in stream_fps_dict: - match = stream_pattern.search(stream_name) - if match and total_cameras > 0: - camera_idx = int(match.group(1)) % total_cameras - meta = idx_to_meta.get(camera_idx, {"camera": stream_name, "workload": "-"}) + camera_idx, branch_workloads = stream_to_entry.get(stream_name, (None, None)) + if camera_idx is not None: + meta = dict(idx_to_meta.get(camera_idx, {"camera": stream_name, "workload": "-"})) + if branch_workloads: + meta["workload"] = ", ".join(branch_workloads) else: meta = {"camera": stream_name, "workload": "-"} per_stream_meta[stream_name] = meta @@ -418,7 +466,8 @@ def clean_up_pipeline_logs(results_dir): matching_files = glob.glob(os.path.join(results_dir, 'pipeline*_*.log')) \ + glob.glob(os.path.join(results_dir, 'gst*_*.log')) \ + glob.glob(os.path.join(results_dir, 'rs*_*.jsonl')) \ - + glob.glob(os.path.join(results_dir, 'qmassa*-*.json')) + + glob.glob(os.path.join(results_dir, 'qmassa*-*.json')) \ + + glob.glob(os.path.join(results_dir, 'pipeline_streams_*.json')) if len(matching_files) > 0: for log_file in matching_files: os.remove(log_file) @@ -683,9 +732,12 @@ def validate_and_setup_env(env_vars, target_fps_list): 'ERROR: run acceptance criterion (consecutive passes) should be greater than 0') if is_env_non_empty(env_vars, MEASUREMENT_WINDOW_SECONDS_KEY) and int( - env_vars[MEASUREMENT_WINDOW_SECONDS_KEY]) <= 0: + env_vars[MEASUREMENT_WINDOW_SECONDS_KEY]) < MIN_SAMPLES_PER_STREAM: raise ArgumentError( - 'ERROR: measurement interval seconds should be greater than 0') + f'ERROR: measurement interval seconds should be at least ' + f'{MIN_SAMPLES_PER_STREAM}: gvafpscounter reports about one sample ' + f'per second, each pipeline log needs {MIN_SAMPLES_PER_STREAM} samples, ' + f'and collection extends to at most twice the interval') if is_env_non_empty(env_vars, PASS_TOLERANCE_RATIO_KEY): pass_tolerance_ratio = float(env_vars[PASS_TOLERANCE_RATIO_KEY]) @@ -812,7 +864,7 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, below_text = "-" print( f"{name:<{stream_w}}{camera:<{camera_w}}{workload:<{workload_w}}{target:>8.2f}" - f"{pass_mark:>22.2f}{measured:>16.6f}{p10:>9.2f}{p90:>9.2f}" + f"{pass_mark:>22.2f}{measured:>16.2f}{p10:>9.2f}{p90:>9.2f}" f"{below_text:>36}{result:>8}") print("") print( @@ -847,7 +899,7 @@ def print_stream_density_report(num_pipelines, stream_fps_dict, f"over consecutive measurement intervals. " f"{targets_clause} The {stream_descriptor} was {binding_name} " f"({binding_camera}{workload_clause}), targeting {bt:.2f} FPS and " - f"measuring {bm:.6f} FPS against its {bp:.2f} FPS throughput threshold, leaving " + f"measuring {bm:.2f} FPS against its {bp:.2f} FPS throughput threshold, leaving " f"{absolute_headroom:.2f} FPS " f"({relative_headroom:.1f}% relative headroom).") @@ -865,8 +917,10 @@ def run_pipeline_iterations( container_name: Name of the container to run. target_fps: Target FPS to achieve. Returns: - num_pipelines: Number of pipelines used. - meet_target_fps: Whether the target FPS was achieved. + num_pipelines: Number of pipelines used; for STATUS_INCONCLUSIVE, the + last lane count whose interval passed (0 if none did). + status: STATUS_PASS, STATUS_FAIL or STATUS_INCONCLUSIVE. + streams_sustained: Streams with valid FPS data at num_pipelines. ''' INIT_DURATION = int(env_vars[INIT_DURATION_KEY]) num_pipelines = 1 @@ -900,6 +954,8 @@ def run_pipeline_iterations( fail_window_count = 0 pass_window_count = 0 streams_sustained = 0 + last_passing_pipelines = 0 + last_passing_streams = 0 # Measure memory usage of a single pipeline per_pipeline_memory_mb = measure_pipeline_memory( @@ -930,7 +986,7 @@ def run_pipeline_iterations( num_pipelines = num_pipelines - increments if num_pipelines < 1: num_pipelines = 1 - return num_pipelines, False, streams_sustained + return num_pipelines, STATUS_FAIL, streams_sustained # Bring down previous iteration's containers before starting new ones print("Stopping previous containers before scaling...") @@ -961,16 +1017,19 @@ def run_pipeline_iterations( num_pipelines = num_pipelines - increments if num_pipelines < 1: num_pipelines = 1 - return num_pipelines, False, streams_sustained + return num_pipelines, STATUS_FAIL, streams_sustained # once we have all non-empty pipeline log files # capture where each stream log ends so only samples produced # during the measurement interval feed the decision - (window_start_offsets, window_end_offsets, - actual_measurement_seconds, sample_counts, - enough_samples) = collect_measurement_window( - num_pipelines, results_dir, container_name, - measurement_window_seconds) - if not enough_samples: + inconclusive_retries = 0 + while True: + (window_start_offsets, window_end_offsets, + actual_measurement_seconds, sample_counts, + enough_samples) = collect_measurement_window( + num_pipelines, results_dir, container_name, + measurement_window_seconds) + if enough_samples: + break print( "INCONCLUSIVE: Measurement interval did not collect " f"{MIN_SAMPLES_PER_STREAM} samples from every pipeline log " @@ -978,7 +1037,22 @@ def run_pipeline_iterations( f"(limit {measurement_window_seconds * 2} s).") for log_file, count in sorted(sample_counts.items()): print(f" {os.path.basename(log_file)}: {count} samples") - return num_pipelines, False, streams_sustained + if inconclusive_retries >= MAX_INCONCLUSIVE_RETRIES: + last_passing_text = ( + f"Last passing density: {last_passing_pipelines} lane(s) " + f"({last_passing_streams} streams), not confirmed by the run " + f"acceptance criterion." + if last_passing_pipelines else "No measurement interval had passed.") + print( + f"Result: Inconclusive - {inconclusive_retries + 1} consecutive " + f"measurement intervals at {num_pipelines} lane(s) were inconclusive " + f"and were not used for a pass/fail decision. {last_passing_text}") + return last_passing_pipelines, STATUS_INCONCLUSIVE, last_passing_streams + inconclusive_retries += 1 + print( + f"INFO: Retrying the measurement interval at {num_pipelines} lane(s) " + f"without restarting the pipelines " + f"(retry {inconclusive_retries}/{MAX_INCONCLUSIVE_RETRIES}).") # --- Calculate FPS and latency metrics --- total_fps, min_stream_fps, stream_fps_dict, stream_samples_dict = calculate_multi_stream_fps( num_pipelines, results_dir, container_name, env_vars, @@ -1016,6 +1090,9 @@ def run_pipeline_iterations( if fps >= pass_thresholds[name] } all_streams_meet_target = bool(stream_fps_dict) and len(passing_streams) == len(stream_fps_dict) + if all_streams_meet_target: + last_passing_pipelines = num_pipelines + last_passing_streams = streams_sustained if os.getenv("STREAM_DENSITY_DEBUG", "0") == "1": print('pass_thresholds:', pass_thresholds) print('passing_streams:', passing_streams) @@ -1159,7 +1236,8 @@ def run_pipeline_iterations( # f"with input target_fps = {target_fps}" #) - return num_pipelines, meet_target_fps, streams_sustained + status = STATUS_PASS if meet_target_fps else STATUS_FAIL + return num_pipelines, status, streams_sustained def run_stream_density(env_vars, compose_files, target_fps_list, @@ -1176,12 +1254,13 @@ def run_stream_density(env_vars, compose_files, target_fps_list, the corresponding target FPS Returns: results as a list of tuples (target_fps, container_name, - num_pipelines, meet_target_fps) where + num_pipelines, status, + streams_sustained) where target_fps: the desire frames per second to maintain for pipeline container_name: the corresponding container name for the pipeline - num_pipelines: maximum number of pipelines to achieve TARGET_FPS - meet_target_fps: boolean to indicate whether the returned - number_pipelines can achieve the TARGET_FPS goal or not + num_pipelines: maximum number of pipelines to achieve TARGET_FPS, + or the last passing count when status is STATUS_INCONCLUSIVE + status: STATUS_PASS, STATUS_FAIL or STATUS_INCONCLUSIVE ''' results = [] validate_and_setup_env(env_vars, target_fps_list) @@ -1207,7 +1286,7 @@ def run_stream_density(env_vars, compose_files, target_fps_list, env_vars[CONTAINER_NAME_KEY] = container_name # stream density main logic: try: - num_pipelines, meet_target_fps, streams_sustained = run_pipeline_iterations( + num_pipelines, status, streams_sustained = run_pipeline_iterations( env_vars, compose_files, results_dir, container_name, target_fps, explicit_target_fps ) @@ -1216,7 +1295,7 @@ def run_stream_density(env_vars, compose_files, target_fps_list, target_fps, container_name, num_pipelines, - meet_target_fps, + status, streams_sustained ) ) diff --git a/benchmark-scripts/stream_density_test.py b/benchmark-scripts/stream_density_test.py index b9cf448..98a9a3f 100644 --- a/benchmark-scripts/stream_density_test.py +++ b/benchmark-scripts/stream_density_test.py @@ -4,12 +4,10 @@ * SPDX-License-Identifier: Apache-2.0 ''' -import mock -import subprocess # nosec B404 import unittest import tempfile import json -from unittest.mock import patch, mock_open, MagicMock +from unittest.mock import patch, mock_open import stream_density from stream_density import validate_and_setup_env, ArgumentError from stream_density import ( @@ -50,7 +48,8 @@ def test_build_per_stream_target_fps_mixed_values(self): temp_config_path = temp_file.name try: - with patch.dict(os.environ, {'CAMERA_STREAM': temp_config_path}, clear=False): + with patch.dict(os.environ, {'CAMERA_STREAM': temp_config_path, + 'STREAM_MANIFEST_PATH': temp_config_path + '.missing'}, clear=False): result = stream_density.build_per_stream_target_fps( stream_fps_dict, default_target_fps=15 ) @@ -77,7 +76,8 @@ def test_build_per_stream_target_fps_missing_config_uses_default(self): with tempfile.TemporaryDirectory() as tmpdir: missing_path = os.path.join(tmpdir, 'non-existent-camera-config-for-test.json') - with patch.dict(os.environ, {'CAMERA_STREAM': missing_path}, clear=False): + with patch.dict(os.environ, {'CAMERA_STREAM': missing_path, + 'STREAM_MANIFEST_PATH': missing_path}, clear=False): result = stream_density.build_per_stream_target_fps( stream_fps_dict, default_target_fps=22.0 ) @@ -89,6 +89,67 @@ def test_build_per_stream_target_fps_missing_config_uses_default(self): } self.assertEqual(result, expected) + def _run_with_manifest(self, cameras, manifest_streams, stream_count, func): + with tempfile.TemporaryDirectory() as tmpdir: + config_path = os.path.join(tmpdir, 'cameras.json') + manifest_path = os.path.join(tmpdir, 'pipeline_streams.json') + with open(config_path, 'w') as f: + json.dump({'lane_config': {'cameras': cameras}}, f) + with open(manifest_path, 'w') as f: + json.dump({'streams': manifest_streams}, f) + stream_fps_dict = {f'pipeline_stream{i}': 10.0 for i in range(stream_count)} + env_vars = {'CAMERA_STREAM': config_path, 'STREAM_MANIFEST_PATH': manifest_path} + with redirect_stdout(io.StringIO()): + return func(stream_fps_dict, env_vars) + + @staticmethod + def _manifest(lane_camera_pairs): + return [{'log_index': i, 'lane': lane, 'camera_index': cam} + for i, (lane, cam) in enumerate(lane_camera_pairs)] + + def test_target_fps_uses_manifest_for_multiple_lanes(self): + cameras = [{'camera_id': f'cam{i + 1}', 'fps': 15} for i in range(6)] + cameras[5]['fps'] = 5 + for lanes in (1, 2, 5): + with self.subTest(lanes=lanes): + manifest = self._manifest([(l, c) for l in range(lanes) for c in range(6)]) + result = self._run_with_manifest( + cameras, manifest, 6 * lanes, + lambda d, e: stream_density.build_per_stream_target_fps(d, 15, env_vars=e)) + slow = sorted(int(n[len('pipeline_stream'):]) for n, v in result.items() if v == 5.0) + self.assertEqual(slow, [6 * l + 5 for l in range(lanes)]) + + def test_camera_meta_handles_multi_branch_and_skipped_cameras(self): + # cam2 is lp_vlm (no branch); cam3 has two branches per lane + cameras = [ + {'camera_id': 'cam1', 'workloads': ['a'], 'fps': 15}, + {'camera_id': 'cam2', 'workloads': ['lp_vlm'], 'fps': 15}, + {'camera_id': 'cam3', 'workloads': ['b', 'c'], 'fps': 5}, + ] + manifest = self._manifest([(l, c) for l in range(2) for c in (0, 2, 2)]) + for entry, workloads in zip(manifest, [['a'], ['b'], ['c']] * 2): + entry['workloads'] = workloads + meta = self._run_with_manifest( + cameras, manifest, 6, + lambda d, e: stream_density.build_per_stream_camera_meta(d, env_vars=e)) + self.assertEqual([meta[f'pipeline_stream{i}']['camera'] for i in range(6)], + ['cam1', 'cam3', 'cam3', 'cam1', 'cam3', 'cam3']) + self.assertEqual([meta[f'pipeline_stream{i}']['workload'] for i in range(6)], + ['a', 'b', 'c', 'a', 'b', 'c']) + targets = self._run_with_manifest( + cameras, manifest, 6, + lambda d, e: stream_density.build_per_stream_target_fps(d, 15, env_vars=e)) + self.assertEqual([targets[f'pipeline_stream{i}'] for i in range(6)], + [15.0, 5.0, 5.0, 15.0, 5.0, 5.0]) + + def test_manifest_log_count_mismatch_raises(self): + cameras = [{'camera_id': 'cam1', 'fps': 15}] + manifest = self._manifest([(0, 0), (1, 0)]) + with self.assertRaises(ValueError): + self._run_with_manifest( + cameras, manifest, 3, + lambda d, e: stream_density.build_per_stream_target_fps(d, 15, env_vars=e)) + def test_is_env_non_empty(self): sys_env = os.environ.copy() sys_env["EMPTY"] = "" @@ -291,13 +352,14 @@ def test_report_uses_weighted_rate_and_displays_p10_and_p90(self): self.assertIn('measured (avg)', report) self.assertIn('p10', report) self.assertIn('p90', report) - self.assertIn('14.0000', report) + self.assertIn('14.00', report) + self.assertNotIn('14.0000', report) self.assertIn('10.00', report) self.assertIn('15.00', report) self.assertIn('seconds below throughput threshold', report) self.assertIn('2.00 s (20%)', report) self.assertIn('Result: Fail', report) - self.assertIn('measuring 14.000000 FPS', report) + self.assertIn('measuring 14.00 FPS', report) def test_sweep_fails_on_weighted_rate_even_when_p90_passes(self): responses = [ @@ -327,7 +389,7 @@ def test_sweep_fails_on_weighted_rate_even_when_p90_passes(self): num_pipelines, passed, _ = stream_density.run_pipeline_iterations( env_vars, ['docker-compose.yml'], '/tmp/results', 'gst', 15.0) - self.assertEqual((num_pipelines, passed), (1, True)) + self.assertEqual((num_pipelines, passed), (1, stream_density.STATUS_PASS)) self.assertEqual(calculate.call_count, 3) self.assertEqual(report.call_args.args[1], {'pipeline_stream0': 15.0}) @@ -437,7 +499,7 @@ def test_print_stream_density_report_summary(self): rows = {line.split()[0]: line for line in report.splitlines() if line.startswith('pipeline_stream')} self.assertIn('14.25', rows['pipeline_stream0']) - self.assertIn('14.600000', rows['pipeline_stream0']) + self.assertIn('14.60', rows['pipeline_stream0']) self.assertIn('12.00', rows['pipeline_stream0']) self.assertIn('16.00', rows['pipeline_stream0']) self.assertIn('5.00 s (50%)', rows['pipeline_stream0']) @@ -446,7 +508,7 @@ def test_print_stream_density_report_summary(self): self.assertIn( 'The lowest-throughput stream was pipeline_stream0', report) self.assertIn('targeting 15.00 FPS', report) - self.assertIn('measuring 14.600000 FPS', report) + self.assertIn('measuring 14.60 FPS', report) self.assertIn('against its 14.25 FPS throughput threshold', report) def test_count_valid_streams(self): @@ -470,18 +532,37 @@ def test_count_valid_streams_five_lanes_of_six_cameras(self): self.assertEqual( stream_density.count_valid_streams(stream_fps_dict), 30) + def test_build_per_stream_target_fps_camera_ordered_logs_above_one_lane(self): + # Camera-ordered logs, where index % cameras picks the wrong camera above one lane + cameras = [{'camera_id': f'cam{i + 1}', 'fps': 15} for i in range(6)] + cameras[5]['fps'] = 5 + for lanes, expected_slow in ((2, [10, 11]), (5, [25, 26, 27, 28, 29])): + with self.subTest(lanes=lanes): + manifest = self._manifest([(lane, cam) for cam in range(6) for lane in range(lanes)]) + result = self._run_with_manifest( + cameras, manifest, 6 * lanes, + lambda d, e: stream_density.build_per_stream_target_fps(d, 15, env_vars=e)) + slow = sorted(int(name[len('pipeline_stream'):]) + for name, target in result.items() if target == 5.0) + self.assertEqual(slow, expected_slow) + self.assertEqual(list(result.values()).count(15.0), 6 * lanes - lanes) + def test_clean_up_pipeline_logs(self): test_results_dir = './test_results_clean' testFile1 = os.path.join( test_results_dir, 'pipeline12345656_abc.log') testFile2 = os.path.join( test_results_dir, 'pipeline98765432_def.log') + manifestFile = os.path.join( + test_results_dir, 'pipeline_streams_20261009092918074066_gst0.json') try: os.makedirs(test_results_dir) with open(testFile1, 'w') as file: file.write('this is a test') with open(testFile2, 'w') as file: file.write('another file for testing') + with open(manifestFile, 'w') as file: + file.write('{"streams": []}') stream_density.clean_up_pipeline_logs( test_results_dir) self.assertFalse( @@ -490,6 +571,9 @@ def test_clean_up_pipeline_logs(self): self.assertFalse( os.path.exists(testFile2), f"file still exists: {testFile2}") + self.assertFalse( + os.path.exists(manifestFile), + f"file still exists: {manifestFile}") except Exception as ex: self.fail(f"ERROR: found exception {ex}") finally: @@ -497,6 +581,8 @@ def test_clean_up_pipeline_logs(self): os.remove(testFile1) if os.path.exists(testFile2): os.remove(testFile2) + if os.path.exists(manifestFile): + os.remove(manifestFile) if not os.listdir(test_results_dir): os.rmdir(test_results_dir) @@ -625,7 +711,7 @@ def test_pipeline_iterations( "container_name": "above_fps_target", "target_fps": 14.0, "expected_num_pipelines": 1, - "expected_meet_target_fps": True, + "expected_status": stream_density.STATUS_PASS, "expected_streams_sustained": 1, "calculate_fps_side_effect": [ (15.0, 15.0, {"pipeline_stream0": 15.0}, {}), @@ -644,46 +730,51 @@ def test_pipeline_iterations( "container_name": "below_fps_target", "target_fps": 15.0, "expected_num_pipelines": 1, - "expected_meet_target_fps": False, + "expected_status": stream_density.STATUS_FAIL, "expected_streams_sustained": 1, "calculate_fps_side_effect": [ (10.0, 10.0, {"pipeline_stream0": 10.0}, {}), (10.0, 10.0, {"pipeline_stream0": 10.0}, {}), ] }, + # Test case 3: inconclusive twice at the first lane count. { "env_vars": {"INIT_DURATION": "10"}, "compose_files": ["docker-compose.yml"], "results_dir": "/path/to/results", "container_name": "insufficient_samples", "target_fps": 15.0, - "expected_num_pipelines": 1, - "expected_meet_target_fps": False, + "expected_num_pipelines": 0, + "expected_status": stream_density.STATUS_INCONCLUSIVE, "expected_streams_sustained": 0, "measurement_window_result": ( {}, {}, 200.0, {"pipeline_stream0_lane1_gst.log": 99}, False), "calculate_fps_side_effect": [], "expect_no_fps_calculation": True, + "expected_window_calls": 2, }, ] for i, test_case in enumerate(test_cases): with self.subTest(f"Test case {i + 1}"): - env_vars = test_case["env_vars"] + # Keep a manifest left by a real run in src/pipelines out of this test + env_vars = dict(test_case["env_vars"], + STREAM_MANIFEST_PATH="/nonexistent/pipeline_streams.json") compose_files = test_case["compose_files"] results_dir = test_case["results_dir"] container_name = test_case["container_name"] target_fps = test_case["target_fps"] mock_calculate_fps.reset_mock() + mock_collect_window.reset_mock() mock_calculate_fps.side_effect = test_case[ "calculate_fps_side_effect"] mock_collect_window.return_value = test_case.get( "measurement_window_result", ({}, {}, 100.0, {'pipeline_stream0_lane_gst.log': 100}, True)) - num_pipelines, meet_target_fps, streams_sustained = ( + num_pipelines, status, streams_sustained = ( stream_density.run_pipeline_iterations( env_vars, compose_files, results_dir, container_name, target_fps) @@ -691,12 +782,80 @@ def test_pipeline_iterations( self.assertEqual( num_pipelines, test_case["expected_num_pipelines"]) - self.assertEqual( - meet_target_fps, test_case["expected_meet_target_fps"]) + self.assertEqual(status, test_case["expected_status"]) self.assertEqual( streams_sustained, test_case["expected_streams_sustained"]) if test_case.get("expect_no_fps_calculation"): mock_calculate_fps.assert_not_called() + if "expected_window_calls" in test_case: + self.assertEqual(mock_collect_window.call_count, + test_case["expected_window_calls"]) + + def _run_iterations_with_windows(self, window_results, fps_responses): + env_vars = {'INIT_DURATION': '0', 'PIPELINE_INC': '1', + 'CONSECUTIVE_FAIL_WINDOWS': '1', + 'CONSECUTIVE_PASS_WINDOWS': '1', + 'STREAM_MANIFEST_PATH': '/nonexistent/pipeline_streams.json'} + with patch('stream_density.measure_pipeline_memory', return_value=100), \ + patch('stream_density.clean_up_pipeline_logs'), \ + patch('stream_density.check_can_add_pipelines', return_value=True), \ + patch('stream_density.benchmark.docker_compose_containers') as compose, \ + patch('stream_density.time.sleep'), \ + patch('stream_density.check_non_empty_result_logs'), \ + patch('stream_density.collect_measurement_window', + side_effect=window_results) as collect, \ + patch('stream_density.calculate_multi_stream_fps', + side_effect=fps_responses), \ + patch('stream_density.calculate_pipeline_latency', return_value=(0.0, 0.0)), \ + patch('stream_density.build_per_stream_target_fps', + side_effect=lambda d, *a, **k: {n: 15.0 for n in d}), \ + patch('stream_density.build_per_stream_camera_meta', return_value={}), \ + patch('stream_density.print_stream_density_report'): + with redirect_stdout(io.StringIO()) as output: + result = stream_density.run_pipeline_iterations( + env_vars, ['docker-compose.yml'], '/tmp/results', 'gst', 15.0) + return result, collect.call_count, compose.call_count, output.getvalue() + + def test_inconclusive_interval_reports_last_passing_density(self): + enough = ({}, {}, 100.0, {'pipeline_stream0.log': 100}, True) + short = ({}, {}, 200.0, {'pipeline_stream0.log': 99}, False) + two_streams = {'pipeline_stream0': 15.0, 'pipeline_stream1': 15.0} + result, collect_calls, _, output = self._run_iterations_with_windows( + [enough, enough, short, short], + [(15.0, 15.0, {'pipeline_stream0': 15.0}, {}), + (30.0, 15.0, two_streams, {})]) + + # Passed at 1 and 2 lanes; 3 lanes was inconclusive twice + self.assertEqual(result, (2, stream_density.STATUS_INCONCLUSIVE, 2)) + self.assertEqual(collect_calls, 4) + self.assertIn('Result: Inconclusive', output) + self.assertIn('Last passing density: 2 lane(s) (2 streams)', output) + + def test_inconclusive_interval_is_retried_without_restarting(self): + enough = ({}, {}, 100.0, {'pipeline_stream0.log': 100}, True) + short = ({}, {}, 200.0, {'pipeline_stream0.log': 99}, False) + result, collect_calls, compose_calls, output = self._run_iterations_with_windows( + [short, enough, enough, enough], + [(15.0, 15.0, {'pipeline_stream0': 15.0}, {}), + (10.0, 10.0, {'pipeline_stream1': 10.0}, {}), + (15.0, 15.0, {'pipeline_stream0': 15.0}, {})]) + + self.assertEqual(result[1], stream_density.STATUS_PASS) + self.assertEqual(collect_calls, 4) + self.assertIn('Retrying the measurement interval at 1 lane(s)', output) + # Three lane steps (1, 2, 1); the retry adds no extra down/up pair + self.assertEqual(compose_calls, 3 * 2) + + def test_validate_rejects_measurement_interval_below_sample_floor(self): + for seconds, should_raise in (('50', True), ('99', True), ('100', False)): + with self.subTest(seconds=seconds): + env_vars = {RESULTS_DIR_KEY: '/some/path', + 'MEASUREMENT_WINDOW_SECONDS': seconds} + if should_raise: + with self.assertRaises(ArgumentError): + validate_and_setup_env(env_vars, [15.0]) + else: + validate_and_setup_env(env_vars, [15.0]) @patch('time.sleep', return_value=None) @patch('stream_density.validate_and_setup_env') @@ -719,12 +878,12 @@ def test_run_stream_density( "target_fps_list": [15.0, 25.0], "container_names_list": ["container1", "container2"], "run_pipeline_side_effect": [ - (5, True, 10), # For container1 - (7, False, 12) # For container2 + (5, stream_density.STATUS_PASS, 10), # For container1 + (7, stream_density.STATUS_FAIL, 12) # For container2 ], "expected_results": [ - (15.0, "container1", 5, True, 10), - (25.0, "container2", 7, False, 12) + (15.0, "container1", 5, stream_density.STATUS_PASS, 10), + (25.0, "container2", 7, stream_density.STATUS_FAIL, 12) ], # Expected number of compose down calls "expected_down_call_count": 2