From dc8f822de61cbf01a21920e43e03347ec0d2d7dd Mon Sep 17 00:00:00 2001 From: speriaswamy-amd Date: Fri, 14 Aug 2026 16:44:50 -0400 Subject: [PATCH 1/2] chore(config): strip _comment_*/_example_* clutter now that cvs man documents these AIMVT-276 --- .../health/mi300_health_config.json | 5 - .../preflight/README_preflight_config.md | 226 ++---------------- .../preflight/preflight_config.json | 67 +----- cvs/input/config_file/rccl/rccl_config.json | 12 +- .../megatron/mi35x_megatron_llama_single.json | 3 - .../mi3xx_megatron_llama_distributed.json | 19 -- .../megatron/mi3xx_megatron_llama_single.json | 3 - docs/reference/configuration-files/health.rst | 10 - .../configuration-files/megatron.rst | 92 ------- 9 files changed, 25 insertions(+), 412 deletions(-) diff --git a/cvs/input/config_file/health/mi300_health_config.json b/cvs/input/config_file/health/mi300_health_config.json index b6d17c1c5..3eacbd41d 100644 --- a/cvs/input/config_file/health/mi300_health_config.json +++ b/cvs/input/config_file/health/mi300_health_config.json @@ -3,7 +3,6 @@ "path": "/opt/amd/agfhc", "package_tar_ball": "/home/{user-id}/PACKAGES/agfhc-mi300x_1.22.0_ub2204.tar.bz2", "install_dir": "/home/{user-id}/INSTALL/agfhc/", - "_comments_log_dir": "log_dir has to be a NON NFS local file system", "log_dir": "/root/agfhc_logs", "hbm_test_duration": "00:01:30" }, @@ -12,7 +11,6 @@ "git_install_path": "/home/{user-id}/INSTALL/", "git_url": "https://github.com/ROCm/TransferBench.git", "git_tag": "v1.67.00", - "_comment_rocm_path": "ROCm installation path. Set the placeholder changeme to auto-detect from /opt/rocm or /opt/rocm/core-*", "rocm_path": "", "results": { "bytes_to_transfer": "268435456", @@ -34,13 +32,10 @@ "git_install_path": "/home/{user-id}/INSTALL/rvs", "git_url": "https://github.com/ROCm/ROCmValidationSuite.git -b release/rocm-rel-7.2", "nfs_install": "True", - "_comment_rocm_path": "ROCm installation path.Set to placeholder changeme auto-detect from /opt/rocm or /opt/rocm/core-*", "rocm_path": "", - "_comment_rocm_runtime_lib_path": "Optional. Colon-separated dirs prepended to LD_LIBRARY_PATH for every rvs invocation (also used during the install ldd check). Use this when the rvs binary under /opt/rocm/extras-7 was built against a newer ROCm than /opt/rocm symlinks to and you need it to load amd_smi/rocm_smi from a side-by-side install (e.g. /home/{user-id}/install/lib:/home/{user-id}/install/lib/rocm_sysdeps:/home/{user-id}/install/lib/llvm/lib). Leave empty for default loader behavior.", "rocm_runtime_lib_path": "", "config_path_mi300x": "/opt/rocm/share/rocm-validation-suite/conf/MI300X", "config_path_default": "/opt/rocm/share/rocm-validation-suite/conf", - "_comment_rvs_test_level": "RVS test level configuration (0-5). 0: Run individual tests (skip level test), 1-5: Run LEVEL config test if RVS >= 1.3.0, else run individual tests. Default is 4.", "rvs_test_level": 4, "tests": [ { diff --git a/cvs/input/config_file/preflight/README_preflight_config.md b/cvs/input/config_file/preflight/README_preflight_config.md index 75f47ce92..5ba22a1b4 100644 --- a/cvs/input/config_file/preflight/README_preflight_config.md +++ b/cvs/input/config_file/preflight/README_preflight_config.md @@ -99,9 +99,19 @@ preflight/ ## Configuration Parameters -### Complete Parameter Reference +Every parameter -- with its type, real default, constraints and an example -- +is documented by `cvs man`: -All parameters below are optional and have sensible defaults. The sample configuration file includes all available parameters with their default values and inline comments explaining their purpose. +```bash +cvs man preflight_checks # every parameter +cvs man preflight_checks gid_index # a single parameter +cvs man preflight_checks --json # machine-readable +``` + +That reference is generated from `PreflightConfigFile` in +`cvs/parsers/schemas.py`, so it cannot drift from the code the way a +hand-written table does. All parameters are optional and have sensible +defaults. ### Important Update: RDMA Connectivity Testing @@ -115,93 +125,7 @@ All parameters below are optional and have sensible defaults. The sample configu **Updated parameter names**: Configuration parameters now use accurate names (`ibv_test_timeout`, `ibv_test_port_range`) that reflect the use of `ibv_rc_pingpong` for testing. -### RDMA Batching (`connectivity_check.rdma`) - -- **`nodes_per_full_mesh_group`** (default: 128) - - Group size for parallel RDMA connectivity testing (2-512 nodes per group) - - Smaller groups use fewer resources per node but require more rounds - - Adjust based on cluster size and resource constraints - -### Debug Settings (`debug`) - -- **`scriptlet`** (default: false) - - Enable ScriptLet debug mode: preserve generated scripts/logs on remote nodes - - For RDMA connectivity, wraps each ibv_rc_pingpong server in strace - - Creates per-test traces under /tmp/preflight/strace_server__.log - - **Warning**: Can be expensive at scale due to strace overhead - -### Node Check Settings (`node_check`) - -- **`enabled`** (default: `true`) - - Enables GPU visibility, AMDGPU/KFD, kernel-health, and ROCm validation - - Set to `false` to skip node-local health checks - -- **`gpus_per_node`** (default: `4`) - - Exact number of AMD GPUs expected on every node - - GPU visibility is generation-independent and can run on older or newer AMD hardware - -- **`expected_rocm_version`** (default: "6.2.0") - - Expected ROCm version across all cluster nodes - - Must match the output of `amd-smi version` on all nodes - - Format: "major.minor.patch" (e.g., "6.2.0", "5.7.1") - -### Connectivity Check Settings (`connectivity_check`) - -#### RDMA Settings (`connectivity_check.rdma`) - -- **`connectivity_mode`** (default: "basic") - - **"basic"**: Test adjacent node pairs (fast, ~14% coverage for 8 nodes) - - **"full_mesh"**: Test all possible node pairs (comprehensive, 100% coverage) - - **"skip"**: Skip RDMA interface presence, GID validation, and pairwise connectivity - -- **`gid_index`** (default: "3") - - GID index to check on all configured RDMA interfaces - - Typically "3" for RoCE (RDMA over Converged Ethernet) - - Must be a valid GID index for your InfiniBand/RoCE setup - -- **`interfaces`** (default: `["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"]`) - - List of RDMA device names that should be present on all cluster nodes - - Examples: - - `["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"]` - Standard 4-interface setup - - `["mlx5_0", "mlx5_1"]` - Mellanox 2-interface setup - -- **`ibv_test_timeout`** (default: 90) - - Timeout in seconds for each ibv_rc_pingpong connectivity test - - Integer value (seconds), used directly as configured - - Uses `ibv_rc_pingpong` (direct InfiniBand verbs) for RCCL-compatible testing - - Increase for slower networks or high-latency connections - -- **`ibv_test_port_range`** (default: "10000-50000") - - Port range for ibv_rc_pingpong tests to avoid conflicts - - Format: "start-end" (e.g., "10000-50000", "10000-10999") - - Ensure ports are not blocked by firewalls - -- **`inter_full_mesh_group_pairs_per_wave`** (default: "auto") - - Max ordered group-pairs (Gi→Gj keys) per wave during inter-group RDMA testing - - "auto" calculates as max(1, num_groups - 1) - - Can be set to a specific integer to control wave size and reduce memory/CPU load - -- **`prune_failure_threshold`** (default: 0.5) - - Prune nodes whose fraction of peers with ≥1 FAIL intra test is ≥ this value - - Range: 0.0 to 1.0 (0.5 = 50% failure threshold) - - Helps remove problematic nodes before inter-group testing - - Lower values (0.2-0.3) are more aggressive at removing problematic nodes - -- **`port_retry_max`** (default: 3) - - Max retry attempts for pairs whose logs show PORT_LISTEN_FAILED - - Range: 0-10 retries with new TCP ports after each wave - - Helps handle port conflicts during large-scale testing - -- **`port_retry_gap`** (default: 1000) - - Port gap when remapping ports for PORT_LISTEN_FAILED retries - - Range: 1-65535 - - Starts at (max port in batch) + this gap to reduce overlap with ephemeral ports - -- **`exclude_failed_interface_nodes`** (default: "true") - - Legacy hint for reporting: preflight now prunes interface/GID-failed nodes automatically - - Interface failures are excluded from mesh testing regardless of this flag - -##### Legacy RDMA paths — deprecated +### Legacy RDMA paths — deprecated Existing RDMA users may temporarily retain `node_check.gid_index` and `node_check.rdma_interfaces`. CVS normalizes them to @@ -210,15 +134,12 @@ and emits a deprecation warning. If a legacy and canonical value are both present, they must match. New configurations should use the canonical RDMA paths; the compatibility paths will be removed in a future release. -#### IFoE Settings (`connectivity_check.ifoe`) — MI4XX scale-up fabric - -IFoE validation is organized into fabric admission, strict L2 connectivity, -and TransferBench data-path validation. CVS owns `afmctl` discovery, privilege -handling, BDF and port discovery, strict coverage, traffic selection, timeout -derivation, and result parsing. +### IFoE settings that are no longer configurable -The earlier configuration shape exposed those implementation details directly. -They now follow this fixed policy: +IFoE validation (`connectivity_check.ifoe`) used to expose implementation +details -- fabric discovery, privilege handling, port selection, timeouts -- +directly as config keys. Those keys are no longer read; CVS now derives all +of this automatically and follows a fixed policy: | Previous setting | Current CVS behavior | |---|---| @@ -232,117 +153,6 @@ They now follow this fixed policy: | `loss_threshold_pct` | Fail on any reported loss or incomplete coverage | | `per_ping_timeout` / `ssh_timeout` | Derive conservative timeouts from the requested workload | -- **`fabric_checks`** (default: `false`) - - Enables MI4XX-only AIFM/AFM/vPOD, station-mask, and IFoE port admission checks - - Set to `true` only on MI4XX systems; it remains disabled for MI3XX systems - - Requires `node_check.enabled: true` - -##### L2 ping (`connectivity_check.ifoe.l2ping`) - -Runs `afmctl test ping` with strict full-mesh coverage on every admitted IFoE -port and validates per-port and aggregate summary accounting. - -- **`enabled`** (default: `false`) - - Enables the mandatory L2 connectivity gate before TransferBench and RDMA -- **`pings_per_port`** (default: `3`) - - Number of ping samples sent per selected IFoE port pair - -##### TransferBench (`connectivity_check.ifoe.transferbench`) - -- **`enabled`** (default: `false`) - - Enables the TransferBench IFoE data-path gate before RDMA -- **`scope`** (default: `"node"`) - - `"node"` runs an independent smoketest on each node - - `"cluster"` runs one multi-rank test across the admitted cluster -- **`profile`** (default: `"smoketest"`) - - Selects the CVS-supported test profile; `"smoketest"` is currently supported -- **`message_sizes`** (default: `["1K", "16M"]`) - - Message sizes exercised by the selected profile -- **`iterations`** (default: `2`) - - Validated iterations per test and message size -- **`warmup_iterations`** (default: `0`) - - Warmup iterations performed before validation - -#### Node Smoke Settings (`node_smoke`) — opt-in (Primus Tier 1) - -Runs Primus `node_smoke` on each reachable node via `primus-cli direct --single -- node_smoke` -over parallel SSH (no Slurm required). Reference: Primus `docs/node-smoke-test-instruction.md` -on branch `dev/preflight-direct-test`. - -- **`connectivity_mode`** (default: `"skip"`) - - `"run"` — execute node_smoke on every reachable node - - `"skip"` — preflight records a SKIPPED result and does not invoke Primus -- **`auto_setup`** (default: `true`) - - Clone/update Primus and create the venv with minimal deps (ROCm PyTorch) before node_smoke -- **`setup_timeout`** (default: `600`) - - SSH timeout (seconds) for the per-node Primus auto_setup step -- **`force_reclone`** (default: `false`) - - Remove `primus_dir` and clone fresh on every run (destructive) -- **`shared_install`** (default: `true`) - - Leader node clones/installs on shared NFS home; other nodes wait (recommended for shared home) -- **`pip_install_mode`** (default: `"minimal"`) - - `"minimal"` — ROCm PyTorch only; `"requirements"` — `pip install -r requirements.txt`; `"skip"` — venv only -- **`torch_pip_index_url`** (default: `"https://download.pytorch.org/whl/rocm6.2"`) - - PyTorch wheel index for minimal install; match your ROCm version -- **`primus_git_url`** (default: `"https://github.com/AMD-AIG-AIMA/Primus.git"`) -- **`primus_git_branch`** (default: `"dev/preflight-direct-test"`) -- **`primus_git_recurse_submodules`** (default: `false`) -- **`primus_dir`** (default: `"/home/{user-id}/INSTALL/Primus"`) - - Required when `connectivity_mode` is `"run"`; `{user-id}` is resolved at runtime -- **`venv_activate`** (default: `"/home/{user-id}/envs/preflight/.venv/bin/activate"`) - - Required when `connectivity_mode` is `"run"` -- **`gpus_per_node`** (default: `8`) -- **`master_port`** (default: `1234`) -- **`dump_path`** (default: `""`) - - Per-node smoke JSON output; empty uses `/node_smoke` -- **`expected_rdma_nics`** (default: `null`) - - Defaults to `len(node_check.rdma_interfaces)` when null -- **`ulimit_l_min_gb`** (default: `32`) — FAIL below this memlock limit; `0` disables -- **`shm_min_gb`** (default: `8`) — FAIL below this `/dev/shm` size; `0` disables -- **`skip_dmesg`** (default: `false`) -- **`allow_foreign_procs`** (default: `false`) -- **`allowed_procs`** (default: `"gpuagent,rocm-smi-daemon,amd-smi,dcgm-exporter"`) -- **`require_tools`** (default: `""`) — empty = warn only -- **`nccl_socket_ifname`** / **`gloo_socket_ifname`** (default: `""`) -- **`nccl_ib_hca`** (default: `""`) — defaults to comma-joined `node_check.rdma_interfaces` -- **`nccl_ib_gid_index`** (default: `null`) — defaults to `node_check.gid_index` -- **`ssh_timeout`** (default: `300`) -- **`extra_args`** (default: `[]`) — additional flags forwarded to primus-cli - -#### Tier 2 perf sanity (`node_smoke.tier2_perf`) — optional - -When `tier2_perf` is `true`, preflight forwards `--tier2-perf` to Primus `node_smoke`, enabling all three Tier 2 checks on each node (same as `launch_nodesmoke_ssh.sh -- --tier2-perf`): - -1. **Large GEMM TFLOPS floor** — 8192³ bf16 `torch.matmul`; FAIL below `gemm_tflops_min` (default 600) -2. **HBM D2D bandwidth** — 512 MB device-to-device copy; FAIL below `hbm_gbs_min` (default 2000 GB/s) -3. **Local multi-GPU RCCL all-reduce** — node-local only; FAIL below `rccl_gbs_min` (default 100 GB/s) - -Set `NCCL_IB_HCA`, `NCCL_SOCKET_IFNAME`, and `NCCL_IB_GID_INDEX` (via `node_smoke` config or cluster `env_vars`) before enabling Tier 2 — RCCL init enumerates every transport even though the all-reduce is local-only. - -- **`tier2_perf`** (default: `false`) — master switch; maps to `--tier2-perf` -- **`gemm_tflops_min`** (default: `600`) — `--gemm-tflops-min` -- **`hbm_gbs_min`** (default: `2000`) — `--hbm-gbs-min` -- **`rccl_gbs_min`** (default: `100`) — `--rccl-gbs-min` -- **`rccl_size_mb`** (default: `64`) — `--rccl-size-mb` -- **`rccl_timeout_sec`** (default: `120`) — `--rccl-timeout-sec` - -Tier 2 runs need a longer SSH budget; when `tier2_perf` is enabled the effective timeout is at least 600 seconds even if `ssh_timeout` is lower. - -### Reporting Settings (`reporting`) - -- **`generate_html_report`** (default: `true`) - - Whether to generate detailed HTML report - - Set to `false` to disable HTML report generation - -- **`artifacts_root_dir`** (default: `"/tmp/{user-id}/preflight"`) - - Root directory where preflight artifacts are saved - - Includes HTML reports and RDMA full_mesh workspace logs under `rdma_connectivity_workspace/` - - Must be writable by the user running the tests - -- **`generate_rdma_pairs_csv`** (default: `true`) - - Whether to generate CSV file with failed RDMA pairs alongside HTML report - - Set to `false` to disable CSV generation - ## Usage Examples ### Basic 8-Node Cluster Check diff --git a/cvs/input/config_file/preflight/preflight_config.json b/cvs/input/config_file/preflight/preflight_config.json index 3ae64b6c7..3a4f9b025 100644 --- a/cvs/input/config_file/preflight/preflight_config.json +++ b/cvs/input/config_file/preflight/preflight_config.json @@ -1,114 +1,65 @@ { "preflight": { "node_check": { - "_comment": "Individual node validation (parallel across nodes)", "enabled": true, - "_comment_enabled": "Enable GPU visibility, AMDGPU/KFD, kernel-health, and ROCm validation.", - "_example_gpus_per_node": 4, "gpus_per_node": 4, - "_comment_gpus_per_node": "Exact number of AMD GPUs expected on every node.", - "_example_expected_rocm_version": "6.2.0", - "expected_rocm_version": "", - "_comment_expected_rocm_version": "Expected ROCm version across all cluster nodes. Must match the output of 'amd-smi version' on all nodes. Format: 'major.minor.patch' (e.g., '6.2.0', '5.7.1')." + "expected_rocm_version": "" }, "connectivity_check": { - "_comment": "Inter-node connectivity tests by protocol", "rdma": { - "_comment": "RDMA connectivity testing parameters using ibv_rc_pingpong (direct IB verbs) for RCCL-compatible testing", "connectivity_mode": "skip", - "_comment_connectivity_mode": "RDMA connectivity testing mode. Options: 'basic' (test adjacent node pairs, fast, ~14% coverage for 8 nodes), 'full_mesh' (test all possible node pairs, comprehensive, 100% coverage), 'skip' (skip RDMA connectivity testing entirely).", - "_example_gid_index": "3", "gid_index": "3", - "_comment_gid_index": "GID index to check on all RDMA interfaces. Typically '3' for RoCE (RDMA over Converged Ethernet). Must be a valid GID index for your InfiniBand/RoCE setup.", - "_example_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0", "rocep158s0", "rocep190s0", "rocep206s0", "rocep222s0"], "interfaces": ["enp4s0np0"], - "_comment_interfaces": "List of specific RDMA interface names that should be present on all cluster nodes. Examples: ['rocep28s0', 'rocep62s0'] for standard setup, ['mlx5_0', 'mlx5_1'] for Mellanox, ['ib0', 'ib1'] for generic InfiniBand.", "nodes_per_full_mesh_group": 32, - "_comment_nodes_per_full_mesh_group": "Number of nodes in each full-mesh partition group for parallel RDMA testing (2-128). Smaller groups use fewer resources per node but require more rounds. Adjust based on cluster size and resource constraints.", "ibv_test_timeout": 90, - "_comment_ibv_test_timeout": "Timeout in seconds for each ibv_rc_pingpong connectivity test. Uses ibv_rc_pingpong (direct IB verbs) instead of rping for RCCL-compatible testing. Increase for slower networks or high-latency connections.", "ibv_test_port_range": "10000-50000", - "_comment_ibv_test_port_range": "Port range for ibv_rc_pingpong tests to avoid conflicts. Format: 'start-end' (e.g., '10000-50000', '10000-10999'). Ensure ports are not blocked by firewalls.", "inter_full_mesh_group_pairs_per_wave": "auto", - "_comment_inter_full_mesh_group_pairs_per_wave": "Max ordered full-mesh group-pairs (Gi→Gj keys) per wave during inter-group RDMA testing. 'auto' calculates as max(1, num_groups - 1). Can be set to a specific integer to control wave size and reduce memory/CPU load.", "prune_failure_threshold": 0.5, - "_comment_prune_failure_threshold": "Prune nodes whose fraction of peers with ≥1 FAIL intra test is ≥ this value. Range: 0.0 to 1.0 (0.5 = 50% failure threshold). Helps remove problematic nodes before inter-group testing. Lower values (0.2-0.3) are more aggressive.", "port_retry_max": 3, - "_comment_port_retry_max": "Max retry attempts for pairs whose logs show PORT_LISTEN_FAILED. Range: 0-10 retries with new TCP ports after each wave. Helps handle port conflicts during large-scale testing.", "port_retry_gap": 1000, - "_comment_port_retry_gap": "Port gap when remapping ports for PORT_LISTEN_FAILED retries. Range: 1-65535. Starts at (max port in batch) + this gap to reduce overlap with ephemeral ports.", - "exclude_failed_interface_nodes": "true", - "_comment_exclude_failed_interface_nodes": "Legacy hint for reporting: preflight now prunes interface/GID-failed nodes from the SSH host list automatically. Interface failures are excluded from mesh testing regardless of this flag." + "exclude_failed_interface_nodes": "true" }, "ifoe": { - "_comment": "IFoE L2 connectivity testing via 'afmctl test ping'. Opt-in: defaults to 'skip'.", - - "_comment_connectivity_mode": "IFoE L2 ping mode. Options: 'run' (execute afmctl L2 ping on every reachable node) or 'skip' (default; preflight will not invoke afmctl). Enable once afmctl and the IFoE driver are available on every node.", - "_comment_afmctl_path": "Absolute path or PATH-resolved name of the afmctl binary on each cluster node. Examples: 'afmctl', '/usr/local/bin/afmctl'.", - "_comment_use_sudo": "When true, afmctl is invoked with sudo. Enable if afmctl needs root on the cluster image.", - "_comment_bdf_discovery": "How to determine which accelerator BDFs to ping on each node. 'auto' runs 'afmctl show device' on each node and uses the BDFs reported there. 'config' uses only the explicit 'bdfs' list below.", - "_example_bdfs": ["0001:01:00.1"], - "_comment_bdfs": "Optional explicit list of accelerator BDFs (e.g. ['0001:01:00.1']) shared across the cluster. Leave empty to defer to bdf_discovery='auto'.", - "_comment_dst_accelerators": "Destination accelerator IDs passed to --dst-accelerator. One afmctl ping is issued per (bdf, dst_accelerator). Use [0] for a single-accelerator destination; use a list like [0, 1] to ping multiple peers.", - "_comment_ports": "Ports passed to -p. Use 'all' (default; omits -p so afmctl tests every port), a string like '0,1,2' or '0-7', or a list [0,1,2].", - "_comment_pings_per_port": "Value for -c (pings per port pair). Larger values smooth over transient losses.", - "_comment_per_ping_timeout": "Optional value for afmctl's -t flag (per-ping timeout). Leave null to use afmctl's default.", - "_comment_traffic_types": "Traffic categories to enforce when evaluating PASS/FAIL. Maps to afmctl's --traffic-type (request, response, non-ifoe). When all three are selected (default) --traffic-type is omitted so afmctl exercises every category.", - "_comment_loss_threshold_pct": "Maximum tolerated packet loss percentage per traffic type. Defaults to 0.0 (any failure marks the node as FAIL).", - "_comment_ssh_timeout": "Overall SSH timeout (seconds) for each afmctl invocation. Increase for large port counts or high pings_per_port values.", "fabric_checks": false, - "_comment_fabric_checks": "Enable MI4XX-only AIFM/AFM/vPOD, station-mask, and IFoE port admission checks.", - - "_comment_policy": "CVS owns afmctl discovery, privilege handling, BDF and port discovery, strict coverage, traffic selection, and timeout derivation.", "l2ping": { - "_comment": "Strict afmctl L2 connectivity gate executed before TransferBench and RDMA", "enabled": true, - "_comment_enabled": "Enable IFoE L2 connectivity validation.", - "pings_per_port": 3, - "_comment_pings_per_port": "Ping samples sent per selected IFoE port pair." + "pings_per_port": 3 }, "transferbench": { - "_comment": "TransferBench IFoE data-path validation executed before RDMA", "enabled": true, - "_comment_enabled": "Enable the TransferBench preflight gate.", "scope": "node", - "_comment_scope": "Run independent tests per node or one multi-rank cluster test. Options: node, cluster.", "profile": "smoketest", - "_comment_profile": "CVS-supported TransferBench validation profile. Currently: smoketest.", "message_sizes": ["1K", "16M"], - "_comment_message_sizes": "Message sizes exercised by the selected profile.", "iterations": 2, - "_comment_iterations": "Validated iterations per test and message size.", - "warmup_iterations": 0, - "_comment_warmup_iterations": "Warmup iterations performed before validation." + "warmup_iterations": 0 } } }, @@ -226,23 +177,17 @@ }, "reporting": { - "_comment": "Post-test reporting and output", "generate_html_report": true, - "_comment_generate_html_report": "Whether to generate detailed HTML report. Set to 'false' to disable HTML report generation.", "artifacts_root_dir": "/tmp/{user-id}/preflight", - "_comment_artifacts_root_dir": "Root directory for preflight artifacts. HTML reports are saved here, and RDMA full_mesh ScriptLet logs use /rdma_connectivity_workspace/// on each node (NFS-friendly). Must be writable by the user running the tests.", - "generate_rdma_pairs_csv": false, - "_comment_generate_rdma_pairs_csv": "Whether to generate CSV file with failed RDMA pairs alongside HTML report. Set to 'false' to disable CSV generation. Useful for analyzing connectivity patterns and failures." + "generate_rdma_pairs_csv": false }, "debug": { - "_comment": "Debug and troubleshooting options", - "scriptlet": false, - "_comment_scriptlet": "Enable ScriptLet debug mode: preserve generated scripts/logs on remote nodes. For RDMA connectivity, wraps each ibv_rc_pingpong server in strace with per-test traces under /tmp/preflight/strace_server__.log. WARNING: Can be expensive at scale due to strace overhead." + "scriptlet": false } } } diff --git a/cvs/input/config_file/rccl/rccl_config.json b/cvs/input/config_file/rccl/rccl_config.json index d853f9543..10c5738e8 100644 --- a/cvs/input/config_file/rccl/rccl_config.json +++ b/cvs/input/config_file/rccl/rccl_config.json @@ -3,7 +3,6 @@ "mpi_params": { "no_of_nodes": "2", "no_of_local_ranks": "8", - "_comment_mpi_pml": "MPI Point-to-Point Messaging Layer: 'auto' (auto-detect UCX support), 'ucx' (force UCX), or 'ob1' (force OpenIB/TCP fallback). This remains in config because it is an MPI launch option, not an env-script setting.", "mpi_pml": "auto", "mpi_dir": "/home/{user-id}/openmpi/bin", "mpi_oob_port": "eth0", @@ -13,7 +12,6 @@ "_comment_net_dev_list": "Leave empty for auto-detection from backend NICs, or set explicitly e.g. bnxt_re0:1,bnxt_re1:1,.." }, - "_comment_env_script": "RCCL/NCCL/UCX tuning parameters. Path configurations (mpi_dir, rccl_tests_dir) are now in mpi_params and rccl_test_params respectively.", "env_source_script": "/home/{user-id}/thor2_env_script.sh", "rccl_test_params": { @@ -27,30 +25,23 @@ "no_of_iterations": "20", "no_of_cycles": "1", "check_iteration_count": "1", - "_comment_rccl_test_flags": "Optional rccl-tests binary flags. rccl_timeout = rccl-tests internal timer passed as -T (seconds; omit to not set). output_algo_proto_channels = boolean toggle for rccl-tests' -A 1 algorithm/protocol/channels diagnostic output.", "rccl_timeout": "1800", "output_algo_proto_channels": false, "data_types": ["float"] }, - "_comment_cvs_params": "The rccl_result_file.json will be appended by rccl_test_params.data_types. When it's float the file name will be rccl_result_file_float.json on the head node.", "cvs_params": { "cluster_snapshot_debug": "False", - "_comment_nic_model": "Set nic_model for validations (eg., 'thor' , 'ainic', 'connectx')", "nic_model": "thor", "verify_bus_bw": "False", "verify_bw_dip": "True", "verify_lat_dip": "True", - "_comment_cvs_exec_timeout": "Optional. CVS-side outer cap on the SSH-exec call wrapping mpirun (seconds, default 2400).", "cvs_exec_timeout": "2400", "rccl_result_file": "/home/{user-id}/rccl_result_file.json", "pairwise_min_bw": "300", - "_comment_pairwise_min_bw": "Minimum bus BW in GB/s required for Phase 2 incremental admission (Phase 1 pairwise never gates on bandwidth, matching the original bash tool). Set to 0 to skip the BW check.", - "pairwise_results_file": "/tmp/rccl_pairwise_results.json", - "_comment_pairwise_results_file": "Local JSON file where rccl_pairwise.py persists the Phase 1 pass/fail lists and the Phase 2 final valid/excluded host lists." + "pairwise_results_file": "/tmp/rccl_pairwise_results.json" }, - "_comment_regression": "New regression object format using NCCL_* environment variable names. Creates Cartesian product of all combinations. Channel config uses 'min-max' format (e.g., '16-16') or 'default'. Tree algorithm is automatically filtered to only run with all_reduce_perf collective.", "regression": { "NCCL_ALGO": [ "Ring", "Tree" ], "NCCL_PROTO": [ "Simple" ], @@ -60,7 +51,6 @@ "NCCL_MAX_NCHANNELS": [ "8", "16" ] }, - "_comments_results": "expected results below are for 2 node cluster, will vary based on cluster size", "results": { "all_reduce_perf": { "bus_bw": { diff --git a/cvs/input/config_file/training/megatron/mi35x_megatron_llama_single.json b/cvs/input/config_file/training/megatron/mi35x_megatron_llama_single.json index 26e997a71..e6aaf79e0 100644 --- a/cvs/input/config_file/training/megatron/mi35x_megatron_llama_single.json +++ b/cvs/input/config_file/training/megatron/mi35x_megatron_llama_single.json @@ -4,14 +4,11 @@ { "container_image": "rocm/megatron-lm:v25.9_gfx950", "container_name": "megatron_llama3.1_310", - "_example_nnodes": "4", "nnodes": "-no of nodes to run singlenode training", "master_address": "localhost", - "_example_training_iterations": "30", "training_iterations": "", "hf_token_file": "/home/{user-id}/.hf_token", "shm_size": "128G", - "_comments_data_cache_dir": "This path should be accessible from all nodes like a common FS like NFS for distributed training", "data_cache_dir": "/home/{user-id}/cache", "megatron_root": "/workspace/Megatron-LM", "training_scripts": { diff --git a/cvs/input/config_file/training/megatron/mi3xx_megatron_llama_distributed.json b/cvs/input/config_file/training/megatron/mi3xx_megatron_llama_distributed.json index 0a669100c..cba81a8cc 100644 --- a/cvs/input/config_file/training/megatron/mi3xx_megatron_llama_distributed.json +++ b/cvs/input/config_file/training/megatron/mi3xx_megatron_llama_distributed.json @@ -2,32 +2,21 @@ "config": { - "_comments__": "Config file created for 4 nodes, change expected results based on number of nodes", "container_image": "rocm/megatron-lm:v25.5_py310", "container_name": "megatron_llama3.1_310", "distributed_training": "True", - "_example_nnodes": "4", "nnodes": "", - "_example_master_address": "X.X.X.X", "master_address": "", - "_example_training_iterations": "30", "training_iterations": "", - "_example_nic_type": "ainic|thor2|cx7", "nic_type": "", - "_example_nccl_ib_hca_list": "bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7", "nccl_ib_hca_list": "", - "_example_nccl_socket_ifname": "ens51f1np1", "nccl_socket_ifname": "", - "_example_gloo_socket_ifname": "ens51f1np1", "gloo_socket_ifname": "", - "_example_nccl_ib_gid_index": "3", "nccl_ib_gid_index": "", - "_example_hca_id_pattern": "bnxt_|rocep|mlx5_", "hca_id_pattern": "bnxt_|rocep", "nccl_debug": "ERROR", "hf_token_file": "/home/{user-id}/.hf_token", "shm_size": "128G", - "_comments_data_cache_dir": "This path should be accessible from all nodes like a common FS like NFS for distributed training", "data_cache_dir": "/home/{user-id}/cache", "megatron_root": "/workspace/Megatron-LM", "training_scripts": { @@ -73,8 +62,6 @@ "fsdp": "0", "result_dict": { - "_example_throughput_per_gpu": "610.0", - "_example_tokens_per_gpu": "12000.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -93,8 +80,6 @@ "fsdp": "0", "result_dict": { - "_example_throughput_per_gpu": "620.0", - "_example_tokens_per_gpu": "14000.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -116,8 +101,6 @@ "fsdp": "0", "result_dict": { - "_example_throughput_per_gpu": "530.0", - "_example_tokens_per_gpu": "1100.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -136,8 +119,6 @@ "fsdp": "0", "result_dict": { - "_example_throughput_per_gpu": "550.0", - "_example_tokens_per_gpu": "1200.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } diff --git a/cvs/input/config_file/training/megatron/mi3xx_megatron_llama_single.json b/cvs/input/config_file/training/megatron/mi3xx_megatron_llama_single.json index 255d2d0f9..866637c26 100644 --- a/cvs/input/config_file/training/megatron/mi3xx_megatron_llama_single.json +++ b/cvs/input/config_file/training/megatron/mi3xx_megatron_llama_single.json @@ -4,14 +4,11 @@ { "container_image": "rocm/megatron-lm:v25.5_py310", "container_name": "megatron_llama3.1_310", - "_example_nnodes": "4", "nnodes": "-no of nodes to run singlenode training", "master_address": "", - "_example_training_iterations": "30", "training_iterations": "", "hf_token_file": "/home/{user-id}/.hf_token", "shm_size": "128G", - "_comments_data_cache_dir": "This path should be accessible from all nodes like a common FS like NFS for distributed training", "data_cache_dir": "/home/{user-id}/cache", "megatron_root": "/workspace/Megatron-LM", "training_scripts": { diff --git a/docs/reference/configuration-files/health.rst b/docs/reference/configuration-files/health.rst index a4a0ea95c..cd87b4d21 100644 --- a/docs/reference/configuration-files/health.rst +++ b/docs/reference/configuration-files/health.rst @@ -25,7 +25,6 @@ Here's a code snippet of the ``mi300_health_config.json`` file for reference: "path": "/opt/amd/agfhc", "package_tar_ball": "/home/{user-id}/PACKAGES/agfhc-mi300x_1.22.0_ub2204.tar.bz2", "install_dir": "/home/{user-id}/INSTALL/agfhc/", - "_comments_log_dir": "log_dir has to be a NON NFS local file system", "log_dir": "/root/agfhc_logs", "hbm_test_duration": "00:01:30" }, @@ -35,7 +34,6 @@ Here's a code snippet of the ``mi300_health_config.json`` file for reference: "git_install_path": "/home/{user-id}/INSTALL/", "git_url": "https://github.com/ROCm/TransferBench.git", "git_tag": "v1.67.00", - "_comment_rocm_path": "ROCm installation path. Set the placeholder changeme to auto-detect from /opt/rocm or /opt/rocm/core-*", "rocm_path": "", "results": { @@ -60,11 +58,9 @@ Here's a code snippet of the ``mi300_health_config.json`` file for reference: "git_install_path": "/home/{user-id}/INSTALL/rvs", "git_url": "https://github.com/ROCm/ROCmValidationSuite.git", "nfs_install": "True", - "_comment_rocm_path": "ROCm installation path.Set to placeholder changeme auto-detect from /opt/rocm or /opt/rocm/core-*", "rocm_path": "", "config_path_mi300x": "/opt/rocm/share/rocm-validation-suite/conf/MI300X", "config_path_default": "/opt/rocm/share/rocm-validation-suite/conf", - "_comment_rvs_test_level": "RVS test level configuration (0-5). 0: Run individual tests (skip level test), 1-5: Run LEVEL config test if RVS >= 1.3.0, else run individual tests. Default is 4.", "rvs_test_level": 4, "tests": [ { @@ -166,9 +162,6 @@ AGFHC * - ``install_dir`` - ``/home/{user-id}/INSTALL/agfhc/`` - Path where AGFHC runs - * - ``_comments_log_dir`` - - ``/home/{user-id}/INSTALL/agfhc/`` - - Path where AGFHC runs * - ``log_dir`` - ``/root/agfhc_logs`` - Log directory @@ -266,9 +259,6 @@ ROCm Validation Suite (RVS) * - ``config_path_default`` - ``/opt/rocm/share/`` |br| ``rocm-validation`` |br| ``-suite/conf`` - Default path for RVS - * - ``_comment_rvs_test_level`` - - "RVS test level configuration (0-5). 0: Run individual tests (skip level test), 1-5: Run LEVEL config test if RVS >= 1.3.0, else run individual tests. Default is 4." - - RVS test comments * - ``rvs_test_level`` - 4 - Test level diff --git a/docs/reference/configuration-files/megatron.rst b/docs/reference/configuration-files/megatron.rst index 71117320e..2497e9a4a 100644 --- a/docs/reference/configuration-files/megatron.rst +++ b/docs/reference/configuration-files/megatron.rst @@ -40,14 +40,11 @@ This is the ``mi3xx_megatron_llama_single.json`` configuration file: { "container_image": "rocm/megatron-lm:v25.5_py310", "container_name": "megatron_llama3.1_310", - "_example_nnodes": "4", "nnodes": "-no of nodes to run singlenode training", "master_address": "", - "_example_training_iterations": "30", "training_iterations": "", "hf_token_file": "/home/{user-id}/.hf_token", "shm_size": "128G", - "_comments_data_cache_dir": "This path should be accessible from all nodes like a common FS like NFS for distributed training", "data_cache_dir": "/home/{user-id}/cache", "megatron_root": "/workspace/Megatron-LM", "training_scripts": { @@ -185,18 +182,12 @@ Use the parameters in these tables to configure the training file. * - ``container_name`` - ``megatron_llama3.1_310`` - Name assigned to the container instance - * - ``_example_nnodes`` - - 4 - - Example of number of cluster nodes participating in the job * - ``Nnodes`` - "````-no of nodes to run singlenode training" - Number of nodes in the distributed job * - ``master_address`` - ```` - IP of the master/coordinator node - * - ``_example_training_iterations`` - - 30 - - Example of number of training iterations/steps to run in this test * - ``training_iterations`` - ```` - Number of training iterations/steps to run in this test @@ -206,9 +197,6 @@ Use the parameters in these tables to configure the training file. * - ``shm_size`` - 256G - Docker shared memory size mounted into container - * - ``_comments_data_cache_dir`` - - "This path should be accessible from all nodes like a common FS like NFS for distributed training" - - Comment explaining ``data_cache_dir`` must be accessible from all nodes * - ``data_cache_dir`` - ``/home/{user-id}/cache`` - Dataset/cache directory @@ -492,14 +480,11 @@ The ``mi35x_megatron_llama_single.json`` config file is used to run Megatron on { "container_image": "rocm/megatron-lm:v25.9_gfx950", "container_name": "megatron_llama3.1_310", - "_example_nnodes": "4", "nnodes": "-no of nodes to run singlenode training", "master_address": "localhost", - "_example_training_iterations": "30", "training_iterations": "", "hf_token_file": "/home/{user-id}/.hf_token", "shm_size": "128G", - "_comments_data_cache_dir": "This path should be accessible from all nodes like a common FS like NFS for distributed training", "data_cache_dir": "/home/{user-id}/cache", "megatron_root": "/workspace/Megatron-LM", "training_scripts": { @@ -626,18 +611,12 @@ Use the parameters in these tables to configure the training file. * - ``container_name`` - ``megatron_llama3.1_310`` - Name assigned to the container instance - * - ``_example_nnodes`` - - 4 - - Example of number of cluster nodes participating in the job * - ``Nnodes`` - "````-no of nodes to run singlenode training" - Number of nodes in the distributed job * - ``master_address`` - ```` - IP of the master/coordinator node - * - ``_example_training_iterations`` - - 30 - - Example of number of training iterations/steps to run in this test * - ``training_iterations`` - ```` - Number of training iterations/steps to run in this test @@ -647,9 +626,6 @@ Use the parameters in these tables to configure the training file. * - ``shm_size`` - 128G - Docker shared memory size mounted into container - * - ``_comments_data_cache_dir`` - - "This path should be accessible from all nodes like a common FS like NFS for distributed training" - - Comment explaining ``data_cache_dir`` must be accessible from all nodes * - ``data_cache_dir`` - ``/home/{user-id}/cache`` - Dataset/cache directory @@ -925,32 +901,21 @@ This is the multi-node ``mi3xx_megatron_llama_distributed.json`` configuration f "config": { - "_comments__": "Config file created for 4 nodes, change expected results based on number of nodes", "container_image": "rocm/megatron-lm:v25.5_py310", "container_name": "megatron_llama3.1_310", "distributed_training": "True", - "_example_nnodes": "4", "nnodes": "", - "_example_master_address": "X.X.X.X", "master_address": "", - "_example_training_iterations": "30", "training_iterations": "", - "_example_nic_type": "ainic|thor2|cx7", "nic_type": "", - "_example_nccl_ib_hca_list": "bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7", "nccl_ib_hca_list": "", - "_example_nccl_socket_ifname": "ens51f1np1", "nccl_socket_ifname": "", - "_example_gloo_socket_ifname": "ens51f1np1", "gloo_socket_ifname": "", - "_example_nccl_ib_gid_index": "3", "nccl_ib_gid_index": "", - "_example_hca_id_pattern": "bnxt_|rocep|mlx5_", "hca_id_pattern": "bnxt_|rocep", "nccl_debug": "ERROR", "hf_token_file": "/home/{user-id}/.hf_token", "shm_size": "128G", - "_comments_data_cache_dir": "This path should be accessible from all nodes like a common FS like NFS for distributed training", "data_cache_dir": "/home/{user-id}/cache", "megatron_root": "/workspace/Megatron-LM", "training_scripts": { @@ -996,8 +961,6 @@ This is the multi-node ``mi3xx_megatron_llama_distributed.json`` configuration f "fsdp": "0", "result_dict": { - "_example_throughput_per_gpu": "610.0", - "_example_tokens_per_gpu": "12000.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -1016,8 +979,6 @@ This is the multi-node ``mi3xx_megatron_llama_distributed.json`` configuration f "fsdp": "0", "result_dict": { - "_example_throughput_per_gpu": "620.0", - "_example_tokens_per_gpu": "14000.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -1039,8 +1000,6 @@ This is the multi-node ``mi3xx_megatron_llama_distributed.json`` configuration f "fsdp": "0", "result_dict": { - "_example_throughput_per_gpu": "530.0", - "_example_tokens_per_gpu": "1100.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -1059,8 +1018,6 @@ This is the multi-node ``mi3xx_megatron_llama_distributed.json`` configuration f "fsdp": "0", "result_dict": { - "_example_throughput_per_gpu": "550.0", - "_example_tokens_per_gpu": "1200.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -1090,9 +1047,6 @@ Use the parameters in these tables to configure the training file. * - Configuration parameters - Default values - Description - * - ``_comments__`` - - "Config file created for 4 nodes, change expected results based on number of nodes" - - A generic comment * - ``container_image`` - ``rocm/megatron-lm:v25.5_py310`` - Docker image used to run Megatron-LM @@ -1102,65 +1056,30 @@ Use the parameters in these tables to configure the training file. * - ``distributed_training`` - True - "True"/"False": Ehether to run training across multiple nodes - * - ``_example_nnodes`` - - 4 - - Example of number of cluster nodes participating in the job * - ``Nnodes`` - ```` - Number of cluster nodes participating in the distributed job - * - ``_example_master_address`` - - "X.X.X.X" - - Example IP of the master/coordinator node * - ``master_address`` - ```` - IP of the master/coordinator node - * - ``_example_training_iterations`` - - 30 - - Example of number of training iterations/steps to run in this test * - ``training_iterations`` - ```` - Number of training iterations/steps to run in this test - * - ``_example_nic_type`` - - ``ainic|thor2|cx7`` - - Example of NIC hardware type * - ``nic_type`` - ```` - NIC hardware type - * - ``_example_nccl_ib_hca_list`` - - Values: - - ``bnxt_re0`` - - ``bnxt_re1`` - - ``bnxt_re2`` - - ``bnxt_re3`` - - ``bnxt_re4`` - - ``bnxt_re5`` - - ``bnxt_re6`` - - ``bnxt_re7`` - - Example of a comma-separated list of InfiniBand HCA device names to use for NCCL/communication (multi-rail support) * - ``nccl_ib_hca_list`` - ```` - Comma-separated list of InfiniBand HCA device names to use for NCCL/communication (multi-rail support) - * - ``_example_nccl_socket_ifname`` - - ``ens51f1np1`` - - Example of a network interface name used by NCCL Network interface name used by NCCL / control channels * - ``nccl_socket_ifname`` - ```` - Network interface name used by NCCL Network interface name used by NCCL / control channels - * - ``_example_gloo_socket_ifname`` - - ``ens51f1np1`` - - Example of a network interface name used by Gloo control channels * - ``gloo_socket_ifname`` - ```` - Network interface name used by Gloo control channels * - ``nccl_ib_gid_index`` - ```` - GID index used for IB addressing (selects which GID) - * - ``_example_nccl_ib_gid_index`` - - 3 - - Example of GID index used for IB addressing (selects which GID entry on the HCA to use) - * - ``_example_hca_id_pattern`` - - ``bnxt_|rocep|mlx5_`` - - Example of HCA-id pattern used to verify the libbnxt copy succeeded inside the container * - ``hca_id_pattern`` - ``bnxt_|rocep`` - ``|``-separated list of NIC-name prefixes (e.g. ``bnxt_``, ``rocep``, ``mlx5_``) checked against ``ibv_devinfo`` ``hca_id:`` lines after the libbnxt copy. Each segment is treated as a literal prefix (regex special chars are escaped by the lib), so use ``|`` only as the separator -- not as part of a regex pattern within a segment. Add ``|mlx5_`` for Mellanox/RoCE NICs. @@ -1173,9 +1092,6 @@ Use the parameters in these tables to configure the training file. * - ``shm_size`` - 128G - Docker shared memory size - * - ``_comments_data_cache_dir`` - - "This path should be accessible from all nodes like a common FS like NFS for distributed training" - - A comment explaining ``data_cache_dir`` must be accessible from all nodes (NFS/shared FS). * - ``data_cache_dir`` - ``/home/{user-id}/cache`` - Dataset/cache directory (should be shared across nodes for distributed training unless using per-node copies) @@ -1282,8 +1198,6 @@ This section also contains the ``result_dict`` parameter. It describes the expec "result_dict": { - "_example_throughput_per_gpu": "610.0", - "_example_tokens_per_gpu": "12000.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -1340,8 +1254,6 @@ This section also contains the ``result_dict`` parameter. It describes the expec "result_dict": { - "_example_throughput_per_gpu": "610.0", - "_example_tokens_per_gpu": "14000.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -1398,8 +1310,6 @@ This section also contains the ``result_dict`` parameter. It describes the expec "result_dict": { - "_example_throughput_per_gpu": "530.0", - "_example_tokens_per_gpu": "1100.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } @@ -1456,8 +1366,6 @@ This section also contains the ``result_dict`` parameter. It describes the expec "result_dict": { - "_example_throughput_per_gpu": "550.0", - "_example_tokens_per_gpu": "1200.0", "throughput_per_gpu": "", "tokens_per_gpu": "" } From f329fc51f5a2071ffe1236744fe120eef7e6bbeb Mon Sep 17 00:00:00 2001 From: speriaswamy-amd Date: Fri, 14 Aug 2026 16:46:00 -0400 Subject: [PATCH 2/2] fix(config): finish stripping residual _comment_* keys missed by unstaged edits The prior commit's git add ran before the last two manual conflict resolutions on these upstream-diverged files, so it captured a partially stripped intermediate state. AIMVT-276 --- .../preflight/preflight_config.json | 38 +------------------ cvs/input/config_file/rccl/rccl_config.json | 4 +- 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/cvs/input/config_file/preflight/preflight_config.json b/cvs/input/config_file/preflight/preflight_config.json index 3a4f9b025..cfdba30ba 100644 --- a/cvs/input/config_file/preflight/preflight_config.json +++ b/cvs/input/config_file/preflight/preflight_config.json @@ -65,8 +65,6 @@ }, "node_smoke": { - "_comment": "Primus node_smoke checks via primus-cli direct (opt-in; default skip). See Primus docs/node-smoke-test-instruction.md", - "_setup_comment": "Primus clone/venv setup runs automatically when auto_setup is true (default). Manual equivalent:", "_setup_step_1": "git clone --recurse-submodules https://github.com/AMD-AIG-AIMA/Primus.git /home/{user-id}/INSTALL/Primus", "_setup_step_2": "cd /home/{user-id}/INSTALL/Primus && git checkout dev/preflight-direct-test", @@ -74,106 +72,72 @@ "_setup_note": "Paths use {user-id} resolved at runtime. Set auto_setup to false to skip automatic install.", "auto_setup": true, - "_comment_auto_setup": "When true, clone/update Primus and create venv with minimal deps (torch) on each node before node_smoke.", "setup_timeout": 600, - "_comment_setup_timeout": "SSH timeout (seconds) for the per-node Primus auto_setup step (clone + pip install).", "force_reclone": false, - "_comment_force_reclone": "When true, rm -rf primus_dir and clone fresh on every run (destructive). With shared_install (default), only the leader node reclones.", "shared_install": true, - "_comment_shared_install": "When true (default), only the first node clones/updates Primus and installs the venv on shared NFS home; other nodes wait. Set false only if primus_dir and venv_activate are local per node.", "pip_install_mode": "minimal", - "_comment_pip_install_mode": "Venv deps after clone: 'minimal' (torch only), 'requirements' (pip install -r requirements.txt), or 'skip' (venv only).", "torch_pip_index_url": "https://download.pytorch.org/whl/rocm6.2", - "_comment_torch_pip_index_url": "PyTorch wheel index for minimal install. Match your ROCm version (e.g. rocm6.2, rocm7.1).", "primus_git_url": "https://github.com/AMD-AIG-AIMA/Primus.git", - "_comment_primus_git_url": "Primus repository URL for one-time clone.", "primus_git_branch": "dev/preflight-direct-test", - "_comment_primus_git_branch": "Git branch to checkout after clone. node_smoke and primus-cli direct preflight live on this branch.", "primus_git_recurse_submodules": false, - "_comment_primus_git_recurse_submodules": "Clone submodules (Megatron, etc.). false is recommended for node_smoke — submodules are not required and slow setup.", "primus_dir": "/home/{user-id}/INSTALL/Primus", - "_comment_primus_dir": "Path where Primus is cloned on each cluster node. Must match the clone target in setup step 1. Required when connectivity_mode is 'run'.", "venv_activate": "/home/{user-id}/envs/preflight/.venv/bin/activate", - "_comment_venv_activate": "Path to the Python virtualenv activate script used by primus-cli direct. Required when connectivity_mode is 'run'.", "connectivity_mode": "skip", - "_comment_connectivity_mode": "Options: 'run' (host/GPU/RDMA roll-call via node_smoke) or 'skip' (default).", "gpus_per_node": 8, - "_comment_gpus_per_node": "Expected GPU count per node (exported as GPUS_PER_NODE and passed to node_smoke --expected-gpus).", "master_port": 1234, - "_comment_master_port": "MASTER_PORT for the distributed env primus-cli sets up across SSH-launched ranks.", "dump_path": "", - "_comment_dump_path": "Directory for per-node smoke/*.json output. Leave empty to use /node_smoke.", "expected_rdma_nics": null, - "_comment_expected_rdma_nics": "Hard-fail when training RDMA NIC count differs. null defaults to len(node_check.rdma_interfaces). Example: 8.", "ulimit_l_min_gb": 32, - "_comment_ulimit_l_min_gb": "FAIL when RLIMIT_MEMLOCK is below this many GiB. 0 disables.", "shm_min_gb": 8, - "_comment_shm_min_gb": "FAIL when /dev/shm is below this many GiB. 0 disables.", "skip_dmesg": false, - "_comment_skip_dmesg": "Skip the dmesg recent-error scan (use inside unprivileged containers).", "allow_foreign_procs": false, - "_comment_allow_foreign_procs": "Do not FAIL on foreign GPU processes. Recommended inside containers where proc names resolve to N/A.", "allowed_procs": "gpuagent,rocm-smi-daemon,amd-smi,dcgm-exporter", - "_comment_allowed_procs": "Comma-separated process names allowed to hold GPUs without failing the node.", "require_tools": "", - "_comment_require_tools": "Comma-separated tools that must exist in PATH for PASS (amd-smi, rocm-smi, lsof). Empty = warn only.", "nccl_socket_ifname": "", - "_comment_nccl_socket_ifname": "Optional NCCL_SOCKET_IFNAME / GLOO_SOCKET_IFNAME override for node_smoke.", "gloo_socket_ifname": "", - "_comment_gloo_socket_ifname": "Optional GLOO_SOCKET_IFNAME override (defaults to nccl_socket_ifname when empty).", "nccl_ib_hca": "", - "_comment_nccl_ib_hca": "Optional NCCL_IB_HCA override. Defaults to comma-joined node_check.rdma_interfaces.", "nccl_ib_gid_index": null, - "_comment_nccl_ib_gid_index": "Optional NCCL_IB_GID_INDEX override. Defaults to node_check.gid_index.", "ssh_timeout": 300, - "_comment_ssh_timeout": "SSH timeout in seconds for each node's node_smoke invocation (~30s Tier 1; use 600+ with tier2_perf).", "tier2_perf": false, - "_comment_tier2_perf": "Enable Tier 2 perf sanity (--tier2-perf): 8192³ GEMM TFLOPS, HBM D2D copy bandwidth, local multi-GPU RCCL all-reduce. Requires NCCL_IB_HCA / NCCL_SOCKET_IFNAME (see launch_nodesmoke_ssh.sh).", "gemm_tflops_min": 600, - "_comment_gemm_tflops_min": "Tier 2 FAIL below this large GEMM TFLOPS (--gemm-tflops-min). MI300X healthy nodes typically exceed 600.", "hbm_gbs_min": 2000, - "_comment_hbm_gbs_min": "Tier 2 FAIL below this HBM device-to-device bandwidth in GB/s (--hbm-gbs-min). MI300X healthy ≈ 4500–5000.", "rccl_gbs_min": 100, - "_comment_rccl_gbs_min": "Tier 2 FAIL below this local multi-GPU RCCL all-reduce bandwidth in GB/s (--rccl-gbs-min).", "rccl_size_mb": 64, - "_comment_rccl_size_mb": "Tier 2 local RCCL all-reduce tensor size in MB (--rccl-size-mb).", "rccl_timeout_sec": 120, - "_comment_rccl_timeout_sec": "Tier 2 local RCCL all-reduce hard timeout in seconds (--rccl-timeout-sec).", - "extra_args": [], - "_comment_extra_args": "Additional node_smoke CLI flags forwarded to primus-cli. Example: [\"--no-clean-dump-path\"]." + "extra_args": [] }, "reporting": { diff --git a/cvs/input/config_file/rccl/rccl_config.json b/cvs/input/config_file/rccl/rccl_config.json index 10c5738e8..a1c548d75 100644 --- a/cvs/input/config_file/rccl/rccl_config.json +++ b/cvs/input/config_file/rccl/rccl_config.json @@ -7,9 +7,7 @@ "mpi_dir": "/home/{user-id}/openmpi/bin", "mpi_oob_port": "eth0", "ucx_tls": "rc,self,sm,tcp", - "_comment_ucx_tls": "When user requested UCX either leave this parameter value blank for auto assignment or assign values e.g. rc,self,sm,tcp..", - "net_dev_list": "", - "_comment_net_dev_list": "Leave empty for auto-detection from backend NICs, or set explicitly e.g. bnxt_re0:1,bnxt_re1:1,.." + "net_dev_list": "" }, "env_source_script": "/home/{user-id}/thor2_env_script.sh",