-
Notifications
You must be signed in to change notification settings - Fork 109
DRAFT: Add Zstd GPU CI test infrastructure (Phase 1) #108
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
brohan203
wants to merge
6
commits into
microsoft:development
Choose a base branch
from
brohan203:users/rohanborkar/ds_testInfra_phase1
base: development
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
9313df4
Add Zstd GPU CI test infrastructure (Phase 1)
9266056
zstdgpu_demo: non-fatal CPU-sim break handler + non-zero exit on corr…
1456908
histogram script handles inf values
8768a5e
zstdgpu_ci_tests: address PR #108 review feedback (@chmann-micro)
d94e83a
small cleanup, no change to functionality
ee9e029
zstdgpu_demo: trim comments and drop redundant exit message
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| #!/usr/bin/env python3 | ||
| """ | ||
| Generate histogram PNGs from zstdgpu_demo performance CSV output. | ||
|
|
||
| Usage: | ||
| python generate_histogram.py --input <csv_file> --output <png_file> [--title <title>] | ||
|
|
||
| Consumes the wide-format CSV emitted by zstdgpu_demo's --out-csv flag: | ||
| RunIdx, Stage 0 (us), Stage 0 :: <scope> (us), ..., Readback 0 (us), | ||
| Stage 1 (us), ..., Stage 2 (us), Bandwidth (GB/s) | ||
|
|
||
| Plots a histogram of the 'Bandwidth (GB/s)' column. Per-stage timing columns | ||
| are preserved in the CSV but not plotted here (the histogram is the summary | ||
| view; users wanting per-stage detail can read the CSV directly). | ||
| """ | ||
|
|
||
| import argparse | ||
| import csv | ||
| import math | ||
| import sys | ||
|
|
||
|
|
||
| def try_import_matplotlib(): | ||
| try: | ||
| import matplotlib | ||
| matplotlib.use("Agg") # Non-interactive backend for CI | ||
| import matplotlib.pyplot as plt | ||
| return plt | ||
| except ImportError: | ||
| print( | ||
| "WARNING: matplotlib not installed. Install with: pip install matplotlib", | ||
| file=sys.stderr, | ||
| ) | ||
| return None | ||
|
|
||
|
|
||
| # Match Pavel's --out-csv column header verbatim ("Bandwidth (GB/s)"). | ||
| # Kept case-insensitive and whitespace-tolerant in case the schema spelling | ||
| # drifts upstream — the eyeballed match is "bandwidth". | ||
| def _is_bandwidth_column(col_name: str) -> bool: | ||
| return col_name is not None and "bandwidth" in col_name.strip().lower() | ||
|
|
||
|
|
||
| def main(): | ||
| parser = argparse.ArgumentParser(description="Generate histogram from zstdgpu perf CSV") | ||
| parser.add_argument("--input", required=True, help="Path to input CSV file") | ||
| parser.add_argument("--output", required=True, help="Path to output PNG file") | ||
| parser.add_argument("--title", default="Bandwidth", help="Chart title") | ||
| args = parser.parse_args() | ||
|
|
||
| plt = try_import_matplotlib() | ||
| if plt is None: | ||
| return 1 | ||
|
|
||
| data = [] | ||
| skipped_non_finite = 0 | ||
| bandwidth_col = None | ||
| with open(args.input, newline="") as f: | ||
| reader = csv.DictReader(f) | ||
| if reader.fieldnames is None: | ||
| print(f"CSV has no header row: {args.input}", file=sys.stderr) | ||
| return 1 | ||
| # Pick the bandwidth column by name (resilient to header drift). | ||
| for col in reader.fieldnames: | ||
| if _is_bandwidth_column(col): | ||
| bandwidth_col = col | ||
| break | ||
| if bandwidth_col is None: | ||
| print( | ||
| f"No Bandwidth column found in {args.input} " | ||
| f"(headers: {reader.fieldnames})", | ||
| file=sys.stderr, | ||
| ) | ||
| return 1 | ||
|
|
||
| for row in reader: | ||
| raw = row.get(bandwidth_col, "") | ||
| if raw is None or raw == "": | ||
| continue | ||
| try: | ||
| val = float(raw) | ||
| except ValueError: | ||
| # Pavel may write empty strings for skipped iterations; ignore. | ||
| continue | ||
| if math.isfinite(val): | ||
| data.append(val) | ||
| else: | ||
| skipped_non_finite += 1 | ||
|
|
||
| if skipped_non_finite > 0: | ||
| print( | ||
| f"WARNING: Skipped {skipped_non_finite} non-finite (Inf/NaN) value(s) from {args.input}", | ||
| file=sys.stderr, | ||
| ) | ||
|
|
||
| if not data: | ||
| print( | ||
| f"No finite Bandwidth (GB/s) data found in {args.input}", | ||
| file=sys.stderr, | ||
| ) | ||
| return 1 | ||
|
|
||
| plt.figure() | ||
| plt.hist(data, bins=20) | ||
| plt.xlabel("Bandwidth (GB/s)") | ||
| plt.ylabel("Count") | ||
| plt.title(args.title) | ||
| plt.savefig(args.output) | ||
| plt.close() | ||
| print(f"Generated: {args.output}") | ||
| return 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| sys.exit(main()) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,234 @@ | ||
| /** | ||
| * Copyright (c) Microsoft. All rights reserved. | ||
| * This code is licensed under the MIT License (MIT). | ||
| * THIS CODE IS PROVIDED *AS IS* WITHOUT WARRANTY OF | ||
| * ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING ANY | ||
| * IMPLIED WARRANTIES OF FITNESS FOR A PARTICULAR | ||
| * PURPOSE, MERCHANTABILITY, OR NON-INFRINGEMENT. | ||
| */ | ||
|
|
||
| // Entry point for the Zstd GPU CI tests. Thin GTest wrapper that shells out | ||
| // to zstdgpu_demo.exe to validate Zstd GPU decompression shaders. | ||
| // | ||
| // main() parses custom CLI flags (--content-path, --demo-path, etc.), | ||
| // validates them (hard failure with non-zero exit on any misconfiguration — | ||
| // never silently skip), discovers .zst files under the content path exactly | ||
| // once, then hands off to GTest which runs the parameterized suite defined | ||
| // in zstdgpu_ci_tests.cpp. Each test spawns the demo as a child process. | ||
| // | ||
| // If the content path is missing, unreadable, or contains no .zst files, or | ||
| // if the demo executable is missing, the process exits non-zero before any | ||
| // test runs. This intentionally prevents a "green" run with zero coverage. | ||
| // | ||
| // This file also owns the g_testConfig storage and the file discovery | ||
| // implementation declared in zstdgpu_ci_tests.h. | ||
|
|
||
| #include "zstdgpu_ci_tests.h" | ||
| #include <gtest/gtest.h> | ||
| #include <algorithm> | ||
| #include <cstring> | ||
| #include <filesystem> | ||
| #include <iostream> | ||
| #include <string> | ||
| #include <system_error> | ||
|
|
||
| // Global config, declared extern in zstdgpu_ci_tests.h. Set once in main(), | ||
| // read from test bodies. | ||
| TestConfig g_testConfig; | ||
|
|
||
| // File discovery | ||
| // | ||
| // Uses the non-throwing overloads of recursive_directory_iterator so a single | ||
| // unreadable subdirectory anywhere in the tree does not abort the entire walk | ||
| // (which would take down the process before any test runs — see review | ||
| // feedback on the throwing overload). Per-entry errors are logged as warnings | ||
| // and the walk continues. | ||
|
|
||
| std::vector<std::string> DiscoverZstFiles(const std::string& contentPath) | ||
| { | ||
| std::vector<std::string> files; | ||
|
|
||
| if (contentPath.empty() || !std::filesystem::exists(contentPath) || !std::filesystem::is_directory(contentPath)) | ||
| { | ||
| return files; | ||
| } | ||
|
|
||
| std::error_code ec; | ||
| auto iter = std::filesystem::recursive_directory_iterator( | ||
| contentPath, | ||
| std::filesystem::directory_options::skip_permission_denied, | ||
| ec); | ||
| if (ec) | ||
| { | ||
| std::cerr << "Warning: could not open '" << contentPath << "' for scanning: " | ||
| << ec.message() << "\n"; | ||
| return files; | ||
| } | ||
|
|
||
| auto endIter = std::filesystem::recursive_directory_iterator(); | ||
| while (iter != endIter) | ||
| { | ||
| std::error_code entryEc; | ||
| if (iter->is_regular_file(entryEc) && !entryEc) | ||
| { | ||
| if (iter->path().extension() == ".zst") | ||
| { | ||
| files.push_back(iter->path().string()); | ||
| } | ||
| } | ||
| else if (entryEc) | ||
| { | ||
| std::cerr << "Warning: skipping '" << iter->path().string() << "': " | ||
| << entryEc.message() << "\n"; | ||
| } | ||
| iter.increment(ec); | ||
| if (ec) | ||
| { | ||
| std::cerr << "Warning: recursive walk aborted early after '" | ||
| << iter->path().string() << "': " << ec.message() << "\n"; | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| std::sort(files.begin(), files.end()); | ||
| return files; | ||
| } | ||
|
|
||
| // CLI and entry point | ||
|
|
||
| // QOL for diagnostics. For running manually | ||
| // Activate with --help-ci to avoid conflicting with GTest's own --help output. | ||
| static void PrintUsage(const char* exe) | ||
| { | ||
| std::cout << "Usage: " << exe << " [gtest_options] [options]\n" | ||
| << "\n" | ||
| << "Options:\n" | ||
| << " --content-path <dir> Directory containing .zst test files (required)\n" | ||
| << " --demo-path <path> Path to zstdgpu_demo.exe (required)\n" | ||
| << " --log-dir <dir> Directory for logs and CSV output\n" | ||
| << " --log-file <path> Consolidated text log file\n" | ||
| << " --run-count <N> Perf test iteration count (default: 40)\n" | ||
| << " --timeout <seconds> Per-test process timeout (default: no timeout)\n" | ||
| << std::endl; | ||
| } | ||
|
|
||
| // Prints an error prefixed with "Error:" to stderr and returns the exit code so | ||
| // main() can 'return Fail(...)' in a single expression. Keeps validation blocks | ||
| // short and consistent. | ||
| static int Fail(const std::string& msg) | ||
| { | ||
| std::cerr << "Error: " << msg << std::endl; | ||
| return 1; | ||
| } | ||
|
|
||
| // Parse custom flags out of argv before we hand argv to GTest. GTest's own | ||
| // InitGoogleTest() runs later and will consume its own flags (e.g. | ||
| // --gtest_filter). Returns true on success; on --help-ci, prints usage and | ||
| // sets shouldExit=true so main() can return 0 cleanly. | ||
| static bool ParseArgs(int argc, char** argv, TestConfig& config, bool& shouldExit) | ||
| { | ||
| shouldExit = false; | ||
| for (int i = 1; i < argc; ++i) | ||
| { | ||
| if (std::strcmp(argv[i], "--content-path") == 0 && i + 1 < argc) | ||
| { | ||
| config.contentPath = argv[++i]; | ||
| } | ||
| else if (std::strcmp(argv[i], "--demo-path") == 0 && i + 1 < argc) | ||
| { | ||
| config.demoPath = argv[++i]; | ||
| } | ||
| else if (std::strcmp(argv[i], "--log-dir") == 0 && i + 1 < argc) | ||
| { | ||
| config.logDir = argv[++i]; | ||
| } | ||
| else if (std::strcmp(argv[i], "--log-file") == 0 && i + 1 < argc) | ||
| { | ||
| config.logFile = argv[++i]; | ||
| } | ||
| else if (std::strcmp(argv[i], "--run-count") == 0 && i + 1 < argc) | ||
| { | ||
| config.runCount = std::atoi(argv[++i]); | ||
| if (config.runCount <= 0) | ||
| config.runCount = 40; | ||
| } | ||
| else if (std::strcmp(argv[i], "--timeout") == 0 && i + 1 < argc) | ||
| { | ||
| config.timeoutSeconds = std::atoi(argv[++i]); | ||
| if (config.timeoutSeconds < 0) | ||
| config.timeoutSeconds = 0; | ||
| } | ||
| else if (std::strcmp(argv[i], "--help-ci") == 0) | ||
| { | ||
| PrintUsage(argv[0]); | ||
| shouldExit = true; | ||
| return true; | ||
| } | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| // Fail-loud validation + one-shot filesystem discovery. Any misconfiguration | ||
| // means the run cannot produce meaningful test coverage, so we return a | ||
| // non-zero exit code before any test instantiates. Silent skipping would let | ||
| // broken pipelines pass green with zero coverage. | ||
| // | ||
| // On success also creates the log directory (defaulting to cwd) so downstream | ||
| // writes don't have to check. | ||
| static int ValidateAndDiscover(TestConfig& config) | ||
| { | ||
| if (config.demoPath.empty()) | ||
|
brohan203 marked this conversation as resolved.
|
||
| return Fail("--demo-path is required."); | ||
| if (!std::filesystem::exists(config.demoPath)) | ||
| return Fail("--demo-path '" + config.demoPath + "' does not exist."); | ||
| if (!std::filesystem::is_regular_file(config.demoPath)) | ||
| return Fail("--demo-path '" + config.demoPath + "' is not a regular file."); | ||
|
|
||
| if (config.contentPath.empty()) | ||
| return Fail("--content-path is required."); | ||
| if (!std::filesystem::exists(config.contentPath)) | ||
| return Fail("--content-path '" + config.contentPath + "' does not exist."); | ||
| if (!std::filesystem::is_directory(config.contentPath)) | ||
| return Fail("--content-path '" + config.contentPath + "' is not a directory."); | ||
|
|
||
| // Discover exactly once. Cached on TestConfig for GetTestFiles() to reuse | ||
| // when instantiating the parameterized fixture — avoids walking the tree | ||
| // twice. | ||
| config.discoveredFiles = DiscoverZstFiles(config.contentPath); | ||
| if (config.discoveredFiles.empty()) | ||
| return Fail("--content-path '" + config.contentPath + "' contains no .zst files."); | ||
|
|
||
| std::cout << "Discovered " << config.discoveredFiles.size() << " .zst file(s) at '" | ||
| << config.contentPath << "'.\n"; | ||
|
|
||
| if (config.logDir.empty()) | ||
| { | ||
| config.logDir = std::filesystem::current_path().string(); | ||
| } | ||
| if (!std::filesystem::exists(config.logDir)) | ||
| { | ||
| std::filesystem::create_directories(config.logDir); | ||
| } | ||
|
|
||
| return 0; | ||
| } | ||
|
|
||
| int main(int argc, char** argv) | ||
| { | ||
| TestConfig config; | ||
|
|
||
| bool shouldExit = false; | ||
| if (!ParseArgs(argc, argv, config, shouldExit)) | ||
| return 1; | ||
| if (shouldExit) | ||
| return 0; // --help-ci printed usage and asked to exit cleanly | ||
|
|
||
| if (int rc = ValidateAndDiscover(config); rc != 0) | ||
| return rc; | ||
|
|
||
| g_testConfig = std::move(config); | ||
|
|
||
| testing::InitGoogleTest(&argc, argv); | ||
| testing::GTEST_FLAG(catch_exceptions) = false; | ||
| return RUN_ALL_TESTS(); | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.