diff --git a/.github/workflows/claude.yaml b/.github/workflows/claude.yaml index d07f4be..6eee176 100644 --- a/.github/workflows/claude.yaml +++ b/.github/workflows/claude.yaml @@ -25,12 +25,12 @@ jobs: id-token: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 1 - name: Run Claude PR Action - uses: anthropics/claude-code-action@beta + uses: anthropics/claude-code-action@v1 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} timeout_minutes: "60" \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d3623cf..218a375 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,10 +23,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5.6.0 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -44,7 +44,7 @@ jobs: pip install pytest>=6.2.5 pytest-cov>=2.12.0 nose>=1.3.7 - name: setup-stackql - uses: stackql/setup-stackql@v2.2.3 + uses: stackql/setup-stackql@v2.3.1 with: use_wrapper: true diff --git a/.gitignore b/.gitignore index 173aff7..a784856 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ ENV/ # stackql .stackql/ stackql +stackql.exe stackql-*.sh .env nohup.out diff --git a/CHANGELOG.md b/CHANGELOG.md index 6033a75..fad689a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## v3.8.4 (2026-06-29) + +### Updates + +- Routed all binary downloads through `releases.stackql.io` (including macOS) and added a `pystackql/{version}` User-Agent so downloads can be identified +- Updated the test suite and test runners for Linux and Windows + ## v3.8.2 (2025-11-09) ### New Features diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b806bac --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,35 @@ +# CLAUDE.md + +PyStackQL - a Python wrapper for the [StackQL](https://stackql.io) query engine, which runs SQL against cloud and SaaS providers. Published to PyPI as `pystackql`. + +## Layout + +- `pystackql/core/` - main logic + - `stackql.py` - `StackQL` class, the primary public interface (`execute`, `executeStmt`, `executeQueriesAsync`, `properties`, `upgrade`, `test_connection`) + - `query.py` - `QueryExecutor` / `AsyncQueryExecutor` + - `server.py` - `ServerConnection` (server mode via psycopg/postgres wire protocol) + - `output.py` - `OutputFormatter` (output formats: `dict`, `pandas`, `csv`, `markdownkv`) + - `binary.py` - `BinaryManager` (locates/downloads the stackql binary) + - `error_detector.py` - `ErrorDetector`, matches messages against patterns in `pystackql/errors.yaml` +- `pystackql/utils/` - platform, binary, download, auth, and param helpers (re-exported from `utils/__init__.py`) +- `pystackql/magic_ext/` - Jupyter magics: `StackqlMagic` (local) and `StackqlServerMagic` (server), sharing `base.py` +- `pystackql/__init__.py` - public API: `StackQL`, `StackqlMagic`, `StackqlServerMagic` + +## Two execution modes + +- **Local mode** (default): downloads/runs the stackql binary as a subprocess. +- **Server mode** (`server_mode=True`): connects to a running stackql server over the postgres wire protocol. `csv` output and several local-only options are unsupported here. + +## Testing + +Tests use the no-auth Homebrew provider and provider-agnostic literal queries (avoid adding auth-requiring tests). + +- Non-server tests: `python run_tests.py` (optionally pass specific `tests/test_*.py` files, `-v`) +- Server tests: start a server first (`stackql srv --pgsrv.address 127.0.0.1 --pgsrv.port 5466`), then `python run_server_tests.py` +- CI (`.github/workflows/test.yaml`) runs both across Linux/macOS/Windows and Python 3.9-3.13. + +## Conventions + +- Supports Python 3.9-3.13 on Windows, macOS, and Linux - keep changes cross-platform. +- `README.rst` is reStructuredText (the PyPI readme); docs in `docs/` build to ReadTheDocs from Sphinx-style docstrings. Update the `StackQL.__init__` docstring when changing constructor params. +- Bump `version` in `pyproject.toml` for releases; record changes in `CHANGELOG.md`. diff --git a/README.rst b/README.rst index e84b8fd..cf753fc 100644 --- a/README.rst +++ b/README.rst @@ -144,14 +144,14 @@ To build the package, you will need to install the following packages: :: - pip install build + pip install build twine Then, from the root directory of the repository, run: :: rm -rf dist/* - python3 -m build + python3 -m build The package will be built in the ``dist`` directory. diff --git a/launch_venv.sh b/launch_venv.sh deleted file mode 100644 index 741d635..0000000 --- a/launch_venv.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/bin/bash -# launch_venv.sh - Script to create, set up and activate a Python virtual environment for PyStackQL - -# Use simpler code without colors when running with sh -if [ -n "$BASH_VERSION" ]; then - # Color definitions for bash - GREEN='\033[0;32m' - YELLOW='\033[1;33m' - BLUE='\033[0;34m' - RED='\033[0;31m' - NC='\033[0m' # No Color - - # Function to print colored text in bash - cecho() { - printf "%b%s%b\n" "$1" "$2" "$NC" - } -else - # No colors for sh - cecho() { - echo "$2" - } -fi - -# Default virtual environment name -VENV_NAME=".venv" -REQUIREMENTS_FILE="requirements.txt" - -# Function to check if command exists -command_exists() { - command -v "$1" >/dev/null 2>&1 -} - -# Banner -cecho "$BLUE" "=======================================" -cecho "$BLUE" " PyStackQL Development Environment " -cecho "$BLUE" "=======================================" -echo "" - -# Check for Python -if ! command_exists python3; then - cecho "$RED" "Error: Python 3 is not installed." - echo "Please install Python 3 and try again." - exit 1 -fi - -# Print Python version -cecho "$YELLOW" "Using Python:" -python3 --version -echo "" - -# Create virtual environment if it doesn't exist -if [ ! -d "$VENV_NAME" ]; then - cecho "$YELLOW" "Creating virtual environment in ${VENV_NAME}..." - python3 -m venv "$VENV_NAME" - if [ $? -ne 0 ]; then - cecho "$RED" "Error: Failed to create virtual environment." - exit 1 - fi - cecho "$GREEN" "Virtual environment created successfully." -else - cecho "$YELLOW" "Using existing virtual environment in ${VENV_NAME}" -fi - -# Determine the activate script based on OS -case "$OSTYPE" in - msys*|win*|cygwin*) - # Windows - ACTIVATE_SCRIPT="$VENV_NAME/Scripts/activate" - ;; - *) - # Unix-like (Linux, macOS) - ACTIVATE_SCRIPT="$VENV_NAME/bin/activate" - ;; -esac - -# Check if activation script exists -if [ ! -f "$ACTIVATE_SCRIPT" ]; then - cecho "$RED" "Error: Activation script not found at $ACTIVATE_SCRIPT" - echo "The virtual environment may be corrupt. Try removing the $VENV_NAME directory and running this script again." - exit 1 -fi - -# Source the activation script -cecho "$YELLOW" "Activating virtual environment..." -. "$ACTIVATE_SCRIPT" -if [ $? -ne 0 ]; then - cecho "$RED" "Error: Failed to activate virtual environment." - exit 1 -fi - -# Install/upgrade pip, setuptools, and wheel -cecho "$YELLOW" "Upgrading pip, setuptools, and wheel..." -pip install --upgrade pip setuptools wheel -if [ $? -ne 0 ]; then - cecho "$RED" "Warning: Failed to upgrade pip, setuptools, or wheel. Continuing anyway." -fi - -# Check if requirements.txt exists -if [ ! -f "$REQUIREMENTS_FILE" ]; then - cecho "$RED" "Error: $REQUIREMENTS_FILE not found." - echo "Please make sure the file exists in the current directory." - cecho "$YELLOW" "Continuing with an activated environment without installing dependencies." -else - # Install requirements - cecho "$YELLOW" "Installing dependencies from $REQUIREMENTS_FILE..." - pip install -r "$REQUIREMENTS_FILE" - if [ $? -ne 0 ]; then - cecho "$RED" "Warning: Some dependencies may have failed to install." - else - cecho "$GREEN" "Dependencies installed successfully." - fi -fi - -# Install the package in development mode if setup.py exists -if [ -f "setup.py" ]; then - cecho "$YELLOW" "Installing PyStackQL in development mode..." - pip install . - if [ $? -ne 0 ]; then - cecho "$RED" "Warning: Failed to install package in development mode." - else - cecho "$GREEN" "Package installed in development mode." - fi -fi - -# Success message -echo "" -cecho "$GREEN" "Virtual environment is now set up and activated!" -cecho "$YELLOW" "You can use PyStackQL and run tests." -echo "" -cecho "$BLUE" "To run tests:" -echo " python run_tests.py" -echo "" -cecho "$BLUE" "To deactivate the virtual environment when done:" -echo " deactivate" -echo "" -cecho "$BLUE" "=======================================" - -# Keep the terminal open with the activated environment -# The script will be source'd, so the environment stays active -exec "${SHELL:-bash}" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 49f5698..e4a0337 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pystackql" -version = "3.8.2" +version = "3.8.4" description = "A Python interface for StackQL" readme = "README.rst" authors = [ diff --git a/pystackql/core/binary.py b/pystackql/core/binary.py index 023f671..fbee913 100644 --- a/pystackql/core/binary.py +++ b/pystackql/core/binary.py @@ -39,6 +39,10 @@ def __init__(self, download_dir=None): else: # Use provided download_dir or default self.download_dir = download_dir if download_dir else get_download_dir() + # Create a user-supplied download directory if it does not exist + # (get_download_dir already ensures the default location exists). + if not os.path.exists(self.download_dir): + os.makedirs(self.download_dir, exist_ok=True) self.bin_path = os.path.join(self.download_dir, get_binary_name(self.system)) # Check if binary exists and get version diff --git a/pystackql/core/stackql.py b/pystackql/core/stackql.py index 1e0be4d..0902799 100644 --- a/pystackql/core/stackql.py +++ b/pystackql/core/stackql.py @@ -170,11 +170,23 @@ def __init__(self, self.server_port = server_port self.server_connection = ServerConnection(server_address, server_port) else: - # Local mode - execute the binary locally - # Get all parameters from local variables (excluding 'self') - local_params = locals().copy() - local_params.pop('self') - + # Local mode - execute the binary locally. + # Combine the explicit constructor arguments with any additional + # **kwargs (e.g. download_dir, api_timeout, proxy settings, custom_auth). + # kwargs is spread flat so these reach setup_local_mode as top-level + # keys rather than being nested under a 'kwargs' entry. + local_params = { + 'server_mode': server_mode, + 'server_address': server_address, + 'server_port': server_port, + 'output': output, + 'sep': sep, + 'header': header, + 'debug': debug, + 'debug_log_file': debug_log_file, + **kwargs, + } + # Set up local mode - this sets the instance attributes and returns params self.params = setup_local_mode(self, **local_params) diff --git a/pystackql/utils/download.py b/pystackql/utils/download.py index 5fd71ea..ee7196a 100644 --- a/pystackql/utils/download.py +++ b/pystackql/utils/download.py @@ -11,6 +11,25 @@ import platform import requests +from .package import get_package_version + +# Base URL for all stackql binary downloads. This is a Cloudflare proxy in front +# of the GitHub releases, used so downloads can be attributed and cached. +RELEASES_BASE_URL = 'https://releases.stackql.io/stackql/latest' + + +def get_user_agent(): + """Builds the User-Agent string used for stackql binary downloads. + + The versioned identifier (e.g. ``pystackql/3.8.2``) lets the proxy logs + attribute downloads to pystackql. + + Returns: + str: The User-Agent header value + """ + version = get_package_version("pystackql") or "unknown" + return f"pystackql/{version}" + def get_download_dir(): """Gets the directory to download the stackql binary. @@ -38,11 +57,11 @@ def get_download_url(): machine_val = platform.machine() if system_val == 'Linux' and machine_val == 'x86_64': - return 'https://releases.stackql.io/stackql/latest/stackql_linux_amd64.zip' + return f'{RELEASES_BASE_URL}/stackql_linux_amd64.zip' elif system_val == 'Windows': - return 'https://releases.stackql.io/stackql/latest/stackql_windows_amd64.zip' + return f'{RELEASES_BASE_URL}/stackql_windows_amd64.zip' elif system_val == 'Darwin': - return 'https://storage.googleapis.com/stackql-public-releases/latest/stackql_darwin_multiarch.pkg' + return f'{RELEASES_BASE_URL}/stackql_darwin_multiarch.pkg' else: raise Exception(f"ERROR: [get_download_url] unsupported OS type: {system_val} {machine_val}") @@ -59,7 +78,8 @@ def download_file(url, path, showprogress=True): Exception: If the download fails """ try: - r = requests.get(url, stream=True) + headers = {"User-Agent": get_user_agent()} + r = requests.get(url, stream=True, headers=headers) r.raise_for_status() total_size_in_bytes = int(r.headers.get('content-length', 0)) block_size = 1024 diff --git a/pystackql/utils/helpers.py b/pystackql/utils/helpers.py deleted file mode 100644 index de0c329..0000000 --- a/pystackql/utils/helpers.py +++ /dev/null @@ -1,284 +0,0 @@ -# pystackql/utils/helpers.py - -""" -Utility functions for PyStackQL package. - -This module contains helper functions for binary management, platform detection, -and other utilities needed by the PyStackQL package. -""" - -import subprocess -import platform -import json -import site -import os -import requests -import zipfile - -# Conditional import for package metadata retrieval -try: - from importlib.metadata import version, PackageNotFoundError -except ImportError: - # This is for Python versions earlier than 3.8 - from importlib_metadata import version, PackageNotFoundError - - -def is_binary_local(system_platform): - """Checks if the binary exists at the specified local path. - - Args: - system_platform (str): The operating system platform - - Returns: - bool: True if the binary exists at the expected local path - """ - if system_platform == 'Linux' and os.path.exists('/usr/local/bin/stackql'): - return True - return False - - -def get_package_version(package_name): - """Gets the version of the specified package. - - Args: - package_name (str): The name of the package - - Returns: - str: The version of the package or None if not found - """ - try: - pkg_version = version(package_name) - if pkg_version is None: - print(f"Warning: Retrieved version for '{package_name}' is None!") - return pkg_version - except PackageNotFoundError: - print(f"Warning: Package '{package_name}' not found!") - return None - - -def get_platform(): - """Gets the current platform information. - - Returns: - tuple: (platform_string, system_value) - - platform_string: A string with platform details - - system_value: The operating system name - """ - system_val = platform.system() - machine_val = platform.machine() - platform_val = platform.platform() - python_version_val = platform.python_version() - return ( - f"{system_val} {machine_val} ({platform_val}), Python {python_version_val}", - system_val - ) - - -def get_download_dir(): - """Gets the directory to download the stackql binary. - - Returns: - str: The directory path - """ - # check if site.getuserbase() dir exists - if not os.path.exists(site.getuserbase()): - # if not, create it - os.makedirs(site.getuserbase()) - return site.getuserbase() - - -def get_binary_name(system_platform): - """Gets the binary name based on the platform. - - Args: - system_platform (str): The operating system platform - - Returns: - str: The name of the binary - """ - if system_platform.startswith('Windows'): - return r'stackql.exe' - elif system_platform.startswith('Darwin'): - return r'stackql/Payload/stackql' - else: - return r'stackql' - - -def get_download_url(): - """Gets the download URL for the stackql binary based on the platform. - - Returns: - str: The download URL - - Raises: - Exception: If the platform is not supported - """ - system_val = platform.system() - machine_val = platform.machine() - - if system_val == 'Linux' and machine_val == 'x86_64': - return 'https://releases.stackql.io/stackql/latest/stackql_linux_amd64.zip' - elif system_val == 'Windows': - return 'https://releases.stackql.io/stackql/latest/stackql_windows_amd64.zip' - elif system_val == 'Darwin': - return 'https://storage.googleapis.com/stackql-public-releases/latest/stackql_darwin_multiarch.pkg' - else: - raise Exception(f"ERROR: [get_download_url] unsupported OS type: {system_val} {machine_val}") - - -def download_file(url, path, showprogress=True): - """Downloads a file from a URL to a local path. - - Args: - url (str): The URL to download from - path (str): The local path to save the file to - showprogress (bool, optional): Whether to show a progress bar. Defaults to True. - - Raises: - Exception: If the download fails - """ - try: - r = requests.get(url, stream=True) - r.raise_for_status() - total_size_in_bytes = int(r.headers.get('content-length', 0)) - block_size = 1024 - with open(path, 'wb') as f: - chunks = 0 - for data in r.iter_content(block_size): - chunks += 1 - f.write(data) - downloaded_size = chunks * block_size - progress_bar = '#' * int(downloaded_size / total_size_in_bytes * 20) - if showprogress: - print(f'\r[{progress_bar.ljust(20)}] {int(downloaded_size / total_size_in_bytes * 100)}%', end='') - - print("\nDownload complete.") - except Exception as e: - print(f"ERROR: [download_file] {str(e)}") - exit(1) - - -def setup_binary(download_dir, system_platform, showprogress=False): - """Sets up the stackql binary by downloading and extracting it. - - Args: - download_dir (str): The directory to download to - system_platform (str): The operating system platform - showprogress (bool, optional): Whether to show download progress. Defaults to False. - - Raises: - Exception: If the setup fails - """ - try: - print('installing stackql...') - binary_name = get_binary_name(system_platform) - url = get_download_url() - print(f"Downloading latest version of stackql from {url} to {download_dir}") - - # Paths - archive_file_name = os.path.join(download_dir, os.path.basename(url)) - binary_path = os.path.join(download_dir, binary_name) - - # Download and extract - download_file(url, archive_file_name, showprogress) - - # Handle extraction - if system_platform.startswith('Darwin'): - unpacked_file_name = os.path.join(download_dir, 'stackql') - command = f'pkgutil --expand-full {archive_file_name} {unpacked_file_name}' - if os.path.exists(unpacked_file_name): - os.system(f'rm -rf {unpacked_file_name}') - os.system(command) - - else: # Handle Windows and Linux - with zipfile.ZipFile(archive_file_name, 'r') as zip_ref: - zip_ref.extractall(download_dir) - - # Specific check for Windows to ensure `stackql.exe` is extracted - if system_platform.startswith("Windows"): - if not os.path.exists(binary_path) and os.path.exists(os.path.join(download_dir, "stackql")): - os.rename(os.path.join(download_dir, "stackql"), binary_path) - - # Confirm binary presence and set permissions - if os.path.exists(binary_path): - print(f"StackQL executable successfully located at: {binary_path}") - os.chmod(binary_path, 0o755) - else: - print(f"ERROR: Expected binary '{binary_path}' not found after extraction.") - exit(1) - - except Exception as e: - print(f"ERROR: [setup_binary] {str(e)}") - exit(1) - - -def get_binary_version(bin_path): - """Gets the version of the stackql binary. - - Args: - bin_path (str): The path to the binary - - Returns: - tuple: (version, sha) - - version: The version number - - sha: The git commit sha - - Raises: - FileNotFoundError: If the binary is not found - Exception: If the version cannot be determined - """ - try: - iqlPopen = subprocess.Popen([bin_path] + ["--version"], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - # Use communicate to fetch the outputs and wait for the process to finish - output, _ = iqlPopen.communicate() - # Decode the output - decoded_output = output.decode('utf-8') - # Split to get the version tokens - version_tokens = decoded_output.split('\n')[0].split(' ') - version = version_tokens[1] - sha = version_tokens[3].replace('(', '').replace(')', '') - return version, sha - except FileNotFoundError: - print(f"ERROR: [get_binary_version] {bin_path} not found") - exit(1) - except Exception as e: - error_message = e.args[0] - print(f"ERROR: [get_binary_version] {error_message}") - exit(1) - finally: - # Ensure the subprocess is terminated and streams are closed - iqlPopen.terminate() - if hasattr(iqlPopen, 'stdout') and iqlPopen.stdout: - iqlPopen.stdout.close() - - -def format_auth(auth): - """Formats an authentication object for use with stackql. - - Args: - auth: The authentication object, can be a string or a dict - - Returns: - tuple: (auth_obj, auth_str) - - auth_obj: The authentication object as a dict - - auth_str: The authentication object as a JSON string - - Raises: - Exception: If the authentication object is invalid - """ - try: - if auth is not None: - if isinstance(auth, str): - authobj = json.loads(auth) - authstr = auth - elif isinstance(auth, dict): - authobj = auth - authstr = json.dumps(auth) - return authobj, authstr - else: - raise Exception("ERROR: [format_auth] auth key supplied with no value") - except Exception as e: - error_message = e.args[0] - print(f"ERROR: [format_auth] {error_message}") - exit(1) \ No newline at end of file diff --git a/run_tests.ps1 b/run_tests.ps1 new file mode 100644 index 0000000..ebdbd1f --- /dev/null +++ b/run_tests.ps1 @@ -0,0 +1,173 @@ +#Requires -Version 5 +<# +run_tests.ps1 - Provision a venv, ensure stackql.exe, then run the full +PyStackQL test suite (local + server) on Windows and print a summary. + +Steps: + 1. Create/activate a Python venv and install dependencies + 2. Ensure stackql.exe exists in the current directory (download if not) + 3. Start the stackql server + 4. Run the local tests (run_tests.py) + 5. Run the server tests (run_server_tests.py) + 6. Stop the stackql server + 7. Print a PASS/FAIL summary with totals + +This is the PowerShell/Windows counterpart of run_tests.sh. +Run it with: .\run_tests.ps1 +#> + +$ProgressPreference = 'SilentlyContinue' + +# Run from the script's own directory so relative paths resolve correctly. +Set-Location -Path $PSScriptRoot + +# Configuration +$VenvName = '.venv' +$VenvPython = Join-Path $VenvName 'Scripts\python.exe' +$RequirementsFile = 'requirements.txt' +$LocalRunner = 'run_tests.py' +$ServerRunner = 'run_server_tests.py' +$StartServer = '.\start-stackql-server.ps1' +$StopServer = '.\stop-stackql-server.ps1' +$Installer = 'https://get-stackql.io/install' + +function Write-Section($msg) { Write-Host $msg -ForegroundColor Blue } +function Write-Ok($msg) { Write-Host $msg -ForegroundColor Green } +function Write-Warn($msg) { Write-Host $msg -ForegroundColor Yellow } +function Write-Err($msg) { Write-Host $msg -ForegroundColor Red } +function StatusText($code) { if ($code -eq 0) { 'PASS' } else { 'FAIL' } } +function StatusColor($code) { if ($code -eq 0) { 'Green' } else { 'Red' } } + +# Extract the pytest summary line (e.g. "43 passed, 1 skipped") from a log file +function Get-PytestSummary($logPath) { + $match = Get-Content $logPath -ErrorAction SilentlyContinue | + Select-String -Pattern 'passed|failed|error|no tests ran' | + Select-Object -Last 1 + if ($null -eq $match) { return '' } + return (($match.Line) -replace '=', '' -replace 'in [\d.]+s', '').Trim() +} + +# Banner +Write-Section "=======================================" +Write-Section " PyStackQL Test Runner " +Write-Section "=======================================" +Write-Host "" + +# --- 1. Python venv setup ------------------------------------------------- + +if (-not (Get-Command python -ErrorAction SilentlyContinue)) { + Write-Err "Error: Python is not installed or not on PATH." + exit 1 +} + +Write-Warn "Using Python:" +python --version +Write-Host "" + +if (-not (Test-Path $VenvPython)) { + Write-Warn "Creating virtual environment in $VenvName..." + python -m venv $VenvName + if (-not (Test-Path $VenvPython)) { + Write-Err "Error: Failed to create virtual environment." + exit 1 + } + Write-Ok "Virtual environment created successfully." +} else { + Write-Warn "Using existing virtual environment in $VenvName" +} + +Write-Warn "Upgrading pip, setuptools, and wheel..." +& $VenvPython -m pip install --upgrade pip setuptools wheel | Out-Null + +if (Test-Path $RequirementsFile) { + Write-Warn "Installing dependencies from $RequirementsFile..." + & $VenvPython -m pip install -r $RequirementsFile | Out-Null + if ($LASTEXITCODE -ne 0) { Write-Err "Warning: Some dependencies may have failed to install." } + else { Write-Ok "Dependencies installed successfully." } +} else { + Write-Err "Warning: $RequirementsFile not found, skipping dependency install." +} + +if ((Test-Path 'pyproject.toml') -or (Test-Path 'setup.py')) { + Write-Warn "Installing PyStackQL in development mode..." + & $VenvPython -m pip install -e . | Out-Null + if ($LASTEXITCODE -ne 0) { Write-Err "Warning: Failed to install package in development mode." } + else { Write-Ok "Package installed in development mode." } +} +Write-Host "" + +# --- 2. Ensure a stackql.exe in the current directory --------------------- + +if (Test-Path '.\stackql.exe') { + Write-Ok "Found stackql binary: $((Get-Item '.\stackql.exe').FullName)" +} else { + Write-Warn "No stackql.exe in $($PWD.Path), downloading..." + Invoke-RestMethod $Installer | Invoke-Expression + if (-not (Test-Path '.\stackql.exe')) { + Write-Err "Error: stackql.exe was not installed to $($PWD.Path)." + exit 1 + } + Write-Ok "stackql.exe installed: $((Get-Item '.\stackql.exe').FullName)" +} +Write-Host "" + +# --- 3-6. Start server, run both suites, always stop server --------------- + +$localCode = 1; $localSummary = '' +$serverCode = 1; $serverSummary = '' + +Write-Section "Starting stackql server..." +& $StartServer +Write-Host "" + +try { + # Local tests + Write-Section "=======================================" + Write-Section " Running local tests ($LocalRunner)" + Write-Section "=======================================" + $localLog = New-TemporaryFile + & $VenvPython $LocalRunner | Tee-Object -FilePath $localLog.FullName + $localCode = $LASTEXITCODE + $localSummary = Get-PytestSummary $localLog.FullName + Remove-Item $localLog -Force -ErrorAction SilentlyContinue + Write-Host "" + + # Server tests + Write-Section "=======================================" + Write-Section " Running server tests ($ServerRunner)" + Write-Section "=======================================" + $serverLog = New-TemporaryFile + & $VenvPython $ServerRunner | Tee-Object -FilePath $serverLog.FullName + $serverCode = $LASTEXITCODE + $serverSummary = Get-PytestSummary $serverLog.FullName + Remove-Item $serverLog -Force -ErrorAction SilentlyContinue + Write-Host "" +} +finally { + Write-Section "Stopping stackql server..." + & $StopServer + Write-Host "" +} + +# --- 7. Summary ----------------------------------------------------------- + +if (($localCode -eq 0) -and ($serverCode -eq 0)) { $overallCode = 0 } else { $overallCode = 1 } + +$localText = if ($localSummary) { $localSummary } else { 'no results' } +$serverText = if ($serverSummary) { $serverSummary } else { 'no results' } + +Write-Section "=======================================" +Write-Section " Test Summary " +Write-Section "=======================================" +Write-Host (" {0,-14} " -f 'Local tests:') -NoNewline +Write-Host (StatusText $localCode) -ForegroundColor (StatusColor $localCode) -NoNewline +Write-Host (" ({0})" -f $localText) +Write-Host (" {0,-14} " -f 'Server tests:') -NoNewline +Write-Host (StatusText $serverCode) -ForegroundColor (StatusColor $serverCode) -NoNewline +Write-Host (" ({0})" -f $serverText) +Write-Section "---------------------------------------" +Write-Host (" {0,-14} " -f 'Overall:') -NoNewline +Write-Host (StatusText $overallCode) -ForegroundColor (StatusColor $overallCode) +Write-Section "=======================================" + +exit $overallCode diff --git a/run_tests.py b/run_tests.py index 930ba58..9289308 100644 --- a/run_tests.py +++ b/run_tests.py @@ -44,7 +44,8 @@ def main(): "tests/test_query_execution.py", "tests/test_output_formats.py", "tests/test_magic.py", - "tests/test_async.py" + "tests/test_async.py", + "tests/test_download.py" ]) # Run pytest with the arguments diff --git a/run_tests.sh b/run_tests.sh new file mode 100644 index 0000000..301cc1f --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# run_tests.sh - Provision a venv, ensure a stackql binary, then run the full +# PyStackQL test suite (local + server) on Linux/macOS and print a summary. +# +# Steps: +# 1. Create/activate a Python venv and install dependencies (this process only) +# 2. Ensure a stackql binary exists in the current directory (download if not) +# 3. Start the stackql server +# 4. Run the local tests (run_tests.py) +# 5. Run the server tests (run_server_tests.py) +# 6. Stop the stackql server +# 7. Print a PASS/FAIL summary with totals +# +# This is for bash on Linux/macOS. On Windows use the PowerShell workflow. + +# Use simpler output without colors when not running under bash +if [ -n "$BASH_VERSION" ]; then + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + RED='\033[0;31m' + NC='\033[0m' # No Color + cecho() { + printf "%b%s%b\n" "$1" "$2" "$NC" + } +else + cecho() { + echo "$2" + } +fi + +# Configuration +VENV_NAME=".venv" +REQUIREMENTS_FILE="requirements.txt" +LOCAL_RUNNER="run_tests.py" +SERVER_RUNNER="run_server_tests.py" +START_SERVER_SCRIPT="start-stackql-server.sh" +STOP_SERVER_SCRIPT="stop-stackql-server.sh" +STACKQL_INSTALLER="https://get-stackql.io/install" + +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Extract the pytest summary line (e.g. "43 passed, 1 skipped") from a log file +extract_summary() { + grep -aE '(passed|failed|error|no tests ran)' "$1" \ + | tail -1 \ + | sed -E 's/=+//g; s/in [0-9.]+s//; s/^[[:space:]]+//; s/[[:space:]]+$//' +} + +# Banner +cecho "$BLUE" "=======================================" +cecho "$BLUE" " PyStackQL Test Runner " +cecho "$BLUE" "=======================================" +echo "" + +# --- 1. Python venv setup ------------------------------------------------- + +if ! command_exists python3; then + cecho "$RED" "Error: Python 3 is not installed." + echo "Please install Python 3 and try again." + exit 1 +fi + +cecho "$YELLOW" "Using Python:" +python3 --version +echo "" + +if [ ! -d "$VENV_NAME" ]; then + cecho "$YELLOW" "Creating virtual environment in ${VENV_NAME}..." + python3 -m venv "$VENV_NAME" + if [ $? -ne 0 ]; then + cecho "$RED" "Error: Failed to create virtual environment." + exit 1 + fi + cecho "$GREEN" "Virtual environment created successfully." +else + cecho "$YELLOW" "Using existing virtual environment in ${VENV_NAME}" +fi + +# macOS/Linux activation path +ACTIVATE_SCRIPT="$VENV_NAME/bin/activate" +if [ ! -f "$ACTIVATE_SCRIPT" ]; then + cecho "$RED" "Error: Activation script not found at $ACTIVATE_SCRIPT" + echo "The virtual environment may be corrupt. Remove $VENV_NAME and re-run." + exit 1 +fi + +cecho "$YELLOW" "Activating virtual environment..." +. "$ACTIVATE_SCRIPT" +if [ $? -ne 0 ]; then + cecho "$RED" "Error: Failed to activate virtual environment." + exit 1 +fi + +cecho "$YELLOW" "Upgrading pip, setuptools, and wheel..." +pip install --upgrade pip setuptools wheel >/dev/null +if [ $? -ne 0 ]; then + cecho "$RED" "Warning: Failed to upgrade pip, setuptools, or wheel. Continuing anyway." +fi + +if [ -f "$REQUIREMENTS_FILE" ]; then + cecho "$YELLOW" "Installing dependencies from $REQUIREMENTS_FILE..." + pip install -r "$REQUIREMENTS_FILE" >/dev/null + if [ $? -ne 0 ]; then + cecho "$RED" "Warning: Some dependencies may have failed to install." + else + cecho "$GREEN" "Dependencies installed successfully." + fi +else + cecho "$RED" "Warning: $REQUIREMENTS_FILE not found, skipping dependency install." +fi + +if [ -f "pyproject.toml" ] || [ -f "setup.py" ]; then + cecho "$YELLOW" "Installing PyStackQL in development mode..." + pip install -e . >/dev/null + if [ $? -ne 0 ]; then + cecho "$RED" "Warning: Failed to install package in development mode." + else + cecho "$GREEN" "Package installed in development mode." + fi +fi +echo "" + +# --- 2. Ensure a stackql binary in the current directory ------------------ + +if [ -x "./stackql" ]; then + cecho "$GREEN" "Found stackql binary: $(pwd)/stackql" +else + cecho "$YELLOW" "No stackql binary in $(pwd), downloading..." + curl -fsSL "$STACKQL_INSTALLER" | sh + if [ ! -x "./stackql" ]; then + cecho "$RED" "Error: stackql binary was not installed to $(pwd)." + exit 1 + fi + cecho "$GREEN" "stackql binary installed: $(pwd)/stackql" +fi +echo "" + +# --- 3. Start the stackql server ------------------------------------------ + +# Always attempt to stop the server on exit so we don't leave it running. +trap 'bash "$STOP_SERVER_SCRIPT" >/dev/null 2>&1' EXIT + +cecho "$BLUE" "Starting stackql server..." +bash "$START_SERVER_SCRIPT" +echo "" + +# --- 4. Local tests ------------------------------------------------------- + +cecho "$BLUE" "=======================================" +cecho "$BLUE" " Running local tests ($LOCAL_RUNNER)" +cecho "$BLUE" "=======================================" +LOCAL_LOG=$(mktemp) +python "$LOCAL_RUNNER" 2>&1 | tee "$LOCAL_LOG" +LOCAL_CODE=${PIPESTATUS[0]} +LOCAL_SUMMARY=$(extract_summary "$LOCAL_LOG") +rm -f "$LOCAL_LOG" +echo "" + +# --- 5. Server tests ------------------------------------------------------ + +cecho "$BLUE" "=======================================" +cecho "$BLUE" " Running server tests ($SERVER_RUNNER)" +cecho "$BLUE" "=======================================" +SERVER_LOG=$(mktemp) +python "$SERVER_RUNNER" 2>&1 | tee "$SERVER_LOG" +SERVER_CODE=${PIPESTATUS[0]} +SERVER_SUMMARY=$(extract_summary "$SERVER_LOG") +rm -f "$SERVER_LOG" +echo "" + +# --- 6. Stop the stackql server ------------------------------------------- + +cecho "$BLUE" "Stopping stackql server..." +bash "$STOP_SERVER_SCRIPT" +trap - EXIT +echo "" + +# --- 7. Summary ----------------------------------------------------------- + +if [ "$LOCAL_CODE" -eq 0 ]; then + LOCAL_STATUS="${GREEN}PASS${NC}" +else + LOCAL_STATUS="${RED}FAIL${NC}" +fi +if [ "$SERVER_CODE" -eq 0 ]; then + SERVER_STATUS="${GREEN}PASS${NC}" +else + SERVER_STATUS="${RED}FAIL${NC}" +fi + +if [ "$LOCAL_CODE" -eq 0 ] && [ "$SERVER_CODE" -eq 0 ]; then + OVERALL_STATUS="${GREEN}PASS${NC}" + OVERALL_CODE=0 +else + OVERALL_STATUS="${RED}FAIL${NC}" + OVERALL_CODE=1 +fi + +cecho "$BLUE" "=======================================" +cecho "$BLUE" " Test Summary " +cecho "$BLUE" "=======================================" +printf " %-14s %b (%s)\n" "Local tests:" "$LOCAL_STATUS" "${LOCAL_SUMMARY:-no results}" +printf " %-14s %b (%s)\n" "Server tests:" "$SERVER_STATUS" "${SERVER_SUMMARY:-no results}" +cecho "$BLUE" "---------------------------------------" +printf " %-14s %b\n" "Overall:" "$OVERALL_STATUS" +cecho "$BLUE" "=======================================" + +exit $OVERALL_CODE diff --git a/start-stackql-server.ps1 b/start-stackql-server.ps1 new file mode 100644 index 0000000..cc8f4a8 --- /dev/null +++ b/start-stackql-server.ps1 @@ -0,0 +1,25 @@ +# start-stackql-server.ps1 +# Start the stackql server (port 5466) if it is not already running. +# Matches the 'srv' invocation specifically (via the process command line) so a +# repo path containing the word "stackql" cannot cause false detections. + +$Address = "127.0.0.1" +$Port = 5466 + +$running = Get-CimInstance Win32_Process -Filter "Name = 'stackql.exe'" -ErrorAction SilentlyContinue | + Where-Object { $_.CommandLine -match 'srv' } + +if ($running) { + Write-Host "server is already running" +} else { + Write-Host "starting stackql server on ${Address}:${Port}..." + # Bind to loopback only: the tests connect on 127.0.0.1, and binding to + # loopback avoids the Windows Firewall prompt (loopback is not filtered) and + # does not expose the server to the network. + Start-Process -FilePath ".\stackql.exe" ` + -ArgumentList "-v", "--pgsrv.address=$Address", "--pgsrv.port=$Port", "srv" ` + -RedirectStandardOutput "stackql-server.log" ` + -RedirectStandardError "stackql-server.err.log" ` + -WindowStyle Hidden + Start-Sleep -Seconds 5 +} diff --git a/start-stackql-server.sh b/start-stackql-server.sh index 82f8d96..a4f11fe 100644 --- a/start-stackql-server.sh +++ b/start-stackql-server.sh @@ -1,8 +1,16 @@ -# start server if not running -echo "checking if server is running" -if [ -z "$(ps | grep stackql)" ]; then - nohup ./stackql -v --pgsrv.port=5466 srv & - sleep 5 -else +#!/bin/bash +# Start the stackql server (port 5466) if it is not already running. +# Matches the 'srv' invocation specifically so a repo path containing the word +# "stackql" does not produce false "already running" detections. +ADDRESS=127.0.0.1 +PORT=5466 + +if pgrep -f "stackql.*srv" >/dev/null 2>&1; then echo "server is already running" -fi \ No newline at end of file +else + echo "starting stackql server on ${ADDRESS}:${PORT}..." + # Bind to loopback only: the tests connect on 127.0.0.1, binding to loopback + # avoids firewall prompts and does not expose the server to the network. + nohup ./stackql -v --pgsrv.address=${ADDRESS} --pgsrv.port=${PORT} srv > stackql-server.log 2>&1 & + sleep 5 +fi diff --git a/stop-stackql-server.ps1 b/stop-stackql-server.ps1 new file mode 100644 index 0000000..fd1a4b7 --- /dev/null +++ b/stop-stackql-server.ps1 @@ -0,0 +1,18 @@ +# stop-stackql-server.ps1 +# Stop the stackql server started by start-stackql-server.ps1. +# Match the 'srv' invocation specifically (via the process command line) so we +# don't kill unrelated stackql.exe processes or anything that merely has +# "stackql" in its path. + +$procs = Get-CimInstance Win32_Process -Filter "Name = 'stackql.exe'" -ErrorAction SilentlyContinue | + Where-Object { $_.CommandLine -match 'srv' } + +if (-not $procs) { + Write-Host "stackql server is not running." +} else { + foreach ($p in $procs) { + Write-Host "stopping stackql server (PID: $($p.ProcessId))..." + Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue + } + Write-Host "stackql server stopped." +} diff --git a/stop-stackql-server.sh b/stop-stackql-server.sh index 762f6e8..8daa467 100644 --- a/stop-stackql-server.sh +++ b/stop-stackql-server.sh @@ -1,7 +1,9 @@ #!/bin/bash - -# Find the process ID of the StackQL server -PID=$(pgrep -f "stackql") +# Stop the stackql server started by start-stackql-server.sh. +# Match the 'srv' invocation specifically so we don't kill unrelated processes +# that merely have "stackql" in their command line (e.g. a repo path that +# contains "stackql", or the test runner itself). +PID=$(pgrep -f "stackql.*srv") if [ -z "$PID" ]; then echo "stackql server is not running." @@ -9,4 +11,4 @@ else echo "stopping stackql server (PID: $PID)..." kill $PID echo "stackql server stopped." -fi \ No newline at end of file +fi diff --git a/tests/test_async.py b/tests/test_async.py index 9bcfdba..bc0f51d 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -53,7 +53,7 @@ async def test_execute_queries_async_dict_output(self): # Check that we have the expected formula names assert any("stackql" in str(name) for name in formula_names), "Results should include 'stackql'" - assert any("terraform" in str(name) for name in formula_names), "Results should include 'terraform'" + assert any("git" in str(name) for name in formula_names), "Results should include 'git'" print_test_result(f"Async executeQueriesAsync with dict output test\nRESULT COUNT: {len(results)}", isinstance(results, list) and all(isinstance(item, dict) for item in results), @@ -75,7 +75,7 @@ async def test_execute_queries_async_pandas_output(self): # Check that we have the expected formula names assert any("stackql" in str(name) for name in formula_values), "Results should include 'stackql'" - assert any("terraform" in str(name) for name in formula_values), "Results should include 'terraform'" + assert any("git" in str(name) for name in formula_values), "Results should include 'git'" # Check that numeric columns exist numeric_columns = [ diff --git a/tests/test_constants.py b/tests/test_constants.py index 86caf1c..60d00ad 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -10,6 +10,7 @@ import time import platform import subprocess +import tempfile from termcolor import colored import pandas as pd @@ -28,15 +29,16 @@ EXPECTED_PACKAGE_VERSION_PATTERN = r'^(\d+\.\d+\.\d+)$' EXPECTED_PLATFORM_PATTERN = r'^(Windows|Linux|Darwin) (\w+) \(([^)]+)\), Python (\d+\.\d+\.\d+)$' -# Get custom download directory based on platform +# Get custom download directory for tests def get_custom_download_dir(): - """Return a platform-specific custom download directory.""" - custom_download_dirs = { - 'windows': 'C:\\temp', - 'darwin': '/tmp', - 'linux': '/tmp' - } - return custom_download_dirs.get(platform.system().lower(), '/tmp') + """Return an isolated custom download directory for tests. + + Uses a dedicated subdirectory of the system temp dir so it never collides + with a stackql binary that CI may place directly in the temp root (e.g. + /tmp/stackql), which would otherwise be picked up instead of a fresh + download. + """ + return os.path.join(tempfile.gettempdir(), 'pystackql_test_download') # Basic test queries that don't require authentication LITERAL_INT_QUERY = "SELECT 1 as literal_int_value" @@ -59,9 +61,11 @@ def get_custom_download_dir(): REGISTRY_PULL_HOMEBREW_QUERY = "REGISTRY PULL homebrew" # Async test queries +# Note: both formulas must exist in the Homebrew metrics view. 'terraform' was +# removed from Homebrew core, so 'git' is used as the second known-good formula. ASYNC_QUERIES = [ "SELECT * FROM homebrew.formula.vw_usage_metrics WHERE formula_name = 'stackql'", - "SELECT * FROM homebrew.formula.vw_usage_metrics WHERE formula_name = 'terraform'" + "SELECT * FROM homebrew.formula.vw_usage_metrics WHERE formula_name = 'git'" ] # Pattern to match registry pull response diff --git a/tests/test_download.py b/tests/test_download.py new file mode 100644 index 0000000..9b151bd --- /dev/null +++ b/tests/test_download.py @@ -0,0 +1,75 @@ +# tests/test_download.py + +""" +Download utility tests for PyStackQL. + +These tests verify that all stackql binary downloads are routed through the +releases.stackql.io proxy and that requests carry the distinguishable +`pystackql/{version}` User-Agent. See issue #64. +""" + +import os +import sys +from unittest.mock import patch, MagicMock + +import pytest + +# Add the parent directory to the path so we can import from pystackql +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from pystackql.utils import download + + +class TestDownloadUrl: + """Tests for get_download_url across platforms.""" + + @pytest.mark.parametrize("system,machine,expected", [ + ('Linux', 'x86_64', 'https://releases.stackql.io/stackql/latest/stackql_linux_amd64.zip'), + ('Windows', 'AMD64', 'https://releases.stackql.io/stackql/latest/stackql_windows_amd64.zip'), + ('Darwin', 'arm64', 'https://releases.stackql.io/stackql/latest/stackql_darwin_multiarch.pkg'), + ]) + def test_all_platforms_route_via_proxy(self, system, machine, expected): + """Every supported platform must download from releases.stackql.io.""" + with patch('platform.system', return_value=system), \ + patch('platform.machine', return_value=machine): + url = download.get_download_url() + assert url == expected + assert url.startswith('https://releases.stackql.io/'), \ + f"{system} download must route through the proxy, got {url}" + + def test_unsupported_platform_raises(self): + """An unsupported OS should raise rather than return a URL.""" + with patch('platform.system', return_value='Plan9'), \ + patch('platform.machine', return_value='mips'): + with pytest.raises(Exception): + download.get_download_url() + + +class TestUserAgent: + """Tests for the pystackql User-Agent.""" + + def test_user_agent_format(self): + """User-Agent must be the versioned pystackql identifier.""" + with patch('pystackql.utils.download.get_package_version', return_value='9.9.9'): + assert download.get_user_agent() == 'pystackql/9.9.9' + + def test_user_agent_falls_back_when_version_missing(self): + """A missing package version must not produce an empty identifier.""" + with patch('pystackql.utils.download.get_package_version', return_value=None): + assert download.get_user_agent() == 'pystackql/unknown' + + def test_download_file_sends_user_agent(self, tmp_path): + """download_file must send the pystackql User-Agent header.""" + fake_response = MagicMock() + fake_response.headers = {'content-length': '4'} + fake_response.iter_content.return_value = [b'data'] + fake_response.raise_for_status.return_value = None + + dest = os.path.join(str(tmp_path), 'out.bin') + with patch('pystackql.utils.download.requests.get', return_value=fake_response) as mock_get: + download.download_file('https://releases.stackql.io/stackql/latest/x.zip', + dest, showprogress=False) + + assert mock_get.called + headers = mock_get.call_args.kwargs.get('headers', {}) + assert headers.get('User-Agent', '').startswith('pystackql/')