diff --git a/.github/BRANCH_PROTECTION_RULESETS.md b/.github/BRANCH_PROTECTION_RULESETS.md index 9e085cd..a4d4c6d 100644 --- a/.github/BRANCH_PROTECTION_RULESETS.md +++ b/.github/BRANCH_PROTECTION_RULESETS.md @@ -25,6 +25,18 @@ This repository uses GitHub repository rulesets. Public contributors may open pu `.github/ruleset-tags.json` protects `v*` tags from deletion and force-updates. +## Required status check names + +- Use check names exactly as they appear on pull requests. In this repo, required checks are: + - **Analyze (python)** + - **Unit tests (3.11)** + - **Unit tests (3.12)** + - **Compile + help smoke (macos-latest, 3.11)** + - **Compile + help smoke (windows-latest, 3.11)** + - **Windows unit tests** + - **Windows MSI smoke** + - **No build artifacts tracked** + ## Apply or refresh via API ```bash diff --git a/.github/ruleset-main.json b/.github/ruleset-main.json index 93b4a2a..f51dda2 100644 --- a/.github/ruleset-main.json +++ b/.github/ruleset-main.json @@ -52,6 +52,12 @@ { "context": "Compile + help smoke (windows-latest, 3.11)" }, + { + "context": "Windows unit tests" + }, + { + "context": "Windows MSI smoke" + }, { "context": "No build artifacts tracked" } diff --git a/.github/ruleset-release.json b/.github/ruleset-release.json index 5b82bbf..8258fbc 100644 --- a/.github/ruleset-release.json +++ b/.github/ruleset-release.json @@ -52,6 +52,12 @@ { "context": "Compile + help smoke (windows-latest, 3.11)" }, + { + "context": "Windows unit tests" + }, + { + "context": "Windows MSI smoke" + }, { "context": "No build artifacts tracked" } diff --git a/.github/workflows/bootstrap-winget.yml b/.github/workflows/bootstrap-winget.yml index 7e0f729..205f129 100644 --- a/.github/workflows/bootstrap-winget.yml +++ b/.github/workflows/bootstrap-winget.yml @@ -2,6 +2,12 @@ name: Bootstrap WinGet package "on": workflow_dispatch: + inputs: + release_tag: + description: "Published release tag to submit (for example v0.1.7)" + required: true + default: "v0.1.7" + type: string permissions: contents: read @@ -84,17 +90,59 @@ jobs: $ErrorActionPreference = "Stop" Invoke-WebRequest https://aka.ms/wingetcreate/latest -OutFile wingetcreate.exe -UseBasicParsing + - name: Prepare manifest from published release + id: prepare + shell: pwsh + env: + WINGET_TOKEN: ${{ secrets.WINGET_TOKEN }} + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + $ErrorActionPreference = "Stop" + if ($env:RELEASE_TAG -notmatch '^v(\d+\.\d+\.\d+(?:\.\d+)?)$') { + throw "Release tag must use vX.Y.Z or vX.Y.Z.W." + } + $version = $Matches[1] + $headers = @{ + Authorization = "Bearer $env:WINGET_TOKEN" + Accept = "application/vnd.github+json" + "X-GitHub-Api-Version" = "2022-11-28" + } + $release = Invoke-RestMethod ` + -Headers $headers ` + -Uri "https://api.github.com/repos/wildfoundry/dataplicity-cli/releases/tags/$env:RELEASE_TAG" + $assetName = "dataplicity-cli-$version-windows-x64.msi" + $asset = $release.assets | Where-Object { $_.name -eq $assetName } | Select-Object -First 1 + if (-not $asset) { + throw "Release $env:RELEASE_TAG does not contain $assetName." + } + $downloadPath = Join-Path $env:RUNNER_TEMP $assetName + Invoke-WebRequest $asset.browser_download_url -OutFile $downloadPath -UseBasicParsing + $sha256 = (Get-FileHash $downloadPath -Algorithm SHA256).Hash + $releaseDate = ([DateTime]$release.published_at).ToString("yyyy-MM-dd") + $outputRoot = Join-Path $env:RUNNER_TEMP "winget-manifests" + $manifestOutput = python build/winget/prepare_manifest.py ` + --source-dir "build/winget/manifests/w/Wildfoundry/DataplicityCLI/0.1.6" ` + --output-root $outputRoot ` + --version $version ` + --installer-url $asset.browser_download_url ` + --installer-sha256 $sha256 ` + --release-date $releaseDate + $manifestPath = ($manifestOutput | Select-Object -Last 1).Trim() + "manifest_path=$manifestPath" >> $env:GITHUB_OUTPUT + "package_version=$version" >> $env:GITHUB_OUTPUT + - name: Submit initial manifest shell: pwsh env: WINGET_TOKEN: ${{ secrets.WINGET_TOKEN }} + MANIFEST_PATH: ${{ steps.prepare.outputs.manifest_path }} + PACKAGE_VERSION: ${{ steps.prepare.outputs.package_version }} run: | $ErrorActionPreference = "Stop" - $manifestPath = "build/winget/manifests/w/Wildfoundry/DataplicityCLI/0.1.6" - $prTitle = "New package: Wildfoundry.DataplicityCLI version 0.1.6" + $prTitle = "New package: Wildfoundry.DataplicityCLI version $env:PACKAGE_VERSION" ./wingetcreate.exe submit ` --token $env:WINGET_TOKEN ` --prtitle $prTitle ` --no-open ` - $manifestPath + $env:MANIFEST_PATH diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59f93ab..d3e172d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,21 +64,86 @@ jobs: python -m pip install --upgrade pip pip install -e ".[test]" - name: Run unit tests - # remote_access imports termios, which is not available on Windows. - run: pytest -q --maxfail=1 --ignore=tests/test_remote_access_helpers.py --ignore=tests/test_remote_access_single_command.py + run: pytest -q --maxfail=1 windows-msi-smoke: name: Windows MSI smoke runs-on: windows-latest steps: - uses: actions/checkout@v4 - - name: Validate WiX source and MSI build script + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install . pyinstaller + choco install wixtoolset -y + - name: Build executable and MSI + shell: pwsh + run: | + $env:VERSION = python build/get_version.py + pyinstaller --noconfirm --clean --onefile --name dataplicity ` + --workpath pyinstaller-build --distpath pyinstaller-dist ` + dataplicity_cli/__main__.py + ./pyinstaller-dist/dataplicity.exe --version + ./pyinstaller-dist/dataplicity.exe --help + ./build/windows/build_msi.ps1 -Version $env:VERSION + "MSI_PATH=$((Resolve-Path "dist/dataplicity-cli-$env:VERSION-windows-x64.msi").Path)" >> $env:GITHUB_ENV + - name: Install, verify, and uninstall MSI shell: pwsh run: | - if (!(Test-Path "build/windows/DataplicityCLI.wxs")) { throw "missing wxs" } - if (!(Test-Path "build/windows/build_msi.ps1")) { throw "missing build_msi.ps1" } - $null = [xml](Get-Content "build/windows/DataplicityCLI.wxs") - Write-Host "OK: WiX source parses" + $ErrorActionPreference = "Stop" + $installDir = Join-Path $env:ProgramFiles "Dataplicity\Dataplicity CLI" + $installedExe = Join-Path $installDir "dataplicity.exe" + $installed = $false + try { + $install = Start-Process msiexec.exe ` + -ArgumentList @("/i", "`"$env:MSI_PATH`"", "/qn", "/norestart") ` + -Wait -PassThru + if ($install.ExitCode -ne 0) { + throw "MSI install failed with exit code $($install.ExitCode)." + } + $installed = $true + if (!(Test-Path $installedExe)) { + throw "Installed executable not found at $installedExe." + } + & $installedExe --version + if ($LASTEXITCODE -ne 0) { + throw "Installed executable --version failed." + } + & $installedExe --help + if ($LASTEXITCODE -ne 0) { + throw "Installed executable --help failed." + } + $machinePath = [Environment]::GetEnvironmentVariable("Path", "Machine") + $expectedPathEntry = $installDir -replace '[\\/]+$', '' + $machinePathEntries = $machinePath -split ";" | ForEach-Object { + $_.Trim().Trim('"') -replace '[\\/]+$', '' + } + if ($machinePathEntries -notcontains $expectedPathEntry) { + throw "Installer did not add $installDir to the machine PATH. Machine PATH: $machinePath" + } + } finally { + if ($installed) { + $uninstall = Start-Process msiexec.exe ` + -ArgumentList @("/x", "`"$env:MSI_PATH`"", "/qn", "/norestart") ` + -Wait -PassThru + if ($uninstall.ExitCode -ne 0) { + throw "MSI uninstall failed with exit code $($uninstall.ExitCode)." + } + } + } + if (Test-Path $installedExe) { + throw "Installed executable remains after uninstall." + } + $machinePathAfterUninstall = [Environment]::GetEnvironmentVariable("Path", "Machine") + $pathEntriesAfterUninstall = $machinePathAfterUninstall -split ";" | ForEach-Object { + $_.Trim().Trim('"') -replace '[\\/]+$', '' + } + if ($pathEntriesAfterUninstall -contains $expectedPathEntry) { + throw "Installer PATH entry remains after uninstall: $installDir" + } no-artifacts-tracked: name: No build artifacts tracked diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4f300a..306adf5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,6 +87,16 @@ jobs: timestamp-rfc3161: http://timestamp.acs.microsoft.com timestamp-digest: SHA256 + - name: Verify Windows executable signature + if: runner.os == 'Windows' + shell: pwsh + run: | + $signature = Get-AuthenticodeSignature "pyinstaller-dist/dataplicity.exe" + if ($signature.Status -ne "Valid") { + throw "Executable signature is $($signature.Status): $($signature.StatusMessage)" + } + Write-Host "Executable signed by $($signature.SignerCertificate.Subject)" + - name: Build macOS tarball (for Homebrew) if: runner.os == 'macOS' shell: bash @@ -136,6 +146,20 @@ jobs: timestamp-rfc3161: http://timestamp.acs.microsoft.com timestamp-digest: SHA256 + - name: Verify Windows MSI signature + if: runner.os == 'Windows' + shell: pwsh + run: | + $msi = Get-ChildItem "dist/*.msi" | Select-Object -First 1 + if (-not $msi) { + throw "No MSI found to verify." + } + $signature = Get-AuthenticodeSignature $msi.FullName + if ($signature.Status -ne "Valid") { + throw "MSI signature is $($signature.Status): $($signature.StatusMessage)" + } + Write-Host "MSI signed by $($signature.SignerCertificate.Subject)" + - name: Stage Windows release artifacts if: runner.os == 'Windows' shell: pwsh diff --git a/.github/workflows/update-winget.yml b/.github/workflows/update-winget.yml index 51a4622..263bac1 100644 --- a/.github/workflows/update-winget.yml +++ b/.github/workflows/update-winget.yml @@ -40,7 +40,7 @@ jobs: scripts/verify-release-bot-token.sh WINGET_TOKEN "${EXPECTED_BOT}" - name: Publish to WinGet - uses: vedantmgoyal9/winget-releaser@main + uses: vedantmgoyal9/winget-releaser@7bd472be23763def6e16bd06cc8b1cdfab0e2fd5 with: identifier: Wildfoundry.DataplicityCLI release-tag: ${{ github.event.release.tag_name || inputs.release_tag }} diff --git a/README.md b/README.md index b343d6b..025bf22 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,26 @@ dataplicity --help ### Windows (no Python required) -Download the latest `.msi` from [GitHub Releases](https://github.com/wildfoundry/dataplicity-cli/releases) and install it. It installs `dataplicity.exe` and adds it to `PATH`. +Install the signed x64 MSI from WinGet in PowerShell or Windows Terminal: -``` +```powershell +winget install --id Wildfoundry.DataplicityCLI --exact dataplicity --help ``` +WinGet handles future upgrades and uninstall: + +```powershell +winget upgrade --id Wildfoundry.DataplicityCLI --exact +winget uninstall --id Wildfoundry.DataplicityCLI --exact +``` + +The installer is machine-wide and may request administrator approval. If +WinGet is unavailable, download the latest signed `.msi` from +[GitHub Releases](https://github.com/wildfoundry/dataplicity-cli/releases). +Both install paths add `dataplicity.exe` to `PATH`; open a new terminal after +installation. + ### Python (developer install) If you do have Python available and prefer `pipx`: @@ -211,3 +225,4 @@ dataplicity --install-completion zsh - The `Update WinGet package` workflow publishes new `.msi` releases to WinGet using `Wildfoundry.DataplicityCLI`. - Configure a repository secret named `WINGET_TOKEN` (classic PAT with `public_repo`) and ensure your account has a fork of `microsoft/winget-pkgs`. - WinGet automation updates existing manifests; if this package is not yet in WinGet, submit the first manifest for the current release, then subsequent releases are automated. +- Follow [`docs/windows-release.md`](docs/windows-release.md) before tagging a Windows release or submitting its first WinGet manifest. diff --git a/build/windows/DataplicityCLI.wxs b/build/windows/DataplicityCLI.wxs index a72ea50..800f7da 100644 --- a/build/windows/DataplicityCLI.wxs +++ b/build/windows/DataplicityCLI.wxs @@ -3,7 +3,7 @@ @@ -11,15 +11,16 @@ + InstallScope="perMachine" + Platform="x64" /> - + - + @@ -27,7 +28,7 @@ - + Path: + if not re.fullmatch(r"\d+\.\d+\.\d+(?:\.\d+)?", version): + raise ValueError("Version must be numeric, for example 0.1.7.") + if not re.fullmatch(r"[0-9A-Fa-f]{64}", installer_sha256): + raise ValueError("Installer SHA-256 must contain 64 hexadecimal characters.") + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", release_date): + raise ValueError("Release date must use YYYY-MM-DD.") + + source_version = None + for manifest in source_dir.glob("*.yaml"): + match = re.search(r"^PackageVersion:\s*(\S+)\s*$", manifest.read_text(encoding="utf-8"), re.MULTILINE) + if match: + source_version = match.group(1) + break + if source_version is None: + raise ValueError(f"No PackageVersion found under {source_dir}.") + + output_dir = output_root / version + if output_dir.exists(): + shutil.rmtree(output_dir) + shutil.copytree(source_dir, output_dir) + + for manifest in output_dir.glob("*.yaml"): + text = manifest.read_text(encoding="utf-8") + text = text.replace(source_version, version) + text = re.sub(r"(?m)^ InstallerUrl: .+$", f" InstallerUrl: {installer_url}", text) + text = re.sub( + r"(?m)^ InstallerSha256: .+$", + f" InstallerSha256: {installer_sha256.upper()}", + text, + ) + text = re.sub(r"(?m)^ReleaseDate: .+$", f"ReleaseDate: {release_date}", text) + manifest.write_text(text, encoding="utf-8") + + return output_dir + + +def main() -> None: + parser = argparse.ArgumentParser(description="Prepare a versioned WinGet manifest from the bootstrap template.") + parser.add_argument("--source-dir", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--installer-url", required=True) + parser.add_argument("--installer-sha256", required=True) + parser.add_argument("--release-date", required=True) + args = parser.parse_args() + + output_dir = prepare_manifest( + source_dir=args.source_dir, + output_root=args.output_root, + version=args.version, + installer_url=args.installer_url, + installer_sha256=args.installer_sha256, + release_date=args.release_date, + ) + print(output_dir) + + +if __name__ == "__main__": + main() diff --git a/dataplicity_cli/cli.py b/dataplicity_cli/cli.py index 3526a7c..c2898b4 100644 --- a/dataplicity_cli/cli.py +++ b/dataplicity_cli/cli.py @@ -4,6 +4,7 @@ import datetime as dt import html import json +import os import queue import re import shutil @@ -18,7 +19,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any, Dict, List, Optional, Set, Tuple -from urllib.parse import parse_qs, urlencode, urlparse, urlunparse +from urllib.parse import parse_qs, urlparse import typer from rich.console import Console @@ -299,21 +300,6 @@ def _parse_sso_user_artifact(raw: str) -> Optional[Dict[str, Any]]: return _extract_sso_payload_from_query(parse_qs(text, keep_blank_values=True)) -def _with_callback_hint(url: str, callback_url: str) -> str: - parsed = urlparse(url) - query = parse_qs(parsed.query, keep_blank_values=True) - # Prefer standards/common redirect targets when present so the IdP/browser - # flow can return directly to the loopback listener. - for redirect_key in ("redirect_uri", "redirect_url", "return_to", "return", "next"): - if redirect_key in query: - query[redirect_key] = [callback_url] - break - if "cli_callback_url" in query: - return urlunparse(parsed._replace(query=urlencode(query, doseq=True))) - query["cli_callback_url"] = [callback_url] - return urlunparse(parsed._replace(query=urlencode(query, doseq=True))) - - class _SsoCallbackListener: def __init__(self) -> None: self._queue: "queue.Queue[Dict[str, Any]]" = queue.Queue() @@ -545,38 +531,19 @@ def _apply_tokens_or_none(state: AppContext, payload: Any) -> bool: return True -def _try_complete_sso_from_code(state: AppContext, code_payload: Dict[str, Any]) -> bool: - code = code_payload.get("code") - if not code: - return False - body: Dict[str, Any] = {"code": code} - if code_payload.get("state"): - body["state"] = code_payload["state"] - response = state.api.post("/api/auth/sso/complete/", json_data=body) - if not response.ok: - return False - return _apply_tokens_or_none(state, response.data) - - def _attempt_sso_auto_complete( state: AppContext, listener: Optional[_SsoCallbackListener], *, timeout_seconds: int, ) -> bool: + if listener is None: + return False deadline = time.monotonic() + max(timeout_seconds, 1) while time.monotonic() < deadline: - if listener: - payload = listener.wait_for_payload(timeout_seconds=1.0) - if payload: - if _apply_tokens_or_none(state, payload): - return True - if _try_complete_sso_from_code(state, payload): - return True - response = state.api.get("/api/auth/sso/complete/") - if response.ok and _apply_tokens_or_none(state, response.data): + payload = listener.wait_for_payload(timeout_seconds=1.0) + if payload and _apply_tokens_or_none(state, payload): return True - time.sleep(1.0) return False @@ -1730,8 +1697,21 @@ def auth_sso( state = _ctx(ctx) email = _resolve_email_for_auth(state, provided_email=email) timeout_seconds = _coerce_timeout_seconds(timeout) - response = state.api.post("/api/auth/bootstrap/", json_data={"email": email}) + listener: Optional[_SsoCallbackListener] = None + bootstrap_payload = {"email": email} + if open_browser: + listener = _SsoCallbackListener() + if listener.start() and listener.callback_url: + bootstrap_payload["callback_url"] = listener.callback_url + if not state.json_output: + state.console.print(f"Listening for SSO callback on [blue]{listener.callback_url}[/blue]") + else: + listener = None + + response = state.api.post("/api/auth/bootstrap/", json_data=bootstrap_payload) if not response.ok: + if listener: + listener.stop() message = _friendly_response_message("Unable to start SSO.", response.data, response.text) if state.json_output: _print_json({"ok": False, "detail": message}) @@ -1740,6 +1720,8 @@ def auth_sso( raise typer.Exit(code=1) if not isinstance(response.data, dict) or response.data.get("status") != "sso_redirect": + if listener: + listener.stop() message = "SSO is not enabled for this account." if state.json_output: _print_json({"ok": False, "detail": message}) @@ -1749,6 +1731,8 @@ def auth_sso( redirect_url = response.data.get("redirect_url") if not redirect_url: + if listener: + listener.stop() message = "SSO redirect URL missing." if state.json_output: _print_json({"ok": False, "detail": message}) @@ -1756,21 +1740,11 @@ def auth_sso( _show_error(state.console, message) raise typer.Exit(code=1) - listener: Optional[_SsoCallbackListener] = None - browser_url = redirect_url - if open_browser: - listener = _SsoCallbackListener() - if listener.start() and listener.callback_url: - browser_url = _with_callback_hint(redirect_url, listener.callback_url) - if not state.json_output: - state.console.print(f"Listening for SSO callback on [blue]{listener.callback_url}[/blue]") - else: - listener = None - webbrowser.open(browser_url) - - if not state.json_output: - state.console.print("Waiting for browser sign-in to complete...") try: + if open_browser: + webbrowser.open(redirect_url) + if not state.json_output: + state.console.print("Waiting for browser sign-in to complete...") if _attempt_sso_auto_complete(state, listener, timeout_seconds=timeout_seconds): state.config.last_email = email state.config.preferred_login_method = "sso" @@ -1805,7 +1779,7 @@ def auth_sso( if payload is None: _show_error(state.console, "Could not parse SSO response.") raise typer.Exit(code=1) - if not _apply_tokens_or_none(state, payload) and not _try_complete_sso_from_code(state, payload): + if not _apply_tokens_or_none(state, payload): _show_error(state.console, "No access token found in payload.") raise typer.Exit(code=1) state.config.last_email = email @@ -2698,10 +2672,25 @@ def devices_provisioning_key( state.console.print(message) -async def _resolve_m2m_url(state: AppContext, device_hash: str) -> str: - response = state.api.get(f"/api/remote/devices/{device_hash}/host/") +async def _resolve_m2m_url( + state: AppContext, + device_hash: str, + *, + request_timeout: Optional[int] = None, +) -> str: + path = f"/api/remote/devices/{device_hash}/host/" + if request_timeout is None: + response = state.api.get(path) + else: + response = state.api.request("GET", path, timeout=max(1, request_timeout)) if not response.ok or not isinstance(response.data, dict): + if response.status_code == 0 and request_timeout is not None: + raise RuntimeError( + f"Timed out after {request_timeout}s while resolving remote host." + ) detail = _friendly_response_message("Remote Access host lookup failed.", response.data, response.text) + if detail.strip().lower() == "unknown device": + detail = "Device is offline or unavailable for Remote Access." raise RuntimeError(detail) m2m_url = response.data.get("m2m_url") if not m2m_url: @@ -2726,6 +2715,12 @@ def _resolve_local_port(preferred: Optional[int]) -> int: return int(probe.getsockname()[1]) +def _ssh_host_key_options(strict_host_key_checking: bool) -> List[str]: + if strict_host_key_checking: + return [] + return ["-o", "StrictHostKeyChecking=no", "-o", f"UserKnownHostsFile={os.devnull}"] + + @devices_app.command("terminal") def devices_terminal(ctx: typer.Context, device_hash: Optional[str] = typer.Argument(None)) -> None: """Open an interactive terminal session to a device. @@ -2947,7 +2942,11 @@ def devices_ssh( resolved_local_port = _resolve_local_port(local_port) async def runner() -> None: - ws_url = await asyncio.wait_for(_resolve_m2m_url(state, resolved_hash), timeout=float(connect_timeout)) + ws_url = await _resolve_m2m_url( + state, + resolved_hash, + request_timeout=connect_timeout, + ) m2m = M2MClient(ws_url) await asyncio.wait_for(m2m.connect(), timeout=float(connect_timeout)) forward_task: Optional[asyncio.Task] = None @@ -2997,8 +2996,7 @@ async def open_redirect_channel() -> int: ssh_cmd: List[str] = ["ssh", "-p", str(resolved_local_port)] if identity_file: ssh_cmd.extend(["-i", str(identity_file.expanduser())]) - if not strict_host_key_checking: - ssh_cmd.extend(["-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null"]) + ssh_cmd.extend(_ssh_host_key_options(strict_host_key_checking)) ssh_cmd.extend(ssh_arg or []) ssh_cmd.append(target) if remote_command: @@ -3156,12 +3154,11 @@ async def runner() -> str: verbose = state.debug and (not state.json_output) if verbose: state.console.print(f"[blue]Connecting to {resolved_hash}...[/blue]") - try: - ws_url = await asyncio.wait_for(_resolve_m2m_url(state, resolved_hash), timeout=float(connect_timeout)) - except asyncio.TimeoutError as exc: - raise RuntimeError( - f"Timed out after {connect_timeout}s while resolving remote host." - ) from exc + ws_url = await _resolve_m2m_url( + state, + resolved_hash, + request_timeout=connect_timeout, + ) m2m = M2MClient(ws_url) try: await asyncio.wait_for(m2m.connect(), timeout=float(connect_timeout)) diff --git a/dataplicity_cli/remote_access.py b/dataplicity_cli/remote_access.py index 0f99dcf..4d7a889 100644 --- a/dataplicity_cli/remote_access.py +++ b/dataplicity_cli/remote_access.py @@ -4,29 +4,61 @@ import os import secrets import select +import signal import sys -import termios import time -import tty +from collections import deque +from contextlib import suppress from dataclasses import dataclass -from typing import Awaitable, Callable, Optional +from typing import Any, Awaitable, Callable, Optional from .m2m import M2MClient +try: + import msvcrt +except ImportError: # pragma: no cover - Windows only + msvcrt = None + +try: + import termios + import tty +except ImportError: # pragma: no cover - Windows only + termios = None + tty = None + + +_WINDOWS_SIGNAL_INPUT: deque[bytes] = deque() + + +def _capture_windows_sigint(_signum: int, _frame: Any) -> None: + _WINDOWS_SIGNAL_INPUT.append(b"\x03") + class RawTerminal: def __init__(self) -> None: self._fd: Optional[int] = None self._old: Optional[list] = None + self._old_sigint_handler: Any = None def __enter__(self) -> "RawTerminal": + if msvcrt is not None: + _WINDOWS_SIGNAL_INPUT.clear() + self._old_sigint_handler = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, _capture_windows_sigint) + return self + if termios is None or tty is None: + return self self._fd = sys.stdin.fileno() self._old = termios.tcgetattr(self._fd) tty.setraw(self._fd) return self def __exit__(self, exc_type, exc, tb) -> None: - if self._fd is not None and self._old is not None: + if self._old_sigint_handler is not None: + signal.signal(signal.SIGINT, self._old_sigint_handler) + self._old_sigint_handler = None + _WINDOWS_SIGNAL_INPUT.clear() + if termios is not None and self._fd is not None and self._old is not None: termios.tcsetattr(self._fd, termios.TCSADRAIN, self._old) @@ -45,6 +77,36 @@ class PortForwardEvent: PortForwardEventCallback = Callable[[PortForwardEvent], None] PortForwardChannelFactory = Callable[[], Awaitable[int]] +_WINDOWS_KEY_SEQUENCES = { + "G": b"\x1b[H", + "H": b"\x1b[A", + "I": b"\x1b[5~", + "K": b"\x1b[D", + "M": b"\x1b[C", + "O": b"\x1b[F", + "P": b"\x1b[B", + "Q": b"\x1b[6~", + "S": b"\x1b[3~", +} + + +def _read_windows_console_input() -> bytes: + chunks = [] + while _WINDOWS_SIGNAL_INPUT: + chunks.append(_WINDOWS_SIGNAL_INPUT.popleft()) + if msvcrt is None: + return b"".join(chunks) + while msvcrt.kbhit(): + character = msvcrt.getwch() + if character in {"\x00", "\xe0"}: + scan_code = msvcrt.getwch() + sequence = _WINDOWS_KEY_SEQUENCES.get(scan_code) + if sequence: + chunks.append(sequence) + continue + chunks.append(character.encode("utf-8", errors="replace")) + return b"".join(chunks) + def _detect_protocol(sample: bytes) -> Optional[str]: if not sample: @@ -63,6 +125,12 @@ def _detect_protocol(sample: bytes) -> Optional[str]: return None +async def _close_stream_writer(writer: asyncio.StreamWriter) -> None: + writer.close() + with suppress(ConnectionError, OSError): + await writer.wait_closed() + + async def run_terminal_session(m2m: M2MClient, port: int) -> None: queue = m2m.channel_queue(port) stdin_fd = sys.stdin.fileno() @@ -70,6 +138,14 @@ async def run_terminal_session(m2m: M2MClient, port: int) -> None: stop_event = asyncio.Event() async def stdin_loop() -> None: + if msvcrt is not None: + while not stop_event.is_set(): + data = _read_windows_console_input() + if data: + await m2m.send_route(port, data) + else: + await asyncio.sleep(0.02) + return while not stop_event.is_set(): ready, _, _ = await asyncio.to_thread(select.select, [stdin_fd], [], [], 0.1) if not ready: @@ -323,8 +399,7 @@ async def remote_to_local() -> None: await m2m.close_channel(channel_for_client) except Exception as exc: emit("channel_close_failed", detail=f"{connection_label}: {exc}") - writer.close() - await writer.wait_closed() + await _close_stream_writer(writer) emit("connection_closed", detail=connection_label) server = await asyncio.start_server(handle_client, host="127.0.0.1", port=local_port) diff --git a/docs/windows-release.md b/docs/windows-release.md new file mode 100644 index 0000000..57d7157 --- /dev/null +++ b/docs/windows-release.md @@ -0,0 +1,90 @@ +# Windows release runbook + +This runbook covers the x64, per-machine WiX MSI published as +`Wildfoundry.DataplicityCLI`. Windows arm64 and portable/MSIX packages are not +part of the current release scope. + +Do not submit v0.1.6 to WinGet. Its release uploaded the MSI without the +external WiX cabinet, so the package does not contain the executable payload. +The first WinGet version must be v0.1.7 or newer and must pass the MSI install +smoke that verifies the embedded payload. + +## Distribution metadata + +- Product and command: `Dataplicity CLI` / `dataplicity` +- Package identifier: `Wildfoundry.DataplicityCLI` +- Installer: WiX MSI, x64, per-machine +- License: `BSD-3-Clause` +- Copyright holder: `Wildfoundry Ltd` +- MSI and WinGet publisher: `Wildfoundry Ltd` + +Before the first public WinGet submission, Ops must confirm that the Azure +Artifact Signing certificate subject identifies `Wildfoundry Ltd`. Record the +confirmation in the release issue. + +## Signing configuration + +The `Release` workflow uses GitHub OIDC and Azure Artifact Signing. Do not +export a private key or store a PFX file in GitHub. + +The protected GitHub environment `release-signing` must provide: + +- Secret: `AZURE_CLIENT_ID` +- Variables: `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` +- Variables: `AZURE_ARTIFACT_SIGNING_ENDPOINT` +- Variables: `AZURE_ARTIFACT_SIGNING_ACCOUNT`, + `AZURE_ARTIFACT_SIGNING_PROFILE` + +Ops owns the Azure signing account and must document internally: + +- primary and backup owner; +- certificate/profile expiry date and renewal reminder; +- who can approve the `release-signing` environment; +- recovery steps for a failed or unavailable signing profile. + +Repository access only reveals secret and variable names, never their values. +Validate the configuration by running a release build and checking both +signatures rather than copying values into a ticket. + +## Release checklist + +1. Confirm the version in `pyproject.toml` and `dataplicity_cli/__init__.py` + matches the intended `vX.Y.Z` tag. +2. Require green unit tests, Windows unit tests, and Windows MSI smoke. +3. Complete the manual Windows functional smoke below. +4. Create and push the release tag only after the preceding checks pass. +5. Confirm the GitHub release contains the versioned x64 MSI and + `SHA256SUMS-windows-x64.txt`. +6. Download the published MSI to a clean Windows 10 or 11 VM. +7. Verify the MSI and installed executable: + + ```powershell + Get-AuthenticodeSignature .\dataplicity-cli-X.Y.Z-windows-x64.msi + Get-FileHash .\dataplicity-cli-X.Y.Z-windows-x64.msi -Algorithm SHA256 + ``` + + Both signatures must report `Valid`; the hash must match the release + checksum. +8. Record the OS version, artifact hash, signing subject, timestamp, and smoke + results in the release issue. + +## Manual Windows smoke + +Use a non-production test organisation with one online Linux device and one +offline device. + +- Install the MSI interactively and with `/qn`; verify `dataplicity` is on PATH + in a new PowerShell process. +- Run `dataplicity --version`, `dataplicity --help`, and `dataplicity doctor`. +- Test password login or MFA where enabled, browser SSO, token refresh, + `whoami`, and logout. +- Run `dataplicity devices list` and inspect both online and offline devices. +- Run a harmless command with `dataplicity devices run`. +- Open and close `dataplicity devices terminal`. +- Test `dataplicity devices ssh` with a valid key, a missing key, and a bad key. +- Confirm an offline device fails promptly with an actionable error. +- Upgrade from the previous MSI, then uninstall silently; verify the executable + and machine PATH entry are removed. + +Do not publish to WinGet if signing is invalid or a core smoke item fails. +Document any accepted deferral with an owner, reason, and target release. diff --git a/tests/test_devices_cli_helpers.py b/tests/test_devices_cli_helpers.py index 0eb4d13..908d115 100644 --- a/tests/test_devices_cli_helpers.py +++ b/tests/test_devices_cli_helpers.py @@ -1,7 +1,10 @@ from __future__ import annotations +import os import unittest +from unittest.mock import Mock +from dataplicity_cli.api import ApiResponse from dataplicity_cli.cli import ( _connection_quality_points, _device_is_active, @@ -9,11 +12,20 @@ _extract_devices, _render_latency_sparkline, _render_quality_status_bar, + _resolve_m2m_url, + _ssh_host_key_options, _sort_devices_for_display, ) class DeviceCliHelpersTest(unittest.TestCase): + def test_ssh_host_key_options_use_platform_null_device(self) -> None: + self.assertEqual(_ssh_host_key_options(True), []) + self.assertEqual( + _ssh_host_key_options(False), + ["-o", "StrictHostKeyChecking=no", "-o", f"UserKnownHostsFile={os.devnull}"], + ) + def test_extract_devices_includes_limited_devices_bucket(self) -> None: payload = { "devices": [{"hash_id": "active-1"}], @@ -81,5 +93,47 @@ def test_latency_sparkline_handles_missing_data(self) -> None: self.assertIn("latency unavailable", rendered) +class ResolveM2MUrlTest(unittest.IsolatedAsyncioTestCase): + async def test_unknown_router_device_is_reported_as_offline(self) -> None: + state = Mock() + state.api.get.return_value = ApiResponse( + False, + 404, + {"detail": "unknown device"}, + '{"detail":"unknown device"}', + ) + + with self.assertRaisesRegex( + RuntimeError, + "Device is offline or unavailable for Remote Access", + ): + await _resolve_m2m_url(state, "offline-device") + + async def test_host_lookup_uses_requested_timeout(self) -> None: + state = Mock() + state.api.request.return_value = ApiResponse( + False, + 0, + None, + "request timed out", + ) + + with self.assertRaisesRegex( + RuntimeError, + "Timed out after 10s while resolving remote host", + ): + await _resolve_m2m_url( + state, + "offline-device", + request_timeout=10, + ) + + state.api.request.assert_called_once_with( + "GET", + "/api/remote/devices/offline-device/host/", + timeout=10, + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_remote_access_helpers.py b/tests/test_remote_access_helpers.py index bade0d9..53d7598 100644 --- a/tests/test_remote_access_helpers.py +++ b/tests/test_remote_access_helpers.py @@ -7,9 +7,16 @@ import unittest from pathlib import Path from typing import Dict, Optional -from unittest.mock import patch +from unittest.mock import AsyncMock, Mock, patch -from dataplicity_cli.remote_access import _detect_protocol, run_port_forward, run_remote_file, run_single_command +from dataplicity_cli import remote_access +from dataplicity_cli.remote_access import ( + _close_stream_writer, + _detect_protocol, + run_port_forward, + run_remote_file, + run_single_command, +) class _FakeM2M: @@ -37,6 +44,17 @@ def __init__(self) -> None: self.buffer = io.BytesIO() +class _FakeMsvcrt: + def __init__(self, characters: list[str]) -> None: + self.characters = characters + + def kbhit(self) -> bool: + return bool(self.characters) + + def getwch(self) -> str: + return self.characters.pop(0) + + def _unused_local_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: probe.bind(("127.0.0.1", 0)) @@ -44,6 +62,42 @@ def _unused_local_port() -> int: class RemoteAccessHelpersTest(unittest.IsolatedAsyncioTestCase): + async def test_windows_console_input_maps_text_and_navigation_keys(self) -> None: + fake_msvcrt = _FakeMsvcrt(["a", "\xe0", "H", "\xe0", "M", "\r"]) + + with patch.object(remote_access, "msvcrt", fake_msvcrt): + data = remote_access._read_windows_console_input() + + self.assertEqual(data, b"a\x1b[A\x1b[C\r") + + async def test_windows_raw_terminal_forwards_ctrl_c_as_input(self) -> None: + fake_msvcrt = _FakeMsvcrt([]) + previous_handler = object() + + with ( + patch.object(remote_access, "msvcrt", fake_msvcrt), + patch.object(remote_access.signal, "getsignal", return_value=previous_handler), + patch.object(remote_access.signal, "signal") as set_signal, + ): + with remote_access.RawTerminal(): + sigint_handler = set_signal.call_args_list[0].args[1] + sigint_handler(remote_access.signal.SIGINT, None) + self.assertEqual(remote_access._read_windows_console_input(), b"\x03") + + self.assertEqual( + set_signal.call_args_list[-1].args, + (remote_access.signal.SIGINT, previous_handler), + ) + + async def test_raw_terminal_is_noop_without_posix_terminal_modules(self) -> None: + with ( + patch.object(remote_access, "termios", None), + patch.object(remote_access, "tty", None), + patch.object(remote_access.sys.stdin, "fileno", side_effect=AssertionError("fileno should not be called")), + ): + with remote_access.RawTerminal(): + pass + async def test_run_single_command_rejects_empty_command(self) -> None: fake = _FakeM2M() with self.assertRaises(RuntimeError): @@ -89,6 +143,15 @@ async def test_detect_protocol_classifies_known_signatures(self) -> None: self.assertEqual(_detect_protocol(bytes([0x16, 0x03, 0x03, 0x00])), "TLS") self.assertIsNone(_detect_protocol(b"\x01\x02\x03")) + async def test_close_stream_writer_ignores_connection_reset(self) -> None: + writer = Mock() + writer.wait_closed = AsyncMock(side_effect=ConnectionResetError) + + await _close_stream_writer(writer) + + writer.close.assert_called_once_with() + writer.wait_closed.assert_awaited_once_with() + async def test_run_port_forward_allocates_channel_per_local_client(self) -> None: fake = _FakeM2M() events = [] diff --git a/tests/test_sso_auth_flow.py b/tests/test_sso_auth_flow.py index 24f8efb..6f30c7f 100644 --- a/tests/test_sso_auth_flow.py +++ b/tests/test_sso_auth_flow.py @@ -1,19 +1,32 @@ from __future__ import annotations import unittest +from types import SimpleNamespace from urllib.request import urlopen +from unittest.mock import Mock, patch from dataplicity_cli.cli import ( _SsoCallbackListener, + _attempt_sso_auto_complete, _coerce_timeout_seconds, _extract_sso_payload_from_url, _extract_sso_payload_from_query, _extract_sso_tokens, _parse_sso_user_artifact, - _with_callback_hint, + auth_sso, ) +class _FakeSsoListener: + def __init__(self, payload: dict | None) -> None: + self.payload = payload + + def wait_for_payload(self, timeout_seconds: float) -> dict | None: + _ = timeout_seconds + payload, self.payload = self.payload, None + return payload + + class SsoAuthFlowTest(unittest.TestCase): def test_extract_sso_tokens_supports_nested_tokens(self) -> None: access, refresh = _extract_sso_tokens({"tokens": {"access": "a1", "refresh": "r1"}}) @@ -26,16 +39,6 @@ def test_extract_sso_payload_from_query_reads_payload_json(self) -> None: self.assertEqual(payload.get("access"), "a2") self.assertEqual(payload.get("refresh"), "r2") - def test_with_callback_hint_adds_cli_callback(self) -> None: - url = _with_callback_hint("https://example.com/sso?foo=bar", "http://127.0.0.1:1234/callback") - self.assertIn("foo=bar", url) - self.assertIn("cli_callback_url=", url) - - def test_with_callback_hint_rewrites_next_target(self) -> None: - callback = "http://127.0.0.1:1234/callback" - url = _with_callback_hint("https://example.com/sso?next=%2Fafter-login%2F", callback) - self.assertIn("next=http%3A%2F%2F127.0.0.1%3A1234%2Fcallback", url) - self.assertIn("cli_callback_url=", url) def test_extract_sso_payload_from_url_reads_query_and_fragment(self) -> None: payload = _extract_sso_payload_from_url("https://dataplicity.com/cb?code=abc#state=xyz") self.assertEqual(payload, {"code": "abc", "state": "xyz"}) @@ -53,6 +56,61 @@ def test_callback_listener_captures_query_payload(self) -> None: finally: listener.stop() + def test_auto_complete_uses_loopback_payload_without_backend_polling(self) -> None: + state = SimpleNamespace(api=Mock()) + listener = _FakeSsoListener({"access": "abc", "refresh": "def"}) + + with patch("dataplicity_cli.cli._apply_tokens_or_none", return_value=True) as apply_tokens: + completed = _attempt_sso_auto_complete(state, listener, timeout_seconds=1) + + self.assertTrue(completed) + apply_tokens.assert_called_once_with(state, {"access": "abc", "refresh": "def"}) + state.api.get.assert_not_called() + state.api.post.assert_not_called() + + def test_auto_complete_without_listener_returns_immediately(self) -> None: + state = SimpleNamespace(api=Mock()) + + completed = _attempt_sso_auto_complete(state, None, timeout_seconds=180) + + self.assertFalse(completed) + state.api.get.assert_not_called() + state.api.post.assert_not_called() + + def test_sso_bootstrap_registers_loopback_callback_without_rewriting_redirect(self) -> None: + listener = Mock() + listener.start.return_value = True + listener.callback_url = "http://127.0.0.1:1234/callback" + redirect_url = "https://example.com/sso?redirect_uri=https%3A%2F%2Fexample.com%2Fcomplete" + state = SimpleNamespace( + api=Mock(), + config=SimpleNamespace(last_email=None, preferred_login_method=None, save=Mock()), + config_path="config.json", + json_output=True, + ) + state.api.post.return_value = SimpleNamespace( + ok=True, + data={"status": "sso_redirect", "redirect_url": redirect_url}, + text="", + ) + + with ( + patch("dataplicity_cli.cli._ctx", return_value=state), + patch("dataplicity_cli.cli._resolve_email_for_auth", return_value="user@example.com"), + patch("dataplicity_cli.cli._SsoCallbackListener", return_value=listener), + patch("dataplicity_cli.cli._attempt_sso_auto_complete", return_value=True), + patch("dataplicity_cli.cli.webbrowser.open") as open_browser, + patch("dataplicity_cli.cli._print_json"), + ): + auth_sso(Mock(), email="user@example.com", open_browser=True, timeout=1) + + state.api.post.assert_called_once_with( + "/api/auth/bootstrap/", + json_data={"email": "user@example.com", "callback_url": listener.callback_url}, + ) + open_browser.assert_called_once_with(redirect_url) + listener.stop.assert_called_once_with() + def test_coerce_timeout_seconds_handles_invalid_values(self) -> None: self.assertEqual(_coerce_timeout_seconds(30), 30) self.assertEqual(_coerce_timeout_seconds("45"), 45) diff --git a/tests/test_windows_packaging.py b/tests/test_windows_packaging.py new file mode 100644 index 0000000..81af0fb --- /dev/null +++ b/tests/test_windows_packaging.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import unittest +import xml.etree.ElementTree as ET +from pathlib import Path + + +WIX_SOURCE = Path(__file__).parents[1] / "build" / "windows" / "DataplicityCLI.wxs" +WIX_NAMESPACE = {"wix": "http://schemas.microsoft.com/wix/2006/wi"} + + +class WindowsPackagingTest(unittest.TestCase): + def test_msi_uses_legal_publisher(self) -> None: + root = ET.parse(WIX_SOURCE).getroot() + product = root.find(".//wix:Product", WIX_NAMESPACE) + + self.assertIsNotNone(product) + self.assertEqual(product.get("Manufacturer"), "Wildfoundry Ltd") + + def test_msi_targets_64_bit_program_files(self) -> None: + root = ET.parse(WIX_SOURCE).getroot() + package = root.find(".//wix:Package", WIX_NAMESPACE) + install_root = root.find(".//wix:Directory[@Id='ProgramFiles64Folder']", WIX_NAMESPACE) + executable_component = root.find(".//wix:Component[@Id='MainExecutable']", WIX_NAMESPACE) + + self.assertIsNotNone(package) + self.assertEqual(package.get("Platform"), "x64") + self.assertIsNotNone(install_root) + self.assertIsNotNone(executable_component) + self.assertEqual(executable_component.get("Win64"), "yes") + + def test_msi_embeds_its_cabinet(self) -> None: + root = ET.parse(WIX_SOURCE).getroot() + media_template = root.find(".//wix:MediaTemplate", WIX_NAMESPACE) + + self.assertIsNotNone(media_template) + self.assertEqual(media_template.get("EmbedCab"), "yes") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_winget_manifest.py b/tests/test_winget_manifest.py new file mode 100644 index 0000000..278c7be --- /dev/null +++ b/tests/test_winget_manifest.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).parents[1] / "build" / "winget" / "prepare_manifest.py" +SPEC = importlib.util.spec_from_file_location("prepare_manifest", SCRIPT_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class WingetManifestTest(unittest.TestCase): + def test_prepares_versioned_manifest_with_release_artifact(self) -> None: + source_dir = ( + Path(__file__).parents[1] + / "build" + / "winget" + / "manifests" + / "w" + / "Wildfoundry" + / "DataplicityCLI" + / "0.1.6" + ) + installer_url = ( + "https://github.com/wildfoundry/dataplicity-cli/releases/download/" + "v0.1.7/dataplicity-cli-0.1.7-windows-x64.msi" + ) + installer_sha256 = "a" * 64 + + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = MODULE.prepare_manifest( + source_dir=source_dir, + output_root=Path(temp_dir), + version="0.1.7", + installer_url=installer_url, + installer_sha256=installer_sha256, + release_date="2026-07-20", + ) + + manifests = list(output_dir.glob("*.yaml")) + self.assertEqual(len(manifests), 3) + combined = "\n".join(manifest.read_text(encoding="utf-8") for manifest in manifests) + self.assertNotIn("0.1.6", combined) + self.assertIn("PackageVersion: 0.1.7", combined) + self.assertIn(f" InstallerUrl: {installer_url}", combined) + self.assertIn(f" InstallerSha256: {installer_sha256.upper()}", combined) + self.assertIn("ReleaseDate: 2026-07-20", combined) + + def test_rejects_invalid_sha256(self) -> None: + with self.assertRaises(ValueError): + MODULE.prepare_manifest( + source_dir=Path("unused"), + output_root=Path("unused"), + version="0.1.7", + installer_url="https://example.com/installer.msi", + installer_sha256="invalid", + release_date="2026-07-20", + ) + + +if __name__ == "__main__": + unittest.main()