From a643d7dbeb0e376aff27f0bf649ad9eceaaaf6ab Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Mon, 17 Aug 2026 11:37:18 +0200 Subject: [PATCH 1/3] [Presets] Preset status, in-flight get, and preset export `dstack preset get --json` now returns the requested configuration for a preset whose creation is still running (or was interrupted/failed), with a `status` field distinguishing it from verified presets. `dstack preset export` replaces `dstack preset apply`: it writes the preset's service as a plain `type: service` configuration with its patch files, deployable with `dstack apply -f`. The displayed tok/s/user now derives from mean TPOT rather than p50, which under MTP's right-skewed TPOT overstated the delivered rate by ~25%. Co-Authored-By: Claude Fable 5 --- mkdocs/docs/concepts/presets.md | 43 +++-- mkdocs/docs/reference/cli/dstack/preset.md | 25 ++- src/dstack/_internal/cli/commands/preset.py | 161 ++++++++++-------- src/dstack/_internal/cli/models/presets.py | 14 +- .../_internal/cli/services/presets/apply.py | 100 ----------- .../_internal/cli/services/presets/build.py | 13 +- .../_internal/cli/services/presets/create.py | 10 +- .../_internal/cli/services/presets/export.py | 48 ++++++ .../_internal/cli/services/presets/output.py | 59 +++---- .../_internal/cli/services/presets/session.py | 9 +- .../_internal/cli/services/presets/store.py | 26 +-- .../_internal/cli/services/presets/verify.py | 4 +- .../_internal/cli/commands/test_preset.py | 118 +++++++------ src/tests/_internal/cli/common.py | 6 +- .../cli/services/presets/test_apply.py | 140 --------------- .../cli/services/presets/test_create.py | 4 +- .../cli/services/presets/test_export.py | 72 ++++++++ .../cli/services/presets/test_output.py | 44 +++-- .../cli/services/presets/test_store.py | 2 + 19 files changed, 427 insertions(+), 471 deletions(-) delete mode 100644 src/dstack/_internal/cli/services/presets/apply.py create mode 100644 src/dstack/_internal/cli/services/presets/export.py delete mode 100644 src/tests/_internal/cli/services/presets/test_apply.py create mode 100644 src/tests/_internal/cli/services/presets/test_export.py diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index bad71f134..473fe0837 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -18,7 +18,7 @@ To get the best performance for the given model, hardware, and other constraints Creating a preset requires the `claude` CLI to be installed on the machine where you create a preset. -## Create a preset +## Apply a configuration First, define a preset configuration as a YAML file in your project folder. The filename must end with `.dstack.yml` (e.g. `.dstack.yml` or `preset.dstack.yml` are both acceptable). @@ -74,17 +74,17 @@ Create the preset dsv4-flash? [y/n]: y The command executes entirely locally and uses the locally installed `claude` CLI along with `dstack`'s bundled skills. The agent uses a `dstack` task to find the best serving configuration for the available fleet offers, then submits it as a `dstack` service for a final benchmark. -You can stop watching with `Ctrl`+`C` at any time. The agent keeps running, and `dstack preset logs -f` follows it again. Resume an interrupted creation with `dstack preset create --resume`: +You can stop watching with `Ctrl`+`C` at any time. The agent keeps running, and `dstack preset logs -f` follows it again. Resume an interrupted creation with `dstack preset resume`:
```shell -$ dstack preset create -f preset.dstack.yml --resume a1b2c3d4 +$ dstack preset resume a1b2c3d4 ```
-When resuming, the constraints are read from the original session, not from the configuration file. Editing them and resuming has no effect. To change any of them, create a new preset. +When resuming, the configuration and constraints are read from the original session. To change any of them, create a new preset. To stop a creation and its runs, use `dstack preset stop`. @@ -113,7 +113,7 @@ To stop a creation and its runs, use `dstack preset stop`. Set `fleets` to restrict creation and reuse to specific [fleets](fleets.md). It's highly recommended to specify a fleet with exactly the hardware that you'd like the preset to use. -Alternatively, pass `--fleet` to `dstack preset create` or `dstack preset apply`. +Alternatively, pass `--fleet` to `dstack preset create`. > Profile settings such as `spot_policy`, `max_price`, and `backends` are ignored during preset > creation. Configure them on the fleet instead. @@ -201,14 +201,25 @@ When the session builds on `previous`, the baseline trial reproduces the best co !!! info "Reference" The `preset` configuration supports many more options. See the [`.dstack.yml` reference](../reference/dstack.yml/preset.md). -## Apply a preset +## Export a preset -To deploy a preset as a service, pass the preset configuration and the preset ID to the `dstack preset apply` command: +To deploy a preset, export it as a service configuration with `dstack preset export`:
```shell -$ dstack preset apply -f preset.dstack.yml --id c83375b4 +$ dstack preset export c83375b4 -f qwen.dstack.yml +Preset c83375b4 exported to qwen.dstack.yml (16 files). Deploy it with `dstack apply -f qwen.dstack.yml` +``` + +
+ +The command writes the service configuration along with any files it references, such as patches. Set the service `name` and, optionally, a [gateway](gateways.md) in the exported configuration, then submit it with `dstack apply`: + +
+ +```shell +$ dstack apply -f qwen.dstack.yml Project main User admin Type service @@ -218,8 +229,6 @@ $ dstack preset apply -f preset.dstack.yml --id c83375b4 Retry policy off Idle duration 5m Max duration off - Model deepseek-ai/DeepSeek-V4-Flash (base) - Preset c83375b4 (io=10000/1500 conc=1 tok/s/user=309 tok/s=296 ttft=213ms ctx=1M) # BACKEND RESOURCES INSTANCE TYPE PRICE 1 runpod (US-CA-2) cpu=48 mem=502GB disk=500GB gpu=B200:180GB:2 NVIDIA B200 $11.78 @@ -239,8 +248,8 @@ Use `dstack preset` to list presets: ```shell $ dstack preset list - ID BASE GPU CONSTRAINTS BENCHMARK STATUS SUBMITTED - c83375b4 deepseek-ai/DeepSeek-V4-Flash B200:180GB:2 io=10000/1500 conc=1 tok/s/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED + dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tok/s/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago ```
@@ -251,11 +260,11 @@ By default, `dstack preset` shows creations that are still running, or the most ```shell $ dstack preset list -a - ID BASE GPU CONSTRAINTS BENCHMARK STATUS SUBMITTED - c83375b4 deepseek-ai/DeepSeek-V4-Flash B200:180GB:2 io=10000/1500 conc=1 tok/s/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago - 092c792b Qwen/Qwen3.5-397B-A17B RTXPRO6000:4 io=8K/1K conc=64 tok/s/user=19.6 ttft=3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago - 9ab0fa65 Qwen/Qwen3.6-27B RTXPRO4500:1 io=1K/1K conc=8 tok/s/user=57.1 ttft=499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago - f91d6b60 Qwen/Qwen3-32B RTX5090:32GB:1 io=1K/512 conc=8 tok/s/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago + NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED + dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tok/s/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tok/s/user=19.6 ttft=3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago + qwen36-pro4500 9ab0fa65 Qwen/Qwen3.6-27B io=1K/1K c=8 tok/s/user=57.1 ttft=499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago + qwen3-32b-5090 f91d6b60 Qwen/Qwen3-32B io=1K/512 c=8 tok/s/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago ``` diff --git a/mkdocs/docs/reference/cli/dstack/preset.md b/mkdocs/docs/reference/cli/dstack/preset.md index f899383a6..358a1f3c6 100644 --- a/mkdocs/docs/reference/cli/dstack/preset.md +++ b/mkdocs/docs/reference/cli/dstack/preset.md @@ -1,6 +1,6 @@ # dstack preset -The `dstack preset` commands create, list, apply, and delete local +The `dstack preset` commands create, list, export, and delete local [presets](../../../concepts/presets.md). ## dstack preset list @@ -97,17 +97,32 @@ $ dstack preset get --help -## dstack preset apply +## dstack preset resume -The `dstack preset apply` command selects a matching local preset and -submits its service. +The `dstack preset resume` command resumes an interrupted preset creation. ##### Usage
```shell -$ dstack preset apply --help +$ dstack preset resume --help +#GENERATE# +``` + +
+ +## dstack preset export + +The `dstack preset export` command exports a preset as a service +configuration that `dstack apply` deploys. + +##### Usage + +
+ +```shell +$ dstack preset export --help #GENERATE# ``` diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py index 827452193..7399f7344 100644 --- a/src/dstack/_internal/cli/commands/preset.py +++ b/src/dstack/_internal/cli/commands/preset.py @@ -5,6 +5,7 @@ import time from contextlib import redirect_stderr, suppress from pathlib import Path +from typing import Optional from argcomplete import FilesCompleter # type: ignore[attr-defined] from rich.live import Live @@ -17,11 +18,11 @@ ) from dstack._internal.cli.services.completion import ProjectNameCompleter from dstack._internal.cli.services.configurators import APPLY_STDIN_NAME -from dstack._internal.cli.services.presets.apply import apply_preset from dstack._internal.cli.services.presets.create import ( CreationStopped, create_preset, find_preset_name_holders, + load_session_configuration, plan_preset, reassign_preset_name, reconcile_detached_sessions, @@ -29,9 +30,11 @@ show_preset_session_logs, stop_preset_session, ) +from dstack._internal.cli.services.presets.export import export_preset from dstack._internal.cli.services.presets.output import get_presets_table, print_presets from dstack._internal.cli.services.presets.session import ( list_preset_sessions, + load_preset_session, load_resumable_session, resolve_session_ref, ) @@ -126,11 +129,6 @@ def _register(self) -> None: help="Give the agent a previous session's results to analyze and improve on." " Repeat for several", ) - create_parser.add_argument( - "--resume", - metavar="ID", - help="Resume an interrupted preset creation by its preset ID", - ) create_parser.add_argument( "-y", "--yes", action="store_true", help="Do not ask for confirmation" ) @@ -166,33 +164,35 @@ def _register(self) -> None: ) stop_parser.set_defaults(subfunc=self._stop) - apply_parser = preset_subparsers.add_parser( - "apply", - help="Apply a preset", + resume_parser = preset_subparsers.add_parser( + "resume", + help="Resume an interrupted preset creation", formatter_class=self._parser.formatter_class, ) - _add_configuration_args(apply_parser) - register_profile_args(apply_parser) - apply_parser.add_argument( - "--id", - dest="preset_id", - metavar="ID", - required=True, - help="The preset ID to deploy", - ) - apply_parser.add_argument( - "-y", "--yes", action="store_true", help="Do not ask for confirmation" - ) - apply_parser.add_argument( - "--force", action="store_true", help="Force apply when no changes are detected" + resume_parser.add_argument("preset", metavar="ID", help="The preset ID or name") + resume_parser.add_argument( + "--keep-service", + action="store_true", + help="Leave the verified service running", ) - apply_parser.add_argument( - "-d", "--detach", action="store_true", help="Exit after submitting the service" + resume_parser.set_defaults(subfunc=self._resume) + + export_parser = preset_subparsers.add_parser( + "export", + help="Export a preset as a service configuration", + formatter_class=self._parser.formatter_class, ) - apply_parser.add_argument( - "-v", "--verbose", action="store_true", help="Show all plan properties" + export_parser.add_argument("preset", metavar="ID", help="The preset ID or name") + export_parser.add_argument( + "-f", + "--file", + required=True, + metavar="FILE", + dest="destination", + help="The service configuration file to write", ) - apply_parser.set_defaults(subfunc=self._apply) + export_parser.add_argument("--force", action="store_true", help="Overwrite existing files") + export_parser.set_defaults(subfunc=self._export) delete_parser = preset_subparsers.add_parser( "delete", @@ -297,36 +297,21 @@ def _create(self, args: argparse.Namespace) -> None: configuration = _get_effective_configuration(configuration, args, require_name=False) user_prompt = resolve_preset_prompt(configuration, _prompt_base(args.configuration_file)) store = PresetStore() - resume_session = None - if getattr(args, "resume", None): - resume_session = load_resumable_session(args.resume) - if getattr(args, "trials", None) is not None: - console.print( - "[warning]--trials is ignored when resuming: " - "the constraints are fixed at creation[/]" - ) - if configuration.previous: - console.print( - "[warning]previous is ignored when resuming: " - "the previous sessions are fixed at creation[/]" - ) previous = () - if resume_session is None and configuration.previous: + if configuration.previous: previous = resolve_previous_sessions(configuration.previous) api = Client.from_config(project_name=args.project) - allowed_fleets = None - if resume_session is None: - if configuration.trials is None: - raise ConfigurationError( - "trials is required. Set it in the configuration or pass --trials" - ) - for field in ("max_ttft", "min_context_length", "concurrency"): - if getattr(configuration, field) is None: - raise ConfigurationError(f"{field} is required") - allowed_fleets = plan_preset(api=api, configuration=configuration) - if not _confirm_preset_creation(store, configuration.name, assume_yes=args.yes): - console.print("\nExiting...") - return + if configuration.trials is None: + raise ConfigurationError( + "trials is required. Set it in the configuration or pass --trials" + ) + for field in ("max_ttft", "min_context_length", "concurrency"): + if getattr(configuration, field) is None: + raise ConfigurationError(f"{field} is required") + allowed_fleets = plan_preset(api=api, configuration=configuration) + if not _confirm_preset_creation(store, configuration.name, assume_yes=args.yes): + console.print("\nExiting...") + return try: result = create_preset( api=api, @@ -334,7 +319,6 @@ def _create(self, args: argparse.Namespace) -> None: store=store, keep_service=args.keep_service, debug=args.debug, - resume_session=resume_session, user_prompt=user_prompt, allowed_fleets=allowed_fleets, previous=previous, @@ -347,6 +331,24 @@ def _create(self, args: argparse.Namespace) -> None: if args.keep_service: console.print(f"Final service [code]{result.final_run_name}[/] kept running") + def _resume(self, args: argparse.Namespace) -> None: + session = load_resumable_session(resolve_session_ref(args.preset)) + try: + result = create_preset( + api=Client.from_config(project_name=args.project), + configuration=load_session_configuration(session), + store=PresetStore(), + keep_service=args.keep_service, + resume_session=session, + user_prompt=session.read_prompt(), + ) + except KeyboardInterrupt: + return # the interrupt handler already reported detach / stop + except CreationStopped: + return # stopped from another CLI, which reported the interruption + if args.keep_service: + console.print(f"Final service [code]{result.final_run_name}[/] kept running") + def _logs(self, args: argparse.Namespace) -> None: try: result = show_preset_session_logs( @@ -373,22 +375,27 @@ def _stop(self, args: argparse.Namespace) -> None: def _get(self, args: argparse.Namespace) -> None: self._reconcile() preset = PresetStore().find_by_id_or_name(args.preset) + if preset is None: + preset = _get_unfinished_preset(args.preset) if preset is None: raise CLIError(f"Preset {args.preset!r} does not exist") print(preset.model_dump_json()) - def _apply(self, args: argparse.Namespace) -> None: - self._reconcile() - configuration_path, configuration = _read_configuration_arg(args.configuration_file) - configuration = _get_effective_configuration(configuration, args) - apply_preset( - api=Client.from_config(project_name=args.project), - configuration=configuration, - configuration_path=configuration_path, - preset_id=args.preset_id, - profile_name=args.profile, - command_args=args, - store=PresetStore(), + def _export(self, args: argparse.Namespace) -> None: + store = PresetStore() + preset = store.find_by_id_or_name(args.preset) + if preset is None: + raise CLIError(f"Preset {args.preset!r} does not exist") + written = export_preset( + preset, + preset_dir=store.root / preset.id, + destination=Path(args.destination), + force=args.force, + ) + console.print( + f"Preset [code]{preset.id}[/] exported to [code]{args.destination}[/]" + f" ({len(written)} files). Deploy it with" + f" [code]dstack apply -f {args.destination}[/]" ) def _delete(self, args: argparse.Namespace) -> None: @@ -423,6 +430,24 @@ def _delete(self, args: argparse.Namespace) -> None: console.print(f"Deleted {description}") +def _get_unfinished_preset(ref: str) -> Optional[Preset]: + """The preset as its creation session knows it, for any state but verified.""" + try: + session = load_preset_session(resolve_session_ref(ref)) + except CLIError: + return None + state = session.read_state() + if state is None or state.status == "success": + return None + return Preset( + status=state.status, + id=state.id, + name=state.name, + configuration=load_session_configuration(session), + submitted_at=state.created_at, + ) + + def _add_configuration_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( "-f", diff --git a/src/dstack/_internal/cli/models/presets.py b/src/dstack/_internal/cli/models/presets.py index f7e1f2565..62e72e484 100644 --- a/src/dstack/_internal/cli/models/presets.py +++ b/src/dstack/_internal/cli/models/presets.py @@ -75,7 +75,7 @@ def effective_output_tok_per_s(self) -> float: @property def effective_per_user_tok_per_s(self) -> float: - return 1000 / self.metrics.tpot_ms.p50 + return 1000 / self.metrics.tpot_ms.mean @field_validator("command") @classmethod @@ -108,19 +108,21 @@ class PresetVerificationReplicaGroup(CoreModel): class Preset(CoreModel): - """What was asked (`configuration`), what to deploy (`service`), and - the evidence it works (`verification_data`).""" - + status: Literal["running", "interrupted", "failed", "verified"] id: str name: Optional[str] = None configuration: PresetConfiguration + submitted_at: datetime + + +class VerifiedPreset(Preset): + status: Literal["verified"] = "verified" base: Annotated[str, Field(min_length=1)] model: Annotated[str, Field(min_length=1)] # The largest context the service was verified to serve. context_length: PositiveInt # The session's `trials/` that won verification and became this preset. best_trial: PositiveInt - submitted_at: datetime # The service that passed verification, stripped of this machine's deployment # choices; `apply` submits it with the user's own name, gateway, and profile. service: ServiceConfiguration @@ -152,7 +154,7 @@ def validate_preset(self) -> Self: class PresetListOutput(CoreModel): - presets: list[Preset] + presets: list[VerifiedPreset] def _validate_exact_resources(resources: ResourcesSpec) -> None: diff --git a/src/dstack/_internal/cli/services/presets/apply.py b/src/dstack/_internal/cli/services/presets/apply.py deleted file mode 100644 index 738cb9183..000000000 --- a/src/dstack/_internal/cli/services/presets/apply.py +++ /dev/null @@ -1,100 +0,0 @@ -import argparse -from typing import Optional - -from rich.markup import escape - -from dstack._internal.cli.models.configurations import PresetConfiguration -from dstack._internal.cli.models.presets import Preset -from dstack._internal.cli.services.configurators.run import ServiceConfigurator -from dstack._internal.cli.services.presets.output import ( - format_preset_benchmark, - format_preset_objective, -) -from dstack._internal.cli.services.presets.store import PresetStore -from dstack._internal.cli.utils.common import warn -from dstack._internal.core.errors import CLIError -from dstack._internal.core.models.configurations import ServiceConfiguration -from dstack._internal.core.models.profiles import ProfileParams -from dstack.api import Client - - -def apply_preset( - *, - api: Client, - configuration: PresetConfiguration, - configuration_path: str, - preset_id: str, - profile_name: Optional[str], - command_args: argparse.Namespace, - store: PresetStore, -) -> None: - preset = store.get(preset_id) - if preset is None: - raise CLIError(f"Preset {preset_id} does not exist") - _validate_preset_matches(preset, configuration=configuration) - - configurator = ServiceConfigurator(api_client=api) - service_args = configurator.get_parser().parse_args([]) - service_args.profile = profile_name - service = _build_service(configuration, preset) - run_plan, repo = configurator.get_plan( - conf=service, - configuration_path=configuration_path, - configurator_args=service_args, - ) - configurator.apply_plan( - run_plan=run_plan, - repo=repo, - command_args=command_args, - configurator_args=service_args, - plan_properties={ - "Model": _format_requested_model(configuration), - "Preset": _format_selected_preset(preset), - }, - ) - - -def _validate_preset_matches(preset: Preset, *, configuration: PresetConfiguration) -> None: - model_name = configuration.model.api_model_name - service_model = preset.service.model - if service_model is None or service_model.name.lower() != model_name.lower(): - raise CLIError(f"Preset {preset.id} does not serve {model_name}") - if configuration.min_context_length is not None: - if preset.context_length < configuration.min_context_length: - warn( - f"Preset {preset.id} is verified for context length" - f" {preset.context_length}, below the requested" - f" {configuration.min_context_length}" - ) - if configuration.model.allows_variant_selection: - if preset.base.lower() != model_name.lower(): - raise CLIError(f"Preset {preset.id} does not serve base model {model_name}") - elif preset.model != configuration.model.exact_repo: - raise CLIError(f"Preset {preset.id} does not serve repo {configuration.model.exact_repo}") - - -def _build_service( - configuration: PresetConfiguration, - preset: Preset, -) -> ServiceConfiguration: - service = preset.service.model_copy(deep=True) - service.name = configuration.name - service.gateway = configuration.gateway - service.env.update(configuration.env) - for field in ProfileParams.model_fields: - value = getattr(configuration, field) - if value is not None: - setattr(service, field, value) - return service - - -def _format_requested_model(configuration: PresetConfiguration) -> str: - model = escape(configuration.model.api_model_name) - if configuration.model.allows_variant_selection: - return f"{model} ([secondary]base[/])" - return model - - -def _format_selected_preset(preset: Preset) -> str: - details = f"{format_preset_objective(preset)} {format_preset_benchmark(preset, verbose=True)}" - return f"{escape(preset.id)} ({details})" diff --git a/src/dstack/_internal/cli/services/presets/build.py b/src/dstack/_internal/cli/services/presets/build.py index d4f1e855c..6626ac099 100644 --- a/src/dstack/_internal/cli/services/presets/build.py +++ b/src/dstack/_internal/cli/services/presets/build.py @@ -6,9 +6,9 @@ from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.presets import ( PRESET_EXCLUDED_FIELDS, - Preset, PresetBenchmark, PresetVerificationReplicaGroup, + VerifiedPreset, ) from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.envs import Env, EnvSentinel @@ -39,12 +39,12 @@ def build_preset( preset_id: str, name: Optional[str], submitted_at: datetime, -) -> Preset: +) -> VerifiedPreset: service = _without_excluded_fields(service) configuration = _without_excluded_fields(configuration) configuration.env = Env() set_service_gpu_vendor_from_verification(service, verification_replica_groups) - return Preset( + return VerifiedPreset( name=name, base=base_model, id=preset_id, @@ -59,10 +59,11 @@ def build_preset( ) -def preset_to_yaml_dict(preset: Preset) -> dict[str, Any]: - """`Preset` in the plain types `yaml.safe_dump` accepts.""" +def preset_to_yaml_dict(preset: VerifiedPreset) -> dict[str, Any]: + """`VerifiedPreset` in the plain types `yaml.safe_dump` accepts.""" return { - **preset.model_dump(mode="json", exclude_none=True), + # A saved preset is verified by definition; `status` is wire-only. + **preset.model_dump(mode="json", exclude_none=True, exclude={"status"}), "service": service_configuration_to_yaml_dict(preset.service), } diff --git a/src/dstack/_internal/cli/services/presets/create.py b/src/dstack/_internal/cli/services/presets/create.py index 34e455ad7..189a77de7 100644 --- a/src/dstack/_internal/cli/services/presets/create.py +++ b/src/dstack/_internal/cli/services/presets/create.py @@ -131,7 +131,7 @@ def follow_preset( if lock is None: raise SessionBusyError(f"Preset {preset_id} is being finalized by another process") try: - configuration = _load_session_configuration(session) + configuration = load_session_configuration(session) try: result = asyncio.run( _create_preset( @@ -167,12 +167,12 @@ def follow_preset( release_session_claim(lock) -def _load_session_configuration(session: PresetSession) -> PresetConfiguration: +def load_session_configuration(session: PresetSession) -> PresetConfiguration: configuration_path = session.path / "preset.dstack.yml" if not configuration_path.is_file(): raise CLIError( f"Preset {session.preset_id} has no saved configuration and cannot be" - f" followed; resume it with --resume {session.preset_id} instead" + f" followed; run `dstack preset resume {session.preset_id}` instead" ) try: return PresetConfiguration.model_validate( @@ -789,9 +789,7 @@ def _suspend_agent_session(session: PresetSession) -> None: # The kept workspace must not retain a live credential while suspended. scrub_workspace_token(session) console.print(f"\nPreset [code]{session.preset_id}[/] creation interrupted.") - console.print( - f"Resume it with [code]dstack preset create -f --resume {session.preset_id}[/]." - ) + console.print(f"Resume it with [code]dstack preset resume {session.preset_id}[/].") def _get_build_name(name: Optional[str], model_name: str, suffix: str) -> str: diff --git a/src/dstack/_internal/cli/services/presets/export.py b/src/dstack/_internal/cli/services/presets/export.py new file mode 100644 index 000000000..213d01fa3 --- /dev/null +++ b/src/dstack/_internal/cli/services/presets/export.py @@ -0,0 +1,48 @@ +import shutil +from pathlib import Path + +import yaml + +from dstack._internal.cli.models.presets import VerifiedPreset +from dstack._internal.cli.services.presets.build import service_configuration_to_yaml_dict +from dstack._internal.core.errors import CLIError + + +def export_preset( + preset: VerifiedPreset, + *, + preset_dir: Path, + destination: Path, + force: bool, +) -> list[Path]: + """Writes the preset's service as a `type: service` configuration at + `destination` and copies the files it references next to it, keeping their + relative paths, so the result deploys with plain `dstack apply -f`. + Returns every path written.""" + service = preset.service.model_copy(deep=True) + copies: list[tuple[Path, Path]] = [] + for mapping in service.files: + source = Path(mapping.local_path) + # Loading resolved these against the preset directory; a file stored + # outside it keeps its absolute path and needs no copy. + if not source.is_relative_to(preset_dir): + continue + relative = source.relative_to(preset_dir) + copies.append((source, destination.parent / relative)) + mapping.local_path = relative.as_posix() + written = [destination] + [target for _, target in copies] + if not force: + for target in written: + if target.exists(): + raise CLIError(f"{target} already exists. Use --force to overwrite") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + yaml.safe_dump( + {"type": "service", **service_configuration_to_yaml_dict(service)}, + sort_keys=False, + ) + ) + for source, target in copies: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + return written diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 4f8a1023f..06fb88bc2 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -6,7 +6,7 @@ from dstack._internal.cli.models.configurations import DEFAULT_DATASET from dstack._internal.cli.models.presets import ( - Preset, + VerifiedPreset, ) from dstack._internal.cli.utils.common import add_row_from_dict, console from dstack._internal.utils.common import pretty_date, pretty_resources @@ -83,7 +83,7 @@ def _format_trial_progress(session: Optional[dict[str, Any]], *, in_flight: bool def print_presets( - presets: list[Preset], + presets: list[VerifiedPreset], sessions: Optional[list[dict[str, Any]]] = None, verbose: bool = False, all_presets: bool = False, @@ -98,32 +98,24 @@ def print_presets( def get_presets_table( - presets: list[Preset], + presets: list[VerifiedPreset], sessions: Optional[list[dict[str, Any]]] = None, verbose: bool = False, all_presets: bool = False, limit: Optional[int] = None, ) -> Table: table = Table(box=None) - compact = not verbose - table.add_column("ID", no_wrap=True) - # Compact-view caps keep a long model name from starving the wrapping - # CONSTRAINTS and BENCHMARK columns below. - table.add_column("BASE", no_wrap=True, max_width=24 if compact else None, style="secondary") - table.add_column( - "RESOURCES" if verbose else "GPU", - no_wrap=compact, - max_width=18 if compact else None, - style="secondary", - ) + table.add_column("NAME", no_wrap=True) + table.add_column("ID", no_wrap=True, style="secondary") + table.add_column("BASE", no_wrap=True, style="secondary") + if verbose: + table.add_column("RESOURCES", style="secondary") table.add_column("CONSTRAINTS", min_width=len("io=1K/1K"), overflow="fold") table.add_column("BENCHMARK", min_width=len("tps=1"), overflow="fold") table.add_column("", no_wrap=True) table.add_column("STATUS") table.add_column("SUBMITTED", style="secondary") - if verbose: - table.add_column("NAME", no_wrap=True, style="secondary") - presets_by_base: dict[str, list[Preset]] = defaultdict(list) + presets_by_base: dict[str, list[VerifiedPreset]] = defaultdict(list) repo_to_base: dict[str, str] = {} for preset in presets: presets_by_base[preset.base].append(preset) @@ -200,15 +192,15 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False f"io={_format_token_count(constraints['input_tokens'])}" f"/{_format_token_count(constraints['output_tokens'])}" ) - # Shown even at `0%`: rows are only comparable when it matches. input_tokens = constraints.get("input_tokens") if input_tokens: shared_prefix_tokens = constraints.get("shared_prefix_tokens") or 0 share = round(100 * shared_prefix_tokens / input_tokens) - objective.append(f"prefix={share}%") + if share: + objective.append(f"prefix={share}%") concurrency = (best or {}).get("concurrency") or constraints.get("concurrency") if concurrency: - objective.append(f"conc={concurrency}") + objective.append(f"c={concurrency}") min_context_length = constraints.get("min_context_length") if verbose and isinstance(min_context_length, int): objective.append(f"ctx>={_format_token_count(min_context_length)}") @@ -232,7 +224,7 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False parts.append(f"ctx={_format_token_count(context_length)}") benchmark = " ".join(parts) else: - benchmark = f"conc={best.get('concurrency')} tps={tps}" + benchmark = f"c={best.get('concurrency')} tps={tps}" benchmark = benchmark.strip() if benchmark and breached: # Marked, not only dimmed: colour alone is not a signal. @@ -256,17 +248,16 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False def _add_preset( table: Table, - preset: Preset, + preset: VerifiedPreset, *, verbose: bool, creation: Optional[dict[str, Any]] = None, ) -> None: groups = preset.service.replica_groups - column = "RESOURCES" if verbose else "GPU" row = { "ID": preset.id, "NAME": preset.name or "", - column: _format_resources(groups[0].resources, verbose=verbose), + "RESOURCES": _format_resources(groups[0].resources, verbose=verbose), "BASE": preset.base, "STATUS": _format_status("ready") + _format_trial_progress(creation), "": _format_trial_spark(creation), @@ -286,14 +277,14 @@ def _add_preset( table, { "BASE": f" group={group.name}", - column: _format_resources(group.resources, verbose=verbose), + "RESOURCES": _format_resources(group.resources, verbose=verbose), }, style="secondary", ) def format_preset_objective( - preset: Preset, + preset: VerifiedPreset, *, verbose: bool = False, ) -> str: @@ -309,8 +300,9 @@ def format_preset_objective( f"/{_format_token_count(configuration.effective_output_tokens)}" ) share = round(100 * (configuration.shared_prefix_tokens or 0) / input_tokens) - parts.append(f"prefix={share}%") - parts.append(f"conc={configuration.concurrency or workload.concurrency}") + if share: + parts.append(f"prefix={share}%") + parts.append(f"c={configuration.concurrency or workload.concurrency}") if verbose and configuration.min_context_length is not None: parts.append(f"ctx>={_format_token_count(configuration.min_context_length)}") if verbose and configuration.max_ttft is not None: @@ -318,7 +310,7 @@ def format_preset_objective( return f"[secondary]{' '.join(parts)}[/]" -def _breaches_constraints(preset: Preset) -> bool: +def _breaches_constraints(preset: VerifiedPreset) -> bool: configuration = preset.configuration metrics = preset.benchmark.metrics if configuration.max_ttft is not None and metrics.ttft_ms.p50 > configuration.max_ttft: @@ -328,7 +320,7 @@ def _breaches_constraints(preset: Preset) -> bool: ) -def format_preset_benchmark(preset: Preset, *, verbose: bool = False) -> str: +def format_preset_benchmark(preset: VerifiedPreset, *, verbose: bool = False) -> str: benchmark = preset.benchmark metrics = benchmark.metrics parts = [ @@ -354,10 +346,15 @@ def _format_duration_ms(value: float) -> str: def _format_token_count(value: int) -> str: - """Abbreviates only exact multiples of 1024/1024², so 2048 becomes "2K" but 2050 stays "2050".""" + # The conventional name: exact binary multiples keep binary names + # (32768 is "32K"), anything else rounds as decimal (1500 is "1.5K"). for divisor, suffix in ((1024 * 1024, "M"), (1024, "K")): if value >= divisor and value % divisor == 0: return f"{value // divisor}{suffix}" + if value >= 999_950: + return f"{value / 1_000_000:.1f}".removesuffix(".0") + "M" + if value >= 1000: + return f"{value / 1000:.1f}".removesuffix(".0") + "K" return str(value) diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index 21fb0d482..a33d3efbf 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -93,6 +93,12 @@ def service_dir(self) -> Path: def write_prompt(self, prompt: str) -> None: _write_private_text(self.path / "prompt.md", prompt + "\n") + def read_prompt(self) -> Optional[str]: + path = self.path / "prompt.md" + if not path.is_file(): + return None + return path.read_text(encoding="utf-8").strip() or None + def write_user_prompt(self, user_prompt: str) -> None: _write_private_text(self.path / _USER_PROMPT_FILENAME, user_prompt + "\n") @@ -628,7 +634,8 @@ def _trial_entry( gpu: Optional[str], ) -> dict[str, Any]: ttft = (metrics.get("ttft_ms") or {}).get("p50") - tpot = (metrics.get("tpot_ms") or {}).get("p50") + # Mean, not p50: MTP skews TPOT right, and mean is the delivered decode rate. + tpot = (metrics.get("tpot_ms") or {}).get("mean") context_length = record.get("context_length") return { "tok_s": tok_s, diff --git a/src/dstack/_internal/cli/services/presets/store.py b/src/dstack/_internal/cli/services/presets/store.py index 9bb5a79d2..a9df9756c 100644 --- a/src/dstack/_internal/cli/services/presets/store.py +++ b/src/dstack/_internal/cli/services/presets/store.py @@ -12,7 +12,7 @@ PresetConfiguration, PresetPromptFile, ) -from dstack._internal.cli.models.presets import PRESET_EXCLUDED_FIELDS, Preset +from dstack._internal.cli.models.presets import PRESET_EXCLUDED_FIELDS, VerifiedPreset from dstack._internal.cli.services.presets.build import preset_to_yaml_dict from dstack._internal.cli.utils.common import warn from dstack._internal.core.errors import CLIError, ConfigurationError @@ -31,7 +31,7 @@ class PresetStore: def __init__(self, root: Path | None = None) -> None: self.root = root or get_dstack_dir() / "presets" - def list(self) -> list[Preset]: + def list(self) -> list[VerifiedPreset]: if not self.root.exists(): return [] presets = [] @@ -47,7 +47,7 @@ def list(self) -> list[Preset]: warn(str(e), stderr=True) if len(earlier_version_ids) == 1: warn( - f"Preset {earlier_version_ids[0]} was created before dstack 0.21 and cannot" + f"VerifiedPreset {earlier_version_ids[0]} was created before dstack 0.21 and cannot" f" be read. Delete it with" f" [code]dstack preset delete {earlier_version_ids[0]}[/], or recreate.", stderr=True, @@ -61,7 +61,7 @@ def list(self) -> list[Preset]: ) return sorted(presets, key=lambda preset: (preset.base.lower(), preset.id)) - def get(self, preset_id: str) -> Preset | None: + def get(self, preset_id: str) -> VerifiedPreset | None: _validate_preset_id(preset_id) if not self.root.exists(): return None @@ -70,10 +70,10 @@ def get(self, preset_id: str) -> Preset | None: return None preset = self._load(path) if preset.id != preset_id: - raise CLIError(f"Preset file {path} does not match its path") + raise CLIError(f"VerifiedPreset file {path} does not match its path") return preset - def save(self, preset: Preset) -> Path: + def save(self, preset: VerifiedPreset) -> Path: _validate_preset_id(preset.id) directory = self.root / preset.id directory.mkdir(parents=True, exist_ok=True) @@ -103,16 +103,16 @@ def save(self, preset: Preset) -> Path: pass return path - def find_by_name(self, name: str) -> Preset | None: + def find_by_name(self, name: str) -> VerifiedPreset | None: for preset in self.list(): if preset.name == name: return preset return None - def find_by_id_or_name(self, ref: str) -> Preset | None: + def find_by_id_or_name(self, ref: str) -> VerifiedPreset | None: return self.get(ref) or self.find_by_name(ref) - def release_name(self, name: str) -> Preset | None: + def release_name(self, name: str) -> VerifiedPreset | None: preset = self.find_by_name(name) if preset is None: return None @@ -131,7 +131,7 @@ def delete(self, preset_id: str) -> bool: shutil.rmtree(directory) return True - def _load(self, path: Path) -> Preset: + def _load(self, path: Path) -> VerifiedPreset: data = None try: with path.open(encoding="utf-8") as f: @@ -141,7 +141,7 @@ def _load(self, path: Path) -> Preset: # stores `verified_on`. if isinstance(data, dict) and "validations" in data: upgraded = _upgrade_pre_0_21_2_preset(data, preset_id=path.parent.name) - preset = Preset.model_validate(upgraded) + preset = VerifiedPreset.model_validate(upgraded) except (OSError, ValidationError, yaml.YAMLError) as e: if isinstance(data, dict) and "validations" in data: raise _earlier_version_preset_error(path.parent.name) from e @@ -226,7 +226,7 @@ def _upgrade_pre_0_21_2_preset(data: dict, *, preset_id: str) -> dict: def _earlier_version_preset_error(preset_id: str) -> EarlierVersionPresetError: return EarlierVersionPresetError( - f"Preset {preset_id} was created before dstack 0.21 and cannot be read." + f"VerifiedPreset {preset_id} was created before dstack 0.21 and cannot be read." f" Delete it with `dstack preset delete {preset_id}`, or recreate it." ) @@ -260,7 +260,7 @@ def parse_preset_configuration(stream: TextIO) -> PresetConfiguration: try: data = yaml.safe_load(stream) if not isinstance(data, dict): - raise ConfigurationError("Preset configuration must be a YAML object") + raise ConfigurationError("VerifiedPreset configuration must be a YAML object") # Only checked here: a stored preset serializes `model` as an object, so # the model itself has to keep accepting the form a file may not use. model = data.get("model") diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 1263e45fc..ebb64a8ee 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -11,8 +11,8 @@ PresetAgentSuccess, ) from dstack._internal.cli.models.presets import ( - Preset, PresetVerificationReplicaGroup, + VerifiedPreset, ) from dstack._internal.cli.services.presets.agent import ( PresetAgentProcessOutput, @@ -88,7 +88,7 @@ def build_verified_preset( preset_id: str, name: Optional[str], submitted_at: datetime, -) -> Preset: +) -> VerifiedPreset: """Cross-checks the agent's self-reported final report against the actual run and service state before trusting it to build a preset. The preset's service is taken from the run the server verified, not from anything the agent wrote, and diff --git a/src/tests/_internal/cli/commands/test_preset.py b/src/tests/_internal/cli/commands/test_preset.py index 982f64a4b..6e3da3237 100644 --- a/src/tests/_internal/cli/commands/test_preset.py +++ b/src/tests/_internal/cli/commands/test_preset.py @@ -12,7 +12,13 @@ from dstack._internal.cli.services.presets.store import PresetStore from dstack._internal.core.errors import CLIError from dstack._internal.utils.common import render_datetime_as_api -from tests._internal.cli.common import get_preset, plain_console, run_dstack_cli +from tests._internal.cli.common import ( + get_preset, + get_session_run, + get_session_state, + plain_console, + run_dstack_cli, +) pytestmark = pytest.mark.windows @@ -69,6 +75,60 @@ def test_handles_keyboard_interrupt(self, tmp_path, capsys): # interrupt handler, so the command layer stays silent — no generic line. assert "Operation interrupted by user" not in capsys.readouterr().out + def test_resume_uses_the_configuration_and_prompt_pinned_at_creation(self, tmp_path): + session_dir = tmp_path / ".dstack" / "presets" / "ab12cd34" + session_dir.mkdir(parents=True) + (session_dir / "session.json").write_text( + json.dumps( + get_session_state( + status="interrupted", run=get_session_run(claude_session_id="sid-1") + ).model_dump(mode="json") + ) + ) + (session_dir / "preset.dstack.yml").write_text( + "type: preset\nbase: Qwen/Qwen3.5-27B\nmin_context_length: 8192\n" + ) + (session_dir / "prompt.md").write_text("go deep\n") + + with ( + patch("dstack.api.Client.from_config"), + patch("dstack._internal.cli.commands.preset.create_preset") as create, + ): + exit_code = run_dstack_cli( + ["preset", "resume", "ab12cd34"], home_dir=tmp_path, repo_dir=tmp_path + ) + + assert exit_code == 0 + kwargs = create.call_args.kwargs + assert kwargs["resume_session"].preset_id == "ab12cd34" + assert kwargs["configuration"].model.base == "Qwen/Qwen3.5-27B" + assert kwargs["user_prompt"] == "go deep" + + def test_get_returns_the_requested_configuration_for_an_unfinished_preset( + self, tmp_path, capsys + ): + session_dir = tmp_path / ".dstack" / "presets" / "ab12cd34" + session_dir.mkdir(parents=True) + session_dir.joinpath("session.json").write_text( + json.dumps( + get_session_state(status="running", name="in-flight").model_dump(mode="json") + ) + ) + session_dir.joinpath("preset.dstack.yml").write_text( + "type: preset\nbase: Qwen/Qwen3.5-27B\nmin_context_length: 8192\n" + ) + + exit_code = run_dstack_cli( + ["preset", "get", "--json", "in-flight"], home_dir=tmp_path, repo_dir=tmp_path + ) + + assert exit_code == 0 + data = json.loads(capsys.readouterr().out) + assert data["status"] == "running" + assert data["id"] == "ab12cd34" + assert data["configuration"]["model"]["base"] == "Qwen/Qwen3.5-27B" + assert "service" not in data + def test_create_ends_quietly_when_stopped_from_another_cli(self, tmp_path, capsys): from dstack._internal.cli.services.presets.create import CreationStopped @@ -135,8 +195,8 @@ def test_lists_presets_without_api_client(self, tmp_path): assert "ctx>=" not in "".join(output.split()) assert "io=1K/128" in "".join(output.split()) # 42.1 is the aggregate, verbose-only; the default row shows per-user 1/TPOT. - assert "135" in output - assert "conc=1" in "".join(output.split()) + assert "133" in output + assert "c=1" in "".join(output.split()) assert "tok/s/user=" in "".join(output.split()) assert "ttft=" in "".join(output.split()) assert "ttft=[/]108" in "".join(output.split()) or "ttft=108" in "".join(output.split()) @@ -155,7 +215,7 @@ def test_verbose_list_adds_repo(self, tmp_path): # Verbose adds only the repo row. assert "repo=community/Qwen3.5-27B-GPTQ-Int4" in joined_verbose assert "ctx=32K" in joined_verbose - assert "conc=1" in joined_verbose + assert "c=1" in joined_verbose assert "hardware=" not in joined_verbose def test_deletes_preset_without_api_client(self, tmp_path): @@ -341,56 +401,6 @@ def test_merges_profile_configuration_and_cli_args(self, tmp_path): assert [fleet.format() for fleet in configuration.fleets] == ["cli-fleet"] assert create.call_args.kwargs["debug"] is True - def test_apply_passes_selected_profile_and_preset_id(self, tmp_path): - extra_args = ["--id", "cli-preset"] - (tmp_path / ".dstack").mkdir() - (tmp_path / ".dstack" / "profiles.yml").write_text( - "profiles:\n - name: gpu\n max_price: 0.5\n" - ) - configuration_path = tmp_path / "preset.dstack.yml" - configuration_path.write_text("type: preset\nname: qwen\nbase: Qwen/Qwen3.5-27B\n") - - with ( - patch("dstack.api.Client.from_config"), - patch("dstack._internal.cli.commands.preset.apply_preset") as apply, - ): - args = [ - "preset", - "apply", - "-f", - str(configuration_path), - "--profile", - "gpu", - *extra_args, - ] - exit_code = run_dstack_cli( - args, - home_dir=tmp_path, - repo_dir=tmp_path, - ) - - assert exit_code == 0 - assert apply.call_args.kwargs["profile_name"] == "gpu" - assert apply.call_args.kwargs["preset_id"] == "cli-preset" - assert apply.call_args.kwargs["configuration"].max_price == 0.5 - - def test_apply_requires_preset_id(self, tmp_path, capsys): - configuration_path = tmp_path / "preset.dstack.yml" - configuration_path.write_text("type: preset\nname: qwen\nbase: Qwen/Qwen3.5-27B\n") - - with patch("dstack._internal.cli.commands.preset.apply_preset") as apply: - exit_code = run_dstack_cli( - ["preset", "apply", "-f", str(configuration_path)], - home_dir=tmp_path, - repo_dir=tmp_path, - ) - - assert exit_code == 2 - assert "--id" in capsys.readouterr().err - apply.assert_not_called() - - -class TestPresetNameClaims: def test_create_detaches_the_name_from_the_old_preset(self, tmp_path): preset = get_preset().model_copy(update={"name": "qwen"}) store = PresetStore(tmp_path / ".dstack" / "presets") diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index e98048d5f..c925dca80 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -19,9 +19,9 @@ PresetSessionWorkspace, ) from dstack._internal.cli.models.presets import ( - Preset, PresetBenchmark, PresetVerificationReplicaGroup, + VerifiedPreset, ) from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.models.configurations import ( @@ -110,7 +110,7 @@ def get_preset( *, preset_id: str = "8f3a12c4", context_length: int = 32768, -) -> Preset: +) -> VerifiedPreset: resources = ResourcesSpec.model_validate( { "cpu": "16", @@ -119,7 +119,7 @@ def get_preset( "gpu": {"name": "A6000", "memory": "48GB", "count": 1}, } ) - return Preset( + return VerifiedPreset( configuration=PresetConfiguration.model_validate( { "type": "preset", diff --git a/src/tests/_internal/cli/services/presets/test_apply.py b/src/tests/_internal/cli/services/presets/test_apply.py deleted file mode 100644 index 968ed2005..000000000 --- a/src/tests/_internal/cli/services/presets/test_apply.py +++ /dev/null @@ -1,140 +0,0 @@ -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest - -from dstack._internal.cli.models.configurations import PresetConfiguration -from dstack._internal.cli.services.presets.apply import ( - _build_service, - _validate_preset_matches, - apply_preset, -) -from dstack._internal.core.errors import CLIError -from dstack._internal.core.models.instances import InstanceAvailability -from tests._internal.cli.common import get_preset - -pytestmark = pytest.mark.windows - - -class TestValidatePresetMatches: - def test_accepts_matching_base_model_and_context(self): - preset = get_preset(preset_id="large", context_length=32768) - configuration = PresetConfiguration( - name="qwen", - base="Qwen/Qwen3.5-27B", - min_context_length=8192, - ) - - _validate_preset_matches(preset, configuration=configuration) - - def test_warns_on_insufficient_context_instead_of_failing(self, capsys): - # The preset is chosen by ID and may be the best a session could verify; - # the shortfall is stated and the plan confirmation decides. - preset = get_preset(preset_id="small", context_length=4096) - configuration = PresetConfiguration( - name="qwen", - base="Qwen/Qwen3.5-27B", - min_context_length=8192, - ) - - _validate_preset_matches(preset, configuration=configuration) - - output = capsys.readouterr().out - assert "verified for context length 4096" in output - assert "8192" in output - - def test_exact_request_matches_repo_and_client_facing_name(self): - matching = get_preset(preset_id="matching") - configuration = PresetConfiguration( - name="qwen", - model={ - "repo": "community/Qwen3.5-27B-GPTQ-Int4", - "name": "Qwen/Qwen3.5-27B", - }, - ) - - _validate_preset_matches(matching, configuration=configuration) - with pytest.raises(CLIError, match="does not serve repo"): - _validate_preset_matches( - matching.model_copy(update={"model": "other/repo"}), - configuration=configuration, - ) - - -class TestBuildService: - def test_applies_preset_name_env_gateway_and_constraints(self): - configuration = PresetConfiguration( - name="qwen-production", - base="Qwen/Qwen3.5-27B", - gateway="inference", - env={"HF_TOKEN": "token"}, - fleets=["gpu-fleet"], - max_price=1, - ) - - service = _build_service(configuration, get_preset()) - - assert service.name == "qwen-production" - assert service.gateway == "inference" - assert service.env["HF_TOKEN"] == "token" - assert [fleet.format() for fleet in service.fleets] == ["gpu-fleet"] - assert service.max_price == 1 - - -class TestApplyPreset: - def test_rejects_unknown_preset(self): - with pytest.raises(CLIError, match="does not exist"): - apply_preset( - api=Mock(), - configuration=PresetConfiguration(name="qwen", base="Qwen/Qwen3.5-27B"), - configuration_path="preset.dstack.yml", - preset_id="ee55ff66", - profile_name=None, - command_args=SimpleNamespace(), - store=Mock(get=Mock(return_value=None)), - ) - - def test_applies_the_referenced_preset(self, monkeypatch): - preset = get_preset() - run_plan = _plan(InstanceAvailability.AVAILABLE) - repo = Mock() - service_args = SimpleNamespace(profile="gpu") - configurator = Mock() - configurator.get_parser.return_value.parse_args.return_value = service_args - configurator.get_plan.return_value = (run_plan, repo) - monkeypatch.setattr( - "dstack._internal.cli.services.presets.apply.ServiceConfigurator", - lambda api_client: configurator, - ) - command_args = SimpleNamespace() - - apply_preset( - api=Mock(), - configuration=PresetConfiguration( - name="qwen", - base="Qwen/Qwen3.5-27B", - ), - configuration_path="preset.dstack.yml", - preset_id="8f3a12c4", - profile_name="gpu", - command_args=command_args, - store=Mock(get=Mock(return_value=preset)), - ) - - assert service_args.profile == "gpu" - configurator.apply_plan.assert_called_once_with( - run_plan=run_plan, - repo=repo, - command_args=command_args, - configurator_args=service_args, - plan_properties={ - "Model": "Qwen/Qwen3.5-27B ([secondary]base[/])", - "Preset": "8f3a12c4 ([secondary]io=1K/128 prefix=0% conc=1[/] tok/s/user=135 tok/s=42.1 ttft=108ms ctx=32K)", - }, - ) - - -def _plan(availability: InstanceAvailability): - return SimpleNamespace( - job_plans=[SimpleNamespace(offers=[SimpleNamespace(availability=availability)])] - ) diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index 684ac2ece..fea2b50b3 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -702,7 +702,7 @@ def test_throughput_is_derived_not_read(self): expected = benchmark.metrics.total_output_tokens / benchmark.metrics.duration_seconds assert benchmark.effective_output_tok_per_s == expected # Per-user speed is the steady decode rate, not the aggregate over concurrency. - assert benchmark.effective_per_user_tok_per_s == 1000 / benchmark.metrics.tpot_ms.p50 + assert benchmark.effective_per_user_tok_per_s == 1000 / benchmark.metrics.tpot_ms.mean class TestBuildConstraints: @@ -852,7 +852,7 @@ async def create(**kwargs): state = json.loads((sessions[0] / "session.json").read_text()) assert state["status"] == "interrupted" output = capsys.readouterr().out - assert "--resume" in output + assert "dstack preset resume" in output assert sessions[0].name in output def test_suspend_scrubs_workspace_token(self, tmp_path, capsys): diff --git a/src/tests/_internal/cli/services/presets/test_export.py b/src/tests/_internal/cli/services/presets/test_export.py new file mode 100644 index 000000000..88187a8dd --- /dev/null +++ b/src/tests/_internal/cli/services/presets/test_export.py @@ -0,0 +1,72 @@ +from pathlib import Path + +import pytest +import yaml + +from dstack._internal.cli.services.presets.export import export_preset +from dstack._internal.cli.services.presets.store import PresetStore +from dstack._internal.core.errors import CLIError +from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.files import FilePathMapping +from tests._internal.cli.common import get_preset + +pytestmark = pytest.mark.windows + + +class TestExportPreset: + def test_exports_a_deployable_service_configuration_with_its_files(self, tmp_path: Path): + store = PresetStore(tmp_path / "presets") + preset = get_preset() + preset.service.files = [ + FilePathMapping(local_path="service/1/patches/fix.patch", path="/patches/fix.patch") + ] + preset_dir = store.save(preset).parent + (preset_dir / "service" / "1" / "patches").mkdir(parents=True) + (preset_dir / "service" / "1" / "patches" / "fix.patch").write_text("--- a\n+++ b\n") + destination = tmp_path / "deploy" / "qwen.dstack.yml" + + written = export_preset( + store.get(preset.id), + preset_dir=preset_dir, + destination=destination, + force=False, + ) + + assert written == [ + destination, + tmp_path / "deploy" / "service" / "1" / "patches" / "fix.patch", + ] + data = yaml.safe_load(destination.read_text()) + assert data["type"] == "service" + # Relative to the configuration file, which is how `dstack apply` + # resolves `files` paths. + assert data["files"] == [ + {"local_path": "service/1/patches/fix.patch", "path": "/patches/fix.patch"} + ] + assert (tmp_path / "deploy" / "service" / "1" / "patches" / "fix.patch").read_text() == ( + "--- a\n+++ b\n" + ) + assert ServiceConfiguration.model_validate(data).model is not None + + def test_refuses_to_overwrite_without_force(self, tmp_path: Path): + store = PresetStore(tmp_path / "presets") + preset_dir = store.save(get_preset()).parent + destination = tmp_path / "qwen.dstack.yml" + destination.write_text("existing") + + with pytest.raises(CLIError, match="already exists"): + export_preset( + store.get(get_preset().id), + preset_dir=preset_dir, + destination=destination, + force=False, + ) + assert destination.read_text() == "existing" + + export_preset( + store.get(get_preset().id), + preset_dir=preset_dir, + destination=destination, + force=True, + ) + assert yaml.safe_load(destination.read_text())["type"] == "service" diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index 8d1032923..152f5feb6 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -35,7 +35,7 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): output = output_module.format_preset_benchmark(preset, verbose=True) # Per-user speed is 1/TPOT (p50 7.4ms), not the aggregate over concurrency. - assert output.startswith("tok/s/user=135 ") + assert output.startswith("tok/s/user=133 ") assert output_module.format_preset_objective(preset).startswith("[secondary]io=1K/128 ") assert "ctx=32K" in output assert "ttft=8.15s" in output @@ -48,16 +48,14 @@ def test_shows_the_requested_shared_prefix(self): preset.configuration.shared_prefix_tokens = 768 assert output_module.format_preset_objective(preset) == ( - "[secondary]io=1K/128 prefix=75% conc=1[/]" + "[secondary]io=1K/128 prefix=75% c=1[/]" ) def test_treats_an_unset_shared_prefix_as_none_shared(self): preset = get_preset() assert preset.configuration.shared_prefix_tokens is None - assert output_module.format_preset_objective(preset) == ( - "[secondary]io=1K/128 prefix=0% conc=1[/]" - ) + assert output_module.format_preset_objective(preset) == ("[secondary]io=1K/128 c=1[/]") def test_shows_the_dataset_instead_of_the_request_shape(self): # A dataset defines its own request shape, so the cell names it instead. @@ -65,7 +63,7 @@ def test_shows_the_dataset_instead_of_the_request_shape(self): preset.configuration.dataset = "spec_bench" assert output_module.format_preset_objective(preset) == ( - "[secondary]data=spec_bench conc=1[/]" + "[secondary]data=spec_bench c=1[/]" ) def test_renders_the_requested_workload_not_the_measured_one(self): @@ -83,7 +81,7 @@ def test_renders_the_requested_workload_not_the_measured_one(self): ) assert output_module.format_preset_objective(preset) == ( - "[secondary]data=spec_bench conc=1[/]" + "[secondary]data=spec_bench c=1[/]" ) @@ -97,7 +95,7 @@ def test_preserves_constraints_and_benchmark_at_narrow_width(self, monkeypatch): # Both columns wrap rather than clip, so their full content survives even # when a long model name would otherwise squeeze them out. joined = "".join(output.getvalue().split()) - assert "conc=1" in joined + assert "c=1" in joined assert "ttft=108ms" in joined def test_prints_submitted_column(self, monkeypatch): @@ -113,7 +111,17 @@ def test_prints_submitted_column(self, monkeypatch): def _session_row(session: dict) -> dict: table = Table(box=None) - for column in ("BASE", "ID", "GPU", "CONSTRAINTS", "BENCHMARK", "", "STATUS", "SUBMITTED"): + for column in ( + "BASE", + "ID", + "NAME", + "RESOURCES", + "CONSTRAINTS", + "BENCHMARK", + "", + "STATUS", + "SUBMITTED", + ): table.add_column(column) _add_session(table, session) return { @@ -145,7 +153,7 @@ def test_shows_progress_after_status_and_best_benchmark(self): # Per-user speed is 1/TPOT, the same definition the preset row uses — not # the aggregate over concurrency, which would read 292 here. assert row["BENCHMARK"].startswith("tok/s/user=292") - assert row["GPU"] == "A40:48GB:1" + assert row["RESOURCES"] == "A40:48GB:1" def test_shows_the_shared_prefix_when_the_workload_has_one(self): row = _session_row( @@ -161,7 +169,7 @@ def test_shows_the_shared_prefix_when_the_workload_has_one(self): } ) - assert row["CONSTRAINTS"] == ("[secondary]io=8K/1K prefix=90% conc=162[/]") + assert row["CONSTRAINTS"] == ("[secondary]io=8K/1K prefix=90% c=162[/]") def test_shows_the_dataset_for_a_session_with_a_custom_dataset(self): row = _session_row( @@ -172,10 +180,10 @@ def test_shows_the_dataset_for_a_session_with_a_custom_dataset(self): } ) - assert row["CONSTRAINTS"] == ("[secondary]data=spec_bench conc=4[/]") + assert row["CONSTRAINTS"] == ("[secondary]data=spec_bench c=4[/]") - def test_shows_the_shared_prefix_even_when_requests_are_fully_unique(self): - # `prefix=0%` is not noise: it decides how much of each request the engine + def test_omits_the_shared_prefix_when_requests_are_fully_unique(self): + # A zero prefix is the default; only a non-zero share is information. # can serve from cache, so a row without it cannot be compared to one with. row = _session_row( { @@ -190,7 +198,7 @@ def test_shows_the_shared_prefix_even_when_requests_are_fully_unique(self): } ) - assert "prefix=0%" in row["CONSTRAINTS"] + assert "prefix" not in row["CONSTRAINTS"] def test_shows_zero_progress_without_benchmark(self): row = _session_row( @@ -387,7 +395,7 @@ def test_the_fastest_failed_trial_is_kept_when_nothing_passed(self, tmp_path): "metrics": { "total_output_tokens": tokens, "duration_seconds": 1.0, - "tpot_ms": {"p50": 34.4}, + "tpot_ms": {"mean": 41.7, "p50": 34.4}, "ttft_ms": {"p50": 4300.0}, }, "workload": {"concurrency": 4}, @@ -402,6 +410,8 @@ def test_the_fastest_failed_trial_is_kept_when_nothing_passed(self, tmp_path): assert summary["best"] is None assert summary["best_failed"]["tok_s"] == 300.0 + # Mean, not p50: MTP skews TPOT right, and mean is the delivered rate. + assert summary["best_failed"]["tpot_ms"] == 41.7 def test_a_run_that_met_nothing_still_shows_what_it_measured(self): row = _session_row( @@ -427,7 +437,7 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): ) assert "ttft=4.3s" in row["BENCHMARK"] - assert row["GPU"] == "MI300X:192GB:1" + assert row["RESOURCES"] == "MI300X:192GB:1" # Marked, not just dimmed, so it does not read as a met constraint where # styling is absent or invisible to the reader. assert "*tok/s/user" in row["BENCHMARK"] diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index 51726bfc4..ad3deaa40 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -29,6 +29,8 @@ def test_saves_and_lists_self_contained_preset(self, tmp_path: Path): assert data["id"] == preset.id assert data["model"] == preset.model assert data["submitted_at"] == "2026-01-02T03:04:00Z" + # Wire-only: the stored file never carries `status`. + assert "status" not in data assert "presets" not in data assert store.list() == [preset] assert store.get(preset.id) == preset From 44899e6a780c606a2e4a298ddaf5b32b2c28ef78 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Mon, 17 Aug 2026 15:40:17 +0200 Subject: [PATCH 2/3] [Presets] Make preset a first-class apply configuration `dstack apply` now accepts `type: preset` configurations through a PresetConfigurator registered like every other configuration type, with the full creation flag set and `dstack apply -h preset` help. The preset configuration model moves from the CLI into core models, joins AnyApplyConfiguration so the core parser discriminates it natively, and joins the published IDE schema so editors validate and complete `preset.dstack.yml` files. `dstack preset create` remains as a deprecated thin adapter over the same configurator. The displayed metric labels follow the industry vocabulary: `tps/user` and `tps`. Co-Authored-By: Claude Fable 5 --- mkdocs/docs/concepts/presets.md | 26 +-- mkdocs/docs/reference/cli/dstack/preset.md | 2 + mkdocs/docs/reference/dstack.yml/preset.md | 8 +- src/dstack/_internal/cli/commands/preset.py | 153 +++------------ src/dstack/_internal/cli/models/presets.py | 2 +- .../cli/services/configurators/__init__.py | 10 +- .../cli/services/configurators/base.py | 2 + .../cli/services/configurators/preset.py | 182 ++++++++++++++++++ .../_internal/cli/services/presets/build.py | 2 +- .../_internal/cli/services/presets/create.py | 14 +- .../_internal/cli/services/presets/output.py | 12 +- .../_internal/cli/services/presets/session.py | 2 +- .../_internal/cli/services/presets/store.py | 24 +-- .../_internal/cli/services/presets/verify.py | 2 +- .../_internal/core/models/configurations.py | 5 + .../models/presets.py} | 16 +- .../_internal/cli/commands/test_preset.py | 106 +++++++++- src/tests/_internal/cli/common.py | 2 +- .../cli/services/presets/test_agent.py | 2 +- .../cli/services/presets/test_create.py | 13 +- .../cli/services/presets/test_output.py | 6 +- .../cli/services/presets/test_store.py | 10 +- .../cli/services/presets/test_verify.py | 2 +- .../models/test_presets.py} | 12 +- 24 files changed, 408 insertions(+), 207 deletions(-) create mode 100644 src/dstack/_internal/cli/services/configurators/preset.py rename src/dstack/_internal/{cli/models/configurations.py => core/models/presets.py} (95%) rename src/tests/_internal/{cli/models/test_configurations.py => core/models/test_presets.py} (89%) diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index 473fe0837..3f625b5a3 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -53,12 +53,12 @@ env:
-To create the preset, pass the configuration to the `dstack preset create` command: +To create the preset, pass the configuration to the `dstack apply` command:
```shell -$ dstack preset create -f preset.dstack.yml --fleet b200-fleet +$ dstack apply -f preset.dstack.yml --fleet b200-fleet Create the preset dsv4-flash? [y/n]: y [2026-08-04 11:38:34] Starting preset creation for deepseek-ai/DeepSeek-V4-Flash. Allowed fleets: b200-fleet. [2026-08-04 12:31:19] Trial 3 switched from vLLM to SGLang: 319 tok/s per user, 2.2x the baseline. @@ -69,7 +69,7 @@ Create the preset dsv4-flash? [y/n]: y
> It's highly recommended to specify the exact hardware you want the preset to use, so that the -> optimization is done against that hardware. Point `dstack preset create` to a fleet configured +> optimization is done against that hardware. Point `dstack apply` to a fleet configured > correspondingly, via `fleets` inside the preset configuration or via `--fleet` in the CLI. The command executes entirely locally and uses the locally installed `claude` CLI along with `dstack`'s bundled skills. The agent uses a `dstack` task to find the best serving configuration for the available fleet offers, then submits it as a `dstack` service for a final benchmark. @@ -113,7 +113,7 @@ To stop a creation and its runs, use `dstack preset stop`. Set `fleets` to restrict creation and reuse to specific [fleets](fleets.md). It's highly recommended to specify a fleet with exactly the hardware that you'd like the preset to use. -Alternatively, pass `--fleet` to `dstack preset create`. +Alternatively, pass `--fleet` to `dstack apply`. > Profile settings such as `spot_policy`, `max_price`, and `backends` are ignored during preset > creation. Configure them on the fleet instead. @@ -149,7 +149,7 @@ previous: -Alternatively, pass `--previous` (repeatable) to `dstack preset create`. +Alternatively, pass `--previous` (repeatable) to `dstack apply`. ### Prompt @@ -249,7 +249,7 @@ Use `dstack preset` to list presets: ```shell $ dstack preset list NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED - dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tok/s/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tps/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago ``` @@ -261,15 +261,15 @@ By default, `dstack preset` shows creations that are still running, or the most ```shell $ dstack preset list -a NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED - dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tok/s/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago - qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tok/s/user=19.6 ttft=3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago - qwen36-pro4500 9ab0fa65 Qwen/Qwen3.6-27B io=1K/1K c=8 tok/s/user=57.1 ttft=499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago - qwen3-32b-5090 f91d6b60 Qwen/Qwen3-32B io=1K/512 c=8 tok/s/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago + dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tps/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tps/user=19.6 ttft=3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago + qwen36-pro4500 9ab0fa65 Qwen/Qwen3.6-27B io=1K/1K c=8 tps/user=57.1 ttft=499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago + qwen3-32b-5090 f91d6b60 Qwen/Qwen3-32B io=1K/512 c=8 tps/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago ``` -The `CONSTRAINTS` column is what the creation was asked for, and `BENCHMARK` is the best trial so far. `tok/s/user` is the steady decode rate, measured as one second divided by the median time per output token, so it excludes the time to the first token. +The `CONSTRAINTS` column is what the creation was asked for, and `BENCHMARK` is the best trial so far. `tps/user` is the steady decode rate, measured as one second divided by the mean time per output token, so it excludes the time to the first token. The glyphs after the benchmark are one per trial: height is throughput, a yellow bar is a trial whose benchmark broke a constraint, and a red `·` is one that produced no benchmark at all. The shape shows whether a run converged or wandered. @@ -292,12 +292,12 @@ $ dstack preset delete c83375b4 ## Troubleshooting -To trace the agent's activity, pass `--debug` to `dstack preset create`: +To trace the agent's activity, pass `--debug` to `dstack apply`:
```shell -$ dstack preset create -f preset.dstack.yml --debug +$ dstack apply -f preset.dstack.yml --debug ```
diff --git a/mkdocs/docs/reference/cli/dstack/preset.md b/mkdocs/docs/reference/cli/dstack/preset.md index 358a1f3c6..3aacffe73 100644 --- a/mkdocs/docs/reference/cli/dstack/preset.md +++ b/mkdocs/docs/reference/cli/dstack/preset.md @@ -20,6 +20,8 @@ $ dstack preset list --help ## dstack preset create +> Deprecated: pass a preset configuration to [`dstack apply`](apply.md) instead. + The `dstack preset create` command uses an agent to create and save a verified preset from a preset configuration. diff --git a/mkdocs/docs/reference/dstack.yml/preset.md b/mkdocs/docs/reference/dstack.yml/preset.md index a3ac4176f..1ce978304 100644 --- a/mkdocs/docs/reference/dstack.yml/preset.md +++ b/mkdocs/docs/reference/dstack.yml/preset.md @@ -5,7 +5,7 @@ used to create or apply a [preset](../../concepts/presets.md). ## Root reference -#SCHEMA# dstack._internal.cli.models.configurations.PresetConfiguration +#SCHEMA# dstack._internal.core.models.presets.PresetConfiguration overrides: show_root_heading: false type: @@ -17,7 +17,7 @@ used to create or apply a [preset](../../concepts/presets.md). Allows the creation agent to select a compatible model variant. - #SCHEMA# dstack._internal.cli.models.configurations.PresetModelBase + #SCHEMA# dstack._internal.core.models.presets.PresetModelBase overrides: show_root_heading: false @@ -26,7 +26,7 @@ used to create or apply a [preset](../../concepts/presets.md). Requires an exact model repo or path and optionally sets another client-facing model name. - #SCHEMA# dstack._internal.cli.models.configurations.PresetModelRepo + #SCHEMA# dstack._internal.core.models.presets.PresetModelRepo overrides: show_root_heading: false @@ -34,7 +34,7 @@ used to create or apply a [preset](../../concepts/presets.md). Custom agent instructions. Set to an inline string, or to a file: -#SCHEMA# dstack._internal.cli.models.configurations.PresetPromptFile +#SCHEMA# dstack._internal.core.models.presets.PresetPromptFile overrides: show_root_heading: false diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py index 7399f7344..ee30e9ede 100644 --- a/src/dstack/_internal/cli/commands/preset.py +++ b/src/dstack/_internal/cli/commands/preset.py @@ -11,22 +11,21 @@ from rich.live import Live from dstack._internal.cli.commands import BaseCommand -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.presets import ( Preset, PresetListOutput, ) from dstack._internal.cli.services.completion import ProjectNameCompleter from dstack._internal.cli.services.configurators import APPLY_STDIN_NAME +from dstack._internal.cli.services.configurators.preset import ( + PresetConfigurator, + register_creation_args, +) from dstack._internal.cli.services.presets.create import ( CreationStopped, create_preset, - find_preset_name_holders, load_session_configuration, - plan_preset, - reassign_preset_name, reconcile_detached_sessions, - resolve_previous_sessions, show_preset_session_logs, stop_preset_session, ) @@ -42,11 +41,8 @@ PresetStore, load_preset_configuration, parse_preset_configuration, - resolve_preset_prompt, ) from dstack._internal.cli.services.profile import ( - apply_profile_args, - load_profile_from_args, register_profile_args, ) from dstack._internal.cli.utils.common import ( @@ -56,9 +52,8 @@ console, warn, ) -from dstack._internal.core.errors import CLIError, ConfigurationError, ServerClientError -from dstack._internal.core.models.profiles import ProfileParams -from dstack._internal.core.services import validate_dstack_resource_name +from dstack._internal.core.errors import CLIError +from dstack._internal.core.models.presets import PresetConfiguration from dstack.api import Client @@ -106,29 +101,7 @@ def _register(self) -> None: ) _add_configuration_args(create_parser) register_profile_args(create_parser) - create_parser.add_argument( - "--keep-service", - action="store_true", - help="Leave the verified service running", - ) - create_parser.add_argument( - "--trials", - type=int, - metavar="N", - help="The number of benchmarked trials before the best one is promoted", - ) - create_parser.add_argument( - "--debug", - action="store_true", - help="Save the agent prompt and raw trace", - ) - create_parser.add_argument( - "--previous", - action="append", - metavar="ID", - help="Give the agent a previous session's results to analyze and improve on." - " Repeat for several", - ) + register_creation_args(create_parser) create_parser.add_argument( "-y", "--yes", action="store_true", help="Do not ask for confirmation" ) @@ -292,44 +265,22 @@ def _list_presets_and_sessions( return presets, sessions def _create(self, args: argparse.Namespace) -> None: + warn( + "`dstack preset create` is deprecated; use `dstack apply -f `", + stderr=True, + ) _check_stdin_configuration_confirmable(args) _, configuration = _read_configuration_arg(args.configuration_file) - configuration = _get_effective_configuration(configuration, args, require_name=False) - user_prompt = resolve_preset_prompt(configuration, _prompt_base(args.configuration_file)) - store = PresetStore() - previous = () - if configuration.previous: - previous = resolve_previous_sessions(configuration.previous) - api = Client.from_config(project_name=args.project) - if configuration.trials is None: - raise ConfigurationError( - "trials is required. Set it in the configuration or pass --trials" - ) - for field in ("max_ttft", "min_context_length", "concurrency"): - if getattr(configuration, field) is None: - raise ConfigurationError(f"{field} is required") - allowed_fleets = plan_preset(api=api, configuration=configuration) - if not _confirm_preset_creation(store, configuration.name, assume_yes=args.yes): - console.print("\nExiting...") - return - try: - result = create_preset( - api=api, - configuration=configuration, - store=store, - keep_service=args.keep_service, - debug=args.debug, - user_prompt=user_prompt, - allowed_fleets=allowed_fleets, - previous=previous, - ) - except KeyboardInterrupt: - return # the interrupt handler already reported detach / stop - except CreationStopped: - return # stopped from another CLI, which reported the interruption - # The log already told the story; like a finished run, success is silent. - if args.keep_service: - console.print(f"Final service [code]{result.final_run_name}[/] kept running") + # A thin adapter over the configurator, the same path `dstack apply` takes. + # The create parser has no -d, which the configurator contract reads. + args.detach = False + configurator = PresetConfigurator(api_client=Client.from_config(project_name=args.project)) + configurator.apply_configuration( + conf=configuration, + configuration_path=args.configuration_file, + command_args=args, + configurator_args=args, + ) def _resume(self, args: argparse.Namespace) -> None: session = load_resumable_session(resolve_session_ref(args.preset)) @@ -461,7 +412,7 @@ def _add_configuration_args(parser: argparse.ArgumentParser) -> None: "-n", "--name", metavar="NAME", - help="The service name. Required when the configuration omits name", + help="The preset name", ) @@ -534,40 +485,6 @@ def _session_matches_model( return model == base or repo_to_base.get(model) == base -def _apply_name(configuration: PresetConfiguration, name: str | None, *, required: bool) -> None: - if name is not None: - configuration.name = name - if configuration.name is None: - if required: - raise CLIError( - "The service name is required. Set `name` in the configuration or use --name" - ) - return - try: - validate_dstack_resource_name(configuration.name) - except ServerClientError as e: - raise CLIError(str(e)) from e - - -def _confirm_preset_creation(store: PresetStore, name: str | None, *, assume_yes: bool) -> bool: - """One apply-style confirmation; reassigns the name from any holder on yes.""" - holders = find_preset_name_holders(store, name) if name is not None else None - if holders is not None and holders.preset_ids: - used_by = ", ".join(f"preset [code]{preset_id}[/]" for preset_id in holders.preset_ids) - message = ( - f"The name [code]{name}[/] is already used by {used_by}. Reassign it to a new preset?" - ) - elif name is not None: - message = f"Create the preset [code]{name}[/]?" - else: - message = "Create the preset?" - if not assume_yes and not confirm_ask(message): - return False - if holders is not None: - reassign_preset_name(store, holders) - return True - - def _read_configuration_arg(configuration_file: str) -> tuple[str, PresetConfiguration]: """`-f `, or `-f -` for stdin — the same convention as `dstack apply`.""" if configuration_file == APPLY_STDIN_NAME: @@ -576,34 +493,8 @@ def _read_configuration_arg(configuration_file: str) -> tuple[str, PresetConfigu return str(path.resolve()), load_preset_configuration(path) -def _prompt_base(configuration_file: str) -> Path: - """Prompt files resolve relative to the configuration file; cwd for stdin.""" - if configuration_file == APPLY_STDIN_NAME: - return Path.cwd() - return Path(configuration_file).resolve().parent - - def _check_stdin_configuration_confirmable(args: argparse.Namespace) -> None: # Same rule as `dstack apply`: the confirmation prompt cannot read from a # stdin that is the configuration itself. if not args.yes and args.configuration_file == APPLY_STDIN_NAME: raise CLIError("Cannot read configuration from stdin if -y/--yes is not specified") - - -def _get_effective_configuration( - configuration: PresetConfiguration, - args: argparse.Namespace, - *, - require_name: bool = True, -) -> PresetConfiguration: - _apply_name(configuration, args.name, required=require_name) - if getattr(args, "trials", None) is not None: - configuration.trials = args.trials - if getattr(args, "previous", None): - configuration.previous = list(args.previous) - profile = load_profile_from_args(args=args, repo_dir=Path.cwd()) - for field in ProfileParams.model_fields: - if getattr(configuration, field) is None: - setattr(configuration, field, getattr(profile, field)) - apply_profile_args(args, configuration) - return PresetConfiguration.model_validate(configuration.model_dump()) diff --git a/src/dstack/_internal/cli/models/presets.py b/src/dstack/_internal/cli/models/presets.py index 62e72e484..e7c50ac9e 100644 --- a/src/dstack/_internal/cli/models/presets.py +++ b/src/dstack/_internal/cli/models/presets.py @@ -11,9 +11,9 @@ ) from typing_extensions import Self -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.models.profiles import ProfileParams from dstack._internal.core.models.resources import Range, ResourcesSpec diff --git a/src/dstack/_internal/cli/services/configurators/__init__.py b/src/dstack/_internal/cli/services/configurators/__init__.py index 91768bdcd..cf0d7f08f 100644 --- a/src/dstack/_internal/cli/services/configurators/__init__.py +++ b/src/dstack/_internal/cli/services/configurators/__init__.py @@ -4,9 +4,13 @@ import yaml -from dstack._internal.cli.services.configurators.base import BaseApplyConfigurator +from dstack._internal.cli.services.configurators.base import ( + APPLY_STDIN_NAME, + BaseApplyConfigurator, +) from dstack._internal.cli.services.configurators.fleet import FleetConfigurator from dstack._internal.cli.services.configurators.gateway import GatewayConfigurator +from dstack._internal.cli.services.configurators.preset import PresetConfigurator from dstack._internal.cli.services.configurators.run import ( BaseRunConfigurator, DevEnvironmentConfigurator, @@ -21,9 +25,6 @@ parse_apply_configuration, ) -APPLY_STDIN_NAME = "-" - - apply_configurators_mapping: Dict[ ApplyConfigurationType, Type[BaseApplyConfigurator[AnyApplyConfiguration]] ] = { @@ -35,6 +36,7 @@ FleetConfigurator, GatewayConfigurator, VolumeConfigurator, + PresetConfigurator, ] } diff --git a/src/dstack/_internal/cli/services/configurators/base.py b/src/dstack/_internal/cli/services/configurators/base.py index c2d88b565..f380d9fe0 100644 --- a/src/dstack/_internal/cli/services/configurators/base.py +++ b/src/dstack/_internal/cli/services/configurators/base.py @@ -14,6 +14,8 @@ ArgsParser = Union[argparse._ArgumentGroup, argparse.ArgumentParser] +APPLY_STDIN_NAME = "-" + ApplyConfigurationT = TypeVar("ApplyConfigurationT", bound=AnyApplyConfiguration) diff --git a/src/dstack/_internal/cli/services/configurators/preset.py b/src/dstack/_internal/cli/services/configurators/preset.py new file mode 100644 index 000000000..59437e31c --- /dev/null +++ b/src/dstack/_internal/cli/services/configurators/preset.py @@ -0,0 +1,182 @@ +import argparse +from pathlib import Path + +from dstack._internal.cli.services.configurators.base import ( + APPLY_STDIN_NAME, + ArgsParser, + BaseApplyConfigurator, +) +from dstack._internal.cli.services.presets.create import ( + CreationStopped, + create_preset, + find_preset_name_holders, + plan_preset, + reassign_preset_name, + resolve_previous_sessions, +) +from dstack._internal.cli.services.presets.store import ( + PresetStore, + resolve_preset_prompt, +) +from dstack._internal.cli.services.profile import ( + apply_profile_args, + load_profile_from_args, + register_profile_args, +) +from dstack._internal.cli.utils.common import confirm_ask, console +from dstack._internal.core.errors import CLIError, ConfigurationError, ServerClientError +from dstack._internal.core.models.configurations import ApplyConfigurationType +from dstack._internal.core.models.presets import PresetConfiguration +from dstack._internal.core.models.profiles import ProfileParams +from dstack._internal.core.services import validate_dstack_resource_name + + +class PresetConfigurator(BaseApplyConfigurator[PresetConfiguration]): + TYPE = ApplyConfigurationType.PRESET + + def apply_configuration( + self, + conf: PresetConfiguration, + configuration_path: str, + command_args: argparse.Namespace, + configurator_args: argparse.Namespace, + ): + if command_args.detach: + raise CLIError("-d/--detach is not supported for preset configurations") + conf = self.apply_args(conf, configurator_args) + user_prompt = resolve_preset_prompt(conf, _configuration_prompt_base(configuration_path)) + store = PresetStore() + previous = () + if conf.previous: + previous = resolve_previous_sessions(conf.previous) + if conf.trials is None: + raise ConfigurationError( + "trials is required. Set it in the configuration or pass --trials" + ) + for field in ("max_ttft", "min_context_length", "concurrency"): + if getattr(conf, field) is None: + raise ConfigurationError(f"{field} is required") + allowed_fleets = plan_preset(api=self.api, configuration=conf) + if not _confirm_preset_creation(store, conf.name, assume_yes=command_args.yes): + console.print("\nExiting...") + return + try: + result = create_preset( + api=self.api, + configuration=conf, + store=store, + keep_service=configurator_args.keep_service, + debug=configurator_args.debug, + user_prompt=user_prompt, + allowed_fleets=allowed_fleets, + previous=previous, + ) + except KeyboardInterrupt: + return # the interrupt handler already reported detach / stop + except CreationStopped: + return # stopped from another CLI, which reported the interruption + # The log already told the story; like a finished run, success is silent. + if configurator_args.keep_service: + console.print(f"Final service [code]{result.final_run_name}[/] kept running") + + def delete_configuration( + self, + conf: PresetConfiguration, + configuration_path: str, + command_args: argparse.Namespace, + ): + raise CLIError( + "Deleting a preset configuration is not supported. Use `dstack preset delete `" + ) + + @classmethod + def register_args(cls, parser: argparse.ArgumentParser): + configuration_group = parser.add_argument_group(f"{cls.TYPE.value} Options") + configuration_group.add_argument( + "-n", + "--name", + dest="name", + help="The preset name", + ) + register_creation_args(configuration_group) + register_profile_args(parser) + + def apply_args( + self, conf: PresetConfiguration, args: argparse.Namespace + ) -> PresetConfiguration: + _apply_name(conf, args.name) + if getattr(args, "trials", None) is not None: + conf.trials = args.trials + if getattr(args, "previous", None): + conf.previous = list(args.previous) + profile = load_profile_from_args(args=args, repo_dir=Path.cwd()) + for field in ProfileParams.model_fields: + if getattr(conf, field) is None: + setattr(conf, field, getattr(profile, field)) + apply_profile_args(args, conf) + # A revalidated copy rather than the mutated input: merging the profile + # has to re-run the model validators. + return PresetConfiguration.model_validate(conf.model_dump()) + + +def register_creation_args(parser: ArgsParser) -> None: + parser.add_argument( + "--keep-service", + action="store_true", + help="Leave the verified service running", + ) + parser.add_argument( + "--trials", + type=int, + metavar="N", + help="The number of benchmarked trials before the best one is promoted", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Save the agent prompt and raw trace", + ) + parser.add_argument( + "--previous", + action="append", + metavar="ID", + help="Give the agent a previous session's results to analyze and improve on." + " Repeat for several", + ) + + +def _configuration_prompt_base(configuration_file: str) -> Path: + """Prompt files resolve relative to the configuration file; cwd for stdin.""" + if configuration_file == APPLY_STDIN_NAME: + return Path.cwd() + return Path(configuration_file).resolve().parent + + +def _apply_name(configuration: PresetConfiguration, name: str | None) -> None: + if name is not None: + configuration.name = name + if configuration.name is None: + return + try: + validate_dstack_resource_name(configuration.name) + except ServerClientError as e: + raise CLIError(str(e)) from e + + +def _confirm_preset_creation(store: PresetStore, name: str | None, *, assume_yes: bool) -> bool: + """One apply-style confirmation; reassigns the name from any holder on yes.""" + holders = find_preset_name_holders(store, name) if name is not None else None + if holders is not None and holders.preset_ids: + used_by = ", ".join(f"preset [code]{preset_id}[/]" for preset_id in holders.preset_ids) + message = ( + f"The name [code]{name}[/] is already used by {used_by}. Reassign it to a new preset?" + ) + elif name is not None: + message = f"Create the preset [code]{name}[/]?" + else: + message = "Create the preset?" + if not assume_yes and not confirm_ask(message): + return False + if holders is not None: + reassign_preset_name(store, holders) + return True diff --git a/src/dstack/_internal/cli/services/presets/build.py b/src/dstack/_internal/cli/services/presets/build.py index 6626ac099..00c37a9a8 100644 --- a/src/dstack/_internal/cli/services/presets/build.py +++ b/src/dstack/_internal/cli/services/presets/build.py @@ -3,7 +3,6 @@ import gpuhunt -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.presets import ( PRESET_EXCLUDED_FIELDS, PresetBenchmark, @@ -13,6 +12,7 @@ from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.envs import Env, EnvSentinel from dstack._internal.core.models.instances import Resources +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.models.resources import ( CPUSpec, DiskSpec, diff --git a/src/dstack/_internal/cli/services/presets/create.py b/src/dstack/_internal/cli/services/presets/create.py index 189a77de7..490e9520d 100644 --- a/src/dstack/_internal/cli/services/presets/create.py +++ b/src/dstack/_internal/cli/services/presets/create.py @@ -14,13 +14,6 @@ from rich.table import Table from rich.text import Text -from dstack._internal.cli.models.configurations import ( - DEFAULT_DATASET, - PresetConfiguration, - PresetConstraints, - PresetDatasetConstraints, - PresetRandomConstraints, -) from dstack._internal.cli.models.preset_agent import ( PresetAgentFailure, PresetAgentSuccess, @@ -84,6 +77,13 @@ from dstack._internal.core.models.configurations import TaskConfiguration from dstack._internal.core.models.envs import Env, EnvSentinel from dstack._internal.core.models.fleets import FleetStatus +from dstack._internal.core.models.presets import ( + DEFAULT_DATASET, + PresetConfiguration, + PresetConstraints, + PresetDatasetConstraints, + PresetRandomConstraints, +) from dstack._internal.core.models.runs import RunSpec from dstack.api import Client diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 06fb88bc2..1d86ba5e4 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -4,11 +4,11 @@ from rich.table import Table -from dstack._internal.cli.models.configurations import DEFAULT_DATASET from dstack._internal.cli.models.presets import ( VerifiedPreset, ) from dstack._internal.cli.utils.common import add_row_from_dict, console +from dstack._internal.core.models.presets import DEFAULT_DATASET from dstack._internal.utils.common import pretty_date, pretty_resources _STATUS_DISPLAY = { @@ -210,12 +210,12 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False if isinstance(best, dict): tps = _format_number(best["tok_s"]) if objective: - # Lead with per-user tok/s: comparable across rows regardless of concurrency. + # Lead with per-user tps: comparable across rows regardless of concurrency. tpot_ms = best.get("tpot_ms") if isinstance(tpot_ms, (int, float)) and tpot_ms > 0: - parts.append(f"tok/s/user={_format_number(1000 / tpot_ms)}") + parts.append(f"tps/user={_format_number(1000 / tpot_ms)}") if verbose: - parts.append(f"tok/s={tps}") + parts.append(f"tps={tps}") ttft_ms = best.get("ttft_ms") if isinstance(ttft_ms, (int, float)): parts.append(f"ttft={_format_duration_ms(ttft_ms)}") @@ -324,10 +324,10 @@ def format_preset_benchmark(preset: VerifiedPreset, *, verbose: bool = False) -> benchmark = preset.benchmark metrics = benchmark.metrics parts = [ - f"tok/s/user={_format_number(benchmark.effective_per_user_tok_per_s)}", + f"tps/user={_format_number(benchmark.effective_per_user_tok_per_s)}", ] if verbose: - parts.append(f"tok/s={_format_number(benchmark.effective_output_tok_per_s)}") + parts.append(f"tps={_format_number(benchmark.effective_output_tok_per_s)}") parts += [ f"ttft={_format_duration_ms(metrics.ttft_ms.p50)}", f"ctx={_format_token_count(preset.context_length)}", diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index a33d3efbf..fb9ace9f8 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -17,7 +17,6 @@ from pydantic import ValidationError from rich.text import Text -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.preset_agent import ( PresetSessionFinalize, PresetSessionProcess, @@ -30,6 +29,7 @@ from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.errors import CLIError from dstack._internal.core.models.common import validate_extra_ignore +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.utils.common import get_dstack_dir if TYPE_CHECKING: diff --git a/src/dstack/_internal/cli/services/presets/store.py b/src/dstack/_internal/cli/services/presets/store.py index a9df9756c..6787d43b4 100644 --- a/src/dstack/_internal/cli/services/presets/store.py +++ b/src/dstack/_internal/cli/services/presets/store.py @@ -7,16 +7,16 @@ import yaml from pydantic import ValidationError -from dstack._internal.cli.models.configurations import ( - MAX_PROMPT_LENGTH, - PresetConfiguration, - PresetPromptFile, -) from dstack._internal.cli.models.presets import PRESET_EXCLUDED_FIELDS, VerifiedPreset from dstack._internal.cli.services.presets.build import preset_to_yaml_dict from dstack._internal.cli.utils.common import warn from dstack._internal.core.errors import CLIError, ConfigurationError from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.presets import ( + MAX_PROMPT_LENGTH, + PresetConfiguration, + PresetPromptFile, +) from dstack._internal.utils.common import get_dstack_dir @@ -47,7 +47,7 @@ def list(self) -> list[VerifiedPreset]: warn(str(e), stderr=True) if len(earlier_version_ids) == 1: warn( - f"VerifiedPreset {earlier_version_ids[0]} was created before dstack 0.21 and cannot" + f"Preset {earlier_version_ids[0]} was created before dstack 0.21 and cannot" f" be read. Delete it with" f" [code]dstack preset delete {earlier_version_ids[0]}[/], or recreate.", stderr=True, @@ -70,7 +70,7 @@ def get(self, preset_id: str) -> VerifiedPreset | None: return None preset = self._load(path) if preset.id != preset_id: - raise CLIError(f"VerifiedPreset file {path} does not match its path") + raise CLIError(f"Preset file {path} does not match its path") return preset def save(self, preset: VerifiedPreset) -> Path: @@ -226,7 +226,7 @@ def _upgrade_pre_0_21_2_preset(data: dict, *, preset_id: str) -> dict: def _earlier_version_preset_error(preset_id: str) -> EarlierVersionPresetError: return EarlierVersionPresetError( - f"VerifiedPreset {preset_id} was created before dstack 0.21 and cannot be read." + f"Preset {preset_id} was created before dstack 0.21 and cannot be read." f" Delete it with `dstack preset delete {preset_id}`, or recreate it." ) @@ -260,13 +260,7 @@ def parse_preset_configuration(stream: TextIO) -> PresetConfiguration: try: data = yaml.safe_load(stream) if not isinstance(data, dict): - raise ConfigurationError("VerifiedPreset configuration must be a YAML object") - # Only checked here: a stored preset serializes `model` as an object, so - # the model itself has to keep accepting the form a file may not use. - model = data.get("model") - if isinstance(model, dict) and model.get("name") is None: - key = "base" if "base" in model else "repo" - raise ConfigurationError(f"Use top-level `{key}:` instead of nested `model.{key}`") + raise ConfigurationError("Preset configuration must be a YAML object") configuration = PresetConfiguration.model_validate(data) except ValidationError as e: raise ConfigurationError(e) from e diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index ebb64a8ee..3a763577b 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -5,7 +5,6 @@ from pydantic import BeforeValidator, ConfigDict, TypeAdapter, ValidationError, ValidationInfo -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.preset_agent import ( AnyPresetAgentResult, PresetAgentSuccess, @@ -31,6 +30,7 @@ from dstack._internal.core.errors import CLIError from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.envs import EnvSentinel +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.models.runs import JobStatus, Run, RunStatus diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 4fe18d981..79619e1d7 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -35,6 +35,7 @@ from dstack._internal.core.models.files import FilePathMapping from dstack._internal.core.models.fleets import FleetConfiguration from dstack._internal.core.models.gateways import GatewayConfiguration +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.models.profiles import ( ProfileParams, SpotPolicy, @@ -1428,6 +1429,7 @@ class ApplyConfigurationType(str, Enum): FLEET = "fleet" GATEWAY = "gateway" VOLUME = "volume" + PRESET = "preset" AnyApplyConfiguration = Union[ @@ -1435,6 +1437,7 @@ class ApplyConfigurationType(str, Enum): FleetConfiguration, GatewayConfiguration, AnyVolumeConfiguration, + PresetConfiguration, ] @@ -1444,6 +1447,7 @@ class ApplyConfigurationType(str, Enum): AnyRunConfiguration, FleetConfiguration, GatewayConfiguration, + PresetConfiguration, # Base configurations (further parsing required to get a concrete AnyApplyConfiguration) BaseVolumeConfiguration, ], @@ -1483,6 +1487,7 @@ def parse_apply_configuration(data: dict) -> AnyApplyConfiguration: FleetConfiguration, GatewayConfiguration, VolumeConfiguration, + PresetConfiguration, ] diff --git a/src/dstack/_internal/cli/models/configurations.py b/src/dstack/_internal/core/models/presets.py similarity index 95% rename from src/dstack/_internal/cli/models/configurations.py rename to src/dstack/_internal/core/models/presets.py index 94b1fc578..53cbed34e 100644 --- a/src/dstack/_internal/cli/models/configurations.py +++ b/src/dstack/_internal/core/models/presets.py @@ -1,6 +1,7 @@ from typing import Annotated, Any, Literal, Optional, Union from pydantic import ( + ConfigDict, Field, PositiveInt, field_validator, @@ -95,13 +96,26 @@ def validate_path(cls, value: str) -> str: return value +def _drop_model_from_required(schema: dict) -> None: + # `model` is synthesized from the top-level `base`/`repo` shorthand by a + # before-validator, which JSON Schema consumers never run. + required = [field for field in schema.get("required", []) if field != "model"] + if required: + schema["required"] = required + else: + schema.pop("required", None) + + class PresetConfiguration( ProfileParams, ): + model_config = ConfigDict(json_schema_extra=_drop_model_from_required) + type: Annotated[Literal["preset"], Field(description="The configuration type")] = "preset" + # TODO: Generate a random name when omitted, like runs and fleets do name: Annotated[ Optional[str], - Field(description="The service name. Required unless passed with `--name`"), + Field(description="The preset name"), ] = None model: Annotated[ PresetModelSpec, diff --git a/src/tests/_internal/cli/commands/test_preset.py b/src/tests/_internal/cli/commands/test_preset.py index 6e3da3237..130b6f64b 100644 --- a/src/tests/_internal/cli/commands/test_preset.py +++ b/src/tests/_internal/cli/commands/test_preset.py @@ -29,10 +29,12 @@ def _patched_create_preset(**create_kwargs): with ( patch("dstack.api.Client.from_config"), patch( - "dstack._internal.cli.commands.preset.plan_preset", + "dstack._internal.cli.services.configurators.preset.plan_preset", return_value=("fleet-a",), ), - patch("dstack._internal.cli.commands.preset.create_preset", **create_kwargs) as create, + patch( + "dstack._internal.cli.services.configurators.preset.create_preset", **create_kwargs + ) as create, ): yield create @@ -197,7 +199,7 @@ def test_lists_presets_without_api_client(self, tmp_path): # 42.1 is the aggregate, verbose-only; the default row shows per-user 1/TPOT. assert "133" in output assert "c=1" in "".join(output.split()) - assert "tok/s/user=" in "".join(output.split()) + assert "tps/user=" in "".join(output.split()) assert "ttft=" in "".join(output.split()) assert "ttft=[/]108" in "".join(output.split()) or "ttft=108" in "".join(output.split()) assert "A6000:48GB:1" not in output @@ -435,7 +437,10 @@ def test_create_without_confirmation_exits_before_creating(self, tmp_path): with ( _patched_create_preset() as create, - patch("dstack._internal.cli.commands.preset.confirm_ask", return_value=False), + patch( + "dstack._internal.cli.services.configurators.preset.confirm_ask", + return_value=False, + ), ): exit_code = run_dstack_cli( ["preset", "create", "-f", str(configuration_path)], @@ -466,7 +471,8 @@ def test_create_always_asks_even_without_a_name_conflict(self, tmp_path): with ( _patched_create_preset() as create, patch( - "dstack._internal.cli.commands.preset.confirm_ask", return_value=False + "dstack._internal.cli.services.configurators.preset.confirm_ask", + return_value=False, ) as confirm, ): exit_code = run_dstack_cli( @@ -478,3 +484,93 @@ def test_create_always_asks_even_without_a_name_conflict(self, tmp_path): assert exit_code == 0 confirm.assert_called_once_with("Create the preset?") create.assert_not_called() + + +class TestApplyPresetConfiguration: + _CONFIGURATION = """type: preset +name: file-name +base: Qwen/Qwen3.5-27B +trials: 1 +concurrency: 8 +max_ttft: 5000 +min_context_length: 8192 +""" + + def _write_configuration(self, tmp_path): + path = tmp_path / "preset.dstack.yml" + path.write_text(self._CONFIGURATION) + return path + + def test_creates_the_preset(self, tmp_path): + configuration_path = self._write_configuration(tmp_path) + + with _patched_create_preset() as create: + exit_code = run_dstack_cli( + ["apply", "-y", "-f", str(configuration_path)], + home_dir=tmp_path, + repo_dir=tmp_path, + ) + + assert exit_code == 0 + configuration = create.call_args.kwargs["configuration"] + assert configuration.name == "file-name" + assert configuration.model.base == "Qwen/Qwen3.5-27B" + + def test_accepts_creation_and_profile_arguments(self, tmp_path): + configuration_path = self._write_configuration(tmp_path) + + with _patched_create_preset() as create: + exit_code = run_dstack_cli( + [ + "apply", + "-y", + "-f", + str(configuration_path), + "--name", + "cli-name", + "--trials", + "7", + "--backend", + "gcp", + "--debug", + ], + home_dir=tmp_path, + repo_dir=tmp_path, + ) + + assert exit_code == 0 + configuration = create.call_args.kwargs["configuration"] + assert configuration.name == "cli-name" + assert configuration.trials == 7 + assert configuration.backends == ["gcp"] + assert create.call_args.kwargs["debug"] is True + + def test_rejects_detach(self, tmp_path, capsys): + configuration_path = self._write_configuration(tmp_path) + + with _patched_create_preset() as create: + exit_code = run_dstack_cli( + ["apply", "-y", "-d", "-f", str(configuration_path)], + home_dir=tmp_path, + repo_dir=tmp_path, + ) + + assert exit_code == 1 + create.assert_not_called() + captured = capsys.readouterr() + assert "--detach" in captured.out + captured.err + + def test_rejects_unknown_arguments(self, tmp_path, capsys): + configuration_path = self._write_configuration(tmp_path) + + with _patched_create_preset() as create: + exit_code = run_dstack_cli( + ["apply", "-y", "-f", str(configuration_path), "--nonsense"], + home_dir=tmp_path, + repo_dir=tmp_path, + ) + + assert exit_code == 1 + create.assert_not_called() + captured = capsys.readouterr() + assert "Unrecognized arguments: --nonsense" in captured.out + captured.err diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index c925dca80..081eb8018 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -10,7 +10,6 @@ from rich.theme import Theme from dstack._internal.cli.main import main -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.preset_agent import ( PresetAgentSuccess, PresetSessionFinalize, @@ -29,6 +28,7 @@ ServiceConfiguration, ) from dstack._internal.core.models.instances import Disk, Gpu, Resources +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.models.resources import ResourcesSpec from dstack._internal.core.models.runs import JobStatus, Run, RunStatus, ServiceSpec diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index b353f1bba..742a85373 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -13,7 +13,6 @@ import pytest import yaml -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.preset_agent import PresetSessionProcess from dstack._internal.cli.services.presets.agent import ( ClaudeAuth, @@ -49,6 +48,7 @@ ) from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.errors import CLIError +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.services.configs import ConfigManager from tests._internal.cli.common import get_session_run, get_session_state diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index fea2b50b3..ba3098dc5 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -7,7 +7,6 @@ import pytest from pydantic import ValidationError -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.preset_agent import ( PresetSessionFinalize, PresetSessionProcess, @@ -52,6 +51,7 @@ ) from dstack._internal.core.errors import CLIError from dstack._internal.core.models.envs import EnvSentinel +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.models.runs import Run, RunStatus from tests._internal.cli.common import ( get_preset, @@ -449,7 +449,9 @@ def _args(self, previous): return args def test_flag_overrides_and_property_stands_without_it(self): - from dstack._internal.cli.commands.preset import _get_effective_configuration + from unittest.mock import MagicMock + + from dstack._internal.cli.services.configurators.preset import PresetConfigurator def configuration(): # A fresh object per call: the merger mutates its input. @@ -457,10 +459,9 @@ def configuration(): name="qwen", base="Qwen/Qwen3.5-27B", previous=["from-config"] ) - overridden = _get_effective_configuration( - configuration(), self._args(["from-flag"]), require_name=False - ) - kept = _get_effective_configuration(configuration(), self._args(None), require_name=False) + configurator = PresetConfigurator(api_client=MagicMock()) + overridden = configurator.apply_args(configuration(), self._args(["from-flag"])) + kept = configurator.apply_args(configuration(), self._args(None)) assert overridden.previous == ["from-flag"] assert kept.previous == ["from-config"] diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index 152f5feb6..de71b9ffb 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -35,7 +35,7 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): output = output_module.format_preset_benchmark(preset, verbose=True) # Per-user speed is 1/TPOT (p50 7.4ms), not the aggregate over concurrency. - assert output.startswith("tok/s/user=133 ") + assert output.startswith("tps/user=133 ") assert output_module.format_preset_objective(preset).startswith("[secondary]io=1K/128 ") assert "ctx=32K" in output assert "ttft=8.15s" in output @@ -152,7 +152,7 @@ def test_shows_progress_after_status_and_best_benchmark(self): assert row["STATUS"] == "[bold sea_green3]trialing[/] [secondary](3/3)[/]" # Per-user speed is 1/TPOT, the same definition the preset row uses — not # the aggregate over concurrency, which would read 292 here. - assert row["BENCHMARK"].startswith("tok/s/user=292") + assert row["BENCHMARK"].startswith("tps/user=292") assert row["RESOURCES"] == "A40:48GB:1" def test_shows_the_shared_prefix_when_the_workload_has_one(self): @@ -440,7 +440,7 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): assert row["RESOURCES"] == "MI300X:192GB:1" # Marked, not just dimmed, so it does not read as a met constraint where # styling is absent or invisible to the reader. - assert "*tok/s/user" in row["BENCHMARK"] + assert "*tps/user" in row["BENCHMARK"] class TestFailedTrialSpark: diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index ad3deaa40..18640a06c 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -5,12 +5,12 @@ import pytest import yaml -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.services.presets import store as store_module from dstack._internal.cli.services.presets.store import PresetStore from dstack._internal.core.errors import ConfigurationError from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.files import FilePathMapping +from dstack._internal.core.models.presets import PresetConfiguration from tests._internal.cli.common import get_preset pytestmark = pytest.mark.windows @@ -320,11 +320,13 @@ def test_preserves_literal_env_values(self, tmp_path: Path): class TestParsePresetConfiguration: @pytest.mark.parametrize("key", ["base", "repo"]) - def test_rejects_nested_model_without_name(self, key: str): + def test_accepts_nested_model_without_name(self, key: str): + # The nested form is what stored presets use; user files may use it too. stream = StringIO(f"type: preset\nmodel:\n {key}: Qwen/Qwen3.5-27B\n") - with pytest.raises(ConfigurationError, match=f"`{key}:`"): - store_module.parse_preset_configuration(stream) + configuration = store_module.parse_preset_configuration(stream) + + assert getattr(configuration.model, key) == "Qwen/Qwen3.5-27B" def test_accepts_nested_model_with_name(self): stream = StringIO( diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index 70b9734dc..018254351 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -3,7 +3,6 @@ import pytest from pydantic import ValidationError -from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.models.preset_agent import AnyPresetAgentResult from dstack._internal.cli.services.presets.agent import ( PresetAgentProcessOutput, @@ -20,6 +19,7 @@ from dstack._internal.core.models.common import validate_extra_ignore from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.files import FilePathMapping +from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.models.profiles import ProfileParams from tests._internal.cli.common import ( get_preset, diff --git a/src/tests/_internal/cli/models/test_configurations.py b/src/tests/_internal/core/models/test_presets.py similarity index 89% rename from src/tests/_internal/cli/models/test_configurations.py rename to src/tests/_internal/core/models/test_presets.py index c54045163..f6b1d553c 100644 --- a/src/tests/_internal/cli/models/test_configurations.py +++ b/src/tests/_internal/core/models/test_presets.py @@ -1,7 +1,7 @@ import pytest from pydantic import ValidationError -from dstack._internal.cli.models.configurations import ( +from dstack._internal.core.models.presets import ( PresetConfiguration, PresetModelBase, PresetModelRepo, @@ -101,3 +101,13 @@ def test_defaults_to_the_random_dataset(self): assert configuration.dataset is None assert configuration.effective_dataset == "random" + + +class TestPresetConfigurationSchema: + def test_schema_does_not_require_model(self): + # `model` is filled from the `base`/`repo` shorthand by a before-validator, + # which JSON Schema consumers (IDEs) never run. + schema = PresetConfiguration.model_json_schema() + assert "model" not in schema.get("required", []) + for field in ("model", "base", "repo"): + assert field in schema["properties"] From 51511276da28e65be4c0af728fd66b8878655a87 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Mon, 17 Aug 2026 15:46:38 +0200 Subject: [PATCH 3/3] [Presets] Fix preset list typing pyright reported `_filter_presets` reads `base` and `model`, which only `VerifiedPreset` carries, and every caller passes verified lists; the annotations said `Preset`. Co-Authored-By: Claude Fable 5 --- src/dstack/_internal/cli/commands/preset.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py index ee30e9ede..7e630b9a4 100644 --- a/src/dstack/_internal/cli/commands/preset.py +++ b/src/dstack/_internal/cli/commands/preset.py @@ -14,6 +14,7 @@ from dstack._internal.cli.models.presets import ( Preset, PresetListOutput, + VerifiedPreset, ) from dstack._internal.cli.services.completion import ProjectNameCompleter from dstack._internal.cli.services.configurators import APPLY_STDIN_NAME @@ -250,7 +251,7 @@ def _list(self, args: argparse.Namespace) -> None: def _list_presets_and_sessions( self, *, base: str | None, repo: str | None - ) -> tuple[list[Preset], list[dict]]: + ) -> tuple[list[VerifiedPreset], list[dict]]: self._reconcile() presets = PresetStore().list() sessions = list_preset_sessions() @@ -458,11 +459,11 @@ def _add_list_args(parser: argparse.ArgumentParser) -> None: def _filter_presets( - presets: list[Preset], + presets: list[VerifiedPreset], *, base: str | None, repo: str | None, -) -> list[Preset]: +) -> list[VerifiedPreset]: return [ preset for preset in presets