From 002480a22172ee5848d5315b2cbb5c3bed243ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= <12143866+ondrajodas@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:47:52 +0200 Subject: [PATCH 1/5] docs: add design spec for manage:migrate-orchestrations-to-flow (AJDA-3117) --- ...0-migrate-orchestrations-to-flow-design.md | 301 ++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md diff --git a/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md b/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md new file mode 100644 index 0000000..127562e --- /dev/null +++ b/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md @@ -0,0 +1,301 @@ +# Design: manage:migrate-orchestrations-to-flow (AJDA-3117) + +## Goal + +Add a `cli-utils` command that drives the automated `keboola.orchestrator` → `keboola.flow` +migration across a batch of projects on one stack. The command is a **driver only**: it creates +and supervises `keboola.flow-migration-tool` jobs in customer projects. All migration logic lives +in the component (`keboola/flow-migration-tool`); nothing from it is reimplemented here. + +Per-stack migrations (AJDA-3119 Azure NE, AJDA-3120 GCP US) then become a single supervised run. + +## Verified SDK facts (read from `vendor/`, versions from `composer.lock`) + +All packages needed are already installed — **no composer changes required**: + +| Package | Version | What we use | +|---|---|---| +| `keboola/job-queue-api-php-client` | 5.2.0 | `Client::__construct(string $publicApiUrl, string $storageToken, array $options = [])`; `createJob(JobData): DTO\Job`; `getJob(string $jobId): DTO\Job`; `listJobs(ListJobsOptions): array` (elements are `DTO\Job`); `JobData::__construct(string $componentId, ?string $configId = null, array $configData = [], string $mode = 'run', ...)`; `ListJobsOptions::setComponents(array)/setStatuses(array)/setLimit(int)`; `JobStatuses` enum (`CREATED`, `WAITING`, `PROCESSING`, `TERMINATING`, ... `SUCCESS`, `ERROR`, `WARNING`, `TERMINATED`, `CANCELLED`); `DTO\Job` readonly props: `id`, `status`, `isFinished`, `durationSeconds`, `result`, `url` | +| `keboola/service-client` | 1.5.1 | `new ServiceClient(string $hostnameSuffix)`; `getQueueUrl()` → `https://queue.` | +| `keboola/kbc-manage-api-php-client` | v7.1.1 | `getProject($id)`; `createProjectStorageToken($projectId, array $params)` — generic POST pass-through, accepts `expiresIn`, `canManageBuckets`, `canReadAllFileUploads`, `componentAccess`, `description` | +| `keboola/storage-api-client` | v18.7.0 | `Components::listComponentConfigurations(ListComponentConfigurationsOptions)` with `setComponentId()`/`setIsDeleted(false)` | + +Notes: +- `Client::waitForJobCompletion()` exists but blocks on a single job — unusable for a concurrency + window; we poll with `getJob()` ourselves. +- `listJobs()` has no typed return; the batch runner never touches list elements — the running-job + guard only needs `$jobs !== []` (avoids the `$job['id']`-on-DTO trap present in + `QueueMassTerminateJobs`). +- `DTO\Job::fromApiResponse()` requires many keys — fakes in tests will construct results through + it with a full response fixture, or the fake client returns pre-built `Job` instances. + +## Command + +``` +php cli.php manage:migrate-orchestrations-to-flow [-f|--force] [] + [--projects-file=PATH] [--concurrency=10] [--poll-interval=5] [--report=PATH] +``` + +### Arguments + +| Argument | Type | Description | +|---|---|---| +| `token` | REQUIRED | Manage API token | +| `url` | REQUIRED | Stack URL incl. scheme, e.g. `https://connection.north-europe.azure.keboola.com` | +| `projects` | OPTIONAL | Comma-separated project IDs, or `@path/to/file` (one ID per line) | + +`token` and `url` come first to stay compatible with `manage:call-on-stacks` +(`AllStacksIterator` builds ` `). + +### Options + +| Option | Default | Description | +|---|---|---| +| `-f`, `--force` | off | Real migration (`parameters.dryRun: false`). Without it, jobs run with `dryRun: true` | +| `--projects-file=PATH` | — | Alternative to `@file` in the argument | +| `--concurrency=N` | 10 | Max migration jobs in flight | +| `--poll-interval=N` | 5 | Seconds between poll sweeps | +| `--report=PATH` | `flow-migration--.csv` | CSV report path | + +The component's `parameters.migrate.*` sub-flags are **not** exposed: the command always requests +a full migration and relies on the component defaults. + +**Important semantic difference from the usual cli-utils dry-run:** even without `--force` the +command creates a *real* `keboola.flow-migration-tool` job (with `dryRun: true`) in every eligible +project and creates a real ephemeral storage token. The command prints a prominent notice about +this at startup, and the README documents it (relevant for PAYGO billing — see hand-off items). + +### Input resolution and validation + +- Exactly one source of project IDs must be given: the `projects` argument (inline list or + `@file`) or `--projects-file`. Both, or neither → error message + exit 1. +- File format: one ID per line; blank lines and lines starting with `#` are ignored. +- Every ID must pass `ctype_digit()`; any invalid entry → error naming the offending value, exit 1. +- Duplicates are removed (first occurrence wins) so a re-run with a sloppy list cannot double-submit. +- `--concurrency` ≥ 1, `--poll-interval` ≥ 1, both integers; otherwise exit 1. +- `url` must parse to a host beginning with `connection.`; the hostname suffix for `ServiceClient` + is that host minus the `connection.` prefix (e.g. `north-europe.azure.keboola.com`). This keeps + the issue-mandated full-URL argument *and* resolves the Queue API URL via `keboola/service-client` + (no `connection` → `queue` string replace on the URL). + +## Architecture + +Four small classes in `src/Keboola/Console/Command/` (PSR-0: namespace +`Keboola\Console\Command`, path = file name), plus registration in `cli.php`: + +``` +MigrateOrchestrationsToFlow (Symfony Command — thin shell) + ├─ parses/validates input, resolves project ID list + ├─ builds ManageApi\Client, ServiceClient, FlowMigrationProjectClientsFactory + ├─ opens the CSV report (append mode, header if new/empty) and wires the + │ per-result callback: CSV row + progress line to stdout + ├─ runs FlowMigrationBatchRunner + └─ prints final summary, returns exit code (1 if any project failed) + +FlowMigrationBatchRunner (plain class — ALL batch logic, unit-tested) + ├─ per-project pipeline (skip rules, job submission) + ├─ concurrency window + polling loop + └─ emits one FlowMigrationProjectResult per input project via callback, + returns aggregate summary counts + +FlowMigrationProjectClientsFactory (plain class — the only network seam) + ├─ getProject(string $projectId): array (Manage API) + └─ createProjectClients(string $projectId): FlowMigrationProjectClients + creates the ephemeral storage token, returns Components + JobQueueClient + bound to that token + +FlowMigrationProjectClients (tiny readonly DTO: Components + JobQueueClient) +FlowMigrationProjectResult (tiny readonly DTO: projectId, jobId, status, + durationSeconds, error + isFailed()) +``` + +Rationale: the repo's testable-logic pattern (`DataAppOrchestratorTaskMigrator` + +`FakeComponents`) extended one step — because per-project clients are created with per-project +ephemeral tokens, the runner cannot receive clients directly; it receives a factory. Tests +subclass the factory and the SDK clients without calling parent constructors (exactly how +`FakeComponents` already works). No interfaces — the codebase does not use them. + +### Ephemeral token (created per project, before any Storage/Queue call) + +```php +$manageClient->createProjectStorageToken($projectId, [ + 'description' => 'AJDA-3117 keboola.orchestrator -> keboola.flow migration (batch driver)', + 'expiresIn' => 43200, // 12 h: job may wait in queue and runs long; expiry mid-migration is worse than a short-lived privileged token + 'canManageBuckets' => true, + 'canReadAllFileUploads' => true, + 'componentAccess' => [ + 'keboola.orchestrator', + 'keboola.flow', + 'keboola.scheduler', + 'keboola.flow-migration-tool', + ], +]); +``` + +Broad rights are deliberate (trigger/notification migration touches project-level resources); +the token expires on its own, no cleanup step. + +### Per-project pipeline (inside the runner, at submission time) + +1. `getProject()` — `isDisabled` → result `skipped-disabled`. Manage API 404 (deleted project) + → also `skipped-disabled` (issue counts disabled+deleted together). Other Manage errors → + `error` result (project failed, batch continues). +2. Create ephemeral token + per-project clients via the factory. Failure → `error` result. +3. `listComponentConfigurations(componentId: keboola.orchestrator, isDeleted: false)` — + empty → `skipped-no-orchestrations` (no job created; avoids hundreds of empty jobs in + customers' job history). +4. Queue guard: `listJobs(components: [keboola.flow-migration-tool], statuses: [CREATED, + WAITING, PROCESSING, TERMINATING], limit: 1)` — non-empty → `skipped-job-running`. + (`TERMINATING` added on top of the issue's three: a terminating job may still be executing + migration writes, and skipping it strictly reduces overlap risk.) +5. `createJob(new JobData('keboola.flow-migration-tool', configData: [...]))` — via `configData`, + so no stored configuration is left behind in the project: + + ```json + { + "parameters": { + "mode": "project", + "orchestrationIds": [], + "skipBroken": true, + "dryRun": + } + } + ``` + + (`orchestrationIds: []` and `skipBroken: true` are required by the component's config + definition in `project` mode.) Job enters the in-flight window with its own `JobQueueClient`. + +### Concurrency window + polling + +``` +pending = input project queue +inFlight = projectId → {jobId, queueClient, startedAtWallClock, consecutivePollFailures} + +while pending not empty or inFlight not empty: + fill: while |inFlight| < concurrency and pending: submit next + (skips/errors resolve immediately → result callback, do not occupy a slot) + if inFlight empty: continue + sleep(pollInterval) # injected callable, no-op in tests + for each inFlight job: getJob() + finished → result callback (terminal status), free the slot + poll exception → consecutivePollFailures++; after 3 consecutive failures + mark project error ("job still running server-side, polling gave up"), + free the slot; a successful poll resets the counter +``` + +- `durationSeconds` in the result: `Job->durationSeconds` when the API provides it, otherwise + wall-clock from submission. +- No per-job or global timeout (operator supervises; token bounds the run at 12 h anyway). +- No signal handling — the incrementally-appended CSV already makes an interrupted run auditable. +- `sleep` is injected as a `callable` (default `sleep(...)`) so unit tests run instantly and can + assert poll cadence. + +### CSV report + +- Path from `--report`, default `flow-migration--.csv` in cwd. +- Opened in append mode; header written only when the file is new or empty. +- Written with `fputcsv(..., separator: ';')` — error messages containing `;`/newlines get quoted + correctly; no new dependency. +- Header + one row **per input project** (including skips, for a complete audit of the list): + + ``` + projectId;jobId;status;durationSeconds;error + ``` + + `status` ∈ job terminal status (`success`, `warning`, `error`, `terminated`, `cancelled`) + or `skipped-disabled` | `skipped-no-orchestrations` | `skipped-job-running` | `error`. + `jobId` is empty for rows without a job. Rows are appended the moment each project resolves. + +### Progress output and summary + +- One stdout line per event (`[HH:ii:ss]` prefix): job submitted (with job id + job URL), project + skipped (with reason), project finished (status + duration), poll warnings. +- Final summary: attempted / migrated (job `success`; `warning` reported as migrated-with-warning) + / skipped-no-orchestrations / skipped-disabled / skipped-job-running / failed + (job `error`|`terminated`|`cancelled`, or driver-side error). +- **Exit code 1 if at least one project failed, else 0.** A failing project never aborts the batch. + +### Re-runs + +Re-running the same list is the intended recovery path: the component's +`AlreadyMigratedValidator` reports already-migrated orchestrations as `skipped`, and the queue +guard skips projects with a live migration job. No resume state is kept by the driver. + +## Error handling summary + +| Failure | Behavior | +|---|---| +| Invalid input (IDs, options, both/neither project sources) | message + exit 1, nothing executed | +| `getProject` 404 | `skipped-disabled` (deleted) | +| `getProject` other error, token creation, config listing, guard, `createJob` failure | `error` result for that project, batch continues | +| Poll failure | tolerated 3 consecutive times per job, then `error` result; job keeps running server-side | +| Job terminal `error`/`terminated`/`cancelled` | `failed` in summary, exit code 1 | + +## Testing + +`tests/FlowMigrationBatchRunnerTest.php` + fakes (PSR-4 `Keboola\Console\Tests\`): + +- `FakeJobQueueClient extends JobQueueClient\Client` — constructor override (no parent call, same + trick as `FakeComponents`), records `createJob` calls, scripted `getJob` status sequences + (e.g. `processing, processing, success`), scripted `listJobs` guard responses, can throw on + demand for poll-failure tests. +- `FakeFlowMigrationProjectClientsFactory extends FlowMigrationProjectClientsFactory` — scripted + projects (disabled / deleted / erroring), records `createProjectClients` calls (asserts no token + is created for disabled projects), returns `FlowMigrationProjectClients` built from + `FakeComponents` (reused as-is for the orchestrator-config listing) + `FakeJobQueueClient`. + +Scenarios (assert emitted results, summary counts, recorded API calls, sleep-callable cadence): +1. happy path — N projects, jobs created with exact `configData` (incl. `dryRun` true/false by + force flag), results in completion order; +2. all three skip rules, each without a job being created (and without a token for disabled); +3. concurrency window never exceeds the limit and refills as jobs finish; +4. one project's job ends `error` → batch continues, summary flags failure; +5. driver-side error (token creation throws) → `error` result, batch continues; +6. poll failures: 2 consecutive then success → no failure; 3 consecutive → `error` result; +7. duplicate project IDs in input are submitted once. + +The Symfony command shell itself is not unit-tested (repo convention); `composer phpcs`, +`composer phpstan` (level 9), `composer tests` via `docker compose` must pass. + +## Documentation + +README.md, section "Project manipulation", after "Migrate data-apps orchestrator/flow tasks…": +usage line, Arguments/Options, Behavior — including the explicit warning that dry-run still +creates real jobs and real ephemeral tokens in customer projects, the CSV format, re-run safety, +and exit-code semantics. + +## Decisions made without the reporter (recorded, with rationale) + +1. **`url` stays a full connection URL; hostname suffix is derived** (strip `connection.` from the + parsed host) — keeps `manage:call-on-stacks` compatibility and the issue's signature while + using `ServiceClient` for the Queue URL. +2. **`TERMINATING` added to the guard statuses** — a terminating job may still write; strictly safer. +3. **CSV gets a row for every input project including skips** — makes the report a complete audit; + the issue's "resolvable job ID for every project" holds for every project that got a job. +4. **`warning` terminal status counts as migrated** (reported distinctly) — the component finished; + exit code stays 0. The CSV carries the raw status either way. +5. **Poll failures tolerated 3× consecutively, then the project is marked failed** — an unbounded + retry could hang the batch forever; the client already retries 5xx internally 3×. +6. **Duplicates deduplicated, `#`-comment lines allowed in the projects file** — hundreds-of-IDs + lists are hand-assembled; cheap robustness. +7. **No interactive confirmation** — unlike the `all` mode of the data-apps migration command, + the blast radius here is always an explicit project list. +8. **`fputcsv` over `keboola/csv`** — append semantics with header-once needs a plain handle; + no new dependency. +9. **English spec/plan/docs** — repo and git content are English per user's git conventions. + +## Out of scope (YAGNI) + +- No `all` projects mode — per-stack batches are driven from explicit lists (AJDA-3119/3120). +- No exposure of `parameters.migrate.*` sub-flags. +- No resume file/state beyond the CSV; no signal handling; no per-job timeout. +- No per-orchestration counts in the CSV — the component does not expose them in the job result + (issue open question; possible follow-up in `keboola/flow-migration-tool`). + +## Hand-off items (require live stack access — not implementation work) + +The five "Must verify before the first live batch" items from AJDA-3117 (billing exclusion on +PAYGO, notification listing visibility for ephemeral tokens, trigger `runWithTokenId` permission, +token lifetime vs. queue wait, `keboola.flow` availability without a per-project feature) plus +acceptance criterion 9 must be confirmed by Ondrej on a live project and recorded in a Linear +comment. The command itself is designed so these checks can run as a one-project batch first. From 08de81baa9ab995c041dfc4e24428e4919ac4646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= <12143866+ondrajodas@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:01:51 +0200 Subject: [PATCH 2/5] docs: add implementation plan for manage:migrate-orchestrations-to-flow (AJDA-3117) --- ...26-08-10-migrate-orchestrations-to-flow.md | 2154 +++++++++++++++++ 1 file changed, 2154 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-10-migrate-orchestrations-to-flow.md diff --git a/docs/superpowers/plans/2026-08-10-migrate-orchestrations-to-flow.md b/docs/superpowers/plans/2026-08-10-migrate-orchestrations-to-flow.md new file mode 100644 index 0000000..d42b6e4 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-migrate-orchestrations-to-flow.md @@ -0,0 +1,2154 @@ +# manage:migrate-orchestrations-to-flow Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `cli-utils` command that drives the automated `keboola.orchestrator` → `keboola.flow` migration across a batch of projects on one stack by creating and supervising `keboola.flow-migration-tool` jobs (AJDA-3117). + +**Architecture:** A thin Symfony Command (`MigrateOrchestrationsToFlow`) wires input parsing, CSV reporting, and summary output around a fully unit-tested plain class (`FlowMigrationBatchRunner`) that owns all batch logic: per-project skip rules, job submission, a bounded concurrency window, and polling. The only network seam is `FlowMigrationProjectClientsFactory` (Manage API + per-project ephemeral-token clients), which tests replace with fakes that skip the parent constructor — the same trick `tests/FakeComponents.php` already uses. + +**Tech Stack:** PHP 8.3, Symfony Console 7.4, `keboola/kbc-manage-api-php-client` v7.1.1, `keboola/job-queue-api-php-client` 5.2.0, `keboola/service-client` 1.5.1, `keboola/storage-api-client` v18.7.0, PHPUnit 11. **No composer changes needed — everything is already installed.** + +**Spec:** `docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md` + +## Global Constraints + +- All commands run via `docker compose run --rm dev ...` (dev image has a bind mount; run `docker compose run --rm dev composer install` once first). +- PSR-0 autoloading: class `Keboola\Console\Command\X` **must** live at `src/Keboola/Console/Command/X.php`. Tests are PSR-4: `Keboola\Console\Tests\` → `tests/`. +- Commands are registered manually in `cli.php` — no auto-discovery. +- phpstan level 9 must stay clean (`composer phpstan`, covers `src/` only). +- PSR-2 must stay clean repo-wide: `./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` (CI checks `tests/` too). Keep lines under 120 chars. +- Argument order ` ...` is mandatory (compatibility with `manage:call-on-stacks`). +- Dry-run by default behind `-f`/`--force` — but note the twist: here "dry-run" still creates real jobs (with `parameters.dryRun: true`) and real ephemeral tokens. +- Do **not** use constructor property promotion or `readonly` — `src/` uses classic properties exclusively and phpcs PSR-2 is configured for that style. +- Git commits: conventional format, English, **no AI attribution of any kind**. +- Exact strings that must be used verbatim: + - command name: `manage:migrate-orchestrations-to-flow` + - token description: `AJDA-3117 keboola.orchestrator to keboola.flow migration (batch driver)` + - token: `expiresIn` 43200, `canManageBuckets` true, `canReadAllFileUploads` true, `componentAccess` = `keboola.orchestrator`, `keboola.flow`, `keboola.scheduler`, `keboola.flow-migration-tool` + - job payload parameters: `{"mode": "project", "orchestrationIds": [], "skipBroken": true, "dryRun": }` + - CSV header: `projectId;jobId;status;durationSeconds;error` (`;` delimiter) + +**Verified SDK facts (do not re-derive, they were read from `vendor/`):** +- `Keboola\JobQueueClient\Client::__construct(string $publicApiUrl, string $storageToken, array $options = [])`; `createJob(JobData): DTO\Job`; `getJob(string): DTO\Job`; `listJobs(ListJobsOptions): array` (native `array` return type — safe to compare `!== []`). +- `Keboola\JobQueueClient\JobData::__construct(string $componentId, ?string $configId = null, array $configData = [], string $mode = 'run', ...)`; `getArray()` keys: `component`, `config`, `mode`, `configRowIds`, `tag`, `branchId`, `orchestrationJobId`, `parentRunId`, `configData`. +- `Keboola\JobQueueClient\DTO\Job` is `readonly` with private constructor; build instances via `Job::fromApiResponse(array)` — it reads **all** of these keys without `??`: `id, runId, parentRunId, project, token, status, desiredStatus, mode, component, config, configData, configRowIds, tag, createdTime, startTime, endTime, durationSeconds, result, usageData, isFinished, url, branchId, variableValuesId, variableValuesData, backend, executor, metrics, behavior, parallelism, type, orchestrationJobId, orchestrationTaskId, onlyOrchestrationTaskIds, previousJobId`. `project` needs `['id' => string]`, `token` needs `['id' => string, 'description' => ?string]`; `variableValuesData`, `backend`, `behavior` accept `[]`. +- `Keboola\JobQueueClient\JobStatuses` is a string-backed enum: `CREATED, PROCESSING, TERMINATING, TERMINATED, WAITING, SUCCESS, ERROR, WARNING, CANCELLED`. +- `Keboola\ServiceClient\ServiceClient::__construct(string $hostnameSuffix)`; `getQueueUrl(): string` returns `https://queue.`. +- `Keboola\ManageApi\Client::getProject($id)` and `createProjectStorageToken($projectId, array $params)` have **no declared return types** (implicit mixed) — direct offset access like `$tokenInfo['token']` passes phpstan level 9 (proven by the identical pattern in `MigrateDataAppsOrchestratorTasks.php:199`). Do not add `is_array()` guards on their results — phpstan would not flag either way, and the existing code style omits them. +- `Keboola\ManageApi\ClientException` extends `\Exception`; HTTP status is available via `getCode()`. + +--- + +### Task 1: Result and clients DTOs + +**Files:** +- Create: `src/Keboola/Console/Command/FlowMigrationProjectResult.php` +- Create: `src/Keboola/Console/Command/FlowMigrationProjectClients.php` +- Test: `tests/FlowMigrationProjectResultTest.php` + +**Interfaces:** +- Consumes: `Keboola\JobQueueClient\JobStatuses` (vendor enum), `Keboola\StorageApi\Components`, `Keboola\JobQueueClient\Client` (vendor classes). +- Produces: + - `FlowMigrationProjectResult::__construct(string $projectId, ?string $jobId, string $status, ?int $durationSeconds, ?string $error)` with public typed properties `$projectId`, `$jobId`, `$status`, `$durationSeconds`, `$error`; methods `isSkipped(): bool`, `isFailed(): bool`; constants `STATUS_SKIPPED_DISABLED = 'skipped-disabled'`, `STATUS_SKIPPED_NO_ORCHESTRATIONS = 'skipped-no-orchestrations'`, `STATUS_SKIPPED_JOB_RUNNING = 'skipped-job-running'`, `STATUS_ERROR = 'error'`. + - `FlowMigrationProjectClients::__construct(Components $components, Client $queueClient)` with public typed properties `$components`, `$queueClient`. + +- [ ] **Step 1: Install dependencies (once)** + +Run: `docker compose run --rm dev composer install` +Expected: exits 0, `vendor/` present. + +- [ ] **Step 2: Write the failing test** + +Create `tests/FlowMigrationProjectResultTest.php`: + +```php +assertSame($expectedSkipped, $result->isSkipped()); + $this->assertSame($expectedFailed, $result->isFailed()); + } + + /** + * @return iterable + */ + public static function provideStatuses(): iterable + { + yield 'job success is neither skipped nor failed' => ['success', false, false]; + yield 'job warning counts as migrated, not failed' => ['warning', false, false]; + yield 'job error is failed' => ['error', false, true]; + yield 'job terminated is failed' => ['terminated', false, true]; + yield 'job cancelled is failed' => ['cancelled', false, true]; + yield 'skipped disabled' => [FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, true, false]; + yield 'skipped no orchestrations' => [FlowMigrationProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, true, false]; + yield 'skipped job running' => [FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING, true, false]; + } +} +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationProjectResultTest.php` +Expected: FAIL — `Class "Keboola\Console\Command\FlowMigrationProjectResult" not found` + +- [ ] **Step 4: Write the implementation** + +Create `src/Keboola/Console/Command/FlowMigrationProjectResult.php`: + +```php +projectId = $projectId; + $this->jobId = $jobId; + $this->status = $status; + $this->durationSeconds = $durationSeconds; + $this->error = $error; + } + + public function isSkipped(): bool + { + return in_array($this->status, [ + self::STATUS_SKIPPED_DISABLED, + self::STATUS_SKIPPED_NO_ORCHESTRATIONS, + self::STATUS_SKIPPED_JOB_RUNNING, + ], true); + } + + public function isFailed(): bool + { + // Anything that is not a skip and not a successful terminal job status is a failure — + // unexpected statuses fail loud rather than passing silently. + return !$this->isSkipped() + && !in_array($this->status, [JobStatuses::SUCCESS->value, JobStatuses::WARNING->value], true); + } +} +``` + +Create `src/Keboola/Console/Command/FlowMigrationProjectClients.php`: + +```php +components = $components; + $this->queueClient = $queueClient; + } +} +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationProjectResultTest.php` +Expected: PASS (8 tests) + +- [ ] **Step 6: Static analysis and code style** + +Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` +Expected: both exit 0. + +- [ ] **Step 7: Commit** + +```bash +git add src/Keboola/Console/Command/FlowMigrationProjectResult.php \ + src/Keboola/Console/Command/FlowMigrationProjectClients.php \ + tests/FlowMigrationProjectResultTest.php +git commit -m "feat: add flow migration result and per-project clients DTOs" +``` + +--- + +### Task 2: FlowMigrationProjectClientsFactory (network seam) + +**Files:** +- Create: `src/Keboola/Console/Command/FlowMigrationProjectClientsFactory.php` + +**Interfaces:** +- Consumes: `FlowMigrationProjectClients` (Task 1), `Keboola\ManageApi\Client`, `Keboola\StorageApi\Client`, `Keboola\StorageApi\Components`, `Keboola\JobQueueClient\Client`. +- Produces: + - `FlowMigrationProjectClientsFactory::__construct(Keboola\ManageApi\Client $manageClient, string $connectionUrl, string $queueApiUrl)` + - `getProject(string $projectId): array` — Manage API project detail (throws `Keboola\ManageApi\ClientException`, 404 = deleted project) + - `createProjectClients(string $projectId): FlowMigrationProjectClients` — creates the ephemeral token and both clients + +This class is pure network wiring with no branching logic — it is **not** unit-tested (would only test the mock). It is replaced by a fake in Task 3 and its constants are asserted indirectly through the command smoke test in Task 7. Both public methods must stay non-final and overridable. + +- [ ] **Step 1: Write the implementation** + +Create `src/Keboola/Console/Command/FlowMigrationProjectClientsFactory.php`: + +```php +manageClient = $manageClient; + $this->connectionUrl = $connectionUrl; + $this->queueApiUrl = $queueApiUrl; + } + + /** + * @return array Manage API project detail + */ + public function getProject(string $projectId): array + { + return $this->manageClient->getProject($projectId); + } + + public function createProjectClients(string $projectId): FlowMigrationProjectClients + { + $tokenInfo = $this->manageClient->createProjectStorageToken($projectId, [ + 'description' => self::TOKEN_DESCRIPTION, + 'expiresIn' => self::TOKEN_EXPIRES_IN_SECONDS, + 'canManageBuckets' => true, + 'canReadAllFileUploads' => true, + 'componentAccess' => self::TOKEN_COMPONENT_ACCESS, + ]); + + $storageClient = new StorageClient([ + 'url' => $this->connectionUrl, + 'token' => $tokenInfo['token'], + ]); + + return new FlowMigrationProjectClients( + new Components($storageClient), + new JobQueueClient($this->queueApiUrl, $tokenInfo['token']) + ); + } +} +``` + +- [ ] **Step 2: Static analysis and code style** + +Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` +Expected: both exit 0. (If phpstan complains about `$tokenInfo['token']`, something changed in the vendor package — compare with the working pattern in `src/Keboola/Console/Command/MigrateDataAppsOrchestratorTasks.php:199-213` and match it.) + +- [ ] **Step 3: Commit** + +```bash +git add src/Keboola/Console/Command/FlowMigrationProjectClientsFactory.php +git commit -m "feat: add per-project clients factory with ephemeral token creation" +``` + +--- + +### Task 3: Test fakes for the queue client and the clients factory + +**Files:** +- Create: `tests/FakeJobQueueClient.php` +- Create: `tests/FakeFlowMigrationProjectClientsFactory.php` +- Test: `tests/FakeJobQueueClientTest.php` + +**Interfaces:** +- Consumes: `FlowMigrationProjectClients`, `FlowMigrationProjectClientsFactory` (Tasks 1-2), vendor `Client`, `DTO\Job`, `JobData`, `ListJobsOptions`. +- Produces (used by Tasks 4-6): + - `FakeJobQueueClient::__construct(array $createJobReturns = [], array $getJobSequences = [], array $listJobsReturn = [])` — scripted returns; `$getJobSequences` maps jobId → list of `Job|Throwable` consumed one per poll. + - public inspection fields: `array $createdJobs` (list of `JobData->getArray()` payloads), `int $listJobsCalls`, `array $calls` (ordered log of `['createJob', ]` / `['getJob', ]` / `['listJobs']` entries). + - `FakeJobQueueClient::makeJob(string $id, string $status, ?int $durationSeconds = null, ?array $result = null): Job` — builds a real `DTO\Job` fixture; `isFinished` is true for terminal statuses. + - `FakeFlowMigrationProjectClientsFactory::__construct(array $projects, array $projectClients = [])` — `$projects`: projectId → project detail array or `Throwable` to throw; `$projectClients`: projectId → `FlowMigrationProjectClients` or `Throwable`; public field `array $createClientsCalls` (list of projectIds). + +- [ ] **Step 1: Write the failing sanity test** + +Create `tests/FakeJobQueueClientTest.php`: + +```php + 'ok']); + + $this->assertFalse($running->isFinished); + $this->assertTrue($finished->isFinished); + $this->assertSame('success', $finished->status); + $this->assertSame(42, $finished->durationSeconds); + $this->assertSame(['message' => 'ok'], $finished->result); + } + + public function testGetJobConsumesScriptedSequenceAndThrowsThrowables(): void + { + $fake = new FakeJobQueueClient([], ['job-1' => [ + new RuntimeException('network blip'), + FakeJobQueueClient::makeJob('job-1', 'success'), + ]]); + + try { + $fake->getJob('job-1'); + $this->fail('First scripted outcome should throw'); + } catch (RuntimeException $e) { + $this->assertSame('network blip', $e->getMessage()); + } + + $this->assertSame('success', $fake->getJob('job-1')->status); + $this->assertSame([['getJob', 'job-1'], ['getJob', 'job-1']], $fake->calls); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FakeJobQueueClientTest.php` +Expected: FAIL — `Class "Keboola\Console\Tests\FakeJobQueueClient" not found` + +- [ ] **Step 3: Write the fakes** + +Create `tests/FakeJobQueueClient.php`: + +```php +> recorded JobData->getArray() payloads */ + public array $createdJobs = []; + + /** @var array> ordered call log: [method, id] */ + public array $calls = []; + + public int $listJobsCalls = 0; + + /** @var array */ + private array $createJobReturns; + + /** @var array> */ + private array $getJobSequences; + + /** @var array */ + private array $listJobsReturn; + + /** + * @param array $createJobReturns successive createJob() returns + * @param array> $getJobSequences jobId => successive getJob() outcomes + * @param array $listJobsReturn returned by every listJobs() call + */ + public function __construct(array $createJobReturns = [], array $getJobSequences = [], array $listJobsReturn = []) + { + $this->createJobReturns = $createJobReturns; + $this->getJobSequences = $getJobSequences; + $this->listJobsReturn = $listJobsReturn; + } + + public function createJob(JobData $jobData): Job + { + $this->createdJobs[] = $jobData->getArray(); + $job = array_shift($this->createJobReturns); + if ($job === null) { + throw new RuntimeException('FakeJobQueueClient: no scripted createJob return left'); + } + $this->calls[] = ['createJob', $job->id]; + + return $job; + } + + public function getJob(string $jobId): Job + { + $this->calls[] = ['getJob', $jobId]; + $sequence = $this->getJobSequences[$jobId] ?? []; + if ($sequence === []) { + throw new RuntimeException(sprintf('FakeJobQueueClient: no scripted getJob outcome left for "%s"', $jobId)); + } + $outcome = array_shift($sequence); + $this->getJobSequences[$jobId] = $sequence; + if ($outcome instanceof Throwable) { + throw $outcome; + } + + return $outcome; + } + + public function listJobs(ListJobsOptions $listOptions): array + { + $this->calls[] = ['listJobs']; + $this->listJobsCalls++; + + return $this->listJobsReturn; + } + + /** + * Builds a real DTO\Job through its public factory so the fixture stays in sync with the SDK. + * + * @param array|null $result + */ + public static function makeJob(string $id, string $status, ?int $durationSeconds = null, ?array $result = null): Job + { + $terminalStatuses = ['success', 'error', 'warning', 'terminated', 'cancelled']; + + return Job::fromApiResponse([ + 'id' => $id, + 'runId' => $id, + 'parentRunId' => '', + 'project' => ['id' => '123'], + 'token' => ['id' => '456', 'description' => 'test token'], + 'status' => $status, + 'desiredStatus' => 'processing', + 'mode' => 'run', + 'component' => 'keboola.flow-migration-tool', + 'config' => null, + 'configData' => null, + 'configRowIds' => null, + 'tag' => null, + 'createdTime' => '2026-08-10T10:00:00+00:00', + 'startTime' => null, + 'endTime' => null, + 'durationSeconds' => $durationSeconds, + 'result' => $result, + 'usageData' => null, + 'isFinished' => in_array($status, $terminalStatuses, true), + 'url' => sprintf('https://queue.example.com/jobs/%s', $id), + 'branchId' => null, + 'variableValuesId' => null, + 'variableValuesData' => [], + 'backend' => [], + 'behavior' => [], + 'executor' => null, + 'metrics' => null, + 'parallelism' => null, + 'type' => 'standard', + 'orchestrationJobId' => null, + 'orchestrationTaskId' => null, + 'onlyOrchestrationTaskIds' => null, + 'previousJobId' => null, + ]); + } +} +``` + +Create `tests/FakeFlowMigrationProjectClientsFactory.php`: + +```php + projectIds passed to createProjectClients() */ + public array $createClientsCalls = []; + + /** @var array|Throwable> */ + private array $projects; + + /** @var array */ + private array $projectClients; + + /** + * @param array|Throwable> $projects projectId => Manage project detail, or Throwable to throw + * @param array $projectClients projectId => clients, or Throwable + */ + public function __construct(array $projects, array $projectClients = []) + { + $this->projects = $projects; + $this->projectClients = $projectClients; + } + + public function getProject(string $projectId): array + { + if (!array_key_exists($projectId, $this->projects)) { + throw new RuntimeException(sprintf('FakeFlowMigrationProjectClientsFactory: unknown project "%s"', $projectId)); + } + $project = $this->projects[$projectId]; + if ($project instanceof Throwable) { + throw $project; + } + + return $project; + } + + public function createProjectClients(string $projectId): FlowMigrationProjectClients + { + $this->createClientsCalls[] = $projectId; + if (!array_key_exists($projectId, $this->projectClients)) { + throw new RuntimeException(sprintf('FakeFlowMigrationProjectClientsFactory: no clients for project "%s"', $projectId)); + } + $clients = $this->projectClients[$projectId]; + if ($clients instanceof Throwable) { + throw $clients; + } + + return $clients; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FakeJobQueueClientTest.php` +Expected: PASS (2 tests). If `Job::fromApiResponse` throws about a missing key, the SDK changed — add the missing key with a null/empty value to `makeJob()`. + +- [ ] **Step 5: Code style** + +Run: `docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` +Expected: exit 0. + +- [ ] **Step 6: Commit** + +```bash +git add tests/FakeJobQueueClient.php tests/FakeFlowMigrationProjectClientsFactory.php tests/FakeJobQueueClientTest.php +git commit -m "test: add fakes for job queue client and flow migration clients factory" +``` + +--- + +### Task 4: FlowMigrationBatchRunner — core loop, submission, polling + +**Files:** +- Create: `src/Keboola/Console/Command/FlowMigrationBatchRunner.php` +- Test: `tests/FlowMigrationBatchRunnerTest.php` + +**Interfaces:** +- Consumes: Tasks 1-3 classes; vendor `JobData`, `JobStatuses`, `ListJobsOptions`, `ListComponentConfigurationsOptions`, `DTO\Job`; `tests/FakeComponents.php` (existing, constructor `new FakeComponents(array $configsByComponent)`). +- Produces: + - `FlowMigrationBatchRunner::__construct(FlowMigrationProjectClientsFactory $clientsFactory, int $concurrency, int $pollIntervalSeconds, ?callable $sleep = null)` — `$sleep` signature `callable(int): void`, defaults to PHP `sleep()`. + - `run(array $projectIds, bool $force, OutputInterface $output, callable $onProjectFinished): array` — `$onProjectFinished` receives one `FlowMigrationProjectResult` per input project; returns summary shape `array{attempted: int, migrated: int, migratedWithWarning: int, skippedNoOrchestrations: int, skippedDisabled: int, skippedJobRunning: int, failed: int}`. + - public constants `ORCHESTRATOR_COMPONENT_ID = 'keboola.orchestrator'`, `MIGRATION_COMPONENT_ID = 'keboola.flow-migration-tool'`. + +In this task the runner handles enabled projects with orchestrations and no live migration job (the happy pipeline: guard query → createJob → poll → terminal result). Skip rules come in Task 5, poll-failure tolerance in Task 6. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/FlowMigrationBatchRunnerTest.php`: + +```php + */ + private array $results = []; + + /** @var array */ + private array $sleeps = []; + + private function collector(): Closure + { + return function (FlowMigrationProjectResult $result): void { + $this->results[] = $result; + }; + } + + private function sleepRecorder(): Closure + { + return function (int $seconds): void { + $this->sleeps[] = $seconds; + }; + } + + /** + * @return array enabled project detail as returned by the Manage API + */ + private static function enabledProject(string $id): array + { + return ['id' => $id, 'name' => 'Project ' . $id, 'isDisabled' => false]; + } + + private static function clientsWith(FakeJobQueueClient $queueClient, bool $hasOrchestrations = true): FlowMigrationProjectClients + { + $configs = $hasOrchestrations + ? ['keboola.orchestrator' => [['id' => 'orch-1', 'name' => 'Daily load', 'configuration' => []]]] + : []; + + return new FlowMigrationProjectClients(new FakeComponents($configs), $queueClient); + } + + public function testHappyPathCreatesJobAndReportsSuccess(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [ + FakeJobQueueClient::makeJob('job-1', 'processing'), + FakeJobQueueClient::makeJob('job-1', 'success', 42), + ]] + ); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + // Exact job payload - this is the whole contract with keboola.flow-migration-tool. + $this->assertCount(1, $queueClient->createdJobs); + $this->assertSame('keboola.flow-migration-tool', $queueClient->createdJobs[0]['component']); + $this->assertNull($queueClient->createdJobs[0]['config']); + $this->assertSame('run', $queueClient->createdJobs[0]['mode']); + $this->assertSame( + ['parameters' => [ + 'mode' => 'project', + 'orchestrationIds' => [], + 'skipBroken' => true, + 'dryRun' => false, + ]], + $queueClient->createdJobs[0]['configData'] + ); + // The live-job guard ran exactly once before submission. + $this->assertSame(1, $queueClient->listJobsCalls); + + $this->assertCount(1, $this->results); + $this->assertSame('100', $this->results[0]->projectId); + $this->assertSame('job-1', $this->results[0]->jobId); + $this->assertSame('success', $this->results[0]->status); + $this->assertSame(42, $this->results[0]->durationSeconds); + $this->assertNull($this->results[0]->error); + + // Two poll sweeps (processing, then success), each preceded by one poll-interval sleep. + $this->assertSame([5, 5], $this->sleeps); + + $this->assertSame(1, $summary['attempted']); + $this->assertSame(1, $summary['migrated']); + $this->assertSame(0, $summary['failed']); + } + + public function testWithoutForceJobRunsWithDryRunTrue(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'success', 1)]] + ); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $runner->run(['100'], false, new BufferedOutput(), $this->collector()); + + $configData = $queueClient->createdJobs[0]['configData']; + $this->assertIsArray($configData); + $this->assertTrue($configData['parameters']['dryRun']); + } + + public function testJobEndingInErrorMarksProjectFailedButBatchContinues(): void + { + $queueClient1 = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'error', 10, ['message' => 'boom'])]] + ); + $queueClient2 = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-2', 'created')], + ['job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 20)]] + ); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100'), '200' => self::enabledProject('200')], + ['100' => self::clientsWith($queueClient1), '200' => self::clientsWith($queueClient2)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); + + $this->assertCount(2, $this->results); + $byProject = []; + foreach ($this->results as $result) { + $byProject[$result->projectId] = $result; + } + $this->assertSame('error', $byProject['100']->status); + $this->assertSame('boom', $byProject['100']->error); + $this->assertTrue($byProject['100']->isFailed()); + $this->assertSame('success', $byProject['200']->status); + + $this->assertSame(2, $summary['attempted']); + $this->assertSame(1, $summary['migrated']); + $this->assertSame(1, $summary['failed']); + } + + public function testWarningJobCountsAsMigratedWithWarning(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'warning', 5, ['message' => 'partial'])]] + ); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame('warning', $this->results[0]->status); + $this->assertFalse($this->results[0]->isFailed()); + $this->assertSame(1, $summary['migratedWithWarning']); + $this->assertSame(0, $summary['migrated']); + $this->assertSame(0, $summary['failed']); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` +Expected: FAIL — `Class "Keboola\Console\Command\FlowMigrationBatchRunner" not found` + +- [ ] **Step 3: Write the implementation** + +Create `src/Keboola/Console/Command/FlowMigrationBatchRunner.php`: + +```php +clientsFactory = $clientsFactory; + $this->concurrency = $concurrency; + $this->pollIntervalSeconds = $pollIntervalSeconds; + $this->sleep = $sleep ?? function (int $seconds): void { + sleep($seconds); + }; + } + + /** + * @param array $projectIds + * @param callable(FlowMigrationProjectResult): void $onProjectFinished invoked once per input project + * @return array{ + * attempted: int, + * migrated: int, + * migratedWithWarning: int, + * skippedNoOrchestrations: int, + * skippedDisabled: int, + * skippedJobRunning: int, + * failed: int + * } + */ + public function run(array $projectIds, bool $force, OutputInterface $output, callable $onProjectFinished): array + { + $pending = array_values(array_unique($projectIds)); + $summary = [ + 'attempted' => count($pending), + 'migrated' => 0, + 'migratedWithWarning' => 0, + 'skippedNoOrchestrations' => 0, + 'skippedDisabled' => 0, + 'skippedJobRunning' => 0, + 'failed' => 0, + ]; + /** @var array $inFlight */ + $inFlight = []; + + while ($pending !== [] || $inFlight !== []) { + while (count($inFlight) < $this->concurrency && $pending !== []) { + $projectId = array_shift($pending); + $submission = $this->submitProject($projectId, $force, $output); + if ($submission instanceof FlowMigrationProjectResult) { + $this->recordResult($submission, $summary, $output, $onProjectFinished); + continue; + } + $inFlight[$projectId] = $submission; + } + + if ($inFlight === []) { + continue; + } + + ($this->sleep)($this->pollIntervalSeconds); + $this->pollInFlightJobs($inFlight, $summary, $output, $onProjectFinished); + } + + return $summary; + } + + /** + * Runs the per-project pipeline up to job creation. Returns an in-flight slot on success, + * or an immediately-final FlowMigrationProjectResult (skip or driver-side error). + * + * @return FlowMigrationProjectResult|array{jobId: string, queueClient: JobQueueClient, startedAt: float, pollFailures: int} + */ + private function submitProject(string $projectId, bool $force, OutputInterface $output) + { + try { + $clients = $this->clientsFactory->createProjectClients($projectId); + + $liveJobs = $clients->queueClient->listJobs( + (new ListJobsOptions()) + ->setComponents([self::MIGRATION_COMPONENT_ID]) + ->setStatuses(self::LIVE_JOB_STATUSES) + ->setLimit(1) + ); + if ($liveJobs !== []) { + return new FlowMigrationProjectResult( + $projectId, + null, + FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING, + null, + 'a keboola.flow-migration-tool job is already running in this project' + ); + } + + $job = $clients->queueClient->createJob(new JobData( + self::MIGRATION_COMPONENT_ID, + null, + [ + 'parameters' => [ + 'mode' => 'project', + 'orchestrationIds' => [], + 'skipBroken' => true, + 'dryRun' => !$force, + ], + ] + )); + } catch (Throwable $e) { + return new FlowMigrationProjectResult( + $projectId, + null, + FlowMigrationProjectResult::STATUS_ERROR, + null, + $e->getMessage() + ); + } + + $this->writeLine($output, sprintf('Project %s: created job %s (%s)', $projectId, $job->id, $job->url)); + + return [ + 'jobId' => $job->id, + 'queueClient' => $clients->queueClient, + 'startedAt' => microtime(true), + 'pollFailures' => 0, + ]; + } + + /** + * @param array $inFlight + * @param array{ + * attempted: int, + * migrated: int, + * migratedWithWarning: int, + * skippedNoOrchestrations: int, + * skippedDisabled: int, + * skippedJobRunning: int, + * failed: int + * } $summary + * @param callable(FlowMigrationProjectResult): void $onProjectFinished + */ + private function pollInFlightJobs( + array &$inFlight, + array &$summary, + OutputInterface $output, + callable $onProjectFinished + ): void { + foreach (array_keys($inFlight) as $projectId) { + $slot = $inFlight[$projectId]; + $job = $slot['queueClient']->getJob($slot['jobId']); + + if (!$job->isFinished) { + continue; + } + + unset($inFlight[$projectId]); + $durationSeconds = $job->durationSeconds ?? (int) round(microtime(true) - $slot['startedAt']); + $this->recordResult( + new FlowMigrationProjectResult( + $projectId, + $slot['jobId'], + $job->status, + $durationSeconds, + $this->extractJobError($job) + ), + $summary, + $output, + $onProjectFinished + ); + } + } + + private function extractJobError(Job $job): ?string + { + if ($job->isSuccess()) { + return null; + } + $result = $job->result; + if (is_array($result) && isset($result['message']) && is_scalar($result['message'])) { + return (string) $result['message']; + } + + return null; + } + + /** + * @param array{ + * attempted: int, + * migrated: int, + * migratedWithWarning: int, + * skippedNoOrchestrations: int, + * skippedDisabled: int, + * skippedJobRunning: int, + * failed: int + * } $summary + * @param callable(FlowMigrationProjectResult): void $onProjectFinished + */ + private function recordResult( + FlowMigrationProjectResult $result, + array &$summary, + OutputInterface $output, + callable $onProjectFinished + ): void { + $summaryKey = match ($result->status) { + JobStatuses::SUCCESS->value => 'migrated', + JobStatuses::WARNING->value => 'migratedWithWarning', + FlowMigrationProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS => 'skippedNoOrchestrations', + FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED => 'skippedDisabled', + FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING => 'skippedJobRunning', + default => 'failed', + }; + $summary[$summaryKey]++; + + $this->writeLine($output, sprintf( + 'Project %s: %s%s%s', + $result->projectId, + $result->status, + $result->durationSeconds !== null ? sprintf(' in %d s', $result->durationSeconds) : '', + $result->error !== null ? sprintf(' (%s)', $result->error) : '' + )); + + $onProjectFinished($result); + } + + private function writeLine(OutputInterface $output, string $message): void + { + $output->writeln(sprintf('[%s] %s', date('H:i:s'), $message)); + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` +Expected: PASS (4 tests) + +- [ ] **Step 5: Static analysis and code style** + +Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` +Expected: both exit 0. + +- [ ] **Step 6: Commit** + +```bash +git add src/Keboola/Console/Command/FlowMigrationBatchRunner.php tests/FlowMigrationBatchRunnerTest.php +git commit -m "feat: add flow migration batch runner with concurrency window and polling" +``` + +--- + +### Task 5: Batch runner — skip rules and input deduplication + +**Files:** +- Modify: `src/Keboola/Console/Command/FlowMigrationBatchRunner.php` (method `submitProject`, plus two `use` imports) +- Test: `tests/FlowMigrationBatchRunnerTest.php` (append methods) + +**Interfaces:** +- Consumes: `Keboola\ManageApi\ClientException` (HTTP status via `getCode()`); everything from Task 4. +- Produces: final `submitProject()` behavior — skip order is: disabled/deleted → (token+clients) → no orchestrator configs → live migration job → createJob. No token is created for disabled/deleted projects. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/FlowMigrationBatchRunnerTest.php` (add `use Keboola\ManageApi\ClientException as ManageClientException;` and `use RuntimeException;` to the imports): + +```php + public function testSkipsDisabledProjectWithoutCreatingTokenOrJob(): void + { + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => ['id' => '100', 'name' => 'Off', 'isDisabled' => true]] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + // No ephemeral token may be created for a disabled project. + $this->assertSame([], $factory->createClientsCalls); + $this->assertSame(FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, $this->results[0]->status); + $this->assertNull($this->results[0]->jobId); + $this->assertSame(1, $summary['skippedDisabled']); + $this->assertSame(0, $summary['failed']); + $this->assertSame([], $this->sleeps); + } + + public function testSkipsDeletedProjectOnManage404(): void + { + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => new ManageClientException('Project not found', 404)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame([], $factory->createClientsCalls); + $this->assertSame(FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, $this->results[0]->status); + $this->assertSame(1, $summary['skippedDisabled']); + } + + public function testManageErrorOtherThan404MarksProjectFailed(): void + { + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => new ManageClientException('Internal error', 500)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame(FlowMigrationProjectResult::STATUS_ERROR, $this->results[0]->status); + $this->assertSame('Internal error', $this->results[0]->error); + $this->assertSame(1, $summary['failed']); + } + + public function testSkipsProjectWithoutOrchestratorConfigurations(): void + { + $queueClient = new FakeJobQueueClient(); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient, false)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + // No queue API call and no job for a project with nothing to migrate. + $this->assertSame(0, $queueClient->listJobsCalls); + $this->assertSame([], $queueClient->createdJobs); + $this->assertSame(FlowMigrationProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, $this->results[0]->status); + $this->assertSame(1, $summary['skippedNoOrchestrations']); + } + + public function testSkipsProjectWithLiveMigrationJob(): void + { + $queueClient = new FakeJobQueueClient( + [], + [], + [FakeJobQueueClient::makeJob('existing-job', 'processing')] + ); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame([], $queueClient->createdJobs); + $this->assertSame(FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING, $this->results[0]->status); + $this->assertSame(1, $summary['skippedJobRunning']); + } + + public function testDeduplicatesInputProjectIds(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'success', 1)]] + ); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100', '100', '100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame(1, $summary['attempted']); + $this->assertCount(1, $queueClient->createdJobs); + $this->assertCount(1, $this->results); + } +``` + +- [ ] **Step 2: Run tests to verify the new ones fail** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` +Expected: `testDeduplicatesInputProjectIds` PASSES already (dedup shipped in Task 4's `run()`); the disabled/404/500 tests FAIL (`getProject` is never called, so the fake's clients-map miss makes them land in `failed`, not `skippedDisabled`); the no-orchestrations test FAILS (no configuration check exists yet); the live-job test PASSES already. Failing count: 4. + +- [ ] **Step 3: Extend the implementation** + +In `src/Keboola/Console/Command/FlowMigrationBatchRunner.php`, add one import +(`ListComponentConfigurationsOptions` is already imported since Task 4): + +```php +use Keboola\ManageApi\ClientException as ManageClientException; +``` + +Replace the whole `submitProject()` method with: + +```php + /** + * Runs the per-project pipeline up to job creation. Returns an in-flight slot on success, + * or an immediately-final FlowMigrationProjectResult (skip or driver-side error). + * + * Order matters: the disabled/deleted check runs before any token is created, and the + * configuration check runs before the queue guard so empty projects never appear in + * customers' job history. + * + * @return FlowMigrationProjectResult|array{jobId: string, queueClient: JobQueueClient, startedAt: float, pollFailures: int} + */ + private function submitProject(string $projectId, bool $force, OutputInterface $output) + { + try { + $project = $this->clientsFactory->getProject($projectId); + } catch (ManageClientException $e) { + if ($e->getCode() === 404) { + return new FlowMigrationProjectResult( + $projectId, + null, + FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, + null, + 'project is deleted' + ); + } + + return new FlowMigrationProjectResult( + $projectId, + null, + FlowMigrationProjectResult::STATUS_ERROR, + null, + $e->getMessage() + ); + } + + if (isset($project['isDisabled']) && $project['isDisabled']) { + return new FlowMigrationProjectResult( + $projectId, + null, + FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, + null, + 'project is disabled' + ); + } + + try { + $clients = $this->clientsFactory->createProjectClients($projectId); + + $configurations = $clients->components->listComponentConfigurations( + (new ListComponentConfigurationsOptions()) + ->setComponentId(self::ORCHESTRATOR_COMPONENT_ID) + ->setIsDeleted(false) + ); + if (count($configurations) === 0) { + return new FlowMigrationProjectResult( + $projectId, + null, + FlowMigrationProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, + null, + 'no keboola.orchestrator configurations' + ); + } + + $liveJobs = $clients->queueClient->listJobs( + (new ListJobsOptions()) + ->setComponents([self::MIGRATION_COMPONENT_ID]) + ->setStatuses(self::LIVE_JOB_STATUSES) + ->setLimit(1) + ); + if ($liveJobs !== []) { + return new FlowMigrationProjectResult( + $projectId, + null, + FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING, + null, + 'a keboola.flow-migration-tool job is already running in this project' + ); + } + + $job = $clients->queueClient->createJob(new JobData( + self::MIGRATION_COMPONENT_ID, + null, + [ + 'parameters' => [ + 'mode' => 'project', + 'orchestrationIds' => [], + 'skipBroken' => true, + 'dryRun' => !$force, + ], + ] + )); + } catch (Throwable $e) { + return new FlowMigrationProjectResult( + $projectId, + null, + FlowMigrationProjectResult::STATUS_ERROR, + null, + $e->getMessage() + ); + } + + $this->writeLine($output, sprintf('Project %s: created job %s (%s)', $projectId, $job->id, $job->url)); + + return [ + 'jobId' => $job->id, + 'queueClient' => $clients->queueClient, + 'startedAt' => microtime(true), + 'pollFailures' => 0, + ]; + } +``` + +Note: `listComponentConfigurations()` has no declared return type in the SDK (implicit mixed), so `count()` on it is phpstan-safe — the same access pattern as `DataAppOrchestratorTaskMigrator` uses. Update the existing Task 4 tests' fixtures if needed: they already provide `'100' => self::enabledProject('100')` in the `$projects` map, so `getProject()` succeeds there — no changes expected. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` +Expected: PASS (10 tests) + +- [ ] **Step 5: Static analysis and code style** + +Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` +Expected: both exit 0. + +- [ ] **Step 6: Commit** + +```bash +git add src/Keboola/Console/Command/FlowMigrationBatchRunner.php tests/FlowMigrationBatchRunnerTest.php +git commit -m "feat: add skip rules for disabled, empty and already-migrating projects" +``` + +--- + +### Task 6: Batch runner — poll-failure tolerance and concurrency window verification + +**Files:** +- Modify: `src/Keboola/Console/Command/FlowMigrationBatchRunner.php` (constant + method `pollInFlightJobs`) +- Test: `tests/FlowMigrationBatchRunnerTest.php` (append methods) + +**Interfaces:** +- Consumes: everything from Tasks 4-5. +- Produces: `pollInFlightJobs()` tolerates up to 2 consecutive `getJob()` failures per job (a success resets the counter); the 3rd consecutive failure resolves the project as `STATUS_ERROR` with the job id preserved in the result. Constant `MAX_CONSECUTIVE_POLL_FAILURES = 3` (private). + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/FlowMigrationBatchRunnerTest.php`: + +```php + public function testTwoConsecutivePollFailuresAreToleratedAndJobFinishes(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [ + new RuntimeException('blip 1'), + new RuntimeException('blip 2'), + FakeJobQueueClient::makeJob('job-1', 'success', 7), + ]] + ); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame('success', $this->results[0]->status); + $this->assertSame(1, $summary['migrated']); + $this->assertSame(0, $summary['failed']); + $this->assertSame([5, 5, 5], $this->sleeps); + } + + public function testThreeConsecutivePollFailuresMarkProjectFailedWithJobIdKept(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [ + new RuntimeException('down 1'), + new RuntimeException('down 2'), + new RuntimeException('down 3'), + ]] + ); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame(FlowMigrationProjectResult::STATUS_ERROR, $this->results[0]->status); + // The job id must survive into the report - the job may still be running server-side. + $this->assertSame('job-1', $this->results[0]->jobId); + $this->assertIsString($this->results[0]->error); + $this->assertStringContainsString('polling gave up', $this->results[0]->error); + $this->assertSame(1, $summary['failed']); + } + + public function testConcurrencyWindowCapsInFlightJobsAndRefills(): void + { + // One shared fake for both projects makes the cross-project call order observable. + $queueClient = new FakeJobQueueClient( + [ + FakeJobQueueClient::makeJob('job-1', 'created'), + FakeJobQueueClient::makeJob('job-2', 'created'), + ], + [ + 'job-1' => [ + FakeJobQueueClient::makeJob('job-1', 'processing'), + FakeJobQueueClient::makeJob('job-1', 'success', 1), + ], + 'job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 1)], + ] + ); + $clients = self::clientsWith($queueClient); + $factory = new FakeFlowMigrationProjectClientsFactory( + ['100' => self::enabledProject('100'), '200' => self::enabledProject('200')], + ['100' => $clients, '200' => $clients] + ); + $runner = new FlowMigrationBatchRunner($factory, 1, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); + + // With concurrency=1, job-2 must not be created until job-1 has finished. + $this->assertSame( + [ + ['listJobs'], + ['createJob', 'job-1'], + ['getJob', 'job-1'], + ['getJob', 'job-1'], + ['listJobs'], + ['createJob', 'job-2'], + ['getJob', 'job-2'], + ], + $queueClient->calls + ); + $this->assertSame(2, $summary['migrated']); + } +``` + +- [ ] **Step 2: Run tests to verify the new ones fail** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` +Expected: the two poll-failure tests FAIL (the `RuntimeException` from `getJob()` currently escapes `run()` uncaught). The concurrency test should PASS already (window logic shipped in Task 4) — it is the regression guard for this behavior. + +- [ ] **Step 3: Extend the implementation** + +In `src/Keboola/Console/Command/FlowMigrationBatchRunner.php`, add below `LIVE_JOB_STATUSES`: + +```php + // A transient Queue API outage must not fail a project instantly (the SDK already retries + // 5xx internally), but an unbounded retry could hang the batch forever - so give up after + // this many consecutive failed polls and leave the job to finish server-side. + private const MAX_CONSECUTIVE_POLL_FAILURES = 3; +``` + +Replace the whole `pollInFlightJobs()` method with: + +```php + /** + * @param array $inFlight + * @param array{ + * attempted: int, + * migrated: int, + * migratedWithWarning: int, + * skippedNoOrchestrations: int, + * skippedDisabled: int, + * skippedJobRunning: int, + * failed: int + * } $summary + * @param callable(FlowMigrationProjectResult): void $onProjectFinished + */ + private function pollInFlightJobs( + array &$inFlight, + array &$summary, + OutputInterface $output, + callable $onProjectFinished + ): void { + foreach (array_keys($inFlight) as $projectId) { + $slot = $inFlight[$projectId]; + try { + $job = $slot['queueClient']->getJob($slot['jobId']); + } catch (Throwable $e) { + $inFlight[$projectId]['pollFailures']++; + $this->writeLine($output, sprintf( + 'Project %s: polling job %s failed (%d/%d): %s', + $projectId, + $slot['jobId'], + $inFlight[$projectId]['pollFailures'], + self::MAX_CONSECUTIVE_POLL_FAILURES, + $e->getMessage() + )); + if ($inFlight[$projectId]['pollFailures'] >= self::MAX_CONSECUTIVE_POLL_FAILURES) { + unset($inFlight[$projectId]); + $this->recordResult( + new FlowMigrationProjectResult( + $projectId, + $slot['jobId'], + FlowMigrationProjectResult::STATUS_ERROR, + null, + sprintf( + 'polling gave up after %d consecutive failures, job may still be running: %s', + self::MAX_CONSECUTIVE_POLL_FAILURES, + $e->getMessage() + ) + ), + $summary, + $output, + $onProjectFinished + ); + } + continue; + } + + $inFlight[$projectId]['pollFailures'] = 0; + + if (!$job->isFinished) { + continue; + } + + unset($inFlight[$projectId]); + $durationSeconds = $job->durationSeconds ?? (int) round(microtime(true) - $slot['startedAt']); + $this->recordResult( + new FlowMigrationProjectResult( + $projectId, + $slot['jobId'], + $job->status, + $durationSeconds, + $this->extractJobError($job) + ), + $summary, + $output, + $onProjectFinished + ); + } + } +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` +Expected: PASS (13 tests) + +- [ ] **Step 5: Static analysis and code style** + +Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` +Expected: both exit 0. + +- [ ] **Step 6: Commit** + +```bash +git add src/Keboola/Console/Command/FlowMigrationBatchRunner.php tests/FlowMigrationBatchRunnerTest.php +git commit -m "feat: tolerate transient poll failures and verify concurrency window" +``` + +--- + +### Task 7: The Symfony command, registration and input-parsing tests + +**Files:** +- Create: `src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php` +- Modify: `cli.php` (one `use` line + one `add()` line) +- Test: `tests/MigrateOrchestrationsToFlowTest.php` + +**Interfaces:** +- Consumes: `FlowMigrationBatchRunner`, `FlowMigrationProjectClientsFactory`, `FlowMigrationProjectResult` (Tasks 1-6); `Keboola\ManageApi\Client`, `Keboola\ServiceClient\ServiceClient`. +- Produces: command `manage:migrate-orchestrations-to-flow` registered in `cli.php`. Private helpers (tested via reflection, matching the `MigrateDataAppsOrchestratorTasksTest` pattern): `hostnameSuffixFromUrl(string): ?string`, `parseProjectIdList(string): ?array`, `parseProjectIdsFile(string): ?array`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/MigrateOrchestrationsToFlowTest.php`: + +```php +|string|null + */ + private function invokePrivate(string $method, string $argument): array|string|null + { + $command = new MigrateOrchestrationsToFlow(); + $reflection = (new ReflectionClass($command))->getMethod($method); + $reflection->setAccessible(true); + + /** @var array|string|null $result */ + $result = $reflection->invoke($command, $argument); + + return $result; + } + + #[DataProvider('provideUrls')] + public function testHostnameSuffixFromUrl(string $url, ?string $expected): void + { + $this->assertSame($expected, $this->invokePrivate('hostnameSuffixFromUrl', $url)); + } + + /** + * @return iterable + */ + public static function provideUrls(): iterable + { + yield 'azure ne stack' => [ + 'https://connection.north-europe.azure.keboola.com', + 'north-europe.azure.keboola.com', + ]; + yield 'aws us stack' => ['https://connection.keboola.com', 'keboola.com']; + yield 'trailing slash is fine' => ['https://connection.keboola.com/', 'keboola.com']; + yield 'missing connection prefix' => ['https://queue.keboola.com', null]; + yield 'not a url' => ['not-a-url', null]; + yield 'bare connection host' => ['https://connection.', null]; + } + + #[DataProvider('provideProjectLists')] + public function testParseProjectIdList(string $input, ?array $expected): void + { + $this->assertSame($expected, $this->invokePrivate('parseProjectIdList', $input)); + } + + /** + * @return iterable|null}> + */ + public static function provideProjectLists(): iterable + { + yield 'plain list' => ['1,2,3', ['1', '2', '3']]; + yield 'whitespace is trimmed' => ['1, 2 ,3', ['1', '2', '3']]; + yield 'duplicates are removed' => ['1,2,1', ['1', '2']]; + yield 'non-numeric entry invalidates the list' => ['1,foo', null]; + yield 'decimal is rejected' => ['1.2', null]; + yield 'negative is rejected' => ['-1', null]; + yield 'empty string is rejected' => ['', null]; + } + + #[DataProvider('provideProjectFiles')] + public function testParseProjectIdsFile(string $contents, ?array $expected): void + { + $this->assertSame($expected, $this->invokePrivate('parseProjectIdsFile', $contents)); + } + + /** + * @return iterable|null}> + */ + public static function provideProjectFiles(): iterable + { + yield 'one id per line' => ["100\n200\n", ['100', '200']]; + yield 'blank lines and comments are ignored' => ["100\n\n# staging batch\n200\n", ['100', '200']]; + yield 'windows line endings' => ["100\r\n200\r\n", ['100', '200']]; + yield 'duplicates are removed' => ["100\n200\n100\n", ['100', '200']]; + yield 'non-numeric line invalidates the file' => ["100\nfoo\n", null]; + yield 'empty file is a valid empty list' => ['', []]; + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/MigrateOrchestrationsToFlowTest.php` +Expected: FAIL — `Class "Keboola\Console\Command\MigrateOrchestrationsToFlow" not found` + +- [ ] **Step 3: Write the command** + +Create `src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php`: + +```php + keboola.flow migration (AJDA-3117). + * All migration logic lives in the keboola.flow-migration-tool component; this command only + * creates and supervises its jobs across a list of projects on one stack. + */ +class MigrateOrchestrationsToFlow extends Command +{ + const ARG_TOKEN = 'token'; + const ARG_URL = 'url'; + const ARG_PROJECTS = 'projects'; + const OPT_FORCE = 'force'; + const OPT_PROJECTS_FILE = 'projects-file'; + const OPT_CONCURRENCY = 'concurrency'; + const OPT_POLL_INTERVAL = 'poll-interval'; + const OPT_REPORT = 'report'; + + private const CSV_HEADER = ['projectId', 'jobId', 'status', 'durationSeconds', 'error']; + private const CSV_DELIMITER = ';'; + + protected function configure(): void + { + $this + ->setName('manage:migrate-orchestrations-to-flow') + ->setDescription( + 'Run the automated keboola.orchestrator -> keboola.flow migration for a batch of projects' + ) + ->addArgument(self::ARG_TOKEN, InputArgument::REQUIRED, 'Manage API token') + ->addArgument( + self::ARG_URL, + InputArgument::REQUIRED, + 'Stack URL, e.g. https://connection.north-europe.azure.keboola.com' + ) + ->addArgument( + self::ARG_PROJECTS, + InputArgument::OPTIONAL, + 'Comma-separated project IDs, or @path/to/file with one ID per line' + ) + ->addOption( + self::OPT_FORCE, + 'f', + InputOption::VALUE_NONE, + 'Run the real migration; without it jobs are created with dryRun: true' + ) + ->addOption( + self::OPT_PROJECTS_FILE, + null, + InputOption::VALUE_REQUIRED, + 'File with one project ID per line (alternative to @file in the argument)' + ) + ->addOption( + self::OPT_CONCURRENCY, + null, + InputOption::VALUE_REQUIRED, + 'Max migration jobs in flight at once', + '10' + ) + ->addOption( + self::OPT_POLL_INTERVAL, + null, + InputOption::VALUE_REQUIRED, + 'Seconds between job status polls', + '5' + ) + ->addOption( + self::OPT_REPORT, + null, + InputOption::VALUE_REQUIRED, + 'CSV report path (default: flow-migration--.csv)' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $token = $input->getArgument(self::ARG_TOKEN); + assert(is_string($token)); + $url = $input->getArgument(self::ARG_URL); + assert(is_string($url)); + $force = (bool) $input->getOption(self::OPT_FORCE); + + $hostnameSuffix = $this->hostnameSuffixFromUrl($url); + if ($hostnameSuffix === null) { + $output->writeln(sprintf( + 'Invalid stack URL "%s": expected a URL like https://connection.keboola.com', + $url + )); + return 1; + } + + $projectIds = $this->resolveProjectIds($input, $output); + if ($projectIds === null) { + return 1; + } + + $concurrency = $this->parsePositiveIntOption($input, self::OPT_CONCURRENCY); + $pollInterval = $this->parsePositiveIntOption($input, self::OPT_POLL_INTERVAL); + if ($concurrency === null || $pollInterval === null) { + $output->writeln('Options --concurrency and --poll-interval must be positive integers'); + return 1; + } + + $reportPath = $input->getOption(self::OPT_REPORT); + if (!is_string($reportPath) || $reportPath === '') { + $reportPath = sprintf('flow-migration-%s-%s.csv', $hostnameSuffix, date('Ymd-His')); + } + + $output->writeln($force + ? 'Running in FORCE mode: migration jobs run with dryRun: false.' + : 'Running in dry-run mode: migration jobs run with dryRun: true. Use -f for the real migration.'); + $output->writeln('NOTE: even in dry-run mode a real keboola.flow-migration-tool job and a real' + . ' ephemeral storage token are created in every eligible project.'); + $output->writeln(sprintf('Projects: %d, concurrency: %d, poll interval: %d s', count($projectIds), $concurrency, $pollInterval)); + $output->writeln(sprintf('Report: %s', $reportPath)); + $output->writeln(''); + + $manageClient = new ManageClient(['url' => $url, 'token' => $token]); + $serviceClient = new ServiceClient($hostnameSuffix); + $clientsFactory = new FlowMigrationProjectClientsFactory($manageClient, $url, $serviceClient->getQueueUrl()); + + $reportHandle = fopen($reportPath, 'a'); + if ($reportHandle === false) { + $output->writeln(sprintf('Cannot open report file "%s" for writing', $reportPath)); + return 1; + } + if (ftell($reportHandle) === 0) { + fputcsv($reportHandle, self::CSV_HEADER, self::CSV_DELIMITER, '"', '\\'); + } + + $runner = new FlowMigrationBatchRunner($clientsFactory, $concurrency, $pollInterval); + $summary = $runner->run( + $projectIds, + $force, + $output, + function (FlowMigrationProjectResult $result) use ($reportHandle): void { + fputcsv( + $reportHandle, + [ + $result->projectId, + $result->jobId ?? '', + $result->status, + $result->durationSeconds !== null ? (string) $result->durationSeconds : '', + $result->error ?? '', + ], + self::CSV_DELIMITER, + '"', + '\\' + ); + // Flush per row so an interrupted run still leaves an auditable report. + fflush($reportHandle); + } + ); + fclose($reportHandle); + + $output->writeln(''); + $output->writeln(sprintf( + "DONE\nProjects attempted: %d\nMigrated (job success): %d\nMigrated with warning: %d\n" + . "Skipped (no orchestrations): %d\nSkipped (disabled/deleted): %d\n" + . "Skipped (migration job already running): %d\nFailed: %d", + $summary['attempted'], + $summary['migrated'], + $summary['migratedWithWarning'], + $summary['skippedNoOrchestrations'], + $summary['skippedDisabled'], + $summary['skippedJobRunning'], + $summary['failed'] + )); + + return $summary['failed'] > 0 ? 1 : 0; + } + + /** + * Derives the ServiceClient hostname suffix from a full connection URL, e.g. + * "https://connection.north-europe.azure.keboola.com" -> "north-europe.azure.keboola.com". + * Returns null when the URL does not look like a stack connection URL. + */ + private function hostnameSuffixFromUrl(string $url): ?string + { + $host = parse_url($url, PHP_URL_HOST); + if (!is_string($host) || !str_starts_with($host, 'connection.')) { + return null; + } + $suffix = substr($host, strlen('connection.')); + + return $suffix === '' ? null : $suffix; + } + + /** + * Resolves the project ID list from exactly one source: the argument + * (inline list or @file) or --projects-file. Prints an error and returns null otherwise. + * + * @return array|null + */ + private function resolveProjectIds(InputInterface $input, OutputInterface $output): ?array + { + $projectsArg = $input->getArgument(self::ARG_PROJECTS); + $projectsFile = $input->getOption(self::OPT_PROJECTS_FILE); + + $hasArg = is_string($projectsArg) && $projectsArg !== ''; + $hasFileOption = is_string($projectsFile) && $projectsFile !== ''; + + if ($hasArg === $hasFileOption) { + $output->writeln( + 'Provide exactly one source of project IDs: the argument or --projects-file' + ); + return null; + } + + $projectIds = null; + + if ($hasArg) { + assert(is_string($projectsArg)); + if (str_starts_with($projectsArg, '@')) { + $projectsFile = substr($projectsArg, 1); + $hasFileOption = true; + } else { + $projectIds = $this->parseProjectIdList($projectsArg); + } + } + + if ($hasFileOption) { + assert(is_string($projectsFile)); + $contents = @file_get_contents($projectsFile); + if ($contents === false) { + $output->writeln(sprintf('Cannot read projects file "%s"', $projectsFile)); + return null; + } + $projectIds = $this->parseProjectIdsFile($contents); + } + + if ($projectIds === null || $projectIds === []) { + $output->writeln('Projects list is empty or contains a non-numeric ID'); + return null; + } + + return $projectIds; + } + + /** + * @return array|null null when any entry is not a plain non-negative integer + */ + private function parseProjectIdList(string $raw): ?array + { + return $this->validateAndDeduplicate(array_map('trim', explode(',', $raw))); + } + + /** + * One ID per line; blank lines and lines starting with "#" are ignored. + * + * @return array|null null when any remaining line is not a plain non-negative integer + */ + private function parseProjectIdsFile(string $contents): ?array + { + $lines = preg_split('/\R/', $contents); + $ids = []; + foreach ($lines === false ? [] : $lines as $line) { + $line = trim($line); + if ($line === '' || str_starts_with($line, '#')) { + continue; + } + $ids[] = $line; + } + + return $this->validateAndDeduplicate($ids); + } + + /** + * @param array $ids + * @return array|null + */ + private function validateAndDeduplicate(array $ids): ?array + { + foreach ($ids as $id) { + if (!ctype_digit($id)) { + return null; + } + } + + return array_values(array_unique($ids)); + } + + private function parsePositiveIntOption(InputInterface $input, string $name): ?int + { + $value = $input->getOption($name); + if (!is_string($value) || !ctype_digit($value) || (int) $value < 1) { + return null; + } + + return (int) $value; + } +} +``` + +- [ ] **Step 4: Register the command in cli.php** + +In `cli.php`, add to the `use` block (after the `MigrateDataAppsOrchestratorTasks` line): + +```php +use Keboola\Console\Command\MigrateOrchestrationsToFlow; +``` + +and after `$application->add(new MigrateDataAppsOrchestratorTasks());`: + +```php +$application->add(new MigrateOrchestrationsToFlow()); +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/MigrateOrchestrationsToFlowTest.php` +Expected: PASS (19 tests) + +- [ ] **Step 6: Smoke-test the command wiring** + +Run: `docker compose run --rm dev php cli.php list | grep migrate-orchestrations-to-flow` +Expected: one line with `manage:migrate-orchestrations-to-flow`. + +Run: `docker compose run --rm dev php cli.php manage:migrate-orchestrations-to-flow some-token https://connection.keboola.com` +Expected: prints `Provide exactly one source of project IDs...` and exits with code 1 (verify with `echo $?` — note `docker compose run` propagates the container exit code). + +Run: `docker compose run --rm dev php cli.php manage:migrate-orchestrations-to-flow some-token https://connection.keboola.com 1,foo` +Expected: prints `Projects list is empty or contains a non-numeric ID`, exit code 1. + +- [ ] **Step 7: Static analysis and code style** + +Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` +Expected: both exit 0. + +- [ ] **Step 8: Commit** + +```bash +git add src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php cli.php tests/MigrateOrchestrationsToFlowTest.php +git commit -m "feat: add manage:migrate-orchestrations-to-flow batch driver command" +``` + +--- + +### Task 8: README documentation and full quality gate + +**Files:** +- Modify: `README.md` (new section under "Project manipulation", directly after the "Migrate data-apps orchestrator/flow tasks to data-app-control" section that ends at the line before `### Mass enablement of dynamic backends for multiple projects`) + +**Interfaces:** +- Consumes: the finished command (Task 7). +- Produces: user-facing documentation; a fully green build. + +- [ ] **Step 1: Add the README section** + +Insert into `README.md` after the "Migrate data-apps orchestrator/flow tasks to data-app-control" section: + +````markdown +### Migrate keboola.orchestrator configurations to keboola.flow + +Batch driver for the automated `keboola.orchestrator` → `keboola.flow` migration +(see [AJDA-3117](https://linear.app/keboola/issue/AJDA-3117)). All migration logic lives in the +`keboola.flow-migration-tool` component; this command only creates one migration job per project +and supervises the batch. Safe to re-run with the same list: already-migrated orchestrations are +reported as skipped by the component, and projects with a live migration job are skipped here. + +``` +php cli.php manage:migrate-orchestrations-to-flow [-f|--force] [] \ + [--projects-file=PATH] [--concurrency=10] [--poll-interval=5] [--report=PATH] +``` + +Arguments: +- `token` (required): Manage API token. +- `url` (required): Stack URL, including `https://` (e.g. `https://connection.north-europe.azure.keboola.com`). +- `projects` (optional): Comma-separated project IDs (e.g. `1,7,146`), or `@path/to/file` with one ID + per line (blank lines and `#` comments are ignored). Exactly one of `projects`/`--projects-file` + must be given. + +Options: +- `--force` / `-f`: Run the real migration. Without it, jobs are created with `dryRun: true`. + **Note:** even without `--force` a real `keboola.flow-migration-tool` job and a real ephemeral + storage token are created in every eligible project — on PAYGO stacks mind the billing. +- `--projects-file=PATH`: File with one project ID per line (alternative to `@file` in the argument). +- `--concurrency=N` (default 10): Max migration jobs in flight at once. +- `--poll-interval=N` (default 5): Seconds between job status polls. +- `--report=PATH` (default `flow-migration--.csv`): CSV report path. + +Behavior: +- For each project: skips disabled/deleted projects; creates an ephemeral 12h storage token + (`canManageBuckets`, `canReadAllFileUploads`, component access to `keboola.orchestrator`, + `keboola.flow`, `keboola.scheduler`, `keboola.flow-migration-tool`); skips projects with no + `keboola.orchestrator` configurations (no empty jobs in customers' job history); skips projects + where a `keboola.flow-migration-tool` job is already created/waiting/processing/terminating. +- Creates the migration job via `configData` (no stored configuration is left behind) with + `parameters: {mode: "project", orchestrationIds: [], skipBroken: true, dryRun: }`. +- Keeps at most `--concurrency` jobs in flight, polls each job and refills the window as jobs finish. +- Appends a CSV row (`projectId;jobId;status;durationSeconds;error`) the moment each project + resolves, so an interrupted run is still auditable. Every input project gets a row; skipped + projects carry the skip reason in `status`/`error` and an empty `jobId`. +- A failing project never aborts the batch. Exit code is `1` if at least one project failed + (job `error`/`terminated`/`cancelled` or a driver-side error), `0` otherwise. +- Final summary: projects attempted / migrated / migrated with warning / skipped (no + orchestrations, disabled, job already running) / failed. +```` + +Note: the block above is fenced with four backticks only so it survives inside this plan file — in `README.md` itself use plain triple-backtick fences exactly like the surrounding sections. + +- [ ] **Step 2: Full quality gate** + +Run: +```bash +docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n . +docker compose run --rm dev composer phpstan +docker compose run --rm dev composer tests +``` +Expected: all three exit 0; the full test suite passes (existing tests plus the 42 new ones). + +- [ ] **Step 3: Commit** + +```bash +git add README.md +git commit -m "docs: document manage:migrate-orchestrations-to-flow command" +``` + +--- + +## Out of scope / hand-off + +- The five "Must verify before the first live batch" items from AJDA-3117 (PAYGO billing exclusion + for `keboola.flow-migration-tool`, notification-subscription visibility for ephemeral tokens, + trigger `runWithTokenId` permission, token lifetime vs. queue wait, `keboola.flow` availability + without a per-project feature) require live stack access — they are Ondrej's hand-off checklist, + to be confirmed on a real project (start with a one-project batch) and recorded in a Linear comment. +- Per-orchestration migrated/skipped/failed counts in the CSV: not possible until + `keboola/flow-migration-tool` exposes them in the job result (open question in the issue; + potential follow-up there, not here). From e67284b3484f600955cbb89e0253f8773b68d7b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= <12143866+ondrajodas@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:39:00 +0200 Subject: [PATCH 3/5] feat: AJDA-3117 add manage:migrate-orchestrations-to-flow batch driver Drives keboola.flow-migration-tool jobs across a list of projects on one stack; all migration logic stays in the component. Per project it creates an ephemeral storage token, skips disabled/deleted projects, projects without keboola.orchestrator configurations and projects with a live migration job, then supervises the job in a bounded concurrency window. Every input project gets a CSV row appended the moment it resolves, so an interrupted run stays auditable. Dry-run by default behind -f/--force (the job itself runs with dryRun: true), a failing project never aborts the batch and the exit code is 1 when any project failed. --- README.md | 48 ++ cli.php | 2 + ...0-migrate-orchestrations-to-flow-design.md | 35 +- .../Command/FlowMigration/BatchRunner.php | 441 ++++++++++++++++ .../Command/FlowMigration/ProjectClients.php | 23 + .../FlowMigration/ProjectClientsFactory.php | 72 +++ .../Command/FlowMigration/ProjectResult.php | 56 +++ .../Command/MigrateOrchestrationsToFlow.php | 408 +++++++++++++++ tests/FlowMigration/BatchRunnerTest.php | 475 ++++++++++++++++++ tests/FlowMigration/FakeJobQueueClient.php | 138 +++++ .../FlowMigration/FakeJobQueueClientTest.php | 41 ++ .../FakeProjectClientsFactory.php | 67 +++ tests/FlowMigration/ProjectResultTest.php | 40 ++ tests/MigrateOrchestrationsToFlowTest.php | 129 +++++ 14 files changed, 1960 insertions(+), 15 deletions(-) create mode 100644 src/Keboola/Console/Command/FlowMigration/BatchRunner.php create mode 100644 src/Keboola/Console/Command/FlowMigration/ProjectClients.php create mode 100644 src/Keboola/Console/Command/FlowMigration/ProjectClientsFactory.php create mode 100644 src/Keboola/Console/Command/FlowMigration/ProjectResult.php create mode 100644 src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php create mode 100644 tests/FlowMigration/BatchRunnerTest.php create mode 100644 tests/FlowMigration/FakeJobQueueClient.php create mode 100644 tests/FlowMigration/FakeJobQueueClientTest.php create mode 100644 tests/FlowMigration/FakeProjectClientsFactory.php create mode 100644 tests/FlowMigration/ProjectResultTest.php create mode 100644 tests/MigrateOrchestrationsToFlowTest.php diff --git a/README.md b/README.md index f95a596..e05a472 100644 --- a/README.md +++ b/README.md @@ -488,6 +488,54 @@ Behavior: - Prints a summary: projects checked/disabled/errored, configurations scanned/touched, tasks migrated/skipped (unsupported vs. unresolvable). +### Migrate keboola.orchestrator configurations to keboola.flow +Batch driver for the automated `keboola.orchestrator` → `keboola.flow` migration +(see [AJDA-3117](https://linear.app/keboola/issue/AJDA-3117)). All migration logic lives in the +`keboola.flow-migration-tool` component; this command only creates one migration job per project +and supervises the batch. Safe to re-run with the same list: already-migrated orchestrations are +reported as skipped by the component, and projects with a live migration job are skipped here. + +``` +php cli.php manage:migrate-orchestrations-to-flow [-f|--force] [] \ + [--projects-file=PATH] [--concurrency=10] [--poll-interval=5] [--report=PATH] +``` + +Arguments: +- `token` (required): Manage API token. +- `url` (required): Stack URL, including `https://` (e.g. `https://connection.north-europe.azure.keboola.com`). +- `projects` (optional): Comma-separated project IDs (e.g. `1,7,146`), or `@path/to/file` with one ID + per line (blank lines and `#` comments are ignored). Exactly one of `projects`/`--projects-file` + must be given. + +Options: +- `--force` / `-f`: Run the real migration. Without it, jobs are created with `dryRun: true`. + **Note:** even without `--force` a real `keboola.flow-migration-tool` job and a real ephemeral + storage token are created in every eligible project — on PAYGO stacks mind the billing. +- `--projects-file=PATH`: File with one project ID per line (alternative to `@file` in the argument). +- `--concurrency=N` (default 10): Max migration jobs in flight at once. +- `--poll-interval=N` (default 5): Seconds between job status polls. +- `--report=PATH` (default `flow-migration--.csv`): CSV report path. + +Behavior: +- For each project: skips disabled/deleted projects; creates an ephemeral 12h storage token + (`canManageBuckets`, `canReadAllFileUploads`, component access to `keboola.orchestrator`, + `keboola.flow`, `keboola.scheduler`, `keboola.flow-migration-tool`); skips projects with no + `keboola.orchestrator` configurations (no empty jobs in customers' job history); skips projects + where a `keboola.flow-migration-tool` job is already created/waiting/processing/terminating. +- Creates the migration job via `configData` (no stored configuration is left behind) with + `parameters: {mode: "project", orchestrationIds: [], skipBroken: true, dryRun: }`. +- Keeps at most `--concurrency` jobs in flight, polls each job and refills the window as jobs finish. + A transient poll failure is tolerated up to 3 consecutive times per job; after that the project is + reported as failed and the job is left to finish server-side (its job ID stays in the report). +- Appends a CSV row (`projectId;jobId;status;durationSeconds;error`) the moment each project + resolves, so an interrupted run is still auditable. Every input project gets a row; skipped + projects carry the skip reason in `status`/`error` and an empty `jobId`. Re-running with the same + `--report` path appends to the existing file without repeating the header. +- A failing project never aborts the batch. Exit code is `1` if at least one project failed + (job `error`/`terminated`/`cancelled` or a driver-side error), `0` otherwise. +- Final summary: projects attempted / migrated / migrated with warning / skipped (no + orchestrations, disabled, job already running) / failed. + ### Mass enablement of dynamic backends for multiple projects Prerequisities: https://keboola.atlassian.net/wiki/spaces/KB/pages/2135982081/Enable+Dynamic+Backends#Enable-for-project diff --git a/cli.php b/cli.php index 2908e23..4fd4f42 100644 --- a/cli.php +++ b/cli.php @@ -21,6 +21,7 @@ use Keboola\Console\Command\MassProjectEnableDynamicBackends; use Keboola\Console\Command\MassProjectExtendExpiration; use Keboola\Console\Command\MigrateDataAppsOrchestratorTasks; +use Keboola\Console\Command\MigrateOrchestrationsToFlow; use Keboola\Console\Command\OrganizationIntoMaintenanceMode; use Keboola\Console\Command\OrganizationResetWorkspacePasswords; use Keboola\Console\Command\OrganizationsAddFeature; @@ -48,6 +49,7 @@ $application->add(new MassProjectExtendExpiration()); $application->add(new MassProjectEnableDynamicBackends()); $application->add(new MigrateDataAppsOrchestratorTasks()); +$application->add(new MigrateOrchestrationsToFlow()); $application->add(new AddFeature()); $application->add(new CleanupLeakedTestFeatures()); $application->add(new AllStacksIterator()); diff --git a/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md b/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md index 127562e..aa4eca1 100644 --- a/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md +++ b/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md @@ -80,33 +80,36 @@ this at startup, and the README documents it (relevant for PAYGO billing — see ## Architecture -Four small classes in `src/Keboola/Console/Command/` (PSR-0: namespace -`Keboola\Console\Command`, path = file name), plus registration in `cli.php`: +Only the Symfony command itself lives directly in `src/Keboola/Console/Command/`; its four +helper classes go into the `FlowMigration/` subnamespace (PSR-0: namespace +`Keboola\Console\Command\FlowMigration` → `src/Keboola/Console/Command/FlowMigration/`), which +keeps the flat command directory a list of commands. The helper class names therefore drop the +redundant `FlowMigration` prefix the namespace already carries. Registration goes in `cli.php`: ``` -MigrateOrchestrationsToFlow (Symfony Command — thin shell) +Command/MigrateOrchestrationsToFlow (Symfony Command — thin shell) ├─ parses/validates input, resolves project ID list - ├─ builds ManageApi\Client, ServiceClient, FlowMigrationProjectClientsFactory + ├─ builds ManageApi\Client, ServiceClient, FlowMigration\ProjectClientsFactory ├─ opens the CSV report (append mode, header if new/empty) and wires the │ per-result callback: CSV row + progress line to stdout - ├─ runs FlowMigrationBatchRunner + ├─ runs FlowMigration\BatchRunner └─ prints final summary, returns exit code (1 if any project failed) -FlowMigrationBatchRunner (plain class — ALL batch logic, unit-tested) +Command/FlowMigration/BatchRunner (plain class — ALL batch logic, unit-tested) ├─ per-project pipeline (skip rules, job submission) ├─ concurrency window + polling loop - └─ emits one FlowMigrationProjectResult per input project via callback, + └─ emits one ProjectResult per input project via callback, returns aggregate summary counts -FlowMigrationProjectClientsFactory (plain class — the only network seam) +Command/FlowMigration/ProjectClientsFactory (plain class — the only network seam) ├─ getProject(string $projectId): array (Manage API) - └─ createProjectClients(string $projectId): FlowMigrationProjectClients + └─ createProjectClients(string $projectId): ProjectClients creates the ephemeral storage token, returns Components + JobQueueClient bound to that token -FlowMigrationProjectClients (tiny readonly DTO: Components + JobQueueClient) -FlowMigrationProjectResult (tiny readonly DTO: projectId, jobId, status, - durationSeconds, error + isFailed()) +Command/FlowMigration/ProjectClients (tiny DTO: Components + JobQueueClient) +Command/FlowMigration/ProjectResult (tiny DTO: projectId, jobId, status, + durationSeconds, error + isFailed()) ``` Rationale: the repo's testable-logic pattern (`DataAppOrchestratorTaskMigrator` + @@ -233,15 +236,17 @@ guard skips projects with a live migration job. No resume state is kept by the d ## Testing -`tests/FlowMigrationBatchRunnerTest.php` + fakes (PSR-4 `Keboola\Console\Tests\`): +`tests/FlowMigration/BatchRunnerTest.php` + fakes, mirroring the src layout in the PSR-4 +subnamespace `Keboola\Console\Tests\FlowMigration\` (shared `FakeComponents` stays in +`Keboola\Console\Tests\`): - `FakeJobQueueClient extends JobQueueClient\Client` — constructor override (no parent call, same trick as `FakeComponents`), records `createJob` calls, scripted `getJob` status sequences (e.g. `processing, processing, success`), scripted `listJobs` guard responses, can throw on demand for poll-failure tests. -- `FakeFlowMigrationProjectClientsFactory extends FlowMigrationProjectClientsFactory` — scripted +- `FakeProjectClientsFactory extends ProjectClientsFactory` — scripted projects (disabled / deleted / erroring), records `createProjectClients` calls (asserts no token - is created for disabled projects), returns `FlowMigrationProjectClients` built from + is created for disabled projects), returns `ProjectClients` built from `FakeComponents` (reused as-is for the orchestrator-config listing) + `FakeJobQueueClient`. Scenarios (assert emitted results, summary counts, recorded API calls, sleep-callable cadence): diff --git a/src/Keboola/Console/Command/FlowMigration/BatchRunner.php b/src/Keboola/Console/Command/FlowMigration/BatchRunner.php new file mode 100644 index 0000000..54291c4 --- /dev/null +++ b/src/Keboola/Console/Command/FlowMigration/BatchRunner.php @@ -0,0 +1,441 @@ +clientsFactory = $clientsFactory; + // A window smaller than one job would never let the run loop drain the pending queue, + // i.e. it would hang the batch forever - clamp instead of spinning. + $this->concurrency = max(1, $concurrency); + $this->pollIntervalSeconds = $pollIntervalSeconds; + $this->sleep = $sleep ?? function (int $seconds): void { + sleep($seconds); + }; + } + + /** + * @param array $projectIds + * @param callable(ProjectResult): void $onProjectFinished invoked once per input project + * @return BatchSummary + */ + public function run(array $projectIds, bool $force, OutputInterface $output, callable $onProjectFinished): array + { + $pending = array_values(array_unique($projectIds)); + $summary = [ + 'attempted' => count($pending), + 'migrated' => 0, + 'migratedWithWarning' => 0, + 'skippedNoOrchestrations' => 0, + 'skippedDisabled' => 0, + 'skippedJobRunning' => 0, + 'failed' => 0, + ]; + /** @var array $inFlight */ + $inFlight = []; + + while ($pending !== [] || $inFlight !== []) { + while (count($inFlight) < $this->concurrency && $pending !== []) { + $submission = $this->submitProject(array_shift($pending), $force, $output); + if ($submission instanceof ProjectResult) { + $this->recordResult($submission, $summary, $output, $onProjectFinished); + continue; + } + $inFlight[] = $submission; + } + + if ($inFlight === []) { + continue; + } + + ($this->sleep)($this->pollIntervalSeconds); + $inFlight = $this->pollInFlightJobs($inFlight, $summary, $output, $onProjectFinished); + } + + return $summary; + } + + /** + * Runs the per-project pipeline up to job creation. Returns an in-flight slot on success, + * or an immediately-final ProjectResult (skip or driver-side error). + * + * Order matters: the disabled/deleted check runs before any token is created, and the + * configuration check runs before the queue guard so empty projects never appear in + * customers' job history. + * + * @return ProjectResult|InFlightJob + */ + private function submitProject(string $projectId, bool $force, OutputInterface $output) + { + try { + $resolved = $this->checkProjectIsActive($projectId); + if ($resolved !== null) { + return $resolved; + } + + $clients = $this->clientsFactory->createProjectClients($projectId); + + $resolved = $this->checkProjectNeedsMigration($projectId, $clients); + if ($resolved !== null) { + return $resolved; + } + + $job = $clients->queueClient->createJob($this->buildMigrationJobData($force)); + } catch (Throwable $e) { + return new ProjectResult( + $projectId, + null, + ProjectResult::STATUS_ERROR, + null, + $e->getMessage() + ); + } + + $this->writeLine($output, sprintf('Project %s: created job %s (%s)', $projectId, $job->id, $job->url)); + + return [ + 'projectId' => $projectId, + 'jobId' => $job->id, + 'queueClient' => $clients->queueClient, + 'startedAt' => microtime(true), + 'pollFailures' => 0, + ]; + } + + /** + * Runs before any ephemeral token is created. A deleted project (Manage API 404) is reported + * together with disabled ones - the migration has nothing to do in either case. Any other + * failure is rethrown so the caller turns it into a per-project error result. + * + * @return ProjectResult|null non-null when the project must not be migrated + */ + private function checkProjectIsActive(string $projectId): ?ProjectResult + { + try { + $project = $this->clientsFactory->getProject($projectId); + } catch (ManageClientException $e) { + if ($e->getCode() !== 404) { + throw $e; + } + + return new ProjectResult( + $projectId, + null, + ProjectResult::STATUS_SKIPPED_DISABLED, + null, + 'project is deleted' + ); + } + + if (isset($project['isDisabled']) && $project['isDisabled']) { + return new ProjectResult( + $projectId, + null, + ProjectResult::STATUS_SKIPPED_DISABLED, + null, + 'project is disabled' + ); + } + + return null; + } + + /** + * @return ProjectResult|null non-null when no new migration job should be created + */ + private function checkProjectNeedsMigration( + string $projectId, + ProjectClients $clients + ): ?ProjectResult { + $configurations = $clients->components->listComponentConfigurations( + (new ListComponentConfigurationsOptions()) + ->setComponentId(self::ORCHESTRATOR_COMPONENT_ID) + ->setIsDeleted(false) + ); + if (count($configurations) === 0) { + return new ProjectResult( + $projectId, + null, + ProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, + null, + 'no keboola.orchestrator configurations' + ); + } + + $liveJobs = $clients->queueClient->listJobs( + (new ListJobsOptions()) + ->setComponents([self::MIGRATION_COMPONENT_ID]) + ->setStatuses(self::LIVE_JOB_STATUSES) + ->setLimit(1) + ); + if ($liveJobs !== []) { + return new ProjectResult( + $projectId, + null, + ProjectResult::STATUS_SKIPPED_JOB_RUNNING, + null, + 'a keboola.flow-migration-tool job is already running in this project' + ); + } + + return null; + } + + /** + * The migration is requested through configData so no stored configuration is left behind + * in the customer's project. orchestrationIds and skipBroken are required by the component's + * config definition in "project" mode. + */ + private function buildMigrationJobData(bool $force): JobData + { + return new JobData( + self::MIGRATION_COMPONENT_ID, + null, + [ + 'parameters' => [ + 'mode' => 'project', + 'orchestrationIds' => [], + 'skipBroken' => true, + 'dryRun' => !$force, + ], + ] + ); + } + + /** + * Polls every in-flight job once and returns the slots that are still running. + * + * @param array $inFlight + * @param BatchSummary $summary + * @param callable(ProjectResult): void $onProjectFinished + * @return array + */ + private function pollInFlightJobs( + array $inFlight, + array &$summary, + OutputInterface $output, + callable $onProjectFinished + ): array { + $stillRunning = []; + foreach ($inFlight as $slot) { + $polled = $this->pollOneJob($slot, $summary, $output, $onProjectFinished); + if ($polled !== null) { + $stillRunning[] = $polled; + } + } + + return $stillRunning; + } + + /** + * @param InFlightJob $slot + * @param BatchSummary $summary + * @param callable(ProjectResult): void $onProjectFinished + * @return InFlightJob|null null once the project is resolved and its slot is freed + */ + private function pollOneJob( + array $slot, + array &$summary, + OutputInterface $output, + callable $onProjectFinished + ): ?array { + try { + $job = $slot['queueClient']->getJob($slot['jobId']); + } catch (Throwable $e) { + return $this->handlePollFailure($slot, $e, $summary, $output, $onProjectFinished); + } + + $slot['pollFailures'] = 0; + + if (!$job->isFinished) { + return $slot; + } + + $this->recordResult($this->buildFinishedJobResult($slot, $job), $summary, $output, $onProjectFinished); + + return null; + } + + /** + * Keeps the slot in flight until MAX_CONSECUTIVE_POLL_FAILURES is reached, then resolves the + * project as failed while preserving the job id - the job itself may still finish server-side. + * + * @param InFlightJob $slot + * @param BatchSummary $summary + * @param callable(ProjectResult): void $onProjectFinished + * @return InFlightJob|null + */ + private function handlePollFailure( + array $slot, + Throwable $error, + array &$summary, + OutputInterface $output, + callable $onProjectFinished + ): ?array { + $slot['pollFailures']++; + $this->writeLine($output, sprintf( + 'Project %s: polling job %s failed (%d/%d): %s', + $slot['projectId'], + $slot['jobId'], + $slot['pollFailures'], + self::MAX_CONSECUTIVE_POLL_FAILURES, + $error->getMessage() + )); + + if ($slot['pollFailures'] < self::MAX_CONSECUTIVE_POLL_FAILURES) { + return $slot; + } + + $this->recordResult( + new ProjectResult( + $slot['projectId'], + $slot['jobId'], + ProjectResult::STATUS_ERROR, + null, + sprintf( + 'polling gave up after %d consecutive failures, job may still be running: %s', + self::MAX_CONSECUTIVE_POLL_FAILURES, + $error->getMessage() + ) + ), + $summary, + $output, + $onProjectFinished + ); + + return null; + } + + /** + * @param InFlightJob $slot + */ + private function buildFinishedJobResult(array $slot, Job $job): ProjectResult + { + return new ProjectResult( + $slot['projectId'], + $slot['jobId'], + $job->status, + $job->durationSeconds ?? (int) round(microtime(true) - $slot['startedAt']), + $this->extractJobError($job) + ); + } + + private function extractJobError(Job $job): ?string + { + if ($job->isSuccess()) { + return null; + } + $result = $job->result; + if (is_array($result) && isset($result['message']) && is_scalar($result['message'])) { + return (string) $result['message']; + } + + return null; + } + + /** + * @param BatchSummary $summary + * @param callable(ProjectResult): void $onProjectFinished + */ + private function recordResult( + ProjectResult $result, + array &$summary, + OutputInterface $output, + callable $onProjectFinished + ): void { + $summary[$this->summaryKeyFor($result)]++; + + $this->writeLine($output, sprintf( + 'Project %s: %s%s%s', + $result->projectId, + $result->status, + $result->durationSeconds !== null ? sprintf(' in %d s', $result->durationSeconds) : '', + $result->error !== null ? sprintf(' (%s)', $result->error) : '' + )); + + $onProjectFinished($result); + } + + /** + * @return key-of + */ + private function summaryKeyFor(ProjectResult $result): string + { + return match ($result->status) { + JobStatuses::SUCCESS->value => 'migrated', + JobStatuses::WARNING->value => 'migratedWithWarning', + ProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS => 'skippedNoOrchestrations', + ProjectResult::STATUS_SKIPPED_DISABLED => 'skippedDisabled', + ProjectResult::STATUS_SKIPPED_JOB_RUNNING => 'skippedJobRunning', + default => 'failed', + }; + } + + private function writeLine(OutputInterface $output, string $message): void + { + $output->writeln(sprintf('[%s] %s', date('H:i:s'), $message)); + } +} diff --git a/src/Keboola/Console/Command/FlowMigration/ProjectClients.php b/src/Keboola/Console/Command/FlowMigration/ProjectClients.php new file mode 100644 index 0000000..fb9b7d6 --- /dev/null +++ b/src/Keboola/Console/Command/FlowMigration/ProjectClients.php @@ -0,0 +1,23 @@ +components = $components; + $this->queueClient = $queueClient; + } +} diff --git a/src/Keboola/Console/Command/FlowMigration/ProjectClientsFactory.php b/src/Keboola/Console/Command/FlowMigration/ProjectClientsFactory.php new file mode 100644 index 0000000..b41b06d --- /dev/null +++ b/src/Keboola/Console/Command/FlowMigration/ProjectClientsFactory.php @@ -0,0 +1,72 @@ +manageClient = $manageClient; + $this->connectionUrl = $connectionUrl; + $this->queueApiUrl = $queueApiUrl; + } + + /** + * @return array Manage API project detail + */ + public function getProject(string $projectId): array + { + return $this->manageClient->getProject($projectId); + } + + public function createProjectClients(string $projectId): ProjectClients + { + $tokenInfo = $this->manageClient->createProjectStorageToken($projectId, [ + 'description' => self::TOKEN_DESCRIPTION, + 'expiresIn' => self::TOKEN_EXPIRES_IN_SECONDS, + 'canManageBuckets' => true, + 'canReadAllFileUploads' => true, + 'componentAccess' => self::TOKEN_COMPONENT_ACCESS, + ]); + + $storageClient = new StorageClient([ + 'url' => $this->connectionUrl, + 'token' => $tokenInfo['token'], + ]); + + return new ProjectClients( + new Components($storageClient), + new JobQueueClient($this->queueApiUrl, $tokenInfo['token']) + ); + } +} diff --git a/src/Keboola/Console/Command/FlowMigration/ProjectResult.php b/src/Keboola/Console/Command/FlowMigration/ProjectResult.php new file mode 100644 index 0000000..38bbd50 --- /dev/null +++ b/src/Keboola/Console/Command/FlowMigration/ProjectResult.php @@ -0,0 +1,56 @@ +projectId = $projectId; + $this->jobId = $jobId; + $this->status = $status; + $this->durationSeconds = $durationSeconds; + $this->error = $error; + } + + public function isSkipped(): bool + { + return in_array($this->status, [ + self::STATUS_SKIPPED_DISABLED, + self::STATUS_SKIPPED_NO_ORCHESTRATIONS, + self::STATUS_SKIPPED_JOB_RUNNING, + ], true); + } + + public function isFailed(): bool + { + // Anything that is not a skip and not a successful terminal job status is a failure - + // unexpected statuses fail loud rather than passing silently. + return !$this->isSkipped() + && !in_array($this->status, [JobStatuses::SUCCESS->value, JobStatuses::WARNING->value], true); + } +} diff --git a/src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php b/src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php new file mode 100644 index 0000000..73d5caa --- /dev/null +++ b/src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php @@ -0,0 +1,408 @@ + keboola.flow migration (AJDA-3117). + * All migration logic lives in the keboola.flow-migration-tool component; this command only + * creates and supervises its jobs across a list of projects on one stack. + */ +class MigrateOrchestrationsToFlow extends Command +{ + const ARG_TOKEN = 'token'; + const ARG_URL = 'url'; + const ARG_PROJECTS = 'projects'; + const OPT_FORCE = 'force'; + const OPT_PROJECTS_FILE = 'projects-file'; + const OPT_CONCURRENCY = 'concurrency'; + const OPT_POLL_INTERVAL = 'poll-interval'; + const OPT_REPORT = 'report'; + + private const ERROR_ONE_PROJECT_SOURCE = + 'Provide exactly one source of project IDs: the argument or --projects-file'; + + private const CSV_HEADER = ['projectId', 'jobId', 'status', 'durationSeconds', 'error']; + private const CSV_DELIMITER = ';'; + private const CSV_ENCLOSURE = '"'; + // No proprietary escaping (same default as keboola/csv): quotes are doubled, so an API error + // message containing \" cannot break the row for Excel/Sheets or any RFC-4180 parser. + private const CSV_ESCAPE = ''; + + protected function configure(): void + { + $this + ->setName('manage:migrate-orchestrations-to-flow') + ->setDescription( + 'Run the automated keboola.orchestrator -> keboola.flow migration for a batch of projects' + ) + ->addArgument(self::ARG_TOKEN, InputArgument::REQUIRED, 'Manage API token') + ->addArgument( + self::ARG_URL, + InputArgument::REQUIRED, + 'Stack URL, e.g. https://connection.north-europe.azure.keboola.com' + ) + ->addArgument( + self::ARG_PROJECTS, + InputArgument::OPTIONAL, + 'Comma-separated project IDs, or @path/to/file with one ID per line' + ) + ->addOption( + self::OPT_FORCE, + 'f', + InputOption::VALUE_NONE, + 'Run the real migration; without it jobs are created with dryRun: true' + ) + ->addOption( + self::OPT_PROJECTS_FILE, + null, + InputOption::VALUE_REQUIRED, + 'File with one project ID per line (alternative to @file in the argument)' + ) + ->addOption( + self::OPT_CONCURRENCY, + null, + InputOption::VALUE_REQUIRED, + 'Max migration jobs in flight at once', + '10' + ) + ->addOption( + self::OPT_POLL_INTERVAL, + null, + InputOption::VALUE_REQUIRED, + 'Seconds between job status polls', + '5' + ) + ->addOption( + self::OPT_REPORT, + null, + InputOption::VALUE_REQUIRED, + 'CSV report path (default: flow-migration--.csv)' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $token = $input->getArgument(self::ARG_TOKEN); + assert(is_string($token)); + $url = $input->getArgument(self::ARG_URL); + assert(is_string($url)); + $force = (bool) $input->getOption(self::OPT_FORCE); + + $hostnameSuffix = $this->hostnameSuffixFromUrl($url); + if ($hostnameSuffix === null) { + $output->writeln(sprintf( + 'Invalid stack URL "%s": expected a URL like https://connection.keboola.com', + $url + )); + return 1; + } + + $projectIds = $this->resolveProjectIds($input, $output); + if ($projectIds === null) { + return 1; + } + + $concurrency = $this->parsePositiveIntOption($input, self::OPT_CONCURRENCY); + $pollInterval = $this->parsePositiveIntOption($input, self::OPT_POLL_INTERVAL); + if ($concurrency === null || $pollInterval === null) { + $output->writeln('Options --concurrency and --poll-interval must be positive integers'); + return 1; + } + + $reportPath = $this->resolveReportPath($input, $hostnameSuffix); + $this->printRunNotice($output, $force, count($projectIds), $concurrency, $pollInterval, $reportPath); + + $reportHandle = $this->openReport($reportPath, $output); + if ($reportHandle === null) { + return 1; + } + + $manageClient = new ManageClient(['url' => $url, 'token' => $token]); + $clientsFactory = new ProjectClientsFactory( + $manageClient, + $url, + (new ServiceClient($hostnameSuffix))->getQueueUrl() + ); + + $runner = new BatchRunner($clientsFactory, $concurrency, $pollInterval); + $summary = $runner->run( + $projectIds, + $force, + $output, + function (ProjectResult $result) use ($reportHandle): void { + $this->appendReportRow($reportHandle, $result); + } + ); + fclose($reportHandle); + + $this->printSummary($output, $summary); + + return $summary['failed'] > 0 ? 1 : 0; + } + + /** + * Derives the ServiceClient hostname suffix from a full connection URL, e.g. + * "https://connection.north-europe.azure.keboola.com" -> "north-europe.azure.keboola.com". + * Returns null when the URL does not look like a stack connection URL. + * + * @return non-empty-string|null + */ + private function hostnameSuffixFromUrl(string $url): ?string + { + $host = parse_url($url, PHP_URL_HOST); + if (!is_string($host) || !str_starts_with($host, 'connection.')) { + return null; + } + $suffix = substr($host, strlen('connection.')); + + return $suffix === '' ? null : $suffix; + } + + /** + * Resolves the project ID list from exactly one source: the argument + * (inline list or @file) or --projects-file. Prints an error and returns null otherwise. + * + * @return array|null + */ + private function resolveProjectIds(InputInterface $input, OutputInterface $output): ?array + { + $inlineList = $this->optionalStringInput($input->getArgument(self::ARG_PROJECTS)); + $filePath = $this->optionalStringInput($input->getOption(self::OPT_PROJECTS_FILE)); + + if ($inlineList !== null && $filePath !== null) { + $output->writeln(self::ERROR_ONE_PROJECT_SOURCE); + return null; + } + + // "@path" in the argument is shorthand for --projects-file=path. + if ($inlineList !== null && str_starts_with($inlineList, '@')) { + $filePath = substr($inlineList, 1); + $inlineList = null; + } + + if ($filePath !== null) { + return $this->readProjectIdsFile($filePath, $output); + } + + if ($inlineList === null) { + $output->writeln(self::ERROR_ONE_PROJECT_SOURCE); + return null; + } + + return $this->rejectEmptyProjectIds($this->parseProjectIdList($inlineList), $output); + } + + /** + * @return non-empty-string|null null for a missing or empty console input value + */ + private function optionalStringInput(mixed $value): ?string + { + return is_string($value) && $value !== '' ? $value : null; + } + + /** + * @return array|null + */ + private function readProjectIdsFile(string $path, OutputInterface $output): ?array + { + $contents = @file_get_contents($path); + if ($contents === false) { + $output->writeln(sprintf('Cannot read projects file "%s"', $path)); + return null; + } + + return $this->rejectEmptyProjectIds($this->parseProjectIdsFile($contents), $output); + } + + /** + * @param array|null $projectIds null when parsing found a non-numeric ID + * @return array|null + */ + private function rejectEmptyProjectIds(?array $projectIds, OutputInterface $output): ?array + { + if ($projectIds === null || $projectIds === []) { + $output->writeln('Projects list is empty or contains a non-numeric ID'); + return null; + } + + return $projectIds; + } + + /** + * @return array|null null when any entry is not a plain non-negative integer + */ + private function parseProjectIdList(string $raw): ?array + { + return $this->validateAndDeduplicate(array_map('trim', explode(',', $raw))); + } + + /** + * One ID per line; blank lines and lines starting with "#" are ignored. + * + * @return array|null null when any remaining line is not a plain non-negative integer + */ + private function parseProjectIdsFile(string $contents): ?array + { + $lines = preg_split('/\R/', $contents); + $ids = []; + foreach ($lines === false ? [] : $lines as $line) { + $line = trim($line); + if ($line === '' || str_starts_with($line, '#')) { + continue; + } + $ids[] = $line; + } + + return $this->validateAndDeduplicate($ids); + } + + /** + * @param array $ids + * @return array|null + */ + private function validateAndDeduplicate(array $ids): ?array + { + foreach ($ids as $id) { + if (!ctype_digit($id)) { + return null; + } + } + + return array_values(array_unique($ids)); + } + + private function parsePositiveIntOption(InputInterface $input, string $name): ?int + { + $value = $input->getOption($name); + if (!is_string($value) || !ctype_digit($value) || (int) $value < 1) { + return null; + } + + return (int) $value; + } + + private function resolveReportPath(InputInterface $input, string $hostnameSuffix): string + { + $reportPath = $input->getOption(self::OPT_REPORT); + if (is_string($reportPath) && $reportPath !== '') { + return $reportPath; + } + + return sprintf('flow-migration-%s-%s.csv', $hostnameSuffix, date('Ymd-His')); + } + + private function printRunNotice( + OutputInterface $output, + bool $force, + int $projectCount, + int $concurrency, + int $pollInterval, + string $reportPath + ): void { + if ($force) { + $output->writeln('Running in FORCE mode: migration jobs run with dryRun: false.'); + } else { + $output->writeln( + 'Running in dry-run mode: migration jobs run with dryRun: true. Use -f for the real migration.' + ); + // The usual cli-utils dry-run changes nothing at all - this one still spends a job slot + // (and on PAYGO stacks, credits) in every eligible project, so say it out loud. + $output->writeln('NOTE: even in dry-run mode a real keboola.flow-migration-tool job and a real' + . ' ephemeral storage token are created in every eligible project.'); + } + $output->writeln(sprintf( + 'Projects: %d, concurrency: %d, poll interval: %d s', + $projectCount, + $concurrency, + $pollInterval + )); + $output->writeln(sprintf('Report: %s', $reportPath)); + $output->writeln(''); + } + + /** + * Opens the CSV report in append mode and writes the header only for a new or empty file, so + * re-running with the same --report path keeps one continuous, valid CSV. + * + * @return resource|null null when the file cannot be opened + */ + private function openReport(string $reportPath, OutputInterface $output) + { + $needsHeader = !is_file($reportPath) || filesize($reportPath) === 0; + + $reportHandle = fopen($reportPath, 'a'); + if ($reportHandle === false) { + $output->writeln(sprintf('Cannot open report file "%s" for writing', $reportPath)); + return null; + } + + if ($needsHeader) { + fputcsv($reportHandle, self::CSV_HEADER, self::CSV_DELIMITER, self::CSV_ENCLOSURE, self::CSV_ESCAPE); + } + + return $reportHandle; + } + + /** + * @param resource $reportHandle + */ + private function appendReportRow($reportHandle, ProjectResult $result): void + { + fputcsv( + $reportHandle, + [ + $result->projectId, + $result->jobId ?? '', + $result->status, + $result->durationSeconds !== null ? (string) $result->durationSeconds : '', + $result->error ?? '', + ], + self::CSV_DELIMITER, + self::CSV_ENCLOSURE, + self::CSV_ESCAPE + ); + // Flush per row so an interrupted run still leaves an auditable report. + fflush($reportHandle); + } + + /** + * @param array{ + * attempted: int, + * migrated: int, + * migratedWithWarning: int, + * skippedNoOrchestrations: int, + * skippedDisabled: int, + * skippedJobRunning: int, + * failed: int + * } $summary + */ + private function printSummary(OutputInterface $output, array $summary): void + { + $output->writeln(''); + $output->writeln(sprintf( + "DONE\nProjects attempted: %d\nMigrated (job success): %d\nMigrated with warning: %d\n" + . "Skipped (no orchestrations): %d\nSkipped (disabled/deleted): %d\n" + . "Skipped (migration job already running): %d\nFailed: %d", + $summary['attempted'], + $summary['migrated'], + $summary['migratedWithWarning'], + $summary['skippedNoOrchestrations'], + $summary['skippedDisabled'], + $summary['skippedJobRunning'], + $summary['failed'] + )); + } +} diff --git a/tests/FlowMigration/BatchRunnerTest.php b/tests/FlowMigration/BatchRunnerTest.php new file mode 100644 index 0000000..96c88a3 --- /dev/null +++ b/tests/FlowMigration/BatchRunnerTest.php @@ -0,0 +1,475 @@ + */ + private array $results = []; + + /** @var array */ + private array $sleeps = []; + + private function collector(): Closure + { + return function (ProjectResult $result): void { + $this->results[] = $result; + }; + } + + private function sleepRecorder(): Closure + { + return function (int $seconds): void { + $this->sleeps[] = $seconds; + }; + } + + /** + * @return array enabled project detail as returned by the Manage API + */ + private static function enabledProject(string $id): array + { + return ['id' => $id, 'name' => 'Project ' . $id, 'isDisabled' => false]; + } + + private static function clientsWith( + FakeJobQueueClient $queueClient, + bool $hasOrchestrations = true + ): ProjectClients { + $configs = $hasOrchestrations + ? ['keboola.orchestrator' => [['id' => 'orch-1', 'name' => 'Daily load', 'configuration' => []]]] + : []; + + return new ProjectClients(new FakeComponents($configs), $queueClient); + } + + public function testHappyPathCreatesJobAndReportsSuccess(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [ + FakeJobQueueClient::makeJob('job-1', 'processing'), + FakeJobQueueClient::makeJob('job-1', 'success', 42), + ]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + // Exact job payload - this is the whole contract with keboola.flow-migration-tool. + $this->assertCount(1, $queueClient->createdJobs); + $this->assertSame('keboola.flow-migration-tool', $queueClient->createdJobs[0]['component']); + $this->assertNull($queueClient->createdJobs[0]['config']); + $this->assertSame('run', $queueClient->createdJobs[0]['mode']); + $this->assertSame( + ['parameters' => [ + 'mode' => 'project', + 'orchestrationIds' => [], + 'skipBroken' => true, + 'dryRun' => false, + ]], + $queueClient->createdJobs[0]['configData'] + ); + // The live-job guard ran exactly once before submission. + $this->assertSame(1, $queueClient->listJobsCalls); + + $this->assertCount(1, $this->results); + $this->assertSame('100', $this->results[0]->projectId); + $this->assertSame('job-1', $this->results[0]->jobId); + $this->assertSame('success', $this->results[0]->status); + $this->assertSame(42, $this->results[0]->durationSeconds); + $this->assertNull($this->results[0]->error); + + // Two poll sweeps (processing, then success), each preceded by one poll-interval sleep. + $this->assertSame([5, 5], $this->sleeps); + + $this->assertSame(1, $summary['attempted']); + $this->assertSame(1, $summary['migrated']); + $this->assertSame(0, $summary['failed']); + } + + public function testWithoutForceJobRunsWithDryRunTrue(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'success', 1)]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $runner->run(['100'], false, new BufferedOutput(), $this->collector()); + + $configData = $queueClient->createdJobs[0]['configData']; + $this->assertIsArray($configData); + $this->assertTrue($configData['parameters']['dryRun']); + } + + public function testJobEndingInErrorMarksProjectFailedButBatchContinues(): void + { + $queueClient1 = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'error', 10, ['message' => 'boom'])]] + ); + $queueClient2 = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-2', 'created')], + ['job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 20)]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100'), '200' => self::enabledProject('200')], + ['100' => self::clientsWith($queueClient1), '200' => self::clientsWith($queueClient2)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); + + $this->assertCount(2, $this->results); + $byProject = []; + foreach ($this->results as $result) { + $byProject[$result->projectId] = $result; + } + $this->assertSame('error', $byProject['100']->status); + $this->assertSame('boom', $byProject['100']->error); + $this->assertTrue($byProject['100']->isFailed()); + $this->assertSame('success', $byProject['200']->status); + + $this->assertSame(2, $summary['attempted']); + $this->assertSame(1, $summary['migrated']); + $this->assertSame(1, $summary['failed']); + } + + public function testWarningJobCountsAsMigratedWithWarning(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'warning', 5, ['message' => 'partial'])]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame('warning', $this->results[0]->status); + $this->assertFalse($this->results[0]->isFailed()); + $this->assertSame(1, $summary['migratedWithWarning']); + $this->assertSame(0, $summary['migrated']); + $this->assertSame(0, $summary['failed']); + } + + public function testSkipsDisabledProjectWithoutCreatingTokenOrJob(): void + { + $factory = new FakeProjectClientsFactory( + ['100' => ['id' => '100', 'name' => 'Off', 'isDisabled' => true]] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + // No ephemeral token may be created for a disabled project. + $this->assertSame([], $factory->createClientsCalls); + $this->assertSame(ProjectResult::STATUS_SKIPPED_DISABLED, $this->results[0]->status); + $this->assertNull($this->results[0]->jobId); + $this->assertSame(1, $summary['skippedDisabled']); + $this->assertSame(0, $summary['failed']); + $this->assertSame([], $this->sleeps); + } + + public function testSkipsDeletedProjectOnManage404(): void + { + $factory = new FakeProjectClientsFactory( + ['100' => new ManageClientException('Project not found', 404)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame([], $factory->createClientsCalls); + $this->assertSame(ProjectResult::STATUS_SKIPPED_DISABLED, $this->results[0]->status); + $this->assertSame(1, $summary['skippedDisabled']); + } + + public function testManageErrorOtherThan404MarksProjectFailed(): void + { + $factory = new FakeProjectClientsFactory( + ['100' => new ManageClientException('Internal error', 500)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame(ProjectResult::STATUS_ERROR, $this->results[0]->status); + $this->assertSame('Internal error', $this->results[0]->error); + $this->assertSame(1, $summary['failed']); + } + + public function testTransportFailureOnProjectLookupDoesNotAbortTheBatch(): void + { + // A DNS/connect failure surfaces as a Guzzle ConnectException, not a Manage API + // ClientException - it must still resolve as a per-project error. + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-2', 'created')], + ['job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 2)]] + ); + $factory = new FakeProjectClientsFactory( + [ + '100' => new ConnectException('cURL error 6: Could not resolve host', new Request('GET', '/')), + '200' => self::enabledProject('200'), + ], + ['200' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); + + $byProject = []; + foreach ($this->results as $result) { + $byProject[$result->projectId] = $result; + } + $this->assertSame(ProjectResult::STATUS_ERROR, $byProject['100']->status); + $this->assertSame('success', $byProject['200']->status); + $this->assertSame(1, $summary['failed']); + $this->assertSame(1, $summary['migrated']); + } + + public function testSkipsProjectWithoutOrchestratorConfigurations(): void + { + $queueClient = new FakeJobQueueClient(); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient, false)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + // No queue API call and no job for a project with nothing to migrate. + $this->assertSame(0, $queueClient->listJobsCalls); + $this->assertSame([], $queueClient->createdJobs); + $this->assertSame(ProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, $this->results[0]->status); + $this->assertSame(1, $summary['skippedNoOrchestrations']); + } + + public function testSkipsProjectWithLiveMigrationJob(): void + { + $queueClient = new FakeJobQueueClient( + [], + [], + [FakeJobQueueClient::makeJob('existing-job', 'processing')] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame([], $queueClient->createdJobs); + $this->assertSame(ProjectResult::STATUS_SKIPPED_JOB_RUNNING, $this->results[0]->status); + $this->assertSame(1, $summary['skippedJobRunning']); + } + + public function testDriverSideErrorWhenTokenCreationFailsAndBatchContinues(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-2', 'created')], + ['job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 3)]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100'), '200' => self::enabledProject('200')], + [ + '100' => new ManageClientException('Cannot create token', 403), + '200' => self::clientsWith($queueClient), + ] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); + + $byProject = []; + foreach ($this->results as $result) { + $byProject[$result->projectId] = $result; + } + $this->assertSame(ProjectResult::STATUS_ERROR, $byProject['100']->status); + $this->assertSame('Cannot create token', $byProject['100']->error); + $this->assertSame('success', $byProject['200']->status); + $this->assertSame(1, $summary['failed']); + $this->assertSame(1, $summary['migrated']); + } + + public function testDeduplicatesInputProjectIds(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'success', 1)]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100', '100', '100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame(1, $summary['attempted']); + $this->assertCount(1, $queueClient->createdJobs); + $this->assertCount(1, $this->results); + } + + public function testNonPositiveConcurrencyStillDrainsTheQueueInsteadOfHanging(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'success', 1)]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 0, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame(1, $summary['migrated']); + } + + public function testTwoConsecutivePollFailuresAreToleratedAndJobFinishes(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [ + new RuntimeException('blip 1'), + new RuntimeException('blip 2'), + FakeJobQueueClient::makeJob('job-1', 'success', 7), + ]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame('success', $this->results[0]->status); + $this->assertSame(1, $summary['migrated']); + $this->assertSame(0, $summary['failed']); + $this->assertSame([5, 5, 5], $this->sleeps); + } + + public function testThreeConsecutivePollFailuresMarkProjectFailedWithJobIdKept(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [ + new RuntimeException('down 1'), + new RuntimeException('down 2'), + new RuntimeException('down 3'), + ]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame(ProjectResult::STATUS_ERROR, $this->results[0]->status); + // The job id must survive into the report - the job may still be running server-side. + $this->assertSame('job-1', $this->results[0]->jobId); + $this->assertIsString($this->results[0]->error); + $this->assertStringContainsString('polling gave up', $this->results[0]->error); + $this->assertSame(1, $summary['failed']); + } + + public function testPollFailureCounterResetsAfterASuccessfulPoll(): void + { + $queueClient = new FakeJobQueueClient( + [FakeJobQueueClient::makeJob('job-1', 'created')], + ['job-1' => [ + new RuntimeException('blip 1'), + new RuntimeException('blip 2'), + FakeJobQueueClient::makeJob('job-1', 'processing'), + new RuntimeException('blip 3'), + new RuntimeException('blip 4'), + FakeJobQueueClient::makeJob('job-1', 'success', 9), + ]] + ); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100')], + ['100' => self::clientsWith($queueClient)] + ); + $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + + $this->assertSame('success', $this->results[0]->status); + $this->assertSame(1, $summary['migrated']); + $this->assertSame(0, $summary['failed']); + } + + public function testConcurrencyWindowCapsInFlightJobsAndRefills(): void + { + // One shared fake for both projects makes the cross-project call order observable. + $queueClient = new FakeJobQueueClient( + [ + FakeJobQueueClient::makeJob('job-1', 'created'), + FakeJobQueueClient::makeJob('job-2', 'created'), + ], + [ + 'job-1' => [ + FakeJobQueueClient::makeJob('job-1', 'processing'), + FakeJobQueueClient::makeJob('job-1', 'success', 1), + ], + 'job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 1)], + ] + ); + $clients = self::clientsWith($queueClient); + $factory = new FakeProjectClientsFactory( + ['100' => self::enabledProject('100'), '200' => self::enabledProject('200')], + ['100' => $clients, '200' => $clients] + ); + $runner = new BatchRunner($factory, 1, 5, $this->sleepRecorder()); + + $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); + + // With concurrency=1, job-2 must not be created until job-1 has finished. + $this->assertSame( + [ + ['listJobs'], + ['createJob', 'job-1'], + ['getJob', 'job-1'], + ['getJob', 'job-1'], + ['listJobs'], + ['createJob', 'job-2'], + ['getJob', 'job-2'], + ], + $queueClient->calls + ); + $this->assertSame(2, $summary['migrated']); + } +} diff --git a/tests/FlowMigration/FakeJobQueueClient.php b/tests/FlowMigration/FakeJobQueueClient.php new file mode 100644 index 0000000..8df294e --- /dev/null +++ b/tests/FlowMigration/FakeJobQueueClient.php @@ -0,0 +1,138 @@ +> recorded JobData->getArray() payloads */ + public array $createdJobs = []; + + /** @var array> ordered call log: [method, id] */ + public array $calls = []; + + public int $listJobsCalls = 0; + + /** @var array */ + private array $createJobReturns; + + /** @var array> */ + private array $getJobSequences; + + /** @var array */ + private array $listJobsReturn; + + /** + * @param array $createJobReturns successive createJob() returns + * @param array> $getJobSequences jobId => successive getJob() outcomes + * @param array $listJobsReturn returned by every listJobs() call + */ + public function __construct(array $createJobReturns = [], array $getJobSequences = [], array $listJobsReturn = []) + { + $this->createJobReturns = $createJobReturns; + $this->getJobSequences = $getJobSequences; + $this->listJobsReturn = $listJobsReturn; + } + + public function createJob(JobData $jobData): Job + { + $this->createdJobs[] = $jobData->getArray(); + $job = array_shift($this->createJobReturns); + if ($job === null) { + throw new RuntimeException('FakeJobQueueClient: no scripted createJob return left'); + } + $this->calls[] = ['createJob', $job->id]; + + return $job; + } + + public function getJob(string $jobId): Job + { + $this->calls[] = ['getJob', $jobId]; + $sequence = $this->getJobSequences[$jobId] ?? []; + if ($sequence === []) { + throw new RuntimeException( + sprintf('FakeJobQueueClient: no scripted getJob outcome left for "%s"', $jobId) + ); + } + $outcome = array_shift($sequence); + $this->getJobSequences[$jobId] = $sequence; + if ($outcome instanceof Throwable) { + throw $outcome; + } + + return $outcome; + } + + public function listJobs(ListJobsOptions $listOptions): array + { + $this->calls[] = ['listJobs']; + $this->listJobsCalls++; + + return $this->listJobsReturn; + } + + /** + * Builds a real DTO\Job through its public factory so the fixture stays in sync with the SDK. + * + * @param array|null $result + */ + public static function makeJob( + string $id, + string $status, + ?int $durationSeconds = null, + ?array $result = null + ): Job { + $terminalStatuses = ['success', 'error', 'warning', 'terminated', 'cancelled']; + + return Job::fromApiResponse([ + 'id' => $id, + 'runId' => $id, + 'parentRunId' => '', + 'project' => ['id' => '123'], + 'token' => ['id' => '456', 'description' => 'test token'], + 'status' => $status, + 'desiredStatus' => 'processing', + 'mode' => 'run', + 'component' => 'keboola.flow-migration-tool', + 'config' => null, + 'configData' => null, + 'configRowIds' => null, + 'tag' => null, + 'createdTime' => '2026-08-10T10:00:00+00:00', + 'startTime' => null, + 'endTime' => null, + 'durationSeconds' => $durationSeconds, + 'result' => $result, + 'usageData' => null, + 'isFinished' => in_array($status, $terminalStatuses, true), + 'url' => sprintf('https://queue.example.com/jobs/%s', $id), + 'branchId' => null, + 'variableValuesId' => null, + 'variableValuesData' => [], + 'backend' => [], + 'behavior' => [], + 'executor' => null, + 'metrics' => null, + 'parallelism' => null, + 'type' => 'standard', + 'orchestrationJobId' => null, + 'orchestrationTaskId' => null, + 'onlyOrchestrationTaskIds' => null, + 'previousJobId' => null, + ]); + } +} diff --git a/tests/FlowMigration/FakeJobQueueClientTest.php b/tests/FlowMigration/FakeJobQueueClientTest.php new file mode 100644 index 0000000..3c6eaf8 --- /dev/null +++ b/tests/FlowMigration/FakeJobQueueClientTest.php @@ -0,0 +1,41 @@ + 'ok']); + + $this->assertFalse($running->isFinished); + $this->assertTrue($finished->isFinished); + $this->assertSame('success', $finished->status); + $this->assertSame(42, $finished->durationSeconds); + $this->assertSame(['message' => 'ok'], $finished->result); + } + + public function testGetJobConsumesScriptedSequenceAndThrowsThrowables(): void + { + $fake = new FakeJobQueueClient([], ['job-1' => [ + new RuntimeException('network blip'), + FakeJobQueueClient::makeJob('job-1', 'success'), + ]]); + + try { + $fake->getJob('job-1'); + $this->fail('First scripted outcome should throw'); + } catch (RuntimeException $e) { + $this->assertSame('network blip', $e->getMessage()); + } + + $this->assertSame('success', $fake->getJob('job-1')->status); + $this->assertSame([['getJob', 'job-1'], ['getJob', 'job-1']], $fake->calls); + } +} diff --git a/tests/FlowMigration/FakeProjectClientsFactory.php b/tests/FlowMigration/FakeProjectClientsFactory.php new file mode 100644 index 0000000..4fd91e7 --- /dev/null +++ b/tests/FlowMigration/FakeProjectClientsFactory.php @@ -0,0 +1,67 @@ + projectIds passed to createProjectClients() */ + public array $createClientsCalls = []; + + /** @var array|Throwable> */ + private array $projects; + + /** @var array */ + private array $projectClients; + + /** + * @param array|Throwable> $projects projectId => project detail, or Throwable to throw + * @param array $projectClients projectId => clients, or Throwable + */ + public function __construct(array $projects, array $projectClients = []) + { + $this->projects = $projects; + $this->projectClients = $projectClients; + } + + public function getProject(string $projectId): array + { + if (!array_key_exists($projectId, $this->projects)) { + throw new RuntimeException( + sprintf('FakeProjectClientsFactory: unknown project "%s"', $projectId) + ); + } + $project = $this->projects[$projectId]; + if ($project instanceof Throwable) { + throw $project; + } + + return $project; + } + + public function createProjectClients(string $projectId): ProjectClients + { + $this->createClientsCalls[] = $projectId; + if (!array_key_exists($projectId, $this->projectClients)) { + throw new RuntimeException( + sprintf('FakeProjectClientsFactory: no clients for project "%s"', $projectId) + ); + } + $clients = $this->projectClients[$projectId]; + if ($clients instanceof Throwable) { + throw $clients; + } + + return $clients; + } +} diff --git a/tests/FlowMigration/ProjectResultTest.php b/tests/FlowMigration/ProjectResultTest.php new file mode 100644 index 0000000..66e8714 --- /dev/null +++ b/tests/FlowMigration/ProjectResultTest.php @@ -0,0 +1,40 @@ +assertSame($expectedSkipped, $result->isSkipped()); + $this->assertSame($expectedFailed, $result->isFailed()); + } + + /** + * @return iterable + */ + public static function provideStatuses(): iterable + { + yield 'job success is neither skipped nor failed' => ['success', false, false]; + yield 'job warning counts as migrated, not failed' => ['warning', false, false]; + yield 'job error is failed' => ['error', false, true]; + yield 'job terminated is failed' => ['terminated', false, true]; + yield 'job cancelled is failed' => ['cancelled', false, true]; + yield 'skipped disabled' => [ProjectResult::STATUS_SKIPPED_DISABLED, true, false]; + yield 'skipped no orchestrations' => [ + ProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, + true, + false, + ]; + yield 'skipped job running' => [ProjectResult::STATUS_SKIPPED_JOB_RUNNING, true, false]; + } +} diff --git a/tests/MigrateOrchestrationsToFlowTest.php b/tests/MigrateOrchestrationsToFlowTest.php new file mode 100644 index 0000000..17f2825 --- /dev/null +++ b/tests/MigrateOrchestrationsToFlowTest.php @@ -0,0 +1,129 @@ +|string|null + */ + private function invokePrivate(string $method, string $argument): array|string|null + { + $command = new MigrateOrchestrationsToFlow(); + $reflection = (new ReflectionClass($command))->getMethod($method); + $reflection->setAccessible(true); + + /** @var array|string|null $result */ + $result = $reflection->invoke($command, $argument); + + return $result; + } + + #[DataProvider('provideUrls')] + public function testHostnameSuffixFromUrl(string $url, ?string $expected): void + { + $this->assertSame($expected, $this->invokePrivate('hostnameSuffixFromUrl', $url)); + } + + /** + * @return iterable + */ + public static function provideUrls(): iterable + { + yield 'azure ne stack' => [ + 'https://connection.north-europe.azure.keboola.com', + 'north-europe.azure.keboola.com', + ]; + yield 'aws us stack' => ['https://connection.keboola.com', 'keboola.com']; + yield 'trailing slash is fine' => ['https://connection.keboola.com/', 'keboola.com']; + yield 'missing connection prefix' => ['https://queue.keboola.com', null]; + yield 'not a url' => ['not-a-url', null]; + yield 'bare connection host' => ['https://connection.', null]; + } + + /** + * @param array|null $expected + */ + #[DataProvider('provideProjectLists')] + public function testParseProjectIdList(string $input, ?array $expected): void + { + $this->assertSame($expected, $this->invokePrivate('parseProjectIdList', $input)); + } + + /** + * @return iterable|null}> + */ + public static function provideProjectLists(): iterable + { + yield 'plain list' => ['1,2,3', ['1', '2', '3']]; + yield 'whitespace is trimmed' => ['1, 2 ,3', ['1', '2', '3']]; + yield 'duplicates are removed' => ['1,2,1', ['1', '2']]; + yield 'non-numeric entry invalidates the list' => ['1,foo', null]; + yield 'decimal is rejected' => ['1.2', null]; + yield 'negative is rejected' => ['-1', null]; + yield 'empty string is rejected' => ['', null]; + } + + /** + * @param array|null $expected + */ + #[DataProvider('provideProjectFiles')] + public function testParseProjectIdsFile(string $contents, ?array $expected): void + { + $this->assertSame($expected, $this->invokePrivate('parseProjectIdsFile', $contents)); + } + + /** + * @return iterable|null}> + */ + public static function provideProjectFiles(): iterable + { + yield 'one id per line' => ["100\n200\n", ['100', '200']]; + yield 'blank lines and comments are ignored' => ["100\n\n# staging batch\n200\n", ['100', '200']]; + yield 'windows line endings' => ["100\r\n200\r\n", ['100', '200']]; + yield 'duplicates are removed' => ["100\n200\n100\n", ['100', '200']]; + yield 'non-numeric line invalidates the file' => ["100\nfoo\n", null]; + yield 'empty file is a valid empty list' => ['', []]; + } + + /** + * The report is the audit artifact of a batch run, so a quote or backslash in an API error + * message must not be able to break a row for a standard CSV parser. + */ + public function testAppendReportRowKeepsTheRowParsableWithQuotesInTheError(): void + { + $command = new MigrateOrchestrationsToFlow(); + $method = (new ReflectionClass($command))->getMethod('appendReportRow'); + $method->setAccessible(true); + $handle = fopen('php://memory', 'w+'); + $this->assertIsResource($handle); + + $method->invoke( + $command, + $handle, + new ProjectResult('123', 'job-1', 'error', 7, 'Orchestration \"Daily load\" failed; retry') + ); + + rewind($handle); + $contents = stream_get_contents($handle); + fclose($handle); + + $this->assertIsString($contents); + $this->assertSame( + '123;job-1;error;7;"Orchestration \""Daily load\"" failed; retry"' . "\n", + $contents + ); + $this->assertSame( + ['123', 'job-1', 'error', '7', 'Orchestration \"Daily load\" failed; retry'], + str_getcsv(trim($contents), ';', '"', '') + ); + } +} From 66679a8d5123683d68151d9fe8782df5af3eb994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= <12143866+ondrajodas@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:40:27 +0200 Subject: [PATCH 4/5] fix: AJDA-3117 apply review feedback to migrate-orchestrations-to-flow - ephemeral project token: 12 h -> 1 h, and full project rights (canManageBuckets, canManageTokens, canReadAllFileUploads, canPurgeTrash). componentAccess is dropped: it is not part of the Manage API project token contract, so restricting the token that way had no effect. The job runs for minutes and, started from configData, waits only in the shared queue, so an hour covers the whole run with a fully privileged token kept short-lived. - FakeJobQueueClient records the listJobs() query parameters and the live-job guard test asserts them; the scripted return is independent of the query, so a guard asking for the wrong component or statuses used to pass regardless. - drop the unreachable max(1, concurrency) clamp - the command already rejects a non-positive --concurrency, so the runner has no caller that can hit it. - trim tests: merge the disabled/deleted skip cases into one batch, drop coverage already provided elsewhere, and remove the fake's own test suite. - stop tracking the internal design and plan docs under docs/superpowers. --- .gitignore | 1 + README.md | 6 +- ...26-08-10-migrate-orchestrations-to-flow.md | 2154 ----------------- ...0-migrate-orchestrations-to-flow-design.md | 306 --- .../Command/FlowMigration/BatchRunner.php | 7 +- .../FlowMigration/ProjectClientsFactory.php | 23 +- tests/FlowMigration/BatchRunnerTest.php | 129 +- tests/FlowMigration/FakeJobQueueClient.php | 10 + .../FlowMigration/FakeJobQueueClientTest.php | 41 - tests/MigrateOrchestrationsToFlowTest.php | 34 - 10 files changed, 46 insertions(+), 2665 deletions(-) delete mode 100644 docs/superpowers/plans/2026-08-10-migrate-orchestrations-to-flow.md delete mode 100644 docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md delete mode 100644 tests/FlowMigration/FakeJobQueueClientTest.php diff --git a/.gitignore b/.gitignore index 82d7bb9..c1ea49a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ composer.phar *.env.json /.phpunit.result.cache /phpunit.xml +/docs/superpowers diff --git a/README.md b/README.md index e05a472..7a3af1b 100644 --- a/README.md +++ b/README.md @@ -517,9 +517,9 @@ Options: - `--report=PATH` (default `flow-migration--.csv`): CSV report path. Behavior: -- For each project: skips disabled/deleted projects; creates an ephemeral 12h storage token - (`canManageBuckets`, `canReadAllFileUploads`, component access to `keboola.orchestrator`, - `keboola.flow`, `keboola.scheduler`, `keboola.flow-migration-tool`); skips projects with no +- For each project: skips disabled/deleted projects; creates an ephemeral 1h storage token with full + project rights (`canManageBuckets`, `canManageTokens`, `canReadAllFileUploads`, `canPurgeTrash`) so + the component cannot be short of a permission mid-migration; skips projects with no `keboola.orchestrator` configurations (no empty jobs in customers' job history); skips projects where a `keboola.flow-migration-tool` job is already created/waiting/processing/terminating. - Creates the migration job via `configData` (no stored configuration is left behind) with diff --git a/docs/superpowers/plans/2026-08-10-migrate-orchestrations-to-flow.md b/docs/superpowers/plans/2026-08-10-migrate-orchestrations-to-flow.md deleted file mode 100644 index d42b6e4..0000000 --- a/docs/superpowers/plans/2026-08-10-migrate-orchestrations-to-flow.md +++ /dev/null @@ -1,2154 +0,0 @@ -# manage:migrate-orchestrations-to-flow Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a `cli-utils` command that drives the automated `keboola.orchestrator` → `keboola.flow` migration across a batch of projects on one stack by creating and supervising `keboola.flow-migration-tool` jobs (AJDA-3117). - -**Architecture:** A thin Symfony Command (`MigrateOrchestrationsToFlow`) wires input parsing, CSV reporting, and summary output around a fully unit-tested plain class (`FlowMigrationBatchRunner`) that owns all batch logic: per-project skip rules, job submission, a bounded concurrency window, and polling. The only network seam is `FlowMigrationProjectClientsFactory` (Manage API + per-project ephemeral-token clients), which tests replace with fakes that skip the parent constructor — the same trick `tests/FakeComponents.php` already uses. - -**Tech Stack:** PHP 8.3, Symfony Console 7.4, `keboola/kbc-manage-api-php-client` v7.1.1, `keboola/job-queue-api-php-client` 5.2.0, `keboola/service-client` 1.5.1, `keboola/storage-api-client` v18.7.0, PHPUnit 11. **No composer changes needed — everything is already installed.** - -**Spec:** `docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md` - -## Global Constraints - -- All commands run via `docker compose run --rm dev ...` (dev image has a bind mount; run `docker compose run --rm dev composer install` once first). -- PSR-0 autoloading: class `Keboola\Console\Command\X` **must** live at `src/Keboola/Console/Command/X.php`. Tests are PSR-4: `Keboola\Console\Tests\` → `tests/`. -- Commands are registered manually in `cli.php` — no auto-discovery. -- phpstan level 9 must stay clean (`composer phpstan`, covers `src/` only). -- PSR-2 must stay clean repo-wide: `./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` (CI checks `tests/` too). Keep lines under 120 chars. -- Argument order ` ...` is mandatory (compatibility with `manage:call-on-stacks`). -- Dry-run by default behind `-f`/`--force` — but note the twist: here "dry-run" still creates real jobs (with `parameters.dryRun: true`) and real ephemeral tokens. -- Do **not** use constructor property promotion or `readonly` — `src/` uses classic properties exclusively and phpcs PSR-2 is configured for that style. -- Git commits: conventional format, English, **no AI attribution of any kind**. -- Exact strings that must be used verbatim: - - command name: `manage:migrate-orchestrations-to-flow` - - token description: `AJDA-3117 keboola.orchestrator to keboola.flow migration (batch driver)` - - token: `expiresIn` 43200, `canManageBuckets` true, `canReadAllFileUploads` true, `componentAccess` = `keboola.orchestrator`, `keboola.flow`, `keboola.scheduler`, `keboola.flow-migration-tool` - - job payload parameters: `{"mode": "project", "orchestrationIds": [], "skipBroken": true, "dryRun": }` - - CSV header: `projectId;jobId;status;durationSeconds;error` (`;` delimiter) - -**Verified SDK facts (do not re-derive, they were read from `vendor/`):** -- `Keboola\JobQueueClient\Client::__construct(string $publicApiUrl, string $storageToken, array $options = [])`; `createJob(JobData): DTO\Job`; `getJob(string): DTO\Job`; `listJobs(ListJobsOptions): array` (native `array` return type — safe to compare `!== []`). -- `Keboola\JobQueueClient\JobData::__construct(string $componentId, ?string $configId = null, array $configData = [], string $mode = 'run', ...)`; `getArray()` keys: `component`, `config`, `mode`, `configRowIds`, `tag`, `branchId`, `orchestrationJobId`, `parentRunId`, `configData`. -- `Keboola\JobQueueClient\DTO\Job` is `readonly` with private constructor; build instances via `Job::fromApiResponse(array)` — it reads **all** of these keys without `??`: `id, runId, parentRunId, project, token, status, desiredStatus, mode, component, config, configData, configRowIds, tag, createdTime, startTime, endTime, durationSeconds, result, usageData, isFinished, url, branchId, variableValuesId, variableValuesData, backend, executor, metrics, behavior, parallelism, type, orchestrationJobId, orchestrationTaskId, onlyOrchestrationTaskIds, previousJobId`. `project` needs `['id' => string]`, `token` needs `['id' => string, 'description' => ?string]`; `variableValuesData`, `backend`, `behavior` accept `[]`. -- `Keboola\JobQueueClient\JobStatuses` is a string-backed enum: `CREATED, PROCESSING, TERMINATING, TERMINATED, WAITING, SUCCESS, ERROR, WARNING, CANCELLED`. -- `Keboola\ServiceClient\ServiceClient::__construct(string $hostnameSuffix)`; `getQueueUrl(): string` returns `https://queue.`. -- `Keboola\ManageApi\Client::getProject($id)` and `createProjectStorageToken($projectId, array $params)` have **no declared return types** (implicit mixed) — direct offset access like `$tokenInfo['token']` passes phpstan level 9 (proven by the identical pattern in `MigrateDataAppsOrchestratorTasks.php:199`). Do not add `is_array()` guards on their results — phpstan would not flag either way, and the existing code style omits them. -- `Keboola\ManageApi\ClientException` extends `\Exception`; HTTP status is available via `getCode()`. - ---- - -### Task 1: Result and clients DTOs - -**Files:** -- Create: `src/Keboola/Console/Command/FlowMigrationProjectResult.php` -- Create: `src/Keboola/Console/Command/FlowMigrationProjectClients.php` -- Test: `tests/FlowMigrationProjectResultTest.php` - -**Interfaces:** -- Consumes: `Keboola\JobQueueClient\JobStatuses` (vendor enum), `Keboola\StorageApi\Components`, `Keboola\JobQueueClient\Client` (vendor classes). -- Produces: - - `FlowMigrationProjectResult::__construct(string $projectId, ?string $jobId, string $status, ?int $durationSeconds, ?string $error)` with public typed properties `$projectId`, `$jobId`, `$status`, `$durationSeconds`, `$error`; methods `isSkipped(): bool`, `isFailed(): bool`; constants `STATUS_SKIPPED_DISABLED = 'skipped-disabled'`, `STATUS_SKIPPED_NO_ORCHESTRATIONS = 'skipped-no-orchestrations'`, `STATUS_SKIPPED_JOB_RUNNING = 'skipped-job-running'`, `STATUS_ERROR = 'error'`. - - `FlowMigrationProjectClients::__construct(Components $components, Client $queueClient)` with public typed properties `$components`, `$queueClient`. - -- [ ] **Step 1: Install dependencies (once)** - -Run: `docker compose run --rm dev composer install` -Expected: exits 0, `vendor/` present. - -- [ ] **Step 2: Write the failing test** - -Create `tests/FlowMigrationProjectResultTest.php`: - -```php -assertSame($expectedSkipped, $result->isSkipped()); - $this->assertSame($expectedFailed, $result->isFailed()); - } - - /** - * @return iterable - */ - public static function provideStatuses(): iterable - { - yield 'job success is neither skipped nor failed' => ['success', false, false]; - yield 'job warning counts as migrated, not failed' => ['warning', false, false]; - yield 'job error is failed' => ['error', false, true]; - yield 'job terminated is failed' => ['terminated', false, true]; - yield 'job cancelled is failed' => ['cancelled', false, true]; - yield 'skipped disabled' => [FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, true, false]; - yield 'skipped no orchestrations' => [FlowMigrationProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, true, false]; - yield 'skipped job running' => [FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING, true, false]; - } -} -``` - -- [ ] **Step 3: Run test to verify it fails** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationProjectResultTest.php` -Expected: FAIL — `Class "Keboola\Console\Command\FlowMigrationProjectResult" not found` - -- [ ] **Step 4: Write the implementation** - -Create `src/Keboola/Console/Command/FlowMigrationProjectResult.php`: - -```php -projectId = $projectId; - $this->jobId = $jobId; - $this->status = $status; - $this->durationSeconds = $durationSeconds; - $this->error = $error; - } - - public function isSkipped(): bool - { - return in_array($this->status, [ - self::STATUS_SKIPPED_DISABLED, - self::STATUS_SKIPPED_NO_ORCHESTRATIONS, - self::STATUS_SKIPPED_JOB_RUNNING, - ], true); - } - - public function isFailed(): bool - { - // Anything that is not a skip and not a successful terminal job status is a failure — - // unexpected statuses fail loud rather than passing silently. - return !$this->isSkipped() - && !in_array($this->status, [JobStatuses::SUCCESS->value, JobStatuses::WARNING->value], true); - } -} -``` - -Create `src/Keboola/Console/Command/FlowMigrationProjectClients.php`: - -```php -components = $components; - $this->queueClient = $queueClient; - } -} -``` - -- [ ] **Step 5: Run test to verify it passes** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationProjectResultTest.php` -Expected: PASS (8 tests) - -- [ ] **Step 6: Static analysis and code style** - -Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` -Expected: both exit 0. - -- [ ] **Step 7: Commit** - -```bash -git add src/Keboola/Console/Command/FlowMigrationProjectResult.php \ - src/Keboola/Console/Command/FlowMigrationProjectClients.php \ - tests/FlowMigrationProjectResultTest.php -git commit -m "feat: add flow migration result and per-project clients DTOs" -``` - ---- - -### Task 2: FlowMigrationProjectClientsFactory (network seam) - -**Files:** -- Create: `src/Keboola/Console/Command/FlowMigrationProjectClientsFactory.php` - -**Interfaces:** -- Consumes: `FlowMigrationProjectClients` (Task 1), `Keboola\ManageApi\Client`, `Keboola\StorageApi\Client`, `Keboola\StorageApi\Components`, `Keboola\JobQueueClient\Client`. -- Produces: - - `FlowMigrationProjectClientsFactory::__construct(Keboola\ManageApi\Client $manageClient, string $connectionUrl, string $queueApiUrl)` - - `getProject(string $projectId): array` — Manage API project detail (throws `Keboola\ManageApi\ClientException`, 404 = deleted project) - - `createProjectClients(string $projectId): FlowMigrationProjectClients` — creates the ephemeral token and both clients - -This class is pure network wiring with no branching logic — it is **not** unit-tested (would only test the mock). It is replaced by a fake in Task 3 and its constants are asserted indirectly through the command smoke test in Task 7. Both public methods must stay non-final and overridable. - -- [ ] **Step 1: Write the implementation** - -Create `src/Keboola/Console/Command/FlowMigrationProjectClientsFactory.php`: - -```php -manageClient = $manageClient; - $this->connectionUrl = $connectionUrl; - $this->queueApiUrl = $queueApiUrl; - } - - /** - * @return array Manage API project detail - */ - public function getProject(string $projectId): array - { - return $this->manageClient->getProject($projectId); - } - - public function createProjectClients(string $projectId): FlowMigrationProjectClients - { - $tokenInfo = $this->manageClient->createProjectStorageToken($projectId, [ - 'description' => self::TOKEN_DESCRIPTION, - 'expiresIn' => self::TOKEN_EXPIRES_IN_SECONDS, - 'canManageBuckets' => true, - 'canReadAllFileUploads' => true, - 'componentAccess' => self::TOKEN_COMPONENT_ACCESS, - ]); - - $storageClient = new StorageClient([ - 'url' => $this->connectionUrl, - 'token' => $tokenInfo['token'], - ]); - - return new FlowMigrationProjectClients( - new Components($storageClient), - new JobQueueClient($this->queueApiUrl, $tokenInfo['token']) - ); - } -} -``` - -- [ ] **Step 2: Static analysis and code style** - -Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` -Expected: both exit 0. (If phpstan complains about `$tokenInfo['token']`, something changed in the vendor package — compare with the working pattern in `src/Keboola/Console/Command/MigrateDataAppsOrchestratorTasks.php:199-213` and match it.) - -- [ ] **Step 3: Commit** - -```bash -git add src/Keboola/Console/Command/FlowMigrationProjectClientsFactory.php -git commit -m "feat: add per-project clients factory with ephemeral token creation" -``` - ---- - -### Task 3: Test fakes for the queue client and the clients factory - -**Files:** -- Create: `tests/FakeJobQueueClient.php` -- Create: `tests/FakeFlowMigrationProjectClientsFactory.php` -- Test: `tests/FakeJobQueueClientTest.php` - -**Interfaces:** -- Consumes: `FlowMigrationProjectClients`, `FlowMigrationProjectClientsFactory` (Tasks 1-2), vendor `Client`, `DTO\Job`, `JobData`, `ListJobsOptions`. -- Produces (used by Tasks 4-6): - - `FakeJobQueueClient::__construct(array $createJobReturns = [], array $getJobSequences = [], array $listJobsReturn = [])` — scripted returns; `$getJobSequences` maps jobId → list of `Job|Throwable` consumed one per poll. - - public inspection fields: `array $createdJobs` (list of `JobData->getArray()` payloads), `int $listJobsCalls`, `array $calls` (ordered log of `['createJob', ]` / `['getJob', ]` / `['listJobs']` entries). - - `FakeJobQueueClient::makeJob(string $id, string $status, ?int $durationSeconds = null, ?array $result = null): Job` — builds a real `DTO\Job` fixture; `isFinished` is true for terminal statuses. - - `FakeFlowMigrationProjectClientsFactory::__construct(array $projects, array $projectClients = [])` — `$projects`: projectId → project detail array or `Throwable` to throw; `$projectClients`: projectId → `FlowMigrationProjectClients` or `Throwable`; public field `array $createClientsCalls` (list of projectIds). - -- [ ] **Step 1: Write the failing sanity test** - -Create `tests/FakeJobQueueClientTest.php`: - -```php - 'ok']); - - $this->assertFalse($running->isFinished); - $this->assertTrue($finished->isFinished); - $this->assertSame('success', $finished->status); - $this->assertSame(42, $finished->durationSeconds); - $this->assertSame(['message' => 'ok'], $finished->result); - } - - public function testGetJobConsumesScriptedSequenceAndThrowsThrowables(): void - { - $fake = new FakeJobQueueClient([], ['job-1' => [ - new RuntimeException('network blip'), - FakeJobQueueClient::makeJob('job-1', 'success'), - ]]); - - try { - $fake->getJob('job-1'); - $this->fail('First scripted outcome should throw'); - } catch (RuntimeException $e) { - $this->assertSame('network blip', $e->getMessage()); - } - - $this->assertSame('success', $fake->getJob('job-1')->status); - $this->assertSame([['getJob', 'job-1'], ['getJob', 'job-1']], $fake->calls); - } -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FakeJobQueueClientTest.php` -Expected: FAIL — `Class "Keboola\Console\Tests\FakeJobQueueClient" not found` - -- [ ] **Step 3: Write the fakes** - -Create `tests/FakeJobQueueClient.php`: - -```php -> recorded JobData->getArray() payloads */ - public array $createdJobs = []; - - /** @var array> ordered call log: [method, id] */ - public array $calls = []; - - public int $listJobsCalls = 0; - - /** @var array */ - private array $createJobReturns; - - /** @var array> */ - private array $getJobSequences; - - /** @var array */ - private array $listJobsReturn; - - /** - * @param array $createJobReturns successive createJob() returns - * @param array> $getJobSequences jobId => successive getJob() outcomes - * @param array $listJobsReturn returned by every listJobs() call - */ - public function __construct(array $createJobReturns = [], array $getJobSequences = [], array $listJobsReturn = []) - { - $this->createJobReturns = $createJobReturns; - $this->getJobSequences = $getJobSequences; - $this->listJobsReturn = $listJobsReturn; - } - - public function createJob(JobData $jobData): Job - { - $this->createdJobs[] = $jobData->getArray(); - $job = array_shift($this->createJobReturns); - if ($job === null) { - throw new RuntimeException('FakeJobQueueClient: no scripted createJob return left'); - } - $this->calls[] = ['createJob', $job->id]; - - return $job; - } - - public function getJob(string $jobId): Job - { - $this->calls[] = ['getJob', $jobId]; - $sequence = $this->getJobSequences[$jobId] ?? []; - if ($sequence === []) { - throw new RuntimeException(sprintf('FakeJobQueueClient: no scripted getJob outcome left for "%s"', $jobId)); - } - $outcome = array_shift($sequence); - $this->getJobSequences[$jobId] = $sequence; - if ($outcome instanceof Throwable) { - throw $outcome; - } - - return $outcome; - } - - public function listJobs(ListJobsOptions $listOptions): array - { - $this->calls[] = ['listJobs']; - $this->listJobsCalls++; - - return $this->listJobsReturn; - } - - /** - * Builds a real DTO\Job through its public factory so the fixture stays in sync with the SDK. - * - * @param array|null $result - */ - public static function makeJob(string $id, string $status, ?int $durationSeconds = null, ?array $result = null): Job - { - $terminalStatuses = ['success', 'error', 'warning', 'terminated', 'cancelled']; - - return Job::fromApiResponse([ - 'id' => $id, - 'runId' => $id, - 'parentRunId' => '', - 'project' => ['id' => '123'], - 'token' => ['id' => '456', 'description' => 'test token'], - 'status' => $status, - 'desiredStatus' => 'processing', - 'mode' => 'run', - 'component' => 'keboola.flow-migration-tool', - 'config' => null, - 'configData' => null, - 'configRowIds' => null, - 'tag' => null, - 'createdTime' => '2026-08-10T10:00:00+00:00', - 'startTime' => null, - 'endTime' => null, - 'durationSeconds' => $durationSeconds, - 'result' => $result, - 'usageData' => null, - 'isFinished' => in_array($status, $terminalStatuses, true), - 'url' => sprintf('https://queue.example.com/jobs/%s', $id), - 'branchId' => null, - 'variableValuesId' => null, - 'variableValuesData' => [], - 'backend' => [], - 'behavior' => [], - 'executor' => null, - 'metrics' => null, - 'parallelism' => null, - 'type' => 'standard', - 'orchestrationJobId' => null, - 'orchestrationTaskId' => null, - 'onlyOrchestrationTaskIds' => null, - 'previousJobId' => null, - ]); - } -} -``` - -Create `tests/FakeFlowMigrationProjectClientsFactory.php`: - -```php - projectIds passed to createProjectClients() */ - public array $createClientsCalls = []; - - /** @var array|Throwable> */ - private array $projects; - - /** @var array */ - private array $projectClients; - - /** - * @param array|Throwable> $projects projectId => Manage project detail, or Throwable to throw - * @param array $projectClients projectId => clients, or Throwable - */ - public function __construct(array $projects, array $projectClients = []) - { - $this->projects = $projects; - $this->projectClients = $projectClients; - } - - public function getProject(string $projectId): array - { - if (!array_key_exists($projectId, $this->projects)) { - throw new RuntimeException(sprintf('FakeFlowMigrationProjectClientsFactory: unknown project "%s"', $projectId)); - } - $project = $this->projects[$projectId]; - if ($project instanceof Throwable) { - throw $project; - } - - return $project; - } - - public function createProjectClients(string $projectId): FlowMigrationProjectClients - { - $this->createClientsCalls[] = $projectId; - if (!array_key_exists($projectId, $this->projectClients)) { - throw new RuntimeException(sprintf('FakeFlowMigrationProjectClientsFactory: no clients for project "%s"', $projectId)); - } - $clients = $this->projectClients[$projectId]; - if ($clients instanceof Throwable) { - throw $clients; - } - - return $clients; - } -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FakeJobQueueClientTest.php` -Expected: PASS (2 tests). If `Job::fromApiResponse` throws about a missing key, the SDK changed — add the missing key with a null/empty value to `makeJob()`. - -- [ ] **Step 5: Code style** - -Run: `docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` -Expected: exit 0. - -- [ ] **Step 6: Commit** - -```bash -git add tests/FakeJobQueueClient.php tests/FakeFlowMigrationProjectClientsFactory.php tests/FakeJobQueueClientTest.php -git commit -m "test: add fakes for job queue client and flow migration clients factory" -``` - ---- - -### Task 4: FlowMigrationBatchRunner — core loop, submission, polling - -**Files:** -- Create: `src/Keboola/Console/Command/FlowMigrationBatchRunner.php` -- Test: `tests/FlowMigrationBatchRunnerTest.php` - -**Interfaces:** -- Consumes: Tasks 1-3 classes; vendor `JobData`, `JobStatuses`, `ListJobsOptions`, `ListComponentConfigurationsOptions`, `DTO\Job`; `tests/FakeComponents.php` (existing, constructor `new FakeComponents(array $configsByComponent)`). -- Produces: - - `FlowMigrationBatchRunner::__construct(FlowMigrationProjectClientsFactory $clientsFactory, int $concurrency, int $pollIntervalSeconds, ?callable $sleep = null)` — `$sleep` signature `callable(int): void`, defaults to PHP `sleep()`. - - `run(array $projectIds, bool $force, OutputInterface $output, callable $onProjectFinished): array` — `$onProjectFinished` receives one `FlowMigrationProjectResult` per input project; returns summary shape `array{attempted: int, migrated: int, migratedWithWarning: int, skippedNoOrchestrations: int, skippedDisabled: int, skippedJobRunning: int, failed: int}`. - - public constants `ORCHESTRATOR_COMPONENT_ID = 'keboola.orchestrator'`, `MIGRATION_COMPONENT_ID = 'keboola.flow-migration-tool'`. - -In this task the runner handles enabled projects with orchestrations and no live migration job (the happy pipeline: guard query → createJob → poll → terminal result). Skip rules come in Task 5, poll-failure tolerance in Task 6. - -- [ ] **Step 1: Write the failing tests** - -Create `tests/FlowMigrationBatchRunnerTest.php`: - -```php - */ - private array $results = []; - - /** @var array */ - private array $sleeps = []; - - private function collector(): Closure - { - return function (FlowMigrationProjectResult $result): void { - $this->results[] = $result; - }; - } - - private function sleepRecorder(): Closure - { - return function (int $seconds): void { - $this->sleeps[] = $seconds; - }; - } - - /** - * @return array enabled project detail as returned by the Manage API - */ - private static function enabledProject(string $id): array - { - return ['id' => $id, 'name' => 'Project ' . $id, 'isDisabled' => false]; - } - - private static function clientsWith(FakeJobQueueClient $queueClient, bool $hasOrchestrations = true): FlowMigrationProjectClients - { - $configs = $hasOrchestrations - ? ['keboola.orchestrator' => [['id' => 'orch-1', 'name' => 'Daily load', 'configuration' => []]]] - : []; - - return new FlowMigrationProjectClients(new FakeComponents($configs), $queueClient); - } - - public function testHappyPathCreatesJobAndReportsSuccess(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [ - FakeJobQueueClient::makeJob('job-1', 'processing'), - FakeJobQueueClient::makeJob('job-1', 'success', 42), - ]] - ); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - // Exact job payload - this is the whole contract with keboola.flow-migration-tool. - $this->assertCount(1, $queueClient->createdJobs); - $this->assertSame('keboola.flow-migration-tool', $queueClient->createdJobs[0]['component']); - $this->assertNull($queueClient->createdJobs[0]['config']); - $this->assertSame('run', $queueClient->createdJobs[0]['mode']); - $this->assertSame( - ['parameters' => [ - 'mode' => 'project', - 'orchestrationIds' => [], - 'skipBroken' => true, - 'dryRun' => false, - ]], - $queueClient->createdJobs[0]['configData'] - ); - // The live-job guard ran exactly once before submission. - $this->assertSame(1, $queueClient->listJobsCalls); - - $this->assertCount(1, $this->results); - $this->assertSame('100', $this->results[0]->projectId); - $this->assertSame('job-1', $this->results[0]->jobId); - $this->assertSame('success', $this->results[0]->status); - $this->assertSame(42, $this->results[0]->durationSeconds); - $this->assertNull($this->results[0]->error); - - // Two poll sweeps (processing, then success), each preceded by one poll-interval sleep. - $this->assertSame([5, 5], $this->sleeps); - - $this->assertSame(1, $summary['attempted']); - $this->assertSame(1, $summary['migrated']); - $this->assertSame(0, $summary['failed']); - } - - public function testWithoutForceJobRunsWithDryRunTrue(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'success', 1)]] - ); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $runner->run(['100'], false, new BufferedOutput(), $this->collector()); - - $configData = $queueClient->createdJobs[0]['configData']; - $this->assertIsArray($configData); - $this->assertTrue($configData['parameters']['dryRun']); - } - - public function testJobEndingInErrorMarksProjectFailedButBatchContinues(): void - { - $queueClient1 = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'error', 10, ['message' => 'boom'])]] - ); - $queueClient2 = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-2', 'created')], - ['job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 20)]] - ); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100'), '200' => self::enabledProject('200')], - ['100' => self::clientsWith($queueClient1), '200' => self::clientsWith($queueClient2)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); - - $this->assertCount(2, $this->results); - $byProject = []; - foreach ($this->results as $result) { - $byProject[$result->projectId] = $result; - } - $this->assertSame('error', $byProject['100']->status); - $this->assertSame('boom', $byProject['100']->error); - $this->assertTrue($byProject['100']->isFailed()); - $this->assertSame('success', $byProject['200']->status); - - $this->assertSame(2, $summary['attempted']); - $this->assertSame(1, $summary['migrated']); - $this->assertSame(1, $summary['failed']); - } - - public function testWarningJobCountsAsMigratedWithWarning(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'warning', 5, ['message' => 'partial'])]] - ); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame('warning', $this->results[0]->status); - $this->assertFalse($this->results[0]->isFailed()); - $this->assertSame(1, $summary['migratedWithWarning']); - $this->assertSame(0, $summary['migrated']); - $this->assertSame(0, $summary['failed']); - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` -Expected: FAIL — `Class "Keboola\Console\Command\FlowMigrationBatchRunner" not found` - -- [ ] **Step 3: Write the implementation** - -Create `src/Keboola/Console/Command/FlowMigrationBatchRunner.php`: - -```php -clientsFactory = $clientsFactory; - $this->concurrency = $concurrency; - $this->pollIntervalSeconds = $pollIntervalSeconds; - $this->sleep = $sleep ?? function (int $seconds): void { - sleep($seconds); - }; - } - - /** - * @param array $projectIds - * @param callable(FlowMigrationProjectResult): void $onProjectFinished invoked once per input project - * @return array{ - * attempted: int, - * migrated: int, - * migratedWithWarning: int, - * skippedNoOrchestrations: int, - * skippedDisabled: int, - * skippedJobRunning: int, - * failed: int - * } - */ - public function run(array $projectIds, bool $force, OutputInterface $output, callable $onProjectFinished): array - { - $pending = array_values(array_unique($projectIds)); - $summary = [ - 'attempted' => count($pending), - 'migrated' => 0, - 'migratedWithWarning' => 0, - 'skippedNoOrchestrations' => 0, - 'skippedDisabled' => 0, - 'skippedJobRunning' => 0, - 'failed' => 0, - ]; - /** @var array $inFlight */ - $inFlight = []; - - while ($pending !== [] || $inFlight !== []) { - while (count($inFlight) < $this->concurrency && $pending !== []) { - $projectId = array_shift($pending); - $submission = $this->submitProject($projectId, $force, $output); - if ($submission instanceof FlowMigrationProjectResult) { - $this->recordResult($submission, $summary, $output, $onProjectFinished); - continue; - } - $inFlight[$projectId] = $submission; - } - - if ($inFlight === []) { - continue; - } - - ($this->sleep)($this->pollIntervalSeconds); - $this->pollInFlightJobs($inFlight, $summary, $output, $onProjectFinished); - } - - return $summary; - } - - /** - * Runs the per-project pipeline up to job creation. Returns an in-flight slot on success, - * or an immediately-final FlowMigrationProjectResult (skip or driver-side error). - * - * @return FlowMigrationProjectResult|array{jobId: string, queueClient: JobQueueClient, startedAt: float, pollFailures: int} - */ - private function submitProject(string $projectId, bool $force, OutputInterface $output) - { - try { - $clients = $this->clientsFactory->createProjectClients($projectId); - - $liveJobs = $clients->queueClient->listJobs( - (new ListJobsOptions()) - ->setComponents([self::MIGRATION_COMPONENT_ID]) - ->setStatuses(self::LIVE_JOB_STATUSES) - ->setLimit(1) - ); - if ($liveJobs !== []) { - return new FlowMigrationProjectResult( - $projectId, - null, - FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING, - null, - 'a keboola.flow-migration-tool job is already running in this project' - ); - } - - $job = $clients->queueClient->createJob(new JobData( - self::MIGRATION_COMPONENT_ID, - null, - [ - 'parameters' => [ - 'mode' => 'project', - 'orchestrationIds' => [], - 'skipBroken' => true, - 'dryRun' => !$force, - ], - ] - )); - } catch (Throwable $e) { - return new FlowMigrationProjectResult( - $projectId, - null, - FlowMigrationProjectResult::STATUS_ERROR, - null, - $e->getMessage() - ); - } - - $this->writeLine($output, sprintf('Project %s: created job %s (%s)', $projectId, $job->id, $job->url)); - - return [ - 'jobId' => $job->id, - 'queueClient' => $clients->queueClient, - 'startedAt' => microtime(true), - 'pollFailures' => 0, - ]; - } - - /** - * @param array $inFlight - * @param array{ - * attempted: int, - * migrated: int, - * migratedWithWarning: int, - * skippedNoOrchestrations: int, - * skippedDisabled: int, - * skippedJobRunning: int, - * failed: int - * } $summary - * @param callable(FlowMigrationProjectResult): void $onProjectFinished - */ - private function pollInFlightJobs( - array &$inFlight, - array &$summary, - OutputInterface $output, - callable $onProjectFinished - ): void { - foreach (array_keys($inFlight) as $projectId) { - $slot = $inFlight[$projectId]; - $job = $slot['queueClient']->getJob($slot['jobId']); - - if (!$job->isFinished) { - continue; - } - - unset($inFlight[$projectId]); - $durationSeconds = $job->durationSeconds ?? (int) round(microtime(true) - $slot['startedAt']); - $this->recordResult( - new FlowMigrationProjectResult( - $projectId, - $slot['jobId'], - $job->status, - $durationSeconds, - $this->extractJobError($job) - ), - $summary, - $output, - $onProjectFinished - ); - } - } - - private function extractJobError(Job $job): ?string - { - if ($job->isSuccess()) { - return null; - } - $result = $job->result; - if (is_array($result) && isset($result['message']) && is_scalar($result['message'])) { - return (string) $result['message']; - } - - return null; - } - - /** - * @param array{ - * attempted: int, - * migrated: int, - * migratedWithWarning: int, - * skippedNoOrchestrations: int, - * skippedDisabled: int, - * skippedJobRunning: int, - * failed: int - * } $summary - * @param callable(FlowMigrationProjectResult): void $onProjectFinished - */ - private function recordResult( - FlowMigrationProjectResult $result, - array &$summary, - OutputInterface $output, - callable $onProjectFinished - ): void { - $summaryKey = match ($result->status) { - JobStatuses::SUCCESS->value => 'migrated', - JobStatuses::WARNING->value => 'migratedWithWarning', - FlowMigrationProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS => 'skippedNoOrchestrations', - FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED => 'skippedDisabled', - FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING => 'skippedJobRunning', - default => 'failed', - }; - $summary[$summaryKey]++; - - $this->writeLine($output, sprintf( - 'Project %s: %s%s%s', - $result->projectId, - $result->status, - $result->durationSeconds !== null ? sprintf(' in %d s', $result->durationSeconds) : '', - $result->error !== null ? sprintf(' (%s)', $result->error) : '' - )); - - $onProjectFinished($result); - } - - private function writeLine(OutputInterface $output, string $message): void - { - $output->writeln(sprintf('[%s] %s', date('H:i:s'), $message)); - } -} -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` -Expected: PASS (4 tests) - -- [ ] **Step 5: Static analysis and code style** - -Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` -Expected: both exit 0. - -- [ ] **Step 6: Commit** - -```bash -git add src/Keboola/Console/Command/FlowMigrationBatchRunner.php tests/FlowMigrationBatchRunnerTest.php -git commit -m "feat: add flow migration batch runner with concurrency window and polling" -``` - ---- - -### Task 5: Batch runner — skip rules and input deduplication - -**Files:** -- Modify: `src/Keboola/Console/Command/FlowMigrationBatchRunner.php` (method `submitProject`, plus two `use` imports) -- Test: `tests/FlowMigrationBatchRunnerTest.php` (append methods) - -**Interfaces:** -- Consumes: `Keboola\ManageApi\ClientException` (HTTP status via `getCode()`); everything from Task 4. -- Produces: final `submitProject()` behavior — skip order is: disabled/deleted → (token+clients) → no orchestrator configs → live migration job → createJob. No token is created for disabled/deleted projects. - -- [ ] **Step 1: Write the failing tests** - -Append to `tests/FlowMigrationBatchRunnerTest.php` (add `use Keboola\ManageApi\ClientException as ManageClientException;` and `use RuntimeException;` to the imports): - -```php - public function testSkipsDisabledProjectWithoutCreatingTokenOrJob(): void - { - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => ['id' => '100', 'name' => 'Off', 'isDisabled' => true]] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - // No ephemeral token may be created for a disabled project. - $this->assertSame([], $factory->createClientsCalls); - $this->assertSame(FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, $this->results[0]->status); - $this->assertNull($this->results[0]->jobId); - $this->assertSame(1, $summary['skippedDisabled']); - $this->assertSame(0, $summary['failed']); - $this->assertSame([], $this->sleeps); - } - - public function testSkipsDeletedProjectOnManage404(): void - { - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => new ManageClientException('Project not found', 404)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame([], $factory->createClientsCalls); - $this->assertSame(FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, $this->results[0]->status); - $this->assertSame(1, $summary['skippedDisabled']); - } - - public function testManageErrorOtherThan404MarksProjectFailed(): void - { - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => new ManageClientException('Internal error', 500)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame(FlowMigrationProjectResult::STATUS_ERROR, $this->results[0]->status); - $this->assertSame('Internal error', $this->results[0]->error); - $this->assertSame(1, $summary['failed']); - } - - public function testSkipsProjectWithoutOrchestratorConfigurations(): void - { - $queueClient = new FakeJobQueueClient(); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient, false)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - // No queue API call and no job for a project with nothing to migrate. - $this->assertSame(0, $queueClient->listJobsCalls); - $this->assertSame([], $queueClient->createdJobs); - $this->assertSame(FlowMigrationProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, $this->results[0]->status); - $this->assertSame(1, $summary['skippedNoOrchestrations']); - } - - public function testSkipsProjectWithLiveMigrationJob(): void - { - $queueClient = new FakeJobQueueClient( - [], - [], - [FakeJobQueueClient::makeJob('existing-job', 'processing')] - ); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame([], $queueClient->createdJobs); - $this->assertSame(FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING, $this->results[0]->status); - $this->assertSame(1, $summary['skippedJobRunning']); - } - - public function testDeduplicatesInputProjectIds(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'success', 1)]] - ); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100', '100', '100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame(1, $summary['attempted']); - $this->assertCount(1, $queueClient->createdJobs); - $this->assertCount(1, $this->results); - } -``` - -- [ ] **Step 2: Run tests to verify the new ones fail** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` -Expected: `testDeduplicatesInputProjectIds` PASSES already (dedup shipped in Task 4's `run()`); the disabled/404/500 tests FAIL (`getProject` is never called, so the fake's clients-map miss makes them land in `failed`, not `skippedDisabled`); the no-orchestrations test FAILS (no configuration check exists yet); the live-job test PASSES already. Failing count: 4. - -- [ ] **Step 3: Extend the implementation** - -In `src/Keboola/Console/Command/FlowMigrationBatchRunner.php`, add one import -(`ListComponentConfigurationsOptions` is already imported since Task 4): - -```php -use Keboola\ManageApi\ClientException as ManageClientException; -``` - -Replace the whole `submitProject()` method with: - -```php - /** - * Runs the per-project pipeline up to job creation. Returns an in-flight slot on success, - * or an immediately-final FlowMigrationProjectResult (skip or driver-side error). - * - * Order matters: the disabled/deleted check runs before any token is created, and the - * configuration check runs before the queue guard so empty projects never appear in - * customers' job history. - * - * @return FlowMigrationProjectResult|array{jobId: string, queueClient: JobQueueClient, startedAt: float, pollFailures: int} - */ - private function submitProject(string $projectId, bool $force, OutputInterface $output) - { - try { - $project = $this->clientsFactory->getProject($projectId); - } catch (ManageClientException $e) { - if ($e->getCode() === 404) { - return new FlowMigrationProjectResult( - $projectId, - null, - FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, - null, - 'project is deleted' - ); - } - - return new FlowMigrationProjectResult( - $projectId, - null, - FlowMigrationProjectResult::STATUS_ERROR, - null, - $e->getMessage() - ); - } - - if (isset($project['isDisabled']) && $project['isDisabled']) { - return new FlowMigrationProjectResult( - $projectId, - null, - FlowMigrationProjectResult::STATUS_SKIPPED_DISABLED, - null, - 'project is disabled' - ); - } - - try { - $clients = $this->clientsFactory->createProjectClients($projectId); - - $configurations = $clients->components->listComponentConfigurations( - (new ListComponentConfigurationsOptions()) - ->setComponentId(self::ORCHESTRATOR_COMPONENT_ID) - ->setIsDeleted(false) - ); - if (count($configurations) === 0) { - return new FlowMigrationProjectResult( - $projectId, - null, - FlowMigrationProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, - null, - 'no keboola.orchestrator configurations' - ); - } - - $liveJobs = $clients->queueClient->listJobs( - (new ListJobsOptions()) - ->setComponents([self::MIGRATION_COMPONENT_ID]) - ->setStatuses(self::LIVE_JOB_STATUSES) - ->setLimit(1) - ); - if ($liveJobs !== []) { - return new FlowMigrationProjectResult( - $projectId, - null, - FlowMigrationProjectResult::STATUS_SKIPPED_JOB_RUNNING, - null, - 'a keboola.flow-migration-tool job is already running in this project' - ); - } - - $job = $clients->queueClient->createJob(new JobData( - self::MIGRATION_COMPONENT_ID, - null, - [ - 'parameters' => [ - 'mode' => 'project', - 'orchestrationIds' => [], - 'skipBroken' => true, - 'dryRun' => !$force, - ], - ] - )); - } catch (Throwable $e) { - return new FlowMigrationProjectResult( - $projectId, - null, - FlowMigrationProjectResult::STATUS_ERROR, - null, - $e->getMessage() - ); - } - - $this->writeLine($output, sprintf('Project %s: created job %s (%s)', $projectId, $job->id, $job->url)); - - return [ - 'jobId' => $job->id, - 'queueClient' => $clients->queueClient, - 'startedAt' => microtime(true), - 'pollFailures' => 0, - ]; - } -``` - -Note: `listComponentConfigurations()` has no declared return type in the SDK (implicit mixed), so `count()` on it is phpstan-safe — the same access pattern as `DataAppOrchestratorTaskMigrator` uses. Update the existing Task 4 tests' fixtures if needed: they already provide `'100' => self::enabledProject('100')` in the `$projects` map, so `getProject()` succeeds there — no changes expected. - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` -Expected: PASS (10 tests) - -- [ ] **Step 5: Static analysis and code style** - -Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` -Expected: both exit 0. - -- [ ] **Step 6: Commit** - -```bash -git add src/Keboola/Console/Command/FlowMigrationBatchRunner.php tests/FlowMigrationBatchRunnerTest.php -git commit -m "feat: add skip rules for disabled, empty and already-migrating projects" -``` - ---- - -### Task 6: Batch runner — poll-failure tolerance and concurrency window verification - -**Files:** -- Modify: `src/Keboola/Console/Command/FlowMigrationBatchRunner.php` (constant + method `pollInFlightJobs`) -- Test: `tests/FlowMigrationBatchRunnerTest.php` (append methods) - -**Interfaces:** -- Consumes: everything from Tasks 4-5. -- Produces: `pollInFlightJobs()` tolerates up to 2 consecutive `getJob()` failures per job (a success resets the counter); the 3rd consecutive failure resolves the project as `STATUS_ERROR` with the job id preserved in the result. Constant `MAX_CONSECUTIVE_POLL_FAILURES = 3` (private). - -- [ ] **Step 1: Write the failing tests** - -Append to `tests/FlowMigrationBatchRunnerTest.php`: - -```php - public function testTwoConsecutivePollFailuresAreToleratedAndJobFinishes(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [ - new RuntimeException('blip 1'), - new RuntimeException('blip 2'), - FakeJobQueueClient::makeJob('job-1', 'success', 7), - ]] - ); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame('success', $this->results[0]->status); - $this->assertSame(1, $summary['migrated']); - $this->assertSame(0, $summary['failed']); - $this->assertSame([5, 5, 5], $this->sleeps); - } - - public function testThreeConsecutivePollFailuresMarkProjectFailedWithJobIdKept(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [ - new RuntimeException('down 1'), - new RuntimeException('down 2'), - new RuntimeException('down 3'), - ]] - ); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new FlowMigrationBatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame(FlowMigrationProjectResult::STATUS_ERROR, $this->results[0]->status); - // The job id must survive into the report - the job may still be running server-side. - $this->assertSame('job-1', $this->results[0]->jobId); - $this->assertIsString($this->results[0]->error); - $this->assertStringContainsString('polling gave up', $this->results[0]->error); - $this->assertSame(1, $summary['failed']); - } - - public function testConcurrencyWindowCapsInFlightJobsAndRefills(): void - { - // One shared fake for both projects makes the cross-project call order observable. - $queueClient = new FakeJobQueueClient( - [ - FakeJobQueueClient::makeJob('job-1', 'created'), - FakeJobQueueClient::makeJob('job-2', 'created'), - ], - [ - 'job-1' => [ - FakeJobQueueClient::makeJob('job-1', 'processing'), - FakeJobQueueClient::makeJob('job-1', 'success', 1), - ], - 'job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 1)], - ] - ); - $clients = self::clientsWith($queueClient); - $factory = new FakeFlowMigrationProjectClientsFactory( - ['100' => self::enabledProject('100'), '200' => self::enabledProject('200')], - ['100' => $clients, '200' => $clients] - ); - $runner = new FlowMigrationBatchRunner($factory, 1, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); - - // With concurrency=1, job-2 must not be created until job-1 has finished. - $this->assertSame( - [ - ['listJobs'], - ['createJob', 'job-1'], - ['getJob', 'job-1'], - ['getJob', 'job-1'], - ['listJobs'], - ['createJob', 'job-2'], - ['getJob', 'job-2'], - ], - $queueClient->calls - ); - $this->assertSame(2, $summary['migrated']); - } -``` - -- [ ] **Step 2: Run tests to verify the new ones fail** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` -Expected: the two poll-failure tests FAIL (the `RuntimeException` from `getJob()` currently escapes `run()` uncaught). The concurrency test should PASS already (window logic shipped in Task 4) — it is the regression guard for this behavior. - -- [ ] **Step 3: Extend the implementation** - -In `src/Keboola/Console/Command/FlowMigrationBatchRunner.php`, add below `LIVE_JOB_STATUSES`: - -```php - // A transient Queue API outage must not fail a project instantly (the SDK already retries - // 5xx internally), but an unbounded retry could hang the batch forever - so give up after - // this many consecutive failed polls and leave the job to finish server-side. - private const MAX_CONSECUTIVE_POLL_FAILURES = 3; -``` - -Replace the whole `pollInFlightJobs()` method with: - -```php - /** - * @param array $inFlight - * @param array{ - * attempted: int, - * migrated: int, - * migratedWithWarning: int, - * skippedNoOrchestrations: int, - * skippedDisabled: int, - * skippedJobRunning: int, - * failed: int - * } $summary - * @param callable(FlowMigrationProjectResult): void $onProjectFinished - */ - private function pollInFlightJobs( - array &$inFlight, - array &$summary, - OutputInterface $output, - callable $onProjectFinished - ): void { - foreach (array_keys($inFlight) as $projectId) { - $slot = $inFlight[$projectId]; - try { - $job = $slot['queueClient']->getJob($slot['jobId']); - } catch (Throwable $e) { - $inFlight[$projectId]['pollFailures']++; - $this->writeLine($output, sprintf( - 'Project %s: polling job %s failed (%d/%d): %s', - $projectId, - $slot['jobId'], - $inFlight[$projectId]['pollFailures'], - self::MAX_CONSECUTIVE_POLL_FAILURES, - $e->getMessage() - )); - if ($inFlight[$projectId]['pollFailures'] >= self::MAX_CONSECUTIVE_POLL_FAILURES) { - unset($inFlight[$projectId]); - $this->recordResult( - new FlowMigrationProjectResult( - $projectId, - $slot['jobId'], - FlowMigrationProjectResult::STATUS_ERROR, - null, - sprintf( - 'polling gave up after %d consecutive failures, job may still be running: %s', - self::MAX_CONSECUTIVE_POLL_FAILURES, - $e->getMessage() - ) - ), - $summary, - $output, - $onProjectFinished - ); - } - continue; - } - - $inFlight[$projectId]['pollFailures'] = 0; - - if (!$job->isFinished) { - continue; - } - - unset($inFlight[$projectId]); - $durationSeconds = $job->durationSeconds ?? (int) round(microtime(true) - $slot['startedAt']); - $this->recordResult( - new FlowMigrationProjectResult( - $projectId, - $slot['jobId'], - $job->status, - $durationSeconds, - $this->extractJobError($job) - ), - $summary, - $output, - $onProjectFinished - ); - } - } -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/FlowMigrationBatchRunnerTest.php` -Expected: PASS (13 tests) - -- [ ] **Step 5: Static analysis and code style** - -Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` -Expected: both exit 0. - -- [ ] **Step 6: Commit** - -```bash -git add src/Keboola/Console/Command/FlowMigrationBatchRunner.php tests/FlowMigrationBatchRunnerTest.php -git commit -m "feat: tolerate transient poll failures and verify concurrency window" -``` - ---- - -### Task 7: The Symfony command, registration and input-parsing tests - -**Files:** -- Create: `src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php` -- Modify: `cli.php` (one `use` line + one `add()` line) -- Test: `tests/MigrateOrchestrationsToFlowTest.php` - -**Interfaces:** -- Consumes: `FlowMigrationBatchRunner`, `FlowMigrationProjectClientsFactory`, `FlowMigrationProjectResult` (Tasks 1-6); `Keboola\ManageApi\Client`, `Keboola\ServiceClient\ServiceClient`. -- Produces: command `manage:migrate-orchestrations-to-flow` registered in `cli.php`. Private helpers (tested via reflection, matching the `MigrateDataAppsOrchestratorTasksTest` pattern): `hostnameSuffixFromUrl(string): ?string`, `parseProjectIdList(string): ?array`, `parseProjectIdsFile(string): ?array`. - -- [ ] **Step 1: Write the failing tests** - -Create `tests/MigrateOrchestrationsToFlowTest.php`: - -```php -|string|null - */ - private function invokePrivate(string $method, string $argument): array|string|null - { - $command = new MigrateOrchestrationsToFlow(); - $reflection = (new ReflectionClass($command))->getMethod($method); - $reflection->setAccessible(true); - - /** @var array|string|null $result */ - $result = $reflection->invoke($command, $argument); - - return $result; - } - - #[DataProvider('provideUrls')] - public function testHostnameSuffixFromUrl(string $url, ?string $expected): void - { - $this->assertSame($expected, $this->invokePrivate('hostnameSuffixFromUrl', $url)); - } - - /** - * @return iterable - */ - public static function provideUrls(): iterable - { - yield 'azure ne stack' => [ - 'https://connection.north-europe.azure.keboola.com', - 'north-europe.azure.keboola.com', - ]; - yield 'aws us stack' => ['https://connection.keboola.com', 'keboola.com']; - yield 'trailing slash is fine' => ['https://connection.keboola.com/', 'keboola.com']; - yield 'missing connection prefix' => ['https://queue.keboola.com', null]; - yield 'not a url' => ['not-a-url', null]; - yield 'bare connection host' => ['https://connection.', null]; - } - - #[DataProvider('provideProjectLists')] - public function testParseProjectIdList(string $input, ?array $expected): void - { - $this->assertSame($expected, $this->invokePrivate('parseProjectIdList', $input)); - } - - /** - * @return iterable|null}> - */ - public static function provideProjectLists(): iterable - { - yield 'plain list' => ['1,2,3', ['1', '2', '3']]; - yield 'whitespace is trimmed' => ['1, 2 ,3', ['1', '2', '3']]; - yield 'duplicates are removed' => ['1,2,1', ['1', '2']]; - yield 'non-numeric entry invalidates the list' => ['1,foo', null]; - yield 'decimal is rejected' => ['1.2', null]; - yield 'negative is rejected' => ['-1', null]; - yield 'empty string is rejected' => ['', null]; - } - - #[DataProvider('provideProjectFiles')] - public function testParseProjectIdsFile(string $contents, ?array $expected): void - { - $this->assertSame($expected, $this->invokePrivate('parseProjectIdsFile', $contents)); - } - - /** - * @return iterable|null}> - */ - public static function provideProjectFiles(): iterable - { - yield 'one id per line' => ["100\n200\n", ['100', '200']]; - yield 'blank lines and comments are ignored' => ["100\n\n# staging batch\n200\n", ['100', '200']]; - yield 'windows line endings' => ["100\r\n200\r\n", ['100', '200']]; - yield 'duplicates are removed' => ["100\n200\n100\n", ['100', '200']]; - yield 'non-numeric line invalidates the file' => ["100\nfoo\n", null]; - yield 'empty file is a valid empty list' => ['', []]; - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/MigrateOrchestrationsToFlowTest.php` -Expected: FAIL — `Class "Keboola\Console\Command\MigrateOrchestrationsToFlow" not found` - -- [ ] **Step 3: Write the command** - -Create `src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php`: - -```php - keboola.flow migration (AJDA-3117). - * All migration logic lives in the keboola.flow-migration-tool component; this command only - * creates and supervises its jobs across a list of projects on one stack. - */ -class MigrateOrchestrationsToFlow extends Command -{ - const ARG_TOKEN = 'token'; - const ARG_URL = 'url'; - const ARG_PROJECTS = 'projects'; - const OPT_FORCE = 'force'; - const OPT_PROJECTS_FILE = 'projects-file'; - const OPT_CONCURRENCY = 'concurrency'; - const OPT_POLL_INTERVAL = 'poll-interval'; - const OPT_REPORT = 'report'; - - private const CSV_HEADER = ['projectId', 'jobId', 'status', 'durationSeconds', 'error']; - private const CSV_DELIMITER = ';'; - - protected function configure(): void - { - $this - ->setName('manage:migrate-orchestrations-to-flow') - ->setDescription( - 'Run the automated keboola.orchestrator -> keboola.flow migration for a batch of projects' - ) - ->addArgument(self::ARG_TOKEN, InputArgument::REQUIRED, 'Manage API token') - ->addArgument( - self::ARG_URL, - InputArgument::REQUIRED, - 'Stack URL, e.g. https://connection.north-europe.azure.keboola.com' - ) - ->addArgument( - self::ARG_PROJECTS, - InputArgument::OPTIONAL, - 'Comma-separated project IDs, or @path/to/file with one ID per line' - ) - ->addOption( - self::OPT_FORCE, - 'f', - InputOption::VALUE_NONE, - 'Run the real migration; without it jobs are created with dryRun: true' - ) - ->addOption( - self::OPT_PROJECTS_FILE, - null, - InputOption::VALUE_REQUIRED, - 'File with one project ID per line (alternative to @file in the argument)' - ) - ->addOption( - self::OPT_CONCURRENCY, - null, - InputOption::VALUE_REQUIRED, - 'Max migration jobs in flight at once', - '10' - ) - ->addOption( - self::OPT_POLL_INTERVAL, - null, - InputOption::VALUE_REQUIRED, - 'Seconds between job status polls', - '5' - ) - ->addOption( - self::OPT_REPORT, - null, - InputOption::VALUE_REQUIRED, - 'CSV report path (default: flow-migration--.csv)' - ); - } - - protected function execute(InputInterface $input, OutputInterface $output): int - { - $token = $input->getArgument(self::ARG_TOKEN); - assert(is_string($token)); - $url = $input->getArgument(self::ARG_URL); - assert(is_string($url)); - $force = (bool) $input->getOption(self::OPT_FORCE); - - $hostnameSuffix = $this->hostnameSuffixFromUrl($url); - if ($hostnameSuffix === null) { - $output->writeln(sprintf( - 'Invalid stack URL "%s": expected a URL like https://connection.keboola.com', - $url - )); - return 1; - } - - $projectIds = $this->resolveProjectIds($input, $output); - if ($projectIds === null) { - return 1; - } - - $concurrency = $this->parsePositiveIntOption($input, self::OPT_CONCURRENCY); - $pollInterval = $this->parsePositiveIntOption($input, self::OPT_POLL_INTERVAL); - if ($concurrency === null || $pollInterval === null) { - $output->writeln('Options --concurrency and --poll-interval must be positive integers'); - return 1; - } - - $reportPath = $input->getOption(self::OPT_REPORT); - if (!is_string($reportPath) || $reportPath === '') { - $reportPath = sprintf('flow-migration-%s-%s.csv', $hostnameSuffix, date('Ymd-His')); - } - - $output->writeln($force - ? 'Running in FORCE mode: migration jobs run with dryRun: false.' - : 'Running in dry-run mode: migration jobs run with dryRun: true. Use -f for the real migration.'); - $output->writeln('NOTE: even in dry-run mode a real keboola.flow-migration-tool job and a real' - . ' ephemeral storage token are created in every eligible project.'); - $output->writeln(sprintf('Projects: %d, concurrency: %d, poll interval: %d s', count($projectIds), $concurrency, $pollInterval)); - $output->writeln(sprintf('Report: %s', $reportPath)); - $output->writeln(''); - - $manageClient = new ManageClient(['url' => $url, 'token' => $token]); - $serviceClient = new ServiceClient($hostnameSuffix); - $clientsFactory = new FlowMigrationProjectClientsFactory($manageClient, $url, $serviceClient->getQueueUrl()); - - $reportHandle = fopen($reportPath, 'a'); - if ($reportHandle === false) { - $output->writeln(sprintf('Cannot open report file "%s" for writing', $reportPath)); - return 1; - } - if (ftell($reportHandle) === 0) { - fputcsv($reportHandle, self::CSV_HEADER, self::CSV_DELIMITER, '"', '\\'); - } - - $runner = new FlowMigrationBatchRunner($clientsFactory, $concurrency, $pollInterval); - $summary = $runner->run( - $projectIds, - $force, - $output, - function (FlowMigrationProjectResult $result) use ($reportHandle): void { - fputcsv( - $reportHandle, - [ - $result->projectId, - $result->jobId ?? '', - $result->status, - $result->durationSeconds !== null ? (string) $result->durationSeconds : '', - $result->error ?? '', - ], - self::CSV_DELIMITER, - '"', - '\\' - ); - // Flush per row so an interrupted run still leaves an auditable report. - fflush($reportHandle); - } - ); - fclose($reportHandle); - - $output->writeln(''); - $output->writeln(sprintf( - "DONE\nProjects attempted: %d\nMigrated (job success): %d\nMigrated with warning: %d\n" - . "Skipped (no orchestrations): %d\nSkipped (disabled/deleted): %d\n" - . "Skipped (migration job already running): %d\nFailed: %d", - $summary['attempted'], - $summary['migrated'], - $summary['migratedWithWarning'], - $summary['skippedNoOrchestrations'], - $summary['skippedDisabled'], - $summary['skippedJobRunning'], - $summary['failed'] - )); - - return $summary['failed'] > 0 ? 1 : 0; - } - - /** - * Derives the ServiceClient hostname suffix from a full connection URL, e.g. - * "https://connection.north-europe.azure.keboola.com" -> "north-europe.azure.keboola.com". - * Returns null when the URL does not look like a stack connection URL. - */ - private function hostnameSuffixFromUrl(string $url): ?string - { - $host = parse_url($url, PHP_URL_HOST); - if (!is_string($host) || !str_starts_with($host, 'connection.')) { - return null; - } - $suffix = substr($host, strlen('connection.')); - - return $suffix === '' ? null : $suffix; - } - - /** - * Resolves the project ID list from exactly one source: the argument - * (inline list or @file) or --projects-file. Prints an error and returns null otherwise. - * - * @return array|null - */ - private function resolveProjectIds(InputInterface $input, OutputInterface $output): ?array - { - $projectsArg = $input->getArgument(self::ARG_PROJECTS); - $projectsFile = $input->getOption(self::OPT_PROJECTS_FILE); - - $hasArg = is_string($projectsArg) && $projectsArg !== ''; - $hasFileOption = is_string($projectsFile) && $projectsFile !== ''; - - if ($hasArg === $hasFileOption) { - $output->writeln( - 'Provide exactly one source of project IDs: the argument or --projects-file' - ); - return null; - } - - $projectIds = null; - - if ($hasArg) { - assert(is_string($projectsArg)); - if (str_starts_with($projectsArg, '@')) { - $projectsFile = substr($projectsArg, 1); - $hasFileOption = true; - } else { - $projectIds = $this->parseProjectIdList($projectsArg); - } - } - - if ($hasFileOption) { - assert(is_string($projectsFile)); - $contents = @file_get_contents($projectsFile); - if ($contents === false) { - $output->writeln(sprintf('Cannot read projects file "%s"', $projectsFile)); - return null; - } - $projectIds = $this->parseProjectIdsFile($contents); - } - - if ($projectIds === null || $projectIds === []) { - $output->writeln('Projects list is empty or contains a non-numeric ID'); - return null; - } - - return $projectIds; - } - - /** - * @return array|null null when any entry is not a plain non-negative integer - */ - private function parseProjectIdList(string $raw): ?array - { - return $this->validateAndDeduplicate(array_map('trim', explode(',', $raw))); - } - - /** - * One ID per line; blank lines and lines starting with "#" are ignored. - * - * @return array|null null when any remaining line is not a plain non-negative integer - */ - private function parseProjectIdsFile(string $contents): ?array - { - $lines = preg_split('/\R/', $contents); - $ids = []; - foreach ($lines === false ? [] : $lines as $line) { - $line = trim($line); - if ($line === '' || str_starts_with($line, '#')) { - continue; - } - $ids[] = $line; - } - - return $this->validateAndDeduplicate($ids); - } - - /** - * @param array $ids - * @return array|null - */ - private function validateAndDeduplicate(array $ids): ?array - { - foreach ($ids as $id) { - if (!ctype_digit($id)) { - return null; - } - } - - return array_values(array_unique($ids)); - } - - private function parsePositiveIntOption(InputInterface $input, string $name): ?int - { - $value = $input->getOption($name); - if (!is_string($value) || !ctype_digit($value) || (int) $value < 1) { - return null; - } - - return (int) $value; - } -} -``` - -- [ ] **Step 4: Register the command in cli.php** - -In `cli.php`, add to the `use` block (after the `MigrateDataAppsOrchestratorTasks` line): - -```php -use Keboola\Console\Command\MigrateOrchestrationsToFlow; -``` - -and after `$application->add(new MigrateDataAppsOrchestratorTasks());`: - -```php -$application->add(new MigrateOrchestrationsToFlow()); -``` - -- [ ] **Step 5: Run tests to verify they pass** - -Run: `docker compose run --rm dev ./vendor/bin/phpunit tests/MigrateOrchestrationsToFlowTest.php` -Expected: PASS (19 tests) - -- [ ] **Step 6: Smoke-test the command wiring** - -Run: `docker compose run --rm dev php cli.php list | grep migrate-orchestrations-to-flow` -Expected: one line with `manage:migrate-orchestrations-to-flow`. - -Run: `docker compose run --rm dev php cli.php manage:migrate-orchestrations-to-flow some-token https://connection.keboola.com` -Expected: prints `Provide exactly one source of project IDs...` and exits with code 1 (verify with `echo $?` — note `docker compose run` propagates the container exit code). - -Run: `docker compose run --rm dev php cli.php manage:migrate-orchestrations-to-flow some-token https://connection.keboola.com 1,foo` -Expected: prints `Projects list is empty or contains a non-numeric ID`, exit code 1. - -- [ ] **Step 7: Static analysis and code style** - -Run: `docker compose run --rm dev composer phpstan && docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n .` -Expected: both exit 0. - -- [ ] **Step 8: Commit** - -```bash -git add src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php cli.php tests/MigrateOrchestrationsToFlowTest.php -git commit -m "feat: add manage:migrate-orchestrations-to-flow batch driver command" -``` - ---- - -### Task 8: README documentation and full quality gate - -**Files:** -- Modify: `README.md` (new section under "Project manipulation", directly after the "Migrate data-apps orchestrator/flow tasks to data-app-control" section that ends at the line before `### Mass enablement of dynamic backends for multiple projects`) - -**Interfaces:** -- Consumes: the finished command (Task 7). -- Produces: user-facing documentation; a fully green build. - -- [ ] **Step 1: Add the README section** - -Insert into `README.md` after the "Migrate data-apps orchestrator/flow tasks to data-app-control" section: - -````markdown -### Migrate keboola.orchestrator configurations to keboola.flow - -Batch driver for the automated `keboola.orchestrator` → `keboola.flow` migration -(see [AJDA-3117](https://linear.app/keboola/issue/AJDA-3117)). All migration logic lives in the -`keboola.flow-migration-tool` component; this command only creates one migration job per project -and supervises the batch. Safe to re-run with the same list: already-migrated orchestrations are -reported as skipped by the component, and projects with a live migration job are skipped here. - -``` -php cli.php manage:migrate-orchestrations-to-flow [-f|--force] [] \ - [--projects-file=PATH] [--concurrency=10] [--poll-interval=5] [--report=PATH] -``` - -Arguments: -- `token` (required): Manage API token. -- `url` (required): Stack URL, including `https://` (e.g. `https://connection.north-europe.azure.keboola.com`). -- `projects` (optional): Comma-separated project IDs (e.g. `1,7,146`), or `@path/to/file` with one ID - per line (blank lines and `#` comments are ignored). Exactly one of `projects`/`--projects-file` - must be given. - -Options: -- `--force` / `-f`: Run the real migration. Without it, jobs are created with `dryRun: true`. - **Note:** even without `--force` a real `keboola.flow-migration-tool` job and a real ephemeral - storage token are created in every eligible project — on PAYGO stacks mind the billing. -- `--projects-file=PATH`: File with one project ID per line (alternative to `@file` in the argument). -- `--concurrency=N` (default 10): Max migration jobs in flight at once. -- `--poll-interval=N` (default 5): Seconds between job status polls. -- `--report=PATH` (default `flow-migration--.csv`): CSV report path. - -Behavior: -- For each project: skips disabled/deleted projects; creates an ephemeral 12h storage token - (`canManageBuckets`, `canReadAllFileUploads`, component access to `keboola.orchestrator`, - `keboola.flow`, `keboola.scheduler`, `keboola.flow-migration-tool`); skips projects with no - `keboola.orchestrator` configurations (no empty jobs in customers' job history); skips projects - where a `keboola.flow-migration-tool` job is already created/waiting/processing/terminating. -- Creates the migration job via `configData` (no stored configuration is left behind) with - `parameters: {mode: "project", orchestrationIds: [], skipBroken: true, dryRun: }`. -- Keeps at most `--concurrency` jobs in flight, polls each job and refills the window as jobs finish. -- Appends a CSV row (`projectId;jobId;status;durationSeconds;error`) the moment each project - resolves, so an interrupted run is still auditable. Every input project gets a row; skipped - projects carry the skip reason in `status`/`error` and an empty `jobId`. -- A failing project never aborts the batch. Exit code is `1` if at least one project failed - (job `error`/`terminated`/`cancelled` or a driver-side error), `0` otherwise. -- Final summary: projects attempted / migrated / migrated with warning / skipped (no - orchestrations, disabled, job already running) / failed. -```` - -Note: the block above is fenced with four backticks only so it survives inside this plan file — in `README.md` itself use plain triple-backtick fences exactly like the surrounding sections. - -- [ ] **Step 2: Full quality gate** - -Run: -```bash -docker compose run --rm dev ./vendor/bin/phpcs --standard=psr2 --ignore=vendor -n . -docker compose run --rm dev composer phpstan -docker compose run --rm dev composer tests -``` -Expected: all three exit 0; the full test suite passes (existing tests plus the 42 new ones). - -- [ ] **Step 3: Commit** - -```bash -git add README.md -git commit -m "docs: document manage:migrate-orchestrations-to-flow command" -``` - ---- - -## Out of scope / hand-off - -- The five "Must verify before the first live batch" items from AJDA-3117 (PAYGO billing exclusion - for `keboola.flow-migration-tool`, notification-subscription visibility for ephemeral tokens, - trigger `runWithTokenId` permission, token lifetime vs. queue wait, `keboola.flow` availability - without a per-project feature) require live stack access — they are Ondrej's hand-off checklist, - to be confirmed on a real project (start with a one-project batch) and recorded in a Linear comment. -- Per-orchestration migrated/skipped/failed counts in the CSV: not possible until - `keboola/flow-migration-tool` exposes them in the job result (open question in the issue; - potential follow-up there, not here). diff --git a/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md b/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md deleted file mode 100644 index aa4eca1..0000000 --- a/docs/superpowers/specs/2026-08-10-migrate-orchestrations-to-flow-design.md +++ /dev/null @@ -1,306 +0,0 @@ -# Design: manage:migrate-orchestrations-to-flow (AJDA-3117) - -## Goal - -Add a `cli-utils` command that drives the automated `keboola.orchestrator` → `keboola.flow` -migration across a batch of projects on one stack. The command is a **driver only**: it creates -and supervises `keboola.flow-migration-tool` jobs in customer projects. All migration logic lives -in the component (`keboola/flow-migration-tool`); nothing from it is reimplemented here. - -Per-stack migrations (AJDA-3119 Azure NE, AJDA-3120 GCP US) then become a single supervised run. - -## Verified SDK facts (read from `vendor/`, versions from `composer.lock`) - -All packages needed are already installed — **no composer changes required**: - -| Package | Version | What we use | -|---|---|---| -| `keboola/job-queue-api-php-client` | 5.2.0 | `Client::__construct(string $publicApiUrl, string $storageToken, array $options = [])`; `createJob(JobData): DTO\Job`; `getJob(string $jobId): DTO\Job`; `listJobs(ListJobsOptions): array` (elements are `DTO\Job`); `JobData::__construct(string $componentId, ?string $configId = null, array $configData = [], string $mode = 'run', ...)`; `ListJobsOptions::setComponents(array)/setStatuses(array)/setLimit(int)`; `JobStatuses` enum (`CREATED`, `WAITING`, `PROCESSING`, `TERMINATING`, ... `SUCCESS`, `ERROR`, `WARNING`, `TERMINATED`, `CANCELLED`); `DTO\Job` readonly props: `id`, `status`, `isFinished`, `durationSeconds`, `result`, `url` | -| `keboola/service-client` | 1.5.1 | `new ServiceClient(string $hostnameSuffix)`; `getQueueUrl()` → `https://queue.` | -| `keboola/kbc-manage-api-php-client` | v7.1.1 | `getProject($id)`; `createProjectStorageToken($projectId, array $params)` — generic POST pass-through, accepts `expiresIn`, `canManageBuckets`, `canReadAllFileUploads`, `componentAccess`, `description` | -| `keboola/storage-api-client` | v18.7.0 | `Components::listComponentConfigurations(ListComponentConfigurationsOptions)` with `setComponentId()`/`setIsDeleted(false)` | - -Notes: -- `Client::waitForJobCompletion()` exists but blocks on a single job — unusable for a concurrency - window; we poll with `getJob()` ourselves. -- `listJobs()` has no typed return; the batch runner never touches list elements — the running-job - guard only needs `$jobs !== []` (avoids the `$job['id']`-on-DTO trap present in - `QueueMassTerminateJobs`). -- `DTO\Job::fromApiResponse()` requires many keys — fakes in tests will construct results through - it with a full response fixture, or the fake client returns pre-built `Job` instances. - -## Command - -``` -php cli.php manage:migrate-orchestrations-to-flow [-f|--force] [] - [--projects-file=PATH] [--concurrency=10] [--poll-interval=5] [--report=PATH] -``` - -### Arguments - -| Argument | Type | Description | -|---|---|---| -| `token` | REQUIRED | Manage API token | -| `url` | REQUIRED | Stack URL incl. scheme, e.g. `https://connection.north-europe.azure.keboola.com` | -| `projects` | OPTIONAL | Comma-separated project IDs, or `@path/to/file` (one ID per line) | - -`token` and `url` come first to stay compatible with `manage:call-on-stacks` -(`AllStacksIterator` builds ` `). - -### Options - -| Option | Default | Description | -|---|---|---| -| `-f`, `--force` | off | Real migration (`parameters.dryRun: false`). Without it, jobs run with `dryRun: true` | -| `--projects-file=PATH` | — | Alternative to `@file` in the argument | -| `--concurrency=N` | 10 | Max migration jobs in flight | -| `--poll-interval=N` | 5 | Seconds between poll sweeps | -| `--report=PATH` | `flow-migration--.csv` | CSV report path | - -The component's `parameters.migrate.*` sub-flags are **not** exposed: the command always requests -a full migration and relies on the component defaults. - -**Important semantic difference from the usual cli-utils dry-run:** even without `--force` the -command creates a *real* `keboola.flow-migration-tool` job (with `dryRun: true`) in every eligible -project and creates a real ephemeral storage token. The command prints a prominent notice about -this at startup, and the README documents it (relevant for PAYGO billing — see hand-off items). - -### Input resolution and validation - -- Exactly one source of project IDs must be given: the `projects` argument (inline list or - `@file`) or `--projects-file`. Both, or neither → error message + exit 1. -- File format: one ID per line; blank lines and lines starting with `#` are ignored. -- Every ID must pass `ctype_digit()`; any invalid entry → error naming the offending value, exit 1. -- Duplicates are removed (first occurrence wins) so a re-run with a sloppy list cannot double-submit. -- `--concurrency` ≥ 1, `--poll-interval` ≥ 1, both integers; otherwise exit 1. -- `url` must parse to a host beginning with `connection.`; the hostname suffix for `ServiceClient` - is that host minus the `connection.` prefix (e.g. `north-europe.azure.keboola.com`). This keeps - the issue-mandated full-URL argument *and* resolves the Queue API URL via `keboola/service-client` - (no `connection` → `queue` string replace on the URL). - -## Architecture - -Only the Symfony command itself lives directly in `src/Keboola/Console/Command/`; its four -helper classes go into the `FlowMigration/` subnamespace (PSR-0: namespace -`Keboola\Console\Command\FlowMigration` → `src/Keboola/Console/Command/FlowMigration/`), which -keeps the flat command directory a list of commands. The helper class names therefore drop the -redundant `FlowMigration` prefix the namespace already carries. Registration goes in `cli.php`: - -``` -Command/MigrateOrchestrationsToFlow (Symfony Command — thin shell) - ├─ parses/validates input, resolves project ID list - ├─ builds ManageApi\Client, ServiceClient, FlowMigration\ProjectClientsFactory - ├─ opens the CSV report (append mode, header if new/empty) and wires the - │ per-result callback: CSV row + progress line to stdout - ├─ runs FlowMigration\BatchRunner - └─ prints final summary, returns exit code (1 if any project failed) - -Command/FlowMigration/BatchRunner (plain class — ALL batch logic, unit-tested) - ├─ per-project pipeline (skip rules, job submission) - ├─ concurrency window + polling loop - └─ emits one ProjectResult per input project via callback, - returns aggregate summary counts - -Command/FlowMigration/ProjectClientsFactory (plain class — the only network seam) - ├─ getProject(string $projectId): array (Manage API) - └─ createProjectClients(string $projectId): ProjectClients - creates the ephemeral storage token, returns Components + JobQueueClient - bound to that token - -Command/FlowMigration/ProjectClients (tiny DTO: Components + JobQueueClient) -Command/FlowMigration/ProjectResult (tiny DTO: projectId, jobId, status, - durationSeconds, error + isFailed()) -``` - -Rationale: the repo's testable-logic pattern (`DataAppOrchestratorTaskMigrator` + -`FakeComponents`) extended one step — because per-project clients are created with per-project -ephemeral tokens, the runner cannot receive clients directly; it receives a factory. Tests -subclass the factory and the SDK clients without calling parent constructors (exactly how -`FakeComponents` already works). No interfaces — the codebase does not use them. - -### Ephemeral token (created per project, before any Storage/Queue call) - -```php -$manageClient->createProjectStorageToken($projectId, [ - 'description' => 'AJDA-3117 keboola.orchestrator -> keboola.flow migration (batch driver)', - 'expiresIn' => 43200, // 12 h: job may wait in queue and runs long; expiry mid-migration is worse than a short-lived privileged token - 'canManageBuckets' => true, - 'canReadAllFileUploads' => true, - 'componentAccess' => [ - 'keboola.orchestrator', - 'keboola.flow', - 'keboola.scheduler', - 'keboola.flow-migration-tool', - ], -]); -``` - -Broad rights are deliberate (trigger/notification migration touches project-level resources); -the token expires on its own, no cleanup step. - -### Per-project pipeline (inside the runner, at submission time) - -1. `getProject()` — `isDisabled` → result `skipped-disabled`. Manage API 404 (deleted project) - → also `skipped-disabled` (issue counts disabled+deleted together). Other Manage errors → - `error` result (project failed, batch continues). -2. Create ephemeral token + per-project clients via the factory. Failure → `error` result. -3. `listComponentConfigurations(componentId: keboola.orchestrator, isDeleted: false)` — - empty → `skipped-no-orchestrations` (no job created; avoids hundreds of empty jobs in - customers' job history). -4. Queue guard: `listJobs(components: [keboola.flow-migration-tool], statuses: [CREATED, - WAITING, PROCESSING, TERMINATING], limit: 1)` — non-empty → `skipped-job-running`. - (`TERMINATING` added on top of the issue's three: a terminating job may still be executing - migration writes, and skipping it strictly reduces overlap risk.) -5. `createJob(new JobData('keboola.flow-migration-tool', configData: [...]))` — via `configData`, - so no stored configuration is left behind in the project: - - ```json - { - "parameters": { - "mode": "project", - "orchestrationIds": [], - "skipBroken": true, - "dryRun": - } - } - ``` - - (`orchestrationIds: []` and `skipBroken: true` are required by the component's config - definition in `project` mode.) Job enters the in-flight window with its own `JobQueueClient`. - -### Concurrency window + polling - -``` -pending = input project queue -inFlight = projectId → {jobId, queueClient, startedAtWallClock, consecutivePollFailures} - -while pending not empty or inFlight not empty: - fill: while |inFlight| < concurrency and pending: submit next - (skips/errors resolve immediately → result callback, do not occupy a slot) - if inFlight empty: continue - sleep(pollInterval) # injected callable, no-op in tests - for each inFlight job: getJob() - finished → result callback (terminal status), free the slot - poll exception → consecutivePollFailures++; after 3 consecutive failures - mark project error ("job still running server-side, polling gave up"), - free the slot; a successful poll resets the counter -``` - -- `durationSeconds` in the result: `Job->durationSeconds` when the API provides it, otherwise - wall-clock from submission. -- No per-job or global timeout (operator supervises; token bounds the run at 12 h anyway). -- No signal handling — the incrementally-appended CSV already makes an interrupted run auditable. -- `sleep` is injected as a `callable` (default `sleep(...)`) so unit tests run instantly and can - assert poll cadence. - -### CSV report - -- Path from `--report`, default `flow-migration--.csv` in cwd. -- Opened in append mode; header written only when the file is new or empty. -- Written with `fputcsv(..., separator: ';')` — error messages containing `;`/newlines get quoted - correctly; no new dependency. -- Header + one row **per input project** (including skips, for a complete audit of the list): - - ``` - projectId;jobId;status;durationSeconds;error - ``` - - `status` ∈ job terminal status (`success`, `warning`, `error`, `terminated`, `cancelled`) - or `skipped-disabled` | `skipped-no-orchestrations` | `skipped-job-running` | `error`. - `jobId` is empty for rows without a job. Rows are appended the moment each project resolves. - -### Progress output and summary - -- One stdout line per event (`[HH:ii:ss]` prefix): job submitted (with job id + job URL), project - skipped (with reason), project finished (status + duration), poll warnings. -- Final summary: attempted / migrated (job `success`; `warning` reported as migrated-with-warning) - / skipped-no-orchestrations / skipped-disabled / skipped-job-running / failed - (job `error`|`terminated`|`cancelled`, or driver-side error). -- **Exit code 1 if at least one project failed, else 0.** A failing project never aborts the batch. - -### Re-runs - -Re-running the same list is the intended recovery path: the component's -`AlreadyMigratedValidator` reports already-migrated orchestrations as `skipped`, and the queue -guard skips projects with a live migration job. No resume state is kept by the driver. - -## Error handling summary - -| Failure | Behavior | -|---|---| -| Invalid input (IDs, options, both/neither project sources) | message + exit 1, nothing executed | -| `getProject` 404 | `skipped-disabled` (deleted) | -| `getProject` other error, token creation, config listing, guard, `createJob` failure | `error` result for that project, batch continues | -| Poll failure | tolerated 3 consecutive times per job, then `error` result; job keeps running server-side | -| Job terminal `error`/`terminated`/`cancelled` | `failed` in summary, exit code 1 | - -## Testing - -`tests/FlowMigration/BatchRunnerTest.php` + fakes, mirroring the src layout in the PSR-4 -subnamespace `Keboola\Console\Tests\FlowMigration\` (shared `FakeComponents` stays in -`Keboola\Console\Tests\`): - -- `FakeJobQueueClient extends JobQueueClient\Client` — constructor override (no parent call, same - trick as `FakeComponents`), records `createJob` calls, scripted `getJob` status sequences - (e.g. `processing, processing, success`), scripted `listJobs` guard responses, can throw on - demand for poll-failure tests. -- `FakeProjectClientsFactory extends ProjectClientsFactory` — scripted - projects (disabled / deleted / erroring), records `createProjectClients` calls (asserts no token - is created for disabled projects), returns `ProjectClients` built from - `FakeComponents` (reused as-is for the orchestrator-config listing) + `FakeJobQueueClient`. - -Scenarios (assert emitted results, summary counts, recorded API calls, sleep-callable cadence): -1. happy path — N projects, jobs created with exact `configData` (incl. `dryRun` true/false by - force flag), results in completion order; -2. all three skip rules, each without a job being created (and without a token for disabled); -3. concurrency window never exceeds the limit and refills as jobs finish; -4. one project's job ends `error` → batch continues, summary flags failure; -5. driver-side error (token creation throws) → `error` result, batch continues; -6. poll failures: 2 consecutive then success → no failure; 3 consecutive → `error` result; -7. duplicate project IDs in input are submitted once. - -The Symfony command shell itself is not unit-tested (repo convention); `composer phpcs`, -`composer phpstan` (level 9), `composer tests` via `docker compose` must pass. - -## Documentation - -README.md, section "Project manipulation", after "Migrate data-apps orchestrator/flow tasks…": -usage line, Arguments/Options, Behavior — including the explicit warning that dry-run still -creates real jobs and real ephemeral tokens in customer projects, the CSV format, re-run safety, -and exit-code semantics. - -## Decisions made without the reporter (recorded, with rationale) - -1. **`url` stays a full connection URL; hostname suffix is derived** (strip `connection.` from the - parsed host) — keeps `manage:call-on-stacks` compatibility and the issue's signature while - using `ServiceClient` for the Queue URL. -2. **`TERMINATING` added to the guard statuses** — a terminating job may still write; strictly safer. -3. **CSV gets a row for every input project including skips** — makes the report a complete audit; - the issue's "resolvable job ID for every project" holds for every project that got a job. -4. **`warning` terminal status counts as migrated** (reported distinctly) — the component finished; - exit code stays 0. The CSV carries the raw status either way. -5. **Poll failures tolerated 3× consecutively, then the project is marked failed** — an unbounded - retry could hang the batch forever; the client already retries 5xx internally 3×. -6. **Duplicates deduplicated, `#`-comment lines allowed in the projects file** — hundreds-of-IDs - lists are hand-assembled; cheap robustness. -7. **No interactive confirmation** — unlike the `all` mode of the data-apps migration command, - the blast radius here is always an explicit project list. -8. **`fputcsv` over `keboola/csv`** — append semantics with header-once needs a plain handle; - no new dependency. -9. **English spec/plan/docs** — repo and git content are English per user's git conventions. - -## Out of scope (YAGNI) - -- No `all` projects mode — per-stack batches are driven from explicit lists (AJDA-3119/3120). -- No exposure of `parameters.migrate.*` sub-flags. -- No resume file/state beyond the CSV; no signal handling; no per-job timeout. -- No per-orchestration counts in the CSV — the component does not expose them in the job result - (issue open question; possible follow-up in `keboola/flow-migration-tool`). - -## Hand-off items (require live stack access — not implementation work) - -The five "Must verify before the first live batch" items from AJDA-3117 (billing exclusion on -PAYGO, notification listing visibility for ephemeral tokens, trigger `runWithTokenId` permission, -token lifetime vs. queue wait, `keboola.flow` availability without a per-project feature) plus -acceptance criterion 9 must be confirmed by Ondrej on a live project and recorded in a Linear -comment. The command itself is designed so these checks can run as a one-project batch first. diff --git a/src/Keboola/Console/Command/FlowMigration/BatchRunner.php b/src/Keboola/Console/Command/FlowMigration/BatchRunner.php index 54291c4..bdb0332 100644 --- a/src/Keboola/Console/Command/FlowMigration/BatchRunner.php +++ b/src/Keboola/Console/Command/FlowMigration/BatchRunner.php @@ -61,6 +61,9 @@ class BatchRunner /** @var callable(int): void */ private $sleep; + /** + * @param int $concurrency max jobs in flight; must be >= 1, validated by the calling command + */ public function __construct( ProjectClientsFactory $clientsFactory, int $concurrency, @@ -68,9 +71,7 @@ public function __construct( ?callable $sleep = null ) { $this->clientsFactory = $clientsFactory; - // A window smaller than one job would never let the run loop drain the pending queue, - // i.e. it would hang the batch forever - clamp instead of spinning. - $this->concurrency = max(1, $concurrency); + $this->concurrency = $concurrency; $this->pollIntervalSeconds = $pollIntervalSeconds; $this->sleep = $sleep ?? function (int $seconds): void { sleep($seconds); diff --git a/src/Keboola/Console/Command/FlowMigration/ProjectClientsFactory.php b/src/Keboola/Console/Command/FlowMigration/ProjectClientsFactory.php index b41b06d..e335411 100644 --- a/src/Keboola/Console/Command/FlowMigration/ProjectClientsFactory.php +++ b/src/Keboola/Console/Command/FlowMigration/ProjectClientsFactory.php @@ -17,18 +17,10 @@ class ProjectClientsFactory { private const TOKEN_DESCRIPTION = 'AJDA-3117 keboola.orchestrator to keboola.flow migration (batch driver)'; - // 12 hours: the job may wait in the queue and the runtime keeps using this token for the whole - // run - an expiry mid-migration is worse than a longer-lived privileged token. - private const TOKEN_EXPIRES_IN_SECONDS = 43200; - - // Broad rights on purpose: trigger and notification migration touches tables and project-level - // resources, and a migration failing halfway through is worse than a short-lived privileged token. - private const TOKEN_COMPONENT_ACCESS = [ - 'keboola.orchestrator', - 'keboola.flow', - 'keboola.scheduler', - 'keboola.flow-migration-tool', - ]; + // One hour. The job runs for minutes and, being started from configData, does not wait on + // anything project-side - only the shared queue - so this covers the whole run with a wide + // margin while keeping a fully privileged token short-lived. + private const TOKEN_EXPIRES_IN_SECONDS = 3600; private ManageClient $manageClient; private string $connectionUrl; @@ -51,12 +43,17 @@ public function getProject(string $projectId): array public function createProjectClients(string $projectId): ProjectClients { + // Full project rights on purpose. The migration writes configurations, tables, triggers + // (with a runWithTokenId copied from the source trigger, i.e. another token) and + // notification subscriptions, and a migration failing halfway through is worse than a + // short-lived privileged token. Restricting this only risks the component missing something. $tokenInfo = $this->manageClient->createProjectStorageToken($projectId, [ 'description' => self::TOKEN_DESCRIPTION, 'expiresIn' => self::TOKEN_EXPIRES_IN_SECONDS, 'canManageBuckets' => true, + 'canManageTokens' => true, 'canReadAllFileUploads' => true, - 'componentAccess' => self::TOKEN_COMPONENT_ACCESS, + 'canPurgeTrash' => true, ]); $storageClient = new StorageClient([ diff --git a/tests/FlowMigration/BatchRunnerTest.php b/tests/FlowMigration/BatchRunnerTest.php index 96c88a3..bf8ea95 100644 --- a/tests/FlowMigration/BatchRunnerTest.php +++ b/tests/FlowMigration/BatchRunnerTest.php @@ -158,59 +158,27 @@ public function testJobEndingInErrorMarksProjectFailedButBatchContinues(): void $this->assertSame(1, $summary['failed']); } - public function testWarningJobCountsAsMigratedWithWarning(): void + public function testSkipsDisabledAndDeletedProjectsWithoutCreatingTokenOrJob(): void { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'warning', 5, ['message' => 'partial'])]] - ); - $factory = new FakeProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); + $factory = new FakeProjectClientsFactory([ + '100' => ['id' => '100', 'name' => 'Off', 'isDisabled' => true], + // A deleted project surfaces as a Manage API 404 and is reported the same way. + '200' => new ManageClientException('Project not found', 404), + ]); $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame('warning', $this->results[0]->status); - $this->assertFalse($this->results[0]->isFailed()); - $this->assertSame(1, $summary['migratedWithWarning']); - $this->assertSame(0, $summary['migrated']); - $this->assertSame(0, $summary['failed']); - } - - public function testSkipsDisabledProjectWithoutCreatingTokenOrJob(): void - { - $factory = new FakeProjectClientsFactory( - ['100' => ['id' => '100', 'name' => 'Off', 'isDisabled' => true]] - ); - $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); + $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); - // No ephemeral token may be created for a disabled project. + // No ephemeral token may be created for a project that will not be migrated. $this->assertSame([], $factory->createClientsCalls); $this->assertSame(ProjectResult::STATUS_SKIPPED_DISABLED, $this->results[0]->status); + $this->assertSame(ProjectResult::STATUS_SKIPPED_DISABLED, $this->results[1]->status); $this->assertNull($this->results[0]->jobId); - $this->assertSame(1, $summary['skippedDisabled']); + $this->assertSame(2, $summary['skippedDisabled']); $this->assertSame(0, $summary['failed']); $this->assertSame([], $this->sleeps); } - public function testSkipsDeletedProjectOnManage404(): void - { - $factory = new FakeProjectClientsFactory( - ['100' => new ManageClientException('Project not found', 404)] - ); - $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame([], $factory->createClientsCalls); - $this->assertSame(ProjectResult::STATUS_SKIPPED_DISABLED, $this->results[0]->status); - $this->assertSame(1, $summary['skippedDisabled']); - } - public function testManageErrorOtherThan404MarksProjectFailed(): void { $factory = new FakeProjectClientsFactory( @@ -290,34 +258,16 @@ public function testSkipsProjectWithLiveMigrationJob(): void $this->assertSame([], $queueClient->createdJobs); $this->assertSame(ProjectResult::STATUS_SKIPPED_JOB_RUNNING, $this->results[0]->status); $this->assertSame(1, $summary['skippedJobRunning']); - } - - public function testDriverSideErrorWhenTokenCreationFailsAndBatchContinues(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-2', 'created')], - ['job-2' => [FakeJobQueueClient::makeJob('job-2', 'success', 3)]] - ); - $factory = new FakeProjectClientsFactory( - ['100' => self::enabledProject('100'), '200' => self::enabledProject('200')], + // The scripted return does not depend on the query, so assert the query itself: a guard + // asking for another component or for terminal statuses would skip or submit wrongly. + $this->assertSame( [ - '100' => new ManageClientException('Cannot create token', 403), - '200' => self::clientsWith($queueClient), - ] + 'component' => ['keboola.flow-migration-tool'], + 'limit' => 1, + 'status' => ['created', 'waiting', 'processing', 'terminating'], + ], + $queueClient->listJobsQueries[0] ); - $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100', '200'], true, new BufferedOutput(), $this->collector()); - - $byProject = []; - foreach ($this->results as $result) { - $byProject[$result->projectId] = $result; - } - $this->assertSame(ProjectResult::STATUS_ERROR, $byProject['100']->status); - $this->assertSame('Cannot create token', $byProject['100']->error); - $this->assertSame('success', $byProject['200']->status); - $this->assertSame(1, $summary['failed']); - $this->assertSame(1, $summary['migrated']); } public function testDeduplicatesInputProjectIds(): void @@ -339,23 +289,6 @@ public function testDeduplicatesInputProjectIds(): void $this->assertCount(1, $this->results); } - public function testNonPositiveConcurrencyStillDrainsTheQueueInsteadOfHanging(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [FakeJobQueueClient::makeJob('job-1', 'success', 1)]] - ); - $factory = new FakeProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new BatchRunner($factory, 0, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame(1, $summary['migrated']); - } - public function testTwoConsecutivePollFailuresAreToleratedAndJobFinishes(): void { $queueClient = new FakeJobQueueClient( @@ -406,32 +339,6 @@ public function testThreeConsecutivePollFailuresMarkProjectFailedWithJobIdKept() $this->assertSame(1, $summary['failed']); } - public function testPollFailureCounterResetsAfterASuccessfulPoll(): void - { - $queueClient = new FakeJobQueueClient( - [FakeJobQueueClient::makeJob('job-1', 'created')], - ['job-1' => [ - new RuntimeException('blip 1'), - new RuntimeException('blip 2'), - FakeJobQueueClient::makeJob('job-1', 'processing'), - new RuntimeException('blip 3'), - new RuntimeException('blip 4'), - FakeJobQueueClient::makeJob('job-1', 'success', 9), - ]] - ); - $factory = new FakeProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame('success', $this->results[0]->status); - $this->assertSame(1, $summary['migrated']); - $this->assertSame(0, $summary['failed']); - } - public function testConcurrencyWindowCapsInFlightJobsAndRefills(): void { // One shared fake for both projects makes the cross-project call order observable. diff --git a/tests/FlowMigration/FakeJobQueueClient.php b/tests/FlowMigration/FakeJobQueueClient.php index 8df294e..a77d2b6 100644 --- a/tests/FlowMigration/FakeJobQueueClient.php +++ b/tests/FlowMigration/FakeJobQueueClient.php @@ -26,6 +26,15 @@ class FakeJobQueueClient extends JobQueueClient public int $listJobsCalls = 0; + /** + * Query parameters of every listJobs() call. Recorded because the scripted return value is + * independent of the query - without asserting on this, a guard asking for the wrong component + * or the wrong statuses would still pass its test. + * + * @var array> + */ + public array $listJobsQueries = []; + /** @var array */ private array $createJobReturns; @@ -81,6 +90,7 @@ public function listJobs(ListJobsOptions $listOptions): array { $this->calls[] = ['listJobs']; $this->listJobsCalls++; + $this->listJobsQueries[] = $listOptions->getQueryParameters(); return $this->listJobsReturn; } diff --git a/tests/FlowMigration/FakeJobQueueClientTest.php b/tests/FlowMigration/FakeJobQueueClientTest.php deleted file mode 100644 index 3c6eaf8..0000000 --- a/tests/FlowMigration/FakeJobQueueClientTest.php +++ /dev/null @@ -1,41 +0,0 @@ - 'ok']); - - $this->assertFalse($running->isFinished); - $this->assertTrue($finished->isFinished); - $this->assertSame('success', $finished->status); - $this->assertSame(42, $finished->durationSeconds); - $this->assertSame(['message' => 'ok'], $finished->result); - } - - public function testGetJobConsumesScriptedSequenceAndThrowsThrowables(): void - { - $fake = new FakeJobQueueClient([], ['job-1' => [ - new RuntimeException('network blip'), - FakeJobQueueClient::makeJob('job-1', 'success'), - ]]); - - try { - $fake->getJob('job-1'); - $this->fail('First scripted outcome should throw'); - } catch (RuntimeException $e) { - $this->assertSame('network blip', $e->getMessage()); - } - - $this->assertSame('success', $fake->getJob('job-1')->status); - $this->assertSame([['getJob', 'job-1'], ['getJob', 'job-1']], $fake->calls); - } -} diff --git a/tests/MigrateOrchestrationsToFlowTest.php b/tests/MigrateOrchestrationsToFlowTest.php index 17f2825..1b7fb01 100644 --- a/tests/MigrateOrchestrationsToFlowTest.php +++ b/tests/MigrateOrchestrationsToFlowTest.php @@ -4,7 +4,6 @@ namespace Keboola\Console\Tests; -use Keboola\Console\Command\FlowMigration\ProjectResult; use Keboola\Console\Command\MigrateOrchestrationsToFlow; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; @@ -93,37 +92,4 @@ public static function provideProjectFiles(): iterable yield 'non-numeric line invalidates the file' => ["100\nfoo\n", null]; yield 'empty file is a valid empty list' => ['', []]; } - - /** - * The report is the audit artifact of a batch run, so a quote or backslash in an API error - * message must not be able to break a row for a standard CSV parser. - */ - public function testAppendReportRowKeepsTheRowParsableWithQuotesInTheError(): void - { - $command = new MigrateOrchestrationsToFlow(); - $method = (new ReflectionClass($command))->getMethod('appendReportRow'); - $method->setAccessible(true); - $handle = fopen('php://memory', 'w+'); - $this->assertIsResource($handle); - - $method->invoke( - $command, - $handle, - new ProjectResult('123', 'job-1', 'error', 7, 'Orchestration \"Daily load\" failed; retry') - ); - - rewind($handle); - $contents = stream_get_contents($handle); - fclose($handle); - - $this->assertIsString($contents); - $this->assertSame( - '123;job-1;error;7;"Orchestration \""Daily load\"" failed; retry"' . "\n", - $contents - ); - $this->assertSame( - ['123', 'job-1', 'error', '7', 'Orchestration \"Daily load\" failed; retry'], - str_getcsv(trim($contents), ';', '"', '') - ); - } } From 04f4ec55dc26617ed207fbb1a066b69849f20b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= <12143866+ondrajodas@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:04:30 +0200 Subject: [PATCH 5/5] fix: AJDA-3117 drop the already-running migration job guard The guard queried the Queue API for a live keboola.flow-migration-tool job in the project and skipped it. It was never mutual exclusion - createJob can still land right after listJobs returns empty - and the case it did cover (re-running an interrupted batch while its jobs are still in flight) is covered operationally by letting those jobs finish first, which the README now says explicitly. Removes the guard, its ProjectResult status, its summary counter and the listJobs() override in the fake queue client, which the runner no longer calls. --- README.md | 9 +++-- .../Command/FlowMigration/BatchRunner.php | 32 ++-------------- .../Command/FlowMigration/ProjectResult.php | 2 - .../Command/MigrateOrchestrationsToFlow.php | 5 +-- tests/FlowMigration/BatchRunnerTest.php | 37 +------------------ tests/FlowMigration/FakeJobQueueClient.php | 28 +------------- tests/FlowMigration/ProjectResultTest.php | 1 - 7 files changed, 12 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index 7a3af1b..6376fbf 100644 --- a/README.md +++ b/README.md @@ -520,8 +520,7 @@ Behavior: - For each project: skips disabled/deleted projects; creates an ephemeral 1h storage token with full project rights (`canManageBuckets`, `canManageTokens`, `canReadAllFileUploads`, `canPurgeTrash`) so the component cannot be short of a permission mid-migration; skips projects with no - `keboola.orchestrator` configurations (no empty jobs in customers' job history); skips projects - where a `keboola.flow-migration-tool` job is already created/waiting/processing/terminating. + `keboola.orchestrator` configurations (no empty jobs in customers' job history). - Creates the migration job via `configData` (no stored configuration is left behind) with `parameters: {mode: "project", orchestrationIds: [], skipBroken: true, dryRun: }`. - Keeps at most `--concurrency` jobs in flight, polls each job and refills the window as jobs finish. @@ -534,7 +533,11 @@ Behavior: - A failing project never aborts the batch. Exit code is `1` if at least one project failed (job `error`/`terminated`/`cancelled` or a driver-side error), `0` otherwise. - Final summary: projects attempted / migrated / migrated with warning / skipped (no - orchestrations, disabled, job already running) / failed. + orchestrations, disabled) / failed. +- Re-running the same project list is the intended recovery path: the component reports + already-migrated orchestrations as skipped. The command does **not** check for a migration job + already running in the project, so before re-running an interrupted batch let the jobs it already + created finish - two concurrent migrations of one project can both create the same flows. ### Mass enablement of dynamic backends for multiple projects Prerequisities: https://keboola.atlassian.net/wiki/spaces/KB/pages/2135982081/Enable+Dynamic+Backends#Enable-for-project diff --git a/src/Keboola/Console/Command/FlowMigration/BatchRunner.php b/src/Keboola/Console/Command/FlowMigration/BatchRunner.php index bdb0332..9681546 100644 --- a/src/Keboola/Console/Command/FlowMigration/BatchRunner.php +++ b/src/Keboola/Console/Command/FlowMigration/BatchRunner.php @@ -8,7 +8,6 @@ use Keboola\JobQueueClient\DTO\Job; use Keboola\JobQueueClient\JobData; use Keboola\JobQueueClient\JobStatuses; -use Keboola\JobQueueClient\ListJobsOptions; use Keboola\ManageApi\ClientException as ManageClientException; use Keboola\StorageApi\Options\Components\ListComponentConfigurationsOptions; use Symfony\Component\Console\Output\OutputInterface; @@ -24,7 +23,6 @@ * migratedWithWarning: int, * skippedNoOrchestrations: int, * skippedDisabled: int, - * skippedJobRunning: int, * failed: int * } * @phpstan-type InFlightJob array{ @@ -40,15 +38,6 @@ class BatchRunner public const ORCHESTRATOR_COMPONENT_ID = 'keboola.orchestrator'; public const MIGRATION_COMPONENT_ID = 'keboola.flow-migration-tool'; - // TERMINATING is included on top of the issue's created/waiting/processing: a terminating - // job may still be executing migration writes, and skipping it strictly reduces overlap risk. - private const LIVE_JOB_STATUSES = [ - JobStatuses::CREATED, - JobStatuses::WAITING, - JobStatuses::PROCESSING, - JobStatuses::TERMINATING, - ]; - // A transient Queue API outage must not fail a project instantly (the SDK already retries // 5xx internally), but an unbounded retry could hang the batch forever - so give up after // this many consecutive failed polls and leave the job to finish server-side. @@ -92,7 +81,6 @@ public function run(array $projectIds, bool $force, OutputInterface $output, cal 'migratedWithWarning' => 0, 'skippedNoOrchestrations' => 0, 'skippedDisabled' => 0, - 'skippedJobRunning' => 0, 'failed' => 0, ]; /** @var array $inFlight */ @@ -205,6 +193,9 @@ private function checkProjectIsActive(string $projectId): ?ProjectResult } /** + * A project with no keboola.orchestrator configurations gets no job at all, so hundreds of + * empty jobs never show up in customers' job history. + * * @return ProjectResult|null non-null when no new migration job should be created */ private function checkProjectNeedsMigration( @@ -226,22 +217,6 @@ private function checkProjectNeedsMigration( ); } - $liveJobs = $clients->queueClient->listJobs( - (new ListJobsOptions()) - ->setComponents([self::MIGRATION_COMPONENT_ID]) - ->setStatuses(self::LIVE_JOB_STATUSES) - ->setLimit(1) - ); - if ($liveJobs !== []) { - return new ProjectResult( - $projectId, - null, - ProjectResult::STATUS_SKIPPED_JOB_RUNNING, - null, - 'a keboola.flow-migration-tool job is already running in this project' - ); - } - return null; } @@ -430,7 +405,6 @@ private function summaryKeyFor(ProjectResult $result): string JobStatuses::WARNING->value => 'migratedWithWarning', ProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS => 'skippedNoOrchestrations', ProjectResult::STATUS_SKIPPED_DISABLED => 'skippedDisabled', - ProjectResult::STATUS_SKIPPED_JOB_RUNNING => 'skippedJobRunning', default => 'failed', }; } diff --git a/src/Keboola/Console/Command/FlowMigration/ProjectResult.php b/src/Keboola/Console/Command/FlowMigration/ProjectResult.php index 38bbd50..6c855f0 100644 --- a/src/Keboola/Console/Command/FlowMigration/ProjectResult.php +++ b/src/Keboola/Console/Command/FlowMigration/ProjectResult.php @@ -14,7 +14,6 @@ class ProjectResult { public const STATUS_SKIPPED_DISABLED = 'skipped-disabled'; public const STATUS_SKIPPED_NO_ORCHESTRATIONS = 'skipped-no-orchestrations'; - public const STATUS_SKIPPED_JOB_RUNNING = 'skipped-job-running'; public const STATUS_ERROR = 'error'; public string $projectId; @@ -42,7 +41,6 @@ public function isSkipped(): bool return in_array($this->status, [ self::STATUS_SKIPPED_DISABLED, self::STATUS_SKIPPED_NO_ORCHESTRATIONS, - self::STATUS_SKIPPED_JOB_RUNNING, ], true); } diff --git a/src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php b/src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php index 73d5caa..e2ed09b 100644 --- a/src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php +++ b/src/Keboola/Console/Command/MigrateOrchestrationsToFlow.php @@ -385,7 +385,6 @@ private function appendReportRow($reportHandle, ProjectResult $result): void * migratedWithWarning: int, * skippedNoOrchestrations: int, * skippedDisabled: int, - * skippedJobRunning: int, * failed: int * } $summary */ @@ -394,14 +393,12 @@ private function printSummary(OutputInterface $output, array $summary): void $output->writeln(''); $output->writeln(sprintf( "DONE\nProjects attempted: %d\nMigrated (job success): %d\nMigrated with warning: %d\n" - . "Skipped (no orchestrations): %d\nSkipped (disabled/deleted): %d\n" - . "Skipped (migration job already running): %d\nFailed: %d", + . "Skipped (no orchestrations): %d\nSkipped (disabled/deleted): %d\nFailed: %d", $summary['attempted'], $summary['migrated'], $summary['migratedWithWarning'], $summary['skippedNoOrchestrations'], $summary['skippedDisabled'], - $summary['skippedJobRunning'], $summary['failed'] )); } diff --git a/tests/FlowMigration/BatchRunnerTest.php b/tests/FlowMigration/BatchRunnerTest.php index bf8ea95..d3fc1b2 100644 --- a/tests/FlowMigration/BatchRunnerTest.php +++ b/tests/FlowMigration/BatchRunnerTest.php @@ -88,8 +88,6 @@ public function testHappyPathCreatesJobAndReportsSuccess(): void ]], $queueClient->createdJobs[0]['configData'] ); - // The live-job guard ran exactly once before submission. - $this->assertSame(1, $queueClient->listJobsCalls); $this->assertCount(1, $this->results); $this->assertSame('100', $this->results[0]->projectId); @@ -233,43 +231,12 @@ public function testSkipsProjectWithoutOrchestratorConfigurations(): void $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - // No queue API call and no job for a project with nothing to migrate. - $this->assertSame(0, $queueClient->listJobsCalls); + // No job for a project with nothing to migrate. $this->assertSame([], $queueClient->createdJobs); $this->assertSame(ProjectResult::STATUS_SKIPPED_NO_ORCHESTRATIONS, $this->results[0]->status); $this->assertSame(1, $summary['skippedNoOrchestrations']); } - public function testSkipsProjectWithLiveMigrationJob(): void - { - $queueClient = new FakeJobQueueClient( - [], - [], - [FakeJobQueueClient::makeJob('existing-job', 'processing')] - ); - $factory = new FakeProjectClientsFactory( - ['100' => self::enabledProject('100')], - ['100' => self::clientsWith($queueClient)] - ); - $runner = new BatchRunner($factory, 10, 5, $this->sleepRecorder()); - - $summary = $runner->run(['100'], true, new BufferedOutput(), $this->collector()); - - $this->assertSame([], $queueClient->createdJobs); - $this->assertSame(ProjectResult::STATUS_SKIPPED_JOB_RUNNING, $this->results[0]->status); - $this->assertSame(1, $summary['skippedJobRunning']); - // The scripted return does not depend on the query, so assert the query itself: a guard - // asking for another component or for terminal statuses would skip or submit wrongly. - $this->assertSame( - [ - 'component' => ['keboola.flow-migration-tool'], - 'limit' => 1, - 'status' => ['created', 'waiting', 'processing', 'terminating'], - ], - $queueClient->listJobsQueries[0] - ); - } - public function testDeduplicatesInputProjectIds(): void { $queueClient = new FakeJobQueueClient( @@ -367,11 +334,9 @@ public function testConcurrencyWindowCapsInFlightJobsAndRefills(): void // With concurrency=1, job-2 must not be created until job-1 has finished. $this->assertSame( [ - ['listJobs'], ['createJob', 'job-1'], ['getJob', 'job-1'], ['getJob', 'job-1'], - ['listJobs'], ['createJob', 'job-2'], ['getJob', 'job-2'], ], diff --git a/tests/FlowMigration/FakeJobQueueClient.php b/tests/FlowMigration/FakeJobQueueClient.php index a77d2b6..cf50dcb 100644 --- a/tests/FlowMigration/FakeJobQueueClient.php +++ b/tests/FlowMigration/FakeJobQueueClient.php @@ -7,7 +7,6 @@ use Keboola\JobQueueClient\Client as JobQueueClient; use Keboola\JobQueueClient\DTO\Job; use Keboola\JobQueueClient\JobData; -use Keboola\JobQueueClient\ListJobsOptions; use RuntimeException; use Throwable; @@ -24,36 +23,20 @@ class FakeJobQueueClient extends JobQueueClient /** @var array> ordered call log: [method, id] */ public array $calls = []; - public int $listJobsCalls = 0; - - /** - * Query parameters of every listJobs() call. Recorded because the scripted return value is - * independent of the query - without asserting on this, a guard asking for the wrong component - * or the wrong statuses would still pass its test. - * - * @var array> - */ - public array $listJobsQueries = []; - /** @var array */ private array $createJobReturns; /** @var array> */ private array $getJobSequences; - /** @var array */ - private array $listJobsReturn; - /** * @param array $createJobReturns successive createJob() returns * @param array> $getJobSequences jobId => successive getJob() outcomes - * @param array $listJobsReturn returned by every listJobs() call */ - public function __construct(array $createJobReturns = [], array $getJobSequences = [], array $listJobsReturn = []) + public function __construct(array $createJobReturns = [], array $getJobSequences = []) { $this->createJobReturns = $createJobReturns; $this->getJobSequences = $getJobSequences; - $this->listJobsReturn = $listJobsReturn; } public function createJob(JobData $jobData): Job @@ -86,15 +69,6 @@ public function getJob(string $jobId): Job return $outcome; } - public function listJobs(ListJobsOptions $listOptions): array - { - $this->calls[] = ['listJobs']; - $this->listJobsCalls++; - $this->listJobsQueries[] = $listOptions->getQueryParameters(); - - return $this->listJobsReturn; - } - /** * Builds a real DTO\Job through its public factory so the fixture stays in sync with the SDK. * diff --git a/tests/FlowMigration/ProjectResultTest.php b/tests/FlowMigration/ProjectResultTest.php index 66e8714..693c9d5 100644 --- a/tests/FlowMigration/ProjectResultTest.php +++ b/tests/FlowMigration/ProjectResultTest.php @@ -35,6 +35,5 @@ public static function provideStatuses(): iterable true, false, ]; - yield 'skipped job running' => [ProjectResult::STATUS_SKIPPED_JOB_RUNNING, true, false]; } }