diff --git a/.claude/docs/design-system.md b/.claude/docs/design-system.md index 5e60830ce8..8ef3c9d27e 100644 --- a/.claude/docs/design-system.md +++ b/.claude/docs/design-system.md @@ -237,6 +237,9 @@ Three rules keep them inside the column: - Material floors every header at 5rem; that floor is removed, so a narrow column (`#`, a flag) takes only what it needs. - Content tabs are linked (`content.tabs.link`): picking Java on one set switches every set on the page and the choice persists. Tab strips are compact segmented controls sitting 0.35rem above their block. +Where the chips themselves are too long for any column, use a list, not a table. +The generated Helm values pages (`setup_installation/common/helm_chart_values/`) carry keys up to 90 characters, so each value is a definition-list entry inside `.hops-values`: the key chip on a hairline-separated row, its type, default and description under it at table type size. + ## Theme features Set in `mkdocs.yml` under `theme.features`. Current set and why: diff --git a/.github/workflows/mkdocs-test.yml b/.github/workflows/mkdocs-test.yml index f6211bb7a6..623ff4de85 100644 --- a/.github/workflows/mkdocs-test.yml +++ b/.github/workflows/mkdocs-test.yml @@ -31,6 +31,25 @@ jobs: - name: Install Python dependencies run: uv sync --extra cli + # The Helm values pages are filled by gen-helm-values at build time and + # never committed. A committed copy is published as-is whenever a build + # leaves the page alone (no matching chart in Nexus yet), so the check + # resets every page to its placeholder and fails if that changed one. + - name: Check the Helm values pages hold only their placeholders + run: | + hopsworks-docs reset-helm-values + if ! git diff --exit-code --stat -- docs/setup_installation/common/helm_chart_values/; then + echo "::error::Generated Helm values content is committed in docs/setup_installation/common/helm_chart_values/ (files above)." + echo "Those pages are generated during the documentation build and must hold only their placeholder." + echo "To fix it, in your branch run:" + echo " uv run --extra cli hopsworks-docs reset-helm-values" + echo " git add docs/setup_installation/common/helm_chart_values/" + echo " git commit -m 'Reset the Helm values pages to their placeholders'" + echo "The reset keeps any edit outside the generated block (a page title, its intro line, a new page)." + echo "To preview the values locally, run gen-helm-values, then the reset above before committing." + exit 1 + fi + - name: Install Hopsworks Python API run: uv pip install hopsworks-api/python @@ -48,13 +67,18 @@ jobs: NEXUS_USER: ${{ secrets.NEXUS_USER }} NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} BASE_REF: ${{ github.base_ref }} + # --strict turns the generator's warnings (a subchart without a page, an + # upstream chart without a docs link, a missing common value or RonDB + # schema, a RonDB override the schema does not declare, two entries on + # a page with one anchor, a null override with values listed under it) + # into a failed PR check; the deploy workflow only warns. run: | if [ "$BASE_REF" = "main" ]; then # dev docs: latest chart from the (private) dev channel. Fork PRs do # not receive secrets, so skip (keep the page placeholder) when the # Nexus credentials are absent rather than failing the build. if [ -n "$NEXUS_USER" ]; then - hopsworks-docs gen-helm-values \ + hopsworks-docs gen-helm-values --strict \ --repo-url https://nexus.hops.works/repository/hopsworks-helm-dev else echo "Nexus credentials not available (e.g. a fork PR); keeping the values reference placeholder." @@ -62,7 +86,7 @@ jobs: else # release docs: latest patch of the chart version (major.minor) for this docs version. # The released repo is public, so fetch anonymously (clear any creds). - NEXUS_USER= NEXUS_PASSWORD= hopsworks-docs gen-helm-values \ + NEXUS_USER= NEXUS_PASSWORD= hopsworks-docs gen-helm-values --strict \ --repo-url https://nexus.hops.works/repository/hopsworks-helm \ --chart-version "${BASE_REF#branch-}" fi diff --git a/README.md b/README.md index aa7848bed7..82c1364e0f 100644 --- a/README.md +++ b/README.md @@ -92,18 +92,33 @@ After adding your new page in the docs folder, you also need to add it to this f ## Helm chart values reference -The `setup_installation/common/helm_chart_values.md` page renders a placeholder locally; its `## Values` table is injected at build time by the `hopsworks-docs gen-helm-values` step and is **never committed**. +The pages under `setup_installation/common/helm_chart_values/` render placeholders locally; their values tables are injected at build time by the `hopsworks-docs gen-helm-values` step and are **never committed**. +The step reads the `## Values` table of the chart `README.md` plus that of every `charts//README.md`, with `.` put before each subchart key; the first row of a key wins, so a chart whose root README embeds the subchart tables (`helm-docs -u`, every release up to 5.1) and one whose README does not give the same pages. +It writes the rows of each top-level key into the stub page of that name (`kafka.md` gets the `kafka.*` rows), together with the subchart's deployment condition from the root `Chart.yaml` and links to the upstream charts it installs. +A row's default is what Hopsworks deploys: the root `values.yaml` settings for the subchart are merged over the subchart's own defaults the way Helm merges them; a subchart `values.yaml` that PyYAML cannot parse keeps its own defaults, with a warning. +`index.md` gets the common values (`_COMMON_VALUES` in `scripts/helm_values.py`), the overview table, plus the rows of any top-level key that has no stub page; to give a new subchart its own page, add a stub with the generation markers and a `nav:` entry. +For RonDB the page also renders the `values.schema.json` of the pinned RonDB chart, with Hopsworks' overrides (the wrapper subchart's `values.yaml`, then the root `values.yaml`) merged over its defaults the way Helm merges them; an overridden entry also names the RonDB chart's own default. +A values row the generator cannot parse always fails the step. +The PR check runs it with `--strict`, which also fails on a top-level key without a page, an upstream chart repository without a docs link in `_UPSTREAM_DOCS`, a common value the chart no longer has, a missing RonDB schema, a RonDB override of a key the schema does not declare (usually a value RonDB renamed, which the override then stops setting), two entries on a page with the same anchor (usually a helm-docs comment on a key under `rondb.rondb`, which the RonDB schema already lists), or a Hopsworks override that sets a key to null while values under it are listed (Helm removes them, but their entries would still show the chart defaults); the deploy workflow only warns. CI fetches the chart from Nexus rather than the (private) `hopsworks-helm` git repo: release builds (`branch-x.y`) read the public `hopsworks-helm` repo anonymously and pick the latest patch of the chart version matching the docs version, while the development build (`main`) reads the private `hopsworks-helm-dev` repo and uses its newest published chart. -The development path requires the read-only `NEXUS_USER` and `NEXUS_PASSWORD` repository secrets; without them the `main` build's generation step fails. -Older chart versions that predate the chart's `## Values` section keep the page placeholder (the build warns rather than failing). +The development path requires the read-only `NEXUS_USER` and `NEXUS_PASSWORD` repository secrets; without them the `main` build skips the generation step and keeps the placeholders. +Older chart versions that predate the chart's `## Values` section keep the page placeholders (the build warns rather than failing). -To preview the table locally against a chart checkout: +To preview the pages locally against a chart checkout: ```bash +helm dependency build /charts/rondb # only needed for the RonDB chart values uv run --extra cli hopsworks-docs gen-helm-values --chart ``` -This rewrites the page in place, so restore it (`git checkout docs/setup_installation/common/helm_chart_values.md`) before committing. +This rewrites the pages in place, so put the placeholders back before committing: + +```bash +uv run --extra cli hopsworks-docs reset-helm-values +``` + +The reset only touches the text between the generation markers, so edits to a page title or intro survive it. +The PR check runs the same reset and fails if it changes a committed page, since a committed copy of the values would be published as-is by any build that leaves the page alone. ## Checking links diff --git a/docs/css/custom.css b/docs/css/custom.css index 9005e6669d..e86da5af75 100644 --- a/docs/css/custom.css +++ b/docs/css/custom.css @@ -2468,3 +2468,87 @@ body.hops-zoom-lock { background: var(--md-default-bg-color); border: 1px solid var(--hops-border); } + +/* ---- Helm values reference: one entry per chart value ---------- + The generated pages under setup_installation/common/helm_chart_values/ + wrap their definition lists in .hops-values. Keys run to 90 characters, + so they cannot sit in a table (a table chip never wraps, see above); + each key is a hairline-separated row with its type, default and + description under it at table type size. */ +.md-typeset .hops-values dl { + margin: 0; +} +.md-typeset .hops-values dt { + margin: 0; + padding-top: 0.5rem; + border-top: 1px solid var(--hops-border); + font-size: 0.72rem; +} +.md-typeset .hops-values dd { + margin: 0.15rem 0 0.5rem 0; + font-size: 0.68rem; + line-height: 1.45; +} +.md-typeset .hops-values dd p { + margin: 0; +} +/* Deprecated values stay listed (an old override still has to validate) but + recede: a muted label after the key and a muted description. The muting is + the theme's light foreground, not opacity, which took the text below 4.5:1. */ +.md-typeset .hops-values-deprecated { + margin-left: 0.4em; + padding: 0.05em 0.4em; + border: 1px solid var(--hops-border-strong); + border-radius: 0.25rem; + color: var(--md-default-fg-color--light); + font-size: 0.58rem; + letter-spacing: 0.04em; + text-transform: uppercase; + vertical-align: middle; +} +.md-typeset .hops-values dt:has(.hops-values-deprecated) + dd { + color: var(--md-default-fg-color--light); +} +/* Filter box (js/values-filter.js), pinned under the header while the long + pages scroll. */ +.md-typeset .hops-values-filterbar { + position: sticky; + top: 2.4rem; + z-index: 1; + display: flex; + align-items: center; + gap: 0.6rem; + margin: 1rem 0 0.5rem; + padding: 0.4rem 0; + background: var(--md-default-bg-color); +} +.md-typeset .hops-values-filter { + flex: 1; + min-width: 0; + padding: 0.35rem 0.7rem; + border: 1px solid var(--hops-border-strong); + border-radius: 1rem; + background: var(--hops-surface); + color: var(--md-default-fg-color); + font: inherit; + font-size: 0.72rem; +} +.md-typeset .hops-values-filter:focus { + outline: 2px solid var(--hops-accent); + outline-offset: 1px; +} +.md-typeset .hops-values-count { + color: var(--md-default-fg-color--light); + font-size: 0.64rem; + white-space: nowrap; +} +/* A linked key or section lands below the header and the pinned filter bar, + not under it. */ +.md-content:has(.hops-values-filterbar) :is(h2, h3, dt)[id] { + scroll-margin-top: 5.6rem; +} +/* The filter hides a section's "Defaults as YAML" block with the hidden + attribute, which Material's `details { display: flow-root }` outranks. */ +.md-typeset details[hidden] { + display: none; +} diff --git a/docs/index.md b/docs/index.md index 456598c54e..c95e95b4de 100644 --- a/docs/index.md +++ b/docs/index.md @@ -234,7 +234,7 @@ Independent [feature, training and inference pipelines](concepts/fti.md), connec | Task | Start here | | --- | --- | -| :material-rocket-launch-outline: Deploy | [AWS](setup_installation/aws/getting_started.md), [Azure](setup_installation/azure/getting_started.md), [GCP](setup_installation/gcp/getting_started.md), [on-prem](setup_installation/on_prem/contact_hopsworks.md), [Helm values](setup_installation/common/helm_chart_values.md) | +| :material-rocket-launch-outline: Deploy | [AWS](setup_installation/aws/getting_started.md), [Azure](setup_installation/azure/getting_started.md), [GCP](setup_installation/gcp/getting_started.md), [on-prem](setup_installation/on_prem/contact_hopsworks.md), [Helm values][helm-chart-values-reference] | | :material-monitor-dashboard: Operate | [Administration](setup_installation/admin/index.md), [monitoring](setup_installation/admin/monitoring/grafana.md), [alerts](setup_installation/admin/alert.md), [HA and DR](setup_installation/admin/ha-dr/intro.md), [service operations](setup_installation/admin/operationLogs.md) | | :material-wrench-outline: Troubleshoot | [Model serving](user_guides/mlops/serving/troubleshooting.md), [Python deployments](user_guides/projects/python-deployment/troubleshooting.md), [online ingestion](user_guides/fs/feature_group/online_ingestion_observability.md), [Jupyter session capacity](user_guides/projects/jupyter/session_capacity_warnings.md) | | :material-arrow-up-circle-outline: Upgrade | [3.x to 4.0 migration](user_guides/migration/40_migration.md), [Airflow 3 upgrade](user_guides/projects/airflow/airflow3_upgrade.md), [Airflow 3 operator notes](setup_installation/admin/airflow3.md) | @@ -255,7 +255,7 @@ Independent [feature, training and inference pipelines](concepts/fti.md), connec :material-tune:{ .hops-colophon-ico } Configure and query { .hops-colophon-cap } -- [Helm chart values](setup_installation/common/helm_chart_values.md) +- [Helm chart values][helm-chart-values-reference] - [Cluster configuration](setup_installation/admin/variables.md) - [Query engine (Trino)](user_guides/projects/trino/query_engine.md) - [Vector similarity search](user_guides/fs/vector_similarity_search.md) diff --git a/docs/js/values-filter.js b/docs/js/values-filter.js new file mode 100644 index 0000000000..b0cc51ae6f --- /dev/null +++ b/docs/js/values-filter.js @@ -0,0 +1,58 @@ +// Filter box for the generated Helm values pages: narrows every .hops-values +// list to the entries whose key or description contains the query, and hides +// the sections left empty. Without this script every entry stays visible. +document.addEventListener("DOMContentLoaded", function () { + var lists = document.querySelectorAll(".md-typeset .hops-values"); + var terms = document.querySelectorAll(".md-typeset .hops-values dt"); + if (terms.length < 25) return; + + var bar = document.createElement("div"); + bar.className = "hops-values-filterbar"; + var input = document.createElement("input"); + input.type = "search"; + input.className = "hops-values-filter"; + input.placeholder = "Filter " + terms.length + " values by key or description"; + input.setAttribute("aria-label", "Filter values"); + var count = document.createElement("span"); + count.className = "hops-values-count"; + count.setAttribute("aria-live", "polite"); + bar.append(input, count); + + // A list's section is the heading and the "Defaults as YAML" block before it. + function sectionOf(list) { + var parts = []; + var el = list.previousElementSibling; + while (el && el.tagName === "DETAILS") { + parts.push(el); + el = el.previousElementSibling; + } + if (el && /^H[23]$/.test(el.tagName)) parts.push(el); + return parts; + } + + var first = lists[0]; + var firstSection = sectionOf(first); + var before = firstSection.length ? firstSection[firstSection.length - 1] : first; + before.parentNode.insertBefore(bar, before); + + input.addEventListener("input", function () { + var query = input.value.trim().toLowerCase(); + var shown = 0; + terms.forEach(function (dt) { + var dd = dt.nextElementSibling; + var text = (dt.textContent + " " + (dd ? dd.textContent : "")).toLowerCase(); + var match = !query || text.indexOf(query) !== -1; + dt.hidden = !match; + if (dd) dd.hidden = !match; + if (match) shown++; + }); + lists.forEach(function (list) { + var empty = list.querySelector("dt:not([hidden])") === null; + list.hidden = empty; + sectionOf(list).forEach(function (el) { + el.hidden = empty; + }); + }); + count.textContent = query ? shown + " of " + terms.length + " values" : ""; + }); +}); diff --git a/docs/setup_installation/common/helm_chart_values.md b/docs/setup_installation/common/helm_chart_values.md deleted file mode 100644 index a3549604aa..0000000000 --- a/docs/setup_installation/common/helm_chart_values.md +++ /dev/null @@ -1,16 +0,0 @@ -# Helm chart values reference - -This page lists every value you can configure when deploying Hopsworks with the Hopsworks Helm chart. -It is generated from the chart's `README.md`. -On a released version of the docs it matches the Hopsworks Helm chart for that release; on the development docs it reflects the latest chart published to the development channel. - -You set these values in the `values..yaml` file that you pass to `helm install`. -For a guided, end-to-end setup, follow one of the cloud installation guides, such as the [AWS getting started guide][aws-getting-started-with-eks]. -Only a small subset of these values is needed for a typical install; the table below is the exhaustive reference. - -## Configuration values - - -_The values table is generated from the Hopsworks Helm chart during the documentation build._ -_To preview it locally, run `uv run --extra cli hopsworks-docs gen-helm-values --chart `._ - diff --git a/docs/setup_installation/common/helm_chart_values/airflow.md b/docs/setup_installation/common/helm_chart_values/airflow.md new file mode 100644 index 0000000000..25cade3017 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/airflow.md @@ -0,0 +1,9 @@ +# Airflow values { #helm-values-airflow } + +Values under `airflow` configure Apache Airflow, which schedules and orchestrates Hopsworks jobs. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/arrowflight.md b/docs/setup_installation/common/helm_chart_values/arrowflight.md new file mode 100644 index 0000000000..94dee32e8e --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/arrowflight.md @@ -0,0 +1,9 @@ +# Arrow Flight values { #helm-values-arrowflight } + +Values under `arrowflight` configure the Arrow Flight server, which serves fast reads of feature groups and training datasets. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/certs-operator.md b/docs/setup_installation/common/helm_chart_values/certs-operator.md new file mode 100644 index 0000000000..8cd8639a40 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/certs-operator.md @@ -0,0 +1,9 @@ +# Certs operator values { #helm-values-certs-operator } + +Values under `certs-operator` configure the operator that issues the TLS certificates of the Hopsworks services and removes them on uninstall. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/consul.md b/docs/setup_installation/common/helm_chart_values/consul.md new file mode 100644 index 0000000000..b6a1808450 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/consul.md @@ -0,0 +1,9 @@ +# Consul values { #helm-values-consul } + +Values under `consul` configure Consul, which provides service discovery and DNS between the Hopsworks services. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/docker-registry.md b/docs/setup_installation/common/helm_chart_values/docker-registry.md new file mode 100644 index 0000000000..1e7ad84055 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/docker-registry.md @@ -0,0 +1,9 @@ +# Docker registry values { #helm-values-docker-registry } + +Values under `docker-registry` configure the in-cluster Docker registry that stores the images Hopsworks builds, such as project Python environments. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/global.md b/docs/setup_installation/common/helm_chart_values/global.md new file mode 100644 index 0000000000..3c1ce99f12 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/global.md @@ -0,0 +1,9 @@ +# Global values { #helm-values-global } + +Values under `global` are shared by every subchart: image registry and pull secrets, storage class, scheduling, cloud provider and which optional services are enabled. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/grafana.md b/docs/setup_installation/common/helm_chart_values/grafana.md new file mode 100644 index 0000000000..0c8d95253e --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/grafana.md @@ -0,0 +1,9 @@ +# Grafana values { #helm-values-grafana } + +Values under `grafana` configure Grafana and the Hopsworks monitoring dashboards. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/hive.md b/docs/setup_installation/common/helm_chart_values/hive.md new file mode 100644 index 0000000000..da3d763da8 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/hive.md @@ -0,0 +1,9 @@ +# Hive values { #helm-values-hive } + +Values under `hive` configure the Hive metastore, which holds the table metadata of the offline feature store. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/hopsfs.md b/docs/setup_installation/common/helm_chart_values/hopsfs.md new file mode 100644 index 0000000000..7985aab0f9 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/hopsfs.md @@ -0,0 +1,9 @@ +# HopsFS values { #helm-values-hopsfs } + +Values under `hopsfs` configure HopsFS, the distributed file system behind datasets and the offline feature store. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/hopsworks.md b/docs/setup_installation/common/helm_chart_values/hopsworks.md new file mode 100644 index 0000000000..0fe10e873d --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/hopsworks.md @@ -0,0 +1,9 @@ +# Hopsworks values { #helm-values-hopsworks } + +Values under `hopsworks` configure the Hopsworks backend: the Payara worker and admin deployments, ingress, the certificate authority, database migrations and backups. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/hw-kueue.md b/docs/setup_installation/common/helm_chart_values/hw-kueue.md new file mode 100644 index 0000000000..2c309e58de --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/hw-kueue.md @@ -0,0 +1,9 @@ +# Kueue values { #helm-values-hw-kueue } + +Values under `hw-kueue` configure Kueue, the job queueing controller, and the queues Hopsworks schedules jobs with. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/hw-kyverno.md b/docs/setup_installation/common/helm_chart_values/hw-kyverno.md new file mode 100644 index 0000000000..ef2e3ec570 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/hw-kyverno.md @@ -0,0 +1,9 @@ +# Kyverno values { #helm-values-hw-kyverno } + +Values under `hw-kyverno` configure the Kyverno cluster policies and the policy exceptions Hopsworks workloads need. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/index.md b/docs/setup_installation/common/helm_chart_values/index.md new file mode 100644 index 0000000000..65715ea269 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/index.md @@ -0,0 +1,23 @@ +# Helm chart values reference + +This section lists every value you can configure when deploying Hopsworks with the Hopsworks Helm chart, with one page per top-level key. +It is generated from the `README.md` files of the chart and its subcharts, and for RonDB from the `values.schema.json` of the RonDB chart that Hopsworks pins. +On a released version of the docs it matches the Hopsworks Helm chart for that release; on the development docs it reflects the latest chart published to the development channel. + +You set these values in the `values..yaml` file that you pass to `helm install`. +For a guided, end-to-end setup, follow one of the cloud installation guides, such as the [AWS getting started guide][aws-getting-started-with-eks]. +Only a small subset of these values is needed for a typical install: "Common values" below lists the ones the cloud installation guides set, and the pages are the exhaustive reference. + +"All values" lists the pages. +Each page states when its subchart is deployed: when the condition names several values, Helm uses the first one that is set. +"Upstream charts" are the charts a subchart installs from other Helm repositories, and each link opens a chart's documentation for the version Hopsworks pins. +A page lists only the values Hopsworks sets for its upstream charts, except the RonDB page, which lists all of the RonDB chart's values. +A default is the value Hopsworks deploys: the root chart's settings for a subchart are applied over the subchart's own defaults. +Every value has its own link (the `#` next to its key), and each section has its defaults as a values file under "Defaults as YAML". + + + +_The values tables are generated from the Hopsworks Helm chart during the documentation build._ +_To preview them locally, run `uv run --extra cli hopsworks-docs gen-helm-values --chart `, and `uv run hopsworks-docs reset-helm-values` before committing._ + + diff --git a/docs/setup_installation/common/helm_chart_values/judge.md b/docs/setup_installation/common/helm_chart_values/judge.md new file mode 100644 index 0000000000..d494649d12 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/judge.md @@ -0,0 +1,9 @@ +# Judge values { #helm-values-judge } + +Values under `judge` configure Judge, the active cluster arbitrator service. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/kafka.md b/docs/setup_installation/common/helm_chart_values/kafka.md new file mode 100644 index 0000000000..631a89ee79 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/kafka.md @@ -0,0 +1,9 @@ +# Kafka values { #helm-values-kafka } + +Values under `kafka` configure Kafka, run by the Strimzi operator, which carries feature data on its way to the online feature store. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/kserve.md b/docs/setup_installation/common/helm_chart_values/kserve.md new file mode 100644 index 0000000000..f70b666610 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/kserve.md @@ -0,0 +1,9 @@ +# KServe values { #helm-values-kserve } + +Values under `kserve` configure KServe and Knative Serving, which run model deployments. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/minio.md b/docs/setup_installation/common/helm_chart_values/minio.md new file mode 100644 index 0000000000..364d137ec6 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/minio.md @@ -0,0 +1,9 @@ +# MinIO values { #helm-values-minio } + +Values under `minio` configure MinIO, an S3-compatible object store deployed inside the cluster. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/olk.md b/docs/setup_installation/common/helm_chart_values/olk.md new file mode 100644 index 0000000000..a84352fada --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/olk.md @@ -0,0 +1,9 @@ +# OpenSearch values { #helm-values-olk } + +Values under `olk` configure OpenSearch, OpenSearch Dashboards, Logstash and Filebeat, which provide search, the vector index and service logs. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/onlinefs.md b/docs/setup_installation/common/helm_chart_values/onlinefs.md new file mode 100644 index 0000000000..7b912050d1 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/onlinefs.md @@ -0,0 +1,9 @@ +# OnlineFS values { #helm-values-onlinefs } + +Values under `onlinefs` configure OnlineFS, which consumes feature rows from Kafka and writes them to RonDB. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/prometheus.md b/docs/setup_installation/common/helm_chart_values/prometheus.md new file mode 100644 index 0000000000..46a01b6ea1 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/prometheus.md @@ -0,0 +1,9 @@ +# Prometheus values { #helm-values-prometheus } + +Values under `prometheus` configure Prometheus, which collects cluster and service metrics, and the Prometheus adapter, which exposes them to autoscalers. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/ray.md b/docs/setup_installation/common/helm_chart_values/ray.md new file mode 100644 index 0000000000..1abbe93ad0 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/ray.md @@ -0,0 +1,9 @@ +# Ray values { #helm-values-ray } + +Values under `ray` configure the KubeRay operator, which runs the Ray clusters behind Ray jobs and notebooks. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/rondb.md b/docs/setup_installation/common/helm_chart_values/rondb.md new file mode 100644 index 0000000000..c89afbd298 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/rondb.md @@ -0,0 +1,9 @@ +# RonDB values { #helm-values-rondb } + +Values under `rondb` configure RonDB, the online feature store database, installed from the RonDB Helm chart. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/spark.md b/docs/setup_installation/common/helm_chart_values/spark.md new file mode 100644 index 0000000000..c0c4580d09 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/spark.md @@ -0,0 +1,9 @@ +# Spark values { #helm-values-spark } + +Values under `spark` configure the Spark operator, the Spark history server and the remote shuffle service. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/superset.md b/docs/setup_installation/common/helm_chart_values/superset.md new file mode 100644 index 0000000000..bfb5aa99eb --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/superset.md @@ -0,0 +1,9 @@ +# Superset values { #helm-values-superset } + +Values under `superset` configure Apache Superset, the BI dashboards over feature store data. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/trino.md b/docs/setup_installation/common/helm_chart_values/trino.md new file mode 100644 index 0000000000..dfc1d8ef90 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/trino.md @@ -0,0 +1,9 @@ +# Trino values { #helm-values-trino } + +Values under `trino` configure Trino, the SQL query engine, and its test coordinator for user catalogs. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/docs/setup_installation/common/helm_chart_values/vpa.md b/docs/setup_installation/common/helm_chart_values/vpa.md new file mode 100644 index 0000000000..ef68e901d1 --- /dev/null +++ b/docs/setup_installation/common/helm_chart_values/vpa.md @@ -0,0 +1,9 @@ +# Vertical Pod Autoscaler values { #helm-values-vpa } + +Values under `vpa` configure the Vertical Pod Autoscaler: its admission controller, recommender and updater. + + + +_The values table is generated from the Hopsworks Helm chart during the documentation build._ + + diff --git a/mkdocs.yml b/mkdocs.yml index 3dc18e8178..7523942de4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -317,7 +317,33 @@ nav: - Service Operations: setup_installation/admin/operationLogs.md - Query Engine (Trino): setup_installation/admin/trino.md - Superset: setup_installation/admin/superset.md - - Helm Chart Values Reference: setup_installation/common/helm_chart_values.md + - Helm Chart Values Reference: + - setup_installation/common/helm_chart_values/index.md + - global: setup_installation/common/helm_chart_values/global.md + - airflow: setup_installation/common/helm_chart_values/airflow.md + - arrowflight: setup_installation/common/helm_chart_values/arrowflight.md + - certs-operator: setup_installation/common/helm_chart_values/certs-operator.md + - consul: setup_installation/common/helm_chart_values/consul.md + - docker-registry: setup_installation/common/helm_chart_values/docker-registry.md + - grafana: setup_installation/common/helm_chart_values/grafana.md + - hive: setup_installation/common/helm_chart_values/hive.md + - hopsfs: setup_installation/common/helm_chart_values/hopsfs.md + - hopsworks: setup_installation/common/helm_chart_values/hopsworks.md + - hw-kueue: setup_installation/common/helm_chart_values/hw-kueue.md + - hw-kyverno: setup_installation/common/helm_chart_values/hw-kyverno.md + - judge: setup_installation/common/helm_chart_values/judge.md + - kafka: setup_installation/common/helm_chart_values/kafka.md + - kserve: setup_installation/common/helm_chart_values/kserve.md + - minio: setup_installation/common/helm_chart_values/minio.md + - olk: setup_installation/common/helm_chart_values/olk.md + - onlinefs: setup_installation/common/helm_chart_values/onlinefs.md + - prometheus: setup_installation/common/helm_chart_values/prometheus.md + - ray: setup_installation/common/helm_chart_values/ray.md + - rondb: setup_installation/common/helm_chart_values/rondb.md + - spark: setup_installation/common/helm_chart_values/spark.md + - superset: setup_installation/common/helm_chart_values/superset.md + - trino: setup_installation/common/helm_chart_values/trino.md + - vpa: setup_installation/common/helm_chart_values/vpa.md - Python API: python-api - Java API: javadoc - Reference: @@ -391,6 +417,7 @@ extra_javascript: - js/hops-viz.js - js/hops-viz-tooltip.js - js/prov-hover.js + - js/values-filter.js # Machine-readable artifacts for AI agents: llms.txt, llms-full.txt and a raw # Markdown sibling (.md) next to every rendered page. See scripts/llms_hook.py. diff --git a/scripts/__init__.py b/scripts/__init__.py index 1974918c7a..a2740b4b43 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -2,7 +2,7 @@ from .check import check from .gen_config_vars import gen_config_vars -from .helm_values import gen_helm_values +from .helm_values import gen_helm_values, reset_helm_values from .linkchecker import linkchecker from .markdownlint import markdownlint from .serve import serve @@ -17,4 +17,5 @@ cli.command()(linkchecker) cli.command()(snakeoil) cli.command()(gen_helm_values) +cli.command()(reset_helm_values) cli.command()(gen_config_vars) diff --git a/scripts/helm_values.py b/scripts/helm_values.py index 5ec7c99188..e12ff02c27 100644 --- a/scripts/helm_values.py +++ b/scripts/helm_values.py @@ -1,9 +1,13 @@ import base64 import io +import json import re import tarfile +import tempfile +import textwrap import urllib.error import urllib.request +from dataclasses import dataclass, replace from pathlib import Path from typing import Annotated @@ -13,12 +17,102 @@ from packaging.version import InvalidVersion, Version -_DEFAULT_PAGE = ( - _DOCS_ROOT / "docs" / "setup_installation" / "common" / "helm_chart_values.md" +_DEFAULT_PAGES_DIR = ( + _DOCS_ROOT / "docs" / "setup_installation" / "common" / "helm_chart_values" ) +_INDEX = "index.md" +_GLOBAL = "global" _VALUES_HEADING = "\n## Values\n" _BEGIN = "" _END = "" +# What the committed pages hold between the markers (see reset_helm_values). +_PLACEHOLDER = "_The values table is generated from the Hopsworks Helm chart during the documentation build._" +_INDEX_PLACEHOLDER = ( + "_The values tables are generated from the Hopsworks Helm chart during the documentation build._\n" + "_To preview them locally, run `uv run --extra cli hopsworks-docs gen-helm-values --chart `," + " and `uv run hopsworks-docs reset-helm-values` before committing._" +) + +# Test-harness settings (loadtest credentials and the like), not deployment +# configuration. +_EXCLUDED_PREFIXES = ("hopsworks.tests.",) +# Longer defaults (whole subchart overrides as one-line JSON) are shown as a +# collapsed YAML block instead. +_MAX_INLINE_DEFAULT = 80 +# Smaller groups of keys are merged into the page's "General" section, so the +# table of contents is not a list of one-row sections. +_MIN_SECTION_ROWS = 5 + +# Docs of the upstream charts the subcharts install, keyed by Helm repository +# URL without its trailing slash. Artifact Hub where the chart is listed, +# otherwise the chart README at its release tag. +_UPSTREAM_DOCS = { + "https://helm.releases.hashicorp.com": "https://artifacthub.io/packages/helm/hashicorp/{name}/{version}", + "https://grafana.github.io/helm-charts": "https://artifacthub.io/packages/helm/grafana/{name}/{version}", + "https://prometheus-community.github.io/helm-charts": "https://artifacthub.io/packages/helm/prometheus-community/{name}/{version}", + "https://strimzi.io/charts": "https://artifacthub.io/packages/helm/strimzi/{name}/{version}", + "https://ray-project.github.io/kuberay-helm": "https://artifacthub.io/packages/helm/kuberay-operator/{name}/{version}", + "https://trinodb.github.io/charts": "https://artifacthub.io/packages/helm/trino/{name}/{version}", + "https://apache.github.io/superset": "https://artifacthub.io/packages/helm/superset/{name}/{version}", + "oci://registry-1.docker.io/bitnamicharts": "https://artifacthub.io/packages/helm/bitnami/{name}/{version}", + "https://kubeflow.github.io/spark-operator": "https://github.com/kubeflow/spark-operator/blob/v{version}/charts/spark-operator-chart/README.md", + "https://repo.hops.works/master/kueue": "https://github.com/kubernetes-sigs/kueue/blob/v{version}/charts/kueue/README.md", + "https://logicalclocks.github.io/rondb-helm": "https://github.com/logicalclocks/rondb-helm/blob/v{version}/values.schema.json", +} +# Upstream charts whose values.schema.json is rendered on the subchart page, +# keyed by subchart. docs.hopsworks.ai/rondb-helm/ shows whichever version was +# published last, not the one this chart pins. +_RENDERED_SCHEMAS = {"rondb": "rondb"} +# Top-level keys listed under "Other values" on the index by design: the +# library chart has a single override knob. +_NO_PAGE = {"hopsworkslib"} +# Values a typical install sets, taken from the example values files of the +# AWS, Azure and GCP setup guides, with what each one decides. +_COMMON_VALUES = ( + ( + "global._hopsworks.cloudProvider", + "The cloud the cluster runs on; HopsFS, Consul and Hopsworks configure themselves from it.", + ), + ( + "global._hopsworks.storageClassName", + "The storage class of every persistent volume.", + ), + ( + "global._hopsworks.imageRegistry", + "The registry the Hopsworks images are pulled from.", + ), + ("global._hopsworks.imagePullSecrets", "The pull secrets for that registry."), + ( + "global._hopsworks.managedDockerRegistery", + "Use the cloud provider's container registry for the images Hopsworks builds for users.", + ), + ( + "global._hopsworks.managedObjectStorage", + "Use a cloud bucket for HopsFS data and for the RonDB and OpenSearch backups.", + ), + ( + "global._hopsworks.minio.enabled", + "Deploy MinIO in the cluster; turn it off when a cloud bucket is used.", + ), + ( + "global._hopsworks.externalLoadBalancers.enabled", + "Expose services through LoadBalancer Services.", + ), + ("hopsworks.ingress.host", "The host name of the Hopsworks UI and API."), + ("hopsworks.ingress.ingressClassName", "The ingress controller that serves it."), + ( + "hopsworks.velero.backup.enabled", + "Back up Kubernetes resources with Velero, which needs its own install.", + ), + ( + "rondb.rondb.clusterSize", + "The size of the RonDB cluster: data replicas, node groups, MySQL and REST API servers.", + ), +) +# Marks a default that is prose rather than a value (see _default_value). +_PROSE = object() +# Marks a schema key without a default, or a key no override mentions. +_ABSENT = object() # Inline code spans and genuine external links are kept verbatim; everything # else has its square brackets escaped (see _neutralize_markdown_refs). The link @@ -31,6 +125,34 @@ _HOLD_OPEN = chr(0xE000) _HOLD_CLOSE = chr(0xE001) _HOLD_RE = re.compile(re.escape(_HOLD_OPEN) + r"(\d+)" + re.escape(_HOLD_CLOSE)) +# The site has no magiclink extension, so a bare URL in a description renders +# as plain text until it is wrapped as a Markdown autolink. +_BARE_URL = re.compile(r"(?`]+") +_LINK_OR_CODE = re.compile(f"({_CODE_SPAN.pattern}|{_EXTERNAL_LINK.pattern})") +_URL_TRAILER = ".,;:!?'\"" + +# A helm-docs values row: | key | type | default | description |. The default +# is matched first as a whole code span because it can contain " | " itself +# (a shell command, a JSON object). +_ROW = re.compile( + r"\| (?P\S+) \| (?P.*?) \| (?P`.*?`|.*?) \| (?P.*?) ?\|" +) +_TABLE_RULE = re.compile(r"\|[-:| ]+\|") +# Keys quote segments that contain dots: annotations."prometheus.io/path". +_KEY_SEGMENT = re.compile(r'"[^"]*"|[^.]+') +_LIST_INDEX = re.compile(r"\[\d+\]$") +# Any list index in a key: [3] in a README key, [] in a schema array-item key. +_LIST_INDEX_ANY = re.compile(r"\[(\d*)\]") + + +@dataclass(frozen=True) +class _Row: + key: str + type_: str + default: str + description: str + # Extra facts for the entry's first line: schema constraints, overrides. + notes: tuple[str, ...] = () def _neutralize_markdown_refs(text: str) -> str: @@ -53,6 +175,7 @@ def _hold(match: re.Match) -> str: text = _CODE_SPAN.sub(_hold, text) text = _EXTERNAL_LINK.sub(_hold, text) text = text.replace("[", "\\[").replace("]", "\\]") + # Un-stash iteratively: a stashed span can itself contain sentinels for # other stashed spans, which a single pass would leave unresolved and leak # the literal private-use characters into the page. An index that was never @@ -70,6 +193,26 @@ def _restore(m: re.Match) -> str: return text +def _wrap_url(match: re.Match) -> str: + url = match.group(0).rstrip(_URL_TRAILER) + while url.endswith(")") and url.count("(") < url.count(")"): + url = url[:-1].rstrip(_URL_TRAILER) + return f"<{url}>{match.group(0)[len(url) :]}" + + +def _autolink(text: str) -> str: + """Wrap bare URLs as ````, leaving code spans and Markdown links alone. + + Trailing punctuation and an unbalanced closing parenthesis stay outside + the link: "(see https://x/y)." links ``https://x/y``. + """ + parts = _LINK_OR_CODE.split(text) + # re.split puts the captured code spans and links at the odd indices. + for i in range(0, len(parts), 2): + parts[i] = _BARE_URL.sub(_wrap_url, parts[i]) + return "".join(parts) + + def _http_get(url: str, username: str, password: str) -> bytes: """GET a URL, optionally with HTTP basic auth. Fail loudly on errors.""" request = urllib.request.Request(url) @@ -94,7 +237,9 @@ def _select_chart(entries: list[dict], chart_version: str | None) -> dict | None newest-published is the latest dev build. """ if not chart_version: - return max(entries, key=lambda e: str(e.get("created", ""))) if entries else None + return ( + max(entries, key=lambda e: str(e.get("created", ""))) if entries else None + ) candidates = [ e @@ -118,13 +263,13 @@ def _parse(entry: dict) -> Version | None: return max(candidates, key=lambda e: str(e.get("created", ""))) -def _readme_from_registry( +def _archive_from_registry( repo_url: str, chart_version: str | None, username: str, password: str -) -> tuple[str, str, str] | None: - """Resolve a chart in a Helm (Nexus) repo and return (readme, version, appVersion). +) -> bytes | None: + """Resolve a chart in a Helm (Nexus) repo and return its package archive. Returns None (and warns) if nothing suitable is found, so the build keeps - the page placeholder instead of failing. + the page placeholders instead of failing. """ base = repo_url.rstrip("/") # `or {}` guards an empty/invalid index.yaml (safe_load returns None) so a @@ -136,7 +281,7 @@ def _readme_from_registry( if chosen is None: typer.echo( f"WARNING: no chart in {base} matches version " - f"'{chart_version or 'any'}'; leaving the page placeholder.", + f"'{chart_version or 'any'}'; leaving the page placeholders.", err=True, ) return None @@ -145,7 +290,7 @@ def _readme_from_registry( if not urls: typer.echo( f"WARNING: chart {chosen.get('version')} has no download URL; " - "leaving the page placeholder.", + "leaving the page placeholders.", err=True, ) return None @@ -153,35 +298,774 @@ def _readme_from_registry( url = urls[0] if not url.startswith(("http://", "https://")): url = f"{base}/{url.lstrip('/')}" - version = str(chosen.get("version", "")) - app_version = str(chosen.get("appVersion", "")) - typer.echo(f"Using chart {version} (appVersion {app_version}) from {base}") - - archive = _http_get(url, username, password) - with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: - # The chart README is the top-level "/README.md" (one slash); - # deeper matches are subchart READMEs. Require a regular file so a - # directory/symlink/special member never reaches extractfile() (which - # would return None) and crash instead of skipping gracefully. - member = next( - ( - m - for m in tar.getmembers() - if m.isfile() - and m.name.count("/") == 1 - and m.name.endswith("/README.md") - ), - None, + typer.echo( + f"Using chart {chosen.get('version')} " + f"(appVersion {chosen.get('appVersion')}) from {base}" + ) + return _http_get(url, username, password) + + +def _load_yaml(path: Path) -> dict: + if not path.is_file(): + return {} + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + + +def _segments(key: str) -> list[str]: + return _KEY_SEGMENT.findall(key) + + +def _parse_rows(values_section: str, prefix: str = "") -> list[_Row]: + """Parse a helm-docs values table, with ``prefix`` put before every key. + + Only table lines are read: a README rendered from helm-docs' default + template carries a footer after the table. Exits with an error if a table + line does not parse: a page silently missing values is worse than a build + failure that names the row. + """ + rows = [] + unparsed = [] + for line in values_section.splitlines(): + line = line.strip() + is_header = line.startswith("| Key |") or _TABLE_RULE.fullmatch(line) + if not line.startswith("|") or is_header: + continue + match = _ROW.fullmatch(line) + if match is None: + unparsed.append(line) + continue + key, type_, default, description = match.groups() + rows.append(_Row(prefix + key, type_, default, description)) + if unparsed: + for line in unparsed: + typer.echo(f"ERROR: unparseable values row: {line}", err=True) + typer.echo( + "ERROR: the chart README values table no longer matches the " + "helm-docs layout this generator reads (| key | type | default | " + "description |).", + err=True, ) - if member is None: - typer.echo( - "WARNING: README.md not found in the chart package; " - "leaving the page placeholder.", - err=True, + raise typer.Exit(1) + return rows + + +def _resolve_ref(node: dict, defs: dict) -> dict: + ref = node.get("$ref", "") + if not ref.startswith("#/$defs/"): + return node + siblings = {k: v for k, v in node.items() if k != "$ref"} + return {**defs[ref.removeprefix("#/$defs/")], **siblings} + + +def _helm_merge(base: dict, override: dict) -> dict: + """Coalesce two layers of Helm values the way Helm does. + + Maps merge key by key, any other value (lists included) replaces the base + one, and a null removes the key; it is kept here as None, which is what + the chart's templates then see. + """ + merged = dict(base) + for key, value in override.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _helm_merge(merged[key], value) + else: + merged[key] = value + return merged + + +def _json_code(value: object) -> str: + return f"`{json.dumps(value, separators=(',', ':'))}`" + + +def _override_note(chart_default: object, chart: str) -> str: + if chart_default is _ABSENT: + return f"set by Hopsworks, the `{chart}` chart has no default" + if len(_json_code(chart_default)) > _MAX_INLINE_DEFAULT: + return f"Hopsworks overrides the `{chart}` chart default" + return ( + f"Hopsworks overrides the `{chart}` chart default {_json_code(chart_default)}" + ) + + +def _constraints(node: dict) -> tuple[str, ...]: + # `required` is left out: it means "present when the parent object is", + # which the defaults already satisfy, so it would read as "you must set it". + notes = [ + f"{word} {_json_code(node[word])}" + for word in ("minimum", "maximum") + if word in node + ] + if "pattern" in node: + notes.append(f"pattern `{node['pattern']}`") + examples = node.get("examples") or ([node["example"]] if "example" in node else []) + if examples: + notes.append(f"example {_json_code(examples[0])}") + return tuple(notes) + + +def _schema_rows( + schema: dict, prefix: str, overrides: dict | None = None, chart: str = "" +) -> list[_Row]: + """Flatten a values.schema.json into rows keyed under ``prefix``. + + ``overrides`` are the values the parent chart sets for this one; their + effect replaces the schema default, and the entry names the ``chart``'s + own default. Overrides under array items are not applied. Array item + properties are listed as ``key[].field``. Only local ``#/$defs/`` + references are resolved. + """ + defs = schema.get("$defs") or {} + rows: list[_Row] = [] + + def _walk(node: dict, path: str, override: object) -> None: + for name, child in (node.get("properties") or {}).items(): + child = _resolve_ref(child, defs) + key = f"{path}.{name}" + type_ = child.get("type", "") + description = " ".join(str(child.get("description", "")).split()) + if "enum" in child: + type_ = "enum" + allowed = ", ".join(f"`{json.dumps(v)}`" for v in child["enum"]) + description = f"{description} One of: {allowed}.".strip() + if isinstance(type_, list): + type_ = "|".join(type_) + value = child.get("default", _ABSENT) + notes: tuple[str, ...] = () + child_override = ( + override.get(name, _ABSENT) if isinstance(override, dict) else _ABSENT ) + # A map override on an object with declared properties lands on + # the child entries instead. + if child_override is not _ABSENT and not ( + child.get("properties") and isinstance(child_override, dict) + ): + effective = child_override + if isinstance(child_override, dict) and isinstance(value, dict): + effective = _helm_merge(value, child_override) + if effective != value: + notes = (_override_note(value, chart),) + value = effective + default = "" if value is _ABSENT else _json_code(value) + notes += _constraints(child) + rows.append(_Row(key, type_, default, description, notes)) + _walk(child, key, child_override) + if isinstance(child.get("items"), dict): + _walk(_resolve_ref(child["items"], defs), f"{key}[]", _ABSENT) + + _walk(schema, prefix, overrides or {}) + return rows + + +def _undeclared_overrides(schema: dict, overrides: dict, prefix: str) -> list[str]: + """Return the override keys the schema does not declare. + + Below a free-form map (an object without declared properties) any key is + accepted. An undeclared key is usually an override the chart has renamed + or dropped, which then silently stops applying. + """ + defs = schema.get("$defs") or {} + found: list[str] = [] + + def _walk(node: dict, override: object, path: str) -> None: + properties = node.get("properties") + if not properties or not isinstance(override, dict): + return + for name, value in override.items(): + if name in properties: + _walk(_resolve_ref(properties[name], defs), value, f"{path}.{name}") + else: + found.append(f"{path}.{name}") + + _walk(schema, overrides, prefix) + return found + + +def _null_override_problems( + override: object, keys: list[str], prefix: str +) -> list[str]: + """Report the keys an override sets to null although keys under them are listed. + + Helm removes such a key with everything below it, while the entries under + it keep showing their chart defaults and "Defaults as YAML" rebuilds them. + """ + found: list[str] = [] + + def _walk(node: object, path: str) -> None: + if not isinstance(node, dict): + return + for name, value in node.items(): + # Keys quote a segment that contains a dot. + child = f'{path}."{name}"' if "." in str(name) else f"{path}.{name}" + if value is not None: + _walk(value, child) + elif any(k.startswith((f"{child}.", f"{child}[")) for k in keys): + found.append( + f"Hopsworks sets {child} to null, which removes the values " + "listed under it, but their entries still show the chart defaults" + ) + + _walk(override, prefix) + return found + + +def _dependency_schema(subchart: Path, name: str, version: str) -> dict | None: + """Return the values.schema.json of a subchart's vendored dependency. + + A packaged chart has the dependency unpacked under ``charts//``; a + local checkout has the ``-.tgz`` archive ``helm dependency + build`` downloads for the pinned version. None when neither is present. + """ + unpacked = subchart / "charts" / name / "values.schema.json" + if unpacked.is_file(): + return json.loads(unpacked.read_text(encoding="utf-8")) + archive = subchart / "charts" / f"{name}-{version}.tgz" + if not archive.is_file(): + return None + with tarfile.open(archive, mode="r:gz") as tar: + try: + member = tar.extractfile(f"{name}/values.schema.json") + except KeyError: return None - extracted = tar.extractfile(member) - return extracted.read().decode("utf-8"), version, app_version + return json.load(member) if member is not None else None + + +class _BlockDumper(yaml.SafeDumper): + """Dump multi-line strings (embedded config files) as ``|`` blocks.""" + + +def _represent_str(dumper: yaml.SafeDumper, value: str) -> yaml.ScalarNode: + style = "|" if "\n" in value else None + return dumper.represent_scalar("tag:yaml.org,2002:str", value, style=style) + + +_BlockDumper.add_representer(str, _represent_str) + + +def _dump_yaml(value: object, sort_keys: bool = False) -> str: + dumped = yaml.dump( + value, Dumper=_BlockDumper, sort_keys=sort_keys, allow_unicode=True, width=1000 + ).rstrip() + # A lone plain scalar is followed by the "..." document end marker. + return dumped.removesuffix("\n...") + + +def _default_value(default: str) -> object: + """Parse a rendered default back into its value. + + Returns ``_PROSE`` for a default that is not JSON, such as a helm-docs + ``@default`` text ("check values.yaml"). + """ + raw = ( + default[1:-1] if default.startswith("`") and default.endswith("`") else default + ) + if raw == "nil": + return None + try: + return json.loads(raw) + except json.JSONDecodeError: + return _PROSE + + +def _default_block(row: _Row) -> str: + value = _default_value(row.default) + # YAML for every parsed value: a string default such as '["/bin/bash", …]' + # keeps its quotes, so it is not copied into a values file as a list. + if value is _PROSE: + lang, body = "text", row.default.strip("`") + else: + lang, body = "yaml", _dump_yaml(value) + fence = textwrap.indent(f"```{lang}\n{body}\n```", " ") + return f'??? note "Default"\n\n{fence}' + + +def _anchor(key: str) -> str: + """Return the HTML id of a key's entry: ``helm.`` plus the key, URL-safe.""" + path = _LIST_INDEX_ANY.sub(lambda m: f".{m.group(1)}" if m.group(1) else "", key) + return "helm." + re.sub(r"[^A-Za-z0-9._-]+", "-", path.replace('"', "")) + + +def _is_deprecated(row: _Row) -> bool: + return re.match(r"\s*deprecated\b", row.description, re.IGNORECASE) is not None + + +def _render_entries(rows: list[_Row]) -> str: + # A definition list, not a table: a code chip in a table cell never wraps + # (design-system.md, "Tables"), and keys alone run to 90 characters. + # .hops-values in docs/css/custom.css gives it the table's density. + entries = [] + for row in sorted(rows, key=lambda r: (_is_deprecated(r), r.key)): + anchor = _anchor(row.key) + term = f"`{row.key}`" + if _is_deprecated(row): + term += ' Deprecated' + term += ( + f' #' + f" {{ #{anchor} }}" + ) + long_default = len(row.default) > _MAX_INLINE_DEFAULT + facts = [f"Type `{row.type_}`"] if row.type_ else [] + if row.default and not long_default: + facts.append(f"default {_neutralize_markdown_refs(row.default)}") + facts += row.notes + lines = [term, f": {', '.join(facts) or 'Value'}."] + if row.description: + description = _autolink(_neutralize_markdown_refs(row.description)) + lines.append(f" {description}") + if long_default: + lines += ["", textwrap.indent(_default_block(row), " ")] + entries.append("\n".join(lines)) + body = "\n\n".join(entries) + return f'
\n\n{body}\n\n
' + + +def _key_path(key: str) -> list[str | int]: + path: list[str | int] = [] + for segment in _segments(key): + path.append(_LIST_INDEX_ANY.sub("", segment).strip('"')) + path += [int(i) for i in _LIST_INDEX_ANY.findall(segment) if i] + return path + + +def _set_path(tree: dict, path: list[str | int], value: object) -> None: + node: object = tree + for step, next_step in zip(path, path[1:]): + if isinstance(node, list) and isinstance(step, int): + node.extend([None] * (step + 1 - len(node))) + current = node[step] + elif isinstance(node, dict): + current = node.get(step) + else: + return + # A parent's scalar or null default gives way to a listed child. + if not isinstance(current, (dict, list)): + current = {} if isinstance(next_step, str) else [] + node[step] = current + node = current + last = path[-1] + if isinstance(node, list) and isinstance(last, int): + node.extend([None] * (last + 1 - len(node))) + node[last] = value + elif isinstance(node, dict): + node[last] = value + + +def _values_file_block(rows: list[_Row]) -> str | None: + """Return the rows' defaults nested as in a values file, as a collapsed block. + + Parents are set before their children, so a listed child refines an + object default and the keys it does not list stay. Array item fields + (``key[].field``) and prose defaults are left out. None when nothing is + left. + """ + tree: dict = {} + for row in sorted(rows, key=lambda r: len(_key_path(r.key))): + if "[]" in row.key: + continue + value = _default_value(row.default) + if value is not _PROSE: + _set_path(tree, _key_path(row.key), value) + if not tree: + return None + # Sorted, because the insertion order is by depth. + body = _dump_yaml(tree, sort_keys=True) + fence = textwrap.indent(f"```yaml\n{body}\n```", " ") + return f'??? example "Defaults as YAML"\n\n{fence}' + + +def _section(key: str, depth: int) -> str | None: + """Name the group a key belongs to: its first segment below the page. + + ``depth`` is the number of leading segments shared by every key on the + page; None for a key that is the page's own prefix. Underscore namespaces + (``global._hopsworks``) are skipped, and list items + (``wipeWhenUninstall[3]``) group under their list. An object key and its + children share a group. + """ + rest = _segments(key)[depth:] + if len(rest) > 1 and rest[0].startswith("_"): + rest = rest[1:] + return _LIST_INDEX.sub("", rest[0]) if rest else None + + +def _heading(level: str, title: str, anchor: str) -> str: + # Explicit ids, because mkdocs-autorefs resolves [text][id] site-wide: a + # generated "## terminal" would make the Terminal guide's anchor ambiguous. + return f"{level} {title} {{ #{_slug(anchor)} }}" + + +def _slug(anchor: str) -> str: + return re.sub(r"[^a-z0-9_-]+", "-", anchor.lower()).strip("-") + + +def _section_body(rows: list[_Row]) -> str: + block = _values_file_block(rows) + entries = _render_entries(rows) + return f"{block}\n\n{entries}" if block else entries + + +def _render_rows(rows: list[_Row], depth: int, level: str, anchor: str) -> str: + groups: dict[str | None, list[_Row]] = {} + for row in rows: + groups.setdefault(_section(row.key, depth), []).append(row) + sections = sorted( + (name, group) + for name, group in groups.items() + if name is not None and len(group) >= _MIN_SECTION_ROWS + ) + named = {name for name, _ in sections} + general = [row for row in rows if _section(row.key, depth) not in named] + if not sections: + return _section_body(general) + parts = [] + if general: + title = _heading(level, "General", f"{anchor}-general") + parts.append(f"{title}\n\n{_section_body(general)}") + for name, group in sections: + title = _heading(level, name, f"{anchor}-{name}") + parts.append(f"{title}\n\n{_section_body(group)}") + return "\n\n".join(parts) + + +def _key_link(key: str, targets: dict[str, str]) -> str: + return f"[`{key}`]({targets[key]})" if key in targets else f"`{key}`" + + +def _condition_text(condition: str | None, targets: dict[str, str]) -> str: + keys = [p.strip() for p in (condition or "").split(",") if p.strip()] + paths = [_key_link(key, targets) for key in keys] + if not paths: + return "Always deployed." + if len(paths) == 1: + return f"Deployed when {paths[0]} is `true`." + return ( + "Deployed according to the first of these values that is set: " + f"{', '.join(paths)}." + ) + + +def _upstream_link(dependency: dict, problems: list[str]) -> str: + name, version = dependency["name"], str(dependency["version"]) + repository = str(dependency.get("repository", "")).rstrip("/") + label = f"`{name}` {version}" + template = _UPSTREAM_DOCS.get(repository) + if template is None: + problems.append(f"no docs link for chart {name} from {repository}") + return label + return f"[{label}]({template.format(name=name, version=version)})" + + +def _upstream_admonition( + charts: list[tuple[str, dict, str]], listed: dict[str, str] +) -> str: + """Name the key, docs and repository of each upstream chart a page installs. + + ``charts`` holds (values key, dependency, docs link); ``listed`` maps the + values key of a chart whose own values the page renders to a link there. + """ + lines = [] + unlisted = [] + for prefix, dep, link in charts: + line = f"- Values under `{prefix}` go to {link} from `{dep.get('repository')}`" + if prefix in listed: + line += f", and all of them are listed under {listed[prefix]}" + else: + unlisted.append(f"`{prefix}`") + lines.append(f"{line}.") + if unlisted: + keys = unlisted[0] + if len(unlisted) > 1: + keys = f"{', '.join(unlisted[:-1])} and {unlisted[-1]}" + lines += [ + "", + f"Only the values Hopsworks sets under {keys} are listed on this page.", + ] + if len(unlisted) == 1: + lines.append( + "Any other value of the chart can be set there too; the link opens " + "its documentation for the version Hopsworks pins." + ) + else: + lines.append( + "Any other value of the charts can be set under the same keys; each " + "link opens the chart's documentation for the version Hopsworks pins." + ) + body = textwrap.indent("\n".join(lines), " ") + return f'!!! info "Upstream charts"\n\n{body}' + + +def _upstream_note(link: str, listed: str | None) -> str: + if listed: + return f"passed to the {link} chart, whose values are listed under {listed}" + return f"passed to the {link} chart, whose other values are documented there" + + +def _with_deployed_default(row: _Row, deployed: dict) -> _Row: + path = [segment.strip('"') for segment in _segments(row.key)[1:]] + if not path or any("[" in segment for segment in path): + return row + node: object = deployed + for segment in path: + if not isinstance(node, dict) or segment not in node: + return row + node = node[segment] + shown = _default_value(row.default) + if shown is _PROSE or shown == node: + return row + # helm-docs writes null as nil. + return replace(row, default="`nil`" if node is None else _json_code(node)) + + +def _deployed_rows( + rows: list[_Row], subchart: Path, root_override: object +) -> list[_Row]: + """Show what Hopsworks deploys for a subchart's values. + + The README rows carry the subchart's own defaults; the root chart's + values for it are merged over them the way Helm merges them. A + values.yaml PyYAML cannot read (Helm's parser is more lenient) leaves + the rows as they are, with a warning. + """ + if not isinstance(root_override, dict) or not root_override: + return rows + try: + values = _load_yaml(subchart / "values.yaml") + except yaml.YAMLError as exc: + reason = getattr(exc, "problem", None) or exc + typer.echo( + f"WARNING: cannot parse {subchart / 'values.yaml'} ({reason}); " + "showing its own defaults without the root chart's overrides", + err=True, + ) + return rows + deployed = _helm_merge(values, root_override) + return [_with_deployed_default(row, deployed) for row in rows] + + +def _schema_section(prefix: str, name: str, link: str, rows: list[_Row]) -> str: + key = prefix.split(".")[0] + intro = ( + f"These are the values of the {link} chart, set under `{prefix}`.\n" + "The defaults are what Hopsworks deploys: the chart's own, with the " + f"`{key}` and `{prefix}` overrides above applied.\n" + "Where Hopsworks overrides a value, the entry also gives the chart's own default." + ) + anchor = f"helm-values-{prefix}" + title = _heading("##", f"`{name}` chart values", anchor) + return f"{title}\n\n{intro}\n\n{_render_rows(rows, 2, '###', anchor)}" + + +def _common_values(targets: dict[str, str], problems: list[str]) -> str: + lines = ["| Value | What it sets |", "| --- | --- |"] + for key, purpose in _COMMON_VALUES: + if key not in targets: + problems.append(f"common value {key} is not in the chart") + continue + lines.append(f"| {_key_link(key, targets)} | {purpose} |") + title = _heading("##", "Common values", "helm-values-common") + return f"{title}\n\n" + "\n".join(lines) + + +def _inject(page: Path, body: str) -> None: + content = page.read_text(encoding="utf-8") + if _BEGIN not in content or _END not in content: + msg = f"Injection markers ({_BEGIN} / {_END}) not found in {page}" + raise typer.BadParameter(msg) + head = content[: content.index(_BEGIN) + len(_BEGIN)] + tail = content[content.index(_END) :] + page.write_text(f"{head}\n\n{body}\n\n{tail}", encoding="utf-8") + + +@dataclass +class _Page: + stub: Path + rows: list[_Row] + upstream: list[dict] + # (prefix, dependency, rows) of each upstream chart rendered from its schema. + schemas: list[tuple[str, dict, list[_Row]]] + + +def _generate(chart: Path, pages_dir: Path, strict: bool) -> None: + readme = (chart / "README.md").read_text(encoding="utf-8") + if _VALUES_HEADING not in readme: + typer.echo( + "WARNING: '## Values' section not found in the chart README " + "(older chart versions predate it); leaving the page placeholders.", + err=True, + ) + return + meta = _load_yaml(chart / "Chart.yaml") + root_values = _load_yaml(chart / "values.yaml") + version, app_version = meta.get("version", ""), meta.get("appVersion", "") + note = f"_Generated from the Hopsworks Helm chart `{version}`" + note += f" (Hopsworks `{app_version}`)._" if app_version else "._" + conditions = { + dep["name"]: dep.get("condition") for dep in meta.get("dependencies") or [] + } + problems: list[str] = [] + + # A root README rendered without `helm-docs -u` (hopsworks-helm#2469) has + # only the root and global values; each subchart's README has its own, + # keyed from the subchart. With `-u` the root table repeats them all. The + # first row of a key wins, so a key the root chart overrides keeps the + # root's row in both layouts (`-u` lists it twice). The '## Values' + # section is the last one in a README, so take everything after its heading. + sources = [(readme, "")] + for subchart_readme in sorted((chart / "charts").glob("*/README.md")): + prefix = f"{subchart_readme.parent.name}." + sources.append((subchart_readme.read_text(encoding="utf-8"), prefix)) + rows: list[_Row] = [] + known: set[str] = set() + for text, prefix in sources: + if _VALUES_HEADING not in text: + continue + for row in _parse_rows(text.split(_VALUES_HEADING, 1)[1], prefix): + if row.key not in known: + rows.append(row) + known.add(row.key) + by_key: dict[str, list[_Row]] = {} + for row in rows: + if not row.key.startswith(_EXCLUDED_PREFIXES): + by_key.setdefault(_segments(row.key)[0], []).append(row) + + # First pass: which page every key lands on, so pages can link each other. + pages: dict[str, _Page] = {} + stubs = sorted( + (p for p in pages_dir.glob("*.md") if p.name != _INDEX), + key=lambda p: (p.stem != _GLOBAL, p.stem), + ) + for stub in stubs: + key = stub.stem + subchart = chart / "charts" / key + upstream = [ + dep + for dep in _load_yaml(subchart / "Chart.yaml").get("dependencies") or [] + if not str(dep.get("repository", "")).startswith("file://") + ] + rows = by_key.pop(key, []) + root_override = root_values.get(key) + problems += _null_override_problems( + root_override, [row.key for row in rows], key + ) + rows = _deployed_rows(rows, subchart, root_override) + page = _Page(stub, rows, upstream, []) + for dep in upstream: + if _RENDERED_SCHEMAS.get(key) != dep["name"]: + continue + schema = _dependency_schema(subchart, dep["name"], str(dep["version"])) + if schema is None: + problems.append( + f"no values.schema.json for {dep['name']} under {subchart}; " + "run `helm dependency build` there to include it" + ) + continue + dep_key = dep.get("alias") or dep["name"] + prefix = f"{key}.{dep_key}" + # What Hopsworks deploys: the dependency's defaults, then the + # subchart's values for it, then the root chart's. + overrides = _helm_merge( + _load_yaml(subchart / "values.yaml").get(dep_key) or {}, + (root_values.get(key) or {}).get(dep_key) or {}, + ) + problems += [ + f"Hopsworks overrides {path}, which the {dep['name']} " + f"{dep['version']} schema does not declare" + for path in _undeclared_overrides(schema, overrides, prefix) + ] + rows = _schema_rows(schema, prefix, overrides, dep["name"]) + problems += _null_override_problems( + overrides, [row.key for row in rows], prefix + ) + page.schemas.append((prefix, dep, rows)) + pages[key] = page + leftover = [row for rows in by_key.values() for row in rows] + unplaced = sorted(set(by_key) - _NO_PAGE) + if unplaced: + problems.append( + f"no page for top-level keys {', '.join(unplaced)}; add a stub with " + "the generation markers and a nav entry" + ) + targets = {row.key: f"{_INDEX}#{_anchor(row.key)}" for row in leftover} + for page in pages.values(): + by_anchor: dict[str, list[str]] = {} + for row in page.rows + [r for _, _, rows in page.schemas for r in rows]: + by_anchor.setdefault(_anchor(row.key), []).append(row.key) + targets[row.key] = f"{page.stub.name}#{_anchor(row.key)}" + problems += [ + f"{page.stub.name}: {len(keys)} entries for #{anchor} " + f"({', '.join(dict.fromkeys(keys))}); either a README row the rendered " + "schema also lists (a helm-docs comment on a key under it), or keys " + "an anchor cannot tell apart" + for anchor, keys in by_anchor.items() + if len(keys) > 1 + ] + + index_lines = ["| Values | Upstream charts | Keys |", "| --- | --- | --- |"] + for key, page in pages.items(): + links = [_upstream_link(dep, problems) for dep in page.upstream] + charts = [ + (f"{key}.{dep.get('alias') or dep['name']}", dep, link) + for dep, link in zip(page.upstream, links) + ] + listed = { + prefix: f"[`{dep['name']}` chart values](#{_slug(f'helm-values-{prefix}')})" + for prefix, dep, _ in page.schemas + } + # The entry of the key a chart is configured under links its docs too, + # for a reader who lands on it from an anchor below the admonition. + notes = { + prefix: _upstream_note(link, listed.get(prefix)) + for prefix, _, link in charts + } + rows = [ + replace(row, notes=(*row.notes, notes[row.key])) + if row.key in notes + else row + for row in page.rows + ] + parts = [note] + if key in conditions: + parts.append(_condition_text(conditions[key], targets)) + if charts: + parts.append(_upstream_admonition(charts, listed)) + if rows: + parts.append(_render_rows(rows, 1, "##", f"helm-values-{key}")) + else: + parts.append(f"Chart `{version}` has no `{key}` values.") + count = len(page.rows) + by_name = dict(zip((dep["name"] for dep in page.upstream), links)) + for prefix, dep, rows in page.schemas: + parts.append( + _schema_section(prefix, dep["name"], by_name[dep["name"]], rows) + ) + count += len(rows) + _inject(page.stub, "\n\n".join(parts)) + + # An aliased dependency (trino and trinotest) installs the same chart twice. + upstream_cell = ", ".join(dict.fromkeys(links)) + index_lines.append( + f"| [`{key}`][helm-values-{key}] | {upstream_cell} | {count} |" + ) + + index = [ + note, + _common_values(targets, problems), + _heading("##", "All values", "helm-values-pages") + + "\n\n" + + "\n".join(index_lines), + ] + if leftover: + title = _heading("##", "Other values", "helm-values-other") + index.append(f"{title}\n\n{_section_body(leftover)}") + _inject(pages_dir / _INDEX, "\n\n".join(index)) + typer.echo( + f"Injected chart {version} values into {len(stubs)} pages " + f"({len(leftover)} rows without a page: {', '.join(sorted(by_key)) or 'none'})" + ) + for problem in problems: + typer.echo(f"WARNING: {problem}", err=True) + if strict and problems: + typer.echo("ERROR: --strict and the warnings above were raised.", err=True) + raise typer.Exit(1) def gen_helm_values( @@ -209,54 +1093,70 @@ def gen_helm_values( str, typer.Option(envvar="NEXUS_PASSWORD", help="Nexus password (private repos)."), ] = "", - page: Annotated[ + pages_dir: Annotated[ Path, - typer.Option(help="Reference page to inject the values table into."), - ] = _DEFAULT_PAGE, + typer.Option(help="Folder of the reference pages to inject the values into."), + ] = _DEFAULT_PAGES_DIR, + strict: Annotated[ + bool, + typer.Option( + help="Fail when a top-level key has no page, an upstream chart has " + "no docs link, a common value is missing, a rendered schema is " + "absent, an override names a key that schema does not declare, " + "two entries on a page share an anchor, or an override sets a key " + "to null with values listed under it (the PR check)." + ), + ] = False, ) -> None: - """Inject the Helm chart README values table into the reference page. - - Source the chart README either from a local checkout (``--chart``) or, for - CI, from the published chart package in a Nexus Helm repo (``--repo-url``, - optionally ``--chart-version`` for release matching). Slices the ``## Values`` - table out of that README and writes it, with a line recording the chart - version, between the generation markers in ``page``. The result is consumed - by the documentation build and is not committed. If no matching chart or no - ``## Values`` section is found, the page placeholder is left in place rather - than failing the build. + """Inject the Helm chart values into the reference pages, one per top-level key. + + Source the chart either from a local checkout (``--chart``) or, for CI, + from the published chart package in a Nexus Helm repo (``--repo-url``, + optionally ``--chart-version`` for release matching). Each ``.md`` + stub in ``pages_dir`` receives the rows of the chart README ``## Values`` + table under that key, the subchart's deployment condition and links to the + upstream charts it installs; ``index.md`` receives the common values, the + overview table and any rows whose key has no stub. The result is consumed + by the documentation build and is not committed. If no matching chart or + no ``## Values`` section is found, the page placeholders are left in place + rather than failing the build; a values row that does not parse always + fails it. """ if chart: - readme: str = (chart / "README.md").read_text(encoding="utf-8") - meta = yaml.safe_load((chart / "Chart.yaml").read_text(encoding="utf-8")) or {} - version = str(meta.get("version", "")) - app_version = str(meta.get("appVersion", "")) - elif repo_url: - resolved = _readme_from_registry(repo_url, chart_version, username, password) - if resolved is None: - return - readme, version, app_version = resolved - else: + _generate(chart, pages_dir, strict) + return + if not repo_url: raise typer.BadParameter("provide either --chart or --repo-url ") - - if _VALUES_HEADING not in readme: - typer.echo( - "WARNING: '## Values' section not found in the chart README " - "(older chart versions predate it); leaving the page placeholder.", - err=True, - ) + archive = _archive_from_registry(repo_url, chart_version, username, password) + if archive is None: return - # The '## Values' section is the last one in the README, so take everything - # after its heading to EOF -- that is the full Key/Type/Default/Description table. - table = _neutralize_markdown_refs(readme.split(_VALUES_HEADING, 1)[1].strip()) + with tempfile.TemporaryDirectory() as tmp: + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: + tar.extractall(tmp, filter="data") + charts = [p.parent for p in Path(tmp).glob("*/Chart.yaml")] + if len(charts) != 1: + typer.echo( + "WARNING: expected one chart at the top of the package; " + "leaving the page placeholders.", + err=True, + ) + return + _generate(charts[0], pages_dir, strict) - note = f"_Generated from the Hopsworks Helm chart `{version}`" - note += f" (Hopsworks `{app_version}`)._" if app_version else "._" - content = page.read_text() - if _BEGIN not in content or _END not in content: - msg = f"Injection markers ({_BEGIN} / {_END}) not found in {page}" - raise typer.BadParameter(msg) - head = content[: content.index(_BEGIN) + len(_BEGIN)] - tail = content[content.index(_END) :] - page.write_text(f"{head}\n\n{note}\n\n{table}\n\n{tail}") - typer.echo(f"Injected chart {version} values ({table.count(chr(10)) + 1} lines)") +def reset_helm_values( + pages_dir: Annotated[ + Path, + typer.Option(help="Folder of the reference pages to reset."), + ] = _DEFAULT_PAGES_DIR, +) -> None: + """Put the committed placeholder back between the markers of every values page. + + Undoes ``gen-helm-values`` in a working copy, since the generated values + are never committed; text outside the markers is kept. The PR check runs + this and fails when it changes a committed page. + """ + pages = sorted(pages_dir.glob("*.md")) + for page in pages: + _inject(page, _INDEX_PLACEHOLDER if page.name == _INDEX else _PLACEHOLDER) + typer.echo(f"Reset {len(pages)} pages in {pages_dir} to their placeholders")