diff --git a/MODULE.bazel b/MODULE.bazel index e06b31e..a0f4c46 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -28,6 +28,7 @@ bazel_dep(name = "score_bazel_platforms", version = "1.1.0") # C++ Rules for Bazel # ******************************************************************************* bazel_dep(name = "rules_cc", version = "0.2.17") +bazel_dep(name = "toolchains_llvm", version = "1.8.0") # ******************************************************************************* # Needed by S-CORE CI Automatic checks diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index fc6d40d..6400c8a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -63,7 +63,9 @@ "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", - "https://bcr.bazel.build/modules/bazel_features/1.32.0/source.json": "2546c766986a6541f0bacd3e8542a1f621e2b14a80ea9e88c6f89f7eedf64ae1", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/source.json": "31ba776c122b54a2885e23651642e32f087a87bf025465f8040751894b571277", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", @@ -108,6 +110,8 @@ "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/source.json": "41e9e129f80d8c8bf103a7acc337b76e54fad1214ac0a7084bf24f4cd924b8b4", "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", + "https://bcr.bazel.build/modules/helly25_bzl/0.4.3/MODULE.bazel": "9c20052fd3f1fb767c48b78c1bbc46f501a4ca69d14558429d1234e68e449f68", + "https://bcr.bazel.build/modules/helly25_bzl/0.4.3/source.json": "e8c54d81e72633fb6f1d23c7e2d44e0b39b1caef2a256141136a2f63e6204b78", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.6.0/MODULE.bazel": "26ec5118e66a55fef36f8ea39d6415d55fc966671fe4d61a6b9ef0cd6bc7b6a1", "https://bcr.bazel.build/modules/jq.bzl/0.6.0/source.json": "2ed4e35b9fa9505495784114f7637fbde5846ec14af917841b6c045d877f20eb", @@ -265,9 +269,12 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", - "https://bcr.bazel.build/modules/tar.bzl/0.5.1/source.json": "deed3094f7cc779ed1d37a68403847b0e38d9dd9d931e03cb90825f3368b515f", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", "https://bcr.bazel.build/modules/toolchain_utils/1.0.2/MODULE.bazel": "9b8be503a4fcfd3b8b952525bff0869177a5234d5c35dc3e566b9f5ca2f755a1", "https://bcr.bazel.build/modules/toolchain_utils/1.0.2/source.json": "88769ec576dddacafd8cca4631812cf8eead89f10a29d9405d9f7a553de6bf87", + "https://bcr.bazel.build/modules/toolchains_llvm/1.8.0/MODULE.bazel": "68259b66e5fb84f94fa37125ad476c3eb8edf602a34bf58377cde9a16dd8fa98", + "https://bcr.bazel.build/modules/toolchains_llvm/1.8.0/source.json": "70d80fe5b626a3fadf812af41dda4a028640a1184f5a3c7e6cb20130d93ed783", "https://bcr.bazel.build/modules/toolchains_protoc/0.2.1/MODULE.bazel": "2f08433ff5e659069b3a1abfee2377d68f510f2de1da50678ed992c455b4ff91", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", @@ -331,6 +338,8 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.28.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.30.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.32.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.34.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.38.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.4.1/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.9.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.9.1/MODULE.bazel": "not found", @@ -367,6 +376,7 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/googletest/1.11.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/googletest/1.14.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/helly25_bzl/0.4.3/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/jq.bzl/0.1.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/jq.bzl/0.6.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", @@ -536,7 +546,9 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.7.2/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/tar.bzl/0.2.1/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/tar.bzl/0.5.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/tar.bzl/0.6.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/toolchain_utils/1.0.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/toolchains_llvm/1.8.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/toolchains_protoc/0.2.1/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/yq.bzl/0.1.1/MODULE.bazel": "not found", @@ -549,7 +561,7 @@ "moduleExtensions": { "@@aspect_rules_js+//npm:extensions.bzl%pnpm": { "general": { - "bzlTransitiveDigest": "mBV5IQWX7FWfZbz6MJP8c7oawhjJCmyPytourZpcJ94=", + "bzlTransitiveDigest": "34Ox+Fqeuo9xqYojBQmIrY2Op9OjLCSi+URvEpxPck4=", "usagesDigest": "ZYGEy1FrDUNPBzAzD+ujlHkMEsVPMYOvpHm9RhUexUE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1359,7 +1371,7 @@ }, "@@rules_go+//go:extensions.bzl%go_sdk": { "os:linux,arch:amd64": { - "bzlTransitiveDigest": "u1YFz8yPohRPLGnZHit0dAReaKE3BBbOpz7tCIRjfxA=", + "bzlTransitiveDigest": "6+xDMUxUNCx/1D7ofV87ePScYp4E80x0ngSOjdU4Ly4=", "usagesDigest": "igIBXyqNg9Be63Cuu6kZxOeoDRDMqxSv8BcoWiqSh3w=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1437,7 +1449,7 @@ }, "@@rules_go+//go/private:extensions.bzl%non_module_dependencies": { "general": { - "bzlTransitiveDigest": "8bSdTweW+N9XGz2t2QdL82y/0gkMFjf/br4GkckQ/q8=", + "bzlTransitiveDigest": "a7/TN3GFVwbPA9T5+EJlnVWjnyV2Ex2PDVS0ih/2EDs=", "usagesDigest": "JL1XfI3Chd9vaoVGnIlNXDdHjWcf9WYe/Lu+A1HCC74=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1890,7 +1902,7 @@ }, "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { "general": { - "bzlTransitiveDigest": "odt15EU5oQaufFXvA/pJeimD2DG8szS3Yvg8IcgusVs=", + "bzlTransitiveDigest": "v9uEPM3yk6Yg/gZv37OhmzrWEqm2M3nClroI5RmdA+8=", "usagesDigest": "dQ7SQZ7uSSL3vVKSMBKRxKJUm9OVrZZA+S1/QQfT570=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -2040,54 +2052,23 @@ ] } }, - "@@tar.bzl+//tar:extensions.bzl%toolchains": { + "@@toolchains_llvm+//toolchain/extensions:distributions.bzl%llvm_distributions": { "general": { - "bzlTransitiveDigest": "/2afh6fPjq/rcyE/jztQDK3ierehmFFngfvmqyRv72M=", - "usagesDigest": "maF8qsAIqeH1ey8pxP0gNZbvJt34kLZvTFeQ0ntrJVA=", + "bzlTransitiveDigest": "gCdXpBt3HBBc280OILOFheHmO7lXWXJYnPgYiTtyapI=", + "usagesDigest": "VyKGZSw79ryPJ9gt0sqBfIkA7qGOU6tnlDHd7awkb6Q=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "bsd_tar_toolchains": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:toolchain.bzl%tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar_toolchains" - } - }, - "bsd_tar_toolchains_darwin_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_toolchains_darwin_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", + "llvm_distributions_data": { + "repoRuleId": "@@toolchains_llvm+//toolchain/internal:distributions_repo.bzl%llvm_distributions_repo", "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_toolchains_linux_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_toolchains_linux_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_toolchains_windows_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "bsd_tar_toolchains_windows_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_arm64" + "srcs": [ + "@@toolchains_llvm+//toolchain/distributions:pre_github.jsonc", + "@@toolchains_llvm+//toolchain/distributions:github_legacy.jsonc", + "@@toolchains_llvm+//toolchain/distributions:github.jsonc", + "@@toolchains_llvm+//toolchain/distributions:extra.jsonc" + ] } } }, diff --git a/README.md b/README.md index 7022bc4..e97ae21 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ validates the setup, and how QNX-specific authentication and licensing fit in. - [Overview](docs/overview.md) - [Repository layout](docs/repository_layout.md) - [Extension API](docs/extension_api.md) +- [Fast LLVM extraction (opt-in)](docs/fast_llvm_repo.md) - [Generation flow](docs/generation_flow.md) - [Toolchain features](docs/features.md) - [Migration guide (explicit-feature model)](docs/migration_guide.md) diff --git a/docs/extension_api.md b/docs/extension_api.md index 88c9675..3386e71 100644 --- a/docs/extension_api.md +++ b/docs/extension_api.md @@ -182,4 +182,58 @@ behavior explicitly, see the `gcc.toolchain(...)`. See [Toolchain features](features.md#sanitizers-linux-opt-in) for the injected feature names. - QNX toolchains use additional licensing and include-path parameters that do - not apply to Linux toolchains. \ No newline at end of file + not apply to Linux toolchains. + +## LLVM Extension (Optional, Faster Alternative) + +The `llvm` module extension in +`@score_bazel_cpp_toolchains//extensions:llvm.bzl` is an optional, opt-in +alternative for consumers who already use (or plan to use) +[`toolchains_llvm`](https://github.com/bazel-contrib/toolchains_llvm) and want +faster LLVM archive extraction. It is not a replacement for `toolchains_llvm` +in general — it only swaps the repository that provides the LLVM +distribution. See [Fast LLVM repository](fast_llvm_repo.md) for the +rationale and measured performance numbers (~220s → ~70s). + +### Consumer Entry Point + +```starlark +bazel_dep(name = "score_bazel_cpp_toolchains", version = "0.5.4") + +llvm = use_extension("@score_bazel_cpp_toolchains//extensions:llvm.bzl", "llvm") +llvm.toolchain( + name = "llvm_toolchain", + version = "19.1.1", +) + +use_repo(llvm, "llvm_toolchain", "llvm_toolchain_pkg") +``` + +Consumers do not need their own `bazel_dep(toolchains_llvm)`: +`score_bazel_cpp_toolchains` already depends on `toolchains_llvm`, and the +extension loads it internally. + +### `llvm.toolchain(...)` Attributes + +- `name` (mandatory): name of the generated LLVM toolchain repository. The + companion package repository (produced by `fast_llvm_repo`) is generated + alongside it as `_pkg`. +- `version` (mandatory): LLVM version to extract. Must be one of the versions + pinned in [`fast_llvm_repo.bzl`](../rules/fast_llvm_repo.bzl). + +### Activation In A Workspace + +```text +--extra_toolchains=@llvm_toolchain//: +``` + +### Behavior Notes + +- The extension is intended for the root module only, the same restriction as + `gcc`. +- It is an alternative, not a replacement, for `toolchains_llvm`. +- Only the versions and architectures pinned in `fast_llvm_repo.bzl` are + supported; requesting any other version fails at package-fetch time. +- Each `llvm.toolchain(...)` tag creates its own package repo. There is no + sharing or deduplication across multiple toolchain tags requesting the same + version — this is intentionally kept simple. \ No newline at end of file diff --git a/docs/fast_llvm_repo.md b/docs/fast_llvm_repo.md new file mode 100644 index 0000000..b562bde --- /dev/null +++ b/docs/fast_llvm_repo.md @@ -0,0 +1,98 @@ + + +# Fast LLVM repository + +## Purpose + +`fast_llvm_repo` creates the prebuilt LLVM distribution repository consumed by +[`toolchains_llvm`](https://github.com/bazel-contrib/toolchains_llvm). It is a +performance-oriented alternative to the repository that `toolchains_llvm` +normally generates internally. + +The rule is useful when LLVM setup is a significant part of the developer or +CI feedback time. In the measurements that motivated it, LLVM setup decreased +from roughly 220 seconds to 70 seconds. The exact result depends on the host; +the rule is not required for correctness. + +`fast_llvm_repo` is an internal repository rule consumed by the `llvm` module +extension; it is not a public entry point and is not meant to be loaded +directly from a consumer's `MODULE.bazel`. + +## Usage with Bzlmod + +The recommended way to consume this rule is through the `llvm` module +extension in `extensions/llvm.bzl`, which wires `fast_llvm_repo` together with +`toolchains_llvm`'s `llvm_toolchain` for you from a single version value: + +```starlark +module(name = "score_llvm_probe") +bazel_dep(name = "score_bazel_cpp_toolchains", version = "0.5.4") + +llvm = use_extension("@score_bazel_cpp_toolchains//extensions:llvm.bzl", "llvm") +llvm.toolchain(name = "probe_llvm", version = "19.1.1") + +use_repo(llvm, "probe_llvm", "probe_llvm_pkg") +``` + +Consumers do not need their own `bazel_dep(toolchains_llvm)` and only need to +track a single `version` value. See +[Extension API](extension_api.md#llvm-extension-optional-faster-alternative) +for the full attribute reference. + +## What the rule does + +For the requested LLVM version, the rule: + +1. rejects non-Linux hosts and selects the archive for the host architecture, +2. downloads it with a pinned SHA-256 checksum, +3. extracts it with parallel `xz -T0` output streamed to `tar`, using Bash's + `pipefail` to check both process exit codes, +4. renders the BUILD file expected by `toolchains_llvm`. + +The parallel XZ pipeline is the main optimization. It changes the extraction +implementation while leaving the LLVM archive, checksum verification, and +toolchain target layout intact. + +## Why not `http_archive`? + +`http_archive` can functionally download and unpack a `tar.xz`. However, +`toolchains_llvm` already uses Bazel's standard repository download-and-extract +mechanism for the LLVM repository it creates internally. That +`download_and_extract()` path is the slow part this rule is intended to +replace. + +`http_archive` does not provide a switch to use the parallel `xz -T0` pipeline, +and `patch_cmds` run only after extraction. Using `http_archive` would therefore +not provide the performance improvement. + +If the additional extraction time is acceptable, the original +`toolchains_llvm` setup remains the simpler option. This rule exists for the +case where the measured setup time justifies the additional repository-rule +maintenance. + +## Scope and trade-offs + +This is intentionally a small, specialized rule rather than a replacement for +`http_archive` or `toolchains_llvm`: + +- only the versions and Linux architectures listed in + [`fast_llvm_repo.bzl`](../rules/fast_llvm_repo.bzl) are supported; +- the fast path requires `bash`, `tar`, and `xz` on the host; +- the version/checksum table must be kept aligned with `toolchains_llvm`; +- the rule uses `toolchains_llvm`'s internal BUILD template, so upgrades of + that dependency need to be checked; and +- `fast_llvm_repo` itself is not meant to be used directly outside of + `extensions/llvm.bzl`. diff --git a/extensions/llvm.bzl b/extensions/llvm.bzl new file mode 100644 index 0000000..f901ba9 --- /dev/null +++ b/extensions/llvm.bzl @@ -0,0 +1,63 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Module extension for fast prebuilt LLVM toolchains, an alternative to depending on toolchains_llvm directly.""" + +load("@score_bazel_cpp_toolchains//rules:fast_llvm_repo.bzl", "fast_llvm_repo") +load("@toolchains_llvm//toolchain:rules.bzl", "llvm_toolchain") + +_attrs_tc = { + "name": attr.string( + mandatory = True, + doc = "Name of the generated LLVM toolchain repository.", + ), + "version": attr.string( + mandatory = True, + doc = "LLVM version to extract (see rules/fast_llvm_repo.bzl for supported versions).", + ), +} + +def _package_root(package_name): + extension_owner = str(Label("//:MODULE.bazel")).split("//")[0] + return "%s+llvm+%s//" % (extension_owner, package_name) + +def _impl(mctx): + root = None + for mod in mctx.modules: + if not mod.is_root: + fail("Only the root module can use the 'llvm' extension!") + root = mod + + for tag in root.tags.toolchain: + package_name = tag.name + "_pkg" + fast_llvm_repo( + name = package_name, + llvm_version = tag.version, + ) + llvm_toolchain( + name = tag.name, + llvm_version = tag.version, + toolchain_roots = { + "": _package_root(package_name), + }, + ) + +llvm = module_extension( + implementation = _impl, + tag_classes = { + "toolchain": tag_class( + attrs = _attrs_tc, + doc = "Declares an LLVM toolchain extracted via the fast host xz/tar pipeline.", + ), + }, +) diff --git a/rules/fast_llvm_repo.bzl b/rules/fast_llvm_repo.bzl new file mode 100644 index 0000000..2381497 --- /dev/null +++ b/rules/fast_llvm_repo.bzl @@ -0,0 +1,170 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Repository rule for fast extraction of prebuilt LLVM distributions. + +This rule is intended to be used together with `toolchains_llvm`. It creates +the LLVM distribution repository that `toolchains_llvm` normally generates, +but uses the host's parallel `xz` decompressor before streaming into `tar`. +""" + +# The supported LLVM releases. Each entry maps a requested version and host +# architecture to the upstream archive Bazel downloads and verifies. +_LLVM_DISTRIBUTIONS = { + "19.1.0": { + "aarch64": { + # Official prebuilt LLVM package for 64-bit ARM Linux hosts. + "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.0/clang+llvm-19.1.0-aarch64-linux-gnu.tar.xz", + # SHA-256 pinned by toolchains_llvm for this exact upstream archive. + "sha256": "7bb54afd330fe1a1c2d4c593fa1e2dbe2abd9bf34fb3597994ff41e443cf144b", + }, + "x86_64": { + # Official prebuilt LLVM package for 64-bit x86 Linux hosts. + "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.0/LLVM-19.1.0-Linux-X64.tar.xz", + # SHA-256 of that archive. + "sha256": "cee77d641690466a193d9b88c89705de1c02bbad46bde6a3b126793c0a0f2923", + }, + }, + "19.1.1": { + "aarch64": { + # Official prebuilt LLVM package for 64-bit ARM Linux hosts. + "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.1/clang+llvm-19.1.1-aarch64-linux-gnu.tar.xz", + # SHA-256 pinned by toolchains_llvm for this exact upstream archive. + "sha256": "414d2ebef10c5035e9df10a224e81b484dbe17d319373050d0c1b3b1467040d2", + }, + "x86_64": { + # Official prebuilt LLVM package for 64-bit x86 Linux hosts. + "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.1/LLVM-19.1.1-Linux-X64.tar.xz", + "sha256": "8204de000b6a6921f0572e038336601e3225898e9a253c8aaa43b0a5fae8a4ce", + }, + }, + "22.1.7": { + "aarch64": { + # Official prebuilt LLVM package for 64-bit ARM Linux hosts. + "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.7/LLVM-22.1.7-Linux-ARM64.tar.xz", + # SHA-256 from toolchains_llvm's distribution metadata for this + # exact upstream archive. `ctx.download` rejects altered or corrupt + # downloads. + "sha256": "118ca2d3ad9da34367e05735317854e7977db45dc4c02a32af58da64c23b8789", + }, + "x86_64": { + # Official prebuilt LLVM package for 64-bit x86 Linux hosts. + "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.7/LLVM-22.1.7-Linux-X64.tar.xz", + "sha256": "edb0522b41e261819c06ea437d249f9b8acfa413d3805bc9920eec6fb76ff830", + }, + }, +} + +_ARCH_ALIASES = { + "aarch64": "aarch64", + "amd64": "x86_64", + "arm64": "aarch64", + "x86_64": "x86_64", +} + +# Implementation of `fast_llvm_repo`. Repository rules run while Bazel is +# preparing external dependencies, before analysing or building project code. +def _fast_llvm_repo_impl(ctx): + # The distributions below contain Linux binaries and are used as the + # execution root by `toolchains_llvm`; do not create a repository that + # cannot run on the host. + if ctx.os.name.lower() != "linux": + fail("Unsupported host operating system: %s (fast_llvm_repo supports Linux hosts only)" % ctx.os.name) + + # Verify tools needed for extraction are available in the host's PATH. The rule + # will fail with a clear message if any of them is missing. + tar = ctx.which("tar") + if not tar: + fail("tar not found in PATH") + xz = ctx.which("xz") + if not xz: + fail("xz not found in PATH") + bash = ctx.which("bash") + if not bash: + fail("bash not found in PATH") + + # Select the archive native to the host which will execute this toolchain. + host_arch = _ARCH_ALIASES.get(ctx.os.arch) + if not host_arch: + fail("Unsupported host architecture: %s" % ctx.os.arch) + + # Look up the requested release and its host-specific archive. + distributions = _LLVM_DISTRIBUTIONS.get(ctx.attr.llvm_version) + dist = distributions.get(host_arch) if distributions else None + + # Fail with a useful message if MODULE.bazel asks for a version that has no + # URL and checksum entry yet. + if not dist: + fail("Unsupported LLVM version/host architecture: %s/%s" % (ctx.attr.llvm_version, host_arch)) + + # Choose a temporary file inside this external repository for the archive. + archive = ctx.path("llvm.tar.xz") + + # Download the archive and verify its SHA-256 before using it. + ctx.download( + url = dist["url"], + output = archive, + sha256 = dist["sha256"], + ) + + # LLVM's archive has many independently decompressible XZ blocks. Let xz + # decode them in parallel, then stream the resulting tar file directly to + # tar; materializing the uncompressed archive would be wasteful. Bash's + # `pipefail` makes a decompressor failure fail the whole pipeline. + + result = ctx.execute( + [ + bash, + "-o", + "pipefail", + "-c", + "set -e; \"$1\" -T0 -dc \"$2\" | \"$3\" -xf - --strip-components=1 -C \"$4\"", + "fast_llvm_repo", + xz, + archive, + tar, + ctx.path("."), + ], + # Allow the relatively large archive up to 30 minutes to unpack. + timeout = 1800, + # Do not suppress command output from Bazel's repository-rule log. + quiet = False, + ) + + # Stop repository creation if extraction reported an error. + if result.return_code: + fail(result.stderr) + + # The archive is no longer needed after extraction; leave only the LLVM + # distribution files in the external repository. + ctx.delete(archive) + + # Generate the BUILD file consumed by toolchains_llvm. LLVM 16 and later + # store compiler resources in a directory named by the major version. + ctx.template( + "BUILD.bazel", + Label("@toolchains_llvm//toolchain:BUILD.llvm_repo.tpl"), + substitutions = {"{LLVM_VERSION}": ctx.attr.llvm_version.split(".")[0]}, + ) + +# Public repository rule used from MODULE.bazel. Its only user-facing input +# is an LLVM version. +fast_llvm_repo = repository_rule( + implementation = _fast_llvm_repo_impl, + attrs = { + "llvm_version": attr.string( + mandatory = True, + doc = "LLVM release used to select the pinned host archive.", + ), + }, +) diff --git a/tests/MODULE.bazel.lock b/tests/MODULE.bazel.lock index db2f720..0e15ec6 100644 --- a/tests/MODULE.bazel.lock +++ b/tests/MODULE.bazel.lock @@ -68,7 +68,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", - "https://bcr.bazel.build/modules/bazel_features/1.36.0/source.json": "279625cafa5b63cc0a8ee8448d93bc5ac1431f6000c50414051173fd22a6df3c", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/source.json": "31ba776c122b54a2885e23651642e32f087a87bf025465f8040751894b571277", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", @@ -118,6 +119,8 @@ "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/helly25_bzl/0.4.3/MODULE.bazel": "9c20052fd3f1fb767c48b78c1bbc46f501a4ca69d14558429d1234e68e449f68", + "https://bcr.bazel.build/modules/helly25_bzl/0.4.3/source.json": "e8c54d81e72633fb6f1d23c7e2d44e0b39b1caef2a256141136a2f63e6204b78", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.6.0/MODULE.bazel": "26ec5118e66a55fef36f8ea39d6415d55fc966671fe4d61a6b9ef0cd6bc7b6a1", "https://bcr.bazel.build/modules/jq.bzl/0.6.0/source.json": "2ed4e35b9fa9505495784114f7637fbde5846ec14af917841b6c045d877f20eb", @@ -289,10 +292,13 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/tar.bzl/0.7.0/MODULE.bazel": "cc1acd85da33c80e430b65219a620d54d114628df24a618c3a5fa0b65e988da9", "https://bcr.bazel.build/modules/tar.bzl/0.7.0/source.json": "9becb80306f42d4810bfa16379fb48aad0b01ce5342bc12fe47dcd6af3ac4d7a", "https://bcr.bazel.build/modules/toolchain_utils/1.0.2/MODULE.bazel": "9b8be503a4fcfd3b8b952525bff0869177a5234d5c35dc3e566b9f5ca2f755a1", "https://bcr.bazel.build/modules/toolchain_utils/1.0.2/source.json": "88769ec576dddacafd8cca4631812cf8eead89f10a29d9405d9f7a553de6bf87", + "https://bcr.bazel.build/modules/toolchains_llvm/1.8.0/MODULE.bazel": "68259b66e5fb84f94fa37125ad476c3eb8edf602a34bf58377cde9a16dd8fa98", + "https://bcr.bazel.build/modules/toolchains_llvm/1.8.0/source.json": "70d80fe5b626a3fadf812af41dda4a028640a1184f5a3c7e6cb20130d93ed783", "https://bcr.bazel.build/modules/toolchains_protoc/0.2.1/MODULE.bazel": "2f08433ff5e659069b3a1abfee2377d68f510f2de1da50678ed992c455b4ff91", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", @@ -361,6 +367,7 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.32.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.34.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.36.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.38.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.4.1/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.9.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/bazel_features/1.9.1/MODULE.bazel": "not found", @@ -402,6 +409,7 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/googletest/1.14.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/googletest/1.15.2/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/googletest/1.17.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/helly25_bzl/0.4.3/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/jq.bzl/0.1.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/jq.bzl/0.6.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", @@ -592,8 +600,10 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.7.2/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/tar.bzl/0.2.1/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/tar.bzl/0.5.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/tar.bzl/0.6.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/tar.bzl/0.7.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/toolchain_utils/1.0.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/toolchains_llvm/1.8.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/toolchains_protoc/0.2.1/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/yq.bzl/0.1.1/MODULE.bazel": "not found",