Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions packages/contrib/quicksand-ubuntu/hatch_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,12 @@ def initialize(self, version: str, build_data: dict) -> None:
image_path = images_dir / f"ubuntu-{distro_version}-{arch}.qcow2"
dockerfile_path = Path(self.root) / "quicksand_ubuntu" / "docker" / "Dockerfile"

if not image_path.exists():
self.app.display_info(f"Image not found: {image_path.name}, building...")
self._build_image(dockerfile_path, image_path)
# Always delegate to build_image: it reuses the cached qcow2 only when
# its sidecar hash matches the current Dockerfile + agent source, and
# rebuilds otherwise. Gating on mere existence here would package a
# stale image after an agent-only change.
self.app.display_info(f"Ensuring image is up to date: {image_path.name}")
self._build_image(dockerfile_path, image_path)

self.app.display_info(f"Including image: {image_path}")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -322,11 +322,19 @@ def _verify_windows_isolation(self, binary: Path, bin_dir: Path) -> None:
"would break on hosts without it on PATH."
)

def set_platform_wheel_tag(self, build_data: dict) -> None:
def set_platform_wheel_tag(self, build_data: dict, bin_dir: Path | None = None) -> None:
"""Set build_data fields for a platform-specific py3-none wheel.

On Windows ARM64, overrides the tag from ``win_amd64`` to ``win_arm64``
when native hardware is ARM64 (Python may report amd64 under emulation).

On Linux, the manylinux level is derived from the actual glibc symbol
versions the bundled binaries require (via :func:`_linux_manylinux_tag`)
rather than hardcoded. The binaries link against the build runner's
glibc, so the tag must reflect their real floor. A hardcoded
``manylinux_2_17`` lets pip install wheels that then fail to load on
hosts with an older glibc than the runner. ``bin_dir`` (the directory
holding the bundled binaries) is required for Linux wheels.
"""
build_data["pure_python"] = False
platform_tag = sysconfig.get_platform().replace("-", "_").replace(".", "_")
Expand All @@ -349,12 +357,78 @@ def set_platform_wheel_tag(self, build_data: dict) -> None:
except Exception:
pass

# PyPI requires manylinux tags for Linux wheels (PEP 600)
if platform_tag.startswith("linux_"):
platform_tag = platform_tag.replace("linux_", "manylinux_2_17_", 1)
if bin_dir is None:
raise RuntimeError(
"set_platform_wheel_tag requires bin_dir for Linux wheels to "
"derive the manylinux tag from the bundled binaries' glibc "
"requirement."
)
platform_tag = self._linux_manylinux_tag(bin_dir, platform_tag)

build_data["tag"] = f"py3-none-{platform_tag}"

def _linux_manylinux_tag(self, bin_dir: Path, platform_tag: str) -> str:
"""Derive the manylinux platform tag from the bundled ELF binaries.

Scans every ELF file under ``bin_dir`` for the glibc symbol versions it
references and asks auditwheel which manylinux policy that implies. The
result (e.g. ``manylinux_2_38_aarch64``) is the lowest manylinux level
the binaries can actually run on. auditwheel is a hard build dependency
on Linux; if it or the analysis fails we raise rather than guess, since
a wrong tag produces wheels that crash on load instead of being
rejected at install time.
"""
from collections import defaultdict

# Linux-only build deps, absent from the dev venv on other platforms.
from auditwheel.architecture import Architecture # ty: ignore[unresolved-import]
from auditwheel.elfutils import elf_find_versioned_symbols # ty: ignore[unresolved-import]
from auditwheel.libc import Libc # ty: ignore[unresolved-import]
from auditwheel.policy import WheelPolicies # ty: ignore[unresolved-import]
from elftools.common.exceptions import ELFError # ty: ignore[unresolved-import]
from elftools.elf.elffile import ELFFile # ty: ignore[unresolved-import]

arch_name = platform_tag[len("linux_") :]
try:
arch = Architecture(arch_name)
except ValueError as exc:
raise RuntimeError(
f"Unknown architecture {arch_name!r} for manylinux tagging."
) from exc

versioned_symbols: dict[str, set[str]] = defaultdict(set)
elf_count = 0
for path in sorted(bin_dir.rglob("*")):
if path.is_symlink() or not path.is_file():
continue
try:
with path.open("rb") as fh:
if fh.read(4) != b"\x7fELF":
continue
fh.seek(0)
elf = ELFFile(fh)
for soname, version in elf_find_versioned_symbols(elf):
versioned_symbols[soname].add(version)
except (ELFError, OSError):
continue
elf_count += 1

if elf_count == 0:
raise RuntimeError(f"No ELF binaries found under {bin_dir} to derive a glibc tag from.")

policies = WheelPolicies(libc=Libc.GLIBC, arch=arch)
policy_name = policies.versioned_symbols_policy(dict(versioned_symbols)).name
if not policy_name.startswith("manylinux_"):
raise RuntimeError(
f"Bundled binaries require a glibc newer than any manylinux policy "
f"auditwheel knows ({policy_name!r}). Upgrade auditwheel or build "
f"against an older glibc. Required symbol versions: "
f"{dict(versioned_symbols)}"
)
self.app.display_info(f"Derived manylinux tag from glibc usage: {policy_name}")
return policy_name

def force_include_bin_dir(self, bin_dir: Path, root: Path, build_data: dict) -> None:
"""Add all files in bin_dir to the wheel's force_include."""
force_include = build_data.setdefault("force_include", {})
Expand Down
50 changes: 47 additions & 3 deletions packages/dev/quicksand-image-tools/quicksand_image_tools/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,28 @@ def get_agent_source_dir() -> Path:
return AGENT_SOURCE_DIR


def _agent_source_hash() -> str:
"""Hash of the Rust agent source (Cargo manifests + all .rs files).

Folded into the image cache key so that changes to the agent — which the
Dockerfile compiles but does not itself reference — invalidate cached
images. Without this, an agent-only change reuses a stale qcow2.
"""
h = hashlib.sha256()
if AGENT_SOURCE_DIR.exists():
paths = sorted(
p
for p in AGENT_SOURCE_DIR.rglob("*")
if p.is_file()
and "target" not in p.relative_to(AGENT_SOURCE_DIR).parts
and (p.suffix == ".rs" or p.name in ("Cargo.toml", "Cargo.lock"))
)
for p in paths:
h.update(p.relative_to(AGENT_SOURCE_DIR).as_posix().encode())
h.update(p.read_bytes())
return h.hexdigest()


def build_image(
dockerfile: str | Path,
output_path: Path | None = None,
Expand Down Expand Up @@ -79,15 +101,35 @@ def build_image(
dockerfile_content = dockerfile_path.read_text()
context_dir = dockerfile_path.parent

# Compute hash for caching
content_hash = hashlib.sha256(dockerfile_content.encode()).hexdigest()[:16]
# Compute hash for caching. Includes the agent source because the
# Dockerfile compiles the agent from a build-context copy that the
# Dockerfile text doesn't reference — so agent changes must bust the cache.
content_hash = hashlib.sha256(
dockerfile_content.encode() + _agent_source_hash().encode()
).hexdigest()[:16]

explicit_output = output_path is not None
if output_path is None:
output_path = cache / f"custom-{content_hash}.qcow2"

# A sidecar records the content hash of the inputs (Dockerfile + agent
# source) that produced ``output_path``. A caller-supplied output path has
# a fixed name, so without the sidecar a stale image (e.g. built before an
# agent change) would be silently reused. The default ``custom-<hash>``
# path already encodes the hash in its name, so existence alone proves
# freshness there.
hash_sidecar = output_path.with_name(output_path.name + ".buildhash")

# Check cache
if output_path.exists():
if force:
if explicit_output:
cached_hash = hash_sidecar.read_text().strip() if hash_sidecar.exists() else None
stale = cached_hash != content_hash
else:
stale = False
if force or stale:
reason = "forced" if force else "inputs changed"
log.info("Rebuilding image (%s): %s", reason, output_path)
output_path.unlink()
else:
log.info("Using cached image: %s", output_path)
Expand Down Expand Up @@ -128,6 +170,8 @@ def build_image(
_remove_docker_image(tag)

log.info("[5/5] Done!")
# Record the inputs' hash so a later build can detect staleness.
hash_sidecar.write_text(content_hash)
final_size_mb = output_path.stat().st_size / (1024 * 1024)
log.info("Output: %s (%.1f MB)", output_path, final_size_mb)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ tokio-stream = "0.1"
# single open handle and a buffered `tokio::fs::File` seeks when reads and
# writes interleave (ESPIPE on a non-seekable char device), so the transport
# drives one non-blocking fd through `tokio::io::unix::AsyncFd` instead.
# Also used for privilege drop (setuid/setgid/initgroups) and chown/kill in
# multi-user mode.
libc = "0.2"

[profile.release]
Expand Down
Loading
Loading