From d3e0a6e79cacc09ad10fbb15cb7b084080eb5220 Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 02:16:28 +0900 Subject: [PATCH 01/12] Load the native extension lazily and publish a pure-Python wheel Image codecs only need OpenCV, but importing tensorcodec.decoders loaded the Rust extension, so they were unusable where no native wheel exists. VideoDecoder/AudioDecoder now import it on construction and raise a clear ImportError when it is missing. scripts/build_pure_wheel.py builds a py3-none-any wheel with the same [project] metadata; CI builds it, tests it on Windows, Intel macOS and Linux, and publish uploads it with the native wheels. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- .github/workflows/ci.yml | 2 + .github/workflows/publish.yml | 4 +- .github/workflows/pure-python-wheel.yml | 63 +++++++++++++++++++ README.md | 5 +- docs/releasing.md | 8 ++- scripts/build_pure_wheel.py | 82 +++++++++++++++++++++++++ src/tensorcodec/decoders/_decoder.py | 17 ++++- tests/test_native_optional.py | 51 +++++++++++++++ 8 files changed, 223 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/pure-python-wheel.yml create mode 100644 scripts/build_pure_wheel.py create mode 100644 tests/test_native_optional.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aff15a7..a8674c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,8 @@ concurrency: jobs: macos: uses: ./.github/workflows/macos-wheels.yml + pure-python: + uses: ./.github/workflows/pure-python-wheel.yml test: runs-on: ubuntu-24.04 steps: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b85973d..ed71639 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,6 +14,8 @@ concurrency: jobs: macos: uses: ./.github/workflows/macos-wheels.yml + pure-python: + uses: ./.github/workflows/pure-python-wheel.yml build: strategy: fail-fast: false @@ -117,7 +119,7 @@ jobs: path: dist/* if-no-files-found: error publish: - needs: [build, macos] + needs: [build, macos, pure-python] if: inputs.publish runs-on: ubuntu-24.04 environment: diff --git a/.github/workflows/pure-python-wheel.yml b/.github/workflows/pure-python-wheel.yml new file mode 100644 index 0000000..0095ff8 --- /dev/null +++ b/.github/workflows/pure-python-wheel.yml @@ -0,0 +1,63 @@ +name: Pure-Python wheel +on: + workflow_call: +permissions: + contents: read +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - name: Build and check wheel + run: | + python -m pip install twine + python scripts/build_pure_wheel.py --out dist + python -m twine check dist/* + - uses: actions/upload-artifact@v7 + with: + name: pypi-distributions-pure + path: dist/*.whl + if-no-files-found: error + test: + needs: build + strategy: + fail-fast: false + matrix: + include: + - runner: windows-2025 + python: '3.12' + - runner: macos-15-intel + python: '3.12' + - runner: ubuntu-24.04 + python: '3.10' + runs-on: ${{ matrix.runner }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python }} + - uses: actions/download-artifact@v8 + with: + name: pypi-distributions-pure + path: dist + - name: Install FFmpeg for test fixtures + run: | + case "$RUNNER_OS" in + Windows) choco install ffmpeg-full -y --no-progress ;; + macOS) brew install ffmpeg ;; + Linux) sudo apt-get update && sudo apt-get install -y ffmpeg ;; + esac + ffmpeg -hide_banner -version | head -n 1 + - name: Install wheel without a Rust toolchain + run: | + python -m pip install pytest 'Pillow>=11.3' 'opencv-python-headless>=4.13,<5' + python -m pip install --no-index --find-links dist tensorcodec + python -c "import importlib.util as u; assert u.find_spec('tensorcodec._native') is None" + - name: Test image codecs + run: python -m pytest tests/test_images.py tests/test_image_encoders.py tests/test_native_optional.py diff --git a/README.md b/README.md index bed53c0..5bcfd08 100644 --- a/README.md +++ b/README.md @@ -158,8 +158,9 @@ See the [compatibility contract](docs/compatibility.md) and - **Wheels:** Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+. NumPy must also provide a compatible wheel; newer Python versions may require - a newer glibc. macOS 14+ wheels support Apple Silicon (Intel Macs: through 0.1.5). Windows, musl/Alpine - and free-threaded Python wheels are not provided. + a newer glibc. macOS 14+ wheels support Apple Silicon (Intel Macs: through 0.1.5). Elsewhere + (Windows, Intel macOS, musl/Alpine, free-threaded Python) installers select a pure-Python wheel: + image codecs work, while `VideoDecoder`/`AudioDecoder` raise `ImportError` when constructed. - **Exact seeking:** scans packet timestamps when opening the decoder. Incorrect container keyframe flags can produce corrupt frames; repaired input or corrected frame mappings are needed in that case. diff --git a/docs/releasing.md b/docs/releasing.md index de48728..50422aa 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -5,7 +5,9 @@ Binary wheels target Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10 NumPy must also provide a compatible wheel for the selected Python/glibc pair. The wheel bundles shared FFmpeg 7.1.5 and OpenSSL 3.5.9 LTS; its only Python runtime dependency is NumPy. macOS 14+ ARM64 wheels bundle the same minimal -runtime. Windows wheels are not provided. +runtime. Windows wheels are not provided. A pure-Python `py3-none-any` wheel, +built by `scripts/build_pure_wheel.py` with the same metadata, covers every other +platform with image codecs only; installers prefer a matching native wheel. ## Trusted publisher configuration @@ -25,8 +27,8 @@ is needed. Repository visibility does not need to change for a release. ## Release -Run the **Publish to PyPI** workflow on `main`. It builds the portable Linux/macOS wheels -and source distribution, checks package metadata, validates the pinned oracle +Run the **Publish to PyPI** workflow on `main`. It builds the portable Linux/macOS wheels, +the pure-Python wheel and source distribution, checks package metadata, validates the pinned oracle and compares playback before uploading through PyPI Trusted Publishing. It uses the existing GitHub `pypi` environment. Publication fails if authorization is missing, tests fail, or the version has already been uploaded. diff --git a/scripts/build_pure_wheel.py b/scripts/build_pure_wheel.py new file mode 100644 index 0000000..89bfc67 --- /dev/null +++ b/scripts/build_pure_wheel.py @@ -0,0 +1,82 @@ +"""Build the py3-none-any fallback wheel: every Python module, no native extension. + +Maturin always compiles the extension, so the package is staged with the same +[project] metadata and built by hatchling. Installers prefer the native wheels +where their tags match and fall back to this one elsewhere. +""" + +import argparse +import re +import shutil +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +import tomllib + +ROOT = Path(__file__).resolve().parents[1] +BUILD_SYSTEM = """[build-system] +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/tensorcodec"] +""" + + +def project_metadata(text): + """Return the [project] tables verbatim so the fallback cannot drift from the native wheels.""" + tables = re.split(r"(?m)^(?=\[)", text) + project = "".join(table for table in tables if re.match(r"\[project[\].]", table)) + if tomllib.loads(project)["project"] != tomllib.loads(text)["project"]: + raise RuntimeError("could not extract [project] metadata from pyproject.toml") + return project + + +def stage(directory): + shutil.copytree( + ROOT / "src" / "tensorcodec", + directory / "src" / "tensorcodec", + ignore=shutil.ignore_patterns("_native*", "__pycache__", "*.pyc"), + ) + shutil.copytree(ROOT / "licenses", directory / "licenses") + for name in ("LICENSE", "README.md"): + shutil.copy2(ROOT / name, directory / name) + text = (ROOT / "pyproject.toml").read_text() + (directory / "pyproject.toml").write_text(project_metadata(text) + "\n" + BUILD_SYSTEM) + + +def check(wheel): + if not wheel.name.endswith("-py3-none-any.whl"): + raise RuntimeError(f"unexpected wheel tag: {wheel.name}") + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + if any(Path(name).name.startswith("_native") for name in names): + raise RuntimeError("fallback wheel must not contain the native extension") + if "tensorcodec/decoders/_images.py" not in names: + raise RuntimeError("fallback wheel is missing the image decoders") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--out", type=Path, default=ROOT / "dist") + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory() as tmp: + staging, built = Path(tmp) / "stage", Path(tmp) / "dist" + stage(staging) + subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-deps", "--wheel-dir", built, staging], + check=True, + ) + (wheel,) = built.glob("*.whl") + check(wheel) + target = args.out / wheel.name + shutil.copy2(wheel, target) + print(target) + + +if __name__ == "__main__": + main() diff --git a/src/tensorcodec/decoders/_decoder.py b/src/tensorcodec/decoders/_decoder.py index ec19ad7..2fefbcc 100644 --- a/src/tensorcodec/decoders/_decoder.py +++ b/src/tensorcodec/decoders/_decoder.py @@ -13,10 +13,21 @@ from tensorcodec._frame import AudioSamples, Frame, FrameBatch from tensorcodec._metadata import AudioStreamMetadata, VideoStreamMetadata -from tensorcodec._native import Decoder as NativeDecoder from tensorcodec.transforms import _pipeline +def _native_decoder(*args): + try: + from tensorcodec._native import Decoder + except ImportError as error: + raise ImportError( + "VideoDecoder and AudioDecoder require TensorCodec's native extension, which could not be loaded. " + "Native wheels cover Linux x86_64/aarch64 (glibc 2.17+) and macOS 14+ arm64; elsewhere the " + "pure-Python wheel provides only the image codecs." + ) from error + return Decoder(*args) + + def _source(source): if isinstance(source, (str, os.PathLike)): return os.fspath(source) @@ -125,7 +136,7 @@ def __init__( self._seek_mode = seek_mode self._lock = RLock() self._closed = False - self._native = NativeDecoder(_source(source), "video", stream_index, int(num_ffmpeg_threads)) + self._native = _native_decoder(_source(source), "video", stream_index, int(num_ffmpeg_threads)) try: header = self._native.metadata(apply_rotation=output_format != "native") self.stream_index = header["stream_index"] @@ -424,7 +435,7 @@ def __init__(self, source, *, stream_index=None, sample_rate=None, num_channels= raise ValueError(f"{name} must be a positive integer") self._lock = RLock() self._closed = False - self._native = NativeDecoder(_source(source), "audio", stream_index) + self._native = _native_decoder(_source(source), "audio", stream_index) try: header = self._native.metadata() header.pop("time_base_num") diff --git a/tests/test_native_optional.py b/tests/test_native_optional.py new file mode 100644 index 0000000..2d114e3 --- /dev/null +++ b/tests/test_native_optional.py @@ -0,0 +1,51 @@ +"""Image codecs import and run without the native extension, as in the pure-Python wheel.""" + +import subprocess +import sys + + +def run_python(code): + subprocess.run([sys.executable, "-c", code], check=True) + + +def test_native_extension_is_lazy(): + run_python( + """ +import sys +import tensorcodec +import tensorcodec.decoders +import tensorcodec.encoders +assert 'tensorcodec._native' not in sys.modules +""" + ) + + +def test_image_codecs_without_native_extension(): + run_python( + """ +import sys +sys.modules['tensorcodec._native'] = None +import numpy as np +import tensorcodec +from tensorcodec.decoders import ( + AudioDecoder, ImageReadMode, VideoDecoder, decode_avif, decode_gif, decode_image, decode_jpeg, decode_png, + decode_webp, +) +from tensorcodec.encoders import JpegEncoder, PngEncoder + +pixels = np.zeros((3, 4, 5), np.uint8) +pixels[:] = np.array([23, 91, 177], np.uint8)[:, None, None] +encoded = PngEncoder(pixels).to_tensor() +np.testing.assert_array_equal(decode_png(encoded), pixels) +np.testing.assert_array_equal(decode_image(encoded, mode=ImageReadMode.RGB), pixels) +assert decode_jpeg(JpegEncoder(pixels).to_tensor()).shape == pixels.shape + +for decoder in (VideoDecoder, AudioDecoder): + try: + decoder(b'not media') + except ImportError as exc: + assert 'native extension' in str(exc), exc + else: + raise AssertionError(f'{decoder.__name__} worked without the native extension') +""" + ) From 69adadfdebf64f355b46a5b62c019f4190f5a280 Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 02:16:28 +0900 Subject: [PATCH 02/12] Detect BMP in decode_image OpenCV decodes BMP losslessly. Alpha follows Pillow: only 32-bit BI_BITFIELDS with a nonzero alpha mask keeps it, since OpenCV otherwise returns the padding byte as a fourth channel. TIFF stays unsupported because OpenCV premultiplies unassociated alpha and drops gray+alpha. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- docs/images.md | 13 ++++++-- src/tensorcodec/decoders/_images.py | 16 ++++++++- tests/test_images.py | 52 +++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/docs/images.md b/docs/images.md index c31d087..cea4ff9 100644 --- a/docs/images.md +++ b/docs/images.md @@ -1,6 +1,6 @@ # Image codecs -Decode JPEG, PNG, WebP, GIF and AVIF into NumPy arrays, and encode grayscale or +Decode JPEG, PNG, WebP, GIF, AVIF and BMP into NumPy arrays, and encode grayscale or RGB arrays as JPEG or PNG. Decoding returns CHW images or NCHW animations on CPU. ```sh @@ -26,6 +26,11 @@ installation is sufficient; otherwise the `images` extra installs NumPy is the only required dependency for the base package. Image dependencies are separate from the base wheel size. Pillow is used only in tests. +Image codecs do not use the native extension. Platforms without a native wheel +(Windows, Intel macOS, musl, free-threaded Python) install the pure-Python +`py3-none-any` wheel, which provides the full image API; constructing +`VideoDecoder` or `AudioDecoder` there raises `ImportError`. + ## Contract and limits - `decode_image`, `decode_jpeg`, `decode_png`, `decode_webp`, `decode_gif`, @@ -44,7 +49,11 @@ are separate from the base wheel size. Pillow is used only in tests. - AVIF color conversion follows OpenCV. Dropping alpha preserves straight RGB; TorchCodec 0.17.0 premultiplies AVIF RGB in that case. Pixel identity with TorchCodec is not promised across formats, builds or codec versions. -- HEIC is unsupported. +- `decode_image` also detects BMP (no format-specific function). 32-bit + `BI_BITFIELDS` BMPs with a nonzero alpha mask keep alpha; other 32-bit BMPs are + RGB, as in Pillow, because their fourth byte is padding. +- HEIC is unsupported. TIFF is not detected: OpenCV premultiplies unassociated + alpha and drops gray+alpha samples, so lossless decoding cannot be promised. - Other formats depend on the installed OpenCV build. Missing dependencies, unsupported codecs and decode failures raise; no alternate decoder is tried. - Encoders accept nonempty CHW uint8 arrays with 1 or 3 channels. Both provide diff --git a/src/tensorcodec/decoders/_images.py b/src/tensorcodec/decoders/_images.py index 328371d..05c91c0 100644 --- a/src/tensorcodec/decoders/_images.py +++ b/src/tensorcodec/decoders/_images.py @@ -69,6 +69,8 @@ def _format(data): return "avif" if any(b in (b"heic", b"heix", b"heim", b"heis", b"hevc", b"hevx", b"mif1", b"msf1") for b in brands): return "heic" + if data.startswith(b"BM"): + return "bmp" raise ValueError("Unsupported or unrecognized image format") @@ -94,6 +96,14 @@ def _png_channels(data): return channels +def _bmp_alpha(data): + """True for 32-bit BI_BITFIELDS with a nonzero alpha mask, the only BMP alpha Pillow also reads.""" + header = int.from_bytes(data[14:18], "little") + bits = int.from_bytes(data[28:30], "little") + compression = int.from_bytes(data[30:34], "little") + return bits == 32 and compression == 3 and header >= 56 and len(data) >= 70 and any(data[66:70]) + + def _jpeg_components(data): offset = 2 while offset < len(data): @@ -213,6 +223,9 @@ def _image(source, codec, mode, output_dtype): frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB) elif frame.shape[-1] == 4: frame = cv.cvtColor(frame, cv.COLOR_BGRA2RGBA) + if codec == "bmp" and frame.shape[-1] == 4 and not _bmp_alpha(data): + # OpenCV keeps the padding byte of BI_BITFIELDS pixels without an alpha mask. + frame = frame[..., :3] if channels == 2 and frame.shape[-1] == 4: frame = frame[..., [0, 3]] if codec == "avif" and frame.dtype == np.uint16: @@ -258,7 +271,8 @@ def decode_image(source, *, mode="RGB", output_dtype=np.uint8): Sources are paths, bytes or 1-D uint8 arrays. Modes: UNCHANGED, GRAY, GRAY_ALPHA, RGB, RGB_ALPHA (case-insensitive strings or ImageReadMode). output_dtype is uint8, uint16 or 'auto'; integer conversion scales the range. - Requires optional OpenCV >= 4.13. HEIC and animated PNG are unsupported. + BMP is detected too. Requires optional OpenCV >= 4.13. HEIC and animated PNG + are unsupported. """ return _image(source, None, mode, output_dtype) diff --git a/tests/test_images.py b/tests/test_images.py index d990d19..cc9a460 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -509,3 +509,55 @@ def test_low_bit_grayscale_transparency(bits): result = decode_png(data, mode=mode) np.testing.assert_array_equal(result[:-1], np.tile([[[gray, 0]]], (channels, 1, 1))) np.testing.assert_array_equal(result[-1], [[0, 255]]) + + +def bitfields_bmp(rgba, header_size, alpha_mask): + """32-bit BI_BITFIELDS BMP; header_size 40 stores RGB masks only, 124 is BITMAPV5HEADER.""" + height, width, _ = rgba.shape + pixels = rgba[::-1, :, [2, 1, 0, 3]].tobytes() + header = struct.pack(" Date: Mon, 5 Oct 2026 02:20:21 +0900 Subject: [PATCH 03/12] Make image fixtures and fallback CI portable to Windows and Intel macOS FFmpeg's glob pattern type is unavailable on Windows, so the animated GIF fixture uses a numbered sequence. Homebrew's FFmpeg has no libaom, so the fallback jobs use the conda-forge FFmpeg the other workflows already pin. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- .github/workflows/pure-python-wheel.yml | 17 +++++++++-------- tests/test_images.py | 16 ++++------------ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pure-python-wheel.yml b/.github/workflows/pure-python-wheel.yml index 0095ff8..1fc078f 100644 --- a/.github/workflows/pure-python-wheel.yml +++ b/.github/workflows/pure-python-wheel.yml @@ -46,14 +46,15 @@ jobs: with: name: pypi-distributions-pure path: dist - - name: Install FFmpeg for test fixtures - run: | - case "$RUNNER_OS" in - Windows) choco install ffmpeg-full -y --no-progress ;; - macOS) brew install ffmpeg ;; - Linux) sudo apt-get update && sudo apt-get install -y ffmpeg ;; - esac - ffmpeg -hide_banner -version | head -n 1 + # Fixtures need libaom; Homebrew's FFmpeg lacks it, so use conda-forge's like the other jobs. + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.81.0 + run-install: false + global-environments: ffmpeg=7.1.1=gpl_* + global-cache: true + - name: Check fixture FFmpeg + run: ffmpeg -hide_banner -encoders | grep libaom-av1 - name: Install wheel without a Rust toolchain run: | python -m pip install pytest 'Pillow>=11.3' 'opencv-python-headless>=4.13,<5' diff --git a/tests/test_images.py b/tests/test_images.py index cc9a460..4ccc63a 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -54,18 +54,10 @@ def encoded_images(tmp_path_factory): }.items(): run_ffmpeg("-i", root / "source.png", "-frames:v", 1, *options, "-threads", 1, root / f"image.{codec}") Image.fromarray(pixels).save(root / "image.webp", lossless=True) - (root / "second.png").write_bytes(png_bytes(255 - pixels)) - run_ffmpeg( - "-framerate", - 2, - "-pattern_type", - "glob", - "-i", - str(root / "s*.png"), - "-threads", - 1, - root / "animated.gif", - ) + # Numbered sequence: FFmpeg's glob pattern type is unavailable on Windows. + for index, frame in enumerate((pixels, 255 - pixels)): + (root / f"frame{index}.png").write_bytes(png_bytes(frame)) + run_ffmpeg("-framerate", 2, "-i", root / "frame%d.png", "-threads", 1, root / "animated.gif") return root From 70c447da985c2f5fa4e2e455346ea43dac7789ad Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 02:22:37 +0900 Subject: [PATCH 04/12] Put the pixi FFmpeg on PATH in fallback CI jobs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- .github/workflows/pure-python-wheel.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pure-python-wheel.yml b/.github/workflows/pure-python-wheel.yml index 1fc078f..b0af854 100644 --- a/.github/workflows/pure-python-wheel.yml +++ b/.github/workflows/pure-python-wheel.yml @@ -48,11 +48,17 @@ jobs: path: dist # Fixtures need libaom; Homebrew's FFmpeg lacks it, so use conda-forge's like the other jobs. - uses: prefix-dev/setup-pixi@v0.10.2 + env: + PIXI_HOME: ${{ runner.temp }}/pixi with: pixi-version: v0.81.0 run-install: false global-environments: ffmpeg=7.1.1=gpl_* global-cache: true + - name: Put fixture FFmpeg on PATH + env: + FFMPEG_BIN: ${{ runner.temp }}/pixi/envs/ffmpeg/${{ runner.os == 'Windows' && 'Library/bin' || 'bin' }} + run: echo "$FFMPEG_BIN" >> "$GITHUB_PATH" - name: Check fixture FFmpeg run: ffmpeg -hide_banner -encoders | grep libaom-av1 - name: Install wheel without a Rust toolchain From 10196a73b51d4d395e1cb6ccca55e2ed13ab6e44 Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 02:24:56 +0900 Subject: [PATCH 05/12] Expect the explicit AVIF error where OpenCV has no AVIF decoder The Windows opencv-python-headless 4.14 wheel cannot decode AVIF; the dispatch test now checks the explicit error there instead of failing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- docs/images.md | 3 ++- tests/test_images.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/images.md b/docs/images.md index cea4ff9..84feb77 100644 --- a/docs/images.md +++ b/docs/images.md @@ -54,7 +54,8 @@ Image codecs do not use the native extension. Platforms without a native wheel RGB, as in Pillow, because their fourth byte is padding. - HEIC is unsupported. TIFF is not detected: OpenCV premultiplies unassociated alpha and drops gray+alpha samples, so lossless decoding cannot be promised. -- Other formats depend on the installed OpenCV build. Missing dependencies, +- Format support depends on the installed OpenCV build; for example, the + Windows `opencv-python-headless` 4.14 wheel has no AVIF decoder. Missing dependencies, unsupported codecs and decode failures raise; no alternate decoder is tried. - Encoders accept nonempty CHW uint8 arrays with 1 or 3 channels. Both provide `to_file`, `to_file_like` and `to_tensor`; JPEG quality is 1–100 (default 75), diff --git a/tests/test_images.py b/tests/test_images.py index 4ccc63a..51e6923 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -117,6 +117,13 @@ def test_image_sources_and_content_detection(backend, tmp_path, kind): @pytest.mark.parametrize("codec", ["jpeg", "webp", "gif", "avif"]) def test_format_functions_and_dispatch(backend, encoded_images, codec): path = encoded_images / f"image.{codec}" + if codec == "avif" and not backend.__name__.startswith("torchcodec"): + import cv2 + + if not cv2.haveImageReader(str(path)): # e.g. opencv-python-headless 4.14 on Windows + with pytest.raises(RuntimeError, match="codec build support"): + backend.decode_avif(path) + return direct = as_numpy(getattr(backend, f"decode_{codec}")(path)) assert direct.shape == (3, 16, 24) assert direct.dtype == np.uint8 From 8433f0769ac82186553c7ee0bbc64e9d18fa4eb8 Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 02:44:35 +0900 Subject: [PATCH 06/12] Split into tensorcodec (pure Python) and tensorcodec-native (extension) Replaces the copy-and-rebuild fallback wheel. tensorcodec is built by hatchling as a py3-none-any wheel with the Python API and image codecs. native/ is the maturin project for tensorcodec-native, which contains only the FFmpeg extension (import tensorcodec_native) and the bundled-library notices. tensorcodec pins tensorcodec-native to its own version behind a marker matching the native wheel tags, plus a `native` extra; the version test and a runtime check reject skew. Publish uploads tensorcodec-native before tensorcodec. The OpenCV bound is now measured: the image suite, including TorchCodec comparisons, passes with 4.12, 4.13 and 5.0; 4.10/4.11 wheels lack the GIF and AVIF decoders. The extra becomes >=4.12 without the <5 cap. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- .github/workflows/ci.yml | 12 +- .github/workflows/macos-wheels.yml | 10 +- .github/workflows/publish.yml | 32 ++- .github/workflows/pure-python-wheel.yml | 33 +-- .github/workflows/source.yml | 5 +- README.md | 17 +- docs/images.md | 19 +- docs/package_size.md | 3 +- docs/releasing.md | 73 +++++-- docs/system_ffmpeg.md | 16 +- native/LICENSE | 204 ++++++++++++++++++ native/README.md | 8 + .../licenses}/FFmpeg-GPL-3.0.txt | 0 .../licenses}/FFmpeg-LGPL-3.0.txt | 0 .../licenses}/FFmpeg-NOTICE.md | 0 {licenses => native/licenses}/OpenSSL.txt | 0 {licenses => native/licenses}/README.md | 2 +- {licenses => native/licenses}/Zstandard.txt | 0 {licenses => native/licenses}/dav1d.txt | 0 native/pyproject.toml | 32 +++ native/python/tensorcodec_native/__init__.py | 1 + native/src/lib.rs | 1 + pyproject.toml | 44 ++-- scripts/build_linux_wheel.sh | 2 +- scripts/build_macos_wheel.sh | 2 +- scripts/build_pure_wheel.py | 82 ------- scripts/configure_oracle_ffmpeg.py | 6 +- scripts/update_size_comparison.py | 2 +- src/tensorcodec/_opencv.py | 7 +- src/tensorcodec/decoders/_decoder.py | 16 +- src/tensorcodec/decoders/_images.py | 2 +- tests/test_native_optional.py | 29 ++- tests/test_versions.py | 53 +++++ 33 files changed, 513 insertions(+), 200 deletions(-) create mode 100644 native/LICENSE create mode 100644 native/README.md rename {licenses => native/licenses}/FFmpeg-GPL-3.0.txt (100%) rename {licenses => native/licenses}/FFmpeg-LGPL-3.0.txt (100%) rename {licenses => native/licenses}/FFmpeg-NOTICE.md (100%) rename {licenses => native/licenses}/OpenSSL.txt (100%) rename {licenses => native/licenses}/README.md (91%) rename {licenses => native/licenses}/Zstandard.txt (100%) rename {licenses => native/licenses}/dav1d.txt (100%) create mode 100644 native/pyproject.toml create mode 100644 native/python/tensorcodec_native/__init__.py delete mode 100644 scripts/build_pure_wheel.py create mode 100644 tests/test_versions.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8674c6..5d14434 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: run: | python -m venv .venv .venv/bin/python -m pip install --upgrade pip - .venv/bin/pip install numpy pytest ruff 'maturin>=1.8,<2' 'opencv-python-headless>=4.13,<5' 'Pillow>=11.3' + .venv/bin/pip install numpy pytest ruff 'maturin>=1.8,<2' 'opencv-python-headless>=4.12' 'Pillow>=11.3' echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> "$GITHUB_ENV" - name: Configure prebuilt FFmpeg @@ -53,12 +53,14 @@ jobs: "$prefix/bin/ffmpeg" -version - name: Static checks run: | - ruff check src/tensorcodec tests scripts benchmarks/image_codecs.py - ruff format --check src/tensorcodec tests scripts benchmarks/image_codecs.py + ruff check src/tensorcodec native/python tests scripts benchmarks/image_codecs.py + ruff format --check src/tensorcodec native/python tests scripts benchmarks/image_codecs.py cargo fmt --manifest-path native/Cargo.toml --check cargo clippy --manifest-path native/Cargo.toml --locked -- -D warnings - - name: Build extension against prebuilt FFmpeg - run: maturin develop --locked + - name: Build tensorcodec-native against prebuilt FFmpeg and install tensorcodec + run: | + maturin develop -m native/Cargo.toml --locked + pip install -e . - name: Install pinned reference run: python -m pip install torch==2.14.1 torchcodec==0.17.0 --index-url https://download.pytorch.org/whl/cpu - name: Validate independent contract against reference diff --git a/.github/workflows/macos-wheels.yml b/.github/workflows/macos-wheels.yml index 2db8d2d..b6848db 100644 --- a/.github/workflows/macos-wheels.yml +++ b/.github/workflows/macos-wheels.yml @@ -33,7 +33,7 @@ jobs: run: | brew install nasm pkg-config coreutils meson ninja uv venv - uv pip install numpy pytest 'opencv-python-headless>=4.13,<5' 'Pillow>=11.3' 'maturin>=1.8,<2' delocate twine + uv pip install numpy pytest 'opencv-python-headless>=4.12' 'Pillow>=11.3' 'maturin>=1.8,<2' delocate twine echo "$PWD/.venv/bin" >> "$GITHUB_PATH" echo "LIBCLANG_PATH=$(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib" >> "$GITHUB_ENV" echo "$HOME/.pixi/envs/ffmpeg/bin" >> "$GITHUB_PATH" @@ -51,9 +51,11 @@ jobs: key: ${{ steps.native-cache.outputs.cache-primary-key }} - name: Hide build-time FFmpeg libraries run: mv .native-deps/macos .native-deps/macos-build-only + - name: Build tensorcodec wheel for validation + run: uv build --wheel --out-dir pure-dist - name: Validate installed wheel run: | - uv pip install dist/*.whl + uv pip install dist/*.whl pure-dist/*.whl python -m twine check dist/* pytest python scripts/check_wheel_runtime.py generate .runtime-fixtures @@ -61,7 +63,7 @@ jobs: run: | for version in 3.10 3.13; do uv venv "/tmp/runtime-$version" --python "$version" - uv pip install --python "/tmp/runtime-$version/bin/python" dist/*.whl + uv pip install --python "/tmp/runtime-$version/bin/python" dist/*.whl pure-dist/*.whl "/tmp/runtime-$version/bin/python" scripts/check_wheel_runtime.py check .runtime-fixtures done - name: Restore build cache @@ -72,6 +74,6 @@ jobs: fi - uses: actions/upload-artifact@v7 with: - name: pypi-distributions-macos-${{ matrix.arch }} + name: pypi-distributions-native-macos-${{ matrix.arch }} path: dist/*.whl if-no-files-found: error diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ed71639..68a2c1a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -42,7 +42,7 @@ jobs: target: ${{ matrix.target }} manylinux: '2014' command: build - args: --release --locked --auditwheel repair --out dist + args: -m native/Cargo.toml --release --locked --auditwheel repair --out dist before-script-linux: | /opt/python/cp310-cp310/bin/python -m pip install libclang==18.1.1 export LIBCLANG_PATH="$(/opt/python/cp310-cp310/bin/python -c 'from clang import cindex; print(cindex.Config.library_path)')" @@ -81,15 +81,17 @@ jobs: name: wheel-size-${{ matrix.target }} path: reports/wheel-size.json if-no-files-found: warn - - name: Build source distribution + - name: Build tensorcodec-native source distribution if: matrix.target == 'x86_64' - run: uvx --from 'maturin>=1.8,<2' maturin sdist --out dist + run: uvx --from 'maturin>=1.8,<2' maturin sdist -m native/Cargo.toml --out dist + - name: Build tensorcodec wheel for validation + run: uv build --wheel --out-dir pure-dist - name: Install validation tools and wheel run: | sudo apt-get update sudo apt-get install -y ffmpeg uv venv - uv pip install pytest numpy twine 'opencv-python-headless>=4.13,<5' 'Pillow>=11.3' dist/*.whl + uv pip install pytest numpy twine 'opencv-python-headless>=4.12' 'Pillow>=11.3' dist/*.whl pure-dist/*.whl - name: Generate baseline runtime fixtures run: uv run --no-sync python scripts/check_wheel_runtime.py generate .runtime-fixtures - name: Test on glibc 2.17 with Python 3.10 and 3.13 @@ -100,7 +102,7 @@ jobs: "$python" -m venv "/tmp/$abi" runtime="/tmp/$abi/bin/python" if [ "$abi" = cp310-cp310 ]; then "$runtime" -m pip install --only-binary=:all: numpy==1.26.4; fi - "$runtime" -m pip install --only-binary=:all: /io/dist/*.whl + "$runtime" -m pip install --only-binary=:all: /io/dist/*.whl /io/pure-dist/*.whl env -u LD_LIBRARY_PATH "$runtime" /io/scripts/check_wheel_runtime.py check /io/.runtime-fixtures done ' @@ -115,7 +117,7 @@ jobs: uv run --no-sync pytest --compare - uses: actions/upload-artifact@v7 with: - name: pypi-distributions-${{ matrix.target }} + name: pypi-distributions-native-${{ matrix.target }} path: dist/* if-no-files-found: error publish: @@ -131,10 +133,22 @@ jobs: steps: - uses: actions/download-artifact@v8 with: - pattern: pypi-distributions-* + pattern: pypi-distributions-native-* merge-multiple: true - path: dist - - uses: pypa/gh-action-pypi-publish@release/v1 + path: native-dist + - uses: actions/download-artifact@v8 + with: + name: pypi-distributions-pure + path: pure-dist + # Native first: tensorcodec pins tensorcodec-native exactly, so it must not appear before its pin exists. + - name: Publish tensorcodec-native + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: native-dist/ + - name: Publish tensorcodec + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: pure-dist/ update-size-docs: needs: publish if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/pure-python-wheel.yml b/.github/workflows/pure-python-wheel.yml index b0af854..ad4e92a 100644 --- a/.github/workflows/pure-python-wheel.yml +++ b/.github/workflows/pure-python-wheel.yml @@ -1,4 +1,4 @@ -name: Pure-Python wheel +name: tensorcodec distributions on: workflow_call: permissions: @@ -11,16 +11,23 @@ jobs: - uses: actions/setup-python@v7 with: python-version: '3.12' - - name: Build and check wheel + - name: Build and check wheel and sdist run: | - python -m pip install twine - python scripts/build_pure_wheel.py --out dist - python -m twine check dist/* + python -m pip install build twine + python -m build --outdir dist + python -m twine check --strict dist/* + python - <<'EOF' + import glob, zipfile + (wheel,) = glob.glob("dist/tensorcodec-*-py3-none-any.whl") + names = zipfile.ZipFile(wheel).namelist() + assert not [n for n in names if n.endswith((".so", ".pyd", ".dylib")) or "FFmpeg" in n], names + EOF - uses: actions/upload-artifact@v7 with: name: pypi-distributions-pure - path: dist/*.whl + path: dist/* if-no-files-found: error + # Platforms without tensorcodec-native wheels: the marker skips it and image codecs still work. test: needs: build strategy: @@ -29,10 +36,10 @@ jobs: include: - runner: windows-2025 python: '3.12' + opencv: 'opencv-python-headless>=4.12' - runner: macos-15-intel - python: '3.12' - - runner: ubuntu-24.04 python: '3.10' + opencv: 'opencv-python-headless==4.12.0.88' # lower bound runs-on: ${{ matrix.runner }} defaults: run: @@ -61,10 +68,10 @@ jobs: run: echo "$FFMPEG_BIN" >> "$GITHUB_PATH" - name: Check fixture FFmpeg run: ffmpeg -hide_banner -encoders | grep libaom-av1 - - name: Install wheel without a Rust toolchain + - name: Install tensorcodec without a Rust toolchain or tensorcodec-native run: | - python -m pip install pytest 'Pillow>=11.3' 'opencv-python-headless>=4.13,<5' - python -m pip install --no-index --find-links dist tensorcodec - python -c "import importlib.util as u; assert u.find_spec('tensorcodec._native') is None" + python -m pip install pytest 'Pillow>=11.3' '${{ matrix.opencv }}' + python -m pip install --no-index --find-links dist "tensorcodec[images]" + python -c "import importlib.util as u; assert u.find_spec('tensorcodec_native') is None" - name: Test image codecs - run: python -m pytest tests/test_images.py tests/test_image_encoders.py tests/test_native_optional.py + run: python -m pytest tests/test_images.py tests/test_image_encoders.py tests/test_native_optional.py tests/test_versions.py diff --git a/.github/workflows/source.yml b/.github/workflows/source.yml index 7b37b90..c5cd0b2 100644 --- a/.github/workflows/source.yml +++ b/.github/workflows/source.yml @@ -12,8 +12,9 @@ jobs: with: python-version: '3.12' - run: | - python -m pip install 'maturin>=1.8,<2' - maturin sdist --out dist + python -m pip install 'maturin>=1.8,<2' build + maturin sdist -m native/Cargo.toml --out dist + python -m build --sdist --outdir dist - uses: actions/upload-artifact@v7 with: name: tensorcodec-source diff --git a/README.md b/README.md index 5bcfd08..232dea6 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,12 @@ See the [compatibility contract](docs/compatibility.md) and - **Wheels:** Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+. NumPy must also provide a compatible wheel; newer Python versions may require - a newer glibc. macOS 14+ wheels support Apple Silicon (Intel Macs: through 0.1.5). Elsewhere - (Windows, Intel macOS, musl/Alpine, free-threaded Python) installers select a pure-Python wheel: - image codecs work, while `VideoDecoder`/`AudioDecoder` raise `ImportError` when constructed. + a newer glibc. macOS 14+ wheels support Apple Silicon (Intel Macs: through 0.1.5). + `tensorcodec` itself is pure Python; video/audio decoding lives in `tensorcodec-native`, pinned to + the same version and installed automatically on those platforms (CPython only). Elsewhere, e.g. + Windows and Intel macOS, image codecs work and `VideoDecoder`/`AudioDecoder` raise `ImportError`. + Environment markers cannot detect musl or free-threaded CPython on Linux x86_64/aarch64, so + installers there still try `tensorcodec-native` (no wheel; its sdist needs FFmpeg 7 and Rust). - **Exact seeking:** scans packet timestamps when opening the decoder. Incorrect container keyframe flags can produce corrupt frames; repaired input or corrected frame mappings are needed in that case. @@ -176,7 +179,7 @@ See [container behavior](docs/container_robustness.md) for seek limitations and
Build from source and run tests -Source builds require Rust 1.88+, Clang/libclang, pkg-config and FFmpeg 7 development +`uv sync` builds `native/` (the `tensorcodec-native` package) in place. Its source builds require Rust 1.88+, Clang/libclang, pkg-config and FFmpeg 7 development headers/libraries. Python handles API and playback selection; Rust + PyO3 handles FFmpeg. Native decoding releases the GIL, allowing separate decoder instances to run concurrently across Python threads. Calls on the same instance are serialized. @@ -189,14 +192,14 @@ uv sync --group dev --group oracle uv run --group oracle pytest tests/test_video_contract.py tests/test_audio_contract.py --backend torchcodec uv run --group oracle pytest --compare -# Rebuild after changing Rust code. -uv run --group oracle maturin develop --locked --uv +# Rebuild tensorcodec-native after changing Rust code. +uv run --group oracle maturin develop -m native/Cargo.toml --locked --uv ``` Tests generate media with FFmpeg/ffprobe and Python's `wave` module. `--compare` requires the pinned oracle; differential tests otherwise skip. -[Playback rules](docs/playback_semantics.md) · [Release guide](docs/releasing.md) · [Dependency licenses](licenses/README.md) +[Playback rules](docs/playback_semantics.md) · [Release guide](docs/releasing.md) · [Dependency licenses](native/licenses/README.md)
diff --git a/docs/images.md b/docs/images.md index 84feb77..46c5873 100644 --- a/docs/images.md +++ b/docs/images.md @@ -20,16 +20,21 @@ encoded = JpegEncoder(rgb).to_tensor(quality=90) # 1-D uint8 NumPy array ## Installation -The current image backend uses OpenCV 4.13+. An existing compatible `cv2` -installation is sufficient; otherwise the `images` extra installs -`opencv-python-headless`. Use only one OpenCV wheel variant per environment. +The image backend uses OpenCV 4.12 or newer, including 5.x. An existing compatible +`cv2` installation is sufficient; otherwise the `images` extra installs +`opencv-python-headless`. The bound is measured: the image test suite, including +the TorchCodec comparisons, passes with `opencv-python-headless` 4.12.0.88, +4.13.0.92 and 5.0.0.93. With 4.10 and 4.11 everything except GIF and AVIF passes: +4.12 is the first release whose PyPI wheels build both decoders (OpenCV added GIF +in 4.11, but its wheels report `GIF: NO` and `AVIF: NO`). The 4.8 and 4.9 wheels +do not import with NumPy 2. Use only one OpenCV wheel variant per environment. NumPy is the only required dependency for the base package. Image dependencies are separate from the base wheel size. Pillow is used only in tests. -Image codecs do not use the native extension. Platforms without a native wheel -(Windows, Intel macOS, musl, free-threaded Python) install the pure-Python -`py3-none-any` wheel, which provides the full image API; constructing -`VideoDecoder` or `AudioDecoder` there raises `ImportError`. +Image codecs are pure Python and do not need `tensorcodec-native`, so they work +on every platform, including Windows and Intel macOS where `pip install +tensorcodec` installs no native package; constructing `VideoDecoder` or +`AudioDecoder` there raises `ImportError`. ## Contract and limits diff --git a/docs/package_size.md b/docs/package_size.md index b8a2981..3da3f01 100644 --- a/docs/package_size.md +++ b/docs/package_size.md @@ -1,7 +1,8 @@ # Package size policy TensorCodec keeps its NumPy-only Python dependency set and bundles a minimal -FFmpeg/OpenSSL runtime in its default Linux wheels. Size limits prevent additions +FFmpeg/OpenSSL runtime in its default Linux `tensorcodec-native` wheels (the pure-Python +`tensorcodec` wheel adds about 30 KiB and is not size-checked). Size limits prevent additions from silently increasing the distributed binary footprint. ## What is measured diff --git a/docs/releasing.md b/docs/releasing.md index 50422aa..4fe652c 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,13 +1,32 @@ # Publishing TensorCodec -Release version: `0.2.0`. Distribution and import name: `tensorcodec`. -Binary wheels target Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+. -NumPy must also provide a compatible wheel for the selected Python/glibc pair. -The wheel bundles shared FFmpeg 7.1.5 and OpenSSL 3.5.9 LTS; its only Python +Release version: `0.2.0`. One release publishes two PyPI projects from the same +commit and version: + +| Project | Contents | Build | Distributions | +| --- | --- | --- | --- | +| `tensorcodec` | Python API, image codecs (import `tensorcodec`) | hatchling (`uv build`) | `py3-none-any` wheel, sdist | +| `tensorcodec-native` | Rust/FFmpeg extension (import `tensorcodec_native`) | maturin (`native/`) | platform wheels, sdist | + +`tensorcodec` requires `tensorcodec-native==` behind an environment +marker matching the native wheel tags, and the `native` extra requests it +unconditionally. `tests/test_versions.py` fails CI unless `pyproject.toml`, +`native/Cargo.toml` (the native version source), both pins and +`tensorcodec.__version__` agree. At runtime `VideoDecoder`/`AudioDecoder` reject a +`tensorcodec-native` whose version differs. + +Native wheels target Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+ +(abi3). NumPy must also provide a compatible wheel for the selected Python/glibc +pair. The wheel bundles shared FFmpeg 7.1.5 and OpenSSL 3.5.9 LTS; its only Python runtime dependency is NumPy. macOS 14+ ARM64 wheels bundle the same minimal -runtime. Windows wheels are not provided. A pure-Python `py3-none-any` wheel, -built by `scripts/build_pure_wheel.py` with the same metadata, covers every other -platform with image codecs only; installers prefer a matching native wheel. +runtime. Windows wheels are not provided; there, `tensorcodec` provides image +codecs only. Bundled-library notices ship only with `tensorcodec-native` +(`native/licenses/`). + +To release, set the new version in `pyproject.toml` (project version and both +`tensorcodec-native` pins), `native/Cargo.toml` (then `cargo update -p +tensorcodec-native --manifest-path native/Cargo.toml` to refresh the lock file) +and `src/tensorcodec/__init__.py`. ## Trusted publisher configuration @@ -15,23 +34,28 @@ The PyPI project is already registered. Its GitHub Trusted Publisher uses: | Field | Value | | --- | --- | -| PyPI project name | `tensorcodec` | +| PyPI project names | `tensorcodec`, `tensorcodec-native` | | GitHub owner | `epishelf` | | Repository | `tensorcodec` | | Workflow filename | `publish.yml` | | Environment | `pypi` | -Manage this configuration in the project's PyPI Publishing settings when moving +Both projects need this publisher. `tensorcodec-native` does not exist on PyPI yet: +before its first release, add it as a pending publisher (PyPI account → Publishing) +with the same fields. Manage this configuration in each project's PyPI Publishing settings when moving or renaming the repository or workflow. Publishing uses GitHub OIDC; no API token is needed. Repository visibility does not need to change for a release. ## Release -Run the **Publish to PyPI** workflow on `main`. It builds the portable Linux/macOS wheels, -the pure-Python wheel and source distribution, checks package metadata, validates the pinned oracle +Run the **Publish to PyPI** workflow on `main`. It builds the portable Linux/macOS +`tensorcodec-native` wheels and sdist plus the `tensorcodec` wheel and sdist, checks package metadata, validates the pinned oracle and compares playback before uploading through PyPI Trusted Publishing. It uses -the existing GitHub `pypi` environment. Publication fails if authorization is -missing, tests fail, or the version has already been uploaded. +the existing GitHub `pypi` environment. `tensorcodec-native` is uploaded first, so the +exact pin in `tensorcodec` never points at a missing release; if the second upload +fails, rerun only it (`twine upload` of the `pypi-distributions-pure` artifact) rather +than the whole workflow. Publication fails if authorization is missing, tests fail, +or the version has already been uploaded. ```sh gh workflow run publish.yml --repo epishelf/tensorcodec --ref main @@ -39,26 +63,33 @@ gh workflow run publish.yml --repo epishelf/tensorcodec --ref main For a build and full validation without uploading, pass `--field publish=false`. -Check the workflow and https://pypi.org/project/tensorcodec/0.2.0/ before reporting -success. Verify a fresh `uv pip install tensorcodec==0.2.0` and a decode without -Torch/PyAV on both architectures. Update the version before subsequent releases; +Check the workflow, https://pypi.org/project/tensorcodec/ and +https://pypi.org/project/tensorcodec-native/ before reporting success. Verify a fresh +`uv pip install tensorcodec==` pulls the matching `tensorcodec-native` and +decodes video without Torch/PyAV on both Linux architectures and macOS arm64, and +that it installs without `tensorcodec-native` on Windows. Update the version before subsequent releases; PyPI versions cannot be overwritten. The local Linux build is reproducible using `scripts/build_linux_wheel.sh` inside `quay.io/pypa/manylinux2014_x86_64` or `quay.io/pypa/manylinux2014_aarch64` with Rust, maturin, libclang, NASM and Perl. Both native source archives are version- and checksum-pinned. Their licensing -and source links are recorded in `licenses/README.md`. +and source links are recorded in `native/licenses/README.md`. ## CI versus release builds - Ordinary CI uses prebuilt conda-forge FFmpeg 7.1.1 through Pixi, including its - headers and shared libraries. It builds only the TensorCodec extension. + headers and shared libraries. It builds only the `tensorcodec-native` extension + and installs `tensorcodec` from the checkout. +- The `tensorcodec` wheel is built once with `python -m build` and checked with + `twine check --strict`; Windows and Intel macOS jobs install it without + `tensorcodec-native` and run the image tests (one with OpenCV 5, one with 4.x). - PyPI wheels use the smaller LGPL FFmpeg 7.1.5 build plus OpenSSL 3.5.9. Their native prefix is cached by architecture, glibc baseline and build-script checksums. This preserves the wheel's codec set, dependency size and licensing rather than bundling the full conda-forge dependency graph. -- Release validation installs each repaired wheel on glibc 2.17 with Python 3.10 +- Release validation installs each repaired `tensorcodec-native` wheel together with + the `tensorcodec` wheel on glibc 2.17 with Python 3.10 and 3.13 and decodes video/audio without Torch, PyAV or a system FFmpeg. Python 3.10 also checks the minimum NumPy line (1.26.4). Native x86_64 and ARM64 runners also run the full pinned playback oracle comparison. @@ -67,13 +98,13 @@ and source links are recorded in `licenses/README.md`. ## Size checks and published comparison -Final repaired wheels must stay within 15 MiB download and 35 MiB unpacked per +Final repaired `tensorcodec-native` wheels must stay within 15 MiB download and 35 MiB unpacked per architecture. The build job checks `packaging/size-policy.json` and uploads a separate `wheel-size-*` report, including differences from the last published baseline. Size reports must not be placed in `dist/`. After a successful publication, the `update-size-docs` job measures hash-verified -PyPI wheels for the release and pinned TorchCodec 0.17.0. It commits the published +PyPI `tensorcodec-native` wheels for the release and pinned TorchCodec 0.17.0. It commits the published snapshot and README table/badge with a normal push to `main`. Only this documentation job has `contents: write`; the PyPI publisher retains OIDC plus read access. The repository must permit the Actions bot to push these documentation updates. diff --git a/docs/system_ffmpeg.md b/docs/system_ffmpeg.md index 72fb23a..b26d757 100644 --- a/docs/system_ffmpeg.md +++ b/docs/system_ffmpeg.md @@ -7,7 +7,8 @@ uv venv uv pip install tensorcodec ``` -Supported Linux wheels include minimal shared FFmpeg 7.1.5 (with dav1d for AV1) and OpenSSL libraries. +On supported platforms this also installs the matching `tensorcodec-native` +wheel, which includes minimal shared FFmpeg 7.1.5 (with dav1d for AV1) and OpenSSL libraries. No FFmpeg CLI, Pixi, Rust or libclang is required at runtime. This is the recommended installation for a new environment. @@ -37,9 +38,14 @@ test -f "$FFMPEG_DIR/include/libavcodec/avcodec.h" test -f "$FFMPEG_DIR/lib/libavcodec.so.61" uv venv -uv pip install --no-binary tensorcodec 'tensorcodec==0.1.5' +uv pip install --no-binary tensorcodec-native 'tensorcodec[native]' ``` +`tensorcodec` is pure Python; only `tensorcodec-native` is built from source here. +Outside the platforms with native wheels (for example Intel macOS), the `native` +extra requests it explicitly. Releases up to 0.2.0 were a single package, built +from source with `--no-binary tensorcodec`. + The version/build constraint avoids silently selecting an incompatible FFmpeg major. `FFMPEG_DIR` tells the source build where to find headers and libraries; `LD_LIBRARY_PATH` tells the Linux loader where to find the libraries at runtime. @@ -49,14 +55,14 @@ paths, also set `LIBCLANG_PATH` to its library directory. The example uses a GPL-enabled conda-forge build, unlike the minimal LGPL release build. Its additional codecs, dependencies and licensing apply to your environment. -Review [`licenses/README.md`](../licenses/README.md) before redistributing a binary +Review [`native/licenses/README.md`](../native/licenses/README.md) before redistributing a binary built against a different FFmpeg configuration. For development against the same prefix: ```sh uv sync --group dev -uv run maturin develop --locked --uv +uv run maturin develop -m native/Cargo.toml --locked --uv ``` Do not reuse an existing `dist/` wheel while verifying this path: it may contain the @@ -78,7 +84,7 @@ within the color-conversion tolerances described in the ## macOS wheels -macOS 14+ wheels support Apple Silicon, bundling FFmpeg/OpenSSL with `delocate` +`tensorcodec-native` macOS 14+ wheels support Apple Silicon, bundling FFmpeg/OpenSSL with `delocate` (0.1.3 through 0.1.5 also had Intel wheels). CI tests the installed wheels and clean Python 3.10/3.13 environments. Developers can run `scripts/build_macos_wheel.sh` with Rust, Xcode tools, NASM, Meson, Ninja, pkg-config, coreutils, diff --git a/native/LICENSE b/native/LICENSE new file mode 100644 index 0000000..77d7acf --- /dev/null +++ b/native/LICENSE @@ -0,0 +1,204 @@ +Copyright (c) 2026 Suhwan Choi + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/native/README.md b/native/README.md new file mode 100644 index 0000000..65c6b56 --- /dev/null +++ b/native/README.md @@ -0,0 +1,8 @@ +# tensorcodec-native + +The FFmpeg-backed extension behind [TensorCodec](https://github.com/MilkClouds/tensorcodec)'s +`VideoDecoder` and `AudioDecoder`. It has no public API: install `tensorcodec`, which +depends on the matching `tensorcodec-native` version on platforms with native wheels. + +Wheels bundle shared FFmpeg, dav1d and OpenSSL libraries; see +[`licenses/README.md`](https://github.com/MilkClouds/tensorcodec/blob/main/native/licenses/README.md) for their licenses and sources. diff --git a/licenses/FFmpeg-GPL-3.0.txt b/native/licenses/FFmpeg-GPL-3.0.txt similarity index 100% rename from licenses/FFmpeg-GPL-3.0.txt rename to native/licenses/FFmpeg-GPL-3.0.txt diff --git a/licenses/FFmpeg-LGPL-3.0.txt b/native/licenses/FFmpeg-LGPL-3.0.txt similarity index 100% rename from licenses/FFmpeg-LGPL-3.0.txt rename to native/licenses/FFmpeg-LGPL-3.0.txt diff --git a/licenses/FFmpeg-NOTICE.md b/native/licenses/FFmpeg-NOTICE.md similarity index 100% rename from licenses/FFmpeg-NOTICE.md rename to native/licenses/FFmpeg-NOTICE.md diff --git a/licenses/OpenSSL.txt b/native/licenses/OpenSSL.txt similarity index 100% rename from licenses/OpenSSL.txt rename to native/licenses/OpenSSL.txt diff --git a/licenses/README.md b/native/licenses/README.md similarity index 91% rename from licenses/README.md rename to native/licenses/README.md index 664e922..3980012 100644 --- a/licenses/README.md +++ b/native/licenses/README.md @@ -1,6 +1,6 @@ # Bundled native libraries -TensorCodec's own code is Apache-2.0 licensed. Linux and macOS wheels bundle shared FFmpeg +TensorCodec's own code is Apache-2.0 licensed. `tensorcodec-native` Linux and macOS wheels bundle shared FFmpeg 7.1.5 libraries, built without GPL codec libraries using `scripts/build_ffmpeg.sh`. This configuration is LGPL-3.0-or-later. Its notices and both the LGPLv3 and incorporated GPLv3 texts are included here. The exact diff --git a/licenses/Zstandard.txt b/native/licenses/Zstandard.txt similarity index 100% rename from licenses/Zstandard.txt rename to native/licenses/Zstandard.txt diff --git a/licenses/dav1d.txt b/native/licenses/dav1d.txt similarity index 100% rename from licenses/dav1d.txt rename to native/licenses/dav1d.txt diff --git a/native/pyproject.toml b/native/pyproject.toml new file mode 100644 index 0000000..c187d80 --- /dev/null +++ b/native/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "tensorcodec-native" +# Version comes from Cargo.toml and must equal tensorcodec's (tests/test_versions.py). +dynamic = ["version"] +description = "FFmpeg extension for tensorcodec's video and audio decoders" +readme = "README.md" +license = "Apache-2.0 AND LGPL-3.0-or-later AND BSD-2-Clause" +license-files = ["LICENSE", "licenses/*"] +authors = [{ name = "Suhwan Choi", email = "milkclouds00@gmail.com" }] +requires-python = ">=3.10" +dependencies = ["numpy>=1.26"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python :: 3", + "Programming Language :: Rust", + "Topic :: Multimedia :: Video", +] + +[project.urls] +Repository = "https://github.com/MilkClouds/tensorcodec" +Issues = "https://github.com/MilkClouds/tensorcodec/issues" + +[build-system] +requires = ["maturin>=1.8,<2"] +build-backend = "maturin" + +[tool.maturin] +python-source = "python" +module-name = "tensorcodec_native._native" +features = ["pyo3/extension-module"] diff --git a/native/python/tensorcodec_native/__init__.py b/native/python/tensorcodec_native/__init__.py new file mode 100644 index 0000000..af91cb2 --- /dev/null +++ b/native/python/tensorcodec_native/__init__.py @@ -0,0 +1 @@ +"""FFmpeg extension used by tensorcodec's video and audio decoders; not a public API.""" diff --git a/native/src/lib.rs b/native/src/lib.rs index 48a7951..076af89 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -137,6 +137,7 @@ fn closed() -> PyErr { fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add("ffmpeg_version", ffmpeg::version())?; + module.add("__version__", env!("CARGO_PKG_VERSION"))?; Ok(()) } diff --git a/pyproject.toml b/pyproject.toml index 9ed776c..eb747cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,48 +4,45 @@ version = "0.2.0" description = "Video, audio and image codecs with TorchCodec-style APIs and NumPy arrays" readme = "README.md" license = "Apache-2.0" -license-files = ["LICENSE", "licenses/*"] +license-files = ["LICENSE"] authors = [{ name = "Suhwan Choi", email = "milkclouds00@gmail.com" }] requires-python = ">=3.10" -dependencies = ["numpy>=1.26"] +# tensorcodec-native is pinned to this exact version. The marker matches its wheel tags (CPython abi3: +# manylinux x86_64/aarch64, macOS 14+ arm64 = Darwin 23+); elsewhere only the image codecs work. +dependencies = [ + "numpy>=1.26", + "tensorcodec-native==0.2.0; platform_python_implementation == 'CPython' and ((sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64' and platform_release >= '23'))", +] classifiers = [ "Development Status :: 3 - Alpha", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS :: MacOS X", + "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Rust", + "Topic :: Multimedia :: Graphics", "Topic :: Multimedia :: Video", ] [project.optional-dependencies] -images = ["opencv-python-headless>=4.13,<5"] +images = ["opencv-python-headless>=4.12"] +# Unconditional, for platforms outside the marker with a source-built tensorcodec-native. +native = ["tensorcodec-native==0.2.0"] [project.urls] Repository = "https://github.com/MilkClouds/tensorcodec" Issues = "https://github.com/MilkClouds/tensorcodec/issues" [dependency-groups] -dev = ["pytest>=8", "ruff>=0.9", "maturin>=1.8,<2", "opencv-python-headless>=4.13,<5", "Pillow>=11.3"] +dev = ["pytest>=8", "ruff>=0.9", "maturin>=1.8,<2", "opencv-python-headless>=4.12", "Pillow>=11.3"] oracle = ["torch==2.14.1", "torchcodec==0.17.0"] [build-system] -requires = ["maturin>=1.8,<2"] -build-backend = "maturin" +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" -[tool.maturin] -manifest-path = "native/Cargo.toml" -python-source = "src" -module-name = "tensorcodec._native" -features = ["pyo3/extension-module"] -include = [ - { path = "scripts/*.sh", format = "sdist" }, - { path = "scripts/*.py", format = "sdist" }, - { path = "docs/*.md", format = "sdist" }, - { path = "packaging/*.json", format = "sdist" }, - { path = "tests/**/*.py", format = "sdist" }, - { path = "benchmarks/**/*.py", format = "sdist" }, - { path = "benchmarks/*.md", format = "sdist" }, -] +[tool.hatch.build.targets.wheel] +packages = ["src/tensorcodec"] + +[tool.hatch.build.targets.sdist] +only-include = ["src", "tests", "docs", "scripts", "benchmarks", "packaging", "README.md", "LICENSE"] [tool.pytest.ini_options] testpaths = ["tests"] @@ -55,6 +52,7 @@ line-length = 119 target-version = "py310" [tool.uv.sources] +tensorcodec-native = { path = "native", editable = true } torch = { index = "pytorch-cpu" } torchcodec = { index = "pytorch-cpu" } diff --git a/scripts/build_linux_wheel.sh b/scripts/build_linux_wheel.sh index 8123a9d..e04a969 100755 --- a/scripts/build_linux_wheel.sh +++ b/scripts/build_linux_wheel.sh @@ -9,5 +9,5 @@ export LD_LIBRARY_PATH="$build_prefix/openssl/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY scripts/build_ffmpeg.sh "$build_prefix/ffmpeg" export FFMPEG_DIR="$build_prefix/ffmpeg" export LD_LIBRARY_PATH="$FFMPEG_DIR/lib:$LD_LIBRARY_PATH" -maturin build --release --locked --auditwheel repair --compatibility manylinux2014 --out dist +maturin build -m native/Cargo.toml --release --locked --auditwheel repair --compatibility manylinux2014 --out dist python scripts/check_wheel_size.py dist/*.whl diff --git a/scripts/build_macos_wheel.sh b/scripts/build_macos_wheel.sh index db681a3..178840b 100644 --- a/scripts/build_macos_wheel.sh +++ b/scripts/build_macos_wheel.sh @@ -11,6 +11,6 @@ if [ ! -f "$build_prefix/ready" ]; then touch "$build_prefix/ready" fi export FFMPEG_DIR="$build_prefix/ffmpeg" -maturin build --release --locked --out unrepaired +maturin build -m native/Cargo.toml --release --locked --out unrepaired delocate-wheel --require-archs "$(uname -m)" -w dist unrepaired/*.whl python scripts/check_wheel_size.py dist/*.whl diff --git a/scripts/build_pure_wheel.py b/scripts/build_pure_wheel.py deleted file mode 100644 index 89bfc67..0000000 --- a/scripts/build_pure_wheel.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Build the py3-none-any fallback wheel: every Python module, no native extension. - -Maturin always compiles the extension, so the package is staged with the same -[project] metadata and built by hatchling. Installers prefer the native wheels -where their tags match and fall back to this one elsewhere. -""" - -import argparse -import re -import shutil -import subprocess -import sys -import tempfile -import zipfile -from pathlib import Path - -import tomllib - -ROOT = Path(__file__).resolve().parents[1] -BUILD_SYSTEM = """[build-system] -requires = ["hatchling>=1.27"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/tensorcodec"] -""" - - -def project_metadata(text): - """Return the [project] tables verbatim so the fallback cannot drift from the native wheels.""" - tables = re.split(r"(?m)^(?=\[)", text) - project = "".join(table for table in tables if re.match(r"\[project[\].]", table)) - if tomllib.loads(project)["project"] != tomllib.loads(text)["project"]: - raise RuntimeError("could not extract [project] metadata from pyproject.toml") - return project - - -def stage(directory): - shutil.copytree( - ROOT / "src" / "tensorcodec", - directory / "src" / "tensorcodec", - ignore=shutil.ignore_patterns("_native*", "__pycache__", "*.pyc"), - ) - shutil.copytree(ROOT / "licenses", directory / "licenses") - for name in ("LICENSE", "README.md"): - shutil.copy2(ROOT / name, directory / name) - text = (ROOT / "pyproject.toml").read_text() - (directory / "pyproject.toml").write_text(project_metadata(text) + "\n" + BUILD_SYSTEM) - - -def check(wheel): - if not wheel.name.endswith("-py3-none-any.whl"): - raise RuntimeError(f"unexpected wheel tag: {wheel.name}") - with zipfile.ZipFile(wheel) as archive: - names = archive.namelist() - if any(Path(name).name.startswith("_native") for name in names): - raise RuntimeError("fallback wheel must not contain the native extension") - if "tensorcodec/decoders/_images.py" not in names: - raise RuntimeError("fallback wheel is missing the image decoders") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--out", type=Path, default=ROOT / "dist") - args = parser.parse_args() - args.out.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory() as tmp: - staging, built = Path(tmp) / "stage", Path(tmp) / "dist" - stage(staging) - subprocess.run( - [sys.executable, "-m", "pip", "wheel", "--no-deps", "--wheel-dir", built, staging], - check=True, - ) - (wheel,) = built.glob("*.whl") - check(wheel) - target = args.out / wheel.name - shutil.copy2(wheel, target) - print(target) - - -if __name__ == "__main__": - main() diff --git a/scripts/configure_oracle_ffmpeg.py b/scripts/configure_oracle_ffmpeg.py index f6df43b..2b7e4b8 100644 --- a/scripts/configure_oracle_ffmpeg.py +++ b/scripts/configure_oracle_ffmpeg.py @@ -5,10 +5,10 @@ import re from pathlib import Path -spec = importlib.util.find_spec("tensorcodec") -libs = Path(spec.origin).parent.parent / "tensorcodec.libs" +spec = importlib.util.find_spec("tensorcodec_native") +libs = Path(spec.origin).parent.parent / "tensorcodec_native.libs" if not libs.is_dir(): - raise RuntimeError("Expected an installed, repaired TensorCodec wheel") + raise RuntimeError("Expected an installed, repaired tensorcodec-native wheel") for library in libs.glob("lib*.so.*"): match = re.fullmatch(r"(lib[^-]+)-[0-9a-f]+(\.so\.\d+)", library.name) if match and match[1] in {"libavcodec", "libavformat", "libavutil", "libswscale", "libswresample"}: diff --git a/scripts/update_size_comparison.py b/scripts/update_size_comparison.py index 3250627..c665faf 100644 --- a/scripts/update_size_comparison.py +++ b/scripts/update_size_comparison.py @@ -118,7 +118,7 @@ def main() -> None: reference = policy["comparison"] pyav = policy["pyav"] snapshot = { - "tensorcodec": measure_release("tensorcodec", args.version, "cp310"), + "tensorcodec": measure_release("tensorcodec-native", args.version, "cp310"), "torchcodec": measure_release(reference["project"], reference["version"], reference["python_tag"]), "pyav": measure_release(pyav["project"], pyav["version"], pyav["python_tag"]), "torch": measure_release(**policy["torch"]), diff --git a/src/tensorcodec/_opencv.py b/src/tensorcodec/_opencv.py index b20f34e..5a63b15 100644 --- a/src/tensorcodec/_opencv.py +++ b/src/tensorcodec/_opencv.py @@ -6,9 +6,10 @@ def opencv(): import cv2 except ImportError as exc: raise ImportError( - "Image codecs require OpenCV >= 4.13; install tensorcodec[images] " + "Image codecs require OpenCV >= 4.12; install tensorcodec[images] " "or use an existing compatible cv2 installation" ) from exc - if tuple(int(part) for part in cv2.__version__.split(".")[:2]) < (4, 13): - raise ImportError("Image codecs require OpenCV >= 4.13") + # 4.12: first PyPI wheels with GIF and AVIF decoders; 4.10/4.11 fail only those tests (docs/images.md). + if tuple(int(part) for part in cv2.__version__.split(".")[:2]) < (4, 12): + raise ImportError("Image codecs require OpenCV >= 4.12") return cv2 diff --git a/src/tensorcodec/decoders/_decoder.py b/src/tensorcodec/decoders/_decoder.py index 2fefbcc..413a704 100644 --- a/src/tensorcodec/decoders/_decoder.py +++ b/src/tensorcodec/decoders/_decoder.py @@ -18,14 +18,20 @@ def _native_decoder(*args): try: - from tensorcodec._native import Decoder + from tensorcodec_native import _native except ImportError as error: raise ImportError( - "VideoDecoder and AudioDecoder require TensorCodec's native extension, which could not be loaded. " - "Native wheels cover Linux x86_64/aarch64 (glibc 2.17+) and macOS 14+ arm64; elsewhere the " - "pure-Python wheel provides only the image codecs." + "VideoDecoder and AudioDecoder require the tensorcodec-native package, which could not be imported. " + "pip installs it automatically on Linux x86_64/aarch64 (glibc 2.17+) and macOS 14+ arm64 with " + "CPython; elsewhere tensorcodec provides only the image codecs." ) from error - return Decoder(*args) + from tensorcodec import __version__ + + if _native.__version__ != __version__: + raise ImportError( + f"tensorcodec {__version__} requires tensorcodec-native=={__version__}, found {_native.__version__}" + ) + return _native.Decoder(*args) def _source(source): diff --git a/src/tensorcodec/decoders/_images.py b/src/tensorcodec/decoders/_images.py index 05c91c0..2bef3b7 100644 --- a/src/tensorcodec/decoders/_images.py +++ b/src/tensorcodec/decoders/_images.py @@ -271,7 +271,7 @@ def decode_image(source, *, mode="RGB", output_dtype=np.uint8): Sources are paths, bytes or 1-D uint8 arrays. Modes: UNCHANGED, GRAY, GRAY_ALPHA, RGB, RGB_ALPHA (case-insensitive strings or ImageReadMode). output_dtype is uint8, uint16 or 'auto'; integer conversion scales the range. - BMP is detected too. Requires optional OpenCV >= 4.13. HEIC and animated PNG + BMP is detected too. Requires optional OpenCV >= 4.12. HEIC and animated PNG are unsupported. """ return _image(source, None, mode, output_dtype) diff --git a/tests/test_native_optional.py b/tests/test_native_optional.py index 2d114e3..69cd21d 100644 --- a/tests/test_native_optional.py +++ b/tests/test_native_optional.py @@ -1,4 +1,4 @@ -"""Image codecs import and run without the native extension, as in the pure-Python wheel.""" +"""Image codecs work without tensorcodec-native; video/audio fail clearly when it is missing or skewed.""" import subprocess import sys @@ -15,7 +15,7 @@ def test_native_extension_is_lazy(): import tensorcodec import tensorcodec.decoders import tensorcodec.encoders -assert 'tensorcodec._native' not in sys.modules +assert 'tensorcodec_native' not in sys.modules """ ) @@ -24,7 +24,7 @@ def test_image_codecs_without_native_extension(): run_python( """ import sys -sys.modules['tensorcodec._native'] = None +sys.modules['tensorcodec_native'] = None import numpy as np import tensorcodec from tensorcodec.decoders import ( @@ -44,8 +44,27 @@ def test_image_codecs_without_native_extension(): try: decoder(b'not media') except ImportError as exc: - assert 'native extension' in str(exc), exc + assert 'tensorcodec-native package' in str(exc), exc else: - raise AssertionError(f'{decoder.__name__} worked without the native extension') + raise AssertionError(f'{decoder.__name__} worked without tensorcodec-native') +""" + ) + + +def test_version_skew_is_rejected(): + run_python( + """ +import sys +import types +package = types.ModuleType('tensorcodec_native') +package._native = types.SimpleNamespace(__version__='0.0.0', Decoder=None) +sys.modules['tensorcodec_native'] = package +from tensorcodec.decoders import VideoDecoder +try: + VideoDecoder(b'not media') +except ImportError as exc: + assert 'requires tensorcodec-native==' in str(exc) and 'found 0.0.0' in str(exc), exc +else: + raise AssertionError('mismatched tensorcodec-native was accepted') """ ) diff --git a/tests/test_versions.py b/tests/test_versions.py new file mode 100644 index 0000000..364e64b --- /dev/null +++ b/tests/test_versions.py @@ -0,0 +1,53 @@ +"""tensorcodec and tensorcodec-native are released in lockstep from one commit.""" + +import re +from pathlib import Path + +import pytest +from packaging.markers import default_environment +from packaging.requirements import Requirement + +tomllib = pytest.importorskip("tomllib") +ROOT = Path(__file__).resolve().parents[1] + + +def native_requirements(project): + groups = [project["dependencies"], *project["optional-dependencies"].values()] + requirements = [Requirement(item) for group in groups for item in group] + return [r for r in requirements if r.name == "tensorcodec-native"] + + +def test_versions_are_locked(): + project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] + version = project["version"] + cargo = tomllib.loads((ROOT / "native/Cargo.toml").read_text())["package"]["version"] + init = re.search(r'^__version__ = "(.+)"$', (ROOT / "src/tensorcodec/__init__.py").read_text(), re.MULTILINE)[1] + assert cargo == init == version + pins = native_requirements(project) + assert len(pins) == 2 + assert all(str(r.specifier) == f"=={version}" for r in pins) + + +def test_native_license_matches_project_license(): + assert (ROOT / "native/LICENSE").read_bytes() == (ROOT / "LICENSE").read_bytes() + + +@pytest.mark.parametrize( + ("environment", "expected"), + [ + ({"sys_platform": "linux", "platform_machine": "x86_64"}, True), + ({"sys_platform": "linux", "platform_machine": "aarch64"}, True), + ({"sys_platform": "linux", "platform_machine": "armv7l"}, False), + ({"sys_platform": "linux", "platform_machine": "x86_64", "platform_python_implementation": "PyPy"}, False), + ({"sys_platform": "darwin", "platform_machine": "arm64", "platform_release": "23.0.0"}, True), + ({"sys_platform": "darwin", "platform_machine": "arm64", "platform_release": "25.1.0"}, True), + ({"sys_platform": "darwin", "platform_machine": "arm64", "platform_release": "22.6.0"}, False), + ({"sys_platform": "darwin", "platform_machine": "x86_64", "platform_release": "24.0.0"}, False), + ({"sys_platform": "win32", "platform_machine": "AMD64", "platform_release": "2025Server"}, False), + ], +) +def test_native_marker_matches_published_wheels(environment, expected): + project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] + (default,) = [r for r in native_requirements(project) if r.marker is not None] + base = default_environment() | {"platform_python_implementation": "CPython", "platform_release": "6.8.0-azure"} + assert default.marker.evaluate(base | environment) is expected From 1285c801cfcb01bc99e1c09411daa1a67c1eba1c Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 02:53:08 +0900 Subject: [PATCH 07/12] Version the split as 0.3.0; tolerate the sdist's missing native sources 0.2.0 is an existing single-package release: reusing it would collide with its sdist and leave installed 0.2.0 environments un-upgraded. The version test also checks Cargo.lock and skips native checks in the tensorcodec sdist, which does not contain native/. README notes the --no-deps route for Linux environments the marker cannot exclude (musl, free-threaded). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- README.md | 1 + docs/releasing.md | 2 +- native/Cargo.lock | 2 +- native/Cargo.toml | 2 +- pyproject.toml | 6 +++--- src/tensorcodec/__init__.py | 2 +- tests/test_versions.py | 6 ++++++ 7 files changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 232dea6..d4d4428 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ See the [compatibility contract](docs/compatibility.md) and Windows and Intel macOS, image codecs work and `VideoDecoder`/`AudioDecoder` raise `ImportError`. Environment markers cannot detect musl or free-threaded CPython on Linux x86_64/aarch64, so installers there still try `tensorcodec-native` (no wheel; its sdist needs FFmpeg 7 and Rust). + For image codecs only there, use `pip install --no-deps tensorcodec numpy`. - **Exact seeking:** scans packet timestamps when opening the decoder. Incorrect container keyframe flags can produce corrupt frames; repaired input or corrected frame mappings are needed in that case. diff --git a/docs/releasing.md b/docs/releasing.md index 4fe652c..a0de70a 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,6 +1,6 @@ # Publishing TensorCodec -Release version: `0.2.0`. One release publishes two PyPI projects from the same +Release version: `0.3.0`. One release publishes two PyPI projects from the same commit and version: | Project | Contents | Build | Distributions | diff --git a/native/Cargo.lock b/native/Cargo.lock index 1fc1a76..7c6aea9 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -449,7 +449,7 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tensorcodec-native" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ffmpeg-sys-next", "libc", diff --git a/native/Cargo.toml b/native/Cargo.toml index 43dbdf1..596fedb 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tensorcodec-native" -version = "0.2.0" +version = "0.3.0" edition = "2021" rust-version = "1.88" # ffmpeg-sys-next 9 builds with let chains license = "Apache-2.0" diff --git a/pyproject.toml b/pyproject.toml index eb747cb..b15db3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tensorcodec" -version = "0.2.0" +version = "0.3.0" description = "Video, audio and image codecs with TorchCodec-style APIs and NumPy arrays" readme = "README.md" license = "Apache-2.0" @@ -11,7 +11,7 @@ requires-python = ">=3.10" # manylinux x86_64/aarch64, macOS 14+ arm64 = Darwin 23+); elsewhere only the image codecs work. dependencies = [ "numpy>=1.26", - "tensorcodec-native==0.2.0; platform_python_implementation == 'CPython' and ((sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64' and platform_release >= '23'))", + "tensorcodec-native==0.3.0; platform_python_implementation == 'CPython' and ((sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64' and platform_release >= '23'))", ] classifiers = [ "Development Status :: 3 - Alpha", @@ -24,7 +24,7 @@ classifiers = [ [project.optional-dependencies] images = ["opencv-python-headless>=4.12"] # Unconditional, for platforms outside the marker with a source-built tensorcodec-native. -native = ["tensorcodec-native==0.2.0"] +native = ["tensorcodec-native==0.3.0"] [project.urls] Repository = "https://github.com/MilkClouds/tensorcodec" diff --git a/src/tensorcodec/__init__.py b/src/tensorcodec/__init__.py index 9b619cf..26f3a02 100644 --- a/src/tensorcodec/__init__.py +++ b/src/tensorcodec/__init__.py @@ -3,5 +3,5 @@ from tensorcodec import decoders, transforms from tensorcodec._frame import AudioSamples, Frame, FrameBatch -__version__ = "0.2.0" +__version__ = "0.3.0" __all__ = ["AudioSamples", "Frame", "FrameBatch", "decoders", "transforms"] diff --git a/tests/test_versions.py b/tests/test_versions.py index 364e64b..ee17a84 100644 --- a/tests/test_versions.py +++ b/tests/test_versions.py @@ -9,6 +9,8 @@ tomllib = pytest.importorskip("tomllib") ROOT = Path(__file__).resolve().parents[1] +# The tensorcodec sdist ships tests but not native/. +needs_native_sources = pytest.mark.skipif(not (ROOT / "native").is_dir(), reason="needs a repository checkout") def native_requirements(project): @@ -17,10 +19,13 @@ def native_requirements(project): return [r for r in requirements if r.name == "tensorcodec-native"] +@needs_native_sources def test_versions_are_locked(): project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] version = project["version"] cargo = tomllib.loads((ROOT / "native/Cargo.toml").read_text())["package"]["version"] + lock = tomllib.loads((ROOT / "native/Cargo.lock").read_text())["package"] + assert [p["version"] for p in lock if p["name"] == "tensorcodec-native"] == [cargo] init = re.search(r'^__version__ = "(.+)"$', (ROOT / "src/tensorcodec/__init__.py").read_text(), re.MULTILINE)[1] assert cargo == init == version pins = native_requirements(project) @@ -28,6 +33,7 @@ def test_versions_are_locked(): assert all(str(r.specifier) == f"=={version}" for r in pins) +@needs_native_sources def test_native_license_matches_project_license(): assert (ROOT / "native/LICENSE").read_bytes() == (ROOT / "LICENSE").read_bytes() From 0d34565f3c8b171307790ae5e89daf0a08e61236 Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 02:56:40 +0900 Subject: [PATCH 08/12] Drop the macOS release check from the marker; ship native/ in the sdist pip's vendored packaging 22-25 raises InvalidVersion when comparing platform_release on Linux kernels such as 6.8.0-azure, which would break installs on supported Linux. macOS arm64 older than 14 now tries the native sdist, as it did before the split. The tensorcodec sdist includes native/ so its uv source and version tests work from the archive. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- README.md | 2 +- pyproject.toml | 10 ++++++---- tests/test_versions.py | 16 ++++++---------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index d4d4428..21db6f7 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ See the [compatibility contract](docs/compatibility.md) and `tensorcodec` itself is pure Python; video/audio decoding lives in `tensorcodec-native`, pinned to the same version and installed automatically on those platforms (CPython only). Elsewhere, e.g. Windows and Intel macOS, image codecs work and `VideoDecoder`/`AudioDecoder` raise `ImportError`. - Environment markers cannot detect musl or free-threaded CPython on Linux x86_64/aarch64, so + Environment markers cannot detect musl, free-threaded CPython or macOS older than 14, so installers there still try `tensorcodec-native` (no wheel; its sdist needs FFmpeg 7 and Rust). For image codecs only there, use `pip install --no-deps tensorcodec numpy`. - **Exact seeking:** scans packet timestamps when opening the decoder. Incorrect diff --git a/pyproject.toml b/pyproject.toml index b15db3a..5e8d480 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,11 +7,12 @@ license = "Apache-2.0" license-files = ["LICENSE"] authors = [{ name = "Suhwan Choi", email = "milkclouds00@gmail.com" }] requires-python = ">=3.10" -# tensorcodec-native is pinned to this exact version. The marker matches its wheel tags (CPython abi3: -# manylinux x86_64/aarch64, macOS 14+ arm64 = Darwin 23+); elsewhere only the image codecs work. +# tensorcodec-native is pinned to this exact version. The marker follows its wheel tags (CPython abi3, +# manylinux x86_64/aarch64, macOS 14+ arm64); elsewhere only the image codecs work. No platform_release +# check for macOS 14: pip's packaging 22-25 raises on non-PEP 440 Linux kernel releases. dependencies = [ "numpy>=1.26", - "tensorcodec-native==0.3.0; platform_python_implementation == 'CPython' and ((sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64' and platform_release >= '23'))", + "tensorcodec-native==0.3.0; platform_python_implementation == 'CPython' and ((sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", ] classifiers = [ "Development Status :: 3 - Alpha", @@ -42,7 +43,8 @@ build-backend = "hatchling.build" packages = ["src/tensorcodec"] [tool.hatch.build.targets.sdist] -only-include = ["src", "tests", "docs", "scripts", "benchmarks", "packaging", "README.md", "LICENSE"] +# native/ keeps the sdist a complete checkout for the uv source below and tests/test_versions.py. +only-include = ["src", "native", "tests", "docs", "scripts", "benchmarks", "packaging", "README.md", "LICENSE"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/test_versions.py b/tests/test_versions.py index ee17a84..f2db85a 100644 --- a/tests/test_versions.py +++ b/tests/test_versions.py @@ -9,8 +9,6 @@ tomllib = pytest.importorskip("tomllib") ROOT = Path(__file__).resolve().parents[1] -# The tensorcodec sdist ships tests but not native/. -needs_native_sources = pytest.mark.skipif(not (ROOT / "native").is_dir(), reason="needs a repository checkout") def native_requirements(project): @@ -19,7 +17,6 @@ def native_requirements(project): return [r for r in requirements if r.name == "tensorcodec-native"] -@needs_native_sources def test_versions_are_locked(): project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] version = project["version"] @@ -33,7 +30,6 @@ def test_versions_are_locked(): assert all(str(r.specifier) == f"=={version}" for r in pins) -@needs_native_sources def test_native_license_matches_project_license(): assert (ROOT / "native/LICENSE").read_bytes() == (ROOT / "LICENSE").read_bytes() @@ -45,15 +41,15 @@ def test_native_license_matches_project_license(): ({"sys_platform": "linux", "platform_machine": "aarch64"}, True), ({"sys_platform": "linux", "platform_machine": "armv7l"}, False), ({"sys_platform": "linux", "platform_machine": "x86_64", "platform_python_implementation": "PyPy"}, False), - ({"sys_platform": "darwin", "platform_machine": "arm64", "platform_release": "23.0.0"}, True), - ({"sys_platform": "darwin", "platform_machine": "arm64", "platform_release": "25.1.0"}, True), - ({"sys_platform": "darwin", "platform_machine": "arm64", "platform_release": "22.6.0"}, False), - ({"sys_platform": "darwin", "platform_machine": "x86_64", "platform_release": "24.0.0"}, False), - ({"sys_platform": "win32", "platform_machine": "AMD64", "platform_release": "2025Server"}, False), + ({"sys_platform": "darwin", "platform_machine": "arm64"}, True), + ({"sys_platform": "darwin", "platform_machine": "x86_64"}, False), + ({"sys_platform": "win32", "platform_machine": "AMD64"}, False), ], ) def test_native_marker_matches_published_wheels(environment, expected): project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] (default,) = [r for r in native_requirements(project) if r.marker is not None] - base = default_environment() | {"platform_python_implementation": "CPython", "platform_release": "6.8.0-azure"} + # packaging 22-25 (vendored by pip) raise on version comparisons with releases like "6.8.0-azure". + assert "platform_release" not in str(default.marker) + base = default_environment() | {"platform_python_implementation": "CPython"} assert default.marker.evaluate(base | environment) is expected From fc4e79b34650fb1b16b33e05acdde907b7bae829 Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 03:23:47 +0900 Subject: [PATCH 09/12] Remove the tensorcodec `native` extra Where the marker applies it is a no-op; elsewhere it only triggers the same source build as requesting tensorcodec-native directly, while suggesting that it enables video on Windows. The docs now say to install tensorcodec-native== explicitly to build from source. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- README.md | 4 +++- docs/system_ffmpeg.md | 10 +++++----- pyproject.toml | 2 -- tests/test_versions.py | 11 ++++++----- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 21db6f7..b85de58 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,9 @@ See the [compatibility contract](docs/compatibility.md) and Windows and Intel macOS, image codecs work and `VideoDecoder`/`AudioDecoder` raise `ImportError`. Environment markers cannot detect musl, free-threaded CPython or macOS older than 14, so installers there still try `tensorcodec-native` (no wheel; its sdist needs FFmpeg 7 and Rust). - For image codecs only there, use `pip install --no-deps tensorcodec numpy`. + For image codecs only there, use `pip install --no-deps tensorcodec numpy`. To build video/audio + support from source on a platform without native wheels, install `tensorcodec-native==` explicitly (needs Rust and FFmpeg 7; see [external FFmpeg](docs/system_ffmpeg.md)). - **Exact seeking:** scans packet timestamps when opening the decoder. Incorrect container keyframe flags can produce corrupt frames; repaired input or corrected frame mappings are needed in that case. diff --git a/docs/system_ffmpeg.md b/docs/system_ffmpeg.md index b26d757..8bc9cd8 100644 --- a/docs/system_ffmpeg.md +++ b/docs/system_ffmpeg.md @@ -38,13 +38,13 @@ test -f "$FFMPEG_DIR/include/libavcodec/avcodec.h" test -f "$FFMPEG_DIR/lib/libavcodec.so.61" uv venv -uv pip install --no-binary tensorcodec-native 'tensorcodec[native]' +uv pip install 'tensorcodec==0.3.0' 'tensorcodec-native==0.3.0' --no-binary tensorcodec-native ``` -`tensorcodec` is pure Python; only `tensorcodec-native` is built from source here. -Outside the platforms with native wheels (for example Intel macOS), the `native` -extra requests it explicitly. Releases up to 0.2.0 were a single package, built -from source with `--no-binary tensorcodec`. +`tensorcodec` is pure Python; only `tensorcodec-native` is built from source here, +and naming it explicitly also covers platforms without native wheels (for example +Intel macOS), where `tensorcodec` does not depend on it. Releases up to 0.2.0 were +a single package, built from source with `--no-binary tensorcodec`. The version/build constraint avoids silently selecting an incompatible FFmpeg major. `FFMPEG_DIR` tells the source build where to find headers and libraries; diff --git a/pyproject.toml b/pyproject.toml index 5e8d480..c461b73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,8 +24,6 @@ classifiers = [ [project.optional-dependencies] images = ["opencv-python-headless>=4.12"] -# Unconditional, for platforms outside the marker with a source-built tensorcodec-native. -native = ["tensorcodec-native==0.3.0"] [project.urls] Repository = "https://github.com/MilkClouds/tensorcodec" diff --git a/tests/test_versions.py b/tests/test_versions.py index f2db85a..7f7f1a6 100644 --- a/tests/test_versions.py +++ b/tests/test_versions.py @@ -12,7 +12,7 @@ def native_requirements(project): - groups = [project["dependencies"], *project["optional-dependencies"].values()] + groups = [project["dependencies"], *project.get("optional-dependencies", {}).values()] requirements = [Requirement(item) for group in groups for item in group] return [r for r in requirements if r.name == "tensorcodec-native"] @@ -25,9 +25,10 @@ def test_versions_are_locked(): assert [p["version"] for p in lock if p["name"] == "tensorcodec-native"] == [cargo] init = re.search(r'^__version__ = "(.+)"$', (ROOT / "src/tensorcodec/__init__.py").read_text(), re.MULTILINE)[1] assert cargo == init == version - pins = native_requirements(project) - assert len(pins) == 2 - assert all(str(r.specifier) == f"=={version}" for r in pins) + # Only the marked dependency: an extra would just trigger the same source build elsewhere. + (pin,) = native_requirements(project) + assert str(pin.specifier) == f"=={version}" + assert pin.marker is not None def test_native_license_matches_project_license(): @@ -48,7 +49,7 @@ def test_native_license_matches_project_license(): ) def test_native_marker_matches_published_wheels(environment, expected): project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] - (default,) = [r for r in native_requirements(project) if r.marker is not None] + (default,) = native_requirements(project) # packaging 22-25 (vendored by pip) raise on version comparisons with releases like "6.8.0-azure". assert "platform_release" not in str(default.marker) base = default_environment() | {"platform_python_implementation": "CPython"} From b1209f4a6c9e049de9614e69242b0d22b0cced52 Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 03:23:47 +0900 Subject: [PATCH 10/12] One reusable workflow per distribution, called by CI and publish build-python.yml builds and tests tensorcodec (artifact dist-python); build-native.yml builds and validates the manylinux x86_64/aarch64 and macOS arm64 wheels and the sdist of tensorcodec-native (dist-native-*). CI now runs the Linux release builds too: on every push and on PRs touching what they build or validate. publish.yml calls both, checks the release set (one version, six files) and uploads native first. The duplicated sdist steps and source.yml are gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- .github/workflows/build-native.yml | 202 ++++++++++++++++++ ...pure-python-wheel.yml => build-python.yml} | 11 +- .github/workflows/ci.yml | 31 ++- .github/workflows/macos-wheels.yml | 79 ------- .github/workflows/publish.yml | 141 +++--------- .github/workflows/source.yml | 21 -- docs/package_size.md | 8 +- docs/releasing.md | 79 ++++--- 8 files changed, 315 insertions(+), 257 deletions(-) create mode 100644 .github/workflows/build-native.yml rename .github/workflows/{pure-python-wheel.yml => build-python.yml} (91%) delete mode 100644 .github/workflows/macos-wheels.yml delete mode 100644 .github/workflows/source.yml diff --git a/.github/workflows/build-native.yml b/.github/workflows/build-native.yml new file mode 100644 index 0000000..6895709 --- /dev/null +++ b/.github/workflows/build-native.yml @@ -0,0 +1,202 @@ +# Builds and validates every tensorcodec-native distribution. Callers must run build-python.yml first: +# validation installs its dist-python wheel next to the native wheel. +name: Build tensorcodec-native +on: + workflow_call: +permissions: + contents: read +jobs: + linux: + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + target: x86_64 + - runner: ubuntu-24.04-arm + target: aarch64 + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.2.0 + with: + python-version: '3.12' + - uses: actions/cache/restore@v6 + id: native-cache + with: + path: .native-deps + key: manylinux2014-${{ matrix.target }}-${{ hashFiles('scripts/build_nasm.sh', 'scripts/build_openssl.sh', 'scripts/build_dav1d.sh', 'scripts/build_ffmpeg.sh') }} + - name: Build manylinux2014 wheel + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: '2014' + command: build + args: -m native/Cargo.toml --release --locked --auditwheel repair --out dist + before-script-linux: | + /opt/python/cp310-cp310/bin/python -m pip install libclang==18.1.1 + export LIBCLANG_PATH="$(/opt/python/cp310-cp310/bin/python -c 'from clang import cindex; print(cindex.Config.library_path)')" + export BINDGEN_EXTRA_CLANG_ARGS="-I$(gcc -print-file-name=include)" + prefix="$PWD/.native-deps" + export PKG_CONFIG_PATH="$prefix/openssl/lib/pkgconfig" + export LD_LIBRARY_PATH="$prefix/openssl/lib:$prefix/ffmpeg/lib" + if [ ! -f "$prefix/ready" ]; then + yum install -y perl-core + /opt/python/cp310-cp310/bin/python -m pip install meson==1.12.1 ninja==1.13.2 + export PATH="/opt/python/cp310-cp310/bin:$PATH" + if [ "$(uname -m)" = x86_64 ]; then + bash scripts/build_nasm.sh "$prefix/nasm" + export PATH="$prefix/nasm/bin:$PATH" + fi + bash scripts/build_openssl.sh "$prefix/openssl" + bash scripts/build_ffmpeg.sh "$prefix/ffmpeg" + touch "$prefix/ready" + fi + export FFMPEG_DIR="$prefix/ffmpeg" + - name: Check native cache completeness + if: always() + id: native-ready + run: | + if [ -f .native-deps/ready ]; then echo 'ready=true' >> "$GITHUB_OUTPUT"; fi + - uses: actions/cache/save@v6 + if: always() && steps.native-ready.outputs.ready == 'true' && steps.native-cache.outputs.cache-hit != 'true' + with: + path: .native-deps + key: ${{ steps.native-cache.outputs.cache-primary-key }} + - name: Check final wheel size + run: uv run --no-project python scripts/check_wheel_size.py dist/*.whl --output reports/wheel-size.json + - uses: actions/upload-artifact@v7 + if: always() + with: + name: wheel-size-linux-${{ matrix.target }} + path: reports/wheel-size.json + if-no-files-found: warn + - uses: actions/download-artifact@v8 + with: + name: dist-python + path: python-dist + - name: Install validation tools and wheels + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg + uv venv + uv pip install pytest numpy twine 'opencv-python-headless>=4.12' 'Pillow>=11.3' dist/*.whl python-dist/*.whl + - name: Generate baseline runtime fixtures + run: uv run --no-sync python scripts/check_wheel_runtime.py generate .runtime-fixtures + - name: Test on glibc 2.17 with Python 3.10 and 3.13 + run: | + docker run --rm -v "$PWD:/io:ro" quay.io/pypa/manylinux2014_${{ matrix.target }} bash -euxc ' + for abi in cp310-cp310 cp313-cp313; do + python="/opt/python/$abi/bin/python" + "$python" -m venv "/tmp/$abi" + runtime="/tmp/$abi/bin/python" + if [ "$abi" = cp310-cp310 ]; then "$runtime" -m pip install --only-binary=:all: numpy==1.26.4; fi + "$runtime" -m pip install --only-binary=:all: /io/dist/*.whl /io/python-dist/*.whl + env -u LD_LIBRARY_PATH "$runtime" /io/scripts/check_wheel_runtime.py check /io/.runtime-fixtures + done + ' + - name: Install pinned reference + run: uv pip install torch==2.14.1 torchcodec==0.17.0 --index-url https://download.pytorch.org/whl/cpu + - name: Use bundled FFmpeg for the reference + run: uv run --no-sync python scripts/configure_oracle_ffmpeg.py + - name: Validate distribution and playback + run: | + uv run --no-sync python -m twine check --strict dist/* + uv run --no-sync pytest tests/test_video_contract.py tests/test_audio_contract.py --backend torchcodec + uv run --no-sync pytest --compare + - uses: actions/upload-artifact@v7 + with: + name: dist-native-linux-${{ matrix.target }} + path: dist/*.whl + if-no-files-found: error + macos: + strategy: + fail-fast: false + matrix: + include: + - runner: macos-15 + arch: arm64 + runs-on: ${{ matrix.runner }} + env: + MACOSX_DEPLOYMENT_TARGET: '14.0' + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.2.0 + with: + python-version: '3.12' + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: native + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.81.0 + run-install: false + global-environments: ffmpeg=7.1.1=gpl_* + global-cache: true + - name: Install build tools + run: | + brew install nasm pkg-config coreutils meson ninja + uv venv + uv pip install numpy pytest 'opencv-python-headless>=4.12' 'Pillow>=11.3' 'maturin>=1.8,<2' delocate twine + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + echo "LIBCLANG_PATH=$(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib" >> "$GITHUB_ENV" + echo "$HOME/.pixi/envs/ffmpeg/bin" >> "$GITHUB_PATH" + - uses: actions/cache/restore@v6 + id: native-cache + with: + path: .native-deps/macos + key: macos14-${{ matrix.arch }}-${{ hashFiles('scripts/build_openssl.sh', 'scripts/build_dav1d.sh', 'scripts/build_ffmpeg.sh', 'scripts/build_macos_wheel.sh') }} + - name: Build and bundle wheel + run: bash scripts/build_macos_wheel.sh + - uses: actions/cache/save@v6 + if: always() && steps.native-cache.outputs.cache-hit != 'true' && hashFiles('.native-deps/macos/ready') != '' + with: + path: .native-deps/macos + key: ${{ steps.native-cache.outputs.cache-primary-key }} + - name: Hide build-time FFmpeg libraries + run: mv .native-deps/macos .native-deps/macos-build-only + - uses: actions/download-artifact@v8 + with: + name: dist-python + path: python-dist + - name: Validate installed wheels + run: | + uv pip install dist/*.whl python-dist/*.whl + python -m twine check --strict dist/* + pytest + python scripts/check_wheel_runtime.py generate .runtime-fixtures + - name: Test clean Python 3.10 and 3.13 environments + run: | + for version in 3.10 3.13; do + uv venv "/tmp/runtime-$version" --python "$version" + uv pip install --python "/tmp/runtime-$version/bin/python" dist/*.whl python-dist/*.whl + "/tmp/runtime-$version/bin/python" scripts/check_wheel_runtime.py check .runtime-fixtures + done + - name: Restore build cache + if: always() + run: | + if [ -d .native-deps/macos-build-only ]; then + mv .native-deps/macos-build-only .native-deps/macos + fi + - uses: actions/upload-artifact@v7 + with: + name: dist-native-macos-${{ matrix.arch }} + path: dist/*.whl + if-no-files-found: error + sdist: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.2.0 + with: + python-version: '3.12' + - name: Build and check source distribution + run: | + uvx --from 'maturin>=1.8,<2' maturin sdist -m native/Cargo.toml --out dist + uvx twine check --strict dist/* + - uses: actions/upload-artifact@v7 + with: + name: dist-native-sdist + path: dist/*.tar.gz + if-no-files-found: error diff --git a/.github/workflows/pure-python-wheel.yml b/.github/workflows/build-python.yml similarity index 91% rename from .github/workflows/pure-python-wheel.yml rename to .github/workflows/build-python.yml index ad4e92a..db0c6f7 100644 --- a/.github/workflows/pure-python-wheel.yml +++ b/.github/workflows/build-python.yml @@ -1,4 +1,6 @@ -name: tensorcodec distributions +# Builds the pure-Python tensorcodec wheel and sdist (artifact dist-python) and tests image codecs +# on platforms that get no tensorcodec-native. +name: Build tensorcodec on: workflow_call: permissions: @@ -24,11 +26,10 @@ jobs: EOF - uses: actions/upload-artifact@v7 with: - name: pypi-distributions-pure + name: dist-python path: dist/* if-no-files-found: error - # Platforms without tensorcodec-native wheels: the marker skips it and image codecs still work. - test: + test-without-native: needs: build strategy: fail-fast: false @@ -51,7 +52,7 @@ jobs: python-version: ${{ matrix.python }} - uses: actions/download-artifact@v8 with: - name: pypi-distributions-pure + name: dist-python path: dist # Fixtures need libaom; Homebrew's FFmpeg lacks it, so use conda-forge's like the other jobs. - uses: prefix-dev/setup-pixi@v0.10.2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d14434..bba9287 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,10 +10,33 @@ concurrency: group: ci-${{ github.ref }} cancel-in-progress: true jobs: - macos: - uses: ./.github/workflows/macos-wheels.yml - pure-python: - uses: ./.github/workflows/pure-python-wheel.yml + python: + uses: ./.github/workflows/build-python.yml + changes: + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + outputs: + native: ${{ steps.filter.outputs.native }} + steps: + - uses: actions/checkout@v7 + - uses: dorny/paths-filter@v4 + id: filter + with: + filters: | + native: + - 'native/**' + - 'scripts/**' + - 'tests/**' + - '.github/workflows/**' + - 'pyproject.toml' + # Release builds of every tensorcodec-native wheel: on pushes (including main) and on PRs that touch + # what they build or validate. Other PRs rely on the `test` job's development build. + native: + needs: [python, changes] + if: github.event_name != 'pull_request' || needs.changes.outputs.native == 'true' + uses: ./.github/workflows/build-native.yml test: runs-on: ubuntu-24.04 steps: diff --git a/.github/workflows/macos-wheels.yml b/.github/workflows/macos-wheels.yml deleted file mode 100644 index b6848db..0000000 --- a/.github/workflows/macos-wheels.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: macOS wheels -on: - workflow_call: -permissions: - contents: read -jobs: - build: - strategy: - fail-fast: false - matrix: - include: - - runner: macos-15 - arch: arm64 - runs-on: ${{ matrix.runner }} - env: - MACOSX_DEPLOYMENT_TARGET: '14.0' - steps: - - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v10.2.0 - with: - python-version: '3.12' - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - with: - workspaces: native - - uses: prefix-dev/setup-pixi@v0.10.2 - with: - pixi-version: v0.81.0 - run-install: false - global-environments: ffmpeg=7.1.1=gpl_* - global-cache: true - - name: Install build tools - run: | - brew install nasm pkg-config coreutils meson ninja - uv venv - uv pip install numpy pytest 'opencv-python-headless>=4.12' 'Pillow>=11.3' 'maturin>=1.8,<2' delocate twine - echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - echo "LIBCLANG_PATH=$(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib" >> "$GITHUB_ENV" - echo "$HOME/.pixi/envs/ffmpeg/bin" >> "$GITHUB_PATH" - - uses: actions/cache/restore@v6 - id: native-cache - with: - path: .native-deps/macos - key: macos14-${{ matrix.arch }}-${{ hashFiles('scripts/build_openssl.sh', 'scripts/build_dav1d.sh', 'scripts/build_ffmpeg.sh', 'scripts/build_macos_wheel.sh') }} - - name: Build and bundle wheel - run: bash scripts/build_macos_wheel.sh - - uses: actions/cache/save@v6 - if: always() && steps.native-cache.outputs.cache-hit != 'true' && hashFiles('.native-deps/macos/ready') != '' - with: - path: .native-deps/macos - key: ${{ steps.native-cache.outputs.cache-primary-key }} - - name: Hide build-time FFmpeg libraries - run: mv .native-deps/macos .native-deps/macos-build-only - - name: Build tensorcodec wheel for validation - run: uv build --wheel --out-dir pure-dist - - name: Validate installed wheel - run: | - uv pip install dist/*.whl pure-dist/*.whl - python -m twine check dist/* - pytest - python scripts/check_wheel_runtime.py generate .runtime-fixtures - - name: Test clean Python 3.10 and 3.13 environments - run: | - for version in 3.10 3.13; do - uv venv "/tmp/runtime-$version" --python "$version" - uv pip install --python "/tmp/runtime-$version/bin/python" dist/*.whl pure-dist/*.whl - "/tmp/runtime-$version/bin/python" scripts/check_wheel_runtime.py check .runtime-fixtures - done - - name: Restore build cache - if: always() - run: | - if [ -d .native-deps/macos-build-only ]; then - mv .native-deps/macos-build-only .native-deps/macos - fi - - uses: actions/upload-artifact@v7 - with: - name: pypi-distributions-native-macos-${{ matrix.arch }} - path: dist/*.whl - if-no-files-found: error diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 68a2c1a..e2a0db2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,116 +12,37 @@ concurrency: group: pypi-publish cancel-in-progress: false jobs: - macos: - uses: ./.github/workflows/macos-wheels.yml - pure-python: - uses: ./.github/workflows/pure-python-wheel.yml - build: - strategy: - fail-fast: false - matrix: - include: - - runner: ubuntu-24.04 - target: x86_64 - - runner: ubuntu-24.04-arm - target: aarch64 - runs-on: ${{ matrix.runner }} + python: + uses: ./.github/workflows/build-python.yml + native: + needs: python + uses: ./.github/workflows/build-native.yml + check: + needs: [python, native] + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v10.2.0 - with: - python-version: '3.12' - - uses: actions/cache/restore@v6 - id: native-cache - with: - path: .native-deps - key: manylinux2014-${{ matrix.target }}-${{ hashFiles('scripts/build_nasm.sh', 'scripts/build_openssl.sh', 'scripts/build_dav1d.sh', 'scripts/build_ffmpeg.sh') }} - - name: Build portable Linux wheel - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - manylinux: '2014' - command: build - args: -m native/Cargo.toml --release --locked --auditwheel repair --out dist - before-script-linux: | - /opt/python/cp310-cp310/bin/python -m pip install libclang==18.1.1 - export LIBCLANG_PATH="$(/opt/python/cp310-cp310/bin/python -c 'from clang import cindex; print(cindex.Config.library_path)')" - export BINDGEN_EXTRA_CLANG_ARGS="-I$(gcc -print-file-name=include)" - prefix="$PWD/.native-deps" - export PKG_CONFIG_PATH="$prefix/openssl/lib/pkgconfig" - export LD_LIBRARY_PATH="$prefix/openssl/lib:$prefix/ffmpeg/lib" - if [ ! -f "$prefix/ready" ]; then - yum install -y perl-core - /opt/python/cp310-cp310/bin/python -m pip install meson==1.12.1 ninja==1.13.2 - export PATH="/opt/python/cp310-cp310/bin:$PATH" - if [ "$(uname -m)" = x86_64 ]; then - bash scripts/build_nasm.sh "$prefix/nasm" - export PATH="$prefix/nasm/bin:$PATH" - fi - bash scripts/build_openssl.sh "$prefix/openssl" - bash scripts/build_ffmpeg.sh "$prefix/ffmpeg" - touch "$prefix/ready" - fi - export FFMPEG_DIR="$prefix/ffmpeg" - - name: Check native cache completeness - if: always() - id: native-ready - run: | - if [ -f .native-deps/ready ]; then echo 'ready=true' >> "$GITHUB_OUTPUT"; fi - - uses: actions/cache/save@v6 - if: always() && steps.native-ready.outputs.ready == 'true' && steps.native-cache.outputs.cache-hit != 'true' - with: - path: .native-deps - key: ${{ steps.native-cache.outputs.cache-primary-key }} - - name: Check final wheel size - run: uv run --no-project python scripts/check_wheel_size.py dist/*.whl --output reports/wheel-size.json - - uses: actions/upload-artifact@v7 - if: always() + - uses: actions/download-artifact@v8 with: - name: wheel-size-${{ matrix.target }} - path: reports/wheel-size.json - if-no-files-found: warn - - name: Build tensorcodec-native source distribution - if: matrix.target == 'x86_64' - run: uvx --from 'maturin>=1.8,<2' maturin sdist -m native/Cargo.toml --out dist - - name: Build tensorcodec wheel for validation - run: uv build --wheel --out-dir pure-dist - - name: Install validation tools and wheel + pattern: dist-* + path: artifacts + - name: Check the release set run: | - sudo apt-get update - sudo apt-get install -y ffmpeg - uv venv - uv pip install pytest numpy twine 'opencv-python-headless>=4.12' 'Pillow>=11.3' dist/*.whl pure-dist/*.whl - - name: Generate baseline runtime fixtures - run: uv run --no-sync python scripts/check_wheel_runtime.py generate .runtime-fixtures - - name: Test on glibc 2.17 with Python 3.10 and 3.13 - run: | - docker run --rm -v "$PWD:/io:ro" quay.io/pypa/manylinux2014_${{ matrix.target }} bash -euxc ' - for abi in cp310-cp310 cp313-cp313; do - python="/opt/python/$abi/bin/python" - "$python" -m venv "/tmp/$abi" - runtime="/tmp/$abi/bin/python" - if [ "$abi" = cp310-cp310 ]; then "$runtime" -m pip install --only-binary=:all: numpy==1.26.4; fi - "$runtime" -m pip install --only-binary=:all: /io/dist/*.whl /io/pure-dist/*.whl - env -u LD_LIBRARY_PATH "$runtime" /io/scripts/check_wheel_runtime.py check /io/.runtime-fixtures - done - ' - - name: Install pinned reference - run: uv pip install torch==2.14.1 torchcodec==0.17.0 --index-url https://download.pytorch.org/whl/cpu - - name: Use bundled FFmpeg for the reference - run: uv run --no-sync python scripts/configure_oracle_ffmpeg.py - - name: Validate distribution and playback - run: | - uv run --no-sync python -m twine check dist/* - uv run --no-sync pytest tests/test_video_contract.py tests/test_audio_contract.py --backend torchcodec - uv run --no-sync pytest --compare - - uses: actions/upload-artifact@v7 - with: - name: pypi-distributions-native-${{ matrix.target }} - path: dist/* - if-no-files-found: error + python3 - <<'EOF' + import pathlib, re + files = sorted(p for p in pathlib.Path("artifacts").rglob("*") if p.is_file()) + names = [f.name for f in files] + print("\n".join(names)) + versions = {re.match(r"tensorcodec(?:_native)?-([^-]+?)(?:\.tar\.gz|-)", n)[1] for n in names} + assert len(versions) == 1, f"mixed versions: {versions}" + expected = ["tensorcodec-*-py3-none-any.whl", "tensorcodec-*.tar.gz", "tensorcodec_native-*.tar.gz", + "tensorcodec_native-*manylinux*x86_64.whl", "tensorcodec_native-*manylinux*aarch64.whl", + "tensorcodec_native-*macosx*arm64.whl"] + for pattern in expected: + assert len([f for f in files if f.match(pattern)]) == 1, pattern + assert len(files) == len(expected), names + EOF publish: - needs: [build, macos, pure-python] + needs: check if: inputs.publish runs-on: ubuntu-24.04 environment: @@ -133,13 +54,13 @@ jobs: steps: - uses: actions/download-artifact@v8 with: - pattern: pypi-distributions-native-* + pattern: dist-native-* merge-multiple: true path: native-dist - uses: actions/download-artifact@v8 with: - name: pypi-distributions-pure - path: pure-dist + name: dist-python + path: python-dist # Native first: tensorcodec pins tensorcodec-native exactly, so it must not appear before its pin exists. - name: Publish tensorcodec-native uses: pypa/gh-action-pypi-publish@release/v1 @@ -148,7 +69,7 @@ jobs: - name: Publish tensorcodec uses: pypa/gh-action-pypi-publish@release/v1 with: - packages-dir: pure-dist/ + packages-dir: python-dist/ update-size-docs: needs: publish if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/source.yml b/.github/workflows/source.yml deleted file mode 100644 index c5cd0b2..0000000 --- a/.github/workflows/source.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Source artifact -on: - workflow_dispatch: -permissions: - contents: read -jobs: - source: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: '3.12' - - run: | - python -m pip install 'maturin>=1.8,<2' build - maturin sdist -m native/Cargo.toml --out dist - python -m build --sdist --outdir dist - - uses: actions/upload-artifact@v7 - with: - name: tensorcodec-source - path: dist/*.tar.gz diff --git a/docs/package_size.md b/docs/package_size.md index 3da3f01..5763a83 100644 --- a/docs/package_size.md +++ b/docs/package_size.md @@ -25,10 +25,10 @@ uv run --no-project python scripts/check_wheel_size.py dist/*.whl --output repor ``` Each architecture is checked independently. Exactly reaching a limit passes; -exceeding either limit fails. The release workflow runs this after `auditwheel -repair`, before uploading distributions. JSON reports are separate artifacts, not -files in `dist/`. Actions summaries include changes from the committed published -baseline. Ordinary CI tests the checker without building FFmpeg from source. +exceeding either limit fails. The `linux` job of `build-native.yml` runs this after +`auditwheel repair`, in CI and before every upload. JSON reports are separate +artifacts, not files in `dist/`. Actions summaries include changes from the +committed published baseline. The `test` job also unit-tests the checker. Before changing a limit, explain the feature, the measured byte increase on both architectures and why a smaller configuration would not provide the same behavior. diff --git a/docs/releasing.md b/docs/releasing.md index a0de70a..b0d1bdb 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -9,9 +9,8 @@ commit and version: | `tensorcodec-native` | Rust/FFmpeg extension (import `tensorcodec_native`) | maturin (`native/`) | platform wheels, sdist | `tensorcodec` requires `tensorcodec-native==` behind an environment -marker matching the native wheel tags, and the `native` extra requests it -unconditionally. `tests/test_versions.py` fails CI unless `pyproject.toml`, -`native/Cargo.toml` (the native version source), both pins and +marker matching the native wheel tags. `tests/test_versions.py` fails CI unless +`pyproject.toml`, `native/Cargo.toml` (the native version source), `native/Cargo.lock`, the pin and `tensorcodec.__version__` agree. At runtime `VideoDecoder`/`AudioDecoder` reject a `tensorcodec-native` whose version differs. @@ -23,8 +22,8 @@ runtime. Windows wheels are not provided; there, `tensorcodec` provides image codecs only. Bundled-library notices ship only with `tensorcodec-native` (`native/licenses/`). -To release, set the new version in `pyproject.toml` (project version and both -`tensorcodec-native` pins), `native/Cargo.toml` (then `cargo update -p +To release, set the new version in `pyproject.toml` (project version and the +`tensorcodec-native` pin), `native/Cargo.toml` (then `cargo update -p tensorcodec-native --manifest-path native/Cargo.toml` to refresh the lock file) and `src/tensorcodec/__init__.py`. @@ -46,29 +45,44 @@ with the same fields. Manage this configuration in each project's PyPI Publishin or renaming the repository or workflow. Publishing uses GitHub OIDC; no API token is needed. Repository visibility does not need to change for a release. +## Workflows + +Each distribution has one reusable workflow; CI and the release call both. + +| Workflow | Jobs | Artifacts | +| --- | --- | --- | +| `build-python.yml` | `build`: `tensorcodec` wheel and sdist (`python -m build`, `twine check --strict`); `test-without-native`: installs it on Windows and Intel macOS without `tensorcodec-native` and runs the image tests (OpenCV 5 and the 4.12 lower bound) | `dist-python` | +| `build-native.yml` | `linux` (x86_64, aarch64: manylinux2014 via maturin-action), `macos` (arm64), `sdist`; each wheel job validates with the `dist-python` wheel, so callers run `build-python.yml` first | `dist-native-linux-{x86_64,aarch64}`, `dist-native-macos-arm64`, `dist-native-sdist`, `wheel-size-linux-*` | +| `ci.yml` | `python`, `native`, plus `test` (development build against conda-forge FFmpeg, Clippy, oracle comparison) | — | +| `publish.yml` | `python`, `native`, `check` (one version, exactly the six expected files), `publish`, `update-size-docs` | — | + +`ci.yml` runs `native` on every push, including `main`, and on pull requests that +touch `native/`, `scripts/`, `tests/`, workflows or `pyproject.toml`; a release +commit has therefore already passed the same native builds and validation. + ## Release -Run the **Publish to PyPI** workflow on `main`. It builds the portable Linux/macOS -`tensorcodec-native` wheels and sdist plus the `tensorcodec` wheel and sdist, checks package metadata, validates the pinned oracle -and compares playback before uploading through PyPI Trusted Publishing. It uses -the existing GitHub `pypi` environment. `tensorcodec-native` is uploaded first, so the -exact pin in `tensorcodec` never points at a missing release; if the second upload -fails, rerun only it (`twine upload` of the `pypi-distributions-pure` artifact) rather -than the whole workflow. Publication fails if authorization is missing, tests fail, -or the version has already been uploaded. +Run the **Publish to PyPI** workflow on `main`. It runs both build workflows, checks the +release set and uploads through PyPI Trusted Publishing using the existing GitHub +`pypi` environment. `tensorcodec-native` is uploaded first, so the exact pin in +`tensorcodec` never points at a missing release; if the second upload fails, upload +the `dist-python` artifact with `twine upload` rather than rerunning the whole +workflow. Publication fails if authorization is missing, tests fail, or the version +has already been uploaded. ```sh gh workflow run publish.yml --repo epishelf/tensorcodec --ref main ``` -For a build and full validation without uploading, pass `--field publish=false`. +For a build and full validation without uploading, pass `--field publish=false`; +the run's `dist-*` artifacts are then the release candidates. Check the workflow, https://pypi.org/project/tensorcodec/ and https://pypi.org/project/tensorcodec-native/ before reporting success. Verify a fresh `uv pip install tensorcodec==` pulls the matching `tensorcodec-native` and decodes video without Torch/PyAV on both Linux architectures and macOS arm64, and -that it installs without `tensorcodec-native` on Windows. Update the version before subsequent releases; -PyPI versions cannot be overwritten. +that it installs without `tensorcodec-native` on Windows. Update the version before +subsequent releases; PyPI versions cannot be overwritten. The local Linux build is reproducible using `scripts/build_linux_wheel.sh` inside `quay.io/pypa/manylinux2014_x86_64` or @@ -76,31 +90,28 @@ The local Linux build is reproducible using `scripts/build_linux_wheel.sh` insid Both native source archives are version- and checksum-pinned. Their licensing and source links are recorded in `native/licenses/README.md`. -## CI versus release builds +## Development versus release builds -- Ordinary CI uses prebuilt conda-forge FFmpeg 7.1.1 through Pixi, including its +- The `test` job uses prebuilt conda-forge FFmpeg 7.1.1 through Pixi, including its headers and shared libraries. It builds only the `tensorcodec-native` extension and installs `tensorcodec` from the checkout. -- The `tensorcodec` wheel is built once with `python -m build` and checked with - `twine check --strict`; Windows and Intel macOS jobs install it without - `tensorcodec-native` and run the image tests (one with OpenCV 5, one with 4.x). -- PyPI wheels use the smaller LGPL FFmpeg 7.1.5 build plus OpenSSL 3.5.9. - Their native prefix is cached by architecture, glibc baseline and build-script - checksums. This preserves the wheel's codec set, dependency size and licensing rather than bundling the full - conda-forge dependency graph. -- Release validation installs each repaired `tensorcodec-native` wheel together with - the `tensorcodec` wheel on glibc 2.17 with Python 3.10 - and 3.13 and decodes video/audio without Torch, PyAV or a system FFmpeg. Python - 3.10 also checks the minimum NumPy line (1.26.4). Native - x86_64 and ARM64 runners also run the full pinned playback oracle comparison. -- Release validation still tests the installed repaired wheel. The fixture CLI - can be FFmpeg 6 or 7; fixtures explicitly remove auxiliary sentinel packets. +- Release wheels (`build-native.yml`) use the smaller LGPL FFmpeg 7.1.5 build plus + OpenSSL 3.5.9. Their native prefix is cached by architecture, glibc baseline and + build-script checksums. This preserves the wheel's codec set, dependency size and + licensing rather than bundling the full conda-forge dependency graph. +- Each repaired Linux wheel is installed with the `tensorcodec` wheel on glibc 2.17 + with Python 3.10 and 3.13 and decodes video/audio without Torch, PyAV or a system + FFmpeg. Python 3.10 also checks the minimum NumPy line (1.26.4). The x86_64 and + ARM64 runners also run the full pinned playback oracle comparison; the macOS job + runs the test suite and clean Python 3.10/3.13 environments. +- The fixture CLI can be FFmpeg 6 or 7; fixtures explicitly remove auxiliary + sentinel packets. ## Size checks and published comparison Final repaired `tensorcodec-native` wheels must stay within 15 MiB download and 35 MiB unpacked per -architecture. The build job checks `packaging/size-policy.json` and uploads a -separate `wheel-size-*` report, including differences from the last published +architecture. The `linux` job checks `packaging/size-policy.json` and uploads a +separate `wheel-size-linux-*` report, including differences from the last published baseline. Size reports must not be placed in `dist/`. After a successful publication, the `update-size-docs` job measures hash-verified From 2bc8fe5627a99d541e7711bea9a135a4c2537482 Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 03:38:13 +0900 Subject: [PATCH 11/12] Rename tensorcodec-native to tensorcodec-av The compiled distribution is the audio/video codec implementation; images live in tensorcodec. Renamed consistently: PyPI name, import package tensorcodec_av with extension module _av, Cargo package and lib, the native/ directory (now av/), the pin, ImportError messages, tests, workflows (build-av.yml, dist-av-* artifacts, av jobs) and docs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- .github/dependabot.yml | 2 +- .../{build-native.yml => build-av.yml} | 18 +++--- .github/workflows/build-python.yml | 10 ++-- .github/workflows/ci.yml | 28 +++++----- .github/workflows/publish.yml | 24 ++++---- .gitignore | 2 +- README.md | 14 ++--- {native => av}/Cargo.lock | 2 +- {native => av}/Cargo.toml | 4 +- {native => av}/LICENSE | 0 {native => av}/README.md | 6 +- {native => av}/licenses/FFmpeg-GPL-3.0.txt | 0 {native => av}/licenses/FFmpeg-LGPL-3.0.txt | 0 {native => av}/licenses/FFmpeg-NOTICE.md | 0 {native => av}/licenses/OpenSSL.txt | 0 {native => av}/licenses/README.md | 2 +- {native => av}/licenses/Zstandard.txt | 0 {native => av}/licenses/dav1d.txt | 0 {native => av}/pyproject.toml | 6 +- .../python/tensorcodec_av}/__init__.py | 0 {native => av}/src/ffmpeg.rs | 0 {native => av}/src/lib.rs | 2 +- docs/images.md | 4 +- docs/package_size.md | 4 +- docs/releasing.md | 56 +++++++++---------- docs/system_ffmpeg.md | 14 ++--- pyproject.toml | 10 ++-- scripts/build_linux_wheel.sh | 2 +- scripts/build_macos_wheel.sh | 2 +- scripts/configure_oracle_ffmpeg.py | 6 +- scripts/update_size_comparison.py | 2 +- src/tensorcodec/decoders/_decoder.py | 22 ++++---- ...native_optional.py => test_av_optional.py} | 24 ++++---- tests/test_versions.py | 22 ++++---- 34 files changed, 143 insertions(+), 145 deletions(-) rename .github/workflows/{build-native.yml => build-av.yml} (93%) rename {native => av}/Cargo.lock (99%) rename {native => av}/Cargo.toml (90%) rename {native => av}/LICENSE (100%) rename {native => av}/README.md (63%) rename {native => av}/licenses/FFmpeg-GPL-3.0.txt (100%) rename {native => av}/licenses/FFmpeg-LGPL-3.0.txt (100%) rename {native => av}/licenses/FFmpeg-NOTICE.md (100%) rename {native => av}/licenses/OpenSSL.txt (100%) rename {native => av}/licenses/README.md (91%) rename {native => av}/licenses/Zstandard.txt (100%) rename {native => av}/licenses/dav1d.txt (100%) rename {native => av}/pyproject.toml (86%) rename {native/python/tensorcodec_native => av/python/tensorcodec_av}/__init__.py (100%) rename {native => av}/src/ffmpeg.rs (100%) rename {native => av}/src/lib.rs (99%) rename tests/{test_native_optional.py => test_av_optional.py} (65%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b2ab8f9..c49bf3f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,6 @@ updates: interval: "monthly" - package-ecosystem: "cargo" - directory: "/native" + directory: "/av" schedule: interval: "monthly" diff --git a/.github/workflows/build-native.yml b/.github/workflows/build-av.yml similarity index 93% rename from .github/workflows/build-native.yml rename to .github/workflows/build-av.yml index 6895709..57b7288 100644 --- a/.github/workflows/build-native.yml +++ b/.github/workflows/build-av.yml @@ -1,6 +1,6 @@ -# Builds and validates every tensorcodec-native distribution. Callers must run build-python.yml first: -# validation installs its dist-python wheel next to the native wheel. -name: Build tensorcodec-native +# Builds and validates every tensorcodec-av distribution. Callers must run build-python.yml first: +# validation installs its dist-python wheel next to each tensorcodec-av wheel. +name: Build tensorcodec-av on: workflow_call: permissions: @@ -32,7 +32,7 @@ jobs: target: ${{ matrix.target }} manylinux: '2014' command: build - args: -m native/Cargo.toml --release --locked --auditwheel repair --out dist + args: -m av/Cargo.toml --release --locked --auditwheel repair --out dist before-script-linux: | /opt/python/cp310-cp310/bin/python -m pip install libclang==18.1.1 export LIBCLANG_PATH="$(/opt/python/cp310-cp310/bin/python -c 'from clang import cindex; print(cindex.Config.library_path)')" @@ -106,7 +106,7 @@ jobs: uv run --no-sync pytest --compare - uses: actions/upload-artifact@v7 with: - name: dist-native-linux-${{ matrix.target }} + name: dist-av-linux-${{ matrix.target }} path: dist/*.whl if-no-files-found: error macos: @@ -127,7 +127,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 with: - workspaces: native + workspaces: av - uses: prefix-dev/setup-pixi@v0.10.2 with: pixi-version: v0.81.0 @@ -181,7 +181,7 @@ jobs: fi - uses: actions/upload-artifact@v7 with: - name: dist-native-macos-${{ matrix.arch }} + name: dist-av-macos-${{ matrix.arch }} path: dist/*.whl if-no-files-found: error sdist: @@ -193,10 +193,10 @@ jobs: python-version: '3.12' - name: Build and check source distribution run: | - uvx --from 'maturin>=1.8,<2' maturin sdist -m native/Cargo.toml --out dist + uvx --from 'maturin>=1.8,<2' maturin sdist -m av/Cargo.toml --out dist uvx twine check --strict dist/* - uses: actions/upload-artifact@v7 with: - name: dist-native-sdist + name: dist-av-sdist path: dist/*.tar.gz if-no-files-found: error diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml index db0c6f7..c940f11 100644 --- a/.github/workflows/build-python.yml +++ b/.github/workflows/build-python.yml @@ -1,5 +1,5 @@ # Builds the pure-Python tensorcodec wheel and sdist (artifact dist-python) and tests image codecs -# on platforms that get no tensorcodec-native. +# on platforms that get no tensorcodec-av. name: Build tensorcodec on: workflow_call: @@ -29,7 +29,7 @@ jobs: name: dist-python path: dist/* if-no-files-found: error - test-without-native: + test-without-av: needs: build strategy: fail-fast: false @@ -69,10 +69,10 @@ jobs: run: echo "$FFMPEG_BIN" >> "$GITHUB_PATH" - name: Check fixture FFmpeg run: ffmpeg -hide_banner -encoders | grep libaom-av1 - - name: Install tensorcodec without a Rust toolchain or tensorcodec-native + - name: Install tensorcodec without a Rust toolchain or tensorcodec-av run: | python -m pip install pytest 'Pillow>=11.3' '${{ matrix.opencv }}' python -m pip install --no-index --find-links dist "tensorcodec[images]" - python -c "import importlib.util as u; assert u.find_spec('tensorcodec_native') is None" + python -c "import importlib.util as u; assert u.find_spec('tensorcodec_av') is None" - name: Test image codecs - run: python -m pytest tests/test_images.py tests/test_image_encoders.py tests/test_native_optional.py tests/test_versions.py + run: python -m pytest tests/test_images.py tests/test_image_encoders.py tests/test_av_optional.py tests/test_versions.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bba9287..7e81dd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,25 +18,25 @@ jobs: contents: read pull-requests: read outputs: - native: ${{ steps.filter.outputs.native }} + av: ${{ steps.filter.outputs.av }} steps: - uses: actions/checkout@v7 - uses: dorny/paths-filter@v4 id: filter with: filters: | - native: - - 'native/**' + av: + - 'av/**' - 'scripts/**' - 'tests/**' - '.github/workflows/**' - 'pyproject.toml' - # Release builds of every tensorcodec-native wheel: on pushes (including main) and on PRs that touch + # Release builds of every tensorcodec-av wheel: on pushes (including main) and on PRs that touch # what they build or validate. Other PRs rely on the `test` job's development build. - native: + av: needs: [python, changes] - if: github.event_name != 'pull_request' || needs.changes.outputs.native == 'true' - uses: ./.github/workflows/build-native.yml + if: github.event_name != 'pull_request' || needs.changes.outputs.av == 'true' + uses: ./.github/workflows/build-av.yml test: runs-on: ubuntu-24.04 steps: @@ -49,7 +49,7 @@ jobs: components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 with: - workspaces: native + workspaces: av - uses: prefix-dev/setup-pixi@v0.10.2 with: pixi-version: v0.81.0 @@ -76,13 +76,13 @@ jobs: "$prefix/bin/ffmpeg" -version - name: Static checks run: | - ruff check src/tensorcodec native/python tests scripts benchmarks/image_codecs.py - ruff format --check src/tensorcodec native/python tests scripts benchmarks/image_codecs.py - cargo fmt --manifest-path native/Cargo.toml --check - cargo clippy --manifest-path native/Cargo.toml --locked -- -D warnings - - name: Build tensorcodec-native against prebuilt FFmpeg and install tensorcodec + ruff check src/tensorcodec av/python tests scripts benchmarks/image_codecs.py + ruff format --check src/tensorcodec av/python tests scripts benchmarks/image_codecs.py + cargo fmt --manifest-path av/Cargo.toml --check + cargo clippy --manifest-path av/Cargo.toml --locked -- -D warnings + - name: Build tensorcodec-av against prebuilt FFmpeg and install tensorcodec run: | - maturin develop -m native/Cargo.toml --locked + maturin develop -m av/Cargo.toml --locked pip install -e . - name: Install pinned reference run: python -m pip install torch==2.14.1 torchcodec==0.17.0 --index-url https://download.pytorch.org/whl/cpu diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e2a0db2..f5b0818 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,11 +14,11 @@ concurrency: jobs: python: uses: ./.github/workflows/build-python.yml - native: + av: needs: python - uses: ./.github/workflows/build-native.yml + uses: ./.github/workflows/build-av.yml check: - needs: [python, native] + needs: [python, av] runs-on: ubuntu-24.04 steps: - uses: actions/download-artifact@v8 @@ -32,11 +32,11 @@ jobs: files = sorted(p for p in pathlib.Path("artifacts").rglob("*") if p.is_file()) names = [f.name for f in files] print("\n".join(names)) - versions = {re.match(r"tensorcodec(?:_native)?-([^-]+?)(?:\.tar\.gz|-)", n)[1] for n in names} + versions = {re.match(r"tensorcodec(?:_av)?-([^-]+?)(?:\.tar\.gz|-)", n)[1] for n in names} assert len(versions) == 1, f"mixed versions: {versions}" - expected = ["tensorcodec-*-py3-none-any.whl", "tensorcodec-*.tar.gz", "tensorcodec_native-*.tar.gz", - "tensorcodec_native-*manylinux*x86_64.whl", "tensorcodec_native-*manylinux*aarch64.whl", - "tensorcodec_native-*macosx*arm64.whl"] + expected = ["tensorcodec-*-py3-none-any.whl", "tensorcodec-*.tar.gz", "tensorcodec_av-*.tar.gz", + "tensorcodec_av-*manylinux*x86_64.whl", "tensorcodec_av-*manylinux*aarch64.whl", + "tensorcodec_av-*macosx*arm64.whl"] for pattern in expected: assert len([f for f in files if f.match(pattern)]) == 1, pattern assert len(files) == len(expected), names @@ -54,18 +54,18 @@ jobs: steps: - uses: actions/download-artifact@v8 with: - pattern: dist-native-* + pattern: dist-av-* merge-multiple: true - path: native-dist + path: av-dist - uses: actions/download-artifact@v8 with: name: dist-python path: python-dist - # Native first: tensorcodec pins tensorcodec-native exactly, so it must not appear before its pin exists. - - name: Publish tensorcodec-native + # tensorcodec-av first: tensorcodec pins tensorcodec-av exactly, so it must not appear before its pin exists. + - name: Publish tensorcodec-av uses: pypa/gh-action-pypi-publish@release/v1 with: - packages-dir: native-dist/ + packages-dir: av-dist/ - name: Publish tensorcodec uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.gitignore b/.gitignore index a4e80d5..b31b79e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ # ============================================================================ # Rust build artifacts -/native/target/ +/av/target/ /.ffmpeg/ /.native-deps/ *.pyd diff --git a/README.md b/README.md index b85de58..e9e88be 100644 --- a/README.md +++ b/README.md @@ -159,13 +159,13 @@ See the [compatibility contract](docs/compatibility.md) and - **Wheels:** Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+. NumPy must also provide a compatible wheel; newer Python versions may require a newer glibc. macOS 14+ wheels support Apple Silicon (Intel Macs: through 0.1.5). - `tensorcodec` itself is pure Python; video/audio decoding lives in `tensorcodec-native`, pinned to + `tensorcodec` itself is pure Python; video/audio decoding lives in `tensorcodec-av`, pinned to the same version and installed automatically on those platforms (CPython only). Elsewhere, e.g. Windows and Intel macOS, image codecs work and `VideoDecoder`/`AudioDecoder` raise `ImportError`. Environment markers cannot detect musl, free-threaded CPython or macOS older than 14, so - installers there still try `tensorcodec-native` (no wheel; its sdist needs FFmpeg 7 and Rust). + installers there still try `tensorcodec-av` (no wheel; its sdist needs FFmpeg 7 and Rust). For image codecs only there, use `pip install --no-deps tensorcodec numpy`. To build video/audio - support from source on a platform without native wheels, install `tensorcodec-native==` explicitly (needs Rust and FFmpeg 7; see [external FFmpeg](docs/system_ffmpeg.md)). - **Exact seeking:** scans packet timestamps when opening the decoder. Incorrect container keyframe flags can produce corrupt frames; repaired input or corrected @@ -182,7 +182,7 @@ See [container behavior](docs/container_robustness.md) for seek limitations and
Build from source and run tests -`uv sync` builds `native/` (the `tensorcodec-native` package) in place. Its source builds require Rust 1.88+, Clang/libclang, pkg-config and FFmpeg 7 development +`uv sync` builds `av/` (the `tensorcodec-av` package) in place. Its source builds require Rust 1.88+, Clang/libclang, pkg-config and FFmpeg 7 development headers/libraries. Python handles API and playback selection; Rust + PyO3 handles FFmpeg. Native decoding releases the GIL, allowing separate decoder instances to run concurrently across Python threads. Calls on the same instance are serialized. @@ -195,14 +195,14 @@ uv sync --group dev --group oracle uv run --group oracle pytest tests/test_video_contract.py tests/test_audio_contract.py --backend torchcodec uv run --group oracle pytest --compare -# Rebuild tensorcodec-native after changing Rust code. -uv run --group oracle maturin develop -m native/Cargo.toml --locked --uv +# Rebuild tensorcodec-av after changing Rust code. +uv run --group oracle maturin develop -m av/Cargo.toml --locked --uv ``` Tests generate media with FFmpeg/ffprobe and Python's `wave` module. `--compare` requires the pinned oracle; differential tests otherwise skip. -[Playback rules](docs/playback_semantics.md) · [Release guide](docs/releasing.md) · [Dependency licenses](native/licenses/README.md) +[Playback rules](docs/playback_semantics.md) · [Release guide](docs/releasing.md) · [Dependency licenses](av/licenses/README.md)
diff --git a/native/Cargo.lock b/av/Cargo.lock similarity index 99% rename from native/Cargo.lock rename to av/Cargo.lock index 7c6aea9..571a4fc 100644 --- a/native/Cargo.lock +++ b/av/Cargo.lock @@ -448,7 +448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] -name = "tensorcodec-native" +name = "tensorcodec-av" version = "0.3.0" dependencies = [ "ffmpeg-sys-next", diff --git a/native/Cargo.toml b/av/Cargo.toml similarity index 90% rename from native/Cargo.toml rename to av/Cargo.toml index 596fedb..a4da9b8 100644 --- a/native/Cargo.toml +++ b/av/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "tensorcodec-native" +name = "tensorcodec-av" version = "0.3.0" edition = "2021" rust-version = "1.88" # ffmpeg-sys-next 9 builds with let chains license = "Apache-2.0" [lib] -name = "_native" +name = "_av" crate-type = ["cdylib"] [dependencies] diff --git a/native/LICENSE b/av/LICENSE similarity index 100% rename from native/LICENSE rename to av/LICENSE diff --git a/native/README.md b/av/README.md similarity index 63% rename from native/README.md rename to av/README.md index 65c6b56..b6c44d5 100644 --- a/native/README.md +++ b/av/README.md @@ -1,8 +1,8 @@ -# tensorcodec-native +# tensorcodec-av The FFmpeg-backed extension behind [TensorCodec](https://github.com/MilkClouds/tensorcodec)'s `VideoDecoder` and `AudioDecoder`. It has no public API: install `tensorcodec`, which -depends on the matching `tensorcodec-native` version on platforms with native wheels. +depends on the matching `tensorcodec-av` version on platforms with `tensorcodec-av` wheels. Wheels bundle shared FFmpeg, dav1d and OpenSSL libraries; see -[`licenses/README.md`](https://github.com/MilkClouds/tensorcodec/blob/main/native/licenses/README.md) for their licenses and sources. +[`licenses/README.md`](https://github.com/MilkClouds/tensorcodec/blob/main/av/licenses/README.md) for their licenses and sources. diff --git a/native/licenses/FFmpeg-GPL-3.0.txt b/av/licenses/FFmpeg-GPL-3.0.txt similarity index 100% rename from native/licenses/FFmpeg-GPL-3.0.txt rename to av/licenses/FFmpeg-GPL-3.0.txt diff --git a/native/licenses/FFmpeg-LGPL-3.0.txt b/av/licenses/FFmpeg-LGPL-3.0.txt similarity index 100% rename from native/licenses/FFmpeg-LGPL-3.0.txt rename to av/licenses/FFmpeg-LGPL-3.0.txt diff --git a/native/licenses/FFmpeg-NOTICE.md b/av/licenses/FFmpeg-NOTICE.md similarity index 100% rename from native/licenses/FFmpeg-NOTICE.md rename to av/licenses/FFmpeg-NOTICE.md diff --git a/native/licenses/OpenSSL.txt b/av/licenses/OpenSSL.txt similarity index 100% rename from native/licenses/OpenSSL.txt rename to av/licenses/OpenSSL.txt diff --git a/native/licenses/README.md b/av/licenses/README.md similarity index 91% rename from native/licenses/README.md rename to av/licenses/README.md index 3980012..85fcf83 100644 --- a/native/licenses/README.md +++ b/av/licenses/README.md @@ -1,6 +1,6 @@ # Bundled native libraries -TensorCodec's own code is Apache-2.0 licensed. `tensorcodec-native` Linux and macOS wheels bundle shared FFmpeg +TensorCodec's own code is Apache-2.0 licensed. `tensorcodec-av` Linux and macOS wheels bundle shared FFmpeg 7.1.5 libraries, built without GPL codec libraries using `scripts/build_ffmpeg.sh`. This configuration is LGPL-3.0-or-later. Its notices and both the LGPLv3 and incorporated GPLv3 texts are included here. The exact diff --git a/native/licenses/Zstandard.txt b/av/licenses/Zstandard.txt similarity index 100% rename from native/licenses/Zstandard.txt rename to av/licenses/Zstandard.txt diff --git a/native/licenses/dav1d.txt b/av/licenses/dav1d.txt similarity index 100% rename from native/licenses/dav1d.txt rename to av/licenses/dav1d.txt diff --git a/native/pyproject.toml b/av/pyproject.toml similarity index 86% rename from native/pyproject.toml rename to av/pyproject.toml index c187d80..89a8701 100644 --- a/native/pyproject.toml +++ b/av/pyproject.toml @@ -1,8 +1,8 @@ [project] -name = "tensorcodec-native" +name = "tensorcodec-av" # Version comes from Cargo.toml and must equal tensorcodec's (tests/test_versions.py). dynamic = ["version"] -description = "FFmpeg extension for tensorcodec's video and audio decoders" +description = "FFmpeg-based audio/video decoders for tensorcodec" readme = "README.md" license = "Apache-2.0 AND LGPL-3.0-or-later AND BSD-2-Clause" license-files = ["LICENSE", "licenses/*"] @@ -28,5 +28,5 @@ build-backend = "maturin" [tool.maturin] python-source = "python" -module-name = "tensorcodec_native._native" +module-name = "tensorcodec_av._av" features = ["pyo3/extension-module"] diff --git a/native/python/tensorcodec_native/__init__.py b/av/python/tensorcodec_av/__init__.py similarity index 100% rename from native/python/tensorcodec_native/__init__.py rename to av/python/tensorcodec_av/__init__.py diff --git a/native/src/ffmpeg.rs b/av/src/ffmpeg.rs similarity index 100% rename from native/src/ffmpeg.rs rename to av/src/ffmpeg.rs diff --git a/native/src/lib.rs b/av/src/lib.rs similarity index 99% rename from native/src/lib.rs rename to av/src/lib.rs index 076af89..7cba589 100644 --- a/native/src/lib.rs +++ b/av/src/lib.rs @@ -134,7 +134,7 @@ fn closed() -> PyErr { } #[pymodule] -fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { +fn _av(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add("ffmpeg_version", ffmpeg::version())?; module.add("__version__", env!("CARGO_PKG_VERSION"))?; diff --git a/docs/images.md b/docs/images.md index 46c5873..31016b8 100644 --- a/docs/images.md +++ b/docs/images.md @@ -31,9 +31,9 @@ do not import with NumPy 2. Use only one OpenCV wheel variant per environment. NumPy is the only required dependency for the base package. Image dependencies are separate from the base wheel size. Pillow is used only in tests. -Image codecs are pure Python and do not need `tensorcodec-native`, so they work +Image codecs are pure Python and do not need `tensorcodec-av`, so they work on every platform, including Windows and Intel macOS where `pip install -tensorcodec` installs no native package; constructing `VideoDecoder` or +tensorcodec` does not install `tensorcodec-av`; constructing `VideoDecoder` or `AudioDecoder` there raises `ImportError`. ## Contract and limits diff --git a/docs/package_size.md b/docs/package_size.md index 5763a83..34bf4d0 100644 --- a/docs/package_size.md +++ b/docs/package_size.md @@ -1,7 +1,7 @@ # Package size policy TensorCodec keeps its NumPy-only Python dependency set and bundles a minimal -FFmpeg/OpenSSL runtime in its default Linux `tensorcodec-native` wheels (the pure-Python +FFmpeg/OpenSSL runtime in its default Linux `tensorcodec-av` wheels (the pure-Python `tensorcodec` wheel adds about 30 KiB and is not size-checked). Size limits prevent additions from silently increasing the distributed binary footprint. @@ -25,7 +25,7 @@ uv run --no-project python scripts/check_wheel_size.py dist/*.whl --output repor ``` Each architecture is checked independently. Exactly reaching a limit passes; -exceeding either limit fails. The `linux` job of `build-native.yml` runs this after +exceeding either limit fails. The `linux` job of `build-av.yml` runs this after `auditwheel repair`, in CI and before every upload. JSON reports are separate artifacts, not files in `dist/`. Actions summaries include changes from the committed published baseline. The `test` job also unit-tests the checker. diff --git a/docs/releasing.md b/docs/releasing.md index b0d1bdb..b2ca342 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -6,25 +6,25 @@ commit and version: | Project | Contents | Build | Distributions | | --- | --- | --- | --- | | `tensorcodec` | Python API, image codecs (import `tensorcodec`) | hatchling (`uv build`) | `py3-none-any` wheel, sdist | -| `tensorcodec-native` | Rust/FFmpeg extension (import `tensorcodec_native`) | maturin (`native/`) | platform wheels, sdist | +| `tensorcodec-av` | Rust/FFmpeg audio/video decoder extension (import `tensorcodec_av`) | maturin (`av/`) | platform wheels, sdist | -`tensorcodec` requires `tensorcodec-native==` behind an environment -marker matching the native wheel tags. `tests/test_versions.py` fails CI unless -`pyproject.toml`, `native/Cargo.toml` (the native version source), `native/Cargo.lock`, the pin and +`tensorcodec` requires `tensorcodec-av==` behind an environment +marker matching the `tensorcodec-av` wheel tags. `tests/test_versions.py` fails CI unless +`pyproject.toml`, `av/Cargo.toml` (the `tensorcodec-av` version source), `av/Cargo.lock`, the pin and `tensorcodec.__version__` agree. At runtime `VideoDecoder`/`AudioDecoder` reject a -`tensorcodec-native` whose version differs. +`tensorcodec-av` whose version differs. -Native wheels target Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+ +`tensorcodec-av` wheels target Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+ (abi3). NumPy must also provide a compatible wheel for the selected Python/glibc pair. The wheel bundles shared FFmpeg 7.1.5 and OpenSSL 3.5.9 LTS; its only Python runtime dependency is NumPy. macOS 14+ ARM64 wheels bundle the same minimal runtime. Windows wheels are not provided; there, `tensorcodec` provides image -codecs only. Bundled-library notices ship only with `tensorcodec-native` -(`native/licenses/`). +codecs only. Bundled-library notices ship only with `tensorcodec-av` +(`av/licenses/`). To release, set the new version in `pyproject.toml` (project version and the -`tensorcodec-native` pin), `native/Cargo.toml` (then `cargo update -p -tensorcodec-native --manifest-path native/Cargo.toml` to refresh the lock file) +`tensorcodec-av` pin), `av/Cargo.toml` (then `cargo update -p +tensorcodec-av --manifest-path av/Cargo.toml` to refresh the lock file) and `src/tensorcodec/__init__.py`. ## Trusted publisher configuration @@ -33,13 +33,13 @@ The PyPI project is already registered. Its GitHub Trusted Publisher uses: | Field | Value | | --- | --- | -| PyPI project names | `tensorcodec`, `tensorcodec-native` | +| PyPI project names | `tensorcodec`, `tensorcodec-av` | | GitHub owner | `epishelf` | | Repository | `tensorcodec` | | Workflow filename | `publish.yml` | | Environment | `pypi` | -Both projects need this publisher. `tensorcodec-native` does not exist on PyPI yet: +Both projects need this publisher. `tensorcodec-av` does not exist on PyPI yet: before its first release, add it as a pending publisher (PyPI account → Publishing) with the same fields. Manage this configuration in each project's PyPI Publishing settings when moving or renaming the repository or workflow. Publishing uses GitHub OIDC; no API token @@ -51,20 +51,20 @@ Each distribution has one reusable workflow; CI and the release call both. | Workflow | Jobs | Artifacts | | --- | --- | --- | -| `build-python.yml` | `build`: `tensorcodec` wheel and sdist (`python -m build`, `twine check --strict`); `test-without-native`: installs it on Windows and Intel macOS without `tensorcodec-native` and runs the image tests (OpenCV 5 and the 4.12 lower bound) | `dist-python` | -| `build-native.yml` | `linux` (x86_64, aarch64: manylinux2014 via maturin-action), `macos` (arm64), `sdist`; each wheel job validates with the `dist-python` wheel, so callers run `build-python.yml` first | `dist-native-linux-{x86_64,aarch64}`, `dist-native-macos-arm64`, `dist-native-sdist`, `wheel-size-linux-*` | -| `ci.yml` | `python`, `native`, plus `test` (development build against conda-forge FFmpeg, Clippy, oracle comparison) | — | -| `publish.yml` | `python`, `native`, `check` (one version, exactly the six expected files), `publish`, `update-size-docs` | — | +| `build-python.yml` | `build`: `tensorcodec` wheel and sdist (`python -m build`, `twine check --strict`); `test-without-av`: installs it on Windows and Intel macOS without `tensorcodec-av` and runs the image tests (OpenCV 5 and the 4.12 lower bound) | `dist-python` | +| `build-av.yml` | `linux` (x86_64, aarch64: manylinux2014 via maturin-action), `macos` (arm64), `sdist`; each wheel job validates with the `dist-python` wheel, so callers run `build-python.yml` first | `dist-av-linux-{x86_64,aarch64}`, `dist-av-macos-arm64`, `dist-av-sdist`, `wheel-size-linux-*` | +| `ci.yml` | `python`, `av`, plus `test` (development build against conda-forge FFmpeg, Clippy, oracle comparison) | — | +| `publish.yml` | `python`, `av`, `check` (one version, exactly the six expected files), `publish`, `update-size-docs` | — | -`ci.yml` runs `native` on every push, including `main`, and on pull requests that -touch `native/`, `scripts/`, `tests/`, workflows or `pyproject.toml`; a release -commit has therefore already passed the same native builds and validation. +`ci.yml` runs `av` on every push, including `main`, and on pull requests that +touch `av/`, `scripts/`, `tests/`, workflows or `pyproject.toml`; a release +commit has therefore already passed the same `tensorcodec-av` builds and validation. ## Release Run the **Publish to PyPI** workflow on `main`. It runs both build workflows, checks the release set and uploads through PyPI Trusted Publishing using the existing GitHub -`pypi` environment. `tensorcodec-native` is uploaded first, so the exact pin in +`pypi` environment. `tensorcodec-av` is uploaded first, so the exact pin in `tensorcodec` never points at a missing release; if the second upload fails, upload the `dist-python` artifact with `twine upload` rather than rerunning the whole workflow. Publication fails if authorization is missing, tests fail, or the version @@ -78,24 +78,24 @@ For a build and full validation without uploading, pass `--field publish=false`; the run's `dist-*` artifacts are then the release candidates. Check the workflow, https://pypi.org/project/tensorcodec/ and -https://pypi.org/project/tensorcodec-native/ before reporting success. Verify a fresh -`uv pip install tensorcodec==` pulls the matching `tensorcodec-native` and +https://pypi.org/project/tensorcodec-av/ before reporting success. Verify a fresh +`uv pip install tensorcodec==` pulls the matching `tensorcodec-av` and decodes video without Torch/PyAV on both Linux architectures and macOS arm64, and -that it installs without `tensorcodec-native` on Windows. Update the version before +that it installs without `tensorcodec-av` on Windows. Update the version before subsequent releases; PyPI versions cannot be overwritten. The local Linux build is reproducible using `scripts/build_linux_wheel.sh` inside `quay.io/pypa/manylinux2014_x86_64` or `quay.io/pypa/manylinux2014_aarch64` with Rust, maturin, libclang, NASM and Perl. Both native source archives are version- and checksum-pinned. Their licensing -and source links are recorded in `native/licenses/README.md`. +and source links are recorded in `av/licenses/README.md`. ## Development versus release builds - The `test` job uses prebuilt conda-forge FFmpeg 7.1.1 through Pixi, including its - headers and shared libraries. It builds only the `tensorcodec-native` extension + headers and shared libraries. It builds only the `tensorcodec-av` extension and installs `tensorcodec` from the checkout. -- Release wheels (`build-native.yml`) use the smaller LGPL FFmpeg 7.1.5 build plus +- Release wheels (`build-av.yml`) use the smaller LGPL FFmpeg 7.1.5 build plus OpenSSL 3.5.9. Their native prefix is cached by architecture, glibc baseline and build-script checksums. This preserves the wheel's codec set, dependency size and licensing rather than bundling the full conda-forge dependency graph. @@ -109,13 +109,13 @@ and source links are recorded in `native/licenses/README.md`. ## Size checks and published comparison -Final repaired `tensorcodec-native` wheels must stay within 15 MiB download and 35 MiB unpacked per +Final repaired `tensorcodec-av` wheels must stay within 15 MiB download and 35 MiB unpacked per architecture. The `linux` job checks `packaging/size-policy.json` and uploads a separate `wheel-size-linux-*` report, including differences from the last published baseline. Size reports must not be placed in `dist/`. After a successful publication, the `update-size-docs` job measures hash-verified -PyPI `tensorcodec-native` wheels for the release and pinned TorchCodec 0.17.0. It commits the published +PyPI `tensorcodec-av` wheels for the release and pinned TorchCodec 0.17.0. It commits the published snapshot and README table/badge with a normal push to `main`. Only this documentation job has `contents: write`; the PyPI publisher retains OIDC plus read access. The repository must permit the Actions bot to push these documentation updates. diff --git a/docs/system_ffmpeg.md b/docs/system_ffmpeg.md index 8bc9cd8..27c5206 100644 --- a/docs/system_ffmpeg.md +++ b/docs/system_ffmpeg.md @@ -7,7 +7,7 @@ uv venv uv pip install tensorcodec ``` -On supported platforms this also installs the matching `tensorcodec-native` +On supported platforms this also installs the matching `tensorcodec-av` wheel, which includes minimal shared FFmpeg 7.1.5 (with dav1d for AV1) and OpenSSL libraries. No FFmpeg CLI, Pixi, Rust or libclang is required at runtime. This is the recommended installation for a new environment. @@ -38,11 +38,11 @@ test -f "$FFMPEG_DIR/include/libavcodec/avcodec.h" test -f "$FFMPEG_DIR/lib/libavcodec.so.61" uv venv -uv pip install 'tensorcodec==0.3.0' 'tensorcodec-native==0.3.0' --no-binary tensorcodec-native +uv pip install 'tensorcodec==0.3.0' 'tensorcodec-av==0.3.0' --no-binary tensorcodec-av ``` -`tensorcodec` is pure Python; only `tensorcodec-native` is built from source here, -and naming it explicitly also covers platforms without native wheels (for example +`tensorcodec` is pure Python; only `tensorcodec-av` is built from source here, +and naming it explicitly also covers platforms without `tensorcodec-av` wheels (for example Intel macOS), where `tensorcodec` does not depend on it. Releases up to 0.2.0 were a single package, built from source with `--no-binary tensorcodec`. @@ -55,14 +55,14 @@ paths, also set `LIBCLANG_PATH` to its library directory. The example uses a GPL-enabled conda-forge build, unlike the minimal LGPL release build. Its additional codecs, dependencies and licensing apply to your environment. -Review [`native/licenses/README.md`](../native/licenses/README.md) before redistributing a binary +Review [`av/licenses/README.md`](../av/licenses/README.md) before redistributing a binary built against a different FFmpeg configuration. For development against the same prefix: ```sh uv sync --group dev -uv run maturin develop -m native/Cargo.toml --locked --uv +uv run maturin develop -m av/Cargo.toml --locked --uv ``` Do not reuse an existing `dist/` wheel while verifying this path: it may contain the @@ -84,7 +84,7 @@ within the color-conversion tolerances described in the ## macOS wheels -`tensorcodec-native` macOS 14+ wheels support Apple Silicon, bundling FFmpeg/OpenSSL with `delocate` +`tensorcodec-av` macOS 14+ wheels support Apple Silicon, bundling FFmpeg/OpenSSL with `delocate` (0.1.3 through 0.1.5 also had Intel wheels). CI tests the installed wheels and clean Python 3.10/3.13 environments. Developers can run `scripts/build_macos_wheel.sh` with Rust, Xcode tools, NASM, Meson, Ninja, pkg-config, coreutils, diff --git a/pyproject.toml b/pyproject.toml index c461b73..6ac87bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,12 +7,12 @@ license = "Apache-2.0" license-files = ["LICENSE"] authors = [{ name = "Suhwan Choi", email = "milkclouds00@gmail.com" }] requires-python = ">=3.10" -# tensorcodec-native is pinned to this exact version. The marker follows its wheel tags (CPython abi3, +# tensorcodec-av is pinned to this exact version. The marker follows its wheel tags (CPython abi3, # manylinux x86_64/aarch64, macOS 14+ arm64); elsewhere only the image codecs work. No platform_release # check for macOS 14: pip's packaging 22-25 raises on non-PEP 440 Linux kernel releases. dependencies = [ "numpy>=1.26", - "tensorcodec-native==0.3.0; platform_python_implementation == 'CPython' and ((sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", + "tensorcodec-av==0.3.0; platform_python_implementation == 'CPython' and ((sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", ] classifiers = [ "Development Status :: 3 - Alpha", @@ -41,8 +41,8 @@ build-backend = "hatchling.build" packages = ["src/tensorcodec"] [tool.hatch.build.targets.sdist] -# native/ keeps the sdist a complete checkout for the uv source below and tests/test_versions.py. -only-include = ["src", "native", "tests", "docs", "scripts", "benchmarks", "packaging", "README.md", "LICENSE"] +# av/ keeps the sdist a complete checkout for the uv source below and tests/test_versions.py. +only-include = ["src", "av", "tests", "docs", "scripts", "benchmarks", "packaging", "README.md", "LICENSE"] [tool.pytest.ini_options] testpaths = ["tests"] @@ -52,7 +52,7 @@ line-length = 119 target-version = "py310" [tool.uv.sources] -tensorcodec-native = { path = "native", editable = true } +tensorcodec-av = { path = "av", editable = true } torch = { index = "pytorch-cpu" } torchcodec = { index = "pytorch-cpu" } diff --git a/scripts/build_linux_wheel.sh b/scripts/build_linux_wheel.sh index e04a969..85ab0be 100755 --- a/scripts/build_linux_wheel.sh +++ b/scripts/build_linux_wheel.sh @@ -9,5 +9,5 @@ export LD_LIBRARY_PATH="$build_prefix/openssl/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY scripts/build_ffmpeg.sh "$build_prefix/ffmpeg" export FFMPEG_DIR="$build_prefix/ffmpeg" export LD_LIBRARY_PATH="$FFMPEG_DIR/lib:$LD_LIBRARY_PATH" -maturin build -m native/Cargo.toml --release --locked --auditwheel repair --compatibility manylinux2014 --out dist +maturin build -m av/Cargo.toml --release --locked --auditwheel repair --compatibility manylinux2014 --out dist python scripts/check_wheel_size.py dist/*.whl diff --git a/scripts/build_macos_wheel.sh b/scripts/build_macos_wheel.sh index 178840b..8982ce0 100644 --- a/scripts/build_macos_wheel.sh +++ b/scripts/build_macos_wheel.sh @@ -11,6 +11,6 @@ if [ ! -f "$build_prefix/ready" ]; then touch "$build_prefix/ready" fi export FFMPEG_DIR="$build_prefix/ffmpeg" -maturin build -m native/Cargo.toml --release --locked --out unrepaired +maturin build -m av/Cargo.toml --release --locked --out unrepaired delocate-wheel --require-archs "$(uname -m)" -w dist unrepaired/*.whl python scripts/check_wheel_size.py dist/*.whl diff --git a/scripts/configure_oracle_ffmpeg.py b/scripts/configure_oracle_ffmpeg.py index 2b7e4b8..e098748 100644 --- a/scripts/configure_oracle_ffmpeg.py +++ b/scripts/configure_oracle_ffmpeg.py @@ -5,10 +5,10 @@ import re from pathlib import Path -spec = importlib.util.find_spec("tensorcodec_native") -libs = Path(spec.origin).parent.parent / "tensorcodec_native.libs" +spec = importlib.util.find_spec("tensorcodec_av") +libs = Path(spec.origin).parent.parent / "tensorcodec_av.libs" if not libs.is_dir(): - raise RuntimeError("Expected an installed, repaired tensorcodec-native wheel") + raise RuntimeError("Expected an installed, repaired tensorcodec-av wheel") for library in libs.glob("lib*.so.*"): match = re.fullmatch(r"(lib[^-]+)-[0-9a-f]+(\.so\.\d+)", library.name) if match and match[1] in {"libavcodec", "libavformat", "libavutil", "libswscale", "libswresample"}: diff --git a/scripts/update_size_comparison.py b/scripts/update_size_comparison.py index c665faf..51debdc 100644 --- a/scripts/update_size_comparison.py +++ b/scripts/update_size_comparison.py @@ -118,7 +118,7 @@ def main() -> None: reference = policy["comparison"] pyav = policy["pyav"] snapshot = { - "tensorcodec": measure_release("tensorcodec-native", args.version, "cp310"), + "tensorcodec": measure_release("tensorcodec-av", args.version, "cp310"), "torchcodec": measure_release(reference["project"], reference["version"], reference["python_tag"]), "pyav": measure_release(pyav["project"], pyav["version"], pyav["python_tag"]), "torch": measure_release(**policy["torch"]), diff --git a/src/tensorcodec/decoders/_decoder.py b/src/tensorcodec/decoders/_decoder.py index 413a704..4c471cd 100644 --- a/src/tensorcodec/decoders/_decoder.py +++ b/src/tensorcodec/decoders/_decoder.py @@ -16,22 +16,20 @@ from tensorcodec.transforms import _pipeline -def _native_decoder(*args): +def _av_decoder(*args): try: - from tensorcodec_native import _native + from tensorcodec_av import _av except ImportError as error: raise ImportError( - "VideoDecoder and AudioDecoder require the tensorcodec-native package, which could not be imported. " - "pip installs it automatically on Linux x86_64/aarch64 (glibc 2.17+) and macOS 14+ arm64 with " - "CPython; elsewhere tensorcodec provides only the image codecs." + "VideoDecoder and AudioDecoder require tensorcodec-av (the audio/video decoders), which could not be " + "imported. pip installs it automatically on Linux x86_64/aarch64 (glibc 2.17+) and macOS 14+ arm64 " + "with CPython; elsewhere tensorcodec provides only the image codecs." ) from error from tensorcodec import __version__ - if _native.__version__ != __version__: - raise ImportError( - f"tensorcodec {__version__} requires tensorcodec-native=={__version__}, found {_native.__version__}" - ) - return _native.Decoder(*args) + if _av.__version__ != __version__: + raise ImportError(f"tensorcodec {__version__} requires tensorcodec-av=={__version__}, found {_av.__version__}") + return _av.Decoder(*args) def _source(source): @@ -142,7 +140,7 @@ def __init__( self._seek_mode = seek_mode self._lock = RLock() self._closed = False - self._native = _native_decoder(_source(source), "video", stream_index, int(num_ffmpeg_threads)) + self._native = _av_decoder(_source(source), "video", stream_index, int(num_ffmpeg_threads)) try: header = self._native.metadata(apply_rotation=output_format != "native") self.stream_index = header["stream_index"] @@ -441,7 +439,7 @@ def __init__(self, source, *, stream_index=None, sample_rate=None, num_channels= raise ValueError(f"{name} must be a positive integer") self._lock = RLock() self._closed = False - self._native = _native_decoder(_source(source), "audio", stream_index) + self._native = _av_decoder(_source(source), "audio", stream_index) try: header = self._native.metadata() header.pop("time_base_num") diff --git a/tests/test_native_optional.py b/tests/test_av_optional.py similarity index 65% rename from tests/test_native_optional.py rename to tests/test_av_optional.py index 69cd21d..0854d96 100644 --- a/tests/test_native_optional.py +++ b/tests/test_av_optional.py @@ -1,4 +1,4 @@ -"""Image codecs work without tensorcodec-native; video/audio fail clearly when it is missing or skewed.""" +"""Image codecs work without tensorcodec-av; video/audio fail clearly when it is missing or skewed.""" import subprocess import sys @@ -8,23 +8,23 @@ def run_python(code): subprocess.run([sys.executable, "-c", code], check=True) -def test_native_extension_is_lazy(): +def test_av_extension_is_lazy(): run_python( """ import sys import tensorcodec import tensorcodec.decoders import tensorcodec.encoders -assert 'tensorcodec_native' not in sys.modules +assert 'tensorcodec_av' not in sys.modules """ ) -def test_image_codecs_without_native_extension(): +def test_image_codecs_without_tensorcodec_av(): run_python( """ import sys -sys.modules['tensorcodec_native'] = None +sys.modules['tensorcodec_av'] = None import numpy as np import tensorcodec from tensorcodec.decoders import ( @@ -44,9 +44,9 @@ def test_image_codecs_without_native_extension(): try: decoder(b'not media') except ImportError as exc: - assert 'tensorcodec-native package' in str(exc), exc + assert 'require tensorcodec-av' in str(exc), exc else: - raise AssertionError(f'{decoder.__name__} worked without tensorcodec-native') + raise AssertionError(f'{decoder.__name__} worked without tensorcodec-av') """ ) @@ -56,15 +56,15 @@ def test_version_skew_is_rejected(): """ import sys import types -package = types.ModuleType('tensorcodec_native') -package._native = types.SimpleNamespace(__version__='0.0.0', Decoder=None) -sys.modules['tensorcodec_native'] = package +package = types.ModuleType('tensorcodec_av') +package._av = types.SimpleNamespace(__version__='0.0.0', Decoder=None) +sys.modules['tensorcodec_av'] = package from tensorcodec.decoders import VideoDecoder try: VideoDecoder(b'not media') except ImportError as exc: - assert 'requires tensorcodec-native==' in str(exc) and 'found 0.0.0' in str(exc), exc + assert 'requires tensorcodec-av==' in str(exc) and 'found 0.0.0' in str(exc), exc else: - raise AssertionError('mismatched tensorcodec-native was accepted') + raise AssertionError('mismatched tensorcodec-av was accepted') """ ) diff --git a/tests/test_versions.py b/tests/test_versions.py index 7f7f1a6..d4f0752 100644 --- a/tests/test_versions.py +++ b/tests/test_versions.py @@ -1,4 +1,4 @@ -"""tensorcodec and tensorcodec-native are released in lockstep from one commit.""" +"""tensorcodec and tensorcodec-av are released in lockstep from one commit.""" import re from pathlib import Path @@ -11,28 +11,28 @@ ROOT = Path(__file__).resolve().parents[1] -def native_requirements(project): +def av_requirements(project): groups = [project["dependencies"], *project.get("optional-dependencies", {}).values()] requirements = [Requirement(item) for group in groups for item in group] - return [r for r in requirements if r.name == "tensorcodec-native"] + return [r for r in requirements if r.name == "tensorcodec-av"] def test_versions_are_locked(): project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] version = project["version"] - cargo = tomllib.loads((ROOT / "native/Cargo.toml").read_text())["package"]["version"] - lock = tomllib.loads((ROOT / "native/Cargo.lock").read_text())["package"] - assert [p["version"] for p in lock if p["name"] == "tensorcodec-native"] == [cargo] + cargo = tomllib.loads((ROOT / "av/Cargo.toml").read_text())["package"]["version"] + lock = tomllib.loads((ROOT / "av/Cargo.lock").read_text())["package"] + assert [p["version"] for p in lock if p["name"] == "tensorcodec-av"] == [cargo] init = re.search(r'^__version__ = "(.+)"$', (ROOT / "src/tensorcodec/__init__.py").read_text(), re.MULTILINE)[1] assert cargo == init == version # Only the marked dependency: an extra would just trigger the same source build elsewhere. - (pin,) = native_requirements(project) + (pin,) = av_requirements(project) assert str(pin.specifier) == f"=={version}" assert pin.marker is not None -def test_native_license_matches_project_license(): - assert (ROOT / "native/LICENSE").read_bytes() == (ROOT / "LICENSE").read_bytes() +def test_av_license_matches_project_license(): + assert (ROOT / "av/LICENSE").read_bytes() == (ROOT / "LICENSE").read_bytes() @pytest.mark.parametrize( @@ -47,9 +47,9 @@ def test_native_license_matches_project_license(): ({"sys_platform": "win32", "platform_machine": "AMD64"}, False), ], ) -def test_native_marker_matches_published_wheels(environment, expected): +def test_av_marker_matches_published_wheels(environment, expected): project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] - (default,) = native_requirements(project) + (default,) = av_requirements(project) # packaging 22-25 (vendored by pip) raise on version comparisons with releases like "6.8.0-azure". assert "platform_release" not in str(default.marker) base = default_environment() | {"platform_python_implementation": "CPython"} From a9f9e1b6cbf85c43b8ec05089d21ad48c531069a Mon Sep 17 00:00:00 2001 From: MilkClouds Date: Mon, 5 Oct 2026 03:56:03 +0900 Subject: [PATCH 12/12] Trim docs, workflows and tests Docs keep only what users and maintainers need; releasing.md is reduced to the two-package release steps and the recovery step. Workflows drop the path filter (CI always runs build-av.yml), the publish release-set check (needs already require every artifact), the single-entry macOS matrix and redundant checks. Tests merge overlapping cases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YAijSnAE4aAjuS1EcCTwqS --- .github/workflows/build-av.yml | 15 +--- .github/workflows/build-python.yml | 12 +-- .github/workflows/ci.yml | 24 +----- .github/workflows/publish.yml | 26 +------ README.md | 16 ++-- av/README.md | 9 +-- docs/images.md | 27 +++---- docs/package_size.md | 11 ++- docs/releasing.md | 119 ++++++++++------------------- docs/system_ffmpeg.md | 11 +-- pyproject.toml | 6 +- tests/test_av_optional.py | 37 ++------- tests/test_images.py | 6 +- tests/test_versions.py | 22 +++--- 14 files changed, 95 insertions(+), 246 deletions(-) diff --git a/.github/workflows/build-av.yml b/.github/workflows/build-av.yml index 57b7288..86e2d6a 100644 --- a/.github/workflows/build-av.yml +++ b/.github/workflows/build-av.yml @@ -1,5 +1,4 @@ -# Builds and validates every tensorcodec-av distribution. Callers must run build-python.yml first: -# validation installs its dist-python wheel next to each tensorcodec-av wheel. +# Callers run build-python.yml first: validation installs its dist-python wheel. name: Build tensorcodec-av on: workflow_call: @@ -110,13 +109,7 @@ jobs: path: dist/*.whl if-no-files-found: error macos: - strategy: - fail-fast: false - matrix: - include: - - runner: macos-15 - arch: arm64 - runs-on: ${{ matrix.runner }} + runs-on: macos-15 env: MACOSX_DEPLOYMENT_TARGET: '14.0' steps: @@ -146,7 +139,7 @@ jobs: id: native-cache with: path: .native-deps/macos - key: macos14-${{ matrix.arch }}-${{ hashFiles('scripts/build_openssl.sh', 'scripts/build_dav1d.sh', 'scripts/build_ffmpeg.sh', 'scripts/build_macos_wheel.sh') }} + key: macos14-arm64-${{ hashFiles('scripts/build_openssl.sh', 'scripts/build_dav1d.sh', 'scripts/build_ffmpeg.sh', 'scripts/build_macos_wheel.sh') }} - name: Build and bundle wheel run: bash scripts/build_macos_wheel.sh - uses: actions/cache/save@v6 @@ -181,7 +174,7 @@ jobs: fi - uses: actions/upload-artifact@v7 with: - name: dist-av-macos-${{ matrix.arch }} + name: dist-av-macos-arm64 path: dist/*.whl if-no-files-found: error sdist: diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml index c940f11..3da6e25 100644 --- a/.github/workflows/build-python.yml +++ b/.github/workflows/build-python.yml @@ -1,5 +1,3 @@ -# Builds the pure-Python tensorcodec wheel and sdist (artifact dist-python) and tests image codecs -# on platforms that get no tensorcodec-av. name: Build tensorcodec on: workflow_call: @@ -18,12 +16,6 @@ jobs: python -m pip install build twine python -m build --outdir dist python -m twine check --strict dist/* - python - <<'EOF' - import glob, zipfile - (wheel,) = glob.glob("dist/tensorcodec-*-py3-none-any.whl") - names = zipfile.ZipFile(wheel).namelist() - assert not [n for n in names if n.endswith((".so", ".pyd", ".dylib")) or "FFmpeg" in n], names - EOF - uses: actions/upload-artifact@v7 with: name: dist-python @@ -67,9 +59,7 @@ jobs: env: FFMPEG_BIN: ${{ runner.temp }}/pixi/envs/ffmpeg/${{ runner.os == 'Windows' && 'Library/bin' || 'bin' }} run: echo "$FFMPEG_BIN" >> "$GITHUB_PATH" - - name: Check fixture FFmpeg - run: ffmpeg -hide_banner -encoders | grep libaom-av1 - - name: Install tensorcodec without a Rust toolchain or tensorcodec-av + - name: Install tensorcodec without tensorcodec-av run: | python -m pip install pytest 'Pillow>=11.3' '${{ matrix.opencv }}' python -m pip install --no-index --find-links dist "tensorcodec[images]" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e81dd5..d10de2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,30 +12,8 @@ concurrency: jobs: python: uses: ./.github/workflows/build-python.yml - changes: - runs-on: ubuntu-24.04 - permissions: - contents: read - pull-requests: read - outputs: - av: ${{ steps.filter.outputs.av }} - steps: - - uses: actions/checkout@v7 - - uses: dorny/paths-filter@v4 - id: filter - with: - filters: | - av: - - 'av/**' - - 'scripts/**' - - 'tests/**' - - '.github/workflows/**' - - 'pyproject.toml' - # Release builds of every tensorcodec-av wheel: on pushes (including main) and on PRs that touch - # what they build or validate. Other PRs rely on the `test` job's development build. av: - needs: [python, changes] - if: github.event_name != 'pull_request' || needs.changes.outputs.av == 'true' + needs: python uses: ./.github/workflows/build-av.yml test: runs-on: ubuntu-24.04 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f5b0818..827f206 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,32 +17,8 @@ jobs: av: needs: python uses: ./.github/workflows/build-av.yml - check: - needs: [python, av] - runs-on: ubuntu-24.04 - steps: - - uses: actions/download-artifact@v8 - with: - pattern: dist-* - path: artifacts - - name: Check the release set - run: | - python3 - <<'EOF' - import pathlib, re - files = sorted(p for p in pathlib.Path("artifacts").rglob("*") if p.is_file()) - names = [f.name for f in files] - print("\n".join(names)) - versions = {re.match(r"tensorcodec(?:_av)?-([^-]+?)(?:\.tar\.gz|-)", n)[1] for n in names} - assert len(versions) == 1, f"mixed versions: {versions}" - expected = ["tensorcodec-*-py3-none-any.whl", "tensorcodec-*.tar.gz", "tensorcodec_av-*.tar.gz", - "tensorcodec_av-*manylinux*x86_64.whl", "tensorcodec_av-*manylinux*aarch64.whl", - "tensorcodec_av-*macosx*arm64.whl"] - for pattern in expected: - assert len([f for f in files if f.match(pattern)]) == 1, pattern - assert len(files) == len(expected), names - EOF publish: - needs: check + needs: [python, av] if: inputs.publish runs-on: ubuntu-24.04 environment: diff --git a/README.md b/README.md index e9e88be..9f4b152 100644 --- a/README.md +++ b/README.md @@ -159,14 +159,10 @@ See the [compatibility contract](docs/compatibility.md) and - **Wheels:** Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+. NumPy must also provide a compatible wheel; newer Python versions may require a newer glibc. macOS 14+ wheels support Apple Silicon (Intel Macs: through 0.1.5). - `tensorcodec` itself is pure Python; video/audio decoding lives in `tensorcodec-av`, pinned to - the same version and installed automatically on those platforms (CPython only). Elsewhere, e.g. - Windows and Intel macOS, image codecs work and `VideoDecoder`/`AudioDecoder` raise `ImportError`. - Environment markers cannot detect musl, free-threaded CPython or macOS older than 14, so - installers there still try `tensorcodec-av` (no wheel; its sdist needs FFmpeg 7 and Rust). - For image codecs only there, use `pip install --no-deps tensorcodec numpy`. To build video/audio - support from source on a platform without `tensorcodec-av` wheels, install `tensorcodec-av==` explicitly (needs Rust and FFmpeg 7; see [external FFmpeg](docs/system_ffmpeg.md)). + These are `tensorcodec-av` wheels, installed automatically with `tensorcodec` (pure Python). + Elsewhere only image codecs work; to build video/audio from source, install + `tensorcodec-av==` ([external FFmpeg](docs/system_ffmpeg.md)). musl and + free-threaded CPython cannot be told apart by markers, so there use `pip install --no-deps tensorcodec numpy`. - **Exact seeking:** scans packet timestamps when opening the decoder. Incorrect container keyframe flags can produce corrupt frames; repaired input or corrected frame mappings are needed in that case. @@ -182,7 +178,7 @@ See [container behavior](docs/container_robustness.md) for seek limitations and
Build from source and run tests -`uv sync` builds `av/` (the `tensorcodec-av` package) in place. Its source builds require Rust 1.88+, Clang/libclang, pkg-config and FFmpeg 7 development +Source builds of `av/` (`tensorcodec-av`) require Rust 1.88+, Clang/libclang, pkg-config and FFmpeg 7 development headers/libraries. Python handles API and playback selection; Rust + PyO3 handles FFmpeg. Native decoding releases the GIL, allowing separate decoder instances to run concurrently across Python threads. Calls on the same instance are serialized. @@ -195,7 +191,7 @@ uv sync --group dev --group oracle uv run --group oracle pytest tests/test_video_contract.py tests/test_audio_contract.py --backend torchcodec uv run --group oracle pytest --compare -# Rebuild tensorcodec-av after changing Rust code. +# Rebuild after changing Rust code. uv run --group oracle maturin develop -m av/Cargo.toml --locked --uv ``` diff --git a/av/README.md b/av/README.md index b6c44d5..27ded41 100644 --- a/av/README.md +++ b/av/README.md @@ -1,8 +1,5 @@ # tensorcodec-av -The FFmpeg-backed extension behind [TensorCodec](https://github.com/MilkClouds/tensorcodec)'s -`VideoDecoder` and `AudioDecoder`. It has no public API: install `tensorcodec`, which -depends on the matching `tensorcodec-av` version on platforms with `tensorcodec-av` wheels. - -Wheels bundle shared FFmpeg, dav1d and OpenSSL libraries; see -[`licenses/README.md`](https://github.com/MilkClouds/tensorcodec/blob/main/av/licenses/README.md) for their licenses and sources. +FFmpeg extension behind [TensorCodec](https://github.com/MilkClouds/tensorcodec)'s +`VideoDecoder` and `AudioDecoder`; install `tensorcodec`, which depends on the matching version. +Bundled library licenses: [`licenses/README.md`](https://github.com/MilkClouds/tensorcodec/blob/main/av/licenses/README.md). diff --git a/docs/images.md b/docs/images.md index 31016b8..4b693be 100644 --- a/docs/images.md +++ b/docs/images.md @@ -20,21 +20,14 @@ encoded = JpegEncoder(rgb).to_tensor(quality=90) # 1-D uint8 NumPy array ## Installation -The image backend uses OpenCV 4.12 or newer, including 5.x. An existing compatible +The image backend uses OpenCV 4.12+ (tested with 4.12, 4.13 and 5.0); earlier +`opencv-python-headless` wheels lack the GIF and AVIF decoders. An existing compatible `cv2` installation is sufficient; otherwise the `images` extra installs -`opencv-python-headless`. The bound is measured: the image test suite, including -the TorchCodec comparisons, passes with `opencv-python-headless` 4.12.0.88, -4.13.0.92 and 5.0.0.93. With 4.10 and 4.11 everything except GIF and AVIF passes: -4.12 is the first release whose PyPI wheels build both decoders (OpenCV added GIF -in 4.11, but its wheels report `GIF: NO` and `AVIF: NO`). The 4.8 and 4.9 wheels -do not import with NumPy 2. Use only one OpenCV wheel variant per environment. +`opencv-python-headless`. Use only one OpenCV wheel variant per environment. NumPy is the only required dependency for the base package. Image dependencies are separate from the base wheel size. Pillow is used only in tests. -Image codecs are pure Python and do not need `tensorcodec-av`, so they work -on every platform, including Windows and Intel macOS where `pip install -tensorcodec` does not install `tensorcodec-av`; constructing `VideoDecoder` or -`AudioDecoder` there raises `ImportError`. +Image codecs do not need `tensorcodec-av`, so they also work on platforms without its wheels. ## Contract and limits @@ -54,13 +47,11 @@ tensorcodec` does not install `tensorcodec-av`; constructing `VideoDecoder` or - AVIF color conversion follows OpenCV. Dropping alpha preserves straight RGB; TorchCodec 0.17.0 premultiplies AVIF RGB in that case. Pixel identity with TorchCodec is not promised across formats, builds or codec versions. -- `decode_image` also detects BMP (no format-specific function). 32-bit - `BI_BITFIELDS` BMPs with a nonzero alpha mask keep alpha; other 32-bit BMPs are - RGB, as in Pillow, because their fourth byte is padding. -- HEIC is unsupported. TIFF is not detected: OpenCV premultiplies unassociated - alpha and drops gray+alpha samples, so lossless decoding cannot be promised. -- Format support depends on the installed OpenCV build; for example, the - Windows `opencv-python-headless` 4.14 wheel has no AVIF decoder. Missing dependencies, +- `decode_image` also detects BMP; as in Pillow, only 32-bit `BI_BITFIELDS` + BMPs with an alpha mask keep alpha. +- HEIC and TIFF (OpenCV alters unassociated alpha) are unsupported. +- Other formats depend on the installed OpenCV build (the Windows wheel has no + AVIF decoder). Missing dependencies, unsupported codecs and decode failures raise; no alternate decoder is tried. - Encoders accept nonempty CHW uint8 arrays with 1 or 3 channels. Both provide `to_file`, `to_file_like` and `to_tensor`; JPEG quality is 1–100 (default 75), diff --git a/docs/package_size.md b/docs/package_size.md index 34bf4d0..60adf23 100644 --- a/docs/package_size.md +++ b/docs/package_size.md @@ -1,8 +1,7 @@ # Package size policy TensorCodec keeps its NumPy-only Python dependency set and bundles a minimal -FFmpeg/OpenSSL runtime in its default Linux `tensorcodec-av` wheels (the pure-Python -`tensorcodec` wheel adds about 30 KiB and is not size-checked). Size limits prevent additions +FFmpeg/OpenSSL runtime in its default Linux `tensorcodec-av` wheels. Size limits prevent additions from silently increasing the distributed binary footprint. ## What is measured @@ -25,10 +24,10 @@ uv run --no-project python scripts/check_wheel_size.py dist/*.whl --output repor ``` Each architecture is checked independently. Exactly reaching a limit passes; -exceeding either limit fails. The `linux` job of `build-av.yml` runs this after -`auditwheel repair`, in CI and before every upload. JSON reports are separate -artifacts, not files in `dist/`. Actions summaries include changes from the -committed published baseline. The `test` job also unit-tests the checker. +exceeding either limit fails. `build-av.yml` runs this after `auditwheel +repair`, in CI and before uploading distributions. JSON reports are separate artifacts, not +files in `dist/`. Actions summaries include changes from the committed published +baseline. Before changing a limit, explain the feature, the measured byte increase on both architectures and why a smaller configuration would not provide the same behavior. diff --git a/docs/releasing.md b/docs/releasing.md index b2ca342..a0c1a91 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,35 +1,21 @@ # Publishing TensorCodec -Release version: `0.3.0`. One release publishes two PyPI projects from the same -commit and version: - -| Project | Contents | Build | Distributions | -| --- | --- | --- | --- | -| `tensorcodec` | Python API, image codecs (import `tensorcodec`) | hatchling (`uv build`) | `py3-none-any` wheel, sdist | -| `tensorcodec-av` | Rust/FFmpeg audio/video decoder extension (import `tensorcodec_av`) | maturin (`av/`) | platform wheels, sdist | - -`tensorcodec` requires `tensorcodec-av==` behind an environment -marker matching the `tensorcodec-av` wheel tags. `tests/test_versions.py` fails CI unless -`pyproject.toml`, `av/Cargo.toml` (the `tensorcodec-av` version source), `av/Cargo.lock`, the pin and -`tensorcodec.__version__` agree. At runtime `VideoDecoder`/`AudioDecoder` reject a -`tensorcodec-av` whose version differs. - -`tensorcodec-av` wheels target Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+ -(abi3). NumPy must also provide a compatible wheel for the selected Python/glibc -pair. The wheel bundles shared FFmpeg 7.1.5 and OpenSSL 3.5.9 LTS; its only Python +Release version: `0.3.0`. Each release publishes two PyPI projects at the same version: +`tensorcodec` (pure Python, built by hatchling) and `tensorcodec-av` (the FFmpeg +extension in `av/`, built by maturin). `tensorcodec` pins `tensorcodec-av==` +behind a platform marker; bump the version in `pyproject.toml` (twice), `av/Cargo.toml`, +`av/Cargo.lock` and `src/tensorcodec/__init__.py` (`tests/test_versions.py` checks). + +`tensorcodec-av` wheels target Linux x86_64 and ARM64 (aarch64), glibc 2.17+, CPython 3.10+. +NumPy must also provide a compatible wheel for the selected Python/glibc pair. +The wheel bundles shared FFmpeg 7.1.5 and OpenSSL 3.5.9 LTS; its only Python runtime dependency is NumPy. macOS 14+ ARM64 wheels bundle the same minimal -runtime. Windows wheels are not provided; there, `tensorcodec` provides image -codecs only. Bundled-library notices ship only with `tensorcodec-av` -(`av/licenses/`). - -To release, set the new version in `pyproject.toml` (project version and the -`tensorcodec-av` pin), `av/Cargo.toml` (then `cargo update -p -tensorcodec-av --manifest-path av/Cargo.toml` to refresh the lock file) -and `src/tensorcodec/__init__.py`. +runtime. Windows wheels are not provided; there `tensorcodec` has image codecs only. ## Trusted publisher configuration -The PyPI project is already registered. Its GitHub Trusted Publisher uses: +Both PyPI projects use this GitHub Trusted Publisher. Before the first release of +`tensorcodec-av`, add it as a pending publisher with the same fields. | Field | Value | | --- | --- | @@ -39,50 +25,29 @@ The PyPI project is already registered. Its GitHub Trusted Publisher uses: | Workflow filename | `publish.yml` | | Environment | `pypi` | -Both projects need this publisher. `tensorcodec-av` does not exist on PyPI yet: -before its first release, add it as a pending publisher (PyPI account → Publishing) -with the same fields. Manage this configuration in each project's PyPI Publishing settings when moving +Manage this configuration in the project's PyPI Publishing settings when moving or renaming the repository or workflow. Publishing uses GitHub OIDC; no API token is needed. Repository visibility does not need to change for a release. -## Workflows - -Each distribution has one reusable workflow; CI and the release call both. - -| Workflow | Jobs | Artifacts | -| --- | --- | --- | -| `build-python.yml` | `build`: `tensorcodec` wheel and sdist (`python -m build`, `twine check --strict`); `test-without-av`: installs it on Windows and Intel macOS without `tensorcodec-av` and runs the image tests (OpenCV 5 and the 4.12 lower bound) | `dist-python` | -| `build-av.yml` | `linux` (x86_64, aarch64: manylinux2014 via maturin-action), `macos` (arm64), `sdist`; each wheel job validates with the `dist-python` wheel, so callers run `build-python.yml` first | `dist-av-linux-{x86_64,aarch64}`, `dist-av-macos-arm64`, `dist-av-sdist`, `wheel-size-linux-*` | -| `ci.yml` | `python`, `av`, plus `test` (development build against conda-forge FFmpeg, Clippy, oracle comparison) | — | -| `publish.yml` | `python`, `av`, `check` (one version, exactly the six expected files), `publish`, `update-size-docs` | — | - -`ci.yml` runs `av` on every push, including `main`, and on pull requests that -touch `av/`, `scripts/`, `tests/`, workflows or `pyproject.toml`; a release -commit has therefore already passed the same `tensorcodec-av` builds and validation. - ## Release -Run the **Publish to PyPI** workflow on `main`. It runs both build workflows, checks the -release set and uploads through PyPI Trusted Publishing using the existing GitHub -`pypi` environment. `tensorcodec-av` is uploaded first, so the exact pin in -`tensorcodec` never points at a missing release; if the second upload fails, upload -the `dist-python` artifact with `twine upload` rather than rerunning the whole -workflow. Publication fails if authorization is missing, tests fail, or the version -has already been uploaded. +Run the **Publish to PyPI** workflow on `main`. It calls `build-python.yml` and +`build-av.yml` (the same builds and validation CI runs), then uploads `tensorcodec-av` before +`tensorcodec`, so the exact pin never points at a missing release. If only the +second upload fails, `twine upload` the run's `dist-python` artifact. It uses the +existing GitHub `pypi` environment. Publication fails if authorization is +missing, tests fail, or the version has already been uploaded. ```sh gh workflow run publish.yml --repo epishelf/tensorcodec --ref main ``` -For a build and full validation without uploading, pass `--field publish=false`; -the run's `dist-*` artifacts are then the release candidates. +For a build and full validation without uploading, pass `--field publish=false`. -Check the workflow, https://pypi.org/project/tensorcodec/ and -https://pypi.org/project/tensorcodec-av/ before reporting success. Verify a fresh -`uv pip install tensorcodec==` pulls the matching `tensorcodec-av` and -decodes video without Torch/PyAV on both Linux architectures and macOS arm64, and -that it installs without `tensorcodec-av` on Windows. Update the version before -subsequent releases; PyPI versions cannot be overwritten. +Check the workflow and both PyPI project pages before reporting success. Verify a +fresh `uv pip install tensorcodec==` pulls `tensorcodec-av` and decodes +without Torch/PyAV on both Linux architectures and macOS arm64. Update the version before subsequent releases; +PyPI versions cannot be overwritten. The local Linux build is reproducible using `scripts/build_linux_wheel.sh` inside `quay.io/pypa/manylinux2014_x86_64` or @@ -90,28 +55,26 @@ The local Linux build is reproducible using `scripts/build_linux_wheel.sh` insid Both native source archives are version- and checksum-pinned. Their licensing and source links are recorded in `av/licenses/README.md`. -## Development versus release builds - -- The `test` job uses prebuilt conda-forge FFmpeg 7.1.1 through Pixi, including its - headers and shared libraries. It builds only the `tensorcodec-av` extension - and installs `tensorcodec` from the checkout. -- Release wheels (`build-av.yml`) use the smaller LGPL FFmpeg 7.1.5 build plus - OpenSSL 3.5.9. Their native prefix is cached by architecture, glibc baseline and - build-script checksums. This preserves the wheel's codec set, dependency size and - licensing rather than bundling the full conda-forge dependency graph. -- Each repaired Linux wheel is installed with the `tensorcodec` wheel on glibc 2.17 - with Python 3.10 and 3.13 and decodes video/audio without Torch, PyAV or a system - FFmpeg. Python 3.10 also checks the minimum NumPy line (1.26.4). The x86_64 and - ARM64 runners also run the full pinned playback oracle comparison; the macOS job - runs the test suite and clean Python 3.10/3.13 environments. -- The fixture CLI can be FFmpeg 6 or 7; fixtures explicitly remove auxiliary - sentinel packets. +## CI versus release builds + +- Ordinary CI uses prebuilt conda-forge FFmpeg 7.1.1 through Pixi, including its + headers and shared libraries. It builds only the `tensorcodec-av` extension. +- PyPI wheels (`build-av.yml`, also run by CI) use the smaller LGPL FFmpeg 7.1.5 build plus OpenSSL 3.5.9. + Their native prefix is cached by architecture, glibc baseline and build-script + checksums. This preserves the wheel's codec set, dependency size and licensing rather than bundling the full + conda-forge dependency graph. +- Release validation installs each repaired wheel with the `tensorcodec` wheel on glibc 2.17 with Python 3.10 + and 3.13 and decodes video/audio without Torch, PyAV or a system FFmpeg. Python + 3.10 also checks the minimum NumPy line (1.26.4). Native + x86_64 and ARM64 runners also run the full pinned playback oracle comparison. +- Release validation still tests the installed repaired wheel. The fixture CLI + can be FFmpeg 6 or 7; fixtures explicitly remove auxiliary sentinel packets. ## Size checks and published comparison -Final repaired `tensorcodec-av` wheels must stay within 15 MiB download and 35 MiB unpacked per -architecture. The `linux` job checks `packaging/size-policy.json` and uploads a -separate `wheel-size-linux-*` report, including differences from the last published +Final repaired wheels must stay within 15 MiB download and 35 MiB unpacked per +architecture. The build job checks `packaging/size-policy.json` and uploads a +separate `wheel-size-*` report, including differences from the last published baseline. Size reports must not be placed in `dist/`. After a successful publication, the `update-size-docs` job measures hash-verified diff --git a/docs/system_ffmpeg.md b/docs/system_ffmpeg.md index 27c5206..0d4ae30 100644 --- a/docs/system_ffmpeg.md +++ b/docs/system_ffmpeg.md @@ -7,8 +7,7 @@ uv venv uv pip install tensorcodec ``` -On supported platforms this also installs the matching `tensorcodec-av` -wheel, which includes minimal shared FFmpeg 7.1.5 (with dav1d for AV1) and OpenSSL libraries. +Supported platforms get `tensorcodec-av` wheels with minimal shared FFmpeg 7.1.5 (with dav1d for AV1) and OpenSSL libraries. No FFmpeg CLI, Pixi, Rust or libclang is required at runtime. This is the recommended installation for a new environment. @@ -41,10 +40,8 @@ uv venv uv pip install 'tensorcodec==0.3.0' 'tensorcodec-av==0.3.0' --no-binary tensorcodec-av ``` -`tensorcodec` is pure Python; only `tensorcodec-av` is built from source here, -and naming it explicitly also covers platforms without `tensorcodec-av` wheels (for example -Intel macOS), where `tensorcodec` does not depend on it. Releases up to 0.2.0 were -a single package, built from source with `--no-binary tensorcodec`. +Only `tensorcodec-av` is built from source; naming it also covers platforms +without its wheels, such as Intel macOS. The version/build constraint avoids silently selecting an incompatible FFmpeg major. `FFMPEG_DIR` tells the source build where to find headers and libraries; @@ -84,7 +81,7 @@ within the color-conversion tolerances described in the ## macOS wheels -`tensorcodec-av` macOS 14+ wheels support Apple Silicon, bundling FFmpeg/OpenSSL with `delocate` +macOS 14+ wheels support Apple Silicon, bundling FFmpeg/OpenSSL with `delocate` (0.1.3 through 0.1.5 also had Intel wheels). CI tests the installed wheels and clean Python 3.10/3.13 environments. Developers can run `scripts/build_macos_wheel.sh` with Rust, Xcode tools, NASM, Meson, Ninja, pkg-config, coreutils, diff --git a/pyproject.toml b/pyproject.toml index 6ac87bb..8e98b4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,8 @@ license = "Apache-2.0" license-files = ["LICENSE"] authors = [{ name = "Suhwan Choi", email = "milkclouds00@gmail.com" }] requires-python = ">=3.10" -# tensorcodec-av is pinned to this exact version. The marker follows its wheel tags (CPython abi3, -# manylinux x86_64/aarch64, macOS 14+ arm64); elsewhere only the image codecs work. No platform_release -# check for macOS 14: pip's packaging 22-25 raises on non-PEP 440 Linux kernel releases. +# Exact pin; the marker follows the tensorcodec-av wheel tags. No platform_release check (macOS 14): +# pip's packaging 22-25 raises on Linux kernel releases that are not PEP 440 versions. dependencies = [ "numpy>=1.26", "tensorcodec-av==0.3.0; platform_python_implementation == 'CPython' and ((sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", @@ -41,7 +40,6 @@ build-backend = "hatchling.build" packages = ["src/tensorcodec"] [tool.hatch.build.targets.sdist] -# av/ keeps the sdist a complete checkout for the uv source below and tests/test_versions.py. only-include = ["src", "av", "tests", "docs", "scripts", "benchmarks", "packaging", "README.md", "LICENSE"] [tool.pytest.ini_options] diff --git a/tests/test_av_optional.py b/tests/test_av_optional.py index 0854d96..0a9c0ed 100644 --- a/tests/test_av_optional.py +++ b/tests/test_av_optional.py @@ -8,38 +8,17 @@ def run_python(code): subprocess.run([sys.executable, "-c", code], check=True) -def test_av_extension_is_lazy(): - run_python( - """ -import sys -import tensorcodec -import tensorcodec.decoders -import tensorcodec.encoders -assert 'tensorcodec_av' not in sys.modules -""" - ) - - def test_image_codecs_without_tensorcodec_av(): run_python( """ import sys -sys.modules['tensorcodec_av'] = None import numpy as np -import tensorcodec -from tensorcodec.decoders import ( - AudioDecoder, ImageReadMode, VideoDecoder, decode_avif, decode_gif, decode_image, decode_jpeg, decode_png, - decode_webp, -) -from tensorcodec.encoders import JpegEncoder, PngEncoder - -pixels = np.zeros((3, 4, 5), np.uint8) -pixels[:] = np.array([23, 91, 177], np.uint8)[:, None, None] -encoded = PngEncoder(pixels).to_tensor() -np.testing.assert_array_equal(decode_png(encoded), pixels) -np.testing.assert_array_equal(decode_image(encoded, mode=ImageReadMode.RGB), pixels) -assert decode_jpeg(JpegEncoder(pixels).to_tensor()).shape == pixels.shape - +from tensorcodec.decoders import * +from tensorcodec.encoders import PngEncoder +assert 'tensorcodec_av' not in sys.modules +sys.modules['tensorcodec_av'] = None +pixels = np.full((3, 4, 5), 91, np.uint8) +np.testing.assert_array_equal(decode_image(PngEncoder(pixels).to_tensor(), mode=ImageReadMode.RGB), pixels) for decoder in (VideoDecoder, AudioDecoder): try: decoder(b'not media') @@ -56,9 +35,7 @@ def test_version_skew_is_rejected(): """ import sys import types -package = types.ModuleType('tensorcodec_av') -package._av = types.SimpleNamespace(__version__='0.0.0', Decoder=None) -sys.modules['tensorcodec_av'] = package +sys.modules['tensorcodec_av'] = types.SimpleNamespace(_av=types.SimpleNamespace(__version__='0.0.0')) from tensorcodec.decoders import VideoDecoder try: VideoDecoder(b'not media') diff --git a/tests/test_images.py b/tests/test_images.py index 51e6923..f9e8096 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -543,18 +543,14 @@ def test_bmp_matches_pillow(pil_mode, channels): np.testing.assert_array_equal(decode_image(data), rgb.transpose(2, 0, 1)) -def test_bmp_alpha(tmp_path): +def test_bmp_alpha(): from tensorcodec.decoders import decode_image, decode_png rgba = np.random.default_rng(8).integers(0, 256, (5, 7, 4), np.uint8) data = bitfields_bmp(rgba, 124, 0xFF000000) assert Image.open(BytesIO(data)).mode == "RGBA" np.testing.assert_array_equal(decode_image(data, mode="UNCHANGED"), rgba.transpose(2, 0, 1)) - np.testing.assert_array_equal(decode_image(data, mode="RGBA"), rgba.transpose(2, 0, 1)) np.testing.assert_array_equal(decode_image(data), rgba[..., :3].transpose(2, 0, 1)) - path = tmp_path / "image.bmp" - path.write_bytes(data) - np.testing.assert_array_equal(decode_image(path, mode="RGBA"), rgba.transpose(2, 0, 1)) with pytest.raises(RuntimeError, match="expected png, got bmp"): decode_png(data) for header_size, mask in ((124, 0), (40, 0)): diff --git a/tests/test_versions.py b/tests/test_versions.py index d4f0752..04feaf0 100644 --- a/tests/test_versions.py +++ b/tests/test_versions.py @@ -11,10 +11,9 @@ ROOT = Path(__file__).resolve().parents[1] -def av_requirements(project): - groups = [project["dependencies"], *project.get("optional-dependencies", {}).values()] - requirements = [Requirement(item) for group in groups for item in group] - return [r for r in requirements if r.name == "tensorcodec-av"] +def av_requirement(project): + (requirement,) = [r for r in map(Requirement, project["dependencies"]) if r.name == "tensorcodec-av"] + return requirement def test_versions_are_locked(): @@ -25,10 +24,7 @@ def test_versions_are_locked(): assert [p["version"] for p in lock if p["name"] == "tensorcodec-av"] == [cargo] init = re.search(r'^__version__ = "(.+)"$', (ROOT / "src/tensorcodec/__init__.py").read_text(), re.MULTILINE)[1] assert cargo == init == version - # Only the marked dependency: an extra would just trigger the same source build elsewhere. - (pin,) = av_requirements(project) - assert str(pin.specifier) == f"=={version}" - assert pin.marker is not None + assert str(av_requirement(project).specifier) == f"=={version}" def test_av_license_matches_project_license(): @@ -49,8 +45,10 @@ def test_av_license_matches_project_license(): ) def test_av_marker_matches_published_wheels(environment, expected): project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] - (default,) = av_requirements(project) + marker = av_requirement(project).marker # packaging 22-25 (vendored by pip) raise on version comparisons with releases like "6.8.0-azure". - assert "platform_release" not in str(default.marker) - base = default_environment() | {"platform_python_implementation": "CPython"} - assert default.marker.evaluate(base | environment) is expected + assert "platform_release" not in str(marker) + assert ( + marker.evaluate(default_environment() | {"platform_python_implementation": "CPython"} | environment) + is expected + )